diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e31ca4e..5502f41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v7 with: &specification-checkout repository: OpenStatSpec/specification - ref: 79339ec3d8f8aa81789b7e85f6b8afa6f1374e50 + ref: agent/dolt-declaration-validator path: openstatspec-specification - name: Checkout required SPSS engine uses: actions/checkout@v7 @@ -32,6 +32,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip + - run: python -m pip install ./openstatspec-specification - run: python -m pip install ./openstatspec-pyspssio - run: python -m pip install -e ".[dev]" - run: python -m pytest -m "not services" @@ -53,6 +54,7 @@ jobs: - run: python -m pip install --upgrade pip build - run: python -m build - run: python -m venv /tmp/openstatspec-wheel-smoke + - run: /tmp/openstatspec-wheel-smoke/bin/python -m pip install ./openstatspec-specification - run: /tmp/openstatspec-wheel-smoke/bin/python -m pip install ./openstatspec-pyspssio - run: /tmp/openstatspec-wheel-smoke/bin/python -m pip install dist/*.whl - run: /tmp/openstatspec-wheel-smoke/bin/openstatspec capabilities @@ -90,6 +92,7 @@ jobs: with: python-version: "3.12" - run: python -m pip install --upgrade pip + - run: python -m pip install ./openstatspec-specification - run: python -m pip install ./openstatspec-pyspssio - run: python -m pip install -e ".[dev,sql]" - run: python -m pytest -m services @@ -192,6 +195,7 @@ jobs: --workdir /var/lib/dolt \ "$OPENSTATSPEC_DOLT_IMAGE" sql-server -H 0.0.0.0 -l warning - run: python -m pip install --upgrade pip + - run: python -m pip install ./openstatspec-specification - run: python -m pip install ./openstatspec-pyspssio - run: python -m pip install -e ".[dev,sql]" - name: Wait for Dolt diff --git a/CHANGELOG.md b/CHANGELOG.md index edb3058..74e0b61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,17 @@ SPSS profile. - Export of supported dataset semantics to SAV and ZSAV. - SQLite, PostgreSQL, MySQL, and MariaDB profiles, including service-backed PostgreSQL 17/18, MySQL 8.4/9.7, and MariaDB 11.4/11.8/12.3 CI coverage. +- Read-only positive Dolt identity and working-set inspection. Operational + Dolt writes load the shared `openstatspec-specification` declaration + validator and remain fail-closed because the packaged concrete declaration + set is empty. +- Added explicit packaged/directory conformance-source injection and exact + single-match binding across active Dolt product version, adapter + implementation and version, and specification commit before mutation. +- Explicit catalog initialization/migration through `initialize_catalog` and + `openstatspec init`; data operations never auto-create catalog relations. +- Read-only Dolt working-set evidence through `dolt_state_snapshot` and + `openstatspec dolt-state`; core performs no Dolt version-control mutations. - Preflight checks for target profile limits, atomic imports, validation, a command-line interface, and machine-readable capability and loss reports. @@ -121,3 +132,11 @@ SPSS profile. - Multiple-response sets, variable alignment, variable sets, and custom attributes have explicit capability diagnostics; see the SAV profile. - Encrypted SPSS files are not supported. +- Dolt adapter-envelope values are not claims about Dolt server limits. Failure + recovery is compensating and reports deterministic residual inventory; a + verified catalog receives a best-effort failed-operation audit when cleanup + itself fails. +- Export publication uses a durable, audit-recorded prior-file backup through + SQL finalization. Post-success backup retention is reported separately and + does not rewrite a successful export as failed; pre-success failures restore + the prior destination and close the running operation as failed. diff --git a/README.md b/README.md index dead014..6aa7188 100644 --- a/README.md +++ b/README.md @@ -26,21 +26,20 @@ import. capabilities, and loss reports. - `openstatspec.sql`: database connection and wide-table/catalog operations. - `openstatspec.spss`: SAV/ZSAV adapter boundary. -- `openstatspec.transform`: canonical plans, frontend-neutral schema concepts, - and plan validation. -- `openstatspec.frontends.spss`: the SPSS-like syntax frontend and convenience - execution adapter. ## Intended workflow ```python -from openstatspec import export_sav, import_sav +from openstatspec import export_sav, import_sav, initialize_catalog -import_sav("responses.sav", database_url="postgresql+psycopg://user:password@server/database", dataset_id="responses-2026") +database_url = "postgresql+psycopg://user:password@server/database" +initialize_catalog(database_url=database_url) +import_sav("responses.sav", database_url=database_url, dataset_id="responses-2026") export_sav(database_url="postgresql+psycopg://user:password@server/database", dataset_id="responses-2026", destination="responses-roundtrip.sav") ``` ```text +openstatspec init --database-url postgresql+psycopg://... openstatspec import responses.sav --database-url postgresql+psycopg://... --dataset-id responses-2026 openstatspec export --database-url postgresql+psycopg://... --dataset-id responses-2026 --output responses-roundtrip.sav ``` @@ -54,72 +53,52 @@ derived datasets through a public catalog API. It uses a separate profile catalog and never presents SQL output as an imported source dataset. Workflow operations support SQLite only in this milestone and fail closed on PostgreSQL/MySQL/MariaDB; core import/export database support is unchanged. -The core SQLite import/export profile accepts SQLite `>=3.24.0,<4.0.0`; the -optional transformation workflow deliberately has the narrower -`>=3.35.0,<4.0.0` runtime preflight. These independent tiers do not change the -server-profile matrix. Microsoft SQL Server is not supported; its future -dialect is scoped only in the specification's -[MSSQL roadmap](https://github.com/OpenStatSpec/specification/blob/main/docs/mssql-dialect-roadmap.md). See [the SQL transformation workflow](docs/sql-transformation-workflow.md) for Python and CLI examples, migration behavior, hashing, atomicity, and the exact implemented capability boundary. -## SPSS-like transformation frontend - -The SPSS-like frontend lowers supported `RECODE`, `VARIABLE LABELS`, and -`VALUE LABELS` syntax into a language-neutral canonical plan. The in-place -path applies it to the same logical dataset, physical wide table, and metadata -catalog without a derived dataset, copied table, snapshot, or separate -rollback/history layer. Dolt remains the sole versioning layer for Dolt-backed -edits, and the transformer never calls `DOLT_COMMIT`. - -See the [dataset transformations manual](docs/transformations.md) for schema -installation, Python and CLI surfaces, database invariants, audit provenance, -package layout, and extension guidance. Stata and SAS are unimplemented -placeholders. - -## Current support status +## 0.1.0 support status The adapter requires `openstatspec-pyspssio==0.5.1.post2` as its sole SPSS engine. Its import module remains `pyspssio`; the exact source commit is recorded in operation metadata. There is no fallback reader or writer. It supports unencrypted SAV and ZSAV import and SAV/ZSAV export for the semantics exposed by that engine. SQLite is the local reference path. -PostgreSQL, MySQL, MariaDB, and Dolt are each covered by separate service-backed CI -conformance checks. Dolt support is an independent core profile for the -canonical stable range `>=2.2.2,<2.3.0`; earlier patches, other families, -noncanonical versions, and unknown MySQL-wire products fail closed. - -The supported family claims are broader than the deliberately exact CI -evidence points: PostgreSQL 17.x/18.x is exercised at 17.10/18.4, MySQL -8.4.x/9.7.x at 8.4.11/9.7.2, and MariaDB 11.4.x/11.8.x/12.3.x at -11.4.12/11.8.8/12.3.2. Each service job checks the normalized live server -version against its exact matrix entry before that run can count as evidence. -Dolt claims the conservative 2.2.x range `>=2.2.2,<2.3.0`; its full service -suite is exercised independently at exact versions 2.2.2 and 2.2.3 using -immutable container-image digests. - -| Engine/profile | Runtime supported policy | Exact CI-tested versions | -| --- | --- | --- | -| SQLite core / optional workflow | Core `>=3.24.0,<4.0.0`; optional workflow `>=3.35.0,<4.0.0` | Runtime-provided SQLite on Python 3.11–3.14 runners; not a pinned server image | -| PostgreSQL | 17.x and 18.x | 17.10 and 18.4 | -| MySQL | 8.4.x and 9.7.x | 8.4.11 and 9.7.2 | -| MariaDB | 11.4.x, 11.8.x, and 12.3.x | 11.4.12, 11.8.8, and 12.3.2 | -| Dolt | 2.2.x with `>=2.2.2,<2.3.0` | 2.2.2 and 2.2.3 | - -Microsoft SQL Server (MSSQL) remains roadmap-only and is not a supported -runtime profile; see the specification's [MSSQL roadmap](https://github.com/OpenStatSpec/specification/blob/main/docs/mssql-dialect-roadmap.md). - -Use these explicit SQLAlchemy URLs: +PostgreSQL, MySQL, and MariaDB are each covered by separate service-backed CI +conformance checks. Use these explicit SQLAlchemy URLs: - SQLite: `sqlite:///dataset.sqlite` - PostgreSQL: `postgresql+psycopg://user:password@host/database` -- MySQL/MariaDB: `mysql+pymysql://user:password@host/database` -- Dolt `>=2.2.2,<2.3.0`: `mysql+pymysql://user:password@host/database` (detected by server identity) - -The Dolt core profile supports strict wide-table import, validation, and export; -the separate Transformation Workflow is unsupported. +- MySQL/MariaDB/Dolt wire protocol: `mysql+pymysql://user:password@host/database` + +Catalog creation and additive migration are explicit: run `initialize_catalog` +or `openstatspec init` before import, read, validation, or export. Those +operations fail closed on absent, foreign, ambiguous, unverified, or +migration-required catalogs and never auto-create catalog relations. Failure +cleanup uses compensating actions where the server does not provide atomic DDL; +a cleanup failure produces machine-readable residual inventory and a best-effort +failed-operation audit in an otherwise verified catalog. + +Dolt identity requires an exact `Dolt` version comment, non-empty exact +`DOLT_VERSION()`, and an explicit active branch. Dolt writes load the +`openstatspec-specification` companion distribution through +`DoltConformanceSource.packaged()` by default. The packaged concrete +declaration directory is intentionally empty, so every operational Dolt write +path currently fails before mutation. There are no mirrored Python evidence +maps or validator rules. + +Tests and explicitly configured local integrations may inject +`DoltConformanceSource.from_directory(specification_root)`. The same shared +validator must then find exactly one concrete declaration matching the active +Dolt product version, `openstatspec-python`, exact adapter version, and pinned +specification commit. Missing, invalid, empty, or ambiguous sources all fail +closed. The proposed 305-source-variable/306-physical-column envelope is not a +Dolt server limit. The read-only `dolt_state_snapshot()` and +`openstatspec dolt-state` remain available for the bound database/branch +working set, HEAD, status, and three diff summaries with deterministic digests. +Core OpenStatSpec never runs `DOLT_ADD`, `DOLT_COMMIT`, checkout, reset, or +branch-changing operations. Run `openstatspec capabilities` before an integration to inspect the machine-readable feature matrix. Export is deliberately strict: if known @@ -127,6 +106,15 @@ dictionary semantics cannot be reproduced, it stops until you pass the exact diagnostic code with `--allow-loss`. This avoids silent loss while making an intentional lossy export auditable. +Filesystem publication and SQL audit finalization cannot form one atomic 2PC +transaction. Export therefore records a running operation, destination, and +unique durable prior-file backup path before publication; it retains that +backup until the SQL operation reaches `succeeded`. A publication or +finalization failure restores the prior destination and closes the operation +as `failed`. Failure to remove the backup after success never rewrites the +successful operation: it raises `backup_retained` with the durable path and +appends a warning best-effort, so a confidential duplicate is discoverable. + The matrix is also available to Python callers as `openstatspec.capability_matrix()`. It distinguishes supported semantics from unobservable and fail-closed paths; see the SAV profile for the exact diff --git a/pyproject.toml b/pyproject.toml index d16b5f5..9309c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.11" license = {text = "Apache-2.0"} authors = [{name = "OpenStatSpec"}] -dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"] +dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "openstatspec-specification==0.2.0", "sqlglot>=30.14", "rfc8785>=0.1.4"] [project.optional-dependencies] dev = ["pytest>=8"] diff --git a/src/openstatspec/__init__.py b/src/openstatspec/__init__.py index d9754f0..4329bda 100644 --- a/src/openstatspec/__init__.py +++ b/src/openstatspec/__init__.py @@ -2,17 +2,18 @@ from .api import ( apply_spss_in_place, apply_transformation_plan_in_place, - capabilities, capability_matrix, derive_sql_dataset, - execute_sql_transformation, - export_sav, get_dataset, import_sav, inspect, - install_in_place_transformation_schema, list_datasets, + capabilities, capability_matrix, derive_sql_dataset, dolt_state_snapshot, execute_sql_transformation, + export_sav, get_dataset, import_sav, initialize_catalog, inspect, list_datasets, + install_in_place_transformation_schema, register_sql_transformation, reconcile_derived_removals, reconcile_sql_transformation_runs, remove_derived_physical_relation, retire_derived, validate, validate_derived, ) from .core import CapabilityDeclaration, LossReport, UnsupportedOperationError -from .sql.workflow import TransformationError from .frontends.spss import SpssFrontendCompilation, compile_spss_syntax +from .spss import compare_sav_semantics +from .sql import DoltConformanceSource +from .sql.workflow import TransformationError from .transform import ( RecodeMatch, RecodeOperation, RecodeResult, RecodeRule, ReplaceValueLabelsOperation, SetVariableLabelOperation, @@ -21,18 +22,17 @@ ) __all__ = [ - "CapabilityDeclaration", "LossReport", "SpssFrontendCompilation", - "TransformationError", "TransformationFrontendError", + "CapabilityDeclaration", "DoltConformanceSource", "LossReport", "TransformationError", + "SpssFrontendCompilation", "TransformationFrontendError", "RecodeMatch", "RecodeOperation", "RecodeResult", "RecodeRule", "ReplaceValueLabelsOperation", "SetVariableLabelOperation", "TransformationPlan", "TypedValue", "ValueLabel", "VariableDefinition", "VariableSchema", "transformation_plan_from_dict", - "UnsupportedOperationError", "capabilities", "capability_matrix", "apply_spss_in_place", "apply_transformation_plan_in_place", - "compile_spss_syntax", "derive_sql_dataset", - "execute_sql_transformation", "export_sav", - "get_dataset", "import_sav", "inspect", - "install_in_place_transformation_schema", "list_datasets", + "compile_spss_syntax", "install_in_place_transformation_schema", + "UnsupportedOperationError", "capabilities", "capability_matrix", "compare_sav_semantics", + "derive_sql_dataset", "dolt_state_snapshot", "execute_sql_transformation", "export_sav", "get_dataset", + "import_sav", "initialize_catalog", "inspect", "list_datasets", "register_sql_transformation", "reconcile_derived_removals", "reconcile_sql_transformation_runs", "remove_derived_physical_relation", "retire_derived", "validate", diff --git a/src/openstatspec/api.py b/src/openstatspec/api.py index 9de2e21..ba0efc8 100644 --- a/src/openstatspec/api.py +++ b/src/openstatspec/api.py @@ -7,7 +7,13 @@ from .core.results import result from .spss import export_dataset, import_dataset, inspect_source from .spss.sav import engine_identity -from .sql import declared_profiles, validate_dataset +from .sql import ( + DoltConformanceSource, + declared_profiles, + dolt_state_snapshot as _dolt_state_snapshot, + initialize_catalog as _initialize_catalog, + validate_dataset, +) from .sql.catalog_api import catalog_dataset as _catalog_dataset, catalog_datasets as _catalog_datasets from .sql.workflow import ( derive_dataset as _derive_dataset, @@ -33,7 +39,11 @@ -def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]: +def capability_matrix( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: """Return the pyspssio-backed SAV/ZSAV fidelity boundary. supported means that the adapter has a tested faithful path. @@ -86,9 +96,18 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]: "maximum_source_file_bytes": None, "limit_basis": "runtime memory and active SQL connection limits", }, - "active_connection": active_connection(database_url) if database_url else None, + "active_connection": ( + active_connection( + database_url, + dolt_conformance_source=dolt_conformance_source, + ) + if database_url else None + ), "catalog_binding": catalog_binding(database_url) if database_url else None, - "sql_profiles": declared_profiles(database_url), + "sql_profiles": declared_profiles( + database_url, + dolt_conformance_source=dolt_conformance_source, + ), "optional_profiles": { "sql_transformation_workflow": transformation_capabilities(database_url), "spss_in_place_transformation": ( @@ -98,26 +117,84 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]: } return declaration -def capabilities(database_url: str | None = None) -> Mapping[str, Any]: - return capability_matrix(database_url) +def capabilities( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: + return capability_matrix( + database_url, dolt_conformance_source=dolt_conformance_source, + ) def inspect(source: str | Path, /, **options: Any) -> Mapping[str, Any]: return result(inspect_source(source, **options)) -def import_sav(source: str | Path, /, *, database_url: Any, dataset_id: str, **options: Any) -> Mapping[str, Any]: +def dolt_state_snapshot( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: + """Return read-only Dolt branch, HEAD, status, and diff-summary evidence.""" + return result(_dolt_state_snapshot( + database_url=database_url, + dolt_conformance_source=dolt_conformance_source, + )) + + +def initialize_catalog( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: + """Install or explicitly migrate a dedicated OpenStatSpec catalog.""" + return result(_initialize_catalog( + database_url=database_url, + dolt_conformance_source=dolt_conformance_source, + )) + + +def import_sav( + source: str | Path, /, *, database_url: Any, dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> Mapping[str, Any]: """Import one source file into one dedicated wide SQL table.""" - return result(import_dataset(source, database_url=database_url, dataset_id=dataset_id, **options)) + return result(import_dataset( + source, database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + **options, + )) -def export_sav(*, database_url: Any, dataset_id: str, destination: str | Path, **options: Any) -> Mapping[str, Any]: +def export_sav( + *, database_url: Any, dataset_id: str, destination: str | Path, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> Mapping[str, Any]: """Export one database-resident conforming dataset to SAV/ZSAV.""" - return result(export_dataset(database_url=database_url, dataset_id=dataset_id, destination=destination, **options)) + return result(export_dataset( + database_url=database_url, dataset_id=dataset_id, + destination=destination, + dolt_conformance_source=dolt_conformance_source, + **options, + )) -def validate(*, database_url: Any, dataset_id: str, **options: Any) -> Mapping[str, Any]: - return result(validate_dataset(database_url=database_url, dataset_id=dataset_id, **options)) +def validate( + *, + database_url: Any, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> Mapping[str, Any]: + return result(validate_dataset( + database_url=database_url, + dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + **options, + )) def list_datasets(*, database_url: Any, kind: str | None = None) -> Mapping[str, Any]: diff --git a/src/openstatspec/cli.py b/src/openstatspec/cli.py index cc9d5c7..87089d6 100644 --- a/src/openstatspec/cli.py +++ b/src/openstatspec/cli.py @@ -2,15 +2,11 @@ import argparse import json from collections.abc import Sequence -from pathlib import Path from .api import ( - apply_spss_in_place, apply_transformation_plan_in_place, - capability_matrix, derive_sql_dataset, - execute_sql_transformation, - export_sav, get_dataset, import_sav, inspect, list_datasets, - install_in_place_transformation_schema, register_sql_transformation, - validate, validate_derived, + capability_matrix, derive_sql_dataset, dolt_state_snapshot, execute_sql_transformation, + export_sav, get_dataset, import_sav, initialize_catalog, inspect, list_datasets, + register_sql_transformation, validate, validate_derived, ) @@ -29,6 +25,12 @@ def main(argv: Sequence[str] | None = None) -> int: commands = parser.add_subparsers(dest="command", required=True) capability_parser = commands.add_parser("capabilities", help="show supported and lossy feature matrix") capability_parser.add_argument("--database-url", help="include active connection limits") + dolt_state = commands.add_parser( + "dolt-state", help="show read-only Dolt branch, HEAD, status, and diff evidence" + ) + dolt_state.add_argument("--database-url", required=True) + initializer = commands.add_parser("init", help="initialize or migrate a dedicated catalog") + initializer.add_argument("--database-url", required=True) importer = commands.add_parser("import", help="import one SAV/ZSAV file") importer.add_argument("source") importer.add_argument("--database-url", required=True) @@ -77,36 +79,6 @@ def main(argv: Sequence[str] | None = None) -> int: derive.add_argument("--dataset-name") derive.add_argument("--weight-variable") - apply_spss = commands.add_parser( - "apply-spss", - help="compile supported SPSS syntax and apply it in-place", - ) - apply_spss.add_argument("--database-url", required=True) - apply_spss.add_argument("--dataset-id", required=True) - apply_spss.add_argument("--actor", required=True) - apply_spss.add_argument("--expected-branch") - apply_spss.add_argument("--expected-head") - syntax_source = apply_spss.add_mutually_exclusive_group(required=True) - syntax_source.add_argument("--syntax") - syntax_source.add_argument("--syntax-file") - - apply_plan = commands.add_parser( - "apply-plan", - help="apply a canonical transformation plan in-place", - ) - apply_plan.add_argument("--database-url", required=True) - apply_plan.add_argument("--dataset-id", required=True) - apply_plan.add_argument("--actor", required=True) - apply_plan.add_argument("--expected-branch") - apply_plan.add_argument("--expected-head") - apply_plan.add_argument("--plan-file", required=True) - - install_in_place = commands.add_parser( - "install-in-place-schema", - help="install or upgrade the compact in-place apply audit schema", - ) - install_in_place.add_argument("--database-url", required=True) - derived_validator = commands.add_parser("validate-derived", help="validate a derived dataset") derived_validator.add_argument("--database-url", required=True) derived_validator.add_argument("--derived-dataset-id", required=True) @@ -114,6 +86,10 @@ def main(argv: Sequence[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "capabilities": output = capability_matrix(database_url=args.database_url) + elif args.command == "dolt-state": + output = dolt_state_snapshot(database_url=args.database_url) + elif args.command == "init": + output = initialize_catalog(database_url=args.database_url) elif args.command == "import": output = import_sav(args.source, database_url=args.database_url, dataset_id=args.dataset_id) elif args.command == "export": @@ -156,33 +132,6 @@ def main(argv: Sequence[str] | None = None) -> int: transformation_name=args.name, dataset_name=args.dataset_name, weight_variable=args.weight_variable, ) - elif args.command == "apply-spss": - source_text = ( - args.syntax - if args.syntax is not None - else Path(args.syntax_file).read_text(encoding="utf-8") - ) - output = apply_spss_in_place( - database_url=args.database_url, - dataset_id=args.dataset_id, - source_text=source_text, - actor=args.actor, - expected_branch=args.expected_branch, - expected_head=args.expected_head, - ) - elif args.command == "apply-plan": - plan = json.loads(Path(args.plan_file).read_text(encoding="utf-8")) - output = apply_transformation_plan_in_place( - database_url=args.database_url, - dataset_id=args.dataset_id, - plan=plan, - actor=args.actor, - expected_branch=args.expected_branch, - expected_head=args.expected_head, - ) - elif args.command == "install-in-place-schema": - install_in_place_transformation_schema(database_url=args.database_url) - output = {"status": "installed"} else: output = validate_derived( database_url=args.database_url, derived_dataset_id=args.derived_dataset_id, diff --git a/src/openstatspec/core/__init__.py b/src/openstatspec/core/__init__.py index 12d26cf..d032ac4 100644 --- a/src/openstatspec/core/__init__.py +++ b/src/openstatspec/core/__init__.py @@ -1,6 +1,7 @@ from .results import Diagnostic, OperationResult """Pure OpenStatSpec concepts; no file or database adapter code.""" +import hashlib from dataclasses import dataclass, field from typing import Any @@ -9,6 +10,19 @@ class UnsupportedOperationError(NotImplementedError): """Raised when faithful support for a requested operation is unavailable.""" +def safe_error_identity(error: Exception, *, phase: str) -> dict[str, Any]: + """Return a path-free, stable identity for an exception.""" + code = getattr(error, "code", None) + if code is not None and not isinstance(code, (str, int, float, bool)): + code = type(code).__name__ + return { + "type": type(error).__name__, + "code": code, + "phase": phase, + "message_sha256": hashlib.sha256(str(error).encode("utf-8")).hexdigest(), + } + + @dataclass(frozen=True) class LossReport: """Machine-readable fidelity outcome for a completed future operation.""" diff --git a/src/openstatspec/spss/__init__.py b/src/openstatspec/spss/__init__.py index cfcc3aa..dde0b7c 100644 --- a/src/openstatspec/spss/__init__.py +++ b/src/openstatspec/spss/__init__.py @@ -3,20 +3,35 @@ from pathlib import Path from typing import Any +from ..sql.dolt_conformance import DoltConformanceSource from .sav import export_sav_dataset, import_sav_dataset, inspect_sav +from .semantics import compare_sav_semantics def inspect_source(source: str | Path, **options: Any) -> dict[str, Any]: return inspect_sav(source) -def import_dataset(source: str | Path, *, database_url: Any, dataset_id: str, **options: Any) -> dict[str, Any]: - return import_sav_dataset(source=source, database_url=str(database_url), dataset_id=dataset_id) +def import_dataset( + source: str | Path, *, database_url: Any, dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> dict[str, Any]: + return import_sav_dataset( + source=source, database_url=str(database_url), dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + ) -def export_dataset(*, database_url: Any, dataset_id: str, destination: str | Path, **options: Any) -> dict[str, Any]: +def export_dataset( + *, database_url: Any, dataset_id: str, destination: str | Path, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> dict[str, Any]: return export_sav_dataset( - database_url=str(database_url), dataset_id=dataset_id, destination=destination, + database_url=str(database_url), dataset_id=dataset_id, + destination=destination, allow_loss=tuple(options.get("allow_loss", ())), legacy_locale=options.get("legacy_locale"), + dolt_conformance_source=dolt_conformance_source, ) diff --git a/src/openstatspec/spss/raw_dictionary.py b/src/openstatspec/spss/raw_dictionary.py index f77f952..8c0524b 100644 --- a/src/openstatspec/spss/raw_dictionary.py +++ b/src/openstatspec/spss/raw_dictionary.py @@ -6,11 +6,14 @@ from __future__ import annotations import os -import stat +import re +from copy import deepcopy from dataclasses import dataclass from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Iterable +from tempfile import mkstemp +from typing import Callable, Iterable + +import pyspssio class RawDictionaryError(ValueError): @@ -27,6 +30,27 @@ class _Record: end: int +@dataclass(frozen=True) +class _DictionarySemantics: + compatible_names: dict[str, str] + variable_sets: dict[str, list[str]] + multiple_response_sets: dict[str, dict] + + +@dataclass(frozen=True) +class _ReferenceLine: + set_name: bytes + segments: tuple[bytes, ...] + members: tuple[bytes, ...] + + def serialize(self) -> bytes: + output = bytearray(self.segments[0]) + for member, segment in zip(self.members, self.segments[1:]): + output.extend(member) + output.extend(segment) + return bytes(output) + + def read_document_lines(path: str | Path, *, encoding: str) -> list[str]: """Read standard 80-byte SAV document lines without touching case data.""" data = Path(path).read_bytes() @@ -73,12 +97,12 @@ def write_document_lines(path: str | Path, lines: Iterable[str], *, encoding: st def write_compatible_names( path: str | Path, names: dict[str, str], *, encoding: str, ) -> None: - """Set exact legacy short names consistently in type-2/13/14 records. + """Set exact legacy short names and rewrite every standard short-name reference. - IBM I/O has no compatible-name setter. Every requested source name must - have a long-name record, and every very-long-string key must agree with the - type-2 and subtype-13 records. The fully rebuilt dictionary is reparsed - before an atomic replacement is published. + IBM I/O has no compatible-name setter. This narrowly rewrites the fixed + type-2 names and standard subtype-13/14, variable-set, and MR-set records. + The complete candidate is raw-validated and IBM Reader-validated before a + same-directory atomic publish. """ if not names: return @@ -86,126 +110,128 @@ def write_compatible_names( data = target.read_bytes() byte_order, records = _records(data) terminator = next(record for record in records if record.record_type == 999) - long_name_records = [ - record for record in records - if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 13 - ] - if not long_name_records: - raise RawDictionaryError("SAV dictionary has no long-variable-name record.") - if len(long_name_records) != 1: - raise RawDictionaryError("SAV dictionary has more than one long-variable-name record.") - long_names = long_name_records[0] - if _int(data, long_names.start + 8, byte_order) != 1: - raise RawDictionaryError("Invalid SAV long-variable-name record dimensions.") - pairs = _long_name_pairs(data[long_names.start + 16 : long_names.end], encoding) + primary_variables, long_names, pairs, very_long_records = _compatible_name_dictionary( + data, byte_order=byte_order, records=records, encoding=encoding, + ) + _require_unique_names(list(names), "requested source variable") + validated_names = { + source_name: _validated_compatible_name(value) + for source_name, value in names.items() + } replacements: dict[str, str] = {} updated_pairs: list[tuple[str, str]] = [] unresolved = set(names) for short_name, long_name in pairs: - replacement = names.get(long_name) + replacement = validated_names.get(long_name) if replacement is None: updated_pairs.append((short_name, long_name)) continue - replacements[short_name] = _validated_compatible_name(replacement) - updated_pairs.append((replacements[short_name], long_name)) + replacements[short_name.casefold()] = replacement + updated_pairs.append((replacement, long_name)) unresolved.remove(long_name) if unresolved: missing = ", ".join(sorted(unresolved)) raise RawDictionaryError( "Compatible-name update requires a long-name record for: " + missing ) - updated_short_names = [short_name.casefold() for short_name, _ in updated_pairs] - if len(updated_short_names) != len(set(updated_short_names)): - raise RawDictionaryError("Compatible-name update would create duplicate short names.") - type_2_names = _type_2_names(data, records, byte_order) - for short_name in replacements: - if type_2_names.count(short_name) != 1: - raise RawDictionaryError( - f"Expected exactly one type-2 record for compatible name {short_name!r}." - ) - - long_name_by_short = {short_name: long_name for short_name, long_name in pairs} - vls_records = [ - record for record in records - if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 14 + expected_primary_names = [ + replacements.get(short_name.casefold(), short_name) + for _record, short_name in primary_variables ] - parsed_vls: dict[int, tuple[list[tuple[str, bytes]], bool]] = {} - seen_vls: set[str] = set() - vls_replacements: dict[str, str] = {} - vls_source_names: set[str] = set() - for record in vls_records: - if _int(data, record.start + 8, byte_order) != 1: - raise RawDictionaryError("Invalid SAV very-long-string record dimensions.") - entries, trailing_tab = _very_long_string_entries(data[record.start + 16 : record.end]) - parsed_vls[record.start] = (entries, trailing_tab) - for short_name, _ in entries: - normalized = short_name.casefold() - if normalized in seen_vls: - raise RawDictionaryError("Duplicate SAV very-long-string entry.") - seen_vls.add(normalized) - if short_name not in long_name_by_short or type_2_names.count(short_name) != 1: - raise RawDictionaryError( - "SAV type-2/subtype-13/subtype-14 names are inconsistent." - ) - replacement = replacements.get(short_name) - if replacement is not None: - vls_replacements[short_name] = replacement - vls_source_names.add(long_name_by_short[short_name]) - + _require_unique_names(expected_primary_names, "compatible variable") mutable = bytearray(data) - for record in records: - if record.record_type != 2: - continue - short_name = bytes(mutable[record.start + 24 : record.start + 32]).decode(encoding).rstrip(" ") - replacement = replacements.get(short_name) + for record, short_name in primary_variables: + replacement = replacements.get(short_name.casefold()) if replacement is not None: - mutable[record.start + 24 : record.start + 32] = replacement.encode("ascii").ljust(8, b" ") + mutable[record.start + 24 : record.start + 32] = ( + replacement.encode("ascii").ljust(8, b" ") + ) - new_payload = b"\t".join( + long_name_payload = b" ".join( short_name.encode("ascii") + b"=" + long_name.encode(encoding) for short_name, long_name in updated_pairs ) - replacement_records = { - long_names.start: _extension_record( - mutable, long_names, new_payload, byte_order=byte_order, - ), + header = bytearray(mutable[long_names.start : long_names.start + 16]) + header[12:16] = _pack(len(long_name_payload), byte_order) + record_replacements = { + long_names.start: bytes(header) + long_name_payload, } - for record in vls_records: - entries, trailing_tab = parsed_vls[record.start] - rewritten = [ - (vls_replacements.get(short_name, short_name), width) - for short_name, width in entries - ] - vls_payload = _very_long_string_payload(rewritten, trailing_tab=trailing_tab) - replacement_records[record.start] = _extension_record( - mutable, record, vls_payload, byte_order=byte_order, + expected_very_long: list[tuple[tuple[str, bytes], ...]] = [] + for record, very_long_pairs in very_long_records: + updated_very_long = tuple( + (replacements.get(short_name.casefold(), short_name), width) + for short_name, width in very_long_pairs ) + expected_very_long.append(updated_very_long) + payload = _very_long_string_payload(updated_very_long) + very_long_header = bytearray(mutable[record.start : record.start + 16]) + very_long_header[8:12] = _pack(1, byte_order) + very_long_header[12:16] = _pack(len(payload), byte_order) + record_replacements[record.start] = bytes(very_long_header) + payload + reference_replacements, expected_reference_payloads = _rewrite_reference_records( + data, + byte_order=byte_order, + records=records, + encoding=encoding, + compatible_replacements=replacements, + known_compatible_names=[name for _record, name in primary_variables], + known_long_names=[long_name for _short_name, long_name in pairs], + ) + record_replacements.update(reference_replacements) + + semantics_before = _read_dictionary_semantics(target) + expected_compatible_names = dict(semantics_before.compatible_names) + for source_name, compatible_name in validated_names.items(): + if source_name not in expected_compatible_names: + raise RawDictionaryError( + "IBM Reader did not expose the requested source variable: " + + source_name + ) + expected_compatible_names[source_name] = compatible_name + _require_unique_names( + expected_compatible_names.values(), "Reader-compatible variable", + ) chunks: list[bytes] = [] cursor = 0 - for record in sorted( - (record for record in records if record.start in replacement_records), - key=lambda item: item.start, - ): - chunks.append(bytes(mutable[cursor : record.start])) - chunks.append(replacement_records[record.start]) + for record in records: + replacement = record_replacements.get(record.start) + if replacement is None: + continue + chunks.extend((bytes(mutable[cursor : record.start]), replacement)) cursor = record.end chunks.append(bytes(mutable[cursor:])) updated = b"".join(chunks) - updated = _shift_zsav_offsets( + + _assert_compatible_name_rewrite( + updated, byte_order=byte_order, encoding=encoding, + expected_primary_names=expected_primary_names, + expected_pairs=updated_pairs, expected_very_long=expected_very_long, + expected_reference_payloads=expected_reference_payloads, + ) + shifted = _shift_zsav_offsets( updated, original_data_start=terminator.end, delta=len(updated) - len(data), byte_order=byte_order, ) - _assert_compatible_name_consistency( - updated, - requested=names, - vls_source_names=vls_source_names, - encoding=encoding, + _assert_compatible_name_rewrite( + shifted, byte_order=byte_order, encoding=encoding, + expected_primary_names=expected_primary_names, + expected_pairs=updated_pairs, expected_very_long=expected_very_long, + expected_reference_payloads=expected_reference_payloads, + ) + _atomic_write_bytes( + target, + shifted, + validator=lambda candidate: _assert_dictionary_semantics( + candidate, + expected_compatible_names=expected_compatible_names, + expected_variable_sets=semantics_before.variable_sets, + expected_multiple_response_sets=semantics_before.multiple_response_sets, + ), ) - _atomic_write_bytes(target, updated) def write_extended_mrset_labels( @@ -279,184 +305,515 @@ def write_extended_mrset_labels( )) +def _compatible_name_dictionary( + data: bytes, *, byte_order: str, records: list[_Record], encoding: str, +): + primary_variables: list[tuple[_Record, str]] = [] + for record in records: + if record.record_type != 2 or _int(data, record.start + 4, byte_order) < 0: + continue + raw_name = data[record.start + 24 : record.start + 32] + try: + short_name = raw_name.decode("ascii").rstrip(" ") + except UnicodeDecodeError as error: + raise RawDictionaryError("Invalid type-2 compatible variable name.") from error + _validated_compatible_name(short_name) + primary_variables.append((record, short_name)) + _require_unique_names( + [short_name for _record, short_name in primary_variables], + "type-2 compatible variable", + ) + + long_name_records = [ + record for record in records + if record.record_type == 7 + and _int(data, record.start + 4, byte_order) == 13 + ] + if len(long_name_records) != 1: + raise RawDictionaryError("Expected exactly one long-variable-name record.") + long_names = long_name_records[0] + if _int(data, long_names.start + 8, byte_order) != 1: + raise RawDictionaryError("Invalid SAV long-variable-name record element size.") + pairs = _long_name_pairs(data[long_names.start + 16 : long_names.end], encoding) + + primary_keys = { + short_name.casefold() for _record, short_name in primary_variables + } + long_name_keys = {short_name.casefold() for short_name, _long_name in pairs} + missing_type_2 = sorted( + short_name for short_name, _long_name in pairs + if short_name.casefold() not in primary_keys + ) + if missing_type_2: + raise RawDictionaryError( + "Subtype-13 names have no matching type-2 record: " + + ", ".join(missing_type_2) + ) + + very_long_records: list[tuple[_Record, tuple[tuple[str, bytes], ...]]] = [] + all_very_long_names: list[str] = [] + for record in records: + if record.record_type != 7 or _int(data, record.start + 4, byte_order) != 14: + continue + if _int(data, record.start + 8, byte_order) != 1: + raise RawDictionaryError("Invalid SAV very-long-string record element size.") + very_long_pairs = tuple( + _very_long_string_pairs(data[record.start + 16 : record.end]) + ) + very_long_records.append((record, very_long_pairs)) + all_very_long_names.extend(short_name for short_name, _width in very_long_pairs) + _require_unique_names(all_very_long_names, "subtype-14 compatible variable") + missing_long_names = sorted( + short_name for short_name in all_very_long_names + if short_name.casefold() not in long_name_keys + ) + if missing_long_names: + raise RawDictionaryError( + "Subtype-14 names have no matching subtype-13 entry: " + + ", ".join(missing_long_names) + ) + return primary_variables, long_names, pairs, very_long_records + + +def _read_dictionary_semantics(path: Path) -> _DictionarySemantics: + try: + with pyspssio.Reader(str(path), mode="r") as reader: + compatible_names = deepcopy(reader.var_compat_names) + variable_sets = deepcopy(reader.var_sets) + multiple_response_sets = deepcopy(reader.mrsets) + except Exception as error: + raise RawDictionaryError( + "IBM Reader could not inspect compatible-name references." + ) from error + if not isinstance(compatible_names, dict): + raise RawDictionaryError("Invalid IBM Reader compatible-name dictionary.") + if not isinstance(variable_sets, dict): + raise RawDictionaryError("Invalid IBM Reader variable-set dictionary.") + if not isinstance(multiple_response_sets, dict): + raise RawDictionaryError("Invalid IBM Reader multiple-response dictionary.") + for members in variable_sets.values(): + if not isinstance(members, list): + raise RawDictionaryError("Invalid IBM Reader variable-set member list.") + for definition in multiple_response_sets.values(): + if not isinstance(definition, dict) or not isinstance( + definition.get("variable_list"), list, + ): + raise RawDictionaryError("Invalid IBM Reader multiple-response definition.") + return _DictionarySemantics( + compatible_names=compatible_names, + variable_sets=variable_sets, + multiple_response_sets=multiple_response_sets, + ) + + +def _assert_dictionary_semantics( + path: Path, *, expected_compatible_names: dict[str, str], + expected_variable_sets: dict[str, list[str]], + expected_multiple_response_sets: dict[str, dict], +) -> None: + observed = _read_dictionary_semantics(path) + if observed.compatible_names != expected_compatible_names: + raise RawDictionaryError("Compatible-name Reader readback differs from candidate.") + if observed.variable_sets != expected_variable_sets: + raise RawDictionaryError("Variable-set Reader readback changed semantics.") + if observed.multiple_response_sets != expected_multiple_response_sets: + raise RawDictionaryError("Multiple-response Reader readback changed semantics.") + + +def _rewrite_reference_member( + raw_member: bytes, + *, + subtype: int, + encoding: str, + known_compatible: dict[str, str], + known_long: dict[str, str], + encoded_replacements: dict[str, bytes], +) -> bytes: + try: + compatible = raw_member.decode("ascii") + _validated_compatible_name(compatible) + except (UnicodeDecodeError, RawDictionaryError): + compatible = None + if compatible is not None and compatible.casefold() in known_compatible: + return encoded_replacements.get(compatible.casefold(), raw_member) + try: + long_name = raw_member.decode(encoding) + except UnicodeDecodeError as error: + raise RawDictionaryError( + f"Invalid subtype-{subtype} variable member token." + ) from error + if long_name.casefold() in known_long: + return raw_member + raise RawDictionaryError( + f"Subtype-{subtype} references unknown variable {long_name!r}." + ) + + +def _rewrite_reference_records( + data: bytes, *, byte_order: str, records: list[_Record], encoding: str, + compatible_replacements: dict[str, str], known_compatible_names: Iterable[str], + known_long_names: Iterable[str], +) -> tuple[dict[int, bytes], dict[int, tuple[bytes, ...]]]: + known_names = list(known_compatible_names) + _require_unique_names(known_names, "reference-compatible variable") + known = {name.casefold(): name for name in known_names} + encoded_replacements = { + key: value.encode("ascii") for key, value in compatible_replacements.items() + } + long_names = list(known_long_names) + _require_unique_names(long_names, "reference long variable") + known_long = {name.casefold(): name for name in long_names} + replacements: dict[int, bytes] = {} + expected: dict[int, list[bytes]] = {5: [], 7: [], 19: []} + seen_variable_sets: dict[str, str] = {} + seen_mrsets: dict[str, str] = {} + for record in records: + if record.record_type != 7: + continue + subtype = _int(data, record.start + 4, byte_order) + if subtype not in expected: + continue + if _int(data, record.start + 8, byte_order) != 1: + raise RawDictionaryError( + f"Subtype-{subtype} reference record must have element size 1." + ) + payload = data[record.start + 16 : record.end] + lines, suffix = _reference_payload_lines(payload, subtype=subtype) + updated_lines: list[_ReferenceLine] = [] + seen = seen_variable_sets if subtype == 5 else seen_mrsets + for raw_line in lines: + parsed = _parse_reference_line(raw_line, subtype=subtype, encoding=encoding) + try: + set_name = parsed.set_name.decode(encoding) + except UnicodeDecodeError as error: + raise RawDictionaryError( + f"Subtype-{subtype} set name does not match file encoding." + ) from error + set_key = set_name.casefold() + if set_key in seen: + raise RawDictionaryError( + f"Duplicate subtype-{subtype} set name: {seen[set_key]!r} and {set_name!r}." + ) + seen[set_key] = set_name + members: list[bytes] = [] + for raw_member in parsed.members: + members.append(_rewrite_reference_member( + raw_member, + subtype=subtype, + encoding=encoding, + known_compatible=known, + known_long=known_long, + encoded_replacements=encoded_replacements, + )) + updated_lines.append(_ReferenceLine( + set_name=parsed.set_name, + segments=parsed.segments, + members=tuple(members), + )) + new_payload = b"\n".join(line.serialize() for line in updated_lines) + suffix + header = bytearray(data[record.start : record.start + 16]) + header[8:12] = _pack(1, byte_order) + header[12:16] = _pack(len(new_payload), byte_order) + replacements[record.start] = bytes(header) + new_payload + expected[subtype].append(new_payload) + return replacements, { + subtype: tuple(payloads) for subtype, payloads in expected.items() + } + + +def _reference_payload_lines(payload: bytes, *, subtype: int) -> tuple[list[bytes], bytes]: + if not payload: + raise RawDictionaryError(f"Empty subtype-{subtype} reference record.") + body_end = len(payload) + while body_end and payload[body_end - 1] == 0: + body_end -= 1 + suffix = payload[body_end:] + body = payload[:body_end] + if body.endswith(b"\n"): + body = body[:-1] + suffix = b"\n" + suffix + if not body or body.endswith(b"\n") or b"\x00" in body or b"\r" in body: + raise RawDictionaryError(f"Invalid subtype-{subtype} line framing.") + lines = body.split(b"\n") + if any(not line for line in lines): + raise RawDictionaryError(f"Invalid empty subtype-{subtype} definition.") + return lines, suffix + + +def _parse_reference_line(line: bytes, *, subtype: int, encoding: str) -> _ReferenceLine: + separator = line.find(b"=") + if separator <= 0: + raise RawDictionaryError(f"Invalid subtype-{subtype} set definition.") + set_name = line[:separator] + try: + set_name.decode(encoding) + except UnicodeDecodeError as error: + raise RawDictionaryError(f"Invalid subtype-{subtype} set name.") from error + definition = line[separator + 1:] + member_start = 0 if subtype == 5 else _mrset_member_start( + definition, subtype=subtype, encoding=encoding, + ) + absolute_start = separator + 1 + member_start + member_region = line[absolute_start:] + matches = list(re.finditer(rb"[^ \t]+", member_region)) + if not matches: + raise RawDictionaryError(f"Subtype-{subtype} definition has no members.") + segments: list[bytes] = [] + members: list[bytes] = [] + cursor = 0 + for match in matches: + segments.append(line[cursor : absolute_start + match.start()]) + members.append(match.group()) + cursor = absolute_start + match.end() + segments.append(line[cursor:]) + return _ReferenceLine(set_name, tuple(segments), tuple(members)) + + +def _mrset_member_start(definition: bytes, *, subtype: int, encoding: str) -> int: + if definition.startswith(b"C "): + if subtype not in {7, 19}: + raise RawDictionaryError("C multiple-response definition has invalid subtype.") + label_length, label_start = _ascii_length_field(definition, 2) + return _length_delimited_end( + definition, label_start, label_length, encoding=encoding, + description="multiple-response label", + ) + if definition.startswith(b"D"): + if subtype not in {7, 19}: + raise RawDictionaryError("D multiple-response definition has invalid subtype.") + first_space = definition.find(b" ") + if first_space < 2 or not definition[1:first_space].isdigit(): + raise RawDictionaryError("Invalid D multiple-response counted-value length.") + value_length = int(definition[1:first_space]) + label_field = _length_delimited_end( + definition, first_space + 1, value_length, encoding=encoding, + description="multiple-response counted value", return_after_separator=True, + ) + label_length, label_start = _ascii_length_field(definition, label_field) + return _length_delimited_end( + definition, label_start, label_length, encoding=encoding, + description="multiple-response label", + ) + if subtype == 19 and definition.startswith(b"E "): + flags, position = _ascii_token(definition, 2, "extended flags") + if not flags.isdigit(): + raise RawDictionaryError("Invalid extended multiple-response flags.") + value_length, value_start = _ascii_length_field(definition, position) + label_field = _length_delimited_end( + definition, value_start, value_length, encoding=encoding, + description="extended counted value", return_after_separator=True, + ) + label_length, label_start = _ascii_length_field(definition, label_field) + return _length_delimited_end( + definition, label_start, label_length, encoding=encoding, + description="extended multiple-response label", + ) + raise RawDictionaryError(f"Unsupported subtype-{subtype} multiple-response definition.") + + +def _ascii_token(data: bytes, start: int, description: str) -> tuple[bytes, int]: + end = data.find(b" ", start) + if end <= start: + raise RawDictionaryError(f"Invalid {description} field.") + return data[start:end], end + 1 + + +def _ascii_length_field(data: bytes, start: int) -> tuple[int, int]: + token, following = _ascii_token(data, start, "byte-length") + if not token.isdigit(): + raise RawDictionaryError("Invalid multiple-response byte-length field.") + return int(token), following + + +def _length_delimited_end( + data: bytes, start: int, length: int, *, encoding: str, description: str, + return_after_separator: bool = False, +) -> int: + end = start + length + if length < 0 or end >= len(data) or data[end:end + 1] != b" ": + raise RawDictionaryError(f"Invalid {description} byte length.") + try: + data[start:end].decode(encoding) + except UnicodeDecodeError as error: + raise RawDictionaryError(f"Invalid {description} encoding.") from error + following = end + 1 + if return_after_separator: + return following + if following >= len(data): + raise RawDictionaryError(f"{description.capitalize()} has no member list.") + return following + + def _long_name_pairs(payload: bytes, encoding: str) -> list[tuple[str, str]]: if not payload: raise RawDictionaryError("Empty SAV long-variable-name record.") pairs: list[tuple[str, str]] = [] - short_names: set[str] = set() - long_names: set[str] = set() - for raw_pair in payload.split(b"\t"): + for raw_pair in payload.split(b" "): try: raw_short, raw_long = raw_pair.split(b"=", maxsplit=1) - short_name = _dictionary_short_name(raw_short) + short_name = raw_short.decode("ascii") long_name = raw_long.decode(encoding) except (UnicodeDecodeError, ValueError) as error: raise RawDictionaryError("Invalid SAV long-variable-name record.") from error - normalized_short = short_name.casefold() - normalized_long = long_name.casefold() - if not long_name or normalized_short in short_names or normalized_long in long_names: - raise RawDictionaryError("Duplicate or empty SAV long-variable-name entry.") - short_names.add(normalized_short) - long_names.add(normalized_long) + _validated_compatible_name(short_name) + if not long_name or any(character in long_name for character in "\x00 ="): + raise RawDictionaryError("Invalid SAV long-variable-name record.") pairs.append((short_name, long_name)) + _require_unique_names( + [short_name for short_name, _long_name in pairs], + "subtype-13 compatible variable", + ) + _require_unique_names( + [long_name for _short_name, long_name in pairs], + "subtype-13 long variable", + ) return pairs -def _very_long_string_entries(payload: bytes) -> tuple[list[tuple[str, bytes]], bool]: - """Parse subtype-14 entries without accepting ambiguous separators or widths.""" - if not payload: - raise RawDictionaryError("Empty SAV very-long-string record.") - raw_entries = payload.split(b"\t") - trailing_tab = raw_entries[-1] == b"" - if trailing_tab: - raw_entries.pop() - if not raw_entries or any(not entry for entry in raw_entries): - raise RawDictionaryError("Invalid SAV very-long-string separators.") - entries: list[tuple[str, bytes]] = [] - for raw_entry in raw_entries: - if not raw_entry.endswith(b"\x00"): - raise RawDictionaryError("SAV very-long-string entry is missing its NUL terminator.") +def _very_long_string_entries( + payload: bytes, +) -> tuple[list[tuple[str, bytes]], bool]: + """Compatibility view for callers that also consumed the terminator flag.""" + return _very_long_string_pairs(payload), True + +def _very_long_string_pairs(payload: bytes) -> list[tuple[str, bytes]]: + if not payload or not payload.endswith(b" "): + raise RawDictionaryError("Invalid SAV very-long-string record terminator.") + raw_pairs = payload.split(b" ") + if raw_pairs[-1] != b"": + raise RawDictionaryError("Invalid SAV very-long-string record separator.") + pairs: list[tuple[str, bytes]] = [] + for raw_pair in raw_pairs[:-1]: + if not raw_pair.endswith(b"\x00"): + raise RawDictionaryError("Invalid SAV very-long-string entry terminator.") try: - raw_short, raw_width = raw_entry[:-1].split(b"=", maxsplit=1) - short_name = _dictionary_short_name(raw_short) - if not raw_width or not raw_width.isdigit(): - raise ValueError - width = int(raw_width.decode("ascii")) + raw_short, raw_width = raw_pair[:-1].split(b"=", maxsplit=1) + short_name = raw_short.decode("ascii") except (UnicodeDecodeError, ValueError) as error: - raise RawDictionaryError("Invalid SAV very-long-string entry.") from error - if not 256 <= width <= 32767: - raise RawDictionaryError("SAV very-long-string width is outside 256..32767.") - entries.append((short_name, raw_width)) - return entries, trailing_tab - - -def _very_long_string_payload( - entries: Iterable[tuple[str, bytes]], *, trailing_tab: bool, -) -> bytes: - payload = b"\t".join( - short_name.encode("ascii") + b"=" + width + b"\x00" - for short_name, width in entries + raise RawDictionaryError("Invalid SAV very-long-string record.") from error + _validated_compatible_name(short_name) + if not raw_width or not raw_width.isdigit() or int(raw_width) <= 255: + raise RawDictionaryError("Invalid SAV very-long-string width.") + pairs.append((short_name, raw_width)) + _require_unique_names( + [short_name for short_name, _width in pairs], + "subtype-14 compatible variable", ) - return payload + (b"\t" if trailing_tab else b"") + return pairs -def _dictionary_short_name(raw_name: bytes) -> str: - try: - name = raw_name.decode("ascii") - except UnicodeDecodeError as error: - raise RawDictionaryError("SAV dictionary short name is not ASCII.") from error - allowed_first = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@#$" - allowed_rest = allowed_first + b"0123456789_." - if not 1 <= len(raw_name) <= 8 or raw_name[:1] not in allowed_first: - raise RawDictionaryError("Invalid SAV dictionary short name.") - if any(character not in allowed_rest for character in raw_name[1:]): - raise RawDictionaryError("Invalid SAV dictionary short name.") - return name - - -def _type_2_names(data: bytes, records: list[_Record], byte_order: str) -> list[str]: - names: list[str] = [] - for record in records: - if record.record_type != 2 or _int(data, record.start + 4, byte_order) < 0: - continue - raw_name = data[record.start + 24 : record.start + 32].rstrip(b" ") - names.append(_dictionary_short_name(raw_name)) - return names +def _very_long_string_payload(pairs: Iterable[tuple[str, bytes]]) -> bytes: + return b"".join( + short_name.encode("ascii") + b"=" + width + b"\x00 " + for short_name, width in pairs + ) -def _extension_record( - data: bytes, record: _Record, payload: bytes, *, byte_order: str, -) -> bytes: - header = bytearray(data[record.start : record.start + 16]) - header[8:12] = _pack(1, byte_order) - header[12:16] = _pack(len(payload), byte_order) - return bytes(header) + payload +def _require_unique_names(names: Iterable[str], description: str) -> None: + seen: dict[str, str] = {} + for name in names: + key = name.casefold() + if key in seen: + raise RawDictionaryError( + f"Duplicate {description} name: {seen[key]!r} and {name!r}." + ) + seen[key] = name -def _assert_compatible_name_consistency( - data: bytes, - *, - requested: dict[str, str], - vls_source_names: set[str], - encoding: str, +def _assert_compatible_name_rewrite( + data: bytes, *, byte_order: str, encoding: str, + expected_primary_names: list[str], expected_pairs: list[tuple[str, str]], + expected_very_long: list[tuple[tuple[str, bytes], ...]], + expected_reference_payloads: dict[int, tuple[bytes, ...]], ) -> None: - byte_order, records = _records(data) - type_2_names = _type_2_names(data, records, byte_order) - normalized_type_2 = [name.casefold() for name in type_2_names] - long_name_records = [ - record for record in records - if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 13 - ] - if len(long_name_records) != 1: - raise RawDictionaryError("Rewritten SAV has an inconsistent long-variable-name record.") - long_name_record = long_name_records[0] - if _int(data, long_name_record.start + 8, byte_order) != 1: - raise RawDictionaryError("Rewritten SAV has invalid long-variable-name dimensions.") - pairs = _long_name_pairs( - data[long_name_record.start + 16 : long_name_record.end], encoding, + observed_byte_order, observed_records = _records(data) + if observed_byte_order != byte_order: + raise RawDictionaryError("Compatible-name rewrite changed SAV byte order.") + primary, _long_record, pairs, very_long = _compatible_name_dictionary( + data, byte_order=byte_order, records=observed_records, encoding=encoding, + ) + if [short_name for _record, short_name in primary] != expected_primary_names: + raise RawDictionaryError( + "Compatible-name rewrite did not update type-2 records consistently." + ) + if pairs != expected_pairs: + raise RawDictionaryError( + "Compatible-name rewrite did not update subtype-13 consistently." + ) + if [items for _record, items in very_long] != expected_very_long: + raise RawDictionaryError( + "Compatible-name rewrite did not update subtype-14 consistently." + ) + _unused, observed_reference_payloads = _rewrite_reference_records( + data, + byte_order=byte_order, + records=observed_records, + encoding=encoding, + compatible_replacements={}, + known_compatible_names=expected_primary_names, + known_long_names=[long_name for _short_name, long_name in pairs], ) - pair_by_long = {long_name: short_name for short_name, long_name in pairs} - normalized_pair_names = {short_name.casefold() for short_name, _ in pairs} - for short_name, _ in pairs: - if normalized_type_2.count(short_name.casefold()) != 1: - raise RawDictionaryError("Rewritten type-2 and subtype-13 names are inconsistent.") + if observed_reference_payloads != expected_reference_payloads: + raise RawDictionaryError( + "Compatible-name rewrite changed a set reference outside member tokens." + ) - vls_names: list[str] = [] - for record in records: - if record.record_type != 7 or _int(data, record.start + 4, byte_order) != 14: - continue - if _int(data, record.start + 8, byte_order) != 1: - raise RawDictionaryError("Rewritten SAV has invalid very-long-string dimensions.") - entries, _ = _very_long_string_entries(data[record.start + 16 : record.end]) - vls_names.extend(short_name for short_name, _ in entries) - normalized_vls = [name.casefold() for name in vls_names] - if len(normalized_vls) != len(set(normalized_vls)): - raise RawDictionaryError("Rewritten SAV has duplicate very-long-string entries.") - for short_name in normalized_vls: - if short_name not in normalized_pair_names or normalized_type_2.count(short_name) != 1: - raise RawDictionaryError( - "Rewritten type-2/subtype-13/subtype-14 names are inconsistent." - ) - for source_name, requested_name in requested.items(): - replacement = _validated_compatible_name(requested_name) - if pair_by_long.get(source_name) != replacement: - raise RawDictionaryError("Requested compatible name is absent after dictionary rewrite.") - if normalized_type_2.count(replacement.casefold()) != 1: - raise RawDictionaryError("Requested compatible name is inconsistent with type-2 records.") - if ( - source_name in vls_source_names - and normalized_vls.count(replacement.casefold()) != 1 - ): - raise RawDictionaryError("Requested compatible name is inconsistent with subtype-14.") +def _atomic_write_bytes( + target: Path, data: bytes, *, validator: Callable[[Path], None] | None = None, +) -> None: + descriptor, temporary_name = mkstemp( + dir=target.parent, + prefix=f".{target.stem}.", + suffix=f".tmp{target.suffix}", + ) + temporary = Path(temporary_name) + try: + stream = os.fdopen(descriptor, "wb") + descriptor = -1 + with stream: + _write_all(stream, data) + stream.flush() + os.fsync(stream.fileno()) + if validator is not None: + validator(temporary) + os.chmod(temporary, target.stat().st_mode & 0o7777) + os.replace(temporary, target) + _best_effort_fsync_directory(target.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) -def _atomic_write_bytes(target: Path, data: bytes) -> None: - """Publish validated dictionary bytes without exposing a partial replacement.""" - mode = stat.S_IMODE(target.stat().st_mode) - temporary: Path | None = None +def _best_effort_fsync_directory(directory: Path) -> None: + descriptor = -1 try: - with NamedTemporaryFile( - mode="wb", - dir=target.parent, - prefix=f".{target.name}.", - suffix=".tmp", - delete=False, - ) as handle: - temporary = Path(handle.name) - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - os.chmod(temporary, mode) - os.replace(temporary, target) - temporary = None + descriptor = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + os.fsync(descriptor) + except OSError: + pass finally: - if temporary is not None: - temporary.unlink(missing_ok=True) + if descriptor >= 0: + os.close(descriptor) + + +def _write_all(stream, data: bytes) -> None: + remaining = memoryview(data) + while remaining: + written = stream.write(remaining) + if written is None or written <= 0: + raise OSError("Temporary SAV publish made no write progress.") + remaining = remaining[written:] def _validated_compatible_name(value: str) -> str: - encoded = str(value).encode("ascii") + try: + encoded = str(value).encode("ascii") + except UnicodeEncodeError as error: + raise RawDictionaryError( + "SPSS compatible variable names must contain one to eight ASCII bytes." + ) from error if not 1 <= len(encoded) <= 8: raise RawDictionaryError("SPSS compatible variable names must contain one to eight ASCII bytes.") first = encoded[:1] @@ -517,7 +874,7 @@ def _document_record(lines: Iterable[str], *, encoding: str, byte_order: str) -> def _records(data: bytes) -> tuple[str, list[_Record]]: - if len(data) < _HEADER_SIZE or data[:4] not in {b"$FL2", b"$FL3"}: + if len(data) < _HEADER_SIZE or bytes(data[:4]) not in {b"$FL2", b"$FL3"}: raise RawDictionaryError("Not a supported SAV or ZSAV system file.") byte_order = _byte_order(data) offset = _HEADER_SIZE diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 7e51cfa..b4cae12 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -7,12 +7,13 @@ """ import hashlib +import os import json import math -from contextlib import ExitStack +from contextlib import ExitStack, contextmanager from datetime import UTC, datetime from pathlib import Path -from tempfile import TemporaryDirectory +from tempfile import mkstemp, TemporaryDirectory from typing import Any import pandas as pd @@ -37,18 +38,35 @@ set_variable_attribute_pairs, variable_attribute_pairs, ) -from ..core import UnsupportedOperationError -from ..sql.capabilities import effective_profile -from ..sql.profiles import preflight +from ..core import ( + UnsupportedOperationError, + safe_error_identity as _export_error_identity, +) +from ..sql.dolt_conformance import DoltConformanceSource from ..sql.wide import ( create_wide_dataset, + fail_export_operation, + finish_export_operation, physical_name, + read_export_operation_state, read_fidelity_events, read_wide_dataset, + record_export_backup_retained, + record_export_cleanup_failure, record_export_operation, validate_spss_catalog, ) + +class ExportRecoveryError(UnsupportedOperationError): + """Export publication could not restore the destination's prior state.""" + + def __init__(self, code: str, detail: str, *, details: dict[str, Any]) -> None: + super().__init__(f"OpenStatSpec export recovery failed [{code}]: {detail}") + self.code = code + self.details = {"reason": code, **details} + + _UTF8_ENCODINGS = {"UTF-8", "UTF8"} def engine_identity() -> dict[str, str]: """Return the exact pinned SPSS engine identity for audit records.""" @@ -82,10 +100,16 @@ def _dictionary(source_path: Path) -> tuple[dict[str, Any], dict[str, Any]]: ) except RawDictionaryError as error: metadata["_documents"] = None - metadata["_documents_error"] = str(error) + metadata["_documents_error"] = _export_error_identity( + error, + phase="read_sav_documents", + ) except Exception as error: # The source stays importable, but not silently faithful. metadata["_var_sets"] = None - metadata["_var_sets_error"] = str(error) + metadata["_var_sets_error"] = _export_error_identity( + error, + phase="read_sav_variable_sets", + ) return metadata, _engine_loss_report(metadata) @@ -217,7 +241,10 @@ def inspect_sav(source: str | Path) -> dict[str, Any]: } -def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str) -> dict[str, Any]: +def import_sav_dataset( + *, source: str | Path, database_url: str, dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: source_path = Path(source) _require_source(source_path) frame, metadata = pyspssio.read_sav( @@ -233,6 +260,7 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str result = create_wide_dataset( database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, source_name=source_path.name, source_format=source_path.suffix[1:].upper(), rows=_rows(frame, variables), @@ -257,24 +285,256 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str return {**result, "loss_report": loss_report} +def _path_reference(path: Path, *, role: str) -> dict[str, str]: + """Return an opaque path identity without disclosing any path component.""" + absolute_path = os.path.abspath(os.fspath(path)) + return { + "role": role, + "path_sha256": hashlib.sha256( + absolute_path.encode("utf-8") + ).hexdigest(), + } + + +def _path_reference_text(path: Path, *, role: str) -> str: + """Serialize a redacted path identity for legacy text audit columns.""" + return json.dumps( + _path_reference(path, role=role), + sort_keys=True, separators=(",", ":"), + ) + + +def _reserve_export_backup(destination: Path) -> Path: + descriptor, name = mkstemp( + dir=destination.parent, prefix=f".{destination.name}.", + suffix=".previous", + ) + os.close(descriptor) + return Path(name) + + +def _restore_export_destination( + *, destination: Path, backup: Path, had_previous: bool, +) -> None: + if had_previous: + os.replace(backup, destination) + else: + destination.unlink(missing_ok=True) + + +def _raise_export_cleanup_failed( + *, original_error: Exception, cleanup_error: Exception, + phase: str, destination: Path, backup: Path, staged: Path, + had_previous: bool, database_url: str, operation_id: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + inventory = { + "destination": _path_reference(destination, role="destination"), + "destination_exists": destination.exists(), + "backup": _path_reference(backup, role="durable_backup"), + "backup_exists": backup.exists(), + "staged_export": _path_reference(staged, role="staged_export"), + "staged_export_exists": staged.exists(), + } + inventory_sha256 = hashlib.sha256( + json.dumps(inventory, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + action_id = hashlib.sha256( + json.dumps( + { + "destination_path_sha256": inventory["destination"]["path_sha256"], + "phase": phase, + }, + sort_keys=True, separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + recovery = { + "procedure_id": "openstatspec.export-destination-restore.v1", + "action_id": action_id, + "targets": { + "destination": inventory["destination"], + "durable_backup": inventory["backup"], + "staged_export": inventory["staged_export"], + }, + "residual_inventory_sha256": inventory_sha256, + "cleanup_attempted": True, + "cleanup_succeeded": False, + "previous_destination_existed": had_previous, + "durable_backup_survives_staging_cleanup": backup.exists(), + } + cleanup_audit_operation_id = None + cleanup_audit_fault = None + try: + cleanup_audit_operation_id = record_export_cleanup_failure( + database_url=database_url, + destination=_path_reference_text( + destination, role="destination", + ), + original_error=original_error, cleanup_error=cleanup_error, + residual_object_inventory=inventory, + deterministic_recovery_evidence=recovery, + operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as audit_error: + cleanup_audit_fault = _export_error_identity( + audit_error, phase="cleanup_failed_audit", + ) + exception_recovery = { + **recovery, + "cleanup_failed_audit_persisted": cleanup_audit_fault is None, + "cleanup_failed_audit_operation_id": cleanup_audit_operation_id, + "terminal_reporting": ( + "catalog_and_exception" if cleanup_audit_fault is None + else "out_of_band_exception" + ), + } + raise ExportRecoveryError( + "cleanup_failed", + "Export failed and the destination's prior state could not be restored.", + details={ + "subcode": "export_destination_restore_failed", + "original_cause": _export_error_identity( + original_error, phase=f"export_{phase}", + ), + "cleanup_fault": _export_error_identity( + cleanup_error, phase="export_destination_restore", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": exception_recovery, + "audit_fault": cleanup_audit_fault, + "success_forbidden": True, + }, + ) from cleanup_error + + +def _mark_export_failed_after_restore( + *, database_url: str, operation_id: str, error: Exception, phase: str, + destination: Path, backup: Path, had_previous: bool, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Close a running audit after the old destination has been restored.""" + failure = { + "phase": phase, + "cause": _export_error_identity(error, phase=f"export_{phase}"), + "destination": _path_reference(destination, role="destination"), + "durable_backup": _path_reference(backup, role="durable_backup"), + "previous_destination_existed": had_previous, + "destination_restored": True, + } + try: + fail_export_operation( + database_url=database_url, operation_id=operation_id, + failure_details=failure, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as audit_error: + raise ExportRecoveryError( + "failure_audit_failed", + "The destination was restored, but the running export audit could not be closed.", + details={ + "subcode": "export_failure_audit_failed", + "original_cause": failure["cause"], + "audit_fault": _export_error_identity( + audit_error, phase="export_failure_audit", + ), + "residual_object_inventory": { + "destination": _path_reference( + destination, role="destination", + ), + "destination_exists": destination.exists(), + "backup": _path_reference( + backup, role="durable_backup", + ), + "backup_exists": backup.exists(), + "operation_id": operation_id, + }, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.export-audit-reconciliation.v1", + "action_id": operation_id, + "phase": phase, + "destination_restored": True, + "operation_terminal_state_verified": False, + "terminal_reporting": "out_of_band_exception", + }, + "success_forbidden": True, + }, + ) from audit_error + + +@contextmanager +def _export_staging_directory( + *, destination: Path, database_url: str, + publication_state: dict[str, Any], + dolt_conformance_source: DoltConformanceSource | None = None, +): + """Compensate a published file if staging-directory cleanup fails.""" + try: + with TemporaryDirectory( + dir=destination.parent, + prefix=f".{destination.name}.staging.", + ) as export_directory: + yield export_directory + except Exception as staging_error: + if not publication_state.get("published"): + raise + backup = publication_state["backup"] + staged = publication_state["staged"] + had_previous = publication_state["had_previous"] + operation_id = publication_state["operation_id"] + try: + _restore_export_destination( + destination=destination, + backup=backup, + had_previous=had_previous, + ) + except Exception as restore_error: + _raise_export_cleanup_failed( + original_error=staging_error, + cleanup_error=restore_error, + phase="staging_cleanup", + destination=destination, + backup=backup, + staged=staged, + had_previous=had_previous, + database_url=database_url, + operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + _mark_export_failed_after_restore( + database_url=database_url, + operation_id=operation_id, + error=staging_error, + phase="staging_cleanup", + destination=destination, + backup=backup, + had_previous=had_previous, + dolt_conformance_source=dolt_conformance_source, + ) + raise + + def export_sav_dataset( *, database_url: str, dataset_id: str, destination: str | Path, allow_loss: tuple[str, ...] = (), legacy_locale: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: destination_path = Path(destination) if destination_path.suffix.lower() not in {".sav", ".zsav"}: raise UnsupportedOperationError("Export destinations must use the .sav or .zsav extension.") - profile, _active = effective_profile(database_url) dataset, variables, rows = read_wide_dataset( - database_url=database_url, dataset_id=dataset_id, profile=profile, + database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, ) - preflight(profile, variables, rows=rows) validate_spss_catalog( variables, case_weight_variable=dataset.get("case_weight_variable"), multiple_response_sets=dataset.get("multiple_response_sets"), ) - persisted_events = read_fidelity_events(database_url=database_url, dataset_id=dataset_id) + persisted_events = read_fidelity_events( + database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + ) if legacy_locale is not None: persisted_events = tuple( event for event in persisted_events @@ -300,22 +560,244 @@ def export_sav_dataset( ], columns=[variable["source_name"] for variable in variables], ) - try: + accepted_events = tuple( + event for event in loss_report if event["code"] in allow_loss + ) + operation_id = None + publication_state: dict[str, Any] = {"published": False} + with _export_staging_directory( + destination=destination_path, database_url=database_url, + publication_state=publication_state, + dolt_conformance_source=dolt_conformance_source, + ) as export_directory: + staged_destination = Path(export_directory) / destination_path.name _write_with_dictionary_bridge( - destination_path, frame, dataset, variables, legacy_locale=legacy_locale, + staged_destination, frame, dataset, variables, + legacy_locale=legacy_locale, ) - except Exception: - destination_path.unlink(missing_ok=True) - raise - operation_id = record_export_operation( - database_url=database_url, - dataset_id=dataset_id, - destination=str(destination_path), - allowed_fidelity_events=tuple( - event for event in loss_report if event["code"] in allow_loss - ), - operation_details={"engine": engine_identity(), "legacy_locale": legacy_locale}, - ) + had_previous = destination_path.exists() + backup = _reserve_export_backup(destination_path) + if not had_previous: + backup.unlink() + try: + operation_id = record_export_operation( + database_url=database_url, + dataset_id=dataset_id, + destination=_path_reference_text( + destination_path, role="destination", + ), + allowed_fidelity_events=accepted_events, + operation_details={ + "engine": engine_identity(), "legacy_locale": legacy_locale, + "recovery": { + "procedure_id": "openstatspec.export-destination-restore.v1", + "phase": "prepared", + "destination": _path_reference( + destination_path, role="destination", + ), + "durable_backup": _path_reference( + backup, role="durable_backup", + ), + "previous_destination_existed": had_previous, + "publication_finalized": False, + }, + }, + terminal=False, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as audit_error: + try: + backup.unlink(missing_ok=True) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=audit_error, cleanup_error=cleanup_error, + phase="audit_start_placeholder_cleanup", + destination=destination_path, backup=backup, + staged=staged_destination, had_previous=had_previous, + database_url=database_url, + dolt_conformance_source=dolt_conformance_source, + ) + raise + backup_installed = False + try: + if had_previous: + os.replace(destination_path, backup) + backup_installed = True + os.replace(staged_destination, destination_path) + publication_state.update({ + "published": True, + "backup": backup, + "staged": staged_destination, + "had_previous": had_previous, + "operation_id": operation_id, + }) + except Exception as publish_error: + if backup_installed or not had_previous: + try: + _restore_export_destination( + destination=destination_path, backup=backup, + had_previous=had_previous, + ) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=publish_error, cleanup_error=cleanup_error, + phase="publish", destination=destination_path, backup=backup, + staged=staged_destination, had_previous=had_previous, + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + else: + try: + backup.unlink(missing_ok=True) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=publish_error, + cleanup_error=cleanup_error, + phase="publish_placeholder_cleanup", + destination=destination_path, backup=backup, + staged=staged_destination, + had_previous=had_previous, + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + _mark_export_failed_after_restore( + database_url=database_url, operation_id=operation_id, + error=publish_error, phase="publish", destination=destination_path, + backup=backup, had_previous=had_previous, + dolt_conformance_source=dolt_conformance_source, + ) + raise + assert operation_id is not None + finalization_state = None + try: + finish_export_operation( + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as finalization_error: + state_read_fault = None + try: + finalization_state = read_export_operation_state( + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as state_error: + state_read_fault = _export_error_identity( + state_error, phase="export_finalization_state_read", + ) + if ( + finalization_state is not None + and finalization_state["classification"] == "succeeded" + ): + pass + elif ( + finalization_state is not None + and finalization_state["classification"] == "running" + ): + try: + _restore_export_destination( + destination=destination_path, backup=backup, + had_previous=had_previous, + ) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=finalization_error, + cleanup_error=cleanup_error, + phase="audit_finalization", + destination=destination_path, + backup=backup, + staged=staged_destination, + had_previous=had_previous, + database_url=database_url, + operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + _mark_export_failed_after_restore( + database_url=database_url, operation_id=operation_id, + error=finalization_error, phase="audit_finalization", + destination=destination_path, backup=backup, + had_previous=had_previous, + dolt_conformance_source=dolt_conformance_source, + ) + raise + else: + raise ExportRecoveryError( + "audit_finalization_ambiguous", + "The published export and durable backup were preserved because " + "the operation catalogs do not prove whether finalization committed.", + details={ + "subcode": "export_finalization_commit_ambiguous", + "operation_id": operation_id, + "finalization_cause": _export_error_identity( + finalization_error, phase="export_audit_finalization", + ), + "state_read_fault": state_read_fault, + "observed_operation_state": finalization_state, + "residual_object_inventory": { + "destination": _path_reference( + destination_path, role="published_destination", + ), + "destination_exists": destination_path.exists(), + "backup": _path_reference( + backup, role="durable_backup", + ), + "backup_exists": backup.exists(), + }, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.export-audit-reconciliation.v1", + "operation_terminal_state_verified": False, + "automatic_filesystem_recovery_performed": False, + "published_file_preserved": destination_path.exists(), + "durable_backup_preserved": backup.exists(), + "manual_reconciliation_required": True, + "terminal_reporting": "out_of_band_exception", + }, + "success_forbidden": True, + }, + ) from finalization_error + if backup.exists(): + try: + backup.unlink() + except Exception as cleanup_error: + audit_fault = None + try: + record_export_backup_retained( + database_url=database_url, operation_id=operation_id, + destination=_path_reference_text( + destination_path, role="destination", + ), + backup=_path_reference_text( + backup, role="durable_backup", + ), + cleanup_error=cleanup_error, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as warning_error: + audit_fault = _export_error_identity( + warning_error, phase="backup_retained_warning_audit", + ) + raise ExportRecoveryError( + "backup_retained", + "The export succeeded, but its durable prior-file backup could not be removed.", + details={ + "subcode": "post_success_backup_retained", + "operation_id": operation_id, + "operation_status": "succeeded", + "destination": _path_reference( + destination_path, role="destination", + ), + "durable_backup": _path_reference( + backup, role="durable_backup", + ), + "cleanup_fault": _export_error_identity( + cleanup_error, phase="post_success_backup_disposal", + ), + "warning_audit_persisted": audit_fault is None, + "audit_fault": audit_fault, + "success_forbidden": False, + }, + ) + return { "dataset_id": dataset_id, "destination": str(destination_path), @@ -335,10 +817,21 @@ def _write_with_dictionary_bridge( pyspssio engine then copies it into the real SAV or ZSAV writer, which keeps any ZSAV internal dictionary offsets valid. """ + compatible_names = { + str(variable["source_name"]): str(variable["compat_name"]) + for variable in variables + if variable.get("compat_name") + and str(variable["compat_name"]).casefold() + != str(variable["source_name"]).casefold() + } + variable_sets = (dataset.get("source_extensions") or {}).get( + "spss.variable_sets" + ) metadata = _writer_metadata(dataset, variables) documents = _json_load(dataset.get("documents"), []) source_encoding = str(dataset.get("source_encoding") or "UTF-8") legacy_output = _is_non_utf8_encoding(source_encoding) and legacy_locale is not None + output_encoding: str | None = None with ExitStack() as stack: document_source = None if documents: @@ -393,24 +886,18 @@ def _write_with_dictionary_bridge( set_variable_attribute_pairs( writer, variable["source_name"], attribute_pairs(values), ) - variable_sets = (dataset.get("source_extensions") or {}).get("spss.variable_sets") if variable_sets: writer.var_sets = variable_sets if document_source is not None: writer.copy_documents_from(document_source) writer.commit_header() - _require_matching_legacy_encoding(source_encoding, str(writer.file_encoding), legacy_output) + output_encoding = str(writer.file_encoding or "") + if not output_encoding: + raise RawDictionaryError("Writer did not expose its output file encoding.") + _require_matching_legacy_encoding(source_encoding, output_encoding, legacy_output) writer.write_data(frame) - write_compatible_names( - destination, - { - str(variable["source_name"]): str(variable["compat_name"]) - for variable in variables - if variable.get("compat_name") - and str(variable["compat_name"]).casefold() != str(variable["source_name"]).casefold() - }, - encoding="UTF-8", - ) + if output_encoding is None: + raise RawDictionaryError("Writer output encoding was not captured.") extended_labels = { str(name): str(definition["label"]) for name, definition in _json_load(dataset.get("multiple_response_sets"), {}).items() @@ -420,7 +907,12 @@ def _write_with_dictionary_bridge( and definition.get("label") } write_extended_mrset_labels( - destination, extended_labels, encoding=source_encoding, + destination, extended_labels, encoding=output_encoding, + ) + write_compatible_names( + destination, + compatible_names, + encoding=output_encoding, ) def _catalog_format(variable: dict[str, Any], key: str) -> tuple[int, int, int]: diff --git a/src/openstatspec/spss/semantics.py b/src/openstatspec/spss/semantics.py new file mode 100644 index 0000000..3a00c73 --- /dev/null +++ b/src/openstatspec/spss/semantics.py @@ -0,0 +1,452 @@ +"""Public, value-redacting semantic comparison for SPSS SAV/ZSAV files. + +The comparison deliberately reports no observed values or dictionary names. +By default it returns only cardinalities, fixed component names, and statuses. +A caller may explicitly request deterministic SHA-256 receipts; those hashes +are dictionary-guessable for low-entropy metadata and are therefore opt-in. +""" + +from __future__ import annotations + +import hashlib +import math +import numbers +import struct +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pandas as pd +import pyspssio + +from .sav import _dictionary + + +@dataclass(frozen=True) +class _Component: + digest: str + count: int + available: bool = True + + +@dataclass(frozen=True) +class _Snapshot: + case_count: int + variable_count: int + numeric_variable_count: int + string_variable_count: int + components: Mapping[str, _Component] + + +_COMPONENT_ORDER = ( + "adapter_observability", + "variable_order", + "case_order", + "numeric_nonmissing_binary64", + "numeric_system_missing_mask", + "string_values", + "source_encoding", + "file_label", + "case_weight_variable", + "documents", + "file_attributes", + "variable_types", + "variable_labels", + "print_formats", + "write_formats", + "measurement_levels", + "variable_roles", + "variable_alignments", + "display_widths", + "compatible_names", + "ordered_value_labels", + "missing_rules", + "variable_attributes", + "variable_sets", + "multiple_response_sets", +) + + +def compare_sav_semantics( + source: str | Path, exported: str | Path, *, + include_digests: bool = False, +) -> dict[str, Any]: + """Compare all SPSS semantics supported by the adapter. + + Numeric nonmissing values are compared by their float64 bit patterns, so + signed zero is significant. Every NaN payload exposed by ``pyspssio`` is + instead the same semantic SPSS system-missing marker. Ordered dictionary + features retain their order, including documents, value labels, variable + sets, multiple-response sets, their members, and attribute-array members. + + The result never contains case values, labels, names, paths, encodings, or + other source content. ``unavailable`` means the adapter could not observe + a required dictionary component, and therefore equivalence is false. + Deterministic digests are returned only with ``include_digests=True``. + """ + source_snapshot = _snapshot(Path(source)) + exported_snapshot = _snapshot(Path(exported)) + differences: list[str] = [] + components: dict[str, dict[str, Any]] = {} + for name in _COMPONENT_ORDER: + left = source_snapshot.components[name] + right = exported_snapshot.components[name] + if not left.available or not right.available: + status = "unavailable" + elif left.count == right.count and left.digest == right.digest: + status = "equal" + else: + status = "different" + if status != "equal": + differences.append(name) + component_result = { + "status": status, + "source_count": left.count, + "exported_count": right.count, + } + if include_digests: + component_result["source_sha256"] = left.digest + component_result["exported_sha256"] = right.digest + components[name] = component_result + return { + "equivalent": not differences, + "differences": differences, + "counts": { + "source": { + "cases": source_snapshot.case_count, + "variables": source_snapshot.variable_count, + "numeric_variables": source_snapshot.numeric_variable_count, + "string_variables": source_snapshot.string_variable_count, + }, + "exported": { + "cases": exported_snapshot.case_count, + "variables": exported_snapshot.variable_count, + "numeric_variables": exported_snapshot.numeric_variable_count, + "string_variables": exported_snapshot.string_variable_count, + }, + }, + "components": components, + } + + +def _snapshot(path: Path) -> _Snapshot: + frame, observed = pyspssio.read_sav( + str(path), convert_datetimes=False, include_user_missing=True, + ) + dictionary, loss_report = _dictionary(path) + metadata = {**dict(observed), **dictionary} + variable_names = list(frame.columns) + types = dict(metadata.get("var_types") or {}) + numeric_indexes = tuple( + index for index, name in enumerate(variable_names) + if int(types.get(name) or 0) == 0 + ) + string_indexes = tuple( + index for index, name in enumerate(variable_names) + if int(types.get(name) or 0) != 0 + ) + numeric_comparison_indexes = tuple( + sorted(numeric_indexes, key=lambda index: variable_names[index]) + ) + string_comparison_indexes = tuple( + sorted(string_indexes, key=lambda index: variable_names[index]) + ) + case_comparison_indexes = tuple( + sorted(range(len(variable_names)), key=lambda index: variable_names[index]) + ) + numeric_index_set = frozenset(numeric_indexes) + reader_bridge_available = metadata.get("_var_sets") is not None + documents_available = metadata.get("_documents") is not None + + encoding = metadata.get("encoding") + components = { + "variable_order": _component(variable_names, count=len(variable_names)), + "adapter_observability": _component( + tuple(sorted(loss_report)), count=len(loss_report), available=not loss_report, + ), + # Without an external case identifier, order is observed as the ordered + # sequence of complete case digests. Per-type components below make + # value-class differences independently visible. + "case_order": _sequence_component( + _case_digest( + row, variable_names, case_comparison_indexes, numeric_index_set, + ) + for row in frame.itertuples(index=False, name=None) + ), + "numeric_nonmissing_binary64": _sequence_component( + (variable_names[index], _binary64(value)) + for index in numeric_comparison_indexes + for row in frame.itertuples(index=False, name=None) + if not _is_system_missing(value := row[index]) + ), + "numeric_system_missing_mask": _sequence_component( + (variable_names[index], _is_system_missing(row[index])) + for index in numeric_comparison_indexes + for row in frame.itertuples(index=False, name=None) + ), + "string_values": _sequence_component( + (variable_names[index], row[index]) + for index in string_comparison_indexes + for row in frame.itertuples(index=False, name=None) + ), + "source_encoding": _component( + encoding, available=bool(str(encoding or "").strip()), + ), + "file_label": _component(metadata.get("file_label")), + "case_weight_variable": _component(metadata.get("case_weight_var")), + "documents": _component( + list(metadata.get("_documents") or ()), + count=len(metadata.get("_documents") or ()), + available=documents_available, + ), + "file_attributes": _component( + _attribute_mapping(metadata.get("file_attributes")), + count=len(metadata.get("file_attributes") or {}), + available=reader_bridge_available, + ), + "variable_types": _variable_property(metadata, "var_types", variable_names), + "variable_labels": _variable_property(metadata, "var_labels", variable_names), + "print_formats": _variable_property( + metadata, "_print_format_tuples", variable_names, + available=reader_bridge_available, + ), + "write_formats": _variable_property( + metadata, "_write_format_tuples", variable_names, + available=reader_bridge_available, + ), + "measurement_levels": _variable_property( + metadata, "var_measure_levels", variable_names, + ), + "variable_roles": _variable_property(metadata, "var_roles", variable_names), + "variable_alignments": _variable_property( + metadata, "var_alignments", variable_names, + ), + "display_widths": _variable_property( + metadata, "var_column_widths", variable_names, + ), + "compatible_names": _variable_property( + metadata, "var_compat_names", variable_names, + ), + "ordered_value_labels": _component( + _ordered_value_labels(metadata, variable_names), + count=sum( + len(labels) + for labels in (metadata.get("var_value_labels") or {}).values() + ), + ), + "missing_rules": _component( + _missing_rules(metadata, variable_names), + count=len(metadata.get("var_missing_values") or {}), + ), + "variable_attributes": _component( + _variable_attributes(metadata, variable_names), + count=sum( + len(attributes) + for attributes in (metadata.get("var_attributes") or {}).values() + ), + available=reader_bridge_available, + ), + "variable_sets": _component( + _ordered_sets(metadata.get("_var_sets")), + count=len(metadata.get("_var_sets") or {}), + available=reader_bridge_available, + ), + "multiple_response_sets": _component( + _ordered_mrsets(metadata.get("mrsets")), + count=len(metadata.get("mrsets") or {}), + ), + } + return _Snapshot( + case_count=len(frame.index), + variable_count=len(variable_names), + numeric_variable_count=len(numeric_indexes), + string_variable_count=len(string_indexes), + components=components, + ) + + +def _variable_property( + metadata: Mapping[str, Any], key: str, variable_names: Sequence[str], + *, available: bool = True, +) -> _Component: + values = dict(metadata.get(key) or {}) + names = sorted(set(variable_names) | set(values), key=str) + ordered = [ + (name, name in values, values.get(name)) + for name in names + ] + return _component(ordered, count=len(values), available=available) + + +def _ordered_value_labels( + metadata: Mapping[str, Any], variable_names: Sequence[str], +) -> list[tuple[Any, list[tuple[Any, Any]]]]: + values = dict(metadata.get("var_value_labels") or {}) + names = sorted(set(variable_names) | set(values), key=str) + return [ + (name, list((values.get(name) or {}).items())) + for name in names + if name in values + ] + + +def _missing_rules( + metadata: Mapping[str, Any], variable_names: Sequence[str], +) -> list[tuple[Any, tuple[Any, Any, tuple[Any, ...]]]]: + rules = dict(metadata.get("var_missing_values") or {}) + names = sorted(set(variable_names) | set(rules), key=str) + result: list[tuple[Any, tuple[Any, Any, tuple[Any, ...]]]] = [] + for name in names: + if name not in rules: + continue + rule = rules[name] or {} + result.append(( + name, + (rule.get("lo"), rule.get("hi"), tuple(rule.get("values") or ())), + )) + return result + + +def _attribute_mapping(value: Any) -> list[tuple[Any, Any]]: + return sorted(dict(value or {}).items(), key=lambda item: str(item[0])) + + +def _variable_attributes( + metadata: Mapping[str, Any], variable_names: Sequence[str], +) -> list[tuple[Any, list[tuple[Any, Any]]]]: + values = dict(metadata.get("var_attributes") or {}) + names = sorted(set(variable_names) | set(values), key=str) + return [ + (name, _attribute_mapping(values[name])) + for name in names + if name in values + ] + + +def _ordered_sets(value: Any) -> list[tuple[Any, tuple[Any, ...]]]: + return [ + (name, tuple(members or ())) + for name, members in dict(value or {}).items() + ] + + +def _ordered_mrsets(value: Any) -> list[tuple[Any, Any]]: + return [ + (name, _mapping_with_ordered_members(definition)) + for name, definition in dict(value or {}).items() + ] + + +def _mapping_with_ordered_members(value: Any) -> Any: + if not isinstance(value, Mapping): + return value + return [ + (key, tuple(item) if key == "variable_list" and isinstance(item, (list, tuple)) else item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + ] + + +def _case_digest( + row: Sequence[Any], variable_names: Sequence[str], + comparison_indexes: Sequence[int], numeric_indexes: frozenset[int], +) -> bytes: + values: list[Any] = [] + for index in comparison_indexes: + value = row[index] + name = variable_names[index] + if index in numeric_indexes: + values.append( + (name, "numeric-system-missing") + if _is_system_missing(value) + else (name, "numeric-binary64", _binary64(value)) + ) + else: + values.append((name, "string", value)) + return bytes.fromhex(_digest(values)) + + +def _is_system_missing(value: Any) -> bool: + if value is None or value is pd.NA: + return True + if isinstance(value, numbers.Real): + return math.isnan(float(value)) + return False + + +def _binary64(value: Any) -> bytes: + return struct.pack(">d", float(value)) + + +def _component( + value: Any, *, count: int = 1, available: bool = True, +) -> _Component: + return _Component( + digest=_digest(value if available else ("adapter-component-unavailable",)), + count=count, + available=available, + ) + + +def _sequence_component(values: Iterable[Any]) -> _Component: + digest = hashlib.sha256() + digest.update(b"openstatspec-semantic-sequence-v1\x00") + count = 0 + for value in values: + encoded = _canonical_bytes(value) + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + count += 1 + digest.update(b"\xff") + digest.update(count.to_bytes(8, "big")) + return _Component(digest.hexdigest(), count) + + +def _digest(value: Any) -> str: + return hashlib.sha256( + b"openstatspec-semantic-component-v1\x00" + _canonical_bytes(value) + ).hexdigest() + + +def _canonical_bytes(value: Any) -> bytes: + if value is None: + return b"N" + if value is pd.NA: + return b"Q" + if isinstance(value, bool): + return b"B1" if value else b"B0" + if isinstance(value, numbers.Integral): + encoded = str(int(value)).encode("ascii") + return b"I" + len(encoded).to_bytes(8, "big") + encoded + if isinstance(value, numbers.Real): + numeric = float(value) + return b"Q" if math.isnan(numeric) else b"F" + struct.pack(">d", numeric) + if isinstance(value, str): + encoded = value.encode("utf-8") + return b"S" + len(encoded).to_bytes(8, "big") + encoded + if isinstance(value, bytes): + return b"Y" + len(value).to_bytes(8, "big") + value + if isinstance(value, Mapping): + items = sorted( + ((_canonical_bytes(key), _canonical_bytes(item)) for key, item in value.items()), + key=lambda pair: pair[0], + ) + return b"M" + _framed(pair for item in items for pair in item) + if isinstance(value, (list, tuple)): + return b"L" + _framed(_canonical_bytes(item) for item in value) + if hasattr(value, "item"): + return _canonical_bytes(value.item()) + raise TypeError( + "SPSS semantic comparison encountered an unsupported observed value type: " + + type(value).__name__ + ) + + +def _framed(values: Iterable[bytes]) -> bytes: + output = bytearray() + count = 0 + for value in values: + output.extend(len(value).to_bytes(8, "big")) + output.extend(value) + count += 1 + return count.to_bytes(8, "big") + bytes(output) diff --git a/src/openstatspec/sql/__init__.py b/src/openstatspec/sql/__init__.py index 3a841fe..2c74007 100644 --- a/src/openstatspec/sql/__init__.py +++ b/src/openstatspec/sql/__init__.py @@ -3,12 +3,64 @@ from typing import Any from .capabilities import profile_declarations -from .wide import validate_wide_dataset +from .dolt_conformance import DoltConformanceSource +from .wide import ( + dolt_state_snapshot as _dolt_state_snapshot, + initialize_wide_catalog, + validate_wide_dataset, +) -def declared_profiles(database_url: str | None = None) -> dict[str, dict[str, object]]: - return profile_declarations(database_url) +def declared_profiles( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, dict[str, object]]: + return profile_declarations( + database_url, dolt_conformance_source=dolt_conformance_source, + ) -def validate_dataset(*, database_url: Any, dataset_id: str, **options: Any) -> dict[str, Any]: - return validate_wide_dataset(database_url=str(database_url), dataset_id=dataset_id) +def dolt_state_snapshot( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + return _dolt_state_snapshot( + database_url=str(database_url), + dolt_conformance_source=dolt_conformance_source, + ) + + +def initialize_catalog( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + return initialize_wide_catalog( + database_url=str(database_url), + dolt_conformance_source=dolt_conformance_source, + ) + + +def validate_dataset( + *, + database_url: Any, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> dict[str, Any]: + return validate_wide_dataset( + database_url=str(database_url), + dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + ) + + +__all__ = [ + "DoltConformanceSource", + "declared_profiles", + "dolt_state_snapshot", + "initialize_catalog", + "validate_dataset", +] diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 2701660..fdfd20c 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -5,20 +5,74 @@ import re from dataclasses import replace from pathlib import Path -from typing import Any +from typing import Any, Mapping from sqlalchemy import MetaData, create_engine, text from sqlalchemy.engine import make_url +from .dolt_conformance import DoltConformanceSource, effective_limits as dolt_effective_limits from .normative import catalog -from .profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, SqlProfile -from .profiles import profile_for_url, validate_connection_url +from .profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, MYSQL_WIRE_PROFILES, SqlProfile +from .profiles import profile_for_url from ..core import UnsupportedOperationError -SPECIFICATION_COMMIT = "79339ec3d8f8aa81789b7e85f6b8afa6f1374e50" -SPECIFICATION_RELEASE: str | None = "v0.2.0" +# Release/build automation must bind this to the exact commit used to build +# openstatspec-specification. An uncommitted source tree has no truthful pin. +SPECIFICATION_COMMIT: str | None = None +SPECIFICATION_RELEASE: str | None = None + +DOLT_WRITE_CONFORMANCE = { + "declaration_schema_id": "openstatspec-dolt-adapter-declaration-v1", + "write_enabled": False, + "status": "packaged_concrete_declarations_required", +} + + +def _conformance_source( + source: DoltConformanceSource | None, +) -> DoltConformanceSource: + return source or DoltConformanceSource.packaged() + + +def _validated_dolt_declarations( + source: DoltConformanceSource | None, +) -> tuple[dict[str, Any], ...]: + try: + return tuple( + dict(item) + for item in _conformance_source(source).validated_declarations() + ) + except UnsupportedOperationError: + return () + + +def _bound_specification_commit() -> str: + if ( + not isinstance(SPECIFICATION_COMMIT, str) + or re.fullmatch(r"[0-9a-f]{40}", SPECIFICATION_COMMIT) is None + ): + raise UnsupportedOperationError( + "The Python adapter is not bound to an exact " + "openstatspec-specification commit; Dolt write rejected before " + "mutation." + ) + return SPECIFICATION_COMMIT + + +def _dolt_write_enabled( + source: DoltConformanceSource | None = None, + *, + active_product_version: str | None = None, +) -> bool: + conformance = _conformance_source(source) + if active_product_version is None: + return bool(conformance.status()["write_enabled"]) + conformance.require_exact_match( + active_product_version=active_product_version, + specification_commit=_bound_specification_commit(), + ) + return True -_DOLT_2_2_STABLE_VERSION = re.compile(r"2\.2\.(0|[1-9][0-9]*)") SERVER_POLICIES = { "sqlite": { @@ -27,72 +81,61 @@ }, "mysql": { "claimed": ["MySQL 8.4.x", "MySQL 9.7.x"], - "ci": ["MySQL 8.4.11", "MySQL 9.7.2"], - }, - "dolt": { - "claimed": ["Dolt 2.2.x"], - "range": { - "minimum_inclusive": "2.2.2", - "maximum_exclusive": "2.3.0", - }, - "ci": ["Dolt 2.2.2", "Dolt 2.2.3"], - "exact_ci": ["2.2.2", "2.2.3"], + "ci": ["MySQL 8.4.x", "MySQL 9.7.x"], }, "mariadb": { "claimed": ["MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x"], - "ci": ["MariaDB 11.4.12", "MariaDB 11.8.8", "MariaDB 12.3.2"], + "ci": ["MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x"], + }, + "dolt": { + "claimed": [], + "ci": [], }, "postgresql": { "claimed": ["PostgreSQL 17.x", "PostgreSQL 18.x"], - "ci": ["PostgreSQL 17.10", "PostgreSQL 18.4"], + "ci": ["PostgreSQL 17.x", "PostgreSQL 18.x"], }, } -def profile_declarations(database_url: str | None = None) -> dict[str, dict[str, Any]]: +def profile_declarations( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, dict[str, Any]]: """Declare every profile and, optionally, one active connection.""" - active = active_connection(database_url) if database_url else None + source = _conformance_source(dolt_conformance_source) + active = ( + active_connection(database_url, dolt_conformance_source=source) + if database_url else None + ) + dolt_declaration = None + if active and active["profile"] == "dolt": + dolt_declaration = source.require_exact_match( + active_product_version=active["raw_product_version"], + specification_commit=_bound_specification_commit(), + ) return { - "sqlite": _profile("sqlite", SQLITE, active), - "mysql": _profile("mysql", MYSQL, active), - "mariadb": _profile("mariadb", MYSQL, active), - "dolt": _profile("dolt", DOLT, active), - "postgresql": _profile("postgresql", POSTGRESQL, active), + "sqlite": _profile("sqlite", SQLITE, active, source), + "mysql": _profile("mysql", MYSQL, active, source), + "mariadb": _profile("mariadb", MYSQL, active, source), + "dolt": _profile("dolt", DOLT, active, source, dolt_declaration), + "postgresql": _profile("postgresql", POSTGRESQL, active, source), } -def _required_text_probe(connection: Any, statement: str, label: str) -> str: - """Return one required identity value without normalizing absence into text.""" - try: - value = connection.execute(text(statement)).scalar_one() - except Exception as error: - raise UnsupportedOperationError( - f"Active SQL server identity probe {label} failed." - ) from error - if value is None or value is False: - raise UnsupportedOperationError( - f"Active SQL server identity probe {label} returned no value." - ) - raw = str(value) - if not raw.strip(): - raise UnsupportedOperationError( - f"Active SQL server identity probe {label} returned no value." - ) - return raw - - -def active_connection(database_url: str) -> dict[str, Any]: +def active_connection( + database_url: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: engine = create_engine(database_url) with engine.connect() as connection: dialect = connection.dialect.name - raw_comment: str | None = None if dialect == "sqlite": profile_name = "sqlite" - raw_wire_version = _required_text_probe( - connection, "select sqlite_version()", "sqlite_version()", - ) - raw_product_version = raw_wire_version - identity_source = "SELECT sqlite_version()" + raw_version = str(connection.execute(text("select sqlite_version()")).scalar_one()) + identity_source = "select sqlite_version()" compile_options = { name: int(value) for option in connection.exec_driver_sql("pragma compile_options").scalars() @@ -103,107 +146,138 @@ def active_connection(database_url: str) -> dict[str, Any]: observed = {"compile_options": compile_options} elif dialect == "postgresql": profile_name = "postgresql" - raw_wire_version = _required_text_probe( - connection, "show server_version", "server_version", - ) - raw_product_version = raw_wire_version + raw_version = str(connection.execute(text("show server_version")).scalar_one()) identity_source = "SHOW server_version" observed = {} elif dialect in {"mysql", "mariadb"}: - raw_wire_version = _required_text_probe( - connection, "select @@version", "@@version", + wire_version = _required_identity_text( + connection.execute(text("select @@version")).scalar_one(), "@@version", ) - raw_comment = _required_text_probe( - connection, "select @@version_comment", "@@version_comment", + comment = _required_identity_text( + connection.execute(text("select @@version_comment")).scalar_one(), + "@@version_comment", ) - identity_text = f"{raw_wire_version} {raw_comment}".casefold() - if raw_comment.strip().casefold() == "dolt": - profile_name = "dolt" - raw_product_version = _required_text_probe( - connection, "select DOLT_VERSION()", "DOLT_VERSION()", - ) - if not _dolt_version_supported(raw_product_version): + normalized_comment = comment.strip().casefold() + product_version = wire_version + if normalized_comment == "dolt": + if "mariadb" in wire_version.casefold(): raise UnsupportedOperationError( - "The active Dolt product version must be a canonical stable " - "release in the supported range >=2.2.2,<2.3.0." + "Conflicting Dolt and MariaDB active-server identity." ) - identity_source = "SELECT @@version, @@version_comment, DOLT_VERSION()" - elif "mariadb" in identity_text: - profile_name = "mariadb" - raw_product_version = raw_wire_version - identity_source = "SELECT @@version, @@version_comment" - elif "mysql" in raw_comment.casefold(): - profile_name = "mysql" - raw_product_version = raw_wire_version - identity_source = "SELECT @@version, @@version_comment" - else: - raise UnsupportedOperationError( - "The active MySQL-wire server product is unknown or unsupported." + product_version = _required_identity_text( + connection.execute(text("select DOLT_VERSION()")).scalar_one(), + "DOLT_VERSION()", ) - packet_text = _required_text_probe( - connection, "select @@max_allowed_packet", "@@max_allowed_packet", - ) - try: - packet = int(packet_text) - except ValueError as error: - raise UnsupportedOperationError( - "Active SQL server returned an invalid @@max_allowed_packet." - ) from error - if packet <= 0: - raise UnsupportedOperationError( - "Active SQL server returned an invalid @@max_allowed_packet." + profile_name, product = "dolt", "Dolt" + active_branch = _required_identity_text( + connection.execute(text("select ACTIVE_BRANCH()")).scalar_one(), + "ACTIVE_BRANCH()", ) - observed = {"max_allowed_packet": packet} + identity_source = ( + "SELECT @@version, @@version_comment, DOLT_VERSION(), ACTIVE_BRANCH()" + ) + elif "mariadb" in (wire_version + " " + comment).casefold(): + profile_name, product = "mariadb", "MariaDB" + identity_source = "SELECT @@version, @@version_comment" + else: + profile_name, product = "mysql", "MySQL" + identity_source = "SELECT @@version, @@version_comment" + raw_version = product_version + packet = int(connection.execute(text("select @@max_allowed_packet")).scalar_one()) + observed = { + "max_allowed_packet": packet, + **( + {"active_branch": active_branch} + if profile_name == "dolt" else {} + ), + } else: # pragma: no cover - validate_connection_url rejects this first raise ValueError(f"Unsupported active SQL dialect {dialect!r}.") return { "dialect": dialect, "profile": profile_name, - "engine": profile_name, - "product": profile_name, - "transport": ( - "mysql_compatible" if profile_name == "dolt" - else "mysql" if dialect in {"mysql", "mariadb"} - else dialect - ), - "driver": engine.dialect.driver, - "server_version": _normalized_version(profile_name, raw_product_version), - "raw_server_version": raw_product_version, - "raw_wire_version": raw_wire_version, - "raw_product_version": raw_product_version, - "raw_version_comment": raw_comment, + "product": product if dialect in {"mysql", "mariadb"} else profile_name, + "server_version": _normalized_version(raw_version), + "raw_server_version": raw_version, + "raw_wire_version": wire_version if dialect in {"mysql", "mariadb"} else raw_version, + "raw_product_version": raw_version, + "raw_version_comment": comment if dialect in {"mysql", "mariadb"} else None, "identity_source": identity_source, - "claimed_supported": server_version_supported(profile_name, raw_product_version), - "matched_claim": _matched_claim(profile_name, raw_product_version), + "claimed_supported": server_version_supported( + profile_name, + raw_version, + dolt_conformance_source=dolt_conformance_source, + ), + "matched_claim": _matched_claim( + profile_name, + raw_version, + dolt_conformance_source=dolt_conformance_source, + ), "catalog_binding": catalog_binding(database_url), + "working_set_binding": ( + { + "database": catalog_binding(database_url)["namespace"], + "active_branch": active_branch, + } + if profile_name == "dolt" else None + ), "observed": observed, } -def effective_profile(database_url: str) -> tuple[SqlProfile, dict[str, Any]]: +def _required_identity_text(value: Any, source: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise UnsupportedOperationError( + f"Active-server identity probe {source} returned no non-empty text." + ) + return value.strip() + + +def effective_profile( + database_url: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> tuple[SqlProfile, dict[str, Any]]: """Resolve and enforce the profile used by import preflight.""" - configured = validate_connection_url(database_url) - active = active_connection(database_url) + source = _conformance_source(dolt_conformance_source) + configured = profile_for_url(database_url) + active = active_connection(database_url, dolt_conformance_source=source) if configured is not MYSQL and active["profile"] != configured.name: raise UnsupportedOperationError("The active SQL server does not match the configured profile.") - if configured is MYSQL and active["profile"] not in {"mysql", "mariadb", "dolt"}: - raise UnsupportedOperationError("The active SQL server is not MySQL, MariaDB, or Dolt.") - if active["profile"] == "dolt" and make_url(database_url).drivername != "mysql+pymysql": - raise UnsupportedOperationError("Dolt requires an explicit mysql+pymysql URL.") + if configured is MYSQL and active["profile"] not in MYSQL_WIRE_PROFILES: + raise UnsupportedOperationError("The active SQL server is not a claimed MySQL-wire product.") + dolt_declaration = None + if active["profile"] == "dolt": + dolt_declaration = source.require_exact_match( + active_product_version=active["raw_product_version"], + specification_commit=_bound_specification_commit(), + ) if not active["claimed_supported"]: raise UnsupportedOperationError( f"Active {active['profile']} server version {active['server_version']} is not claimed supported." ) - selected = DOLT if active["profile"] == "dolt" else configured - declaration = _profile(active["profile"], selected, active) + if ( + active["profile"] in MYSQL_WIRE_PROFILES + and int(active["observed"]["max_allowed_packet"]) <= 131_072 + ): + raise UnsupportedOperationError( + "Active @@max_allowed_packet is too small for the SQL adapter safety reserve." + ) + configured = DOLT if active["profile"] == "dolt" else configured + declaration = _profile( + active["profile"], configured, active, source, dolt_declaration, + ) limits = declaration["effective_limits"] assert limits is not None return replace( - selected, + configured, name=active["profile"], - max_physical_variables=int(limits["maximum_source_variables"]), + max_source_variables=int(limits["maximum_source_variables"]), max_text_value_bytes=int(limits["maximum_value_bytes"]), - max_row_bytes=int(limits["maximum_row_bytes"]), + max_row_bytes=( + int(limits["maximum_row_bytes"]) if limits["maximum_row_bytes"] is not None else None + ), + max_statement_bytes=limits.get("maximum_statement_bytes"), ), active @@ -231,68 +305,75 @@ def catalog_binding(database_url: str) -> dict[str, Any]: def _profile( - name: str, profile: SqlProfile, active: dict[str, Any] | None, + name: str, + profile: SqlProfile, + active: dict[str, Any] | None, + dolt_conformance_source: DoltConformanceSource | None = None, + dolt_declaration: Mapping[str, Any] | None = None, ) -> dict[str, Any]: + dolt_envelope = name == "dolt" identifier = { "value": profile.identifier_limit, "unit": "characters" if name in {"mysql", "mariadb"} else "bytes", "source": ( "MySQL/MariaDB native identifier limit" if name in {"mysql", "mariadb"} - else "Dolt 2.2.2 observed 64-byte ASCII identifier limit" if name == "dolt" else "PostgreSQL NAMEDATALEN minus one native byte limit" if name == "postgresql" + else "OpenStatSpec Dolt adapter envelope pending pinned live boundary evidence" + if dolt_envelope else "OpenStatSpec profile boundary; SQLite has no fixed native identifier limit" ), "repertoire": "generated ASCII [a-z0-9_] identifiers", } - declared = { - "maximum_physical_columns": profile.max_physical_variables + 1, - "maximum_source_variables": profile.max_physical_variables, + profile_limits = { + "maximum_physical_columns": profile.max_source_variables + 1, + "maximum_source_variables": profile.max_source_variables, + "maximum_statement_bytes": profile.max_statement_bytes, "identifier_limit": identifier, "maximum_value_bytes": profile.max_text_value_bytes, "maximum_row_bytes": profile.max_row_bytes, } - theoretical = ( - {"maximum_value_bytes": 4_294_967_295} - if name == "dolt" else declared - ) - proposed = ( - { - "maximum_physical_columns": declared["maximum_physical_columns"], - "maximum_source_variables": declared["maximum_source_variables"], - "maximum_value_bytes": declared["maximum_value_bytes"], - "maximum_row_bytes": declared["maximum_row_bytes"], + adapter_envelope = None + if dolt_envelope: + adapter_envelope = { + **profile_limits, + "limit_basis": "proposed_adapter_envelope", + "evidence_status": "pending_pinned_live_conformance", } - if name == "dolt" else None - ) - observed_limits = ( - { - "minimum_observed_physical_columns": 307, - "identifier_limit": identifier, - "rejected_identifier_bytes": 65, + theoretical = { + key: None for key in ( + "maximum_physical_columns", "maximum_source_variables", + "maximum_statement_bytes", "identifier_limit", + "maximum_value_bytes", "maximum_row_bytes", + ) + } + theoretical["limit_basis"] = "server_limits_not_claimed" + else: + theoretical = { + **profile_limits, + "limit_basis": "profile_theoretical_engine_ceiling", } - if name == "dolt" else None - ) effective = None status = "not_connected" - if active and active["profile"] == name: - effective = dict(declared) - sources = ( - { - "maximum_source_variables": "proposed Dolt adapter envelope", - "maximum_physical_columns": "proposed Dolt adapter envelope", - "identifier_limit": "observed on exact Dolt 2.2.2", - "maximum_value_bytes": "observed Dolt adapter value envelope", - "maximum_row_bytes": "proposed Dolt adapter envelope", - } - if name == "dolt" - else { - "maximum_source_variables": "profile theoretical engine ceiling", - "maximum_physical_columns": "profile theoretical engine ceiling", - "identifier_limit": identifier["source"], - "maximum_value_bytes": "profile theoretical engine ceiling", - "maximum_row_bytes": "profile theoretical engine ceiling", - } + if active and active["profile"] == name and ( + not dolt_envelope or dolt_declaration is not None + ): + effective = ( + dolt_effective_limits(dolt_declaration) + if dolt_envelope and dolt_declaration is not None + else dict(theoretical) + ) + default_source = ( + "proposed adapter envelope pending pinned live conformance" + if dolt_envelope else "profile theoretical engine ceiling" ) + sources = { + "maximum_source_variables": default_source, + "maximum_statement_bytes": default_source, + "maximum_physical_columns": default_source, + "identifier_limit": identifier["source"], + "maximum_value_bytes": default_source, + "maximum_row_bytes": default_source, + } observed = active["observed"] if name == "sqlite": options = observed["compile_options"] @@ -315,15 +396,11 @@ def _profile( sources["maximum_value_bytes"] = "active PRAGMA compile_options MAX_LENGTH" sources["maximum_row_bytes"] = "active PRAGMA compile_options MAX_LENGTH" status = "active_connection_mixed" - elif name in {"mysql", "mariadb", "dolt"}: + elif name in MYSQL_WIRE_PROFILES: packet = int(observed["max_allowed_packet"]) payload = max(0, (packet - 131_072) // 2) effective["maximum_value_bytes"] = min( - ( - declared["maximum_value_bytes"] - if name == "dolt" else theoretical["maximum_value_bytes"] - ), - payload, + effective["maximum_value_bytes"], payload, ) effective["maximum_statement_bytes"] = payload sources["maximum_value_bytes"] = "active @@max_allowed_packet worst-case payload" @@ -332,110 +409,129 @@ def _profile( else: status = "profile_theoretical_fallback" effective["sources"] = sources + elif active and active["profile"] == name and dolt_envelope: + status = "blocked_pending_pinned_live_conformance" policy = SERVER_POLICIES[name] + dolt_declarations = ( + _validated_dolt_declarations(dolt_conformance_source) + if dolt_envelope else () + ) + dolt_claimed_versions = sorted({ + version + for declaration in dolt_declarations + for version in declaration["claimed_product_versions"] + }) + dolt_tested_versions = sorted({ + version + for declaration in dolt_declarations + for version in declaration["tested_product_versions"] + }) + dolt_status = ( + _conformance_source(dolt_conformance_source).status() + if dolt_envelope else None + ) return { - "profile": name, - "engine": name, - "dialect": "mysql" if name == "dolt" else name, - "transport": "mysql_compatible" if name == "dolt" else name, - "specification_commit": SPECIFICATION_COMMIT, - "specification_status": "released", - "specification_release": SPECIFICATION_RELEASE, - "driver": "psycopg" if name == "postgresql" else "PyMySQL" if name in {"mysql", "mariadb", "dolt"} else "sqlite3", - "claimed_server_versions": policy["claimed"], - "claimed_version_range": policy.get("range"), - "ci_tested_server_versions": policy["ci"], - **( - {"exact_ci_tested_versions": policy["exact_ci"]} - if "exact_ci" in policy else {} + "driver": "psycopg" if name == "postgresql" else "PyMySQL" if name in MYSQL_WIRE_PROFILES else "sqlite3", + "claimed_server_versions": ( + dolt_claimed_versions if dolt_envelope else policy["claimed"] + ), + "ci_tested_server_versions": ( + dolt_tested_versions if dolt_envelope else policy["ci"] + ), + "write_conformance": ( + { + "declaration_schema_id": DOLT_WRITE_CONFORMANCE[ + "declaration_schema_id" + ], + **dict(dolt_status or {}), + "active_declaration_id": ( + dolt_declaration["declaration_id"] + if dolt_declaration is not None else None + ), + } if dolt_envelope else { + "write_enabled": True, + "tested_server_versions": list(policy["ci"]), + "status": "profile_claimed", + } + ), + "operational_write_enabled": ( + dolt_declaration is not None + if dolt_envelope and active and active["profile"] == name + else bool((dolt_status or {}).get("write_enabled")) + if dolt_envelope else True ), "theoretical_limits": theoretical, - "proposed_adapter_limits": proposed, - "observed_limits": observed_limits, + "adapter_envelope": adapter_envelope, "effective_limits": effective, "effective_limits_status": status, - "numeric_type": "DOUBLE PRECISION" if name == "postgresql" else "DOUBLE" if name in {"mysql", "mariadb", "dolt"} else "REAL", + "numeric_type": "DOUBLE PRECISION" if name == "postgresql" else "DOUBLE" if name in MYSQL_WIRE_PROFILES else "REAL", "numeric_value_policy": { - "finite_binary64": "supported", - "nan": "rejected_before_ddl", - "positive_infinity": "rejected_before_ddl", - "negative_infinity": "rejected_before_ddl", - }, - "text_type": "LONGTEXT" if name in {"mysql", "mariadb", "dolt"} else "TEXT", - "ddl_atomic": name not in {"mysql", "mariadb", "dolt"}, - "failure_cleanup": "compensating_cleanup" if name in {"mysql", "mariadb", "dolt"} else "transaction_rollback", - "limit_bases": { - "maximum_physical_columns": ( - "proposed_adapter_envelope" if name == "dolt" else "theoretical_engine_limit" - ), - "maximum_source_variables": ( - "proposed_adapter_envelope" if name == "dolt" else "theoretical_engine_limit" - ), - "identifier_limit": "observed_exact_version" if name == "dolt" else "theoretical_engine_limit", - "maximum_value_bytes": ( - "observed_exact_version" if name == "dolt" else "theoretical_engine_limit" + "sql_null": "canonical_system_missing", + "spss_nan": "canonicalize_to_sql_null_during_spss_decode", + "adapter_input": "finite_binary64_or_null", + "positive_infinity": "reject_before_mutation", + "negative_infinity": "reject_before_mutation", + "live_bit_exact_evidence": ( + "pending_pinned_live_conformance" if dolt_envelope + else "profile_conformance_claim" ), - "maximum_row_bytes": ( - "proposed_adapter_envelope" if name == "dolt" else "theoretical_engine_limit" - ), - "maximum_statement_bytes": "active_connection_observation", }, - "storage_evidence": ( - { - "binary64": { - "type": "DOUBLE", - "classification": "observed_exact_version", - "source": "Dolt 2.2.2 interoperability verification", - "version": "2.2.2", - "maximum_finite_round_trip_exact": True, - }, - "text": { - "type": "LONGTEXT NOT NULL", - "classification": "observed_exact_version", - "source": "Dolt 2.2.2 interoperability verification", - "version": "2.2.2", - "observed_value_bytes": 65_504, - "unit": "bytes", - }, - } - if name == "dolt" else None - ), - "transformation_workflow": "unsupported" if name == "dolt" else None, + "text_type": "LONGTEXT" if name in MYSQL_WIRE_PROFILES else "TEXT", + "ddl_atomic": name not in MYSQL_WIRE_PROFILES, + "failure_cleanup": "compensating_cleanup" if name in MYSQL_WIRE_PROFILES else "transaction_rollback", "physical_table_mapping": "dataset.physical_table_schema + dataset.physical_table_name", "identifier_policy": "deterministic ASCII mapping; source name remains authoritative", } -def server_version_supported(profile: str, raw_version: str) -> bool: - if profile == "dolt": - return _dolt_version_supported(raw_version) +def server_version_supported( + profile: str, + raw_version: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> bool: version = _version_tuple(raw_version) + if profile == "dolt": + try: + return _dolt_write_enabled( + dolt_conformance_source, + active_product_version=raw_version.strip(), + ) + except UnsupportedOperationError: + return False if profile == "sqlite": return (3, 24) <= version[:2] < (4, 0) allowed = { "postgresql": {(17,), (18,)}, "mysql": {(8, 4), (9, 7)}, "mariadb": {(11, 4), (11, 8), (12, 3)}, + "dolt": {(2, 2, 2)}, }[profile] width = len(next(iter(allowed))) return version[:width] in allowed -def _dolt_version_supported(raw_version: str) -> bool: - """Accept only canonical stable Dolt 2.2 patches at or above 2.2.2.""" - match = _DOLT_2_2_STABLE_VERSION.fullmatch(raw_version.strip()) - return match is not None and match.group(1) not in {"0", "1"} - - -def _matched_claim(profile: str, raw_version: str) -> str | None: - if not server_version_supported(profile, raw_version): +def _matched_claim( + profile: str, + raw_version: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> str | None: + if not server_version_supported( + profile, + raw_version, + dolt_conformance_source=dolt_conformance_source, + ): return None + if profile == "dolt": + return raw_version.strip() version = _version_tuple(raw_version) for claim in SERVER_POLICIES[profile]["claimed"]: numbers = _version_tuple(claim) if profile == "sqlite" or version[:2] == numbers[:2] or version[:1] == numbers[:1]: return claim - return SERVER_POLICIES[profile]["claimed"][0] + claims = SERVER_POLICIES[profile]["claimed"] + return claims[0] if claims else None def _version_tuple(raw_version: str) -> tuple[int, ...]: @@ -445,8 +541,5 @@ def _version_tuple(raw_version: str) -> tuple[int, ...]: return tuple(int(part) for part in match.groups(default="0")) -def _normalized_version(profile: str, raw_version: str) -> str: - """Normalize a product version using that product's stable release width.""" - version = _version_tuple(raw_version) - width = 2 if profile == "postgresql" else 3 - return ".".join(str(part) for part in version[:width]) +def _normalized_version(raw_version: str) -> str: + return ".".join(str(part) for part in _version_tuple(raw_version)) diff --git a/src/openstatspec/sql/dolt_conformance.py b/src/openstatspec/sql/dolt_conformance.py new file mode 100644 index 0000000..fa0e5d8 --- /dev/null +++ b/src/openstatspec/sql/dolt_conformance.py @@ -0,0 +1,199 @@ +"""Thin adapter glue for the shared OpenStatSpec Dolt declaration validator.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version as distribution_version +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from ..core import UnsupportedOperationError + + +def _adapter_version() -> str: + source_project = Path(__file__).resolve().parents[3] / "pyproject.toml" + if source_project.is_file(): + project = tomllib.loads(source_project.read_text(encoding="utf-8")) + value = project.get("project", {}).get("version") + if isinstance(value, str) and value: + return value + try: + return distribution_version("openstatspec") + except PackageNotFoundError as error: + raise RuntimeError( + "The openstatspec adapter version is unavailable from both the " + "source project and installed distribution metadata." + ) from error + + +ADAPTER_IMPLEMENTATION_ID = "openstatspec-python" +ADAPTER_VERSION = _adapter_version() + + +def _shared_api() -> tuple[Any, Any, Any, Any]: + try: + from openstatspec_specification.dolt import ( + DoltDeclarationError, + DoltDeclarationSource, + load_validated_dolt_declarations, + select_dolt_declaration, + ) + except (ImportError, ModuleNotFoundError) as error: + raise UnsupportedOperationError( + "The openstatspec-specification companion distribution is not " + "installed; Dolt writes remain disabled." + ) from error + return ( + DoltDeclarationError, + DoltDeclarationSource, + load_validated_dolt_declarations, + select_dolt_declaration, + ) + + +@dataclass(frozen=True) +class DoltConformanceSource: + """Explicit source for validated packaged or directory declarations.""" + + directory: Path | None = None + + @classmethod + def packaged(cls) -> "DoltConformanceSource": + return cls() + + @classmethod + def from_directory(cls, root: str | Path) -> "DoltConformanceSource": + return cls(directory=Path(root)) + + def validated_declarations(self) -> tuple[Mapping[str, Any], ...]: + ( + shared_error, + shared_source_type, + shared_loader, + _shared_selector, + ) = _shared_api() + try: + shared_source = ( + shared_source_type.packaged() + if self.directory is None + else shared_source_type.from_directory(self.directory) + ) + return tuple(shared_loader(shared_source)) + except shared_error as error: + raise UnsupportedOperationError( + "The Dolt declaration source failed shared semantic or " + "resource-integrity validation: " + str(error) + ) from error + + def status(self) -> Mapping[str, Any]: + try: + declarations = self.validated_declarations() + except UnsupportedOperationError as error: + return { + "write_enabled": False, + "declarations_available": False, + "declaration_count": 0, + "status": "blocked_invalid_or_unavailable_declaration_source", + "reason": str(error), + } + return { + "write_enabled": False, + "declarations_available": bool(declarations), + "declaration_count": len(declarations), + "status": ( + "validated_concrete_declarations_available" + if declarations + else "blocked_no_concrete_declarations" + ), + } + + def require_exact_match( + self, + *, + active_product_version: str, + specification_commit: str, + ) -> Mapping[str, Any]: + if ( + not isinstance(specification_commit, str) + or len(specification_commit) != 40 + or any(character not in "0123456789abcdef" for character in specification_commit) + ): + raise UnsupportedOperationError( + "Dolt conformance selection requires an exact lowercase 40-hex " + "specification commit." + ) + declarations = self.validated_declarations() + if not declarations: + raise UnsupportedOperationError( + "The validated Dolt declaration source contains no concrete " + "declarations; write rejected before mutation." + ) + shared_error, _source_type, _loader, shared_selector = _shared_api() + try: + return shared_selector( + declarations, + active_product_version=active_product_version, + adapter_implementation_id=ADAPTER_IMPLEMENTATION_ID, + adapter_version=ADAPTER_VERSION, + specification_commit=specification_commit, + ) + except shared_error as error: + raise UnsupportedOperationError( + "No unique exact Dolt conformance declaration matches the " + "active product, adapter, and specification binding: " + + str(error) + ) from error + + +def effective_limits(declaration: Mapping[str, Any]) -> dict[str, Any]: + """Project the shared declaration's effective layers into adapter limits.""" + + dimensions = declaration["limit_declarations"] + + def effective(name: str) -> Mapping[str, Any]: + matches = tuple( + row for row in dimensions[name] if row.get("basis") == "effective" + ) + if len(matches) != 1: + raise UnsupportedOperationError( + "Validated Dolt declaration has no unique effective " + name + " layer." + ) + return matches[0] + + physical = effective("physical_columns") + source = effective("source_variables") + identifier = effective("identifier") + value = effective("value") + structural = effective("structural_row") + statement = effective("emitted_statement") + return { + "maximum_physical_columns": physical["value"], + "maximum_source_variables": source["value"], + "maximum_statement_bytes": statement["value"], + "identifier_limit": { + "value": identifier["value"], + "unit": identifier["unit"], + "source": "validated concrete Dolt declaration", + "repertoire": declaration["identifier_limit"]["repertoire"], + }, + "maximum_value_bytes": value["value"], + "maximum_row_bytes": ( + structural["value"] + if isinstance(structural["value"], int) + and not isinstance(structural["value"], bool) + else None + ), + "limit_basis": "validated_concrete_dolt_declaration", + "sources": { + name: "declaration:" + str(declaration["declaration_id"]) + for name in ( + "maximum_physical_columns", + "maximum_source_variables", + "maximum_statement_bytes", + "identifier_limit", + "maximum_value_bytes", + "maximum_row_bytes", + ) + }, + } diff --git a/src/openstatspec/sql/normative.py b/src/openstatspec/sql/normative.py index 5af3321..90dc261 100644 --- a/src/openstatspec/sql/normative.py +++ b/src/openstatspec/sql/normative.py @@ -34,6 +34,15 @@ def binary64_type() -> Float: ) +def lossless_text_type() -> Text: + """Use unbounded catalog text on the supported MySQL-wire profiles.""" + return ( + Text() + .with_variant(mysql.LONGTEXT(), "mysql") + .with_variant(mysql.LONGTEXT(), "mariadb") + ) + + @dataclass(frozen=True) class NormativeTables: catalog_identity: Table @@ -76,7 +85,7 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("physical_table_schema", String(255)), Column("physical_table_name", String(255), nullable=False), Column("dataset_name", String(255)), - Column("dataset_label", Text), + Column("dataset_label", lossless_text_type()), Column("source_encoding", String(128)), Column("source_hash", String(128)), Column("source_case_count", BigInteger, nullable=False), @@ -100,7 +109,7 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("physical_name", String(255), nullable=False), Column("storage_kind", String(16), nullable=False), Column("declared_string_width", Integer), - Column("variable_label", Text), + Column("variable_label", lossless_text_type()), Column("print_format_family", String(64)), Column("print_format_width", Integer), Column("print_format_decimals", Integer), @@ -133,8 +142,8 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("ordinal", Integer, nullable=False), Column("code_kind", String(16), nullable=False), Column("numeric_code", binary64_type()), - Column("string_code", Text), - Column("label", Text, nullable=False), + Column("string_code", lossless_text_type()), + Column("label", lossless_text_type(), nullable=False), UniqueConstraint("value_label_set_id", "ordinal"), ) variable_value_label_set = Table( @@ -150,7 +159,7 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("rule_kind", String(32), nullable=False), Column("code_kind", String(16)), Column("numeric_value", binary64_type()), - Column("string_value", Text), + Column("string_value", lossless_text_type()), Column("numeric_lower", binary64_type()), Column("numeric_upper", binary64_type()), Column("lower_special", String(16)), @@ -163,7 +172,7 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("dataset_id", String(36), ForeignKey("dataset.dataset_id"), nullable=False), Column("attribute_name", String(255), nullable=False), Column("array_ordinal", Integer, nullable=False, default=1), - Column("attribute_value", Text, nullable=False), + Column("attribute_value", lossless_text_type(), nullable=False), UniqueConstraint("dataset_id", "attribute_name", "array_ordinal"), ) variable_attribute = Table( @@ -172,7 +181,7 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("variable_id", String(36), ForeignKey("variable.variable_id"), nullable=False), Column("attribute_name", String(255), nullable=False), Column("array_ordinal", Integer, nullable=False, default=1), - Column("attribute_value", Text, nullable=False), + Column("attribute_value", lossless_text_type(), nullable=False), UniqueConstraint("variable_id", "attribute_name", "array_ordinal"), ) document = Table( @@ -180,7 +189,7 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("document_id", String(36), primary_key=True), Column("dataset_id", String(36), ForeignKey("dataset.dataset_id"), nullable=False), Column("source_ordinal", Integer, nullable=False), - Column("document_text", Text, nullable=False), + Column("document_text", lossless_text_type(), nullable=False), UniqueConstraint("dataset_id", "source_ordinal"), ) variable_set = Table( @@ -205,13 +214,13 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("dataset_id", String(36), ForeignKey("dataset.dataset_id"), nullable=False), Column("source_ordinal", Integer, nullable=False), Column("set_name", String(255), nullable=False), - Column("set_label", Text), + Column("set_label", lossless_text_type()), Column("set_kind", String(4), nullable=False), Column("counted_value_kind", String(16)), Column("counted_numeric_value", binary64_type()), - Column("counted_string_value", Text), - Column("category_label_behavior", Text), - Column("label_source", Text), + Column("counted_string_value", lossless_text_type()), + Column("category_label_behavior", lossless_text_type()), + Column("label_source", lossless_text_type()), UniqueConstraint("dataset_id", "source_ordinal"), UniqueConstraint("dataset_id", "set_name"), ) @@ -230,8 +239,8 @@ def catalog(metadata: MetaData) -> NormativeTables: Column("direction", String(16), nullable=False), Column("severity", String(16), nullable=False), Column("event_code", String(128), nullable=False), - Column("source_item", Text), - Column("detail_json", Text, nullable=False), + Column("source_item", lossless_text_type()), + Column("detail_json", lossless_text_type(), nullable=False), Column("created_at", DateTime, nullable=False), ) return NormativeTables( diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index 84142c4..e1800f4 100755 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -4,10 +4,10 @@ Importers use this information for preflight checks before creating a dataset. """ -import math - from collections.abc import Iterable, Mapping from dataclasses import dataclass +import math +from numbers import Real from typing import Any from urllib.parse import urlparse @@ -18,23 +18,30 @@ class SqlProfile: name: str url_schemes: tuple[str, ...] - max_physical_variables: int + max_source_variables: int identifier_limit: int binary64_numeric: bool lossless_text: bool max_text_value_bytes: int - max_row_bytes: int + max_row_bytes: int | None tested_reference: bool = False driver_packages: tuple[str, ...] = () + max_statement_bytes: int | None = None + + @property + def max_physical_variables(self) -> int: + """Compatibility alias for the historical, source-count-named field.""" + return self.max_source_variables def as_dict(self) -> dict[str, object]: return { - "max_physical_variables": self.max_physical_variables, + "max_source_variables": self.max_source_variables, "identifier_limit": self.identifier_limit, "binary64_numeric": self.binary64_numeric, "lossless_text": self.lossless_text, "max_text_value_bytes": self.max_text_value_bytes, "max_row_bytes": self.max_row_bytes, + "max_statement_bytes": self.max_statement_bytes, "tested_reference": self.tested_reference, "driver_packages": list(self.driver_packages), } @@ -48,20 +55,16 @@ def as_dict(self) -> dict[str, object]: "postgresql", ("postgresql", "postgres"), 1_599, 63, True, True, 1_073_741_823, 1_073_741_823, True, ("psycopg",), ) -# SQLAlchemy's generic Text column compiles to MySQL TEXT, not MEDIUMTEXT. -# The contract therefore declares TEXT's 64 KiB payload limit rather than -# promising an unimplemented wider physical type. MYSQL = SqlProfile( "mysql", ("mysql", "mariadb"), 1_016, 64, True, True, 4_294_967_295, 65_535, True, ("PyMySQL",), ) DOLT = SqlProfile( "dolt", (), 305, 64, True, True, - 65_504, 65_504, True, ("PyMySQL",), + 4_294_967_295, None, False, ("PyMySQL",), ) -# Dolt deliberately has no URL scheme. It is selected only after the server -# reached through the MySQL wire family positively identifies itself as Dolt. -PROFILES = (SQLITE, POSTGRESQL, MYSQL, DOLT) +PROFILES = (SQLITE, POSTGRESQL, MYSQL) +MYSQL_WIRE_PROFILES = frozenset({"mysql", "mariadb", "dolt"}) def profile_for_url(database_url: str) -> SqlProfile: @@ -108,12 +111,15 @@ def preflight( """Validate strict target capabilities before any source dataset is created.""" variables = None if isinstance(variables_or_count, int) else list(variables_or_count) variable_count = variables_or_count if isinstance(variables_or_count, int) else len(variables) - if variable_count > profile.max_physical_variables: + if variable_count > profile.max_source_variables: raise _exceeded( - "physical_variable_limit", - f"{profile.name} supports at most {profile.max_physical_variables} " + "source_variable_limit", + f"{profile.name} supports at most {profile.max_source_variables} " "source variables in one strict wide table.", - variable_count=variable_count, maximum=profile.max_physical_variables, + source_count=variable_count, + max_source=profile.max_source_variables, + physical_count=variable_count + 1, + max_physical=profile.max_source_variables + 1, ) if variables is None: return @@ -167,7 +173,7 @@ def preflight( maximum=profile.max_text_value_bytes, ) declared_row_bytes = 8 + sum(_row_storage_bytes(profile, variable) for variable in variables) - if declared_row_bytes > profile.max_row_bytes: + if profile.max_row_bytes is not None and declared_row_bytes > profile.max_row_bytes: raise _exceeded( "declared_row_size_limit", f"the declared SQL row requires {declared_row_bytes} bytes; " @@ -177,20 +183,80 @@ def preflight( for row_ordinal, row in enumerate(rows or (), start=1): row_bytes = 8 for variable in variables: - if variable.get("storage_kind") != "string": - value = row.get(str(variable["physical_name"])) + if variable.get("storage_kind") == "numeric": + physical_name = str(variable["physical_name"]) + if physical_name not in row: + raise _exceeded( + "numeric_value_missing", + f"row {row_ordinal} has no value for {variable['source_name']!r}.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) + value = row[physical_name] + if value is not None and ( + isinstance(value, bool) or not isinstance(value, Real) + ): + raise _exceeded( + "numeric_value_type", + f"row {row_ordinal} value for {variable['source_name']!r} " + "is not a binary64 number or SQL NULL.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + value_type=type(value).__name__, + ) + if isinstance(value, int) and value is not None: + try: + converted = float(value) + except OverflowError as error: + raise _exceeded( + "numeric_value_not_binary64_exact", + f"row {row_ordinal} integer for {variable['source_name']!r} " + "cannot be represented exactly as binary64.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) from error + if not math.isfinite(converted) or int(converted) != value: + raise _exceeded( + "numeric_value_not_binary64_exact", + f"row {row_ordinal} integer for {variable['source_name']!r} " + "cannot be represented exactly as binary64.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) if value is not None and not math.isfinite(float(value)): raise _exceeded( - "nonfinite_numeric_value", + "numeric_value_not_finite", f"row {row_ordinal} value for {variable['source_name']!r} " - "is not a finite binary64 value.", + "is not finite; non-finite adapter inputs are rejected.", row_ordinal=row_ordinal, source_name=variable["source_name"], + classification=( + "nan" if math.isnan(float(value)) + else "positive_infinity" if float(value) > 0 + else "negative_infinity" + ), ) row_bytes += 8 continue - value = row.get(str(variable["physical_name"]), "") - encoded_bytes = len(str(value).encode("utf-8")) + physical_name = str(variable["physical_name"]) + if physical_name not in row: + raise _exceeded( + "string_value_missing", + f"row {row_ordinal} has no value for {variable['source_name']!r}.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) + value = row[physical_name] + if not isinstance(value, str): + raise _exceeded( + "string_value_type", + f"row {row_ordinal} value for {variable['source_name']!r} " + "is not a string; SPSS string missing values must be empty strings.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + value_type=type(value).__name__, + ) + encoded_bytes = len(value.encode("utf-8")) if encoded_bytes > profile.max_text_value_bytes: raise _exceeded( "text_value_limit", @@ -200,8 +266,8 @@ def preflight( row_ordinal=row_ordinal, source_name=variable["source_name"], encoded_bytes=encoded_bytes, maximum=profile.max_text_value_bytes, ) - row_bytes += 20 if profile.name in {"mysql", "mariadb", "dolt"} else encoded_bytes - if row_bytes > profile.max_row_bytes: + row_bytes += 20 if profile.name in MYSQL_WIRE_PROFILES else encoded_bytes + if profile.max_row_bytes is not None and row_bytes > profile.max_row_bytes: raise _exceeded( "row_size_limit", f"row {row_ordinal} requires {row_bytes} bytes; " @@ -209,16 +275,39 @@ def preflight( row_ordinal=row_ordinal, row_bytes=row_bytes, maximum=profile.max_row_bytes, ) + statement_bytes = statement_payload_bytes(row, variables) + if profile.max_statement_bytes is not None and statement_bytes > profile.max_statement_bytes: + raise _exceeded( + "statement_payload_limit", + f"row {row_ordinal} requires {statement_bytes} payload bytes; " + f"{profile.name} permits {profile.max_statement_bytes} per bounded statement.", + row_ordinal=row_ordinal, statement_bytes=statement_bytes, + maximum=profile.max_statement_bytes, + ) def _row_storage_bytes(profile: SqlProfile, variable: Mapping[str, Any]) -> int: if variable.get("storage_kind") == "numeric": return 8 - if profile.name in {"mysql", "mariadb", "dolt"}: + if profile.name in MYSQL_WIRE_PROFILES: return 20 return int(variable.get("string_width") or 0) +def statement_payload_bytes( + row: Mapping[str, Any], variables: Iterable[Mapping[str, Any]], +) -> int: + """Count raw payload bytes for packet-safe bounded insert batches.""" + size = 32 + for variable in variables: + if variable.get("storage_kind") == "numeric": + size += 8 + else: + value = row.get(str(variable["physical_name"]), "") + size += len(str(value).encode("utf-8")) + 8 + return size + + def _physical_name(source_name: str, used: set[str]) -> str: """The profile-independent deterministic OpenStatSpec SQL-name mapping.""" diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index a9e3bad..9448ec2 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1,25 +1,30 @@ """SQLite reference SQL profile for the strict OpenStatSpec wide-table contract.""" +import hashlib import json import math import re import sys +from contextlib import contextmanager from datetime import UTC, datetime from uuid import uuid4 from collections.abc import Iterable, Mapping from typing import Any -from sqlalchemy import delete, BigInteger, Boolean, Column, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update -from sqlalchemy.dialects import mysql, postgresql, sqlite -from ..core import UnsupportedOperationError -from .capabilities import effective_profile -from .profiles import preflight, validate_connection_url +from sqlalchemy import delete, BigInteger, Boolean, Column, DateTime, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update +from ..core import UnsupportedOperationError, safe_error_identity as _safe_error_identity +from .capabilities import active_connection, effective_profile +from .dolt_conformance import DoltConformanceSource +from .profiles import preflight, statement_payload_bytes, validate_connection_url from .normative import ( + binary64_type, + CATALOG_CONTRACT_ID, + CATALOG_SCHEMA_VERSION, catalog as normative_catalog, create as create_normative_catalog, - delete_dataset_representation as delete_normative_dataset, dataset_id_for_name as normative_dataset_id_for_name, finish_operation as finish_normative_operation, + lossless_text_type, record_fidelity_events as record_normative_fidelity_events, record_operation as record_normative_operation, store_imported_dataset as store_normative_dataset, @@ -37,29 +42,257 @@ def __init__(self, code: str, detail: str, *, details: Mapping[str, Any]) -> Non self.details = {"reason": code, **details} +class ImportRecoveryError(UnsupportedOperationError): + """Import recovery could not establish the promised terminal state.""" + + def __init__(self, code: str, detail: str, *, details: Mapping[str, Any]) -> None: + super().__init__(f"OpenStatSpec import recovery failed [{code}]: {detail}") + self.code = code + self.details = {"reason": code, **details} + + def _catalog_error(code: str, detail: str, **details: Any) -> CatalogPreflightError: return CatalogPreflightError(code, detail, details=details) -def string_type(profile: Any) -> Text: - """Use Dolt's tested LONGTEXT storage without changing MySQL/MariaDB DDL.""" - return mysql.LONGTEXT() if profile.name == "dolt" else Text() +def _canonical_sha256(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") + ).hexdigest() -def binary64_type() -> Float: - """Return the required IEEE-754 binary64 SQL type for every profile. +def _verification_fault_identity( + code: str, *, phase: str, evidence: Any, +) -> dict[str, Any]: + return { + "type": "InvariantVerificationError", + "code": code, + "phase": phase, + "message_sha256": _canonical_sha256(evidence), + } + - ``Float()`` is not adequate as a portable declaration: SQLAlchemy compiles - it to ``FLOAT`` for MySQL, which is single precision there. The strict - profile therefore declares the physical type explicitly for every target. - """ - return ( - Float(precision=53) - .with_variant(mysql.DOUBLE(asdecimal=False), "mysql") - .with_variant(mysql.DOUBLE(asdecimal=False), "mariadb") - .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql") - .with_variant(sqlite.REAL(), "sqlite") +def _normalized_dolt_rows( + rows: Iterable[Mapping[str, Any]], *, expected_keys: tuple[str, ...], +) -> list[dict[str, Any]]: + normalized = [] + for row in rows: + raw = dict(row) + if set(raw) != set(expected_keys): + raise UnsupportedOperationError( + "Dolt state probe returned an unexpected column shape." + ) + if expected_keys == ("table_name", "staged", "status"): + if ( + not isinstance(raw["table_name"], str) + or not raw["table_name"].strip() + or raw["staged"] not in {False, True, 0, 1} + or not isinstance(raw["status"], str) + or not raw["status"].strip() + ): + raise UnsupportedOperationError( + "Dolt status probe returned an invalid row value shape." + ) + raw["staged"] = bool(raw["staged"]) + else: + relation_names = (raw["from_table_name"], raw["to_table_name"]) + if ( + not any(isinstance(name, str) and name.strip() for name in relation_names) + or any(name is not None and not isinstance(name, str) for name in relation_names) + or not isinstance(raw["diff_type"], str) + or not raw["diff_type"].strip() + or raw["data_change"] not in {False, True, 0, 1} + or raw["schema_change"] not in {False, True, 0, 1} + ): + raise UnsupportedOperationError( + "Dolt diff-summary probe returned an invalid row value shape." + ) + raw["data_change"] = bool(raw["data_change"]) + raw["schema_change"] = bool(raw["schema_change"]) + normalized.append({key: raw[key] for key in expected_keys}) + return sorted(normalized, key=lambda row: json.dumps(row, sort_keys=True, default=str)) + + +def _dolt_evidence_block( + rows: Iterable[Mapping[str, Any]], *, audit_relations: set[str], + expected_keys: tuple[str, ...], +) -> dict[str, Any]: + normalized = _normalized_dolt_rows(rows, expected_keys=expected_keys) + + def is_audit_row(row: Mapping[str, Any]) -> bool: + if expected_keys == ("table_name", "staged", "status"): + return row["table_name"] in audit_relations and ( + str(row["status"]).strip().casefold() == "modified" + ) + from_name = row.get("from_table_name") + to_name = row.get("to_table_name") + return ( + isinstance(from_name, str) + and from_name == to_name + and from_name in audit_relations + and str(row.get("diff_type") or "").strip().casefold() == "modified" + and bool(row.get("data_change")) + and not bool(row.get("schema_change")) + ) + + audit = [row for row in normalized if is_audit_row(row)] + non_audit = [row for row in normalized if not is_audit_row(row)] + return { + "rows": normalized, + "sha256": _canonical_sha256(normalized), + "audit_catalog_rows": audit, + "audit_catalog_sha256": _canonical_sha256(audit), + "non_audit_rows": non_audit, + "non_audit_sha256": _canonical_sha256(non_audit), + } + + +def _capture_dolt_state( + connection: Any, *, profile_name: str, audit_relations: set[str], +) -> dict[str, Any] | None: + """Capture read-only Dolt version-control state without changing branches or HEAD.""" + if profile_name != "dolt": + return None + identity = connection.exec_driver_sql( + "SELECT DATABASE() AS database_name, ACTIVE_BRANCH() AS active_branch, " + "DOLT_HASHOF('HEAD') AS head_hash" + ).mappings().one() + summaries = {} + for label, left, right in ( + ("head_to_working", "HEAD", "WORKING"), + ("head_to_staged", "HEAD", "STAGED"), + ("staged_to_working", "STAGED", "WORKING"), + ): + rows = connection.exec_driver_sql( + "SELECT from_table_name, to_table_name, diff_type, " + "data_change, schema_change " + f"FROM DOLT_DIFF_SUMMARY('{left}', '{right}') " + "ORDER BY from_table_name, to_table_name, diff_type" + ).mappings().all() + summaries[label] = _dolt_evidence_block( + rows, audit_relations=audit_relations, + expected_keys=( + "from_table_name", "to_table_name", "diff_type", + "data_change", "schema_change", + ), + ) + status = _dolt_evidence_block( + connection.exec_driver_sql( + "SELECT table_name, staged, status FROM dolt_status " + "ORDER BY table_name, staged, status" + ).mappings().all(), + audit_relations=audit_relations, + expected_keys=("table_name", "staged", "status"), ) + for key in ("database_name", "active_branch", "head_hash"): + if not isinstance(identity[key], str) or not identity[key].strip(): + raise UnsupportedOperationError( + f"Dolt state probe returned no non-empty {key}." + ) + result = { + "database": identity["database_name"].strip(), + "active_branch": identity["active_branch"].strip(), + "head": identity["head_hash"].strip(), + "status": status, + "diff_summaries": summaries, + } + result["snapshot_sha256"] = _canonical_sha256(result) + return result + + +def _require_dolt_working_set_binding( + snapshot: dict[str, Any] | None, active: Mapping[str, Any], *, phase: str, +) -> None: + if snapshot is None: + return + binding = active.get("working_set_binding") + if ( + not isinstance(binding, Mapping) + or snapshot["database"] != binding.get("database") + or snapshot["active_branch"] != binding.get("active_branch") + ): + raise UnsupportedOperationError( + f"Dolt database/branch working-set binding mismatch during {phase}." + ) + + +def _require_dolt_success_identity( + before: dict[str, Any] | None, after: dict[str, Any] | None, *, phase: str, +) -> None: + if before is None and after is None: + return + if before is None or after is None or any( + before[key] != after[key] for key in ("database", "active_branch", "head") + ): + raise UnsupportedOperationError( + f"Dolt database/branch/HEAD changed during {phase}." + ) + + +def _dolt_failure_boundary_evidence( + before: dict[str, Any] | None, after: dict[str, Any] | None, +) -> dict[str, Any]: + if before is None and after is None: + return {"applicable": False} + if before is None or after is None: + return {"applicable": True, "verified": False, "reason": "snapshot_missing"} + invariant_failures = [] + for key in ("database", "active_branch", "head"): + if before[key] != after[key]: + invariant_failures.append(f"{key}_changed") + if before["status"]["non_audit_sha256"] != after["status"]["non_audit_sha256"]: + invariant_failures.append("non_audit_status_changed") + for label in sorted(before["diff_summaries"]): + if ( + before["diff_summaries"][label]["non_audit_sha256"] + != after["diff_summaries"][label]["non_audit_sha256"] + ): + invariant_failures.append(f"non_audit_{label}_changed") + return { + "applicable": True, + "verified": not invariant_failures, + "invariant_failures": invariant_failures, + "before": before, + "after": after, + "permitted_delta": "failed-operation audit catalog relations only", + "prohibited_vc_actions": [ + "DOLT_ADD", "DOLT_COMMIT", "checkout", "reset", "branch_change", + ], + } + + +@contextmanager +def _bound_catalog_transaction( + *, engine: Any, profile_name: str, active: Mapping[str, Any], + audit_relations: set[str], phase: str, +) -> Iterable[Any]: + """Bind a write transaction to one Dolt database/branch/HEAD identity.""" + with engine.connect() as connection: + before = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding(before, active, phase=f"{phase} preflight") + if profile_name == "dolt": + raise UnsupportedOperationError( + "Dolt audit-only mutation is blocked until operation-scoped " + "append/update and immutable-history evidence is implemented." + ) + connection.rollback() + with connection.begin(): + yield connection + after = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding(after, active, phase=f"{phase} completion") + _require_dolt_success_identity(before, after, phase=phase) + boundary = _dolt_failure_boundary_evidence(before, after) + if boundary.get("applicable") and not boundary.get("verified"): + raise UnsupportedOperationError( + f"Dolt non-audit working-set state changed during {phase}; " + "the transaction was rolled back." + ) def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: @@ -68,19 +301,19 @@ def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: Column("dataset_id", String(255), primary_key=True), Column("data_table", String(255), nullable=False, unique=True), Column("source_format", String(16), nullable=False), - Column("source_name", Text, nullable=False), - Column("source_table_name", Text), + Column("source_name", lossless_text_type(), nullable=False), + Column("source_table_name", lossless_text_type()), Column("source_sha256", String(64), nullable=False), Column("source_created_at", String(40)), Column("source_modified_at", String(40)), Column("imported_at", String(40), nullable=False), Column("source_encoding", String(128)), Column("case_count", BigInteger, nullable=False), - Column("file_label", Text, nullable=False, default=""), - Column("documents", Text, nullable=False, default="[]"), - Column("file_attributes", Text, nullable=False, default="{}"), + Column("file_label", lossless_text_type(), nullable=False, default=""), + Column("documents", lossless_text_type(), nullable=False, default="[]"), + Column("file_attributes", lossless_text_type(), nullable=False, default="{}"), Column("case_weight_variable", String(255)), - Column("multiple_response_sets", Text, nullable=False, default="{}"), + Column("multiple_response_sets", lossless_text_type(), nullable=False, default="{}"), ) variables = Table( "variable_catalog", metadata, @@ -91,7 +324,7 @@ def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: Column("storage_kind", String(16), nullable=False), Column("readstat_storage_type", String(32)), Column("string_width", Integer), - Column("label", Text, nullable=False, default=""), + Column("label", lossless_text_type(), nullable=False, default=""), Column("format", String(64)), # format remains the legacy print-format mirror; SPSS has a distinct write format. Column("print_format", String(64)), @@ -100,10 +333,10 @@ def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: Column("role", String(32)), Column("alignment", String(32)), Column("display_width", Integer), - Column("attributes", Text, nullable=False, default="{}"), + Column("attributes", lossless_text_type(), nullable=False, default="{}"), Column("compat_name", String(255)), - Column("value_labels", Text, nullable=False, default="{}"), - Column("missing_ranges", Text, nullable=False, default="[]"), + Column("value_labels", lossless_text_type(), nullable=False, default="{}"), + Column("missing_ranges", lossless_text_type(), nullable=False, default="[]"), ) fidelity_events = Table( "fidelity_event_catalog", metadata, @@ -112,8 +345,8 @@ def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: Column("dataset_id", String(255)), Column("direction", String(16), nullable=False), Column("severity", String(16), nullable=False), - Column("detail", Text, nullable=False), - Column("details", Text, nullable=False, default="{}"), + Column("detail", lossless_text_type(), nullable=False), + Column("details", lossless_text_type(), nullable=False, default="{}"), Column("code", String(128), nullable=False), ) operations = Table( @@ -122,11 +355,11 @@ def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: Column("direction", String(16), nullable=False), Column("status", String(16), nullable=False), Column("dataset_id", String(255)), - Column("source", Text), - Column("destination", Text), + Column("source", lossless_text_type()), + Column("destination", lossless_text_type()), Column("created_at", String(40), nullable=False), Column("completed_at", String(40)), - Column("details", Text, nullable=False, default="{}"), + Column("details", lossless_text_type(), nullable=False, default="{}"), ) return datasets, variables, fidelity_events, operations @@ -144,14 +377,17 @@ def _migrate_catalog_columns( the newly observable metadata alongside it. """ inspector = inspect(connection) + text_declaration = str( + lossless_text_type().compile(dialect=connection.dialect) + ) additions = { datasets.name: { - "file_attributes": "TEXT NOT NULL DEFAULT '{}'", + "file_attributes": f"{text_declaration} NOT NULL DEFAULT '{{}}'", "case_weight_variable": "VARCHAR(255)", }, variables.name: { "role": "VARCHAR(32)", - "attributes": "TEXT NOT NULL DEFAULT '{}'", + "attributes": f"{text_declaration} NOT NULL DEFAULT '{{}}'", "compat_name": "VARCHAR(255)", "print_format": "VARCHAR(64)", "write_format": "VARCHAR(64)", @@ -162,7 +398,7 @@ def _migrate_catalog_columns( "use_first_var_label": "BOOLEAN", "counted_value_type": "VARCHAR(16)", "counted_numeric": "DOUBLE", - "counted_text": "TEXT", + "counted_text": text_declaration, }, } preparer = connection.dialect.identifier_preparer @@ -205,14 +441,11 @@ def _record_failed_preflight( multiple_response_catalog: Table, fidelity_event_catalog: Table, operation_catalog: Table, normative: Any, operation_id: str, source_name: str, source_format: str, variable_count: int, profile_name: str, error: Exception, + legacy: tuple[Table, ...], ) -> None: """Persist a failed preflight without creating any source dataset state.""" with engine.begin() as connection: - create_normative_catalog(connection, normative) - metadata.create_all(connection, tables=[ - datasets, variable_catalog, multiple_response_catalog, - fidelity_event_catalog, operation_catalog, - ]) + _require_verified_catalog(connection, normative, legacy) failed_at = datetime.now(UTC).replace(tzinfo=None) record_normative_operation( connection, normative, operation_id=operation_id, @@ -246,16 +479,16 @@ def multiple_response_set_catalog(metadata: MetaData) -> Table: Column("dataset_id", String(255), primary_key=True), Column("set_name", String(255), primary_key=True), Column("member_ordinal", Integer, primary_key=True), - Column("kind", String(16)), Column("label", Text), + Column("kind", String(16)), Column("label", lossless_text_type()), Column("is_dichotomy", Boolean), Column("use_category_labels", Boolean), Column("use_first_var_label", Boolean), - Column("counted_value", Text), + Column("counted_value", lossless_text_type()), Column("counted_value_type", String(16)), Column("counted_numeric", binary64_type()), - Column("counted_text", Text), + Column("counted_text", lossless_text_type()), Column("variable_name", String(255)), - Column("definition", Text, nullable=False), + Column("definition", lossless_text_type(), nullable=False), ) @@ -265,7 +498,7 @@ def source_extension_catalog(metadata: MetaData) -> Table: "source_extension_catalog", metadata, Column("dataset_id", String(255), primary_key=True), Column("extension_key", String(255), primary_key=True), - Column("payload", Text, nullable=False), + Column("payload", lossless_text_type(), nullable=False), ) @@ -283,7 +516,7 @@ def document_catalog(metadata: MetaData) -> Table: "document_catalog", metadata, Column("dataset_id", String(255), primary_key=True), Column("ordinal", Integer, primary_key=True), - Column("text", Text, nullable=False), + Column("text", lossless_text_type(), nullable=False), ) @@ -296,8 +529,8 @@ def value_label_catalog(metadata: MetaData) -> Table: Column("ordinal", Integer, primary_key=True), Column("value_type", String(16), nullable=False), Column("numeric_value", binary64_type()), - Column("text_value", Text), - Column("label", Text, nullable=False), + Column("text_value", lossless_text_type()), + Column("label", lossless_text_type(), nullable=False), ) @@ -311,10 +544,10 @@ def missing_rule_catalog(metadata: MetaData) -> Table: Column("kind", String(16), nullable=False), Column("lower_type", String(16), nullable=False), Column("lower_numeric", binary64_type()), - Column("lower_text", Text), + Column("lower_text", lossless_text_type()), Column("upper_type", String(16), nullable=False), Column("upper_numeric", binary64_type()), - Column("upper_text", Text), + Column("upper_text", lossless_text_type()), Column("lower_inclusive", Boolean, nullable=False, default=True), Column("upper_inclusive", Boolean, nullable=False, default=True), ) @@ -337,7 +570,7 @@ def attribute_catalog(metadata: MetaData) -> Table: Column("attribute_ordinal", Integer, primary_key=True), Column("value_ordinal", Integer, primary_key=True), Column("attribute_name", String(255), nullable=False), - Column("attribute_value", Text, nullable=False), + Column("attribute_value", lossless_text_type(), nullable=False), ) @@ -745,6 +978,1338 @@ def data_table_name(dataset_id: str) -> str: return f"data_{stem[:48]}" +def _catalog_layout(metadata: MetaData) -> tuple[tuple[Table, ...], Any]: + datasets, variables, fidelity_events, operations = catalog(metadata) + multiple_response = multiple_response_set_catalog(metadata) + source_extensions = source_extension_catalog(metadata) + documents, value_labels, missing_rules, attributes = normalized_metadata_tables(metadata) + return ( + datasets, variables, multiple_response, source_extensions, documents, + value_labels, missing_rules, attributes, fidelity_events, operations, + ), normative_catalog(metadata) + +def _normalized_sql_type(inspector: Any, value: Any) -> str: + compiled = " ".join( + str(value.compile(dialect=inspector.bind.dialect)).strip().upper().split() + ) + if inspector.bind.dialect.name in {"mysql", "mariadb"}: + if compiled in {"BOOL", "BOOLEAN", "TINYINT(1)"}: + return "BOOLEAN/TINYINT(1)" + return compiled + + +def _normalized_default(value: Any) -> str | None: + if value is None: + return None + result = " ".join(str(value).strip().split()) + while result.startswith("(") and result.endswith(")"): + result = result[1:-1].strip() + return result + + +def _expected_unique_constraints(table: Table) -> set[tuple[str, ...]]: + return { + tuple(column.name for column in constraint.columns) + for constraint in table.constraints + if getattr(constraint, "__visit_name__", "") == "unique_constraint" + } + + +def _actual_unique_constraints( + inspector: Any, table_name: str, +) -> set[tuple[str, ...]]: + constraints = { + tuple(str(name) for name in item.get("column_names") or ()) + for item in inspector.get_unique_constraints(table_name) + } + constraints.update( + tuple(str(name) for name in item.get("column_names") or ()) + for item in inspector.get_indexes(table_name) + if item.get("unique") + ) + constraints.discard(()) + return constraints + + +def _expected_foreign_keys(table: Table) -> set[tuple[Any, ...]]: + return { + ( + tuple(column.name for column in constraint.columns), + next(iter(constraint.elements)).column.table.name, + tuple(element.column.name for element in constraint.elements), + ) + for constraint in table.foreign_key_constraints + } + + +def _actual_foreign_keys( + inspector: Any, table_name: str, +) -> set[tuple[Any, ...]]: + return { + ( + tuple(str(name) for name in item.get("constrained_columns") or ()), + str(item.get("referred_table") or ""), + tuple(str(name) for name in item.get("referred_columns") or ()), + ) + for item in inspector.get_foreign_keys(table_name) + } + + +def _normalized_check_sql(value: Any) -> str: + result = " ".join(str(value).strip().casefold().split()) + result = result.replace("`", "").replace('"', "") + result = re.sub(r"\[([^]]+)\]", r"\1", result) + while result.startswith("(") and result.endswith(")"): + result = result[1:-1].strip() + return result + + +def _expected_check_constraints(table: Table) -> set[str]: + return { + _normalized_check_sql(constraint.sqltext) + for constraint in table.constraints + if getattr(constraint, "__visit_name__", "") == "table_or_column_check_constraint" + } + + +def _actual_check_constraints(inspector: Any, table_name: str) -> set[str]: + return { + _normalized_check_sql(item.get("sqltext") or "") + for item in inspector.get_check_constraints(table_name) + } + + +_MIGRATED_SERVER_DEFAULTS = { + ("dataset_catalog", "file_attributes"): {None, "'{}'", '"{}"', "{}"}, + ("variable_catalog", "attributes"): {None, "'{}'", '"{}"', "{}"}, +} + + +def _catalog_table_shape_valid( + inspector: Any, table: Table, *, allow_missing: bool, +) -> bool: + actual = { + str(column["name"]): column + for column in inspector.get_columns(table.name) + } + expected = {column.name: column for column in table.columns} + if set(actual) - set(expected): + return False + if not allow_missing and set(actual) != set(expected): + return False + for name in set(actual) & set(expected): + expected_column = expected[name] + actual_column = actual[name] + if ( + _normalized_sql_type(inspector, expected_column.type) + != _normalized_sql_type(inspector, actual_column["type"]) + ): + return False + if bool(actual_column.get("nullable")) != bool(expected_column.nullable): + return False + actual_default = _normalized_default(actual_column.get("default")) + if (table.name, name) in _MIGRATED_SERVER_DEFAULTS: + if actual_default not in _MIGRATED_SERVER_DEFAULTS[(table.name, name)]: + return False + else: + expected_default = _normalized_default( + expected_column.server_default.arg + if expected_column.server_default is not None else None + ) + if actual_default != expected_default: + return False + if actual_column.get("identity") is not None or actual_column.get("computed") is not None: + return False + if expected_column.autoincrement is True and actual_column.get("autoincrement") is not True: + return False + if expected_column.autoincrement is False and actual_column.get("autoincrement") is True: + return False + expected_pk = tuple(column.name for column in table.primary_key.columns) + actual_pk = tuple( + str(name) for name in ( + inspector.get_pk_constraint(table.name).get("constrained_columns") or () + ) + ) + if actual_pk != expected_pk: + return False + if _actual_unique_constraints(inspector, table.name) != _expected_unique_constraints(table): + return False + if _actual_foreign_keys(inspector, table.name) != _expected_foreign_keys(table): + return False + if _actual_check_constraints(inspector, table.name) != _expected_check_constraints(table): + return False + return True + + +def _identity_shape_valid( + inspector: Any, table: Table, *, key_name: str, +) -> bool: + try: + return ( + [column.name for column in table.primary_key.columns] == [key_name] + and _catalog_table_shape_valid(inspector, table, allow_missing=False) + ) + except Exception: + return False + + +def _catalog_existing_shapes_valid( + inspector: Any, tables: Iterable[Table], +) -> bool: + """Require exact existing shape while allowing only absent migration columns.""" + try: + return all( + _catalog_table_shape_valid(inspector, table, allow_missing=True) + for table in tables + ) + except Exception: + return False + + +_MIGRATABLE_CATALOG_COLUMNS = { + "dataset_catalog": {"file_attributes", "case_weight_variable"}, + "variable_catalog": { + "role", "attributes", "compat_name", "print_format", "write_format", + }, + "multiple_response_set_catalog": { + "is_dichotomy", "use_category_labels", "use_first_var_label", + "counted_value_type", "counted_numeric", "counted_text", + }, +} + + +def _catalog_missing_columns( + inspector: Any, tables: Iterable[Table], +) -> dict[str, set[str]]: + missing = {} + for table in tables: + absent = {column.name for column in table.columns} - { + str(column["name"]) + for column in inspector.get_columns(table.name) + } + if absent: + missing[table.name] = absent + return missing + + +def _catalog_missing_columns_are_migratable( + missing: Mapping[str, set[str]], +) -> bool: + return bool(missing) and all( + columns <= _MIGRATABLE_CATALOG_COLUMNS.get(table_name, set()) + for table_name, columns in missing.items() + ) + + +def _registered_physical_relations( + connection: Any, *, existing_tables: set[str], normative: Any, + legacy: Iterable[Table], +) -> tuple[set[str], tuple[Table, ...], set[str], set[str], str]: + """Return owned relations and the optional workflow identity state.""" + legacy = tuple(legacy) + declared_tables = legacy + normative.all() + static_tables = {table.name for table in declared_tables} + physical_tables: set[str] = set() + physical_views: set[str] = set() + inspector = inspect(connection) + if ( + legacy[0].name in existing_tables + and "data_table" in { + str(column["name"]) for column in inspector.get_columns(legacy[0].name) + } + ): + physical_tables.update(str(name) for name in connection.execute( + select(legacy[0].c.data_table) + ).scalars() if name) + if ( + normative.dataset.name in existing_tables + and "physical_table_name" in { + str(column["name"]) for column in inspector.get_columns(normative.dataset.name) + } + ): + physical_tables.update(str(name) for name in connection.execute( + select(normative.dataset.c.physical_table_name) + ).scalars() if name) + + # The optional workflow is another OpenStatSpec-owned relation profile in + # the same dedicated namespace. Import locally to avoid its documented + # dependency on this module during module initialization. + from .workflow import ( # pylint: disable=import-outside-toplevel + PROFILE_ID, PROFILE_SCHEMA_VERSION, workflow_catalog, + ) + workflow = workflow_catalog(MetaData()) + workflow_tables = {table.name for table in workflow.all()} + workflow_identity = workflow.transformation_profile_identity + profile_present = workflow_identity.name in existing_tables + if profile_present: + declared_tables += workflow.all() + static_tables.update(workflow_tables) + if not workflow_tables <= existing_tables: + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + if not _identity_shape_valid( + inspector, workflow_identity, key_name="profile_identity_key", + ): + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + identities = connection.execute(select(workflow_identity)).mappings().all() + if len(identities) != 1: + return static_tables, declared_tables, physical_tables, physical_views, "ambiguous" + if ( + identities[0]["profile_identity_key"] != 1 + or identities[0]["contract_id"] != PROFILE_ID + or identities[0]["schema_version"] != PROFILE_SCHEMA_VERSION + or identities[0]["core_contract_id"] != CATALOG_CONTRACT_ID + ): + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + for row in connection.execute(select( + workflow.derived_dataset.c.physical_relation_name, + workflow.derived_dataset.c.output_mode, + )).mappings(): + name = str(row["physical_relation_name"]) + if row["output_mode"] == "view": + physical_views.add(name) + else: + physical_tables.add(name) + elif workflow_tables & existing_tables: + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + return static_tables, declared_tables, physical_tables, physical_views, "valid" + + +def _catalog_dataset_bijection_state( + connection: Any, *, normative: Any, legacy: Iterable[Table], +) -> str: + """Require one exact legacy-to-normative dataset/table mapping per row.""" + datasets = tuple(legacy)[0] + legacy_rows = [ + (row["dataset_id"], row["data_table"]) + for row in connection.execute(select( + datasets.c.dataset_id, datasets.c.data_table, + )).mappings() + ] + normative_rows = [ + (row["dataset_name"], row["physical_table_name"]) + for row in connection.execute(select( + normative.dataset.c.dataset_name, + normative.dataset.c.physical_table_name, + )).mappings() + ] + for rows in (legacy_rows, normative_rows): + if any( + not isinstance(dataset_name, str) or not dataset_name.strip() + or not isinstance(table_name, str) or not table_name.strip() + for dataset_name, table_name in rows + ): + return "unverified" + dataset_names = [dataset_name for dataset_name, _table_name in rows] + table_names = [table_name for _dataset_name, table_name in rows] + if ( + len(set(dataset_names)) != len(dataset_names) + or len(set(table_names)) != len(table_names) + or len(set(rows)) != len(rows) + ): + return "ambiguous" + return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" + + +def _catalog_state(connection: Any, normative: Any, legacy: Iterable[Table]) -> str: + inspector = inspect(connection) + existing_tables = set(inspector.get_table_names()) + existing_views = set(inspector.get_view_names()) + if existing_tables & existing_views: + return "ambiguous" + existing_relations = existing_tables | existing_views + if normative.catalog_identity.name not in existing_tables: + return "absent" if not existing_relations else "foreign" + try: + identities = connection.execute( + select(normative.catalog_identity) + ).mappings().all() + except Exception: + return "foreign" + if len(identities) != 1: + return "ambiguous" + if not _identity_shape_valid( + inspector, normative.catalog_identity, key_name="catalog_identity_key", + ): + return "foreign" + identity = identities[0] + if ( + identity["catalog_identity_key"] != 1 + or identity["contract_id"] != CATALOG_CONTRACT_ID + or identity["schema_version"] != CATALOG_SCHEMA_VERSION + ): + return "foreign" + static_tables, declared_tables, physical_tables, physical_views, profile_valid = ( + _registered_physical_relations( + connection, existing_tables=existing_tables, + normative=normative, legacy=legacy, + ) + ) + if profile_valid != "valid": + return profile_valid + if ( + physical_tables & physical_views + or static_tables & physical_tables + or static_tables & physical_views + ): + return "ambiguous" + owned_relations = static_tables | physical_tables | physical_views + if existing_relations - owned_relations: + return "foreign" + if physical_tables - existing_tables or physical_views - existing_views: + return "unverified" + if not static_tables <= existing_tables: + return "unverified" + if not _catalog_existing_shapes_valid(inspector, declared_tables): + return "foreign" + missing_columns = _catalog_missing_columns(inspector, declared_tables) + if missing_columns: + return ( + "migration_required" + if _catalog_missing_columns_are_migratable(missing_columns) + else "unverified" + ) + mapping_state = _catalog_dataset_bijection_state( + connection, normative=normative, legacy=legacy, + ) + if mapping_state != "valid": + return mapping_state + return "verified" + + +def _require_verified_catalog( + connection: Any, normative: Any, legacy: Iterable[Table], +) -> None: + state = _catalog_state(connection, normative, legacy) + if state != "verified": + raise UnsupportedOperationError( + f"The selected OpenStatSpec catalog is {state}; run explicit catalog initialization first." + ) + + +def _catalog_snapshot( + connection: Any, +) -> tuple[set[str], dict[str, set[str]]]: + inspector = inspect(connection) + tables = set(inspector.get_table_names()) + return tables, { + table_name: { + str(column["name"]) for column in inspector.get_columns(table_name) + } + for table_name in tables + } + + +def _compensate_catalog_initialization( + connection: Any, *, metadata: MetaData, before_tables: set[str], + before_columns: Mapping[str, set[str]], +) -> None: + """Restore the pre-initialization relation/column inventory after DDL failure.""" + current_tables = set(inspect(connection).get_table_names()) + for table in reversed(metadata.sorted_tables): + if table.name in current_tables and table.name not in before_tables: + table.drop(connection, checkfirst=True) + inspector = inspect(connection) + preparer = connection.dialect.identifier_preparer + for table_name, original_columns in before_columns.items(): + if not inspector.has_table(table_name): + raise RuntimeError( + f"Pre-existing catalog table {table_name!r} disappeared during initialization." + ) + current_columns = { + str(column["name"]) for column in inspect(connection).get_columns(table_name) + } + for column_name in sorted(current_columns - original_columns): + connection.execute(text( + f"ALTER TABLE {preparer.quote(table_name)} " + f"DROP COLUMN {preparer.quote(column_name)}" + )) + + +def _catalog_residual_inventory( + engine: Any, *, before_tables: set[str], + before_columns: Mapping[str, set[str]], +) -> dict[str, Any]: + try: + with engine.connect() as connection: + inspector = inspect(connection) + current_tables = set(inspector.get_table_names()) + added_columns = { + table_name: sorted( + { + str(column["name"]) + for column in inspector.get_columns(table_name) + } - original_columns + ) + for table_name, original_columns in before_columns.items() + if table_name in current_tables + } + return { + "new_tables": sorted(current_tables - before_tables), + "missing_preexisting_tables": sorted(before_tables - current_tables), + "added_columns": { + name: columns for name, columns in added_columns.items() if columns + }, + "views": sorted(inspector.get_view_names()), + } + except Exception as inventory_error: + return {"inspection_error_type": type(inventory_error).__name__} + + +def dolt_state_snapshot( + *, + database_url: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + """Return a read-only, digest-bound snapshot of one active Dolt database.""" + validate_connection_url(database_url) + active = active_connection( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + if active["profile"] != "dolt": + raise UnsupportedOperationError( + "dolt_state_snapshot requires a positively identified Dolt connection." + ) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + engine = create_engine(database_url) + with engine.connect() as connection: + state = _capture_dolt_state( + connection, profile_name="dolt", audit_relations=audit_relations, + ) + assert state is not None + _require_dolt_working_set_binding( + state, active, phase="read-only state capture", + ) + binding = active["working_set_binding"] + return { + "profile": "dolt", + "server_version": active["server_version"], + "read_only": True, + "operational_write_enabled": bool(active["claimed_supported"]), + "working_set_binding": binding, + "state": state, + } + + +def initialize_wide_catalog( + *, + database_url: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + """Install or explicitly migrate a dedicated catalog after server preflight.""" + validate_connection_url(database_url) + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + datasets, variables, multiple_response = legacy[:3] + with engine.connect() as connection: + state = _catalog_state(connection, normative, legacy) + if state not in {"absent", "verified", "migration_required"}: + raise UnsupportedOperationError( + f"The selected database catalog is {state}; initialization is not permitted." + ) + before_tables, before_columns = _catalog_snapshot(connection) + pre_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + _require_dolt_working_set_binding( + pre_dolt_state, active, phase="catalog initialization preflight", + ) + connection.rollback() + try: + with connection.begin(): + create_normative_catalog(connection, normative) + metadata.create_all(connection, tables=list(legacy)) + _migrate_catalog_columns( + connection, datasets, variables, multiple_response, + ) + _require_verified_catalog(connection, normative, legacy) + post_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + _require_dolt_working_set_binding( + post_dolt_state, active, phase="catalog initialization completion", + ) + _require_dolt_success_identity( + pre_dolt_state, post_dolt_state, phase="catalog initialization", + ) + except Exception as install_error: + try: + with connection.begin(): + _compensate_catalog_initialization( + connection, metadata=metadata, before_tables=before_tables, + before_columns=before_columns, + ) + except Exception as cleanup_error: + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + try: + after_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + dolt_boundary = _dolt_failure_boundary_evidence( + pre_dolt_state, after_dolt_state, + ) + except Exception as snapshot_error: + dolt_boundary = { + "applicable": profile.name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", + ), + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog initialization failed and its DDL compensation also failed.", + details={ + "subcode": "catalog_install_cleanup_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _safe_error_identity( + cleanup_error, phase="catalog_compensation", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.catalog-init-compensation.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted( + table.name for table in metadata.tables.values() + ), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "preexisting_unverified_catalog_mutation_forbidden": True, + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from cleanup_error + try: + after_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + dolt_boundary = _dolt_failure_boundary_evidence( + pre_dolt_state, after_dolt_state, + ) + except Exception as snapshot_error: + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + recovery = { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted(metadata.tables), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile.name == "dolt", + "verified": False, + }, + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog compensation completed but Dolt state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": recovery, + "success_forbidden": True, + }, + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + recovery = { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted(metadata.tables), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog compensation did not preserve Dolt failure-boundary invariants.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_catalog_cleanup_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": recovery, + "success_forbidden": True, + }, + ) from install_error + raise + return { + "profile": profile.name, + "server_version": active["server_version"], + "catalog": "verified", + } + + +def _bounded_batches( + rows: list[dict[str, Any]], variables: list[dict[str, Any]], + maximum: int | None, +) -> Iterable[list[dict[str, Any]]]: + if maximum is None: + if rows: + yield rows + return + batch: list[dict[str, Any]] = [] + used = 0 + for row in rows: + size = statement_payload_bytes(row, variables) + if size > maximum: + raise RuntimeError("A preflighted row exceeds the active statement payload limit.") + if batch and used + size > maximum: + yield batch + batch, used = [], 0 + batch.append(row) + used += size + if batch: + yield batch + + +def _delete_normative_import_state( + connection: Any, normative: Any, *, dataset_name: str, + physical_table_name: str, normative_dataset_id: str | None, + normative_dataset_creation_attempted: bool, operation_id: str, +) -> None: + dataset_ids = [] + if normative_dataset_creation_attempted: + dataset_ids = list(connection.execute( + select(normative.dataset.c.dataset_id).where( + normative.dataset.c.dataset_name == dataset_name, + normative.dataset.c.physical_table_name == physical_table_name, + ) + ).scalars()) + if normative_dataset_id is not None and normative_dataset_id not in dataset_ids: + dataset_ids.append(normative_dataset_id) + connection.execute(delete(normative.fidelity_event).where( + normative.fidelity_event.c.operation_id == operation_id + )) + for normative_dataset_id in dataset_ids: + variable_ids = list(connection.execute( + select(normative.variable.c.variable_id) + .where(normative.variable.c.dataset_id == normative_dataset_id) + ).scalars()) + label_set_ids = list(connection.execute( + select(normative.value_label_set.c.value_label_set_id) + .where(normative.value_label_set.c.dataset_id == normative_dataset_id) + ).scalars()) + variable_set_ids = list(connection.execute( + select(normative.variable_set.c.variable_set_id) + .where(normative.variable_set.c.dataset_id == normative_dataset_id) + ).scalars()) + response_set_ids = list(connection.execute( + select(normative.multiple_response_set.c.multiple_response_set_id) + .where(normative.multiple_response_set.c.dataset_id == normative_dataset_id) + ).scalars()) + if variable_set_ids: + connection.execute(delete(normative.variable_set_member).where( + normative.variable_set_member.c.variable_set_id.in_(variable_set_ids) + )) + if response_set_ids: + connection.execute(delete(normative.multiple_response_member).where( + normative.multiple_response_member.c.multiple_response_set_id.in_(response_set_ids) + )) + if variable_ids: + for table in ( + normative.variable_value_label_set, normative.missing_rule, + normative.variable_attribute, + ): + connection.execute(delete(table).where(table.c.variable_id.in_(variable_ids))) + if label_set_ids: + connection.execute(delete(normative.value_label).where( + normative.value_label.c.value_label_set_id.in_(label_set_ids) + )) + for table in ( + normative.dataset_weight_variable, normative.dataset_attribute, + normative.document, normative.variable_set, normative.multiple_response_set, + normative.value_label_set, normative.fidelity_event, normative.variable, + ): + connection.execute(delete(table).where(table.c.dataset_id == normative_dataset_id)) + connection.execute(delete(normative.dataset).where( + normative.dataset.c.dataset_id == normative_dataset_id + )) + connection.execute(delete(normative.operation).where( + normative.operation.c.operation_id == operation_id + )) + + +def _cleanup_import_state( + connection: Any, *, dataset_id: str, operation_id: str, data_table: Table, + state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], +) -> None: + ( + datasets, variables, multiple_response, source_extensions, documents, + value_labels, missing_rules, attributes, fidelity_events, operations, + ) = legacy + _delete_normative_import_state( + connection, normative, dataset_name=dataset_id, + physical_table_name=data_table.name, + normative_dataset_id=state["normative_dataset_id"], + normative_dataset_creation_attempted=state["normative_dataset_creation_attempted"], + operation_id=operation_id, + ) + if state["legacy_dataset_created"]: + for table in ( + multiple_response, source_extensions, documents, value_labels, + missing_rules, attributes, + ): + connection.execute(delete(table).where(table.c.dataset_id == dataset_id)) + connection.execute(delete(variables).where(variables.c.dataset_id == dataset_id)) + connection.execute(delete(datasets).where(datasets.c.dataset_id == dataset_id)) + connection.execute(delete(fidelity_events).where( + fidelity_events.c.operation_id == operation_id + )) + connection.execute(delete(operations).where(operations.c.operation_id == operation_id)) + if state["data_table_creation_attempted"]: + data_table.drop(connection, checkfirst=True) + + +def _record_failed_import_audit( + *, engine: Any, operation_id: str, source_name: str, source_format: str, + variable_count: int, profile_name: str, import_error: Exception, + normative: Any, legacy: tuple[Table, ...], +) -> None: + """Persist only a failed operation and NULL-dataset event after cleanup.""" + fidelity_events, operations = legacy[8:] + failed_event = { + "code": "import_failed", + "detail": "Import failed after mutation began; operation-owned state was removed.", + "severity": "error", + "source_item": source_name, + "details": { + "phase": "mutation", + "profile": profile_name, + "variable_count": variable_count, + "error_type": type(import_error).__name__, + }, + } + with engine.begin() as connection: + _require_verified_catalog(connection, normative, legacy) + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="failed", source_format=source_format, + started_at=failed_at, completed_at=failed_at, + ) + connection.execute(insert(operations).values( + operation_id=operation_id, direction="import", status="failed", + dataset_id=None, source=source_name, created_at=_now(), + completed_at=_now(), details=json.dumps({ + "reason": "runtime_failure", + "variable_count": variable_count, + "error_type": type(import_error).__name__, + }, sort_keys=True), + )) + connection.execute(insert(fidelity_events), _event_rows( + operation_id=operation_id, dataset_id=None, direction="import", + fidelity_events=(failed_event,), + )) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="import", events=(failed_event,), + ) + + +def _record_import_cleanup_failure_audit( + *, engine: Any, operation_id: str, source_name: str, source_format: str, + profile_name: str, import_error: Exception, cleanup_error: Exception, + residual_object_inventory: Mapping[str, Any], + deterministic_recovery_evidence: Mapping[str, Any], + normative: Any, legacy: tuple[Table, ...], +) -> None: + """Best-effort immutable audit for verified-catalog cleanup failure.""" + fidelity_events, operations = legacy[8:] + original = _safe_error_identity(import_error, phase="import_mutation") + cleanup = _safe_error_identity(cleanup_error, phase="compensating_cleanup") + event_details = { + "original_cause": original, + "cleanup_fault": cleanup, + "residual_object_inventory": dict(residual_object_inventory), + "deterministic_recovery_evidence": dict( + deterministic_recovery_evidence + ), + } + event = { + "code": "cleanup_failed", + "detail": "Import cleanup failed; terminal recovery requires out-of-band review.", + "severity": "error", + "source_item": source_name, + "details": event_details, + } + with engine.begin() as connection: + _require_verified_catalog(connection, normative, legacy) + existing = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one_or_none() + normative_existing = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one_or_none() + if (existing is None) != (normative_existing is None): + raise UnsupportedOperationError( + "Import operation catalogs disagree about cleanup-failure state." + ) + if existing is not None: + if ( + existing["direction"] != "import" + or existing["status"] != "running" + or normative_existing["status"] != "started" + ): + raise UnsupportedOperationError( + "Existing import operation is not in an auditable running state." + ) + details = json.loads(existing["details"] or "{}") + details["cleanup_failure"] = event_details + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values( + status="failed", completed_at=_now(), + details=json.dumps(details, sort_keys=True), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=None, direction="import", + fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="import", events=(event,), + ) + else: + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="failed", source_format=source_format, + started_at=failed_at, completed_at=failed_at, + ) + connection.execute(insert(operations).values( + operation_id=operation_id, direction="import", status="failed", + dataset_id=None, source=source_name, created_at=_now(), + completed_at=_now(), details=json.dumps({ + "reason": "cleanup_failed", + "profile": profile_name, **event_details, + }, sort_keys=True), + )) + connection.execute(insert(fidelity_events), _event_rows( + operation_id=operation_id, dataset_id=None, direction="import", + fidelity_events=(event,), + )) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="import", events=(event,), + ) + + +def _import_residual_inventory( + engine: Any, *, dataset_id: str, operation_id: str, data_table: Table, + state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], +) -> dict[str, Any]: + try: + with engine.connect() as connection: + inspector = inspect(connection) + tables = set(inspector.get_table_names()) + + def count_rows(table: Table, condition: Any) -> int | None: + if table.name not in tables: + return None + return len(connection.execute(select(table).where(condition)).all()) + + return { + "data_table": { + "name": data_table.name, + "present": data_table.name in tables, + }, + "legacy_dataset_rows": count_rows( + legacy[0], legacy[0].c.dataset_id == dataset_id, + ), + "legacy_operation_rows": count_rows( + legacy[9], legacy[9].c.operation_id == operation_id, + ), + "legacy_fidelity_event_rows": count_rows( + legacy[8], legacy[8].c.operation_id == operation_id, + ), + "normative_dataset_rows": count_rows( + normative.dataset, + ( + normative.dataset.c.dataset_name == dataset_id + ) & ( + normative.dataset.c.physical_table_name == data_table.name + ), + ), + "normative_operation_rows": count_rows( + normative.operation, + normative.operation.c.operation_id == operation_id, + ), + "normative_fidelity_event_rows": count_rows( + normative.fidelity_event, + normative.fidelity_event.c.operation_id == operation_id, + ), + "mutation_markers": dict(state), + } + except Exception as inventory_error: + return {"inspection_error_type": type(inventory_error).__name__} + + +@contextmanager +def _import_cleanup_guard( + *, engine: Any, dataset_id: str, operation_id: str, data_table: Table, + source_name: str, source_format: str, variable_count: int, + profile_name: str, normative: Any, legacy: tuple[Table, ...], + snapshot_connection: Any, pre_dolt_state: dict[str, Any] | None, +) -> Iterable[dict[str, Any]]: + state: dict[str, Any] = { + "data_table_creation_attempted": False, + "legacy_dataset_created": False, + "normative_dataset_creation_attempted": False, + "normative_dataset_id": None, + } + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + + def capture_boundary() -> dict[str, Any]: + after = _capture_dolt_state( + snapshot_connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + return _dolt_failure_boundary_evidence(pre_dolt_state, after) + + try: + yield state + except Exception as import_error: + try: + with engine.begin() as cleanup_connection: + _cleanup_import_state( + cleanup_connection, dataset_id=dataset_id, + operation_id=operation_id, data_table=data_table, + state=state, normative=normative, legacy=legacy, + ) + except Exception as cleanup_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + try: + pre_audit_dolt_boundary = capture_boundary() + except Exception as snapshot_error: + pre_audit_dolt_boundary = { + "applicable": profile_name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, phase="post_cleanup_dolt_state_capture", + ), + } + audit_recovery = { + "procedure_id": "openstatspec.import-compensation.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "operation_owned_state_targeted": True, + "dolt_failure_boundary": pre_audit_dolt_boundary, + } + cleanup_audit_fault = None + audit_permitted = not ( + pre_audit_dolt_boundary.get("applicable") + and not pre_audit_dolt_boundary.get("verified") + ) + if not audit_permitted: + cleanup_audit_fault = _verification_fault_identity( + "dolt_state_unverified_before_cleanup_failed_audit", + phase="pre_cleanup_failed_audit_boundary", + evidence=pre_audit_dolt_boundary, + ) + dolt_boundary = pre_audit_dolt_boundary + else: + try: + _record_import_cleanup_failure_audit( + engine=engine, operation_id=operation_id, + source_name=source_name, source_format=source_format, + profile_name=profile_name, import_error=import_error, + cleanup_error=cleanup_error, + residual_object_inventory=inventory, + deterministic_recovery_evidence=audit_recovery, + normative=normative, legacy=legacy, + ) + except Exception as audit_error: + cleanup_audit_fault = _safe_error_identity( + audit_error, phase="cleanup_failed_audit", + ) + try: + dolt_boundary = capture_boundary() + except Exception as snapshot_error: + dolt_boundary = { + "applicable": profile_name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, + phase="post_cleanup_audit_dolt_state_capture", + ), + } + raise ImportRecoveryError( + "cleanup_failed", + "Import failed and complete compensating cleanup also failed.", + details={ + "subcode": "import_cleanup_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + cleanup_error, phase="compensating_cleanup", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.import-compensation.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "operation_owned_state_targeted": True, + "cleanup_failed_audit_persisted": cleanup_audit_fault is None, + "terminal_reporting": ( + "catalog_and_exception" if cleanup_audit_fault is None + else "out_of_band_exception" + ), + "dolt_failure_boundary": dolt_boundary, + }, + "audit_fault": cleanup_audit_fault, + "success_forbidden": True, + }, + ) from cleanup_error + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + try: + pre_failed_audit_boundary = capture_boundary() + except Exception as snapshot_error: + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup completed but its pre-audit Dolt boundary could not be captured.", + details={ + "subcode": "pre_failed_audit_dolt_state_capture_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="pre_failed_audit_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "failed_operation_audit_attempted": False, + "terminal_reporting": "out_of_band_exception", + "dolt_failure_boundary": { + "applicable": profile_name == "dolt", + "verified": False, + }, + }, + "audit_fault": None, + "success_forbidden": True, + }, + ) from snapshot_error + if ( + pre_failed_audit_boundary.get("applicable") + and not pre_failed_audit_boundary.get("verified") + ): + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup completed but its pre-audit Dolt boundary is unverified.", + details={ + "subcode": "pre_failed_audit_dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="pre_failed_audit_dolt_state_verification", + evidence=pre_failed_audit_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "failed_operation_audit_attempted": False, + "terminal_reporting": "out_of_band_exception", + "dolt_failure_boundary": pre_failed_audit_boundary, + }, + "audit_fault": None, + "success_forbidden": True, + }, + ) from import_error + try: + _record_failed_import_audit( + engine=engine, operation_id=operation_id, + source_name=source_name, source_format=source_format, + variable_count=variable_count, profile_name=profile_name, + import_error=import_error, normative=normative, legacy=legacy, + ) + except Exception as audit_error: + raise ImportRecoveryError( + "failure_audit_failed", + "Import cleanup succeeded but its failed-operation audit could not be persisted.", + details={ + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + audit_error, phase="failed_operation_audit", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.failed-import-audit.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + }, + "success_forbidden": True, + }, + ) from audit_error + try: + dolt_boundary = capture_boundary() + except Exception as snapshot_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup completed but Dolt failure-boundary state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_audit_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile_name == "dolt", + "verified": False, + }, + }, + "success_forbidden": True, + }, + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup did not preserve the Dolt failure-boundary invariants.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_audit_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from import_error + raise + + def create_wide_dataset( *, database_url: str, dataset_id: str, source_name: str, source_format: str, rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], file_label: str = "", @@ -760,9 +2325,12 @@ def create_wide_dataset( source_extensions: Mapping[str, Any] | None = None, fidelity_events: Iterable[Mapping[str, Any]] = (), operation_details: Mapping[str, Any] | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: validate_connection_url(database_url) - profile, _active_connection = effective_profile(database_url) + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() datasets, variable_catalog, fidelity_event_catalog, operation_catalog = catalog(metadata) @@ -770,269 +2338,286 @@ def create_wide_dataset( multiple_response_catalog = multiple_response_set_catalog(metadata) source_extensions_catalog = source_extension_catalog(metadata) documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog = normalized_metadata_tables(metadata) + legacy = ( + datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, + documents_catalog, value_labels_catalog, missing_rules_catalog, + attributes_catalog, fidelity_event_catalog, operation_catalog, + ) operation_id = str(uuid4()) fidelity_events = tuple(fidelity_events) source_rows = list(rows) - try: - preflight(profile, variables, rows=source_rows) - validate_spss_catalog( - variables, - case_weight_variable=case_weight_variable, - multiple_response_sets=multiple_response_sets, - ) - except Exception as error: - _record_failed_preflight( - engine=engine, metadata=metadata, datasets=datasets, - variable_catalog=variable_catalog, multiple_response_catalog=multiple_response_catalog, - fidelity_event_catalog=fidelity_event_catalog, operation_catalog=operation_catalog, - operation_id=operation_id, source_name=source_name, - source_format=source_format, variable_count=len(variables), - profile_name=profile.name, error=error, normative=normative, - ) - raise data_table = Table( data_table_name(dataset_id), metadata, Column("__case_ordinal", BigInteger, primary_key=True, nullable=False), - *(Column(item["physical_name"], binary64_type() if item["storage_kind"] == "numeric" else string_type(profile), - nullable=item["storage_kind"] == "numeric") for item in variables), + *(Column( + item["physical_name"], + binary64_type() if item["storage_kind"] == "numeric" else lossless_text_type(), + nullable=item["storage_kind"] == "numeric", + ) for item in variables), ) - materialized = [ - {"__case_ordinal": ordinal, **row} - for ordinal, row in enumerate(source_rows, start=1) - ] - normative_dataset_id = str(uuid4()) - namespace_owned = False - data_table_was_absent = False - try: - with engine.begin() as connection: - create_normative_catalog(connection, normative) - namespace_owned = True - metadata.create_all(connection, tables=[ - datasets, variable_catalog, multiple_response_catalog, - source_extensions_catalog, documents_catalog, value_labels_catalog, - missing_rules_catalog, attributes_catalog, fidelity_event_catalog, - operation_catalog, - ]) - _migrate_catalog_columns( - connection, datasets, variable_catalog, multiple_response_catalog, + audit_relations = { + fidelity_event_catalog.name, operation_catalog.name, + normative.fidelity_event.name, normative.operation.name, + } + preflight_state = { + "data_table_creation_attempted": False, + "legacy_dataset_created": False, + "normative_dataset_creation_attempted": False, + "normative_dataset_id": None, + } + with engine.connect() as preflight_connection: + _require_verified_catalog(preflight_connection, normative, legacy) + preflight_dolt_state = _capture_dolt_state( + preflight_connection, profile_name=profile.name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding( + preflight_dolt_state, active, phase="import preflight", + ) + preflight_connection.rollback() + try: + preflight(profile, variables, rows=source_rows) + validate_spss_catalog( + variables, + case_weight_variable=case_weight_variable, + multiple_response_sets=multiple_response_sets, ) - if connection.execute( - select(datasets.c.dataset_id).where( - datasets.c.dataset_id == dataset_id + except Exception as error: + try: + _record_failed_preflight( + engine=engine, metadata=metadata, datasets=datasets, + variable_catalog=variable_catalog, + multiple_response_catalog=multiple_response_catalog, + fidelity_event_catalog=fidelity_event_catalog, + operation_catalog=operation_catalog, operation_id=operation_id, + source_name=source_name, source_format=source_format, + variable_count=len(variables), profile_name=profile.name, + error=error, normative=normative, legacy=legacy, ) - ).first(): - raise ValueError( - f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset." + except Exception as audit_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=preflight_state, + normative=normative, legacy=legacy, ) - if connection.execute( - select(datasets.c.dataset_id).where( - datasets.c.data_table == data_table.name + raise ImportRecoveryError( + "failure_audit_failed", + "Preflight failed and its failed-operation audit could not be persisted.", + details={ + "original_cause": _safe_error_identity( + error, phase="import_preflight", + ), + "cleanup_fault": _safe_error_identity( + audit_error, phase="failed_preflight_audit", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.failed-preflight-audit.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + }, + "success_forbidden": True, + }, + ) from audit_error + try: + post_preflight_dolt_state = _capture_dolt_state( + preflight_connection, profile_name=profile.name, + audit_relations=audit_relations, ) - ).first(): - raise ValueError( - f"Dataset ID {dataset_id!r} collides with an existing physical " - "data-table name; import was not started." + dolt_boundary = _dolt_failure_boundary_evidence( + preflight_dolt_state, post_preflight_dolt_state, ) - if inspect(connection).has_table(data_table.name): - raise ValueError( - f"Physical data-table name {data_table.name!r} is already occupied." + except Exception as snapshot_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=preflight_state, + normative=normative, legacy=legacy, ) - data_table_was_absent = True - record_normative_operation( - connection, normative, operation_id=operation_id, - operation_kind="import", status="started", - source_format=source_format, - ) - connection.execute(insert(operation_catalog).values( - operation_id=operation_id, direction="import", status="running", - dataset_id=dataset_id, source=source_name, created_at=_now(), - details=json.dumps({ - "variable_count": len(variables), - **dict(operation_details or {}), - }, sort_keys=True), - )) - data_table.create(connection) - connection.execute(insert(datasets).values( - dataset_id=dataset_id, data_table=data_table.name, - source_format=source_format, source_name=source_name, - source_encoding=source_encoding, case_count=len(materialized), - source_table_name=source_table_name, source_sha256=source_sha256, - source_created_at=source_created_at, - source_modified_at=source_modified_at, imported_at=imported_at, - file_label=file_label, documents=documents, - file_attributes=file_attributes, - case_weight_variable=case_weight_variable, - multiple_response_sets=multiple_response_sets, - )) - connection.execute( - insert(variable_catalog), - [dict(dataset_id=dataset_id, **item) for item in variables], - ) - docs_rows = document_rows(dataset_id, documents) - if docs_rows: - connection.execute(insert(documents_catalog), docs_rows) - labels_rows = value_label_rows(dataset_id, variables) - if labels_rows: - connection.execute(insert(value_labels_catalog), labels_rows) - missing_rows = missing_rule_rows(dataset_id, variables) - if missing_rows: - connection.execute(insert(missing_rules_catalog), missing_rows) - attributes_rows = attribute_rows( - dataset_id, variables, - file_attributes=file_attribute_values, - variable_attributes=variable_attribute_values, - ) - if attributes_rows: - connection.execute(insert(attributes_catalog), attributes_rows) - mrset_rows = multiple_response_set_rows( - dataset_id, multiple_response_sets, - ) - if mrset_rows: - connection.execute(insert(multiple_response_catalog), mrset_rows) - extension_rows = source_extension_rows( - dataset_id, source_extensions or {}, - ) - if extension_rows: - connection.execute(insert(source_extensions_catalog), extension_rows) - event_rows = _event_rows( - operation_id=operation_id, dataset_id=dataset_id, - direction="import", fidelity_events=fidelity_events, - ) - if event_rows: - connection.execute(insert(fidelity_event_catalog), event_rows) - store_normative_dataset( - connection, normative, dataset_name=dataset_id, - source_format=source_format, physical_table_name=data_table.name, - dataset_label=file_label, source_encoding=source_encoding, - source_hash=source_sha256, source_case_count=len(materialized), - imported_at=imported_at or None, variables=variables, - documents=docs_rows, value_labels=labels_rows, - missing_rules=missing_rows, attributes=attributes_rows, - multiple_response_sets=mrset_rows, - source_extensions=source_extensions or {}, - case_weight_variable=case_weight_variable, - dataset_id=normative_dataset_id, - ) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, - dataset_id=normative_dataset_id, direction="import", - events=fidelity_events, - ) - if materialized: - connection.execute(insert(data_table), materialized) - connection.execute(update(operation_catalog).where( - operation_catalog.c.operation_id == operation_id - ).values(status="succeeded", completed_at=_now())) - finish_normative_operation( - connection, normative, operation_id=operation_id, - status="succeeded", - ) - except Exception as error: - if ( - profile.name in {"mysql", "mariadb", "dolt"} - and namespace_owned - and data_table_was_absent - ): - try: - with engine.begin() as cleanup: - delete_normative_dataset( - cleanup, normative, normative_dataset_id, - ) - cleanup_inspector = inspect(cleanup) - for table in ( - multiple_response_catalog, source_extensions_catalog, - documents_catalog, value_labels_catalog, - missing_rules_catalog, attributes_catalog, - fidelity_event_catalog, variable_catalog, - ): - if cleanup_inspector.has_table(table.name): - cleanup.execute( - delete(table).where(table.c.dataset_id == dataset_id) - ) - if cleanup_inspector.has_table(datasets.name): - cleanup.execute( - delete(datasets).where(datasets.c.dataset_id == dataset_id) - ) - data_table.drop(cleanup, checkfirst=True) - metadata.create_all(cleanup, tables=[ - datasets, variable_catalog, multiple_response_catalog, - source_extensions_catalog, documents_catalog, - value_labels_catalog, missing_rules_catalog, - attributes_catalog, fidelity_event_catalog, - operation_catalog, - ]) - failed_event = ({ - "code": "import_failed", - "detail": str(error), - "severity": "error", - "source_item": source_name, - "details": { - "profile": profile.name, - "phase": "post_ddl", - "cleanup": "complete", - "error_type": type(error).__name__, + raise ImportRecoveryError( + "cleanup_failed", + "Preflight audit completed but Dolt state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + error, phase="import_preflight", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_preflight_audit_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile.name == "dolt", + "verified": False, + }, }, - },) - normative_operation_exists = cleanup.execute(select( - normative.operation.c.operation_id - ).where( - normative.operation.c.operation_id == operation_id - )).first() - if normative_operation_exists: - finish_normative_operation( - cleanup, normative, operation_id=operation_id, - status="failed", - ) - else: - failed_at = datetime.now(UTC).replace(tzinfo=None) - record_normative_operation( - cleanup, normative, operation_id=operation_id, - operation_kind="import", status="failed", - source_format=source_format, started_at=failed_at, - completed_at=failed_at, - ) - mirror_operation_exists = cleanup.execute(select( - operation_catalog.c.operation_id - ).where( - operation_catalog.c.operation_id == operation_id - )).first() - if mirror_operation_exists: - cleanup.execute(update(operation_catalog).where( - operation_catalog.c.operation_id == operation_id - ).values( - status="failed", dataset_id=None, completed_at=_now(), - )) - else: - cleanup.execute(insert(operation_catalog).values( - operation_id=operation_id, direction="import", - status="failed", dataset_id=None, source=source_name, - created_at=_now(), completed_at=_now(), - details=json.dumps({ - "reason": "post_ddl", - "variable_count": len(variables), - }, sort_keys=True), - )) - cleanup.execute( - insert(fidelity_event_catalog), - _event_rows( - operation_id=operation_id, dataset_id=None, - direction="import", fidelity_events=failed_event, + "success_forbidden": True, + }, + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=preflight_state, + normative=normative, legacy=legacy, + ) + raise ImportRecoveryError( + "cleanup_failed", + "Preflight audit changed non-audit Dolt state.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + error, phase="import_preflight", ), - ) - record_normative_fidelity_events( - cleanup, normative, operation_id=operation_id, - dataset_id=None, direction="import", - events=failed_event, - ) - except Exception as cleanup_error: - raise RuntimeError( - f"OpenStatSpec compensating cleanup failed: {cleanup_error}" - ) from cleanup_error - raise - return { - "dataset_id": dataset_id, "data_table": data_table.name, - "case_count": len(materialized), "operation_id": operation_id, - } + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_preflight_audit_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from error + raise + with engine.connect() as mutation_connection: + pre_dolt_state = _capture_dolt_state( + mutation_connection, profile_name=profile.name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding( + pre_dolt_state, active, phase="import mutation preflight", + ) + mutation_connection.rollback() + with _import_cleanup_guard( + engine=engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, source_name=source_name, + source_format=source_format, variable_count=len(variables), + profile_name=profile.name, normative=normative, legacy=legacy, + snapshot_connection=mutation_connection, + pre_dolt_state=pre_dolt_state, + ) as mutation: + with mutation_connection.begin(): + connection = mutation_connection + _require_verified_catalog(connection, normative, legacy) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="started", source_format=source_format, + ) + connection.execute(insert(operation_catalog).values( + operation_id=operation_id, direction="import", status="running", dataset_id=dataset_id, + source=source_name, created_at=_now(), details=json.dumps({"variable_count": len(variables), **dict(operation_details or {})}, sort_keys=True), + )) + if connection.execute(select(datasets.c.dataset_id).where(datasets.c.dataset_id == dataset_id)).first(): + raise ValueError(f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset.") + if connection.execute(select(datasets.c.dataset_id).where(datasets.c.data_table == data_table.name)).first(): + raise ValueError(f"Dataset ID {dataset_id!r} collides with an existing physical data-table name; import was not started.") + mutation["data_table_creation_attempted"] = True + data_table.create(connection) + materialized = [ + {"__case_ordinal": ordinal, **row} + for ordinal, row in enumerate(source_rows, start=1) + ] + connection.execute(insert(datasets).values( + dataset_id=dataset_id, data_table=data_table.name, source_format=source_format, + source_name=source_name, source_encoding=source_encoding, case_count=len(materialized), + source_table_name=source_table_name, + source_sha256=source_sha256, + source_created_at=source_created_at, source_modified_at=source_modified_at, + imported_at=imported_at, + file_label=file_label, documents=documents, + file_attributes=file_attributes, case_weight_variable=case_weight_variable, + multiple_response_sets=multiple_response_sets, + )) + mutation["legacy_dataset_created"] = True + connection.execute(insert(variable_catalog), [dict(dataset_id=dataset_id, **item) for item in variables]) + docs_rows = document_rows(dataset_id, documents) + if docs_rows: + connection.execute(insert(documents_catalog), docs_rows) + labels_rows = value_label_rows(dataset_id, variables) + if labels_rows: + connection.execute(insert(value_labels_catalog), labels_rows) + missing_rows = missing_rule_rows(dataset_id, variables) + if missing_rows: + connection.execute(insert(missing_rules_catalog), missing_rows) + attributes_rows = attribute_rows( + dataset_id, variables, + file_attributes=file_attribute_values, + variable_attributes=variable_attribute_values, + ) + if attributes_rows: + connection.execute(insert(attributes_catalog), attributes_rows) + mrset_rows = multiple_response_set_rows(dataset_id, multiple_response_sets) + if mrset_rows: + connection.execute(insert(multiple_response_catalog), mrset_rows) + extension_rows = source_extension_rows(dataset_id, source_extensions or {}) + if extension_rows: + connection.execute(insert(source_extensions_catalog), extension_rows) + event_rows = _event_rows( + operation_id=operation_id, dataset_id=dataset_id, direction="import", fidelity_events=fidelity_events, + ) + if event_rows: + connection.execute(insert(fidelity_event_catalog), event_rows) + mutation["normative_dataset_creation_attempted"] = True + normative_dataset_id = store_normative_dataset( + connection, normative, dataset_name=dataset_id, + source_format=source_format, physical_table_name=data_table.name, + dataset_label=file_label, source_encoding=source_encoding, + source_hash=source_sha256, source_case_count=len(materialized), + imported_at=imported_at or None, variables=variables, + documents=docs_rows, value_labels=labels_rows, + missing_rules=missing_rows, attributes=attributes_rows, + multiple_response_sets=mrset_rows, + source_extensions=source_extensions or {}, + case_weight_variable=case_weight_variable, + ) + mutation["normative_dataset_id"] = normative_dataset_id + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="import", + events=fidelity_events, + ) + if materialized: + for batch in _bounded_batches( + materialized, variables, profile.max_statement_bytes, + ): + connection.execute(insert(data_table), batch) + connection.execute(update(operation_catalog).where(operation_catalog.c.operation_id == operation_id).values( + status="succeeded", completed_at=_now(), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="succeeded", + ) + post_dolt_state = _capture_dolt_state( + mutation_connection, profile_name=profile.name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding( + post_dolt_state, active, phase="import completion", + ) + _require_dolt_success_identity( + pre_dolt_state, post_dolt_state, phase="import", + ) + return {"dataset_id": dataset_id, "data_table": data_table.name, "case_count": len(materialized), "operation_id": operation_id} def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any: @@ -1045,20 +2630,29 @@ def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any: def read_wide_dataset( - *, database_url: str, dataset_id: str, profile: Any | None = None, + *, + database_url: str, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: - """Read a strict dataset only after resolving the active server profile.""" - if profile is None: - profile, _active = effective_profile(database_url) + """Read a strict dataset, preferring normalized metadata with JSON compatibility fallback.""" + profile, _active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() - datasets, variable_catalog, _, _ = catalog(metadata) + datasets, variable_catalog, fidelity_catalog, operation_catalog = catalog(metadata) + normative = normative_catalog(metadata) multiple_response_catalog = multiple_response_set_catalog(metadata) source_extensions_catalog = source_extension_catalog(metadata) documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog = normalized_metadata_tables(metadata) - with engine.begin() as connection: - metadata.create_all(connection, tables=[datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog]) - _migrate_catalog_columns(connection, datasets, variable_catalog, multiple_response_catalog) + legacy = ( + datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, + documents_catalog, value_labels_catalog, missing_rules_catalog, + attributes_catalog, fidelity_catalog, operation_catalog, + ) + with engine.connect() as connection: + _require_verified_catalog(connection, normative, legacy) dataset = dict(connection.execute(select(datasets).where(datasets.c.dataset_id == dataset_id)).mappings().one()) data_table = Table(dataset["data_table"], MetaData(), autoload_with=connection) variables = [dict(item) for item in connection.execute( @@ -1130,16 +2724,26 @@ def read_wide_dataset( rules_by_variable.setdefault(item["variable_ordinal"], []).append({"lo": lower, "hi": upper}) for ordinal, rules in rules_by_variable.items(): variables_by_ordinal[ordinal]["missing_ranges"] = json.dumps(rules, ensure_ascii=False) + preflight(profile, variables, rows=rows) return dataset, variables, rows -def read_fidelity_events(*, database_url: str, dataset_id: str) -> tuple[dict[str, Any], ...]: +def read_fidelity_events( + *, + database_url: str, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> tuple[dict[str, Any], ...]: """Read import-time fidelity diagnostics for a catalogued dataset.""" + effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() - _, _, fidelity_event_catalog, _ = catalog(metadata) + legacy, normative = _catalog_layout(metadata) + fidelity_event_catalog = legacy[8] with engine.connect() as connection: - fidelity_event_catalog.create(connection, checkfirst=True) + _require_verified_catalog(connection, normative, legacy) events = connection.execute( select(fidelity_event_catalog) .where(fidelity_event_catalog.c.dataset_id == dataset_id) @@ -1152,32 +2756,155 @@ def read_fidelity_events(*, database_url: str, dataset_id: str) -> tuple[dict[st +def record_export_cleanup_failure( + *, database_url: str, destination: str, original_error: Exception, + cleanup_error: Exception, + residual_object_inventory: Mapping[str, Any], + deterministic_recovery_evidence: Mapping[str, Any], + operation_id: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> str: + """Best-effort immutable export cleanup-failure audit.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + fidelity_events, operations = legacy[8:] + requested_operation_id = operation_id + operation_id = operation_id or str(uuid4()) + original = _safe_error_identity(original_error, phase="export") + cleanup = _safe_error_identity(cleanup_error, phase="export_destination_restore") + event_details = { + "original_cause": original, + "cleanup_fault": cleanup, + "residual_object_inventory": dict(residual_object_inventory), + "deterministic_recovery_evidence": dict( + deterministic_recovery_evidence + ), + } + event = { + "code": "cleanup_failed", + "detail": "Export destination recovery failed; out-of-band review is required.", + "severity": "error", + "source_item": destination, + "details": event_details, + } + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="record export cleanup failure", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + if requested_operation_id is not None: + existing = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one_or_none() + normative_existing = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one_or_none() + if ( + existing is None or normative_existing is None + or existing["direction"] != "export" + or existing["status"] != "running" + or normative_existing["status"] != "started" + ): + raise UnsupportedOperationError( + "Existing export operation is not in an auditable terminal-transition state." + ) + details = json.loads(existing["details"] or "{}") + details["cleanup_failure"] = event_details + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values( + status="failed", completed_at=_now(), + details=json.dumps(details, sort_keys=True), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=None, direction="export", + fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="export", events=(event,), + ) + else: + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="export", status="failed", source_format=None, + started_at=failed_at, completed_at=failed_at, + ) + connection.execute(insert(operations).values( + operation_id=operation_id, direction="export", status="failed", + dataset_id=None, destination=destination, created_at=_now(), + completed_at=_now(), details=json.dumps({ + "reason": "cleanup_failed", **event_details, + }, sort_keys=True), + )) + connection.execute(insert(fidelity_events), _event_rows( + operation_id=operation_id, dataset_id=None, direction="export", + fidelity_events=(event,), + )) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="export", events=(event,), + ) + return operation_id + + def record_export_operation( *, database_url: str, dataset_id: str, destination: str, allowed_fidelity_events: Iterable[Mapping[str, Any]], operation_details: Mapping[str, Any] | None = None, + terminal: bool = True, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> str: """Persist a completed export and the fidelity loss explicitly accepted by its caller.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() - datasets, variables, fidelity_events, operations = catalog(metadata) - normative = normative_catalog(metadata) - multiple_response = multiple_response_set_catalog(metadata) + legacy, normative = _catalog_layout(metadata) + datasets, variables, multiple_response = legacy[:3] + fidelity_events, operations = legacy[8:] operation_id = str(uuid4()) events = tuple(allowed_fidelity_events) - with engine.begin() as connection: - metadata.create_all(connection, tables=[datasets, variables, multiple_response, fidelity_events, operations]) - create_normative_catalog(connection, normative) + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="record export operation", + ) as connection: + _require_verified_catalog(connection, normative, legacy) normative_dataset_id = normative_dataset_id_for_name(connection, normative, dataset_id) completed_at = datetime.now(UTC).replace(tzinfo=None) + normative_status = "succeeded" if terminal else "started" + legacy_status = "succeeded" if terminal else "running" record_normative_operation( connection, normative, operation_id=operation_id, - operation_kind="export", status="succeeded", source_format=None, - started_at=completed_at, completed_at=completed_at, + operation_kind="export", status=normative_status, source_format=None, + started_at=completed_at, completed_at=completed_at if terminal else None, ) connection.execute(insert(operations).values( - operation_id=operation_id, direction="export", status="succeeded", dataset_id=dataset_id, - destination=destination, created_at=_now(), completed_at=_now(), + operation_id=operation_id, direction="export", status=legacy_status, dataset_id=dataset_id, + destination=destination, created_at=_now(), + completed_at=_now() if terminal else None, details=json.dumps({"allow_loss": [event["code"] for event in events], **dict(operation_details or {})}, sort_keys=True), )) rows = _event_rows( @@ -1197,12 +2924,268 @@ def record_export_operation( ) return operation_id -def validate_wide_dataset(*, database_url: str, dataset_id: str) -> dict[str, Any]: - profile, _active = effective_profile(database_url) + +def finish_export_operation( + *, + database_url: str, + operation_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Mark a published export successful only after filesystem finalization.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + operations = legacy[9] + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="finish export operation", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + row = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one() + if row["direction"] != "export" or row["status"] != "running": + raise UnsupportedOperationError( + "Only a running export operation can be finalized." + ) + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values(status="succeeded", completed_at=_now())) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="succeeded", + ) + + +def read_export_operation_state( + *, + database_url: str, + operation_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + """Read both export-operation catalogs without changing either one.""" + validate_connection_url(database_url) + effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + operations = legacy[9] + with engine.connect() as connection: + _require_verified_catalog(connection, normative, legacy) + legacy_row = connection.execute(select( + operations.c.direction, + operations.c.status, + ).where( + operations.c.operation_id == operation_id + )).mappings().one_or_none() + normative_row = connection.execute(select( + normative.operation.c.operation_kind, + normative.operation.c.status, + ).where( + normative.operation.c.operation_id == operation_id + )).mappings().one_or_none() + + legacy_state = ( + None if legacy_row is None else { + "direction": legacy_row["direction"], + "status": legacy_row["status"], + } + ) + normative_state = ( + None if normative_row is None else { + "operation_kind": normative_row["operation_kind"], + "status": normative_row["status"], + } + ) + if ( + legacy_state == {"direction": "export", "status": "succeeded"} + and normative_state == { + "operation_kind": "export", "status": "succeeded", + } + ): + classification = "succeeded" + elif ( + legacy_state == {"direction": "export", "status": "running"} + and normative_state == { + "operation_kind": "export", "status": "started", + } + ): + classification = "running" + else: + classification = "ambiguous" + return { + "operation_id": operation_id, + "legacy": legacy_state, + "normative": normative_state, + "classification": classification, + } + + +def fail_export_operation( + *, + database_url: str, + operation_id: str, + failure_details: Mapping[str, Any], + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Close one running export after filesystem compensation succeeded.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + fidelity_events, operations = legacy[8:] + audit_relations = { + fidelity_events.name, operations.name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="fail export operation", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + row = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one() + normative_row = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one() + if ( + row["direction"] != "export" or row["status"] != "running" + or normative_row["operation_kind"] != "export" + or normative_row["status"] != "started" + ): + raise UnsupportedOperationError( + "Only matching running export operations can be failed." + ) + details = json.loads(row["details"] or "{}") + details["failure"] = dict(failure_details) + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values( + status="failed", completed_at=_now(), + details=json.dumps(details, sort_keys=True), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) + event = { + "code": "export_failed", + "detail": "Export publication or finalization failed after audit start.", + "severity": "error", + "source_item": row["destination"], + "details": dict(failure_details), + } + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=row["dataset_id"], + direction="export", fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + normative_dataset_id = normative_dataset_id_for_name( + connection, normative, row["dataset_id"], + ) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="export", events=(event,), + ) + + +def record_export_backup_retained( + *, database_url: str, operation_id: str, destination: str, backup: str, + cleanup_error: Exception, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Append a warning without rewriting a successfully finalized export.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + fidelity_events, operations = legacy[8:] + audit_relations = { + fidelity_events.name, operations.name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="record retained export backup", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + row = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one() + normative_row = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one() + if ( + row["direction"] != "export" or row["status"] != "succeeded" + or normative_row["operation_kind"] != "export" + or normative_row["status"] != "succeeded" + ): + raise UnsupportedOperationError( + "A retained backup warning requires a matching succeeded export." + ) + details = json.loads(row["details"] or "{}") + details["backup_retained"] = { + "destination": destination, "durable_backup": backup, + "cleanup_error_type": type(cleanup_error).__name__, + } + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values(details=json.dumps(details, sort_keys=True))) + event = { + "code": "backup_retained", + "detail": "A successful export retained its durable prior-file backup.", + "severity": "warning", + "source_item": destination, + "details": { + "durable_backup": backup, + "cleanup_error_type": type(cleanup_error).__name__, + }, + } + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=row["dataset_id"], + direction="export", fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + normative_dataset_id = normative_dataset_id_for_name( + connection, normative, row["dataset_id"], + ) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="export", events=(event,), + ) + + +def validate_wide_dataset( + *, + database_url: str, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: dataset, variables, rows = read_wide_dataset( - database_url=database_url, dataset_id=dataset_id, profile=profile, + database_url=database_url, + dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, ) - preflight(profile, variables, rows=rows) validate_spss_catalog( variables, case_weight_variable=dataset.get("case_weight_variable"), @@ -1228,11 +3211,6 @@ def validate_wide_dataset(*, database_url: str, dataset_id: str) -> dict[str, An if item["storage_kind"] == "numeric": if not isinstance(column.type, Float) or not column.nullable: raise ValueError(f"Numeric variable {item['source_name']!r} must be a nullable binary64 column.") - elif profile.name == "dolt": - if not isinstance(column.type, mysql.LONGTEXT) or column.nullable: - raise ValueError( - f"String variable {item['source_name']!r} must be a non-null LONGTEXT column." - ) elif not isinstance(column.type, Text) or column.nullable: raise ValueError(f"String variable {item['source_name']!r} must be a non-null text column.") if [row["__case_ordinal"] for row in rows] != list(range(1, len(rows) + 1)): diff --git a/src/openstatspec/sql/workflow.py b/src/openstatspec/sql/workflow.py index 5b2fd97..8d96702 100644 --- a/src/openstatspec/sql/workflow.py +++ b/src/openstatspec/sql/workflow.py @@ -32,10 +32,11 @@ from ..core import UnsupportedOperationError from .capabilities import SPECIFICATION_COMMIT from .normative import ( - CATALOG_CONTRACT_ID, CATALOG_SCHEMA_VERSION, catalog as core_catalog, + CATALOG_CONTRACT_ID, CATALOG_SCHEMA_VERSION, binary64_type, + catalog as core_catalog, ) from .profiles import preflight, validate_connection_url -from .wide import binary64_type, physical_name +from .wide import physical_name PROFILE_ID = "openstatspec-sql-transformation-workflow-v0.1" PROFILE_SCHEMA_VERSION = 2 diff --git a/tests/conformance.py b/tests/conformance.py index 677f865..528df5f 100755 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -5,8 +5,8 @@ import pandas as pd import pyspssio -from pandas.testing import assert_frame_equal +from openstatspec import compare_sav_semantics from openstatspec.spss.raw_dictionary import write_compatible_names @@ -22,7 +22,7 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] """ destination = Path(destination) long_text = "Õ🙂漢字" * 90 - compatible_text = "\u00d5\U0001f642\u6f22\u5b57" * 30 + long_text_variable = "long_comment_variable" frame = pd.DataFrame({ "discrete_missing": [1.0, 2.0, 3.0, 4.0], "range_only": [-1.0, 0.0, 1.0, 2.0], @@ -30,8 +30,7 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] "highest_range": [-99.0, 0.0, 1.0, 42.0], "code": [3.0, 1.0, 2.0, 1.0], "status": ["NA", "DK", "ok", ""], - "comment": [long_text, "", "näide", long_text], - "very_long_compatible_name": [compatible_text, "", "tail", compatible_text], + long_text_variable: [long_text, "", "näide", long_text], "interview_date": [23123.0, 23124.0, 23125.0, 23126.0], "interview_time": [3661.25, 0.0, 86399.5, 12.0], "interview_datetime": [23123.5, 23124.0, 23125.75, 23126.125], @@ -43,14 +42,11 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] "resp_b": [0.0, 1.0, 1.0, 0.0], }) metadata = { - "var_types": { - "status": 8, "comment": 1024, "very_long_compatible_name": 360, - }, + "var_types": {"status": 8, long_text_variable: 1024}, "var_formats": { "discrete_missing": "F8.0", "range_only": "F8.0", "lowest_range": "F12.1", "highest_range": "F12.1", "code": "F8.0", - "status": "A8", "comment": "A1024", "very_long_compatible_name": "A360", - "interview_date": "DATE11", + "status": "A8", long_text_variable: "A1024", "interview_date": "DATE11", "interview_time": "TIME8", "interview_datetime": "DATETIME20", "interview_dtime": "DTIME10", "formatted_comma": "COMMA12.2", "formatted_dot": "DOT12.2", "formatted_pct": "PCT8.1", @@ -60,8 +56,7 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] "range_only": "Range-only numeric user missing", "lowest_range": "LOWEST-style missing range", "highest_range": "HIGHEST-style missing range", "code": "Ordered numeric code", "status": "String missing code", - "comment": "Long UTF-8 comment", "very_long_compatible_name": "Custom VLS name", - "interview_date": "SPSS numeric date", + long_text_variable: "Long UTF-8 comment", "interview_date": "SPSS numeric date", "interview_time": "SPSS numeric time", "interview_datetime": "SPSS numeric datetime", "interview_dtime": "SPSS numeric duration", "formatted_comma": "SPSS comma format", "formatted_dot": "SPSS dot format", "formatted_pct": "SPSS percent format", @@ -70,7 +65,7 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] "discrete_missing": {3.0: "third", 1.0: "first", 2.0: "second"}, "code": {3.0: "third", 1.0: "first", 2.0: "second"}, "status": {"DK": "don't know", "NA": "not answered", "ok": "valid"}, - "comment": {"coded": "Short code in a very-long string variable"}, + long_text_variable: {"coded": "Short code in a very-long string variable"}, }, "var_missing_values": { "discrete_missing": {"values": [1.0, 2.0, 3.0]}, @@ -82,14 +77,12 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] "var_measure_levels": { "discrete_missing": "scale", "range_only": "scale", "lowest_range": "scale", "highest_range": "scale", "code": "scale", - "status": "nominal", "comment": "nominal", - "very_long_compatible_name": "nominal", "interview_date": "scale", + "status": "nominal", long_text_variable: "nominal", "interview_date": "scale", }, "var_alignments": {column: "left" for column in frame.columns}, "var_column_widths": { "discrete_missing": 8, "range_only": 8, "lowest_range": 12, "highest_range": 12, - "code": 8, "status": 12, "comment": 48, "very_long_compatible_name": 48, - "interview_date": 11, + "code": 8, "status": 12, long_text_variable: 48, "interview_date": 11, "interview_time": 8, "interview_datetime": 20, "interview_dtime": 10, "formatted_comma": 12, "formatted_dot": 12, "formatted_pct": 8, }, @@ -101,54 +94,15 @@ def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any] "case_weight_var": "code", } pyspssio.write_sav(str(destination), frame, metadata=metadata) + compatible_name = "LCOMM" write_compatible_names( - destination, {"very_long_compatible_name": "VLSTEXT"}, encoding="UTF-8", + destination, {long_text_variable: compatible_name}, encoding="UTF-8", ) - return {"long_text": long_text, "file_label": metadata["file_label"]} - - -def sys_float_max() -> float: - return float.fromhex("0x1.fffffffffffffp+1023") - - -def _canonical_missing(metadata: dict[str, Any]) -> dict[str, tuple[tuple[Any, Any, tuple[Any, ...]], ...]]: - result: dict[str, tuple[tuple[Any, Any, tuple[Any, ...]], ...]] = {} - for name, rule in (metadata.get("var_missing_values") or {}).items(): - rule = rule or {} - if "lo" in rule or "hi" in rule: - result[name] = ((rule.get("lo"), rule.get("hi"), tuple(rule.get("values") or ())),) - else: - result[name] = ((None, None, tuple(rule.get("values") or ())),) - return result - - -def _ordered_value_labels(metadata: dict[str, Any]) -> dict[str, tuple[tuple[Any, str], ...]]: - """Retain the label order exposed by the SPSS engine, not merely its mapping.""" return { - name: tuple((value, str(label)) for value, label in labels.items()) - for name, labels in (metadata.get("var_value_labels") or {}).items() + "long_text": long_text, "long_text_variable": long_text_variable, + "compatible_name": compatible_name, "file_label": metadata["file_label"], } -def compare_sav_semantics(source: str | Path, exported: str | Path) -> dict[str, Any]: - source_frame, source_metadata = pyspssio.read_sav(str(source), convert_datetimes=False, include_user_missing=True) - exported_frame, exported_metadata = pyspssio.read_sav(str(exported), convert_datetimes=False, include_user_missing=True) - failures: list[str] = [] - if list(source_frame.columns) != list(exported_frame.columns): - failures.append("variable-order") - try: - assert_frame_equal(source_frame, exported_frame, check_dtype=True, check_like=False) - except AssertionError: - failures.append("values-or-case-order") - for attribute in ( - "encoding", "file_label", "case_weight_var", "file_attributes", "mrsets", "var_types", "var_formats", - "var_labels", "var_alignments", "var_column_widths", "var_measure_levels", "var_roles", - "var_value_labels", "var_attributes", "var_compat_names", - ): - if source_metadata.get(attribute) != exported_metadata.get(attribute): - failures.append(attribute) - if _canonical_missing(source_metadata) != _canonical_missing(exported_metadata): - failures.append("var_missing_values") - if _ordered_value_labels(source_metadata) != _ordered_value_labels(exported_metadata): - failures.append("var_value_label_order") - return {"equivalent": not failures, "differences": failures} +def sys_float_max() -> float: + return float.fromhex("0x1.fffffffffffffp+1023") diff --git a/tests/test_atomic_import.py b/tests/test_atomic_import.py index 88a98a3..d79509a 100755 --- a/tests/test_atomic_import.py +++ b/tests/test_atomic_import.py @@ -5,13 +5,14 @@ from sqlalchemy.exc import IntegrityError import openstatspec.sql.wide as wide -from openstatspec.sql.profiles import DOLT, MYSQL, SQLITE -from openstatspec.sql.wide import create_wide_dataset +from openstatspec.sql.profiles import SQLITE +from openstatspec.sql.wide import create_wide_dataset, initialize_wide_catalog def test_failed_row_insert_leaves_no_catalog_or_data_table(tmp_path) -> None: database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" + initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string", "string_width": 8, "label": "", @@ -28,14 +29,16 @@ def test_failed_row_insert_leaves_no_catalog_or_data_table(tmp_path) -> None: connection = sqlite3.connect(database_path) tables = [row[0] for row in connection.execute("select name from sqlite_master where type = 'table'")] assert "data_broken" not in tables - assert "catalog_identity" in tables - assert connection.execute("select count(*) from dataset").fetchone() == (0,) - assert connection.execute("select count(*) from variable").fetchone() == (0,) + assert "dataset_catalog" in tables + assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) + assert "variable_catalog" in tables + assert connection.execute("select count(*) from variable_catalog").fetchone() == (0,) def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path) -> None: database_path = tmp_path / "preflight.sqlite" database = f"sqlite:///{database_path}" + initialize_wide_catalog(database_url=database) with pytest.raises(Exception, match="Target capability exceeded"): create_wide_dataset( @@ -61,6 +64,7 @@ def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path) def test_identifier_mapping_preflight_records_failure_before_dataset_creation(tmp_path) -> None: database_path = tmp_path / "identifier.sqlite" database = f"sqlite:///{database_path}" + initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "wrong_name", "storage_kind": "string", "string_width": 8, "label": "", @@ -84,9 +88,10 @@ def test_identifier_mapping_preflight_records_failure_before_dataset_creation(tm def test_declared_string_width_preflight_is_atomic_and_diagnostic(tmp_path, monkeypatch) -> None: database_path = tmp_path / "string-width.sqlite" database = f"sqlite:///{database_path}" + initialize_wide_catalog(database_url=database) monkeypatch.setattr( wide, "effective_profile", - lambda _url: (replace(SQLITE, max_text_value_bytes=3), {}), + lambda _url, **kwargs: (replace(SQLITE, max_text_value_bytes=3), {}), ) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", @@ -112,247 +117,3 @@ def test_declared_string_width_preflight_is_atomic_and_diagnostic(tmp_path, monk details = connection.execute("select details from fidelity_event_catalog").fetchone()[0] assert '"reason": "declared_string_width_limit"' in details assert '"string_width": 4' in details - -def test_nonatomic_failure_after_normative_write_cleans_both_catalogs_and_data( - tmp_path, monkeypatch, -) -> None: - database_path = tmp_path / "nonatomic-cleanup.sqlite" - database = f"sqlite:///{database_path}" - variables = [{ - "ordinal": 1, "source_name": "name", "physical_name": "name", - "storage_kind": "string", "string_width": 8, "label": "", - "format": "A8", "measure": "nominal", "alignment": "left", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - monkeypatch.setattr( - wide, "effective_profile", - lambda _url: (replace(MYSQL, name="mysql"), {}), - ) - real_store = wide.store_normative_dataset - - def fail_after_normative_write(*args, **kwargs): - real_store(*args, **kwargs) - raise RuntimeError("fault after normative write") - - monkeypatch.setattr(wide, "store_normative_dataset", fail_after_normative_write) - - with pytest.raises(RuntimeError, match="fault after normative write"): - create_wide_dataset( - database_url=database, dataset_id="cleanup", source_name="fixture.sav", - source_format="SAV", rows=[{"name": "ok"}], variables=variables, - ) - - connection = sqlite3.connect(database_path) - assert "data_cleanup" not in { - row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" - ) - } - existing_tables = { - row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" - ) - } - if "dataset_catalog" in existing_tables: - assert connection.execute( - "select count(*) from dataset_catalog where dataset_id = 'cleanup'" - ).fetchone() == (0,) - if "variable_catalog" in existing_tables: - assert connection.execute( - "select count(*) from variable_catalog where dataset_id = 'cleanup'" - ).fetchone() == (0,) - assert connection.execute( - "select count(*) from dataset where dataset_name = 'cleanup'" - ).fetchone() == (0,) - assert connection.execute( - "select count(*) from variable" - ).fetchone() == (0,) - assert connection.execute( - "select direction, status, dataset_id from operation_catalog" - ).fetchall() == [("import", "failed", None)] - assert connection.execute( - "select direction, severity, code, dataset_id from fidelity_event_catalog" - ).fetchall() == [("import", "error", "import_failed", None)] - assert connection.execute( - "select operation_kind, status from operation" - ).fetchall() == [("import", "failed")] - assert connection.execute( - "select direction, severity, event_code, dataset_id from fidelity_event" - ).fetchall() == [("import", "error", "import_failed", None)] - -def test_occupied_foreign_namespace_fails_without_modification(tmp_path) -> None: - database_path = tmp_path / "foreign.sqlite" - database = f"sqlite:///{database_path}" - connection = sqlite3.connect(database_path) - connection.execute("create table foreign_data (value text not null)") - connection.execute("insert into foreign_data values ('keep')") - connection.commit() - before_schema = connection.execute( - "select name, sql from sqlite_master where type = 'table' order by name" - ).fetchall() - - variables = [{ - "ordinal": 1, "source_name": "name", "physical_name": "name", - "storage_kind": "string", "string_width": 8, "label": "", - "format": "A8", "measure": "nominal", "alignment": "left", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - with pytest.raises(RuntimeError, match="occupied"): - create_wide_dataset( - database_url=database, dataset_id="foreign", source_name="fixture.sav", - source_format="SAV", rows=[{"name": "ok"}], variables=variables, - ) - - assert connection.execute( - "select name, sql from sqlite_master where type = 'table' order by name" - ).fetchall() == before_schema - assert connection.execute("select value from foreign_data").fetchall() == [("keep",)] - - -@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) -def test_dolt_nonfinite_preflight_creates_no_dataset_or_physical_table( - tmp_path, monkeypatch, value, -) -> None: - database_path = tmp_path / "dolt-nonfinite.sqlite" - database = f"sqlite:///{database_path}" - monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, {})) - variables = [{ - "ordinal": 1, "source_name": "value", "physical_name": "value", - "storage_kind": "numeric", "string_width": None, "label": "", - "format": "F8.2", "measure": "scale", "alignment": "right", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - - with pytest.raises(Exception, match="Target capability exceeded"): - create_wide_dataset( - database_url=database, dataset_id="nonfinite", - source_name="nonfinite.sav", source_format="SAV", - rows=[{"value": value}], variables=variables, - ) - - connection = sqlite3.connect(database_path) - assert "data_nonfinite" not in { - row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" - ) - } - assert connection.execute("select count(*) from dataset").fetchone() == (0,) - assert connection.execute("select count(*) from variable").fetchone() == (0,) - assert connection.execute( - "select status from operation" - ).fetchall() == [("failed",)] - assert connection.execute( - "select dataset_id, event_code from fidelity_event" - ).fetchall() == [(None, "target_capability_exceeded")] - - -def test_empty_namespace_dolt_width_failure_initializes_identity_and_one_audit( - tmp_path, monkeypatch, -) -> None: - database_path = tmp_path / "dolt-preflight.sqlite" - database = f"sqlite:///{database_path}" - monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, {})) - - with pytest.raises(Exception, match="Target capability exceeded"): - create_wide_dataset( - database_url=database, dataset_id="too-wide-dolt", - source_name="too-wide-dolt.sav", source_format="SAV", - rows=(), variables=[{}] * 306, - ) - - connection = sqlite3.connect(database_path) - assert connection.execute( - "select contract_id, schema_version from catalog_identity" - ).fetchall() == [("openstatspec-strict-wide-table-v1", 1)] - assert connection.execute( - "select operation_kind, status from operation" - ).fetchall() == [("import", "failed")] - assert connection.execute( - "select dataset_id, event_code from fidelity_event" - ).fetchall() == [(None, "target_capability_exceeded")] - assert connection.execute( - "select dataset_id, status from operation_catalog" - ).fetchall() == [(None, "failed")] - assert connection.execute( - "select dataset_id, code from fidelity_event_catalog" - ).fetchall() == [(None, "target_capability_exceeded")] - - -@pytest.mark.parametrize("failure_point", ["mirror_completion", "normative_completion"]) -def test_nonatomic_failure_during_final_completion_still_cleans_dataset( - tmp_path, monkeypatch, failure_point, -) -> None: - database_path = tmp_path / f"{failure_point}.sqlite" - database = f"sqlite:///{database_path}" - dataset_id = f"cleanup-{failure_point}" - variables = [{ - "ordinal": 1, "source_name": "name", "physical_name": "name", - "storage_kind": "string", "string_width": 8, "label": "", - "format": "A8", "measure": "nominal", "alignment": "left", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - monkeypatch.setattr( - wide, "effective_profile", - lambda _url: (replace(MYSQL, name="mysql"), {}), - ) - triggered = False - if failure_point == "mirror_completion": - real_update = wide.update - - def fail_first_operation_update(table): - nonlocal triggered - if table.name == "operation_catalog" and not triggered: - triggered = True - raise RuntimeError("fault after data insert") - return real_update(table) - - monkeypatch.setattr(wide, "update", fail_first_operation_update) - else: - real_finish = wide.finish_normative_operation - - def fail_first_normative_finish(*args, **kwargs): - nonlocal triggered - if not triggered: - triggered = True - raise RuntimeError("fault during final completion") - return real_finish(*args, **kwargs) - - monkeypatch.setattr( - wide, "finish_normative_operation", fail_first_normative_finish, - ) - - with pytest.raises(RuntimeError, match="fault"): - create_wide_dataset( - database_url=database, dataset_id=dataset_id, - source_name=f"{dataset_id}.sav", source_format="SAV", - rows=[{"name": "ok"}], variables=variables, - ) - - connection = sqlite3.connect(database_path) - assert triggered is True - assert f"data_{dataset_id}" not in { - row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" - ) - } - existing_tables = { - row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" - ) - } - if "dataset_catalog" in existing_tables: - assert connection.execute( - "select count(*) from dataset_catalog where dataset_id = ?", (dataset_id,) - ).fetchone() == (0,) - assert connection.execute( - "select count(*) from dataset where dataset_name = ?", (dataset_id,) - ).fetchone() == (0,) - assert connection.execute( - "select status, dataset_id from operation_catalog" - ).fetchall() == [("failed", None)] - assert connection.execute( - "select status from operation" - ).fetchall() == [("failed",)] - assert connection.execute( - "select code, dataset_id from fidelity_event_catalog" - ).fetchall() == [("import_failed", None)] diff --git a/tests/test_attribute_catalog.py b/tests/test_attribute_catalog.py index 97df2c1..de809c9 100644 --- a/tests/test_attribute_catalog.py +++ b/tests/test_attribute_catalog.py @@ -28,6 +28,7 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su }, ) + openstatspec.initialize_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="attributes") connection = sqlite3.connect(database_path) assert connection.execute( @@ -62,6 +63,7 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su assert exported["var_attributes"] == { "answer": {"Source": "catalog-variable", "Flag": "yes"}, } + openstatspec.initialize_catalog(database_url=f"sqlite:///{imported_again}") openstatspec.import_sav(destination, database_url=f"sqlite:///{imported_again}", dataset_id="again") reimported = sqlite3.connect(imported_again) assert reimported.execute( @@ -74,37 +76,41 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su ).fetchall() == [("Source", "catalog-variable"), ("Flag", "yes")] -def test_attribute_catalog_migrates_old_json_catalog_without_rewriting_it(tmp_path) -> None: +def test_initializer_rejects_unverified_catalog_without_rewriting_it(tmp_path) -> None: source = tmp_path / "legacy.sav" - destination = tmp_path / "legacy-out.sav" database_path = tmp_path / "legacy.sqlite" database = f"sqlite:///{database_path}" pyspssio.write_sav( str(source), pd.DataFrame({"answer": [1.0]}), metadata={"file_attributes": {"File": "legacy"}, "var_attributes": {"answer": {"Var": "legacy"}}}, ) + openstatspec.initialize_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="legacy") connection = sqlite3.connect(database_path) connection.execute("drop table attribute_catalog") connection.commit() + before = connection.execute( + "select type, name, sql from sqlite_master " + "where name not like 'sqlite_%' order by type, name" + ).fetchall() + connection.close() - # Opening an older catalog additively creates the new table and continues to - # use legacy JSON only because that older dataset has no normalized rows. - dataset, variables, _ = read_wide_dataset(database_url=database, dataset_id="legacy") - assert json.loads(dataset["file_attributes"]) == {"File": "legacy"} - assert json.loads(variables[0]["attributes"]) == {"Var": "legacy"} - assert connection.execute( - "select name from sqlite_master where type = 'table' and name = 'attribute_catalog'" - ).fetchone() == ("attribute_catalog",) - openstatspec.export_sav( - database_url=database, dataset_id="legacy", destination=destination, - allow_loss=_REQUIRED_ENGINE_LOSS, - ) - assert pyspssio.read_metadata(str(destination))["file_attributes"] == {"File": "legacy"} + with pytest.raises(UnsupportedOperationError, match="verified OpenStatSpec catalog"): + openstatspec.initialize_catalog(database_url=database) + + connection = sqlite3.connect(database_path) + after = connection.execute( + "select type, name, sql from sqlite_master " + "where name not like 'sqlite_%' order by type, name" + ).fetchall() + connection.close() + assert after == before + assert "attribute_catalog" not in {name for _kind, name, _sql in after} def test_attribute_catalog_preserves_ordered_arrays_through_raw_pyspssio_bridge(tmp_path) -> None: database = f"sqlite:///{tmp_path / 'array.sqlite'}" + openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="array", source_name="array.sav", source_format="SAV", rows=[{"answer": 1.0}], diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py new file mode 100644 index 0000000..f1b985c --- /dev/null +++ b/tests/test_catalog_lifecycle.py @@ -0,0 +1,336 @@ +import sqlite3 + +import pytest +from sqlalchemy import Table + +import openstatspec +import openstatspec.sql.wide as wide +from openstatspec.core import UnsupportedOperationError +from openstatspec.sql.wide import ( + ImportRecoveryError, + _bounded_batches, + create_wide_dataset, + read_wide_dataset, + record_export_operation, + validate_wide_dataset, +) + + +def _variables(): + return [{ + "ordinal": 1, + "source_name": "name", + "physical_name": "name", + "storage_kind": "string", + "string_width": 8, + "label": "", + "format": "A8", + "measure": "nominal", + "alignment": "left", + "display_width": 8, + "value_labels": "{}", + "missing_ranges": "[]", + }] + + +def _table_names(path): + connection = sqlite3.connect(path) + try: + return { + row[0] for row in connection.execute( + "select name from sqlite_master where type = 'table'" + ) + } + finally: + connection.close() + + +def test_import_requires_explicit_catalog_and_creates_no_relations(tmp_path): + path = tmp_path / "absent.sqlite" + database = f"sqlite:///{path}" + + with pytest.raises(UnsupportedOperationError, match="catalog is absent"): + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + assert _table_names(path) == set() + + +@pytest.mark.parametrize( + "operation", + [ + lambda database: read_wide_dataset(database_url=database, dataset_id="missing"), + lambda database: validate_wide_dataset(database_url=database, dataset_id="missing"), + lambda database: record_export_operation( + database_url=database, + dataset_id="missing", + destination="output.sav", + allowed_fidelity_events=(), + ), + ], +) +def test_read_validate_and_export_require_catalog_without_mutation(tmp_path, operation): + path = tmp_path / "absent.sqlite" + database = f"sqlite:///{path}" + + with pytest.raises(UnsupportedOperationError, match="catalog is absent"): + operation(database) + + assert _table_names(path) == set() + + +@pytest.mark.parametrize( + "foreign_sql", + [ + "create table foreign_relation (value integer)", + "create table view_source (value integer); create view foreign_view as select value from view_source", + ], +) +def test_initializer_rejects_foreign_tables_and_views_without_modification( + tmp_path, foreign_sql, +): + path = tmp_path / "foreign.sqlite" + connection = sqlite3.connect(path) + connection.executescript(foreign_sql) + before = connection.execute( + "select type, name, sql from sqlite_master " + "where name not like 'sqlite_%' order by type, name" + ).fetchall() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="catalog is foreign"): + openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") + + connection = sqlite3.connect(path) + after = connection.execute( + "select type, name, sql from sqlite_master " + "where name not like 'sqlite_%' order by type, name" + ).fetchall() + connection.close() + assert after == before + + +def test_initializer_compensates_partial_catalog_install(tmp_path, monkeypatch): + path = tmp_path / "partial.sqlite" + + def fail_migration(*_args, **_kwargs): + raise RuntimeError("injected catalog migration failure") + + monkeypatch.setattr(wide, "_migrate_catalog_columns", fail_migration) + with pytest.raises(RuntimeError, match="injected catalog migration failure"): + openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") + + assert _table_names(path) == set() + + +def test_post_ddl_failure_removes_dataset_state_and_persists_null_dataset_audit( + tmp_path, monkeypatch, +): + path = tmp_path / "runtime.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + original = wide.store_normative_dataset + + def store_then_fail(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected normative failure") + + monkeypatch.setattr(wide, "store_normative_dataset", store_then_fail) + with pytest.raises(RuntimeError, match="injected normative failure"): + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + assert "data_sample" not in _table_names(path) + connection = sqlite3.connect(path) + assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) + assert connection.execute("select count(*) from dataset").fetchone() == (0,) + assert connection.execute( + "select status, dataset_id from operation_catalog" + ).fetchall() == [("failed", None)] + assert connection.execute( + "select code, dataset_id from fidelity_event_catalog" + ).fetchall() == [("import_failed", None)] + assert connection.execute( + "select status from operation" + ).fetchall() == [("failed",)] + assert connection.execute( + "select event_code, dataset_id from fidelity_event" + ).fetchall() == [("import_failed", None)] + connection.close() + + +def test_cleanup_failure_has_machine_readable_error(tmp_path, monkeypatch): + path = tmp_path / "cleanup.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + + def fail_cleanup(*_args, **_kwargs): + raise RuntimeError("injected cleanup failure") + + def fail_mutation(*_args, **_kwargs): + raise RuntimeError("injected mutation failure") + + monkeypatch.setattr(wide, "_cleanup_import_state", fail_cleanup) + monkeypatch.setattr(wide, "store_normative_dataset", fail_mutation) + with pytest.raises(ImportRecoveryError) as error: + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + assert error.value.code == "cleanup_failed" + assert error.value.details["original_cause"]["type"] == "RuntimeError" + assert error.value.details["cleanup_fault"]["type"] == "RuntimeError" + assert error.value.details["success_forbidden"] is True + evidence = error.value.details["deterministic_recovery_evidence"] + assert evidence["procedure_id"] == "openstatspec.import-compensation.v1" + assert evidence["cleanup_attempted"] is True + assert evidence["cleanup_succeeded"] is False + assert evidence["operation_owned_state_targeted"] is True + assert evidence["cleanup_failed_audit_persisted"] is True + assert evidence["terminal_reporting"] == "catalog_and_exception" + assert len(evidence["residual_inventory_sha256"]) == 64 + connection = sqlite3.connect(path) + assert connection.execute( + "select status, dataset_id from operation_catalog" + ).fetchall() == [("failed", None)] + assert connection.execute( + "select code, dataset_id from fidelity_event_catalog" + ).fetchall() == [("cleanup_failed", None)] + connection.close() + + +def test_bounded_batches_never_exceed_statement_payload_limit(): + variables = _variables() + rows = [ + {"name": "aaaa"}, + {"name": "bbbb"}, + {"name": "cccc"}, + ] + single = wide.statement_payload_bytes(rows[0], variables) + + batches = list(_bounded_batches(rows, variables, single * 2)) + + assert batches == [rows[:2], rows[2:]] + + +def test_duplicate_import_failure_preserves_existing_dataset(tmp_path): + path = tmp_path / "duplicate.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="first.sav", + source_format="SAV", + rows=[{"name": "first"}], + variables=_variables(), + ) + + with pytest.raises(ValueError, match="already exists"): + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="second.sav", + source_format="SAV", + rows=[{"name": "second"}], + variables=_variables(), + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select dataset_id, case_count from dataset_catalog" + ).fetchall() == [("sample", 1)] + assert connection.execute( + "select dataset_name, source_case_count from dataset" + ).fetchall() == [("sample", 1)] + assert connection.execute( + "select name from data_sample" + ).fetchall() == [("first",)] + connection.close() + + + +def test_missing_additive_column_requires_explicit_migration_without_audit_mutation( + tmp_path, +): + path = tmp_path / "migration-required.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + connection = sqlite3.connect(path) + connection.execute("alter table variable_catalog drop column compat_name") + connection.commit() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="catalog is migration_required"): + create_wide_dataset( + database_url=database, dataset_id="sample", + source_name="fixture.sav", source_format="SAV", + rows=[{"name": "ok"}], variables=_variables(), + ) + + connection = sqlite3.connect(path) + assert "compat_name" not in { + row[1] for row in connection.execute("pragma table_info(variable_catalog)") + } + assert connection.execute("select count(*) from operation_catalog").fetchone() == (0,) + connection.close() + + result = openstatspec.initialize_catalog(database_url=database) + assert result["catalog"] == "verified" + connection = sqlite3.connect(path) + assert "compat_name" in { + row[1] for row in connection.execute("pragma table_info(variable_catalog)") + } + connection.close() + + +def test_multiple_catalog_identities_are_ambiguous_and_never_mutated(tmp_path): + path = tmp_path / "ambiguous.sqlite" + connection = sqlite3.connect(path) + connection.executescript( + "create table catalog_identity (" + "catalog_identity_key integer primary key, " + "contract_id varchar(128) not null unique, " + "schema_version integer not null, created_at datetime not null);" + "insert into catalog_identity values " + "(1, 'openstatspec-strict-wide-table-v1', 1, '2026-01-01')," + "(2, 'conflicting-contract', 1, '2026-01-01');" + ) + before = connection.execute( + "select * from catalog_identity order by catalog_identity_key" + ).fetchall() + connection.close() + + database = f"sqlite:///{path}" + with pytest.raises(UnsupportedOperationError, match="catalog is ambiguous"): + create_wide_dataset( + database_url=database, dataset_id="sample", + source_name="fixture.sav", source_format="SAV", + rows=[{"name": "ok"}], variables=_variables(), + ) + with pytest.raises(UnsupportedOperationError, match="catalog is ambiguous"): + openstatspec.initialize_catalog(database_url=database) + + connection = sqlite3.connect(path) + assert connection.execute( + "select * from catalog_identity order by catalog_identity_key" + ).fetchall() == before + assert _table_names(path) == {"catalog_identity"} + connection.close() diff --git a/tests/test_cli.py b/tests/test_cli.py index 26d3ff8..8c1da12 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,4 @@ import json -import sqlite3 import openstatspec import openstatspec.cli @@ -13,6 +12,11 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N output = tmp_path / "output.zsav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + assert openstatspec.cli.main(["init", "--database-url", database]) == 0 + initialized = json.loads(capsys.readouterr().out) + assert initialized["catalog"] == "verified" + assert initialized["profile"] == "sqlite" + assert openstatspec.cli.main(["import", str(source), "--database-url", database, "--dataset-id", "fixture"]) == 0 imported = json.loads(capsys.readouterr().out) assert openstatspec.cli.main(["inspect", str(source)]) == 0 @@ -25,7 +29,9 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N assert imported["case_count"] == 1 from openstatspec.core.results import OperationResult - assert isinstance(openstatspec.import_sav(source, database_url=f"sqlite:///{tmp_path / 'typed.sqlite'}", dataset_id="typed"), OperationResult) + typed_database = f"sqlite:///{tmp_path}/typed.sqlite" + openstatspec.initialize_catalog(database_url=typed_database) + assert isinstance(openstatspec.import_sav(source, database_url=typed_database, dataset_id="typed"), OperationResult) assert openstatspec.cli.main(["validate", "--database-url", database, "--dataset-id", "fixture"]) == 0 assert json.loads(capsys.readouterr().out)["valid"] is True @@ -39,9 +45,9 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> None: matrix = openstatspec.capability_matrix() - assert matrix["specification_status"] == "released" - assert matrix["specification_release"] == "v0.2.0" - assert matrix["specification_commit"] == "79339ec3d8f8aa81789b7e85f6b8afa6f1374e50" + assert matrix["specification_status"] == "release_candidate" + assert matrix["specification_release"] is None + assert matrix["specification_commit"] == "6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045" assert matrix["directions"] == ["import", "export", "semantic_round_trip"] assert matrix["active_connection"] is None assert matrix["engine"]["package"] == "openstatspec-pyspssio" @@ -96,63 +102,22 @@ def test_capability_matrix_reports_active_sqlite_limits(tmp_path) -> None: } -def test_cli_installs_schema_and_applies_plan_or_spss( - tmp_path, capsys, -) -> None: - source = tmp_path / "transform-source.sav" - database_path = tmp_path / "transform.sqlite" - database_url = f"sqlite:///{database_path}" - pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) - openstatspec.import_sav( - source, - database_url=database_url, - dataset_id="transform-cli", - ) - connection = sqlite3.connect(database_path) - live_dataset_id = connection.execute( - "SELECT dataset_id FROM dataset" - ).fetchone()[0] - - assert openstatspec.cli.main([ - "install-in-place-schema", - "--database-url", database_url, - ]) == 0 - assert json.loads(capsys.readouterr().out) == {"status": "installed"} - - plan = openstatspec.compile_spss_syntax( - "RECODE answer (1 = 2).", - openstatspec.VariableSchema(( - openstatspec.VariableDefinition("answer", "numeric"), - )), - ).plan - plan_file = tmp_path / "plan.json" - plan_file.write_text( - json.dumps(plan.as_dict()), - encoding="utf-8", +def test_dolt_state_cli_emits_read_only_snapshot(monkeypatch, capsys) -> None: + monkeypatch.setattr( + openstatspec.cli, "dolt_state_snapshot", + lambda **_kwargs: { + "profile": "dolt", + "server_version": "2.2.2", + "read_only": True, + "state": {"database": "synthetic_catalog", "snapshot_sha256": "0" * 64}, + }, ) - assert openstatspec.cli.main([ - "apply-plan", - "--database-url", database_url, - "--dataset-id", live_dataset_id, - "--actor", "cli-test", - "--plan-file", str(plan_file), - ]) == 0 - generic = json.loads(capsys.readouterr().out) - assert generic["source_kind"] == "canonical_plan" assert openstatspec.cli.main([ - "apply-spss", - "--database-url", database_url, - "--dataset-id", live_dataset_id, - "--actor", "cli-test", - "--syntax", "RECODE answer (2 = 3).", + "dolt-state", + "--database-url", "mysql+pymysql://user@host/synthetic_catalog", ]) == 0 - spss = json.loads(capsys.readouterr().out) - assert spss["source_kind"] == "spss_syntax" - - table_name = connection.execute( - "SELECT physical_table_name FROM dataset" - ).fetchone()[0] - assert connection.execute( - f'SELECT answer FROM "{table_name}"' - ).fetchone() == (3.0,) + rendered = json.loads(capsys.readouterr().out) + assert rendered["profile"] == "dolt" + assert rendered["read_only"] is True + assert rendered["state"]["snapshot_sha256"] == "0" * 64 diff --git a/tests/test_document_round_trip.py b/tests/test_document_round_trip.py index 40dacf4..60f624e 100644 --- a/tests/test_document_round_trip.py +++ b/tests/test_document_round_trip.py @@ -1,4 +1,5 @@ from pathlib import Path +import re import shutil import sqlite3 import subprocess @@ -8,11 +9,14 @@ import pytest import openstatspec +from openstatspec.spss import raw_dictionary as raw_dictionary_module from openstatspec.spss.raw_dictionary import ( + RawDictionaryError, read_document_lines, write_compatible_names, write_document_lines, ) +from openstatspec.sql.wide import initialize_wide_catalog def _source_with_documents(path: Path, lines: list[str]) -> None: @@ -20,6 +24,84 @@ def _source_with_documents(path: Path, lines: list[str]) -> None: write_document_lines(path, lines, encoding="UTF-8") +def _reference_payloads(path: Path, subtype: int) -> tuple[bytes, ...]: + data = path.read_bytes() + byte_order, records = raw_dictionary_module._records(data) # pylint: disable=protected-access + payloads = [] + for record in records: + if record.record_type != 7: + continue + observed_subtype = raw_dictionary_module._int( # pylint: disable=protected-access + data, record.start + 4, byte_order, + ) + if observed_subtype != subtype: + continue + assert raw_dictionary_module._int( # pylint: disable=protected-access + data, record.start + 8, byte_order, + ) == 1 + payloads.append(data[record.start + 16 : record.end]) + return tuple(payloads) + + +def _replace_expected_member_tokens( + payloads: tuple[bytes, ...], replacements: dict[str, str], +) -> tuple[bytes, ...]: + updated = [] + for payload in payloads: + for old_name, new_name in replacements.items(): + pattern = rb"(?i)(? None: + data = bytearray(path.read_bytes()) + byte_order, records = raw_dictionary_module._records(data) # pylint: disable=protected-access + matches = [ + record for record in records + if record.record_type == 7 + and raw_dictionary_module._int( # pylint: disable=protected-access + data, record.start + 4, byte_order, + ) == old_subtype + ] + assert len(matches) == 1 + record = matches[0] + data[record.start + 4 : record.start + 8] = raw_dictionary_module._pack( # pylint: disable=protected-access + new_subtype, byte_order, + ) + path.write_bytes(data) + + +def _replace_reference_payload_bytes( + path: Path, subtype: int, old: bytes, new: bytes, +) -> None: + assert len(old) == len(new) + data = bytearray(path.read_bytes()) + byte_order, records = raw_dictionary_module._records(data) # pylint: disable=protected-access + matches = [ + record for record in records + if record.record_type == 7 + and raw_dictionary_module._int( # pylint: disable=protected-access + data, record.start + 4, byte_order, + ) == subtype + ] + assert len(matches) == 1 + record = matches[0] + payload = bytes(data[record.start + 16 : record.end]) + matches = list(re.finditer(re.escape(old), payload, flags=re.IGNORECASE)) + assert len(matches) == 1 + match = matches[0] + data[record.start + 16 : record.end] = ( + payload[:match.start()] + new + payload[match.end():] + ) + path.write_bytes(data) + + @pytest.mark.parametrize("destination_suffix", [".sav", ".zsav"]) def test_document_lines_round_trip_through_sqlite(destination_suffix: str, tmp_path: Path) -> None: source = tmp_path / "source.sav" @@ -28,6 +110,7 @@ def test_document_lines_round_trip_through_sqlite(destination_suffix: str, tmp_p expected = ["Imported from a validated source.", "A second document line."] _source_with_documents(source, expected) + initialize_wide_catalog(database_url=database) imported = openstatspec.import_sav( source, database_url=database, dataset_id=f"documents-{destination_suffix[1:]}", ) @@ -64,6 +147,7 @@ def test_document_lines_import_from_zsav_and_export_to_sav(tmp_path: Path) -> No expected = ["ZSAV document line."] _zsav_with_documents(source, expected, tmp_path / "temporary.sav") + initialize_wide_catalog(database_url=database) imported = openstatspec.import_sav(source, database_url=database, dataset_id="documents-zsav") assert imported.diagnostics == () openstatspec.export_sav( @@ -82,6 +166,7 @@ def test_document_and_compatible_name_round_trip_to_zsav(tmp_path: Path) -> None write_document_lines(source, ["Combined dictionary fixture."], encoding="UTF-8") write_compatible_names(source, {source_name: "ANSWER"}, encoding="UTF-8") + initialize_wide_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="combined") openstatspec.export_sav(database_url=database, dataset_id="combined", destination=destination) @@ -90,6 +175,476 @@ def test_document_and_compatible_name_round_trip_to_zsav(tmp_path: Path) -> None assert pyspssio.read_sav(str(destination))[0][source_name].tolist() == [7.0] +@pytest.mark.parametrize("suffix", [".sav", ".zsav"]) +def test_vls_long_name_and_custom_compatible_name_round_trip( + tmp_path: Path, suffix: str, +) -> None: + source = tmp_path / f"vls-source{suffix}" + destination = tmp_path / f"vls-destination{suffix}" + database = f"sqlite:///{tmp_path / f'vls-{suffix[1:]}.sqlite'}" + source_name = "long_text_response_variable" + compatible_name = "TXTRESP" + payload = "Õ🙂漢字" * 90 + pyspssio.write_sav(str(source), pd.DataFrame({source_name: [payload, "short"]})) + write_compatible_names(source, {source_name: compatible_name}, encoding="UTF-8") + + source_metadata = pyspssio.read_metadata(str(source)) + assert source_metadata["var_types"][source_name] == len(payload.encode("utf-8")) + assert source_metadata["var_compat_names"][source_name] == compatible_name + + initialize_wide_catalog(database_url=database) + openstatspec.import_sav( + source, database_url=database, dataset_id=f"vls-{suffix[1:]}", + ) + openstatspec.export_sav( + database_url=database, dataset_id=f"vls-{suffix[1:]}", destination=destination, + ) + + frame, metadata = pyspssio.read_sav(str(destination), convert_datetimes=False) + assert list(frame.columns) == [source_name] + assert frame[source_name].tolist() == [payload, "short"] + assert metadata["var_types"][source_name] == len(payload.encode("utf-8")) + assert metadata["var_compat_names"][source_name] == compatible_name + + +def test_two_vls_requested_compatible_names_must_be_case_insensitively_unique( + tmp_path: Path, +) -> None: + source = tmp_path / "duplicate-requested-vls.sav" + source_names = ["first_long_text_variable", "second_long_text_variable"] + pyspssio.write_sav( + str(source), + pd.DataFrame({source_name: ["x" * 300] for source_name in source_names}), + ) + original = source.read_bytes() + + with pytest.raises(RawDictionaryError, match="Duplicate compatible variable"): + write_compatible_names( + source, + {source_names[0]: "Shared", source_names[1]: "sHARED"}, + encoding="UTF-8", + ) + + assert source.read_bytes() == original + + +def test_compatible_name_rewrite_rejects_non_unique_type_2_name_without_writing( + tmp_path: Path, +) -> None: + source = tmp_path / "duplicate-compatible-name.sav" + source_name = "long_numeric_response_variable" + pyspssio.write_sav( + str(source), pd.DataFrame({source_name: [1.0], "taken": [2.0]}), + ) + original = source.read_bytes() + + with pytest.raises(RawDictionaryError, match="Duplicate compatible variable"): + write_compatible_names(source, {source_name: "TAKEN"}, encoding="UTF-8") + + assert source.read_bytes() == original + + +def test_compatible_name_rewrite_rejects_malformed_subtype_14_without_writing( + tmp_path: Path, +) -> None: + source = tmp_path / "malformed-vls.sav" + source_name = "long_text_response_variable" + pyspssio.write_sav(str(source), pd.DataFrame({source_name: ["x" * 300]})) + metadata = pyspssio.read_metadata(str(source)) + short_name = metadata["var_compat_names"][source_name].encode("ascii") + valid_entry = short_name + b"=300\x00 " + malformed_entry = short_name + b"=300X " + data = source.read_bytes() + assert data.count(valid_entry) == 1 + source.write_bytes(data.replace(valid_entry, malformed_entry)) + corrupted = source.read_bytes() + + with pytest.raises(RawDictionaryError, match="entry terminator"): + write_compatible_names(source, {source_name: "TEXTRESP"}, encoding="UTF-8") + + assert source.read_bytes() == corrupted + + +def test_compatible_name_rewrite_rejects_duplicate_subtype_14_names( + tmp_path: Path, +) -> None: + source = tmp_path / "duplicate-vls.sav" + source_names = ["first_long_text_variable", "second_long_text_variable"] + pyspssio.write_sav( + str(source), + pd.DataFrame({source_name: ["x" * 300] for source_name in source_names}), + ) + metadata = pyspssio.read_metadata(str(source)) + first_short = metadata["var_compat_names"][source_names[0]].encode("ascii") + second_short = metadata["var_compat_names"][source_names[1]].encode("ascii") + data = source.read_bytes() + second_entry = second_short + b"=300\x00 " + duplicate_entry = first_short + b"=300\x00 " + assert len(second_entry) == len(duplicate_entry) + assert data.count(second_entry) == 1 + source.write_bytes(data.replace(second_entry, duplicate_entry)) + corrupted = source.read_bytes() + + with pytest.raises(RawDictionaryError, match="Duplicate subtype-14"): + write_compatible_names( + source, {source_names[0]: "FIRSTTXT"}, encoding="UTF-8", + ) + + assert source.read_bytes() == corrupted + + +def test_compatible_name_rewrite_updates_subtype_5_variable_set_members( + tmp_path: Path, +) -> None: + source = tmp_path / "variable-set-reference.sav" + source_names = ["long_analysis_variable", "second_analysis_variable"] + with pyspssio.Writer(str(source), mode="w") as writer: + for name in source_names: + writer._add_var(name, 0) # pylint: disable=protected-access + writer.var_sets = {"Analysis": source_names} + writer.commit_header() + writer.write_data(pd.DataFrame({name: [1.0] for name in source_names})) + before = pyspssio.read_metadata(str(source)) + replacements = { + before["var_compat_names"][source_names[0]]: "ANALYZE", + before["var_compat_names"][source_names[1]]: "SECOND", + } + before_payloads = _reference_payloads(source, 5) + + write_compatible_names( + source, + {source_names[0]: "ANALYZE", source_names[1]: "SECOND"}, + encoding="UTF-8", + ) + + with pyspssio.Reader(str(source), mode="r") as reader: + assert reader.var_compat_names[source_names[0]] == "ANALYZE" + assert reader.var_compat_names[source_names[1]] == "SECOND" + assert reader.var_sets == {"Analysis": source_names} + assert _reference_payloads(source, 5) == _replace_expected_member_tokens( + before_payloads, replacements, + ) + + +def test_compatible_name_rewrite_updates_subtype_7_c_and_d_members( + tmp_path: Path, +) -> None: + source = tmp_path / "standard-mrsets.sav" + source_names = [ + "long_category_first", "long_category_second", + "long_dichotomy_first", "long_dichotomy_second", + ] + frame = pd.DataFrame({name: [1.0, 0.0] for name in source_names}) + expected_mrsets = { + "$categories": { + "label": "Category choices", + "variable_list": source_names[:2], + }, + "$dichotomies": { + "label": "Dichotomy choices", + "counted_value": 1, + "variable_list": source_names[2:], + }, + } + pyspssio.write_sav(str(source), frame, metadata={"mrsets": expected_mrsets}) + before = pyspssio.read_metadata(str(source)) + requested = { + source_names[0]: "CATONE", + source_names[2]: "DICHONE", + } + replacements = { + before["var_compat_names"][name]: compatible + for name, compatible in requested.items() + } + before_payloads = _reference_payloads(source, 7) + assert before_payloads + + write_compatible_names(source, requested, encoding="UTF-8") + + after = pyspssio.read_metadata(str(source)) + assert after["mrsets"] == before["mrsets"] + assert {name: after["var_compat_names"][name] for name in requested} == requested + assert _reference_payloads(source, 7) == _replace_expected_member_tokens( + before_payloads, replacements, + ) + + +def test_compatible_name_rewrite_updates_subtype_19_e_members( + tmp_path: Path, +) -> None: + source = tmp_path / "extended-mrset.sav" + source_names = ["long_text_first", "long_text_second"] + frame = pd.DataFrame({ + source_names[0]: ["yes", "no"], + source_names[1]: ["no", "yes"], + }) + pyspssio.write_sav( + str(source), frame, + metadata={ + "var_types": {name: 8 for name in source_names}, + "mrsets": { + "$contact_text": { + "label": "Text contact choices", + "is_dichotomy": True, + "counted_value": "yes", + "use_category_labels": True, + "use_first_var_label": False, + "variable_list": source_names, + }, + }, + }, + ) + before = pyspssio.read_metadata(str(source)) + requested = {source_names[0]: "TEXTONE", source_names[1]: "TEXTTWO"} + replacements = { + before["var_compat_names"][name]: compatible + for name, compatible in requested.items() + } + before_payloads = _reference_payloads(source, 19) + assert before_payloads + + write_compatible_names(source, requested, encoding="UTF-8") + + after = pyspssio.read_metadata(str(source)) + assert after["mrsets"] == before["mrsets"] + assert {name: after["var_compat_names"][name] for name in requested} == requested + assert _reference_payloads(source, 19) == _replace_expected_member_tokens( + before_payloads, replacements, + ) + + +@pytest.mark.parametrize( + ("is_dichotomy", "set_name", "new_compatible"), + [ + (False, "$category19", "CAT19"), + (True, "$dichotomy19", "DICH19"), + ], +) +def test_compatible_name_rewrite_updates_subtype_19_c_and_d_members( + tmp_path: Path, is_dichotomy: bool, set_name: str, new_compatible: str, +) -> None: + source = tmp_path / f"subtype-19-{'d' if is_dichotomy else 'c'}.sav" + source_names = ["long_member_first", "long_member_second"] + definition = { + "label": "Subtype 19 definition", + "variable_list": source_names, + } + if is_dichotomy: + definition["counted_value"] = 1 + pyspssio.write_sav( + str(source), + pd.DataFrame({name: [1.0, 0.0] for name in source_names}), + metadata={"mrsets": {set_name: definition}}, + ) + _change_reference_subtype(source, 7, 19) + before = pyspssio.read_metadata(str(source)) + old_compatible = before["var_compat_names"][source_names[0]] + before_payloads = _reference_payloads(source, 19) + assert before_payloads + + write_compatible_names( + source, {source_names[0]: new_compatible}, encoding="UTF-8", + ) + + after = pyspssio.read_metadata(str(source)) + assert after["mrsets"] == before["mrsets"] + assert after["var_compat_names"][source_names[0]] == new_compatible + assert _reference_payloads(source, 19) == _replace_expected_member_tokens( + before_payloads, {old_compatible: new_compatible}, + ) + + +def test_duplicate_subtype_19_set_name_fails_without_source_change( + tmp_path: Path, +) -> None: + source = tmp_path / "duplicate-subtype-19-set.sav" + source_names = ["long_member_first", "long_member_second"] + pyspssio.write_sav( + str(source), + pd.DataFrame({name: [1.0] for name in source_names}), + metadata={ + "mrsets": { + "$first": {"label": "First", "variable_list": source_names}, + "$secon": {"label": "Second", "variable_list": source_names}, + }, + }, + ) + _change_reference_subtype(source, 7, 19) + _replace_reference_payload_bytes(source, 19, b"$secon=", b"$first=") + original = source.read_bytes() + + with pytest.raises(RawDictionaryError, match="Duplicate subtype-19 set name"): + write_compatible_names( + source, {source_names[0]: "MEMBER1"}, encoding="UTF-8", + ) + + assert source.read_bytes() == original + + +def test_unknown_subtype_19_member_fails_without_source_change( + tmp_path: Path, +) -> None: + source = tmp_path / "unknown-subtype-19-member.sav" + source_names = ["long_member_first", "long_member_second"] + pyspssio.write_sav( + str(source), + pd.DataFrame({name: [1.0] for name in source_names}), + metadata={ + "mrsets": { + "$members": {"label": "Members", "variable_list": source_names}, + }, + }, + ) + _change_reference_subtype(source, 7, 19) + metadata = pyspssio.read_metadata(str(source)) + old_member = metadata["var_compat_names"][source_names[0]] + known_names = {value.casefold() for value in metadata["var_compat_names"].values()} + unknown_member = "U" * len(old_member) + if unknown_member.casefold() in known_names: + unknown_member = "Z" * len(old_member) + assert unknown_member.casefold() not in known_names + _replace_reference_payload_bytes( + source, 19, old_member.encode("ascii"), unknown_member.encode("ascii"), + ) + original = source.read_bytes() + + with pytest.raises(RawDictionaryError, match="references unknown variable"): + write_compatible_names( + source, {source_names[1]: "MEMBER2"}, encoding="UTF-8", + ) + + assert source.read_bytes() == original + + +def test_combined_zsav_reference_rewrite_preserves_offsets_and_semantics( + tmp_path: Path, +) -> None: + source = tmp_path / "combined-references.zsav" + numeric_names = [ + "long_category_first", "long_category_second", + "long_dichotomy_first", "long_dichotomy_second", + ] + text_names = ["long_text_first", "long_text_second"] + vls_name = "long_very_long_text_variable" + all_names = numeric_names + text_names + [vls_name] + with pyspssio.Writer(str(source), mode="w") as writer: + writer.compression = 2 + for name in numeric_names: + writer._add_var(name, 0) # pylint: disable=protected-access + for name in text_names: + writer._add_var(name, 8) # pylint: disable=protected-access + writer._add_var(vls_name, 300) # pylint: disable=protected-access + writer.var_sets = {"Analysis": all_names} + writer.mrsets = { + "$categories": {"label": "Categories", "variable_list": numeric_names[:2]}, + "$dichotomies": { + "label": "Dichotomies", "counted_value": 1, + "variable_list": numeric_names[2:], + }, + "$extended": { + "label": "Extended", "is_dichotomy": True, + "counted_value": "yes", "use_category_labels": True, + "use_first_var_label": False, "variable_list": text_names, + }, + } + writer.commit_header() + writer.write_data(pd.DataFrame({ + **{name: [1.0, 0.0] for name in numeric_names}, + text_names[0]: ["yes", "no"], + text_names[1]: ["no", "yes"], + vls_name: ["x" * 300, "short"], + })) + before = pyspssio.read_metadata(str(source)) + with pyspssio.Reader(str(source), mode="r") as reader: + before_variable_sets = reader.var_sets + requested = { + numeric_names[0]: "CAT", + numeric_names[2]: "DICH", + text_names[0]: "TEXT", + vls_name: "VLS", + } + + write_compatible_names(source, requested, encoding="UTF-8") + + frame, after = pyspssio.read_sav(str(source), convert_datetimes=False) + assert list(frame.columns) == all_names + assert frame[vls_name].tolist() == ["x" * 300, "short"] + assert after["mrsets"] == before["mrsets"] + with pyspssio.Reader(str(source), mode="r") as reader: + assert reader.var_sets == before_variable_sets + assert {name: after["var_compat_names"][name] for name in requested} == requested + assert _reference_payloads(source, 5) + assert _reference_payloads(source, 7) + assert _reference_payloads(source, 19) + + +def test_reader_readback_failure_preserves_source_and_removes_private_candidate( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "readback-failure.sav" + source_name = "long_analysis_variable" + with pyspssio.Writer(str(source), mode="w") as writer: + writer._add_var(source_name, 0) # pylint: disable=protected-access + writer.var_sets = {"Analysis": [source_name]} + writer.commit_header() + writer.write_data(pd.DataFrame({source_name: [1.0]})) + original = source.read_bytes() + real_reader = raw_dictionary_module._read_dictionary_semantics # pylint: disable=protected-access + + def fail_candidate_readback(path): + if Path(path) != source: + assert Path(path).stat().st_mode & 0o777 == 0o600 + raise RawDictionaryError("synthetic Reader readback failure") + return real_reader(path) + + monkeypatch.setattr( + raw_dictionary_module, "_read_dictionary_semantics", fail_candidate_readback, + ) + with pytest.raises(RawDictionaryError, match="synthetic Reader readback failure"): + write_compatible_names(source, {source_name: "ANALYZE"}, encoding="UTF-8") + + assert source.read_bytes() == original + assert list(tmp_path.glob(f".{source.stem}.*.tmp{source.suffix}")) == [] + + +def test_final_shifted_bytes_are_validated_before_publish( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "invalid-shift-result.zsav" + source_name = "long_text_response_variable" + pyspssio.write_sav(str(source), pd.DataFrame({source_name: ["x" * 300]})) + original = source.read_bytes() + + monkeypatch.setattr( + raw_dictionary_module, "_shift_zsav_offsets", + lambda *_args, **_kwargs: b"invalid shifted bytes", + ) + with pytest.raises(RawDictionaryError, match="supported SAV or ZSAV"): + write_compatible_names(source, {source_name: "TXT"}, encoding="UTF-8") + + assert source.read_bytes() == original + + +def test_atomic_publish_failure_preserves_source_and_removes_temporary_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "partial-publish.sav" + source_name = "long_text_response_variable" + pyspssio.write_sav(str(source), pd.DataFrame({source_name: ["x" * 300]})) + original = source.read_bytes() + + def partial_write_then_fail(stream, data): + stream.write(data[:32]) + raise OSError("synthetic partial temporary write") + + monkeypatch.setattr(raw_dictionary_module, "_write_all", partial_write_then_fail) + with pytest.raises(OSError, match="synthetic partial temporary write"): + write_compatible_names(source, {source_name: "TXT"}, encoding="UTF-8") + + assert source.read_bytes() == original + assert list(tmp_path.glob(f".{source.stem}.*.tmp{source.suffix}")) == [] + + def test_windows_1252_values_and_documents_round_trip_when_locale_is_available( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -117,6 +672,7 @@ def test_windows_1252_values_and_documents_round_trip_when_locale_is_available( ) write_document_lines(source, documents, encoding="CP1252") + initialize_wide_catalog(database_url=database) imported = openstatspec.import_sav(source, database_url=database, dataset_id="cp1252") assert {item.code for item in imported.diagnostics} == {"source-encoding-not-preserved"} exported = openstatspec.export_sav( @@ -127,3 +683,76 @@ def test_windows_1252_values_and_documents_round_trip_when_locale_is_available( assert pyspssio.read_metadata(str(destination))["encoding"].casefold() == "windows-1252" assert pyspssio.read_sav(str(destination))[0]["name"].tolist() == [value] assert read_document_lines(destination, encoding="CP1252") == documents + + +def test_legacy_source_unicode_export_uses_actual_output_encoding_for_raw_rewrites( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + localedef = shutil.which("localedef") + charmap = Path("/usr/share/i18n/charmaps/CP1252.gz") + source_definition = Path("/usr/share/i18n/locales/en_US") + if localedef is None or not charmap.exists() or not source_definition.exists(): + pytest.skip("CP1252 locale source is unavailable on this host") + locale_root = tmp_path / "unicode-export-locales" + locale_root.mkdir() + locale_name = "en_US.CP1252" + subprocess.run( + [localedef, "--no-archive", "-i", "en_US", "-f", "CP1252", str(locale_root / locale_name)], + check=True, + ) + monkeypatch.setenv("LOCPATH", str(locale_root)) + source = tmp_path / "legacy-source.sav" + destination = tmp_path / "unicode-output.sav" + database = f"sqlite:///{tmp_path / 'legacy-to-unicode.sqlite'}" + source_names = ["kusimuse_pikk_nimi", "teine_pikk_nimi"] + mr_label = "Mitmikvalik – küsimus" + frame = pd.DataFrame({source_names[0]: [1.0, 0.0], source_names[1]: [0.0, 1.0]}) + pyspssio.write_sav( + str(source), frame, unicode=False, locale=locale_name, + metadata={ + "var_labels": {source_names[0]: "Esimene vastus"}, + "mrsets": { + "$vastused": { + "label": mr_label, + "counted_value": 1, + "use_category_labels": True, + "use_first_var_label": True, + "variable_list": source_names, + }, + }, + }, + ) + source_encoding = str(pyspssio.read_metadata(str(source))["encoding"]) + raw_dictionary_module.write_extended_mrset_labels( + source, {"$vastused": mr_label}, encoding=source_encoding, + ) + write_compatible_names( + source, {source_names[0]: "CUSTOM"}, encoding=source_encoding, + ) + + initialize_wide_catalog(database_url=database) + imported = openstatspec.import_sav( + source, database_url=database, dataset_id="legacy-to-unicode", + ) + assert {diagnostic.code for diagnostic in imported.diagnostics} == { + "source-encoding-not-preserved", + } + exported = openstatspec.export_sav( + database_url=database, + dataset_id="legacy-to-unicode", + destination=destination, + allow_loss=["source-encoding-not-preserved"], + ) + + assert {diagnostic.code for diagnostic in exported.diagnostics} == { + "source-encoding-not-preserved", + } + output_frame, output_metadata = pyspssio.read_sav( + str(destination), convert_datetimes=False, + ) + assert output_metadata["encoding"].casefold() in {"utf-8", "utf8"} + assert output_metadata["var_names"] == source_names + assert output_metadata["var_compat_names"][source_names[0]] == "CUSTOM" + assert output_metadata["mrsets"]["$vastused"]["label"] == mr_label + assert output_metadata["mrsets"]["$vastused"]["variable_list"] == source_names + assert output_frame[source_names[0]].tolist() == [1.0, 0.0] diff --git a/tests/test_dolt_conformance.py b/tests/test_dolt_conformance.py new file mode 100644 index 0000000..77a9062 --- /dev/null +++ b/tests/test_dolt_conformance.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +import inspect +import tomllib +from pathlib import Path + +import pandas as pd +import pytest + +import openstatspec.api as api_module +import openstatspec.spss as spss_module +import openstatspec.spss.sav as sav_module +from openstatspec.core import UnsupportedOperationError +import openstatspec.sql.capabilities as capability_module +import openstatspec.sql.wide as wide +from openstatspec.sql.dolt_conformance import ( + ADAPTER_IMPLEMENTATION_ID, + ADAPTER_VERSION, + DoltConformanceSource, +) + + +def test_packaged_source_is_fail_closed_without_concrete_declarations() -> None: + status = DoltConformanceSource.packaged().status() + assert status["write_enabled"] is False + assert status["declaration_count"] == 0 + + +def test_directory_source_is_explicit_and_invalid_root_fails_closed( + tmp_path: Path, +) -> None: + source = DoltConformanceSource.from_directory(tmp_path) + assert source.directory == tmp_path + status = source.status() + assert status["write_enabled"] is False + assert status["status"] == "blocked_invalid_or_unavailable_declaration_source" + + +def test_packaged_companion_missing_is_reported_without_opening_write_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def missing_companion() -> tuple[object, object, object, object]: + raise UnsupportedOperationError("companion missing") + + monkeypatch.setattr( + "openstatspec.sql.dolt_conformance._shared_api", missing_companion, + ) + declaration = capability_module.profile_declarations()["dolt"] + assert declaration["operational_write_enabled"] is False + assert declaration["write_conformance"]["write_enabled"] is False + assert declaration["claimed_server_versions"] == [] + assert declaration["ci_tested_server_versions"] == [] + + +def test_exact_match_binds_active_product_adapter_and_specification( + monkeypatch: pytest.MonkeyPatch, +) -> None: + matching = { + "declaration_id": "dolt-2.2.2-python-0.1.0", + "active_product_version": "2.2.2", + "adapter_implementation_id": ADAPTER_IMPLEMENTATION_ID, + "adapter_version": ADAPTER_VERSION, + "specification_commit": "a" * 40, + } + monkeypatch.setattr( + DoltConformanceSource, + "validated_declarations", + lambda self: (matching,), + ) + selected = DoltConformanceSource.packaged().require_exact_match( + active_product_version="2.2.2", + specification_commit="a" * 40, + ) + assert selected["declaration_id"] == matching["declaration_id"] + + +def test_exact_match_rejects_multiple_adapter_bindings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = { + "active_product_version": "2.2.2", + "adapter_implementation_id": ADAPTER_IMPLEMENTATION_ID, + "adapter_version": ADAPTER_VERSION, + "specification_commit": "a" * 40, + } + monkeypatch.setattr( + DoltConformanceSource, + "validated_declarations", + lambda self: ( + {**binding, "declaration_id": "one"}, + {**binding, "declaration_id": "two"}, + ), + ) + with pytest.raises(UnsupportedOperationError, match="unique exact"): + DoltConformanceSource.packaged().require_exact_match( + active_product_version="2.2.2", + specification_commit="a" * 40, + ) + + +def test_effective_profile_requires_bound_spec_commit_before_mutation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + class UnexpectedSource: + def require_exact_match( + self, *, active_product_version: str, specification_commit: str, + ) -> None: + calls.append((active_product_version, specification_commit)) + + monkeypatch.setattr( + capability_module, + "active_connection", + lambda database_url, **kwargs: { + "profile": "dolt", + "raw_product_version": "2.2.2", + "server_version": "2.2.2", + "claimed_supported": False, + "observed": {"max_allowed_packet": 1_000_000}, + }, + ) + assert capability_module.SPECIFICATION_COMMIT is None + with pytest.raises(UnsupportedOperationError, match="not bound"): + capability_module.effective_profile( + "mysql+pymysql://example.invalid/catalog", + dolt_conformance_source=UnexpectedSource(), + ) + assert calls == [] + + +def test_every_sql_mutation_entrypoint_accepts_explicit_conformance_source() -> None: + mutation_entrypoints = ( + wide.initialize_wide_catalog, + wide.create_wide_dataset, + wide.record_export_cleanup_failure, + wide.record_export_operation, + wide.finish_export_operation, + wide.fail_export_operation, + wide.record_export_backup_retained, + ) + for entrypoint in mutation_entrypoints: + parameter = inspect.signature(entrypoint).parameters[ + "dolt_conformance_source" + ] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + assert parameter.default is None + + +def test_adapter_binding_version_matches_distribution_metadata() -> None: + pyproject = tomllib.loads( + (Path(__file__).resolve().parents[1] / "pyproject.toml").read_text( + encoding="utf-8" + ) + ) + assert pyproject["project"]["version"] == ADAPTER_VERSION + + +def test_validate_wide_dataset_propagates_explicit_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[object] = [] + + def stop_after_read_preflight(**kwargs: object) -> tuple[object, object, object]: + calls.append(kwargs["dolt_conformance_source"]) + raise UnsupportedOperationError("stop after propagation check") + + monkeypatch.setattr(wide, "read_wide_dataset", stop_after_read_preflight) + with pytest.raises(UnsupportedOperationError, match="propagation check"): + wide.validate_wide_dataset( + database_url="mysql+pymysql://example.invalid/catalog", + dataset_id="synthetic", + dolt_conformance_source=sentinel, + ) + assert calls == [sentinel] + + +def test_public_and_spss_dispatch_propagate_explicit_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[tuple[str, object]] = [] + + def capture_api_import(*args: object, **kwargs: object) -> dict[str, bool]: + calls.append(("api_import", kwargs["dolt_conformance_source"])) + return {"ok": True} + + def capture_api_export(**kwargs: object) -> dict[str, bool]: + calls.append(("api_export", kwargs["dolt_conformance_source"])) + return {"ok": True} + + monkeypatch.setattr(api_module, "import_dataset", capture_api_import) + monkeypatch.setattr(api_module, "export_dataset", capture_api_export) + monkeypatch.setattr(api_module, "result", lambda value: value) + api_module.import_sav( + "synthetic.sav", database_url="sqlite://", dataset_id="synthetic", + dolt_conformance_source=sentinel, + ) + api_module.export_sav( + database_url="sqlite://", dataset_id="synthetic", + destination="synthetic.sav", dolt_conformance_source=sentinel, + ) + + def capture_spss_import(**kwargs: object) -> dict[str, bool]: + calls.append(("spss_import", kwargs["dolt_conformance_source"])) + return {"ok": True} + + def capture_spss_export(**kwargs: object) -> dict[str, bool]: + calls.append(("spss_export", kwargs["dolt_conformance_source"])) + return {"ok": True} + + monkeypatch.setattr(spss_module, "import_sav_dataset", capture_spss_import) + monkeypatch.setattr(spss_module, "export_sav_dataset", capture_spss_export) + spss_module.import_dataset( + "synthetic.sav", database_url="sqlite://", dataset_id="synthetic", + dolt_conformance_source=sentinel, + ) + spss_module.export_dataset( + database_url="sqlite://", dataset_id="synthetic", + destination="synthetic.sav", dolt_conformance_source=sentinel, + ) + assert calls == [ + ("api_import", sentinel), + ("api_export", sentinel), + ("spss_import", sentinel), + ("spss_export", sentinel), + ] + + +def test_sav_import_propagates_explicit_source_to_mutation_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[object] = [] + monkeypatch.setattr(sav_module, "_require_source", lambda path: None) + monkeypatch.setattr( + sav_module.pyspssio, "read_sav", + lambda *args, **kwargs: (pd.DataFrame(), {}), + ) + monkeypatch.setattr(sav_module, "_dictionary", lambda path: ({}, {})) + monkeypatch.setattr(sav_module, "_sha256", lambda path: "0" * 64) + + def capture_create(**kwargs: object) -> dict[str, str]: + calls.append(kwargs["dolt_conformance_source"]) + return {"dataset_id": "synthetic"} + + monkeypatch.setattr(sav_module, "create_wide_dataset", capture_create) + sav_module.import_sav_dataset( + source="synthetic.sav", database_url="sqlite://", + dataset_id="synthetic", dolt_conformance_source=sentinel, + ) + assert calls == [sentinel] + + +def test_sav_export_propagates_explicit_source_to_read_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[object] = [] + + def stop_after_read(**kwargs: object) -> tuple[object, object, object]: + calls.append(kwargs["dolt_conformance_source"]) + raise UnsupportedOperationError("stop after SAV read propagation") + + monkeypatch.setattr(sav_module, "read_wide_dataset", stop_after_read) + with pytest.raises(UnsupportedOperationError, match="SAV read propagation"): + sav_module.export_sav_dataset( + database_url="sqlite://", dataset_id="synthetic", + destination="synthetic.sav", + dolt_conformance_source=sentinel, + ) + assert calls == [sentinel] + + +def test_export_recovery_helpers_propagate_explicit_source( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + sentinel = object() + calls: list[tuple[str, object]] = [] + + def capture_cleanup(**kwargs: object) -> str: + calls.append(("cleanup", kwargs["dolt_conformance_source"])) + return "cleanup-operation" + + def capture_failure(**kwargs: object) -> None: + calls.append(("failure", kwargs["dolt_conformance_source"])) + + monkeypatch.setattr( + sav_module, "record_export_cleanup_failure", capture_cleanup, + ) + monkeypatch.setattr(sav_module, "fail_export_operation", capture_failure) + destination = tmp_path / "destination.sav" + backup = tmp_path / "backup.sav" + staged = tmp_path / "staged.sav" + with pytest.raises(sav_module.ExportRecoveryError, match="cleanup_failed"): + sav_module._raise_export_cleanup_failed( + original_error=RuntimeError("synthetic export failure"), + cleanup_error=RuntimeError("synthetic cleanup failure"), + phase="synthetic", destination=destination, backup=backup, + staged=staged, had_previous=False, database_url="sqlite://", + dolt_conformance_source=sentinel, + ) + sav_module._mark_export_failed_after_restore( + database_url="sqlite://", operation_id="synthetic-operation", + error=RuntimeError("synthetic export failure"), phase="synthetic", + destination=destination, backup=backup, had_previous=False, + dolt_conformance_source=sentinel, + ) + assert calls == [("cleanup", sentinel), ("failure", sentinel)] + diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 3c4cda4..a7ebb76 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -61,6 +61,7 @@ def _submission(source_text: str) -> InPlacePlanSubmission: def catalog(tmp_path): path = tmp_path / "in-place.sqlite" url = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=url) create_wide_dataset( database_url=url, dataset_id="in_place_source", @@ -297,6 +298,7 @@ def test_string_source_can_create_numeric_target(tmp_path) -> None: "print_format": "[1, 8, 0]", "write_format": "[1, 8, 0]", }) + openstatspec.initialize_catalog(database_url=url) create_wide_dataset( database_url=url, dataset_id="string_source", diff --git a/tests/test_loss_reports.py b/tests/test_loss_reports.py index f1916f7..98dce61 100755 --- a/tests/test_loss_reports.py +++ b/tests/test_loss_reports.py @@ -1,5 +1,6 @@ import json import sqlite3 +from pathlib import Path import pandas as pd import pyspssio @@ -7,8 +8,8 @@ import openstatspec from openstatspec.core import UnsupportedOperationError -from openstatspec.sql.wide import create_wide_dataset -from openstatspec.spss.raw_dictionary import write_compatible_names +from openstatspec.sql.wide import create_wide_dataset, initialize_wide_catalog +from openstatspec.spss.raw_dictionary import RawDictionaryError, write_compatible_names from openstatspec.spss import sav as sav_module @@ -23,6 +24,7 @@ def test_persisted_import_fidelity_events_require_consent_after_reopen(tmp_path) approved = tmp_path / "approved.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) imported = openstatspec.import_sav(source, database_url=database, dataset_id="persisted") assert {diagnostic.code for diagnostic in imported.diagnostics} == set(_REQUIRED_ENGINE_LOSS) connection = sqlite3.connect(database_path) @@ -47,6 +49,7 @@ def test_loss_allowed_export_persists_accepted_diagnostics(tmp_path) -> None: source = tmp_path / "source.sav" destination = tmp_path / "accepted.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="accepted") result = openstatspec.export_sav(database_url=database, dataset_id="accepted", destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS) @@ -60,6 +63,7 @@ def test_non_utf8_source_encoding_is_explicit_export_loss(tmp_path) -> None: database_path = tmp_path / "legacy-encoding.sqlite" database = f"sqlite:///{database_path}" destination = tmp_path / "legacy-encoding.sav" + initialize_wide_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="legacy-encoding", source_name="legacy.sav", source_format="SAV", source_encoding="WINDOWS-1252", rows=[{"name": "Muller"}], @@ -82,6 +86,7 @@ def test_explicit_legacy_locale_selects_the_single_engine_route(tmp_path, monkey database_path = tmp_path / "legacy-locale.sqlite" database = f"sqlite:///{database_path}" destination = tmp_path / "legacy-locale.sav" + initialize_wide_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="legacy-locale", source_name="legacy.sav", source_format="SAV", source_encoding="WINDOWS-1252", rows=[{"name": "Muller"}], @@ -120,6 +125,43 @@ def test_legacy_locale_must_emit_the_exact_source_encoding() -> None: with pytest.raises(UnsupportedOperationError, match="instead of required"): sav_module._require_matching_legacy_encoding("WINDOWS-1252", "UTF-8", True) + +def test_dictionary_failures_use_path_free_error_identities( + tmp_path, monkeypatch, +) -> None: + source = tmp_path / "confidential-source.sav" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + + document_secret = f"classified document failure at {tmp_path}" + monkeypatch.setattr( + sav_module, + "read_document_lines", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RawDictionaryError(document_secret) + ), + ) + document_metadata, document_loss = sav_module._dictionary(source) + document_error = document_metadata["_documents_error"] + assert set(document_error) == {"type", "code", "phase", "message_sha256"} + assert document_error["phase"] == "read_sav_documents" + assert document_loss["documents-unreadable"]["details"]["engine_error"] == document_error + assert document_secret not in repr((document_metadata, document_loss)) + + variable_set_secret = f"classified variable-set failure at {tmp_path}" + monkeypatch.setattr( + sav_module, + "format_tuples", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError(variable_set_secret) + ), + ) + variable_metadata, variable_loss = sav_module._dictionary(source) + variable_error = variable_metadata["_var_sets_error"] + assert set(variable_error) == {"type", "code", "phase", "message_sha256"} + assert variable_error["phase"] == "read_sav_variable_sets" + assert variable_loss["variable-sets-unobservable"]["details"]["engine_error"] == variable_error + assert variable_set_secret not in repr((variable_metadata, variable_loss)) + @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) def test_compatible_variable_name_round_trips_from_current_sql_catalog(tmp_path, suffix: str) -> None: source = tmp_path / f"compat-source{suffix}" @@ -132,6 +174,7 @@ def test_compatible_variable_name_round_trips_from_current_sql_catalog(tmp_path, write_compatible_names(source, {source_name: "ANSWER"}, encoding="UTF-8") assert pyspssio.read_metadata(str(source))["var_compat_names"][source_name] == "ANSWER" + initialize_wide_catalog(database_url=database) imported = openstatspec.import_sav(source, database_url=database, dataset_id=f"compat-{suffix[1:]}") assert imported.diagnostics == () connection = sqlite3.connect(database_path) @@ -154,3 +197,308 @@ def test_compatible_variable_name_round_trips_from_current_sql_catalog(tmp_path, metadata = pyspssio.read_metadata(str(destination)) assert metadata["var_compat_names"][source_name] == "EXAMPLE" assert pyspssio.read_sav(str(destination))[0][source_name].tolist() == [1.0] + + +def test_dictionary_rewrite_failure_preserves_existing_destination_and_removes_stage( + tmp_path, monkeypatch, +) -> None: + source = tmp_path / "source.sav" + destination = tmp_path / "partial.sav" + database = f"sqlite:///{tmp_path / 'partial.sqlite'}" + source_name = "long_variable_name" + pyspssio.write_sav(str(source), pd.DataFrame({source_name: [1.0]})) + write_compatible_names(source, {source_name: "ANSWER"}, encoding="UTF-8") + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="partial") + original_destination = b"existing destination" + destination.write_bytes(original_destination) + + def fail_dictionary_rewrite(*_args, **_kwargs): + raise RawDictionaryError("synthetic dictionary rewrite failure") + + monkeypatch.setattr(sav_module, "write_compatible_names", fail_dictionary_rewrite) + with pytest.raises(RawDictionaryError, match="synthetic dictionary rewrite failure"): + openstatspec.export_sav( + database_url=database, dataset_id="partial", destination=destination, + ) + + assert destination.read_bytes() == original_destination + assert list(tmp_path.glob(f".{destination.name}.*")) == [] + + +def test_export_audit_failure_restores_previous_destination(tmp_path, monkeypatch) -> None: + source = tmp_path / "audit-source.sav" + destination = tmp_path / "audit-destination.sav" + database = f"sqlite:///{tmp_path / 'audit.sqlite'}" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="audit") + previous = b"previous destination bytes" + destination.write_bytes(previous) + + def fail_audit(*_args, **_kwargs): + raise RuntimeError("synthetic export audit failure") + + monkeypatch.setattr(sav_module, "record_export_operation", fail_audit) + with pytest.raises(RuntimeError, match="synthetic export audit failure"): + openstatspec.export_sav( + database_url=database, dataset_id="audit", destination=destination, + ) + + assert destination.read_bytes() == previous + assert list(tmp_path.glob(f".{destination.name}.*")) == [] + + +def test_export_restore_failure_persists_null_dataset_cleanup_audit( + tmp_path, monkeypatch, +) -> None: + source = tmp_path / "restore-source.sav" + destination = tmp_path / "restore-destination.sav" + database_path = tmp_path / "restore.sqlite" + database = f"sqlite:///{database_path}" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="restore") + destination.write_bytes(b"previous destination bytes") + + original_replace = sav_module.os.replace + + def fail_publish(source, target): + if ".staging." in str(Path(source).parent): + raise RuntimeError("synthetic publish failure") + return original_replace(source, target) + + monkeypatch.setattr(sav_module.os, "replace", fail_publish) + monkeypatch.setattr( + sav_module, "_restore_export_destination", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("synthetic restore failure")), + ) + + with pytest.raises(sav_module.ExportRecoveryError) as error: + openstatspec.export_sav( + database_url=database, dataset_id="restore", destination=destination, + ) + + assert error.value.code == "cleanup_failed" + assert error.value.details["audit_fault"] is None + evidence = error.value.details["deterministic_recovery_evidence"] + assert evidence["cleanup_failed_audit_persisted"] is True + assert evidence["cleanup_failed_audit_operation_id"] + backup = error.value.details["residual_object_inventory"]["backup"] + assert set(backup) == {"role", "path_sha256"} + assert backup["role"] == "durable_backup" + assert error.value.details["residual_object_inventory"]["backup_exists"] is True + backups = list(tmp_path.glob(f".{destination.name}.*.previous")) + assert len(backups) == 1 + assert str(tmp_path) not in json.dumps(error.value.details, sort_keys=True) + connection = sqlite3.connect(database_path) + assert connection.execute( + "select status, dataset_id from operation_catalog " + "where direction = 'export' order by created_at desc limit 1" + ).fetchone() == ("failed", "restore") + assert connection.execute( + "select code, dataset_id from fidelity_event_catalog " + "where direction = 'export' order by rowid desc limit 1" + ).fetchone() == ("cleanup_failed", None) + persisted_details = json.loads(connection.execute( + "select details from fidelity_event_catalog " + "where direction = 'export' order by rowid desc limit 1" + ).fetchone()[0]) + assert set(persisted_details) == { + "original_cause", "cleanup_fault", + "residual_object_inventory", "deterministic_recovery_evidence", + } + connection.close() + + +def test_export_finalization_failure_restores_previous_destination_and_fails_audit( + tmp_path, monkeypatch, +) -> None: + source = tmp_path / "finalize-source.sav" + destination = tmp_path / "finalize-destination.sav" + database_path = tmp_path / "finalize.sqlite" + database = f"sqlite:///{database_path}" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="finalize") + previous = b"previous destination bytes" + destination.write_bytes(previous) + + monkeypatch.setattr( + sav_module, "finish_export_operation", + lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("synthetic finalization failure") + ), + ) + with pytest.raises(RuntimeError, match="synthetic finalization failure"): + openstatspec.export_sav( + database_url=database, dataset_id="finalize", destination=destination, + ) + + assert destination.read_bytes() == previous + assert list(tmp_path.glob(f".{destination.name}.*")) == [] + connection = sqlite3.connect(database_path) + assert connection.execute( + "select status, dataset_id from operation_catalog " + "where direction = 'export' order by created_at desc limit 1" + ).fetchone() == ("failed", "finalize") + assert connection.execute( + "select code, dataset_id from fidelity_event_catalog " + "where direction = 'export' order by rowid desc limit 1" + ).fetchone() == ("export_failed", "finalize") + connection.close() + + +@pytest.mark.parametrize("had_previous", [False, True]) +def test_staging_cleanup_failure_compensates_published_destination_and_fails_audit( + tmp_path, monkeypatch, had_previous: bool, +) -> None: + source = tmp_path / "cleanup-source.sav" + destination = tmp_path / "cleanup-destination.sav" + database_path = tmp_path / "cleanup.sqlite" + database = f"sqlite:///{database_path}" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="cleanup") + previous = b"previous destination bytes" + if had_previous: + destination.write_bytes(previous) + + real_temporary_directory = sav_module.TemporaryDirectory + + class CleanupFailureTemporaryDirectory: + def __init__(self, *args, **kwargs): + self._delegate = real_temporary_directory(*args, **kwargs) + + def __enter__(self): + return self._delegate.__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + result = self._delegate.__exit__(exc_type, exc_value, traceback) + if exc_type is None: + raise RuntimeError("synthetic staging cleanup failure") + return result + + monkeypatch.setattr( + sav_module, "TemporaryDirectory", CleanupFailureTemporaryDirectory, + ) + with pytest.raises(RuntimeError, match="synthetic staging cleanup failure"): + openstatspec.export_sav( + database_url=database, dataset_id="cleanup", + destination=destination, + ) + + if had_previous: + assert destination.read_bytes() == previous + else: + assert not destination.exists() + assert list(tmp_path.glob(f".{destination.name}.*")) == [] + connection = sqlite3.connect(database_path) + assert connection.execute( + "select status from operation_catalog " + "where direction = 'export' order by created_at desc limit 1" + ).fetchone() == ("failed",) + assert connection.execute( + "select code from fidelity_event_catalog " + "where direction = 'export' order by rowid desc limit 1" + ).fetchone() == ("export_failed",) + details = connection.execute( + "select details from operation_catalog " + "where direction = 'export' order by created_at desc limit 1" + ).fetchone()[0] + assert str(tmp_path) not in details + connection.close() + + +def test_export_finalization_commit_after_send_is_idempotent_success( + tmp_path, monkeypatch, +) -> None: + source = tmp_path / "commit-source.sav" + destination = tmp_path / "commit-destination.sav" + database_path = tmp_path / "commit.sqlite" + database = f"sqlite:///{database_path}" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="commit") + destination.write_bytes(b"previous destination bytes") + real_finish = sav_module.finish_export_operation + + def commit_then_disconnect(**kwargs): + real_finish(**kwargs) + raise ConnectionError("synthetic disconnect after commit") + + monkeypatch.setattr( + sav_module, "finish_export_operation", commit_then_disconnect, + ) + result = openstatspec.export_sav( + database_url=database, dataset_id="commit", destination=destination, + ) + + assert result["operation_id"] + assert pyspssio.read_sav(str(destination))[0]["answer"].tolist() == [1.0] + assert list(tmp_path.glob(f".{destination.name}.*")) == [] + connection = sqlite3.connect(database_path) + assert connection.execute( + "select status from operation_catalog where operation_id = ?", + (result["operation_id"],), + ).fetchone() == ("succeeded",) + assert connection.execute( + "select status from operation where operation_id = ?", + (result["operation_id"],), + ).fetchone() == ("succeeded",) + connection.close() + + +def test_export_finalization_mismatch_preserves_published_file_and_backup( + tmp_path, monkeypatch, +) -> None: + source = tmp_path / "mismatch-source.sav" + destination = tmp_path / "mismatch-destination.sav" + database_path = tmp_path / "mismatch.sqlite" + database = f"sqlite:///{database_path}" + pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) + openstatspec.import_sav(source, database_url=database, dataset_id="mismatch") + previous = b"previous destination bytes" + destination.write_bytes(previous) + + def commit_legacy_only_then_disconnect(*, operation_id, **_kwargs): + connection = sqlite3.connect(database_path) + connection.execute( + "update operation_catalog set status = 'succeeded' " + "where operation_id = ?", + (operation_id,), + ) + connection.commit() + connection.close() + raise ConnectionError("synthetic ambiguous finalization") + + monkeypatch.setattr( + sav_module, "finish_export_operation", + commit_legacy_only_then_disconnect, + ) + with pytest.raises(sav_module.ExportRecoveryError) as error: + openstatspec.export_sav( + database_url=database, dataset_id="mismatch", + destination=destination, + ) + + assert error.value.code == "audit_finalization_ambiguous" + assert pyspssio.read_sav(str(destination))[0]["answer"].tolist() == [1.0] + backups = list(tmp_path.glob(f".{destination.name}.*.previous")) + assert len(backups) == 1 + assert backups[0].read_bytes() == previous + serialized_error = json.dumps(error.value.details, sort_keys=True) + assert str(tmp_path) not in serialized_error + evidence = error.value.details["deterministic_recovery_evidence"] + assert evidence["automatic_filesystem_recovery_performed"] is False + assert evidence["manual_reconciliation_required"] is True + + connection = sqlite3.connect(database_path) + destination_audit = connection.execute( + "select destination from operation_catalog " + "where direction = 'export' order by created_at desc limit 1" + ).fetchone()[0] + assert str(tmp_path) not in destination_audit + assert json.loads(destination_audit)["role"] == "destination" + connection.close() diff --git a/tests/test_normative_catalog.py b/tests/test_normative_catalog.py index 0b1b64e..7255fe3 100644 --- a/tests/test_normative_catalog.py +++ b/tests/test_normative_catalog.py @@ -4,7 +4,7 @@ import pytest -from openstatspec.sql.wide import create_wide_dataset, record_export_operation +from openstatspec.sql.wide import create_wide_dataset, initialize_wide_catalog, record_export_operation from openstatspec.sql.normative import catalog as normative_catalog, create as create_normative from sqlalchemy import MetaData, create_engine from sqlalchemy.dialects.mysql import dialect as mysql_dialect @@ -72,6 +72,7 @@ def variables(): def test_import_writes_complete_normative_catalog(tmp_path): database_path = tmp_path / "normative.sqlite" database_url = f"sqlite:///{database_path}" + initialize_wide_catalog(database_url=database_url) result = create_wide_dataset( database_url=database_url, dataset_id="wave_1", source_name="fixture.sav", source_format="SAV", diff --git a/tests/test_official_conformance.py b/tests/test_official_conformance.py index c2714de..161e4af 100644 --- a/tests/test_official_conformance.py +++ b/tests/test_official_conformance.py @@ -13,8 +13,7 @@ from sqlalchemy import inspect as inspect_database import openstatspec -from conformance import compare_sav_semantics -from openstatspec.sql.capabilities import effective_profile +from openstatspec.sql.profiles import profile_for_url SEMANTIC_EXPECTATIONS = { @@ -232,6 +231,7 @@ def _assert_round_trip( dataset_id = f"official_{profile}_{fixture['id']}_{token}".replace("-", "_") destination = tmp_path / f"{dataset_id}{source.suffix}" + openstatspec.initialize_catalog(database_url=database_url) imported = openstatspec.import_sav( source, database_url=database_url, dataset_id=dataset_id, ) @@ -245,10 +245,9 @@ def _assert_round_trip( database_url=database_url, dataset_id=dataset_id, destination=destination, ) assert exported.diagnostics == () - assert compare_sav_semantics(source, destination) == { - "equivalent": True, - "differences": [], - } + comparison = openstatspec.compare_sav_semantics(source, destination) + assert comparison["equivalent"] is True + assert comparison["differences"] == [] def test_manifest_required_capabilities_are_declared() -> None: @@ -276,7 +275,6 @@ def test_official_manifest_round_trips_through_sqlite( ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), ("OPENSTATSPEC_MYSQL_URL", "mysql"), ("OPENSTATSPEC_MARIADB_URL", "mariadb"), - ("OPENSTATSPEC_DOLT_URL", "dolt"), ], ) @pytest.mark.parametrize(("fixture", "source"), _round_trip_fixtures()) @@ -300,7 +298,7 @@ def _assert_official_preflight_failure( ) assert fixture["directions"] == ["import"] assert set(fixture["expects"]) == PREFLIGHT_EXPECTATIONS - maximum = effective_profile(database_url)[0].max_physical_variables + maximum = profile_for_url(database_url).max_source_variables source = tmp_path / f"preflight-too-wide-{profile}-{uuid4().hex[:8]}.sav" columns = [f"v{ordinal:05d}" for ordinal in range(1, maximum + 2)] pyspssio.write_sav( @@ -309,6 +307,7 @@ def _assert_official_preflight_failure( ) dataset_name = f"official_preflight_failure_{profile}_{uuid4().hex[:8]}" + openstatspec.initialize_catalog(database_url=database_url) with pytest.raises(Exception, match="Target capability exceeded"): openstatspec.import_sav( source, database_url=database_url, dataset_id=dataset_name, @@ -359,7 +358,6 @@ def test_official_preflight_failure_is_atomic_and_diagnostic(tmp_path: Path) -> ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), ("OPENSTATSPEC_MYSQL_URL", "mysql"), ("OPENSTATSPEC_MARIADB_URL", "mariadb"), - ("OPENSTATSPEC_DOLT_URL", "dolt"), ], ) def test_official_preflight_failure_through_server_profiles( diff --git a/tests/test_pyspssio_catalog_authority.py b/tests/test_pyspssio_catalog_authority.py index f160b34..374b42b 100644 --- a/tests/test_pyspssio_catalog_authority.py +++ b/tests/test_pyspssio_catalog_authority.py @@ -25,6 +25,7 @@ def dictionary_with_variable_set(path): return metadata, sav_module._engine_loss_report(metadata) monkeypatch.setattr(sav_module, "_dictionary", dictionary_with_variable_set) + openstatspec.initialize_catalog(database_url=f"sqlite:///{database_path}") imported = openstatspec.import_sav(source, database_url=f"sqlite:///{database_path}", dataset_id="variables") assert {diagnostic.code for diagnostic in imported.diagnostics} == set() connection = sqlite3.connect(database_path) @@ -54,6 +55,7 @@ def test_normalized_mr_catalog_is_authoritative_for_export(tmp_path) -> None: }, }, ) + openstatspec.initialize_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="mr") connection = sqlite3.connect(database_path) rows = connection.execute( @@ -83,6 +85,7 @@ def test_normalized_mr_catalog_is_authoritative_for_export(tmp_path) -> None: def test_fidelity_event_details_survive_reopening_database(tmp_path) -> None: database = f"sqlite:///{tmp_path / 'events.sqlite'}" + openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="events", source_name="events.sav", source_format="SAV", rows=[{"answer": 1.0}], diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index 8b99980..ec92344 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -7,8 +7,9 @@ import pytest import openstatspec -from conformance import compare_sav_semantics, write_supported_semantics_fixture +from conformance import write_supported_semantics_fixture from openstatspec.core import UnsupportedOperationError +from openstatspec.sql.wide import initialize_wide_catalog _REQUIRED_ENGINE_LOSS = [] @@ -38,6 +39,7 @@ def test_pyspssio_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None: }, ) + initialize_wide_catalog(database_url=database) imported = openstatspec.import_sav(source, database_url=database, dataset_id="tiny") assert imported["case_count"] == 2 assert imported["data_table"] == "data_tiny" @@ -89,6 +91,7 @@ def test_file_label_round_trips_through_sqlite_and_export(tmp_path) -> None: str(source), pd.DataFrame({"answer": [1.0]}), metadata={"file_label": label} ) + initialize_wide_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="label") connection = sqlite3.connect(database_path) assert connection.execute( @@ -106,14 +109,27 @@ def test_supported_pyspssio_metadata_round_trips_through_sqlite_for_sav_and_zsav database_path = tmp_path / f"supported-{suffix[1:]}.sqlite" destination = tmp_path / f"supported-roundtrip{suffix}" expected = write_supported_semantics_fixture(source) + database = f"sqlite:///{database_path}" - result = openstatspec.import_sav(source, database_url=f"sqlite:///{database_path}", dataset_id=f"supported-{suffix[1:]}") + initialize_wide_catalog(database_url=database) + result = openstatspec.import_sav(source, database_url=database, dataset_id=f"supported-{suffix[1:]}") assert result["case_count"] == 4 assert openstatspec.validate(database_url=f"sqlite:///{database_path}", dataset_id=f"supported-{suffix[1:]}")["valid"] is True connection = sqlite3.connect(database_path) - assert connection.execute(f"select comment from data_supported_{suffix[1:]} order by __case_ordinal").fetchone() == (expected["long_text"],) + assert connection.execute( + f"select {expected['long_text_variable']} " + f"from data_supported_{suffix[1:]} order by __case_ordinal" + ).fetchone() == (expected["long_text"],) openstatspec.export_sav(database_url=f"sqlite:///{database_path}", dataset_id=f"supported-{suffix[1:]}", destination=destination, allow_loss=_COMPAT_NAME_LOSS) - assert compare_sav_semantics(source, destination) == {"equivalent": True, "differences": []} + comparison = openstatspec.compare_sav_semantics(source, destination) + assert comparison["equivalent"] is True + assert comparison["differences"] == [] + source_metadata = pyspssio.read_metadata(str(source)) + exported_metadata = pyspssio.read_metadata(str(destination)) + assert exported_metadata["var_types"][expected["long_text_variable"]] == ( + source_metadata["var_types"][expected["long_text_variable"]] + ) + assert exported_metadata["var_compat_names"][expected["long_text_variable"]] == expected["compatible_name"] @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) def test_pyspssio_preserves_supported_metadata_for_both_formats(tmp_path, suffix: str) -> None: @@ -124,6 +140,7 @@ def test_import_rejects_physical_table_name_collision_without_partial_catalog(tm database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) + initialize_wide_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="wave-1") with pytest.raises(ValueError, match="collides"): openstatspec.import_sav(source, database_url=database, dataset_id="wave 1") @@ -163,6 +180,7 @@ def test_raw_dictionary_bridge_preserves_distinct_formats_sets_and_attribute_arr writer.commit_header() writer.write_data(pd.DataFrame({"answer": [1.0], "comment": ["yes"]})) + initialize_wide_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="raw") connection = sqlite3.connect(tmp_path / f"raw-{suffix[1:]}.sqlite") assert connection.execute( @@ -201,6 +219,7 @@ def test_very_long_string_round_trips_through_sqlite_and_export(tmp_path, suffix ) assert pyspssio.read_metadata(str(source))["var_types"]["comment"] == payload_width + initialize_wide_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="long") connection = sqlite3.connect(database_path) assert connection.execute( diff --git a/tests/test_semantic_compare.py b/tests/test_semantic_compare.py new file mode 100644 index 0000000..6e4f699 --- /dev/null +++ b/tests/test_semantic_compare.py @@ -0,0 +1,289 @@ +"""Synthetic contract checks for the public value-redacting SAV comparator.""" + +import copy +import struct +from pathlib import Path + +import pandas as pd + +import openstatspec +import openstatspec.spss.semantics as semantic_module + + +def _metadata() -> dict: + return { + "encoding": "UTF-8", + "file_label": "classified-file-label", + "case_weight_var": "numeric_secret", + "var_types": {"numeric_secret": 0, "string_secret": 16}, + "var_labels": { + "numeric_secret": "classified-numeric-label", + "string_secret": "classified-string-label", + }, + "var_measure_levels": { + "numeric_secret": "scale", "string_secret": "nominal", + }, + "var_roles": {"numeric_secret": "target", "string_secret": "input"}, + "var_alignments": {"numeric_secret": "right", "string_secret": "left"}, + "var_column_widths": {"numeric_secret": 8, "string_secret": 16}, + "var_compat_names": {"numeric_secret": "NUMERIC", "string_secret": "STRING"}, + "var_value_labels": { + "numeric_secret": {1.0: "classified-one", 2.0: "classified-two"}, + }, + "var_missing_values": { + "numeric_secret": {"values": [-99.0, -98.0]}, + }, + "mrsets": { + "$classified": { + "label": "classified-mr-label", + "counted_value": 1.0, + "variable_list": ["numeric_secret", "string_secret"], + }, + }, + } + + +def _dictionary() -> dict: + return { + "_documents": ["classified-document-one", "classified-document-two"], + "_var_sets": { + "classified-set-a": ["numeric_secret", "string_secret"], + "classified-set-b": ["string_secret"], + }, + "_print_format_tuples": { + "numeric_secret": (5, 8, 1), "string_secret": (1, 16, 0), + }, + "_write_format_tuples": { + "numeric_secret": (3, 12, 3), "string_secret": (1, 16, 0), + }, + "file_attributes": { + "classified-scalar": "classified-value", + "classified-array": ["first-secret", "second-secret"], + }, + "var_attributes": { + "numeric_secret": { + "classified-origin": "classified-source", + "classified-array": ["first-secret", "second-secret"], + }, + }, + } + + +def _install_observations( + monkeypatch, *, source_frame: pd.DataFrame, exported_frame: pd.DataFrame, + source_metadata: dict | None = None, exported_metadata: dict | None = None, + source_dictionary: dict | None = None, exported_dictionary: dict | None = None, +) -> None: + frames = {"source.sav": source_frame, "exported.sav": exported_frame} + metadata = { + "source.sav": source_metadata or _metadata(), + "exported.sav": exported_metadata or _metadata(), + } + dictionaries = { + "source.sav": source_dictionary or _dictionary(), + "exported.sav": exported_dictionary or _dictionary(), + } + + def read_sav(path: str, **_options): + name = Path(path).name + return frames[name].copy(), copy.deepcopy(metadata[name]) + + def read_dictionary(path: Path): + dictionary = copy.deepcopy(dictionaries[path.name]) + loss_report = {} + if dictionary.get("_documents") is None: + loss_report["documents-unreadable"] = {"code": "documents-unreadable"} + if dictionary.get("_var_sets") is None: + loss_report["variable-sets-unobservable"] = { + "code": "variable-sets-unobservable", + } + return dictionary, loss_report + + monkeypatch.setattr(semantic_module.pyspssio, "read_sav", read_sav) + monkeypatch.setattr(semantic_module, "_dictionary", read_dictionary) + + +def _frame(numeric_values: list[float]) -> pd.DataFrame: + return pd.DataFrame({ + "numeric_secret": numeric_values, + "string_secret": ["classified-value", "", "classified-tail"][:len(numeric_values)], + }) + + +def test_public_comparison_treats_nan_payloads_as_system_missing_and_redacts_values( + monkeypatch, +) -> None: + source_nan = struct.unpack(">d", bytes.fromhex("7ff8000000000001"))[0] + exported_nan = struct.unpack(">d", bytes.fromhex("7ff8000000001234"))[0] + _install_observations( + monkeypatch, + source_frame=_frame([-0.0, source_nan, 2.5]), + exported_frame=_frame([-0.0, exported_nan, 2.5]), + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert result["equivalent"] is True + assert result["differences"] == [] + assert result["counts"]["source"] == { + "cases": 3, "variables": 2, "numeric_variables": 1, "string_variables": 1, + } + assert result["components"]["numeric_nonmissing_binary64"]["source_count"] == 2 + assert result["components"]["numeric_system_missing_mask"]["source_count"] == 3 + serialized_result = repr(result) + for secret in ( + "classified-value", "classified-file-label", "numeric_secret", + "classified-document-one", "classified-set-a", "classified-one", + ): + assert secret not in serialized_result + for component in result["components"].values(): + assert set(component) == {"status", "source_count", "exported_count"} + + receipt = openstatspec.compare_sav_semantics( + "source.sav", "exported.sav", include_digests=True, + ) + for component in receipt["components"].values(): + assert set(component) == { + "status", "source_count", "exported_count", "source_sha256", + "exported_sha256", + } + assert len(component["source_sha256"]) == 64 + assert len(component["exported_sha256"]) == 64 + + +def test_public_comparison_distinguishes_signed_zero_bit_patterns(monkeypatch) -> None: + _install_observations( + monkeypatch, + source_frame=_frame([-0.0]), + exported_frame=_frame([0.0]), + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert result["equivalent"] is False + assert result["differences"] == ["case_order", "numeric_nonmissing_binary64"] + assert result["components"]["numeric_system_missing_mask"]["status"] == "equal" + assert result["components"]["numeric_nonmissing_binary64"]["status"] == "different" + + +def test_public_comparison_covers_ordered_dictionary_semantics(monkeypatch) -> None: + source_metadata = _metadata() + exported_metadata = copy.deepcopy(source_metadata) + exported_metadata["var_value_labels"]["numeric_secret"] = { + 2.0: "classified-two", 1.0: "classified-one", + } + exported_metadata["var_missing_values"]["numeric_secret"]["values"].reverse() + exported_metadata["mrsets"]["$classified"]["variable_list"].reverse() + source_dictionary = _dictionary() + exported_dictionary = copy.deepcopy(source_dictionary) + exported_dictionary["_documents"].reverse() + exported_dictionary["_var_sets"]["classified-set-a"].reverse() + exported_dictionary["_print_format_tuples"]["numeric_secret"] = (5, 9, 1) + exported_dictionary["_write_format_tuples"]["numeric_secret"] = (3, 13, 3) + exported_dictionary["file_attributes"]["classified-array"].reverse() + exported_dictionary["var_attributes"]["numeric_secret"]["classified-array"].reverse() + _install_observations( + monkeypatch, + source_frame=_frame([1.0]), exported_frame=_frame([1.0]), + source_metadata=source_metadata, exported_metadata=exported_metadata, + source_dictionary=source_dictionary, exported_dictionary=exported_dictionary, + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert set(result["differences"]) == { + "documents", "file_attributes", "print_formats", "write_formats", + "ordered_value_labels", "missing_rules", "variable_attributes", + "variable_sets", "multiple_response_sets", + } + + +def test_unobservable_dictionary_components_fail_closed_without_error_text( + monkeypatch, +) -> None: + unreadable = _dictionary() + unreadable["_documents"] = None + unreadable["_documents_error"] = { + "type": "SyntheticReaderError", + "code": None, + "phase": "read_sav_documents", + "message_sha256": "a" * 64, + } + unreadable["_var_sets"] = None + unreadable["_var_sets_error"] = { + "type": "SyntheticBridgeError", + "code": None, + "phase": "read_sav_variable_sets", + "message_sha256": "b" * 64, + } + _install_observations( + monkeypatch, + source_frame=_frame([1.0]), exported_frame=_frame([1.0]), + source_dictionary=unreadable, exported_dictionary=unreadable, + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert result["components"]["adapter_observability"]["status"] == "unavailable" + assert result["equivalent"] is False + assert result["components"]["documents"]["status"] == "unavailable" + assert result["components"]["variable_sets"]["status"] == "unavailable" + assert "a" * 64 not in repr(result) + assert "b" * 64 not in repr(result) + + +def test_missing_source_encoding_fails_closed(monkeypatch) -> None: + metadata = _metadata() + metadata["encoding"] = None + _install_observations( + monkeypatch, + source_frame=_frame([1.0]), exported_frame=_frame([1.0]), + source_metadata=metadata, exported_metadata=metadata, + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert result["equivalent"] is False + assert result["components"]["source_encoding"]["status"] == "unavailable" + + +def test_variable_order_is_reported_without_reclassifying_equal_properties(monkeypatch) -> None: + source = _frame([1.0]) + exported = source[["string_secret", "numeric_secret"]] + _install_observations( + monkeypatch, source_frame=source, exported_frame=exported, + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert result["differences"] == ["variable_order"] + assert result["components"]["variable_labels"]["status"] == "equal" + assert result["components"]["numeric_nonmissing_binary64"]["status"] == "equal" + + +def test_public_comparison_covers_header_and_variable_dictionary(monkeypatch) -> None: + source_metadata = _metadata() + exported_metadata = copy.deepcopy(source_metadata) + exported_metadata["encoding"] = "CP1252" + exported_metadata["file_label"] = "changed-label" + exported_metadata["case_weight_var"] = "string_secret" + exported_metadata["var_types"]["string_secret"] = 32 + exported_metadata["var_labels"]["numeric_secret"] = "changed-variable-label" + exported_metadata["var_measure_levels"]["numeric_secret"] = "ordinal" + exported_metadata["var_roles"]["numeric_secret"] = "input" + exported_metadata["var_alignments"]["numeric_secret"] = "left" + exported_metadata["var_column_widths"]["numeric_secret"] = 12 + exported_metadata["var_compat_names"]["numeric_secret"] = "CHANGED" + _install_observations( + monkeypatch, + source_frame=_frame([1.0]), exported_frame=_frame([1.0]), + source_metadata=source_metadata, exported_metadata=exported_metadata, + ) + + result = openstatspec.compare_sav_semantics("source.sav", "exported.sav") + + assert set(result["differences"]) == { + "source_encoding", "file_label", "case_weight_variable", "variable_types", + "variable_labels", "measurement_levels", "variable_roles", + "variable_alignments", "display_widths", "compatible_names", + } diff --git a/tests/test_spss_catalog_preflight.py b/tests/test_spss_catalog_preflight.py index 1b2aa64..cd707a6 100755 --- a/tests/test_spss_catalog_preflight.py +++ b/tests/test_spss_catalog_preflight.py @@ -42,12 +42,14 @@ def test_import_catalog_preflight_rejects_invalid_weight_atomically( tmp_path, weight_name, mutate, expected_code, ) -> None: database_path = tmp_path / "weight.sqlite" + database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) variables = _variables() mutate(variables) with pytest.raises(CatalogPreflightError) as error: create_wide_dataset( - database_url=f"sqlite:///{database_path}", dataset_id="weight", + database_url=database, dataset_id="weight", source_name="weight.sav", source_format="SAV", rows=[{"weight": 1.0, "answer": 1.0, "text": "ok"}], variables=variables, case_weight_variable=weight_name, @@ -85,10 +87,12 @@ def test_import_catalog_preflight_rejects_invalid_weight_atomically( def test_numeric_weight_does_not_require_scale_measurement_level(tmp_path) -> None: + database = f"sqlite:///{tmp_path}/weight-nominal.sqlite" + openstatspec.initialize_catalog(database_url=database) variables = _variables() variables[1] = {**variables[1], "measure": "nominal"} result = create_wide_dataset( - database_url=f"sqlite:///{tmp_path / 'weight-nominal.sqlite'}", + database_url=database, dataset_id="weight_nominal", source_name="weight.sav", source_format="SAV", rows=[{"weight": 1.0, "answer": 2.0, "text": "ok"}], variables=variables, case_weight_variable="answer", @@ -97,6 +101,7 @@ def test_numeric_weight_does_not_require_scale_measurement_level(tmp_path) -> No def _create_valid_dataset(database: str) -> None: + openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="mr", source_name="mr.sav", source_format="SAV", rows=[{"weight": 1.0, "answer": 1.0, "text": "yes"}], diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 5988265..a103c9d 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -7,15 +7,14 @@ from sqlalchemy.dialects import mysql, postgresql, sqlite from sqlalchemy.schema import CreateTable +import openstatspec.sql.capabilities as capability_module +import openstatspec.sql.wide as wide from openstatspec.sql.normative import catalog as normative_catalog from openstatspec.core import UnsupportedOperationError from openstatspec.sql.profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, preflight, profile_for_url -import openstatspec.sql.capabilities as capabilities -import openstatspec.sql.wide as wide -import openstatspec.spss.sav as sav -from openstatspec.sql.capabilities import active_connection, effective_profile, server_version_supported -from openstatspec.sql.wide import binary64_type, string_type +from openstatspec.sql.capabilities import active_connection, server_version_supported +from openstatspec.sql.wide import binary64_type, lossless_text_type def test_profile_detection_tracks_supported_dialect_urls() -> None: @@ -23,20 +22,11 @@ def test_profile_detection_tracks_supported_dialect_urls() -> None: assert profile_for_url("postgresql+psycopg://user@host/database") is POSTGRESQL assert profile_for_url("mysql+pymysql://user@host/database") is MYSQL assert profile_for_url("mariadb+mariadbconnector://user@host/database") is MYSQL - assert profile_for_url("mysql+pymysql://user@host/dolt_database") is MYSQL - -def test_profile_declarations_publish_released_specification_provenance() -> None: - for declaration in capabilities.profile_declarations().values(): - assert declaration["specification_status"] == "released" - assert declaration["specification_release"] == "v0.2.0" - assert ( - declaration["specification_commit"] - == "79339ec3d8f8aa81789b7e85f6b8afa6f1374e50" - ) + def test_profile_preflight_fails_without_transforming_a_wide_dataset() -> None: with pytest.raises(UnsupportedOperationError, match="Target capability exceeded"): - preflight(POSTGRESQL, POSTGRESQL.max_physical_variables + 1) + preflight(POSTGRESQL, POSTGRESQL.max_source_variables + 1) def test_numeric_columns_compile_to_explicit_binary64_types() -> None: @@ -52,6 +42,66 @@ def test_numeric_columns_compile_to_explicit_binary64_types() -> None: assert "VALUE REAL" in sqlite_ddl +def test_mysql_wire_text_columns_compile_to_declared_longtext() -> None: + table = Table("text_fixture", MetaData(), Column("value", lossless_text_type())) + + mysql_ddl = str(CreateTable(table).compile(dialect=mysql.dialect())).upper() + + assert "VALUE LONGTEXT" in mysql_ddl + assert "VALUE TEXT" not in mysql_ddl + + +@pytest.mark.parametrize("wire_profile", [MYSQL, DOLT], ids=["mysql", "dolt"]) +def test_mysql_wire_additive_metadata_migration_uses_longtext( + monkeypatch, wire_profile, +) -> None: + metadata = MetaData() + datasets, variables, _, _ = wide.catalog(metadata) + multiple_response = wide.multiple_response_set_catalog(metadata) + existing_columns = { + datasets.name: {"case_weight_variable"}, + variables.name: { + "role", "compat_name", "print_format", "write_format", + }, + multiple_response.name: { + "is_dichotomy", "use_category_labels", "use_first_var_label", + "counted_value_type", "counted_numeric", + }, + } + + class SyntheticInspector: + def has_table(self, table_name): + return table_name in existing_columns + + def get_columns(self, table_name): + return [{"name": name} for name in existing_columns[table_name]] + + class SyntheticConnection: + def __init__(self): + self.dialect = mysql.dialect() + self.profile = wire_profile + self.statements = [] + + def execute(self, statement): + self.statements.append(str(statement)) + + connection = SyntheticConnection() + monkeypatch.setattr(wide, "inspect", lambda _connection: SyntheticInspector()) + + wide._migrate_catalog_columns( + connection, datasets, variables, multiple_response, + ) + + statements = "\n".join(connection.statements).upper() + assert len(connection.statements) == 3 + assert "ADD COLUMN FILE_ATTRIBUTES LONGTEXT" in statements + assert "ADD COLUMN ATTRIBUTES LONGTEXT" in statements + assert "ADD COLUMN COUNTED_TEXT LONGTEXT" in statements + assert "ADD COLUMN FILE_ATTRIBUTES TEXT" not in statements + assert "ADD COLUMN ATTRIBUTES TEXT" not in statements + assert "ADD COLUMN COUNTED_TEXT TEXT" not in statements + + def test_unknown_target_is_explicitly_rejected() -> None: with pytest.raises(UnsupportedOperationError, match="No OpenStatSpec SQL profile"): profile_for_url("oracle://host/database") @@ -86,28 +136,31 @@ def test_normative_catalog_compiles_for_every_sql_family() -> None: assert all("CREATE TABLE" in ddl for ddl in compiled) +def test_dolt_adapter_envelope_accepts_305_and_rejects_306_source_variables() -> None: + preflight(DOLT, 305) + with pytest.raises(UnsupportedOperationError) as error: + preflight(DOLT, 306) + assert error.value.details == { + "reason": "source_variable_limit", + "source_count": 306, + "max_source": 305, + "physical_count": 307, + "max_physical": 306, + } + + @pytest.mark.parametrize( ("profile", "version", "supported"), [ - ("mysql", "8.4.0", True), ("mysql", "8.4.999", True), - ("mysql", "9.7.2", True), ("mysql", "8.0.44", False), - ("mysql", "8.5.0", False), ("mysql", "9.8.0", False), - ("dolt", "2.2.2", True), ("dolt", " 2.2.2 ", True), - ("dolt", "2.2.3", True), ("dolt", "2.2.999", True), - ("dolt", "2.2.0", False), ("dolt", "2.2.1", False), - ("dolt", "2.3.0", False), ("dolt", "2.2", False), - ("dolt", "2.2.3-rc1", False), ("dolt", "2.2.3+build.1", False), - ("dolt", "v2.2.3", False), ("dolt", "2.2.03", False), - ("dolt", "garbage", False), - ("mariadb", "11.4.0-MariaDB", True), - ("mariadb", "11.4.999-MariaDB", True), - ("mariadb", "11.8.8-MariaDB", True), - ("mariadb", "12.3.2-MariaDB", True), - ("mariadb", "11.5.0-MariaDB", False), - ("mariadb", "12.4.0-MariaDB", False), - ("postgresql", "17.0", True), ("postgresql", "17.99", True), - ("postgresql", "18.4", True), ("postgresql", "16.10", False), - ("postgresql", "19.0", False), + ("mysql", "8.4.6", True), ("mysql", "9.7.0", True), + ("mysql", "8.0.44", False), + ("mariadb", "11.4.8-MariaDB", True), + ("mariadb", "11.8.3-MariaDB", True), + ("mariadb", "12.3.1-MariaDB", True), + ("postgresql", "17.6", True), ("postgresql", "18.1", True), + ("postgresql", "16.10", False), + ("dolt", "2.2.2", False), ("dolt", " 2.2.2 ", False), + ("dolt", "2.2.2.1", False), ("dolt", "2.2.3", False), ], ) def test_claimed_server_version_policy_is_explicit( @@ -116,104 +169,7 @@ def test_claimed_server_version_policy_is_explicit( assert server_version_supported(profile, version) is supported -@pytest.mark.services -@pytest.mark.parametrize( - ("environment_name", "profile"), - [ - ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), - ("OPENSTATSPEC_MYSQL_URL", "mysql"), - ("OPENSTATSPEC_MARIADB_URL", "mariadb"), - ("OPENSTATSPEC_DOLT_URL", "dolt"), - ], -) -def test_active_server_identity_matches_claimed_ci_profile( - environment_name: str, profile: str, -) -> None: - database_url = os.environ.get(environment_name) - if not database_url: - pytest.skip(f"{environment_name} is not configured") - - active = active_connection(database_url) - assert active["profile"] == profile - assert active["claimed_supported"] is True - assert active["matched_claim"] is not None - - -@pytest.mark.services -@pytest.mark.parametrize( - ("database_environment", "version_environment", "profile"), - [ - ( - "OPENSTATSPEC_POSTGRES_URL", - "OPENSTATSPEC_EXPECTED_POSTGRES_VERSION", - "postgresql", - ), - ( - "OPENSTATSPEC_MYSQL_URL", - "OPENSTATSPEC_EXPECTED_MYSQL_VERSION", - "mysql", - ), - ( - "OPENSTATSPEC_MARIADB_URL", - "OPENSTATSPEC_EXPECTED_MARIADB_VERSION", - "mariadb", - ), - ( - "OPENSTATSPEC_DOLT_URL", - "OPENSTATSPEC_EXPECTED_DOLT_VERSION", - "dolt", - ), - ], -) -def test_live_server_version_matches_exact_ci_evidence( - database_environment: str, version_environment: str, profile: str, -) -> None: - database_url = os.environ.get(database_environment) - if not database_url: - pytest.skip(f"{database_environment} is not configured") - expected_version = os.environ.get(version_environment) - assert expected_version, f"{version_environment} is required for CI provenance" - - active = active_connection(database_url) - - assert active["profile"] == profile - assert active["server_version"] == expected_version - - -def test_server_policy_distinguishes_claimed_families_from_exact_ci_evidence() -> None: - declarations = capabilities.profile_declarations() - assert declarations["mysql"]["claimed_server_versions"] == [ - "MySQL 8.4.x", "MySQL 9.7.x", - ] - assert declarations["mysql"]["ci_tested_server_versions"] == [ - "MySQL 8.4.11", "MySQL 9.7.2", - ] - assert declarations["mariadb"]["claimed_server_versions"] == [ - "MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x", - ] - assert declarations["mariadb"]["ci_tested_server_versions"] == [ - "MariaDB 11.4.12", "MariaDB 11.8.8", "MariaDB 12.3.2", - ] - assert declarations["postgresql"]["claimed_server_versions"] == [ - "PostgreSQL 17.x", "PostgreSQL 18.x", - ] - assert declarations["postgresql"]["ci_tested_server_versions"] == [ - "PostgreSQL 17.10", "PostgreSQL 18.4", - ] - assert declarations["dolt"]["claimed_server_versions"] == ["Dolt 2.2.x"] - assert declarations["dolt"]["claimed_version_range"] == { - "minimum_inclusive": "2.2.2", - "maximum_exclusive": "2.3.0", - } - assert declarations["dolt"]["ci_tested_server_versions"] == [ - "Dolt 2.2.2", "Dolt 2.2.3", - ] - assert declarations["dolt"]["exact_ci_tested_versions"] == [ - "2.2.2", "2.2.3", - ] - - -class _ProbeResult: +class _ScalarResult: def __init__(self, value): self.value = value @@ -221,300 +177,327 @@ def scalar_one(self): return self.value -class _ProbeConnection: - def __init__(self, responses): - self.responses = responses - self.calls = [] - self.dialect = SimpleNamespace(name="mysql") +class _FakeMySqlConnection: + dialect = SimpleNamespace(name="mysql") + + def __init__(self, probes): + self.probes = probes def __enter__(self): return self def __exit__(self, *_args): - return None + return False def execute(self, statement): - query = str(statement) - self.calls.append(query) - value = self.responses[query] - if isinstance(value, Exception): - raise value - return _ProbeResult(value) + return _ScalarResult(self.probes[str(statement).strip().casefold()]) -class _ProbeEngine: - def __init__(self, connection): - self.connection = connection - self.dialect = SimpleNamespace(driver="pymysql") +class _FakeMySqlEngine: + def __init__(self, probes): + self.probes = probes def connect(self): - return self.connection - - -def _mock_mysql_probes(monkeypatch, **overrides): - responses = { - "select @@version": "8.0.31", - "select @@version_comment": "Dolt", - "select DOLT_VERSION()": "2.2.3", - "select @@max_allowed_packet": 1_073_741_824, + return _FakeMySqlConnection(self.probes) + + +def _fake_mysql_identity( + monkeypatch, *, wire="8.0.33", comment="Dolt", dolt="2.2.2", + branch="feature/synthetic", +): + probes = { + "select @@version": wire, + "select @@version_comment": comment, + "select dolt_version()": dolt, + "select active_branch()": branch, + "select @@max_allowed_packet": 16_777_216, } - responses.update(overrides) - connection = _ProbeConnection(responses) - monkeypatch.setattr(capabilities, "create_engine", lambda _url: _ProbeEngine(connection)) - return connection - - -def test_dolt_identity_is_exact_and_publishes_wire_and_product_versions(monkeypatch) -> None: - connection = _mock_mysql_probes( - monkeypatch, **{"select @@version_comment": " dOlT "}, + monkeypatch.setattr( + capability_module, "create_engine", lambda _database_url: _FakeMySqlEngine(probes), ) - active = active_connection("mysql+pymysql://user@host/database") - - assert active["dialect"] == "mysql" - assert active["profile"] == active["engine"] == active["product"] == "dolt" - assert active["transport"] == "mysql_compatible" - assert active["driver"] == "pymysql" - assert active["raw_wire_version"] == "8.0.31" - assert active["raw_product_version"] == active["raw_server_version"] == "2.2.3" - assert active["raw_version_comment"] == " dOlT " - assert active["server_version"] == "2.2.3" - assert active["identity_source"] == "SELECT @@version, @@version_comment, DOLT_VERSION()" - assert active["claimed_supported"] is True - assert connection.calls == [ - "select @@version", "select @@version_comment", - "select DOLT_VERSION()", "select @@max_allowed_packet", - ] +def test_active_identity_recognizes_only_exact_trimmed_casefolded_dolt_comment(monkeypatch): + _fake_mysql_identity(monkeypatch, comment=" dOlT ") + active = capability_module.active_connection("mysql+pymysql://user@host/catalog") + assert active["profile"] == "dolt" + assert active["product"] == "Dolt" + assert active["raw_wire_version"] == "8.0.33" + assert active["raw_product_version"] == "2.2.2" + assert active["claimed_supported"] is False + assert active["working_set_binding"] == { + "database": "catalog", + "active_branch": "feature/synthetic", + } + with pytest.raises(UnsupportedOperationError, match="not bound"): + capability_module.profile_declarations( + "mysql+pymysql://user@host/catalog" + ) -@pytest.mark.parametrize("comment", [None, False, "", " "]) -def test_mysql_wire_identity_requires_a_nonempty_version_comment(monkeypatch, comment) -> None: - connection = _mock_mysql_probes( - monkeypatch, **{"select @@version_comment": comment}, - ) - - with pytest.raises(UnsupportedOperationError, match="@@version_comment"): - active_connection("mysql+pymysql://user@host/database") - - assert "select DOLT_VERSION()" not in connection.calls - assert "select @@max_allowed_packet" not in connection.calls +def test_active_identity_does_not_guess_dolt_from_nonexact_comment(monkeypatch): + _fake_mysql_identity(monkeypatch, comment="Dolt database") + active = capability_module.active_connection("mysql+pymysql://user@host/catalog") + assert active["profile"] == "mysql" -def test_mysql_wire_identity_fails_closed_when_comment_probe_raises(monkeypatch) -> None: - connection = _mock_mysql_probes( - monkeypatch, **{"select @@version_comment": RuntimeError("probe unavailable")}, - ) - with pytest.raises(UnsupportedOperationError, match="@@version_comment"): - active_connection("mysql+pymysql://user@host/database") +@pytest.mark.parametrize( + ("wire", "comment", "dolt", "message"), + [ + ("8.0.33-MariaDB", "Dolt", "2.2.2", "Conflicting Dolt and MariaDB"), + ("8.0.33", "Dolt", "", "DOLT_VERSION"), + ("8.0.33", "", "2.2.2", "@@version_comment"), + ], +) +def test_active_dolt_identity_fails_closed(monkeypatch, wire, comment, dolt, message): + _fake_mysql_identity(monkeypatch, wire=wire, comment=comment, dolt=dolt) + with pytest.raises(UnsupportedOperationError, match=message): + capability_module.active_connection("mysql+pymysql://user@host/catalog") - assert "select DOLT_VERSION()" not in connection.calls +def test_active_dolt_identity_requires_nonempty_active_branch(monkeypatch): + _fake_mysql_identity(monkeypatch, branch="") + with pytest.raises(UnsupportedOperationError, match="ACTIVE_BRANCH"): + capability_module.active_connection( + "mysql+pymysql://user@host/catalog" + ) -def test_dolt_comment_requires_a_nonempty_product_version(monkeypatch) -> None: - _mock_mysql_probes(monkeypatch, **{"select DOLT_VERSION()": None}) - with pytest.raises(UnsupportedOperationError, match=r"DOLT_VERSION\(\)"): - active_connection("mysql+pymysql://user@host/database") +def test_dolt_capabilities_separate_adapter_envelope_from_unclaimed_server_limits(): + declaration = capability_module.profile_declarations()["dolt"] + assert declaration["claimed_server_versions"] == [] + assert declaration["ci_tested_server_versions"] == [] + assert declaration["operational_write_enabled"] is False + assert declaration["write_conformance"] == { + "declaration_schema_id": "openstatspec-dolt-adapter-declaration-v1", + "write_enabled": False, + "declarations_available": False, + "declaration_count": 0, + "status": "blocked_no_concrete_declarations", + "active_declaration_id": None, + } + assert declaration["theoretical_limits"]["limit_basis"] == "server_limits_not_claimed" + assert declaration["theoretical_limits"]["maximum_source_variables"] is None + assert declaration["adapter_envelope"]["maximum_source_variables"] == 305 + assert declaration["adapter_envelope"]["maximum_physical_columns"] == 306 + assert declaration["adapter_envelope"]["evidence_status"] == "pending_pinned_live_conformance" +@pytest.mark.services @pytest.mark.parametrize( - "product_version", + ("environment_name", "profile"), [ - "2.2.0", "2.2.1", "2.3.0", "2.2", - "2.2.3-rc1", "2.2.3+build.1", "v2.2.3", "2.2.03", "garbage", + ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), + ("OPENSTATSPEC_MYSQL_URL", "mysql"), + ("OPENSTATSPEC_MARIADB_URL", "mariadb"), ], ) -def test_dolt_identity_rejects_out_of_range_or_noncanonical_product_versions( - monkeypatch, product_version, +def test_active_server_identity_matches_claimed_ci_profile( + environment_name: str, profile: str, ) -> None: - connection = _mock_mysql_probes( - monkeypatch, **{"select DOLT_VERSION()": product_version}, - ) + database_url = os.environ.get(environment_name) + if not database_url: + pytest.skip(f"{environment_name} is not configured") + active = active_connection(database_url) + assert active["profile"] == profile + assert active["claimed_supported"] is True + assert active["matched_claim"] is not None - with pytest.raises(UnsupportedOperationError, match=r">=2\.2\.2,<2\.3\.0"): - active_connection("mysql+pymysql://user@host/database") - assert connection.calls == [ - "select @@version", "select @@version_comment", "select DOLT_VERSION()", - ] +def _numeric_variables(): + return [{ + "ordinal": 1, "source_name": "score", "physical_name": "score", + "storage_kind": "numeric", + }] -def test_unknown_mysql_wire_product_fails_closed_without_dolt_probe(monkeypatch) -> None: - connection = _mock_mysql_probes( - monkeypatch, - **{ - "select @@version": "8.4.6", - "select @@version_comment": "Percona Server", - }, - ) +def test_dolt_numeric_policy_accepts_finite_or_null_and_rejects_nonfinite_input(): + preflight(DOLT, _numeric_variables(), rows=[{"score": 1.25}, {"score": None}]) + for value, classification in ( + (float("nan"), "nan"), + (float("inf"), "positive_infinity"), + (float("-inf"), "negative_infinity"), + ): + with pytest.raises(UnsupportedOperationError) as error: + preflight(DOLT, _numeric_variables(), rows=[{"score": value}]) + assert error.value.details["reason"] == "numeric_value_not_finite" + assert error.value.details["classification"] == classification + with pytest.raises(UnsupportedOperationError) as error: + preflight(DOLT, _numeric_variables(), rows=[{"score": True}]) + assert error.value.details["reason"] == "numeric_value_type" + + +def test_dolt_numeric_capability_declares_spss_missing_canonicalization(): + policy = capability_module.profile_declarations()["dolt"]["numeric_value_policy"] + assert policy == { + "sql_null": "canonical_system_missing", + "spss_nan": "canonicalize_to_sql_null_during_spss_decode", + "adapter_input": "finite_binary64_or_null", + "positive_infinity": "reject_before_mutation", + "negative_infinity": "reject_before_mutation", + "live_bit_exact_evidence": "pending_pinned_live_conformance", + } - with pytest.raises(UnsupportedOperationError, match="product is unknown"): - active_connection("mysql+pymysql://user@host/database") - assert "select DOLT_VERSION()" not in connection.calls - assert "select @@max_allowed_packet" not in connection.calls +class _MappingRows: + def __init__(self, *, one=None, rows=()): + self._one = one + self._rows = list(rows) + def mappings(self): + return self -def test_non_dolt_products_never_call_the_dolt_function(monkeypatch) -> None: - connection = _mock_mysql_probes( - monkeypatch, - **{ - "select @@version": "8.4.6", - "select @@version_comment": "MySQL Community Server", - }, - ) + def one(self): + return self._one - active = active_connection("mysql+pymysql://user@host/database") + def all(self): + return self._rows - assert active["profile"] == "mysql" - assert active["claimed_supported"] is True - assert "select DOLT_VERSION()" not in connection.calls +class _FakeDoltStateConnection: + dialect = SimpleNamespace(name="mysql") -def test_effective_profile_selects_dolt_without_changing_url_profile(monkeypatch) -> None: - active = { - "profile": "dolt", "server_version": "2.2.3", "claimed_supported": True, - "observed": {"max_allowed_packet": 1_073_741_824}, - } - monkeypatch.setattr(capabilities, "active_connection", lambda _url: active) - - profile, observed = effective_profile("mysql+pymysql://user@host/database") - - assert profile is not MYSQL - assert profile.name == "dolt" - assert profile.url_schemes == () - assert profile.max_physical_variables == 305 - assert profile.max_text_value_bytes == 65_504 - assert profile.max_row_bytes == 65_504 - assert observed is active - - -def test_dolt_declaration_labels_conservative_envelopes() -> None: - declaration = capabilities.profile_declarations()["dolt"] - - assert declaration["dialect"] == "mysql" - assert declaration["profile"] == "dolt" - assert declaration["engine"] == "dolt" - assert declaration["transport"] == "mysql_compatible" - assert declaration["claimed_server_versions"] == ["Dolt 2.2.x"] - assert declaration["claimed_version_range"] == { - "minimum_inclusive": "2.2.2", - "maximum_exclusive": "2.3.0", - } - assert declaration["ci_tested_server_versions"] == [ - "Dolt 2.2.2", "Dolt 2.2.3", - ] - assert declaration["exact_ci_tested_versions"] == ["2.2.2", "2.2.3"] - assert declaration["proposed_adapter_limits"]["maximum_physical_columns"] == 306 - assert declaration["proposed_adapter_limits"]["maximum_source_variables"] == 305 - assert declaration["proposed_adapter_limits"]["maximum_value_bytes"] == 65_504 - assert declaration["proposed_adapter_limits"]["maximum_row_bytes"] == 65_504 - assert declaration["theoretical_limits"]["maximum_value_bytes"] == 4_294_967_295 - assert declaration["observed_limits"]["minimum_observed_physical_columns"] == 307 - assert declaration["observed_limits"]["identifier_limit"]["value"] == 64 - assert declaration["observed_limits"]["rejected_identifier_bytes"] == 65 - assert set(declaration["proposed_adapter_limits"]) == { - "maximum_physical_columns", "maximum_source_variables", - "maximum_value_bytes", "maximum_row_bytes", - } - assert declaration["limit_bases"]["maximum_physical_columns"] == "proposed_adapter_envelope" - assert declaration["limit_bases"]["identifier_limit"] == "observed_exact_version" - assert declaration["limit_bases"]["maximum_value_bytes"] == "observed_exact_version" - assert declaration["limit_bases"]["maximum_statement_bytes"] == "active_connection_observation" - assert declaration["effective_limits"] is None - assert declaration["text_type"] == "LONGTEXT" - assert capabilities.profile_declarations()["mysql"]["text_type"] == "LONGTEXT" - assert capabilities.profile_declarations()["mariadb"]["text_type"] == "LONGTEXT" - assert declaration["ddl_atomic"] is False - assert declaration["failure_cleanup"] == "compensating_cleanup" - assert declaration["numeric_value_policy"] == { - "finite_binary64": "supported", - "nan": "rejected_before_ddl", - "positive_infinity": "rejected_before_ddl", - "negative_infinity": "rejected_before_ddl", - } - assert declaration["storage_evidence"]["binary64"]["maximum_finite_round_trip_exact"] is True - assert declaration["storage_evidence"]["binary64"]["source"] - assert declaration["storage_evidence"]["binary64"]["version"] == "2.2.2" - assert declaration["storage_evidence"]["text"]["observed_value_bytes"] == 65_504 - assert declaration["storage_evidence"]["text"]["source"] - assert declaration["storage_evidence"]["text"]["version"] == "2.2.2" - assert declaration["storage_evidence"]["text"]["unit"] == "bytes" - assert declaration["transformation_workflow"] == "unsupported" + def __init__(self, *, status=(), summary=()): + self.status = list(status) + self.summary = list(summary) + self.calls = [] + def __enter__(self): + return self -def test_dolt_uses_longtext_without_changing_mysql_storage() -> None: - dolt_table = Table("dolt_text", MetaData(), Column("value", string_type(DOLT))) - mysql_table = Table("mysql_text", MetaData(), Column("value", string_type(MYSQL))) + def __exit__(self, *_args): + return False + + def exec_driver_sql(self, statement): + self.calls.append(statement) + normalized = " ".join(statement.split()).casefold() + if normalized.startswith("select database()"): + return _MappingRows(one={ + "database_name": "synthetic_catalog", + "active_branch": "feature/synthetic", + "head_hash": "0123456789abcdef", + }) + if "from dolt_diff_summary" in normalized: + return _MappingRows(rows=self.summary) + if "from dolt_status" in normalized: + return _MappingRows(rows=self.status) + raise AssertionError(statement) + + +class _FakeDoltStateEngine: + def __init__(self, connection): + self.connection = connection - assert "VALUE LONGTEXT" in str(CreateTable(dolt_table).compile(dialect=mysql.dialect())).upper() - mysql_ddl = str(CreateTable(mysql_table).compile(dialect=mysql.dialect())).upper() - assert "VALUE TEXT" in mysql_ddl - assert "LONGTEXT" not in mysql_ddl + def connect(self): + return self.connection -def test_dolt_row_preflight_counts_utf8_values_against_adapter_envelope() -> None: - variables = [{ - "ordinal": 1, "source_name": "value", "physical_name": "value", - "storage_kind": "string", "string_width": 65_504, - }] - preflight(DOLT, variables, rows=[{"value": "x" * 65_504}]) +def test_dolt_state_capture_uses_only_exact_read_only_probes_and_fixed_shapes(): + connection = _FakeDoltStateConnection() + snapshot = wide._capture_dolt_state( + connection, profile_name="dolt", audit_relations={"operation_catalog"}, + ) - with pytest.raises(UnsupportedOperationError) as error: - preflight(DOLT, variables, rows=[{"value": "x" * 65_505}]) - assert error.value.details["reason"] == "text_value_limit" + assert snapshot["database"] == "synthetic_catalog" + assert snapshot["active_branch"] == "feature/synthetic" + assert snapshot["head"] == "0123456789abcdef" + assert len(snapshot["snapshot_sha256"]) == 64 + joined = " ".join(connection.calls).casefold() + assert "dolt_diff_summary('head', 'working')" in joined + assert "dolt_diff_summary('head', 'staged')" in joined + assert "dolt_diff_summary('staged', 'working')" in joined + assert not any( + keyword in joined + for keyword in ("dolt_add", "dolt_commit", "checkout", "reset") + ) -@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) -def test_dolt_numeric_preflight_rejects_nonfinite_values(value) -> None: - variables = [{ - "ordinal": 1, "source_name": "value", "physical_name": "value", - "storage_kind": "numeric", "string_width": None, - }] +def test_dolt_state_capture_fails_closed_on_shape_drift(): + connection = _FakeDoltStateConnection(status=[{"unexpected": "shape"}]) + with pytest.raises(UnsupportedOperationError, match="unexpected column shape"): + wide._capture_dolt_state( + connection, profile_name="dolt", audit_relations=set(), + ) - with pytest.raises(UnsupportedOperationError) as error: - preflight(DOLT, variables, rows=[{"value": value}]) - assert error.value.details == { - "reason": "nonfinite_numeric_value", - "row_ordinal": 1, - "source_name": "value", - } +def test_dolt_failure_boundary_allows_only_audit_catalog_delta(): + before = wide._capture_dolt_state( + _FakeDoltStateConnection(), + profile_name="dolt", audit_relations={"operation_catalog"}, + ) + after = wide._capture_dolt_state( + _FakeDoltStateConnection(status=[{ + "table_name": "operation_catalog", + "staged": False, + "status": "modified", + }]), + profile_name="dolt", audit_relations={"operation_catalog"}, + ) + evidence = wide._dolt_failure_boundary_evidence(before, after) + assert evidence["verified"] is True + changed = dict(after) + changed["active_branch"] = "other-branch" + evidence = wide._dolt_failure_boundary_evidence(before, changed) + assert evidence["verified"] is False + assert "active_branch_changed" in evidence["invariant_failures"] -def test_validate_identity_failure_happens_before_catalog_access(monkeypatch) -> None: - def fail_identity(_url): - raise UnsupportedOperationError("identity unavailable") - monkeypatch.setattr(wide, "effective_profile", fail_identity) +def test_public_dolt_state_snapshot_is_read_only(monkeypatch): + connection = _FakeDoltStateConnection() monkeypatch.setattr( - wide, "create_engine", - lambda _url: pytest.fail("database access continued after identity failure"), + wide, "active_connection", + lambda _url, **_kwargs: { + "profile": "dolt", + "server_version": "2.2.2", + "working_set_binding": { + "database": "synthetic_catalog", + "active_branch": "feature/synthetic", + }, + "claimed_supported": False, + }, ) + monkeypatch.setattr(wide, "create_engine", lambda _url: _FakeDoltStateEngine(connection)) - with pytest.raises(UnsupportedOperationError, match="identity unavailable"): - wide.validate_wide_dataset( - database_url="mysql+pymysql://user@host/database", dataset_id="fixture", - ) + result = wide.dolt_state_snapshot( + database_url="mysql+pymysql://user@host/synthetic_catalog", + ) + assert result["profile"] == "dolt" + assert result["server_version"] == "2.2.2" + assert result["read_only"] is True + assert result["operational_write_enabled"] is False -def test_export_identity_failure_happens_before_read_or_destination(monkeypatch, tmp_path) -> None: - destination = tmp_path / "blocked.sav" - def fail_identity(_url): - raise UnsupportedOperationError("identity unavailable") +def test_dolt_operational_profile_fails_closed_without_live_conformance(monkeypatch): + _fake_mysql_identity(monkeypatch) + with pytest.raises(UnsupportedOperationError, match="not bound"): + capability_module.effective_profile( + "mysql+pymysql://user@host/catalog" + ) - monkeypatch.setattr(sav, "effective_profile", fail_identity) - monkeypatch.setattr( - sav, "read_wide_dataset", - lambda **_kwargs: pytest.fail("catalog read continued after identity failure"), - ) - with pytest.raises(UnsupportedOperationError, match="identity unavailable"): - sav.export_sav_dataset( - database_url="mysql+pymysql://user@host/database", - dataset_id="fixture", destination=destination, - ) +@pytest.mark.parametrize( + ("row", "reason"), + [ + ({"name": None}, "string_value_type"), + ({"name": 123}, "string_value_type"), + ({}, "string_value_missing"), + ], +) +def test_string_preflight_requires_actual_string_values(row, reason): + variables = [{ + "ordinal": 1, "source_name": "name", "physical_name": "name", + "storage_kind": "string", "string_width": 8, + }] + with pytest.raises(UnsupportedOperationError) as error: + preflight(SQLITE, variables, rows=[row]) + assert error.value.details["reason"] == reason - assert not destination.exists() + +def test_string_preflight_accepts_empty_spss_missing_string(): + variables = [{ + "ordinal": 1, "source_name": "name", "physical_name": "name", + "storage_kind": "string", "string_width": 8, + }] + preflight(SQLITE, variables, rows=[{"name": ""}]) diff --git a/tests/test_sql_services.py b/tests/test_sql_services.py index e2e664f..62b337e 100755 --- a/tests/test_sql_services.py +++ b/tests/test_sql_services.py @@ -1,18 +1,14 @@ -"""Real-service conformance checks for PostgreSQL, MySQL, MariaDB, and Dolt.""" +"""Real-service conformance checks for PostgreSQL, MySQL, and MariaDB profiles.""" import os -from uuid import uuid4 import pandas as pd import pyspssio import pytest -from sqlalchemy import create_engine, inspect as inspect_database, text -from sqlalchemy.exc import DBAPIError +from sqlalchemy import create_engine, text import openstatspec -import openstatspec.sql.wide as wide -from openstatspec.core import UnsupportedOperationError -from conformance import compare_sav_semantics, write_supported_semantics_fixture +from conformance import write_supported_semantics_fixture pytestmark = pytest.mark.services @@ -37,26 +33,22 @@ def source_sav(tmp_path): @pytest.mark.parametrize( ("environment_name", "dataset_id"), - [("OPENSTATSPEC_POSTGRES_URL", "profile_pg"), ("OPENSTATSPEC_MYSQL_URL", "profile_mysql"), ("OPENSTATSPEC_MARIADB_URL", "profile_mariadb"), ("OPENSTATSPEC_DOLT_URL", "profile_dolt")], + [("OPENSTATSPEC_POSTGRES_URL", "profile_pg"), ("OPENSTATSPEC_MYSQL_URL", "profile_mysql"), ("OPENSTATSPEC_MARIADB_URL", "profile_mariadb")], ) def test_live_profile_import_validate_and_export(environment_name, dataset_id, source_sav, tmp_path): database_url = os.environ.get(environment_name) if not database_url: pytest.skip(f"{environment_name} is not configured") - runtime_dataset_id = f"{dataset_id}_{uuid4().hex[:8]}" - imported = openstatspec.import_sav( - source_sav, database_url=database_url, dataset_id=runtime_dataset_id, - ) + openstatspec.initialize_catalog(database_url=database_url) + imported = openstatspec.import_sav(source_sav, database_url=database_url, dataset_id=dataset_id) assert imported["case_count"] == 2 - assert openstatspec.validate( - database_url=database_url, dataset_id=runtime_dataset_id, - )["valid"] is True + assert openstatspec.validate(database_url=database_url, dataset_id=dataset_id)["valid"] is True engine = create_engine(database_url) with engine.connect() as connection: assert connection.execute(text(f"SELECT COUNT(*) FROM {imported['data_table']} ")).scalar_one() == 2 - assert connection.execute(text("SELECT COUNT(*) FROM variable_catalog WHERE dataset_id = :dataset_id"), {"dataset_id": runtime_dataset_id}).scalar_one() == 2 + assert connection.execute(text("SELECT COUNT(*) FROM variable_catalog WHERE dataset_id = :dataset_id"), {"dataset_id": dataset_id}).scalar_one() == 2 destination = tmp_path / f"{dataset_id}.sav" - exported = openstatspec.export_sav(database_url=database_url, dataset_id=runtime_dataset_id, destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS) + exported = openstatspec.export_sav(database_url=database_url, dataset_id=dataset_id, destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS) assert destination.exists() assert {diagnostic.code for diagnostic in exported.diagnostics} == set(_REQUIRED_ENGINE_LOSS) frame, metadata = pyspssio.read_sav(str(destination), convert_datetimes=False, include_user_missing=True) @@ -69,258 +61,21 @@ def test_live_profile_import_validate_and_export(environment_name, dataset_id, s @pytest.mark.parametrize( ("environment_name", "dataset_id"), - [("OPENSTATSPEC_POSTGRES_URL", "semantics_pg"), ("OPENSTATSPEC_MYSQL_URL", "semantics_mysql"), ("OPENSTATSPEC_MARIADB_URL", "semantics_mariadb"), ("OPENSTATSPEC_DOLT_URL", "semantics_dolt")], + [("OPENSTATSPEC_POSTGRES_URL", "semantics_pg"), ("OPENSTATSPEC_MYSQL_URL", "semantics_mysql"), ("OPENSTATSPEC_MARIADB_URL", "semantics_mariadb")], ) @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) def test_live_profile_preserves_supported_sav_semantics(environment_name, dataset_id, suffix, tmp_path): database_url = os.environ.get(environment_name) if not database_url: pytest.skip(f"{environment_name} is not configured") - runtime_dataset_id = f"{dataset_id}_{suffix[1:]}_{uuid4().hex[:8]}" - source = tmp_path / f"{runtime_dataset_id}{suffix}" - destination = tmp_path / f"{runtime_dataset_id}-roundtrip{suffix}" + source = tmp_path / f"{dataset_id}{suffix}" + destination = tmp_path / f"{dataset_id}-roundtrip{suffix}" write_supported_semantics_fixture(source) - imported = openstatspec.import_sav( - source, database_url=database_url, dataset_id=runtime_dataset_id, - ) + openstatspec.initialize_catalog(database_url=database_url) + imported = openstatspec.import_sav(source, database_url=database_url, dataset_id=f"{dataset_id}_{suffix[1:]}") assert imported["case_count"] == 4 - assert openstatspec.validate( - database_url=database_url, dataset_id=runtime_dataset_id, - )["valid"] is True - openstatspec.export_sav( - database_url=database_url, dataset_id=runtime_dataset_id, - destination=destination, allow_loss=_COMPAT_NAME_LOSS, - ) - assert compare_sav_semantics(source, destination) == {"equivalent": True, "differences": []} - -def test_live_dolt_conservative_source_width_envelope(tmp_path) -> None: - database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") - if not database_url: - pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - token = uuid4().hex[:8] - accepted_id = f"dolt_width_accepted_{token}" - rejected_id = f"dolt_width_rejected_{token}" - accepted_source = tmp_path / f"{accepted_id}.sav" - rejected_source = tmp_path / f"{rejected_id}.sav" - accepted_columns = [f"v{ordinal:03d}" for ordinal in range(1, 306)] - rejected_columns = [*accepted_columns, "v306"] - pyspssio.write_sav( - str(accepted_source), - pd.DataFrame([[float(ordinal) for ordinal in range(1, 306)]], - columns=accepted_columns), - ) - pyspssio.write_sav( - str(rejected_source), - pd.DataFrame([[float(ordinal) for ordinal in range(1, 307)]], - columns=rejected_columns), - ) - - imported = openstatspec.import_sav( - accepted_source, database_url=database_url, dataset_id=accepted_id, - ) - assert imported["case_count"] == 1 - assert openstatspec.validate( - database_url=database_url, dataset_id=accepted_id, - )["variable_count"] == 305 - - with pytest.raises(UnsupportedOperationError, match="Target capability exceeded"): - openstatspec.import_sav( - rejected_source, database_url=database_url, dataset_id=rejected_id, - ) - - engine = create_engine(database_url) - with engine.connect() as connection: - assert connection.execute(text( - "select count(*) from dataset where dataset_name = :name" - ), {"name": rejected_id}).scalar_one() == 0 - assert connection.execute(text( - "select count(*) from dataset_catalog where dataset_id = :name" - ), {"name": rejected_id}).scalar_one() == 0 - mirror_event = connection.execute(text(""" - select f.dataset_id, f.direction, f.severity, f.code - from fidelity_event_catalog f - join operation_catalog o on o.operation_id = f.operation_id - where o.source = :source - """), {"source": rejected_source.name}).mappings().one() - normative_event = connection.execute(text(""" - select f.dataset_id, f.direction, f.severity, f.event_code - from fidelity_event f - where f.source_item = :source - """), {"source": rejected_source.name}).mappings().one() - assert tuple(mirror_event.values()) == ( - None, "import", "error", "target_capability_exceeded", - ) - assert tuple(normative_event.values()) == ( - None, "import", "error", "target_capability_exceeded", - ) - assert f"data_{rejected_id}" not in inspect_database(engine).get_table_names() - - -def test_live_dolt_post_ddl_fault_has_complete_compensating_cleanup( - tmp_path, monkeypatch, -) -> None: - database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") - if not database_url: - pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - dataset_id = f"dolt_cleanup_{uuid4().hex[:8]}" - source = tmp_path / f"{dataset_id}.sav" - pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) - real_store = wide.store_normative_dataset - - def fail_after_normative_write(*args, **kwargs): - real_store(*args, **kwargs) - raise RuntimeError("injected Dolt post-DDL fault") - - monkeypatch.setattr(wide, "store_normative_dataset", fail_after_normative_write) - - with pytest.raises(RuntimeError, match="injected Dolt post-DDL fault"): - openstatspec.import_sav( - source, database_url=database_url, dataset_id=dataset_id, - ) - - engine = create_engine(database_url) - with engine.connect() as connection: - assert connection.execute(text( - "select count(*) from dataset where dataset_name = :name" - ), {"name": dataset_id}).scalar_one() == 0 - assert connection.execute(text( - "select count(*) from dataset_catalog where dataset_id = :name" - ), {"name": dataset_id}).scalar_one() == 0 - assert connection.execute(text( - "select status, dataset_id from operation_catalog where source = :source" - ), {"source": source.name}).one() == ("failed", None) - assert connection.execute(text(""" - select f.dataset_id, f.direction, f.severity, f.code - from fidelity_event_catalog f - join operation_catalog o on o.operation_id = f.operation_id - where o.source = :source - """), {"source": source.name}).one() == ( - None, "import", "error", "import_failed", - ) - assert f"data_{dataset_id}" not in inspect_database(engine).get_table_names() - - -def test_live_dolt_adapter_value_boundary_is_atomic() -> None: - database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") - if not database_url: - pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - token = uuid4().hex[:8] - accepted_id = f"dolt_value_accepted_{token}" - rejected_id = f"dolt_value_rejected_{token}" - accepted_value = "é" * 32_752 - rejected_value = accepted_value + "x" - assert len(accepted_value.encode("utf-8")) == 65_504 - assert len(rejected_value.encode("utf-8")) == 65_505 - variables = [{ - "ordinal": 1, "source_name": "value", "physical_name": "value", - "storage_kind": "string", "string_width": 65_504, "label": "", - "format": "A65504", "measure": "nominal", "alignment": "left", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - - imported = wide.create_wide_dataset( - database_url=database_url, dataset_id=accepted_id, - source_name="accepted.sav", source_format="SAV", - rows=[{"value": accepted_value}], variables=variables, - ) - assert imported["case_count"] == 1 - - with pytest.raises(UnsupportedOperationError) as caught: - wide.create_wide_dataset( - database_url=database_url, dataset_id=rejected_id, - source_name="rejected.sav", source_format="SAV", - rows=[{"value": rejected_value}], variables=variables, - ) - assert caught.value.details["reason"] == "text_value_limit" - - engine = create_engine(database_url) - accepted_table = wide.data_table_name(accepted_id) - rejected_table = wide.data_table_name(rejected_id) - quote = engine.dialect.identifier_preparer.quote - with engine.connect() as connection: - assert connection.execute(text( - f"SELECT OCTET_LENGTH(value) FROM {quote(accepted_table)}" - )).scalar_one() == 65_504 - assert connection.execute(text( - "SELECT COUNT(*) FROM dataset_catalog WHERE dataset_id = :dataset_id" - ), {"dataset_id": rejected_id}).scalar_one() == 0 - assert connection.execute(text( - "SELECT COUNT(*) FROM dataset WHERE dataset_name = :dataset_id" - ), {"dataset_id": rejected_id}).scalar_one() == 0 - assert rejected_table not in inspect_database(engine).get_table_names() - engine.dispose() - - -def test_live_dolt_published_storage_and_identifier_evidence() -> None: - database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") - if not database_url: - pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - engine = create_engine(database_url) - token = uuid4().hex[:8] - identifier_64 = f"evidence_{token}_" + "i" * (64 - len(f"evidence_{token}_")) - identifier_65 = identifier_64 + "i" - assert len(identifier_64.encode("utf-8")) == 64 - assert len(identifier_65.encode("utf-8")) == 65 - storage_table = f"evidence_storage_{token}" - columns_306_table = f"evidence_columns_306_{token}" - columns_307_table = f"evidence_columns_307_{token}" - quote = engine.dialect.identifier_preparer.quote - - try: - with engine.begin() as connection: - connection.execute(text( - f"CREATE TABLE {quote(identifier_64)} (value INTEGER)" - )) - assert identifier_64 in inspect_database(engine).get_table_names() - with pytest.raises(DBAPIError): - with engine.begin() as connection: - connection.execute(text( - f"CREATE TABLE {quote(identifier_65)} (value INTEGER)" - )) - - maximum_finite = float.fromhex("0x1.fffffffffffffp+1023") - utf8_value = "é" * 32_752 - assert len(utf8_value.encode("utf-8")) == 65_504 - with engine.begin() as connection: - connection.execute(text( - f"CREATE TABLE {quote(storage_table)} " - "(binary64_value DOUBLE NOT NULL, text_value LONGTEXT NOT NULL)" - )) - connection.execute( - text( - f"INSERT INTO {quote(storage_table)} " - "(binary64_value, text_value) VALUES (:binary64, :text_value)" - ), - {"binary64": maximum_finite, "text_value": utf8_value}, - ) - with engine.connect() as connection: - row = connection.execute(text( - f"SELECT binary64_value, text_value, OCTET_LENGTH(text_value) " - f"FROM {quote(storage_table)}" - )).one() - assert row[0] == maximum_finite - assert row[1] == utf8_value - assert row[2] == 65_504 - - for column_count, table_name in ( - (306, columns_306_table), (307, columns_307_table), - ): - columns = ", ".join( - f"{quote(f'c{ordinal:03d}')} INTEGER" - for ordinal in range(1, column_count + 1) - ) - with engine.begin() as connection: - connection.execute(text( - f"CREATE TABLE {quote(table_name)} ({columns})" - )) - assert len(inspect_database(engine).get_columns(table_name)) == column_count - finally: - with engine.begin() as connection: - for table_name in ( - identifier_64, identifier_65, storage_table, - columns_306_table, columns_307_table, - ): - connection.execute(text( - f"DROP TABLE IF EXISTS {quote(table_name)}" - )) - engine.dispose() \ No newline at end of file + assert openstatspec.validate(database_url=database_url, dataset_id=f"{dataset_id}_{suffix[1:]}")["valid"] is True + openstatspec.export_sav(database_url=database_url, dataset_id=f"{dataset_id}_{suffix[1:]}", destination=destination, allow_loss=_COMPAT_NAME_LOSS) + comparison = openstatspec.compare_sav_semantics(source, destination) + assert comparison["equivalent"] is True + assert comparison["differences"] == [] diff --git a/tests/test_sql_workflow.py b/tests/test_sql_workflow.py index a418a70..0b8a7c6 100644 --- a/tests/test_sql_workflow.py +++ b/tests/test_sql_workflow.py @@ -73,6 +73,7 @@ def _variables(): def catalog(tmp_path): path = tmp_path / "workflow.sqlite" url = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=url) create_wide_dataset( database_url=url, dataset_id="source", source_name="source.sav", source_format="SAV", source_sha256="a" * 64, diff --git a/tests/test_vls_compatible_names.py b/tests/test_vls_compatible_names.py index ff1f48c..c2f7056 100644 --- a/tests/test_vls_compatible_names.py +++ b/tests/test_vls_compatible_names.py @@ -65,6 +65,7 @@ def test_vls_custom_compatible_name_round_trips_as_one_variable(tmp_path, suffix raw_dictionary.write_compatible_names( source, {_SOURCE_NAME: _COMPATIBLE_NAME}, encoding="UTF-8", ) + openstatspec.initialize_catalog(database_url=database) imported = openstatspec.import_sav( source, database_url=database, dataset_id=f"vls-{suffix[1:]}", @@ -119,6 +120,7 @@ def test_malformed_vls_export_removes_output_and_records_no_success(tmp_path, mo raw_dictionary.write_compatible_names( source, {_SOURCE_NAME: _COMPATIBLE_NAME}, encoding="UTF-8", ) + openstatspec.initialize_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="failed-vls") real_write = sav_module.write_compatible_names