diff --git a/CHANGELOG.md b/CHANGELOG.md index 802ecf1..186b25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this reference implementation are documented here. ## Unreleased Planned adapter release: `0.5.0`, after lifecycle integration and final specification conformance. +### Fixed + +- Removed the temporary `*_catalog` compatibility schema and made SAV/ZSAV + import, validation, export, fidelity reporting, and in-place metadata edits + use the normative UUID-keyed OpenStatSpec catalog exclusively. +- Existing databases that contain former compatibility relations must be + remediated manually before further operations. +- Imports now reject dataset names that collide with normative UUID identifiers, + cleanup drops only a physical table actually created by the failing import, + and non-preserved SPSS compatible variable names are reported as an explicit + loss requiring export consent. + ### Added @@ -24,8 +36,8 @@ Planned adapter release: `0.5.0`, after lifecycle integration and final specific - Bumped the canonical transformation-plan and SPSS frontend contracts to `v0.2`; canonical JSON and hashes include every sequential operation. -- In-place apply now records variable label, value labels, `F` print/write - format, and measurement level in both normative and compatibility catalogs. +- In-place apply records variable label, value labels, `F` print/write + format, and measurement level only in the normative catalog. - Dolt still requires an exact branch, exact HEAD, and clean working set; successful apply leaves an inspectable diff and never calls `DOLT_COMMIT`. - Dolt declaration validation is implemented by the Python adapter. The diff --git a/README.md b/README.md index 5982760..c0d0ca7 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ For each supported import, one source dataset becomes one dedicated wide SQL table. Cases are rows and source variables are physical SQL columns. The singular UUID-keyed tables from the specification (`dataset`, `variable`, `operation`, `fidelity_event`, and related metadata tables) are the public -catalog contract. Historical `*_catalog` tables are an internal compatibility -layer for the current exporter and are not the standard database interface. -The adapter does not reshape data, create EAV -or long-form tables, or harmonize studies or waves. +catalog contract and the only catalog representation created or read by the +adapter. Databases created with the former `*_catalog` compatibility schema +must be remediated to the normative schema before use. The adapter does not +reshape data, create EAV or long-form tables, or harmonize studies or waves. Unsupported source features, SQL targets, or export paths fail explicitly. There is no silent truncation, type conversion, metadata loss, or partial diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 915a309..f6fa180 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -70,8 +70,8 @@ The gate must prove that: - the exact bounded `COMPUTE`/`IF` program compiles to all seven ordered operations without dropping `FORMATS`, `VARIABLE LEVEL`, or `EXECUTE`; - boolean data results match the equivalent expression and the target's label, - 0/1 value labels, `F1.0` print/write format, and nominal level exist in both - normative and compatibility catalogs; + 0/1 value labels, `F1.0` print/write format, and nominal level exist in the + normative catalog; - injected schema, data, catalog, and audit failures leave no partial apply; - compensation tracks only newly created targets and never drops or rewrites a pre-existing target; diff --git a/docs/transformations.md b/docs/transformations.md index 037aecd..90702ec 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -141,15 +141,15 @@ schema/table identity. It creates no derived dataset, output table, full-table copy, staging table, snapshot, rollback artifact, or recovery/history layer. Assignments and recodes use ordered `UPDATE` statements; later operations see earlier results. Label, value-label, format, and measurement-level operations -update both the normative and compatibility catalogs. +update the normative catalog. A numeric create target is supported atomically on SQLite and PostgreSQL. MySQL, MariaDB, and Dolt reject `target_mode=create` before mutation. On those profiles a separate versioned stage must first provision the nullable numeric -physical column and both catalog representations; the transformation executor -then sees a pre-existing target and performs no schema DDL. -The public operation reports success only after physical data, both metadata -representations, and the compact audit row are mutually complete. +physical column and normative variable row; the transformation executor then +sees a pre-existing target and performs no schema DDL. +The public operation reports success only after physical data, normative +metadata, and the compact audit row are mutually complete. Before Dolt mutation, the executor verifies the expected branch and `HEAD` and requires clean `dolt_status`. Success changes the same working set without diff --git a/src/openstatspec/__init__.py b/src/openstatspec/__init__.py index db64588..eb73a94 100644 --- a/src/openstatspec/__init__.py +++ b/src/openstatspec/__init__.py @@ -15,7 +15,8 @@ from .sql.workflow import TransformationError from .transform import ( AssignOperation, BooleanExpression, ComparisonExpression, - ConditionalAssignOperation, ExecuteOperation, Operand, PredicateExpression, + ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation, + ExecuteOperation, Operand, PredicateExpression, RecodeMatch, RecodeOperation, RecodeResult, RecodeRule, ReplaceValueLabelsOperation, SetFormatOperation, SetMeasurementLevelOperation, SetVariableLabelOperation, @@ -26,7 +27,7 @@ __all__ = [ "AssignOperation", "BooleanExpression", "ComparisonExpression", "ConditionalAssignOperation", "ExecuteOperation", "Operand", - "PredicateExpression", + "PredicateExpression", "CreateVariableOperation", "DeleteVariableOperation", "CapabilityDeclaration", "DoltConformanceSource", "LossReport", "SpssFrontendCompilation", "TransformationError", "TransformationFrontendError", diff --git a/src/openstatspec/api.py b/src/openstatspec/api.py index 656b162..b7bf584 100644 --- a/src/openstatspec/api.py +++ b/src/openstatspec/api.py @@ -84,7 +84,7 @@ def capability_matrix( "multiple_response_sets": "supported", "variable_alignment": "supported", "variable_sets": "supported", - "compatible_variable_names": "supported", + "compatible_variable_names": "requires-explicit-loss-consent", "custom_attributes": { "scalar_values": "supported", "ordered_value_arrays": "supported", diff --git a/src/openstatspec/frontends/spss/__init__.py b/src/openstatspec/frontends/spss/__init__.py index 18a7891..58ed87e 100644 --- a/src/openstatspec/frontends/spss/__init__.py +++ b/src/openstatspec/frontends/spss/__init__.py @@ -1,7 +1,12 @@ """SPSS syntax frontend for canonical OpenStatSpec transformation plans.""" from .binding import bind_spss_syntax -from .compiler import SpssFrontendCompilation, compile_spss_syntax +from .compiler import ( + SPSS_FRONTEND_CONTRACT, + SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT, + SpssFrontendCompilation, + compile_spss_syntax, +) from .syntax import ( SpssSyntaxProgram, normalize_spss_source, @@ -11,10 +16,9 @@ ) -SPSS_FRONTEND_CONTRACT = "openstatspec-spss-syntax-frontend-v0.2" - __all__ = [ "SPSS_FRONTEND_CONTRACT", + "SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT", "SpssFrontendCompilation", "SpssSyntaxProgram", "bind_spss_syntax", diff --git a/src/openstatspec/frontends/spss/binding.py b/src/openstatspec/frontends/spss/binding.py index 0d9f275..cd37d12 100644 --- a/src/openstatspec/frontends/spss/binding.py +++ b/src/openstatspec/frontends/spss/binding.py @@ -8,10 +8,12 @@ from ...transform.errors import SourceSpan, frontend_error from ...transform.plan import ( AssignOperation, BooleanExpression, ComparisonExpression, - ConditionalAssignOperation, ExecuteOperation, Operand, PlanOperation, + ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation, + ExecuteOperation, Operand, PlanOperation, PredicateExpression, RecodeMatch, RecodeOperation, RecodeResult, RecodeRule, ReplaceValueLabelsOperation, SetFormatOperation, SetMeasurementLevelOperation, SetVariableLabelOperation, + TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT, TRANSFORMATION_PLAN_V1_CONTRACT, TransformationPlan, TypedValue, ValueLabel, ) @@ -21,10 +23,11 @@ from ...transform.validation import bind_transformation_plan from .syntax import ( BooleanSyntax, ComparisonSyntax, ComputeCommandSyntax, ExecuteCommandSyntax, - FormatsCommandSyntax, IfCommandSyntax, OperandSyntax, PredicateSyntax, + DeleteVariablesCommandSyntax, FormatsCommandSyntax, IfCommandSyntax, + OperandSyntax, PredicateSyntax, RecodeCommandSyntax, RecodeMatchSyntax, RecodeResultSyntax, SpssSyntaxProgram, SyntaxLiteral, ValueLabelsCommandSyntax, - VariableLabelsCommandSyntax, VariableLevelCommandSyntax, + StringCommandSyntax, VariableLabelsCommandSyntax, VariableLevelCommandSyntax, ) @@ -290,8 +293,6 @@ def _bind_recode( # replace intentionally preserves the existing variable metadata. A later # VALUE LABELS command replaces value labels explicitly. return operations, spans - - def bind_spss_syntax( program: SpssSyntaxProgram, schema: VariableSchema, *, input_alias: str = "parent", ) -> BoundTransformation: @@ -305,6 +306,50 @@ def bind_spss_syntax( operations: list[PlanOperation] = [] spans: list[SourceSpan] = [] for command in program.commands: + if isinstance(command, StringCommandSyntax): + for variable_token in command.variables: + if variable_token.text.startswith("__"): + raise frontend_error( + "reserved_target_name", + f"Target name {variable_token.text!r} is reserved.", + span=variable_token.span, + target=variable_token.text, + ) + if any( + variable.name.casefold() == variable_token.text.casefold() + for variable in variables + ): + raise frontend_error( + "target_already_exists", + f"Target name {variable_token.text!r} already exists.", + span=variable_token.span, + target=variable_token.text, + ) + operations.append(CreateVariableOperation( + variable_token.text, "string", command.width, + )) + spans.append(command.span) + variables.append(VariableDefinition( + variable_token.text, "string", + declared_string_width=command.width, + )) + continue + if isinstance(command, DeleteVariablesCommandSyntax): + for variable_token in command.variables: + index, variable = _resolve( + variables, variable_token.text, variable_token.span, + ) + if len(variables) == 1: + raise frontend_error( + "cannot_delete_last_variable", + "DELETE VARIABLES cannot remove the final dataset variable.", + span=variable_token.span, + variable=variable.name, + ) + operations.append(DeleteVariableOperation(variable.name)) + spans.append(command.span) + del variables[index] + continue if isinstance(command, RecodeCommandSyntax): recodes, recode_spans = _bind_recode(command, variables) operations.extend(recodes) @@ -447,10 +492,15 @@ def bind_spss_syntax( v01_types = ( RecodeOperation, SetVariableLabelOperation, ReplaceValueLabelsOperation, ) + schema_change_types = (CreateVariableOperation, DeleteVariableOperation) contract = ( - TRANSFORMATION_PLAN_V1_CONTRACT - if all(isinstance(operation, v01_types) for operation in operations) - else "openstatspec-transformation-plan-v0.2" + TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT + if any(isinstance(operation, schema_change_types) for operation in operations) + else ( + TRANSFORMATION_PLAN_V1_CONTRACT + if all(isinstance(operation, v01_types) for operation in operations) + else "openstatspec-transformation-plan-v0.2" + ) ) plan = TransformationPlan( tuple(operations), contract=contract, input_alias=input_alias, diff --git a/src/openstatspec/frontends/spss/compiler.py b/src/openstatspec/frontends/spss/compiler.py index 4e003c0..ecd05f2 100644 --- a/src/openstatspec/frontends/spss/compiler.py +++ b/src/openstatspec/frontends/spss/compiler.py @@ -4,7 +4,10 @@ from dataclasses import dataclass -from ...transform.plan import TransformationPlan +from ...transform.plan import ( + TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT, + TransformationPlan, +) from ...transform.schema import BoundTransformation, VariableSchema from .binding import bind_spss_syntax from .syntax import ( @@ -14,6 +17,10 @@ ) +SPSS_FRONTEND_CONTRACT = "openstatspec-spss-syntax-frontend-v0.2" +SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT = "openstatspec-spss-syntax-frontend-v0.3" + + @dataclass(frozen=True) class SpssFrontendCompilation: """One source artifact and its fully bound canonical plan.""" @@ -30,6 +37,12 @@ def plan(self) -> TransformationPlan: def plan_hash(self) -> str: return self.plan.sha256() + @property + def frontend_contract(self) -> str: + if self.plan.contract == TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT: + return SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT + return SPSS_FRONTEND_CONTRACT + def compile_spss_syntax( source: str, diff --git a/src/openstatspec/frontends/spss/execution.py b/src/openstatspec/frontends/spss/execution.py index 1d947da..108bed0 100644 --- a/src/openstatspec/frontends/spss/execution.py +++ b/src/openstatspec/frontends/spss/execution.py @@ -10,7 +10,6 @@ _run_in_place_submission, load_transformation_schema, ) -from . import SPSS_FRONTEND_CONTRACT from .compiler import compile_spss_syntax @@ -37,7 +36,7 @@ def prepare(connection: Any, live_dataset_id: str) -> InPlacePlanSubmission: plan=compilation.plan, source_kind="spss_syntax", source_hash=compilation.source_hash, - frontend_contract=SPSS_FRONTEND_CONTRACT, + frontend_contract=compilation.frontend_contract, ) return _run_in_place_submission( diff --git a/src/openstatspec/frontends/spss/syntax.py b/src/openstatspec/frontends/spss/syntax.py index 04cb179..ed78f93 100644 --- a/src/openstatspec/frontends/spss/syntax.py +++ b/src/openstatspec/frontends/spss/syntax.py @@ -137,6 +137,19 @@ class ExecuteCommandSyntax: span: SourceSpan +@dataclass(frozen=True) +class StringCommandSyntax: + variables: tuple[Token, ...] + width: int + span: SourceSpan + + +@dataclass(frozen=True) +class DeleteVariablesCommandSyntax: + variables: tuple[Token, ...] + span: SourceSpan + + @dataclass(frozen=True) class VariableLabelSyntax: @@ -175,6 +188,7 @@ class ValueLabelsCommandSyntax: RecodeCommandSyntax | ComputeCommandSyntax | IfCommandSyntax | VariableLabelsCommandSyntax | ValueLabelsCommandSyntax | FormatsCommandSyntax | VariableLevelCommandSyntax | ExecuteCommandSyntax + | StringCommandSyntax | DeleteVariablesCommandSyntax ) @@ -549,6 +563,43 @@ def execute(self, start: Token) -> ExecuteCommandSyntax: end = self.expects("period", "Expected '.' after EXECUTE.") return ExecuteCommandSyntax(_joined_span(start.span, end.span)) + def string(self, start: Token) -> StringCommandSyntax: + variables = self.variable_list( + stop_kinds=frozenset({"left_paren", "period", "eof"}), + ) + self.expects("left_paren", "Expected '(' before a STRING width.") + width_token = self.expects( + "identifier", "STRING requires a width such as A20.", + ) + match = re.fullmatch(r"A([0-9]+)", width_token.text, re.IGNORECASE) + if match is None: + raise frontend_error( + "spss_syntax_error", + "STRING supports only character widths such as A20.", + span=width_token.span, + ) + width = int(match.group(1)) + if not 1 <= width <= 32767: + raise frontend_error( + "invalid_string_width", + "STRING width must be between 1 and 32767.", + span=width_token.span, + width=width, + ) + self.expects("right_paren", "Expected ')' after a STRING width.") + end = self.expects("period", "Expected '.' after STRING.") + return StringCommandSyntax( + variables, width, _joined_span(start.span, end.span), + ) + + def delete_variables(self, start: Token) -> DeleteVariablesCommandSyntax: + self.expects_keyword("VARIABLES") + variables = self.variable_list(stop_kinds=frozenset({"period", "eof"})) + end = self.expects("period", "Expected '.' after DELETE VARIABLES.") + return DeleteVariablesCommandSyntax( + variables, _joined_span(start.span, end.span), + ) + def recode_result(self) -> RecodeResultSyntax: if (token := self.accepts_keyword("SYSMIS")) is not None: @@ -692,6 +743,10 @@ def parse(self) -> SpssSyntaxProgram: commands.append(self.formats(start)) elif command == "execute": commands.append(self.execute(start)) + elif command == "string": + commands.append(self.string(start)) + elif command == "delete": + commands.append(self.delete_variables(start)) elif command == "variable": if self.current.kind == "identifier" and self.current.text.casefold() == "level": commands.append(self.variable_level(start)) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 9128782..111d2d3 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -684,6 +684,7 @@ def export_sav_dataset( ) persisted_events = read_fidelity_events( database_url=database_url, dataset_id=dataset_id, + direction="import", dolt_conformance_source=dolt_conformance_source, ) if legacy_locale is not None: @@ -1147,6 +1148,21 @@ def _engine_loss_report(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]: "detail": "pyspssio could not inspect source variable sets; they cannot be preserved silently.", "details": {"engine_error": metadata.get("_var_sets_error", "unknown")}, } + compatible_names = { + str(source_name): str(compatible_name) + for source_name, compatible_name + in dict(metadata.get("var_compat_names") or {}).items() + if compatible_name + and str(source_name).casefold() != str(compatible_name).casefold() + } + if compatible_names: + events["compatible-variable-names-not-preserved"] = { + "code": "compatible-variable-names-not-preserved", + "detail": ( + "The normative catalog does not store SPSS compatible variable names." + ), + "details": {"variable_names": sorted(compatible_names)}, + } if _is_non_utf8_encoding(metadata.get("encoding")): events["source-encoding-not-preserved"] = { "code": "source-encoding-not-preserved", diff --git a/src/openstatspec/sql/catalog_verification.py b/src/openstatspec/sql/catalog_verification.py new file mode 100644 index 0000000..381ce28 --- /dev/null +++ b/src/openstatspec/sql/catalog_verification.py @@ -0,0 +1,283 @@ +"""Strict ownership and structural verification for OpenStatSpec SQL catalogs.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +from sqlalchemy import MetaData, Table, inspect, select + +from ..core import UnsupportedOperationError +from .normative import CATALOG_CONTRACT_ID, CATALOG_SCHEMA_VERSION + + +def _normalized_sql_type(inspector: Any, value: Any) -> str: + compiled = " ".join( + str(value.compile(dialect=inspector.bind.dialect)).strip().upper().split() + ) + if inspector.bind.dialect.name in {"mysql", "mariadb"}: + if compiled in {"BOOL", "BOOLEAN", "TINYINT(1)"}: + return "BOOLEAN/TINYINT(1)" + compiled = re.sub( + r"\b(TINYINT|SMALLINT|MEDIUMINT|INTEGER|INT|BIGINT)\(\d+\)", + r"\1", + compiled, + ) + return compiled + + +def _normalized_default(value: Any) -> str | None: + if value is None: + return None + result = " ".join(str(value).strip().split()) + while result.startswith("(") and result.endswith(")"): + result = result[1:-1].strip() + return result + + +def _expected_unique_constraints(table: Table) -> set[tuple[str, ...]]: + return { + tuple(column.name for column in constraint.columns) + for constraint in table.constraints + if getattr(constraint, "__visit_name__", "") == "unique_constraint" + } + + +def _actual_unique_constraints(inspector: Any, table_name: str) -> set[tuple[str, ...]]: + constraints = { + tuple(str(name) for name in item.get("column_names") or ()) + for item in inspector.get_unique_constraints(table_name) + } + constraints.update( + tuple(str(name) for name in item.get("column_names") or ()) + for item in inspector.get_indexes(table_name) + if item.get("unique") + ) + constraints.discard(()) + return constraints + + +def _expected_foreign_keys(table: Table) -> set[tuple[Any, ...]]: + return { + ( + tuple(column.name for column in constraint.columns), + next(iter(constraint.elements)).column.table.name, + tuple(element.column.name for element in constraint.elements), + ) + for constraint in table.foreign_key_constraints + } + + +def _actual_foreign_keys(inspector: Any, table_name: str) -> set[tuple[Any, ...]]: + return { + ( + tuple(str(name) for name in item.get("constrained_columns") or ()), + str(item.get("referred_table") or ""), + tuple(str(name) for name in item.get("referred_columns") or ()), + ) + for item in inspector.get_foreign_keys(table_name) + } + + +def _normalized_check_sql(value: Any) -> str: + result = " ".join(str(value).strip().casefold().split()) + result = result.replace("`", "").replace('"', "") + result = re.sub(r"\[([^]]+)\]", r"\1", result) + result = re.sub(r"(? set[str]: + return { + _normalized_check_sql(constraint.sqltext) + for constraint in table.constraints + if getattr(constraint, "__visit_name__", "") + == "table_or_column_check_constraint" + } + + +def _actual_check_constraints(inspector: Any, table_name: str) -> set[str]: + return { + _normalized_check_sql(item.get("sqltext") or "") + for item in inspector.get_check_constraints(table_name) + } + + +def _table_shape_valid( + inspector: Any, + table: Table, + *, + allowed_missing: Iterable[str] = (), +) -> bool: + actual = { + str(column["name"]): column + for column in inspector.get_columns(table.name) + } + expected = {column.name: column for column in table.columns} + missing = set(expected) - set(actual) + if set(actual) - set(expected) or missing - set(allowed_missing): + return False + for name in set(actual) & set(expected): + expected_column = expected[name] + actual_column = actual[name] + if ( + _normalized_sql_type(inspector, expected_column.type) + != _normalized_sql_type(inspector, actual_column["type"]) + or bool(actual_column.get("nullable")) != bool(expected_column.nullable) + ): + return False + expected_default = _normalized_default( + expected_column.server_default.arg + if expected_column.server_default is not None else None + ) + if _normalized_default(actual_column.get("default")) != expected_default: + return False + if actual_column.get("identity") is not None or actual_column.get("computed") is not None: + return False + expected_pk = tuple(column.name for column in table.primary_key.columns) + actual_pk = tuple( + str(name) + for name in inspector.get_pk_constraint(table.name).get("constrained_columns") or () + ) + return ( + actual_pk == expected_pk + and _actual_unique_constraints(inspector, table.name) + == _expected_unique_constraints(table) + and _actual_foreign_keys(inspector, table.name) + == _expected_foreign_keys(table) + and _actual_check_constraints(inspector, table.name) + == _expected_check_constraints(table) + ) + + +def _reject(relations: Iterable[str] = ()) -> None: + suffix = f": {', '.join(sorted(relations))}" if relations else "" + raise UnsupportedOperationError( + "The selected database catalog contains foreign, obsolete, or " + f"structurally incompatible relations{suffix}. Remove them manually " + "before continuing." + ) + + +def verify_catalog_relations( + connection: Any, + normative: Any, + *, + allowed_migrations: Mapping[str, set[str]] | None = None, +) -> None: + """Accept only exact normative and validated optional OpenStatSpec profiles.""" + inspector = inspect(connection) + existing_tables = set(inspector.get_table_names()) + existing_views = set(inspector.get_view_names()) + normative_tables = {table.name: table for table in normative.all()} + if not set(normative_tables) <= existing_tables: + _reject(set(normative_tables) - existing_tables) + if any( + not _table_shape_valid(inspector, table) + for table in normative_tables.values() + ): + _reject(normative_tables) + + identities = connection.execute(select(normative.catalog_identity)).mappings().all() + if len(identities) != 1 or ( + identities[0]["catalog_identity_key"] != 1 + or identities[0]["contract_id"] != CATALOG_CONTRACT_ID + or identities[0]["schema_version"] != CATALOG_SCHEMA_VERSION + ): + _reject({normative.catalog_identity.name}) + + owned_tables = set(normative_tables) + owned_views: set[str] = set() + owned_tables.update( + str(name) + for name in connection.execute( + select(normative.dataset.c.physical_table_name) + ).scalars() + if name + ) + + from .workflow import ( + PROFILE_ID, PROFILE_SCHEMA_VERSION, TransformationError, + _assert_trigger_definitions, _derived_trigger_sql, + _validate_workflow_schema, workflow_catalog, + ) + + workflow = workflow_catalog(MetaData()) + workflow_tables = {table.name: table for table in workflow.all()} + workflow_present = set(workflow_tables) & existing_tables + if workflow_present: + if set(workflow_tables) - existing_tables or any( + not _table_shape_valid(inspector, table) + for table in workflow_tables.values() + ): + _reject(workflow_present) + try: + _validate_workflow_schema(connection, workflow) + except TransformationError: + _reject(workflow_present) + identity_rows = connection.execute( + select(workflow.transformation_profile_identity) + ).mappings().all() + if len(identity_rows) != 1 or ( + identity_rows[0]["profile_identity_key"] != 1 + or identity_rows[0]["contract_id"] != PROFILE_ID + or identity_rows[0]["schema_version"] != PROFILE_SCHEMA_VERSION + or identity_rows[0]["core_contract_id"] != CATALOG_CONTRACT_ID + ): + _reject({workflow.transformation_profile_identity.name}) + owned_tables.update(workflow_tables) + physically_removed = set(connection.execute(select( + workflow.derived_dataset_disposition_event.c.derived_dataset_id + ).where( + workflow.derived_dataset_disposition_event.c.event_kind + == "physical_removed" + )).scalars()) + for row in connection.execute(select( + workflow.derived_dataset.c.derived_dataset_id, + workflow.derived_dataset.c.physical_relation_name, + workflow.derived_dataset.c.output_mode, + )).mappings(): + if row["derived_dataset_id"] in physically_removed: + continue + name = str(row["physical_relation_name"]) + if row["output_mode"] == "view": + owned_views.add(name) + else: + owned_tables.add(name) + try: + _assert_trigger_definitions( + connection, + _derived_trigger_sql( + connection, str(row["derived_dataset_id"]), name, + ), + code="derived_corrupt", + ) + except TransformationError: + _reject({name}) + + from .inplace_transform import apply_audit_catalog + + audit = apply_audit_catalog(MetaData()) + if audit.name in existing_tables: + if not _table_shape_valid( + inspector, + audit, + allowed_missing=(allowed_migrations or {}).get(audit.name, set()), + ): + _reject({audit.name}) + owned_tables.add(audit.name) + + foreign = (existing_tables - owned_tables) | (existing_views - owned_views) + missing = (owned_tables - existing_tables) | (owned_views - existing_views) + collisions = (owned_tables & existing_views) | (owned_views & existing_tables) + if foreign or missing or collisions: + _reject(foreign | missing | collisions) diff --git a/src/openstatspec/sql/database_urls.py b/src/openstatspec/sql/database_urls.py new file mode 100644 index 0000000..467a915 --- /dev/null +++ b/src/openstatspec/sql/database_urls.py @@ -0,0 +1,22 @@ +"""Database URL invariants shared by persistent catalog operations.""" + +from sqlalchemy.engine import make_url + +from ..core import UnsupportedOperationError + + +def require_persistent_database_url(database_url: str) -> None: + """Reject SQLite URLs whose catalog disappears with a connection or engine.""" + parsed_url = make_url(database_url) + if parsed_url.get_backend_name() != "sqlite": + return + database = parsed_url.database or "" + mode = str(parsed_url.query.get("mode", "")).lower() + if ( + database in {"", ":memory:"} + or database.lower() == "file::memory:" + or mode == "memory" + ): + raise UnsupportedOperationError( + "OpenStatSpec catalogs require a persistent SQLite database URL." + ) diff --git a/src/openstatspec/sql/inplace_transform.py b/src/openstatspec/sql/inplace_transform.py index 838f9fe..06ffecc 100644 --- a/src/openstatspec/sql/inplace_transform.py +++ b/src/openstatspec/sql/inplace_transform.py @@ -5,7 +5,6 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import UTC, datetime -import json from typing import Any from uuid import uuid4 @@ -17,6 +16,7 @@ from ..transform import ( AssignOperation, BooleanExpression, ComparisonExpression, + CreateVariableOperation, DeleteVariableOperation, ConditionalAssignOperation, ExecuteOperation, Operand, RecodeOperation, RecodeResult, ReplaceValueLabelsOperation, SetFormatOperation, SetMeasurementLevelOperation, @@ -27,10 +27,8 @@ from .capabilities import effective_profile from .dolt_conformance import DoltConformanceSource from .normative import catalog as core_catalog -from .wide import catalog as legacy_catalog -from .wide import ( - normalized_metadata_tables, physical_name, require_verified_catalog, -) +from .profiles import SqlProfile, preflight +from .wide import physical_name, require_verified_catalog from .workflow import TransformationError @@ -61,7 +59,8 @@ def in_place_transformation_capabilities() -> dict[str, Any]: "parent_kinds": ["core"], "mutation": "same_dataset_same_physical_wide_table", "commands": ["RECODE", "COMPUTE", "IF", "VARIABLE LABELS", - "VALUE LABELS", "FORMATS", "VARIABLE LEVEL", "EXECUTE"], + "VALUE LABELS", "FORMATS", "VARIABLE LEVEL", + "STRING", "DELETE VARIABLES", "EXECUTE"], "new_target_column": { "sqlite": True, "postgresql": True, @@ -215,6 +214,10 @@ def _input_schema( and str(row["storage_kind"]) == "numeric" else None ), measurement_level=row["measurement_level"], + declared_string_width=( + int(row["declared_string_width"]) + if row["declared_string_width"] is not None else None + ), ) for row in variables )) @@ -252,32 +255,23 @@ def _target_identity_state( return str(row.dataset_id), schema, table_name, relation_count -def _legacy_identifiers(dataset: dict[str, Any]) -> tuple[str, str]: - dataset_name = dataset.get("dataset_name") +def _physical_table_name(dataset: dict[str, Any]) -> str: table_name = dataset.get("physical_table_name") - if not isinstance(dataset_name, str) or not dataset_name: - raise TransformationError( - "dataset_invalid", "The target lacks its legacy catalog identity." - ) if not isinstance(table_name, str) or not table_name: raise TransformationError( "dataset_invalid", "The target lacks its physical wide-table name." ) - return dataset_name, table_name + return table_name def _replace_value_labels( connection: Any, *, core: Any, - legacy_variable: Table, - legacy_labels: Table, - legacy_dataset_id: str, variable: dict[str, Any], labels: tuple[ValueLabel, ...], ) -> None: variable_id = str(variable["variable_id"]) - ordinal = int(variable["source_ordinal"]) old_set = connection.execute( select(core.variable_value_label_set.c.value_label_set_id).where( core.variable_value_label_set.c.variable_id == variable_id @@ -313,27 +307,104 @@ def _replace_value_labels( string_code=(str(item.value.value) if item.value.type == "string" else None), label=item.label, )) - connection.execute(delete(legacy_labels).where( - legacy_labels.c.dataset_id == legacy_dataset_id, - legacy_labels.c.variable_ordinal == ordinal, + +def _delete_variable_metadata( + connection: Any, + *, + core: Any, + variable: dict[str, Any], +) -> None: + """Delete one variable and every normative catalog row owned by it.""" + variable_id = str(variable["variable_id"]) + variable_set_ids = list(connection.execute( + select(core.variable_set_member.c.variable_set_id).where( + core.variable_set_member.c.variable_id == variable_id + ) + ).scalars()) + response_set_ids = list(connection.execute( + select(core.multiple_response_member.c.multiple_response_set_id).where( + core.multiple_response_member.c.variable_id == variable_id + ) + ).scalars()) + label_set_id = connection.execute( + select(core.variable_value_label_set.c.value_label_set_id).where( + core.variable_value_label_set.c.variable_id == variable_id + ) + ).scalar_one_or_none() + connection.execute(delete(core.dataset_weight_variable).where( + core.dataset_weight_variable.c.variable_id == variable_id + )) + connection.execute(delete(core.variable_set_member).where( + core.variable_set_member.c.variable_id == variable_id + )) + for variable_set_id in variable_set_ids: + remaining_member = connection.execute( + select(core.variable_set_member.c.variable_id).where( + core.variable_set_member.c.variable_set_id == variable_set_id + ) + ).first() + if remaining_member is None: + connection.execute(delete(core.variable_set).where( + core.variable_set.c.variable_set_id == variable_set_id + )) + connection.execute(delete(core.multiple_response_member).where( + core.multiple_response_member.c.variable_id == variable_id + )) + for response_set_id in response_set_ids: + remaining_member = connection.execute( + select(core.multiple_response_member.c.variable_id).where( + core.multiple_response_member.c.multiple_response_set_id + == response_set_id + ) + ).first() + if remaining_member is None: + connection.execute(delete(core.multiple_response_set).where( + core.multiple_response_set.c.multiple_response_set_id + == response_set_id + )) + connection.execute(delete(core.variable_attribute).where( + core.variable_attribute.c.variable_id == variable_id + )) + connection.execute(delete(core.missing_rule).where( + core.missing_rule.c.variable_id == variable_id + )) + connection.execute(delete(core.variable_value_label_set).where( + core.variable_value_label_set.c.variable_id == variable_id + )) + if label_set_id is not None: + still_used = connection.execute( + select(core.variable_value_label_set.c.variable_id).where( + core.variable_value_label_set.c.value_label_set_id == label_set_id + ) + ).first() + if still_used is None: + connection.execute(delete(core.value_label).where( + core.value_label.c.value_label_set_id == label_set_id + )) + connection.execute(delete(core.value_label_set).where( + core.value_label_set.c.value_label_set_id == label_set_id + )) + connection.execute(delete(core.variable).where( + core.variable.c.variable_id == variable_id )) - for label_ordinal, item in enumerate(labels, start=1): - connection.execute(insert(legacy_labels).values( - dataset_id=legacy_dataset_id, - variable_ordinal=ordinal, - ordinal=label_ordinal, - value_type="numeric" if item.value.type == "binary64" else "text", - numeric_value=(item.value.number() if item.value.type == "binary64" else None), - text_value=(str(item.value.value) if item.value.type == "string" else None), - label=item.label, - )) - legacy_json = { - str(_typed_value(item.value)): item.label for item in labels - } - connection.execute(update(legacy_variable).where( - legacy_variable.c.dataset_id == legacy_dataset_id, - legacy_variable.c.ordinal == ordinal, - ).values(value_labels=json.dumps(legacy_json, ensure_ascii=False))) + + +def _compact_variable_ordinals( + connection: Any, + *, + core: Any, + variables: list[dict[str, Any]], +) -> None: + """Keep normative variable source order contiguous after deletion.""" + for source_ordinal, variable in enumerate(variables, start=1): + if int(variable["source_ordinal"]) == source_ordinal: + continue + connection.execute( + update(core.variable) + .where(core.variable.c.variable_id == variable["variable_id"]) + .values(source_ordinal=source_ordinal) + ) + variable["source_ordinal"] = source_ordinal def _failure_boundary(_name: str) -> None: @@ -380,7 +451,9 @@ def _apply_plan_on_connection( submission: InPlacePlanSubmission, actor: str, database_profile: str, + target_profile: SqlProfile, allow_schema_change: bool, + allow_delete_variable: bool, dolt_branch: str | None, dolt_head: str | None, mutation_journal: dict[str, Any] | None = None, @@ -392,13 +465,25 @@ def _apply_plan_on_connection( "The target dataset's physical wide table does not exist.", ) dataset, variables, schema = _input_schema(connection, dataset_id) - legacy_dataset_id, table_name = _legacy_identifiers(dataset) + table_name = _physical_table_name(dataset) plan = submission.plan bound = bind_transformation_plan(plan, schema) - output_by_name = { - variable.name.casefold(): variable - for variable in bound.output_schema.variables - } + output_used_physical = {"__case_ordinal"} + output_variables = [ + { + "ordinal": source_ordinal, + "source_name": variable.name, + "physical_name": physical_name( + variable.name, output_used_physical, + ), + "storage_kind": variable.storage_kind, + "string_width": variable.declared_string_width, + } + for source_ordinal, variable in enumerate( + bound.output_schema.variables, start=1, + ) + ] + preflight(target_profile, output_variables) audit = apply_audit_catalog(MetaData()) if not inspect(connection).has_table("transformation_apply"): raise TransformationError( @@ -416,18 +501,47 @@ def _apply_plan_on_connection( ) create_operations = [ operation for operation in plan.operations - if isinstance(operation, (RecodeOperation, AssignOperation)) - and operation.target_mode == "create" + if isinstance(operation, CreateVariableOperation) + or ( + isinstance(operation, (RecodeOperation, AssignOperation)) + and operation.target_mode == "create" + ) + ] + schema_operations = create_operations + [ + operation for operation in plan.operations + if isinstance(operation, DeleteVariableOperation) ] - if create_operations and not allow_schema_change: + if schema_operations and not allow_schema_change: raise TransformationError( "schema_change_not_atomic", "This database profile has no coherent new-target strategy.", ) - unsupported_targets = [ - operation.target for operation in create_operations - if output_by_name[operation.target.casefold()].storage_kind != "numeric" - ] + if any( + isinstance(operation, DeleteVariableOperation) + for operation in plan.operations + ) and not allow_delete_variable: + raise TransformationError( + "delete_variable_not_supported", + "This SQLite runtime does not support ALTER TABLE DROP COLUMN.", + ) + unsupported_targets: list[str] = [] + for operation_index, operation in enumerate(plan.operations): + if not isinstance(operation, (RecodeOperation, AssignOperation)): + continue + if operation.target_mode != "create": + continue + prefix = TransformationPlan( + plan.operations[:operation_index + 1], + contract=plan.contract, + input_alias=plan.input_alias, + ) + prefix_output = bind_transformation_plan(prefix, schema).output_schema + created = next( + variable for variable in prefix_output.variables + if variable.name.casefold() == operation.target.casefold() + ) + if created.storage_kind != "numeric": + unsupported_targets.append(operation.target) if unsupported_targets: raise TransformationError( "in_place_target_type_unsupported", @@ -435,17 +549,12 @@ def _apply_plan_on_connection( ) core = core_catalog(MetaData()) - legacy_metadata = MetaData() - _, legacy_variable, _, _ = legacy_catalog(legacy_metadata) - _, legacy_labels, _, _ = normalized_metadata_tables(legacy_metadata) relation = Table( table_name, MetaData(), schema=dataset.get("physical_table_schema"), autoload_with=connection, ) by_name = {str(row["source_name"]).casefold(): row for row in variables} used_physical = {str(row["physical_name"]).casefold() for row in variables} - next_ordinal = max(int(row["source_ordinal"]) for row in variables) + 1 - target_rows: list[dict[str, Any]] = [] quote = connection.dialect.identifier_preparer.quote qualified_table = connection.dialect.identifier_preparer.format_table(relation) numeric_type = ( @@ -454,7 +563,6 @@ def _apply_plan_on_connection( if mutation_journal is not None: mutation_journal.update({ "table_schema": dataset.get("physical_table_schema"), - "legacy_dataset_id": legacy_dataset_id, "table_name": table_name, "added_columns": [], "target_rows": [], @@ -483,57 +591,67 @@ def _apply_plan_on_connection( "The Dolt working set changed after locking the dataset.", ) - # All non-transactional DDL precedes every data/catalog mutation. This makes - # Compensation is journal-bounded to target columns and catalog identities - # created by this apply, without resetting unrelated database state. - for operation in create_operations: - target_physical = physical_name(operation.target, used_physical) - connection.exec_driver_sql( - f"ALTER TABLE {qualified_table} ADD COLUMN " - f"{quote(target_physical)} {numeric_type} NULL" - ) - if mutation_journal is not None: - mutation_journal["added_columns"].append(target_physical) - target_row = { - "variable_id": str(uuid4()), - "dataset_id": dataset_id, - "source_ordinal": next_ordinal, - "source_name": operation.target, - "physical_name": target_physical, - "storage_kind": "numeric", - "variable_label": None, - } - next_ordinal += 1 - target_rows.append(target_row) - if mutation_journal is not None: - mutation_journal["target_rows"].append(dict(target_row)) - if create_operations: - _failure_boundary("schema") - - for target_row in target_rows: - connection.execute(insert(core.variable).values(**target_row)) - connection.execute(insert(legacy_variable).values( - dataset_id=legacy_dataset_id, - ordinal=target_row["source_ordinal"], - source_name=target_row["source_name"], - physical_name=target_row["physical_name"], - storage_kind="numeric", - string_width=None, - label="", - attributes="{}", - value_labels="{}", - missing_ranges="[]", - )) - variables.append(target_row) - by_name[str(target_row["source_name"]).casefold()] = target_row - if target_rows: - _failure_boundary("catalog") - relation = Table( - table_name, MetaData(), schema=dataset.get("physical_table_schema"), - autoload_with=connection, - ) - + # Schema changes are allowed only on profiles whose DDL participates in + # this apply transaction, so execute them in canonical operation order. + # That preserves deterministic physical naming across delete/recreate flows. for operation in plan.operations: + creates_target = ( + isinstance(operation, CreateVariableOperation) + or ( + isinstance(operation, (RecodeOperation, AssignOperation)) + and operation.target_mode == "create" + ) + ) + if creates_target: + target_name = ( + operation.variable + if isinstance(operation, CreateVariableOperation) + else operation.target + ) + target_physical = physical_name(target_name, used_physical) + storage_kind = ( + operation.storage_kind + if isinstance(operation, CreateVariableOperation) + else "numeric" + ) + string_width = ( + operation.declared_string_width + if isinstance(operation, CreateVariableOperation) + else None + ) + column_type = numeric_type if storage_kind == "numeric" else "TEXT" + null_clause = ( + "NULL" if storage_kind == "numeric" + else "NOT NULL DEFAULT ''" + ) + connection.exec_driver_sql( + f"ALTER TABLE {qualified_table} ADD COLUMN " + f"{quote(target_physical)} {column_type} {null_clause}" + ) + created_target = { + "variable_id": str(uuid4()), + "dataset_id": dataset_id, + "source_ordinal": len(variables) + 1, + "source_name": target_name, + "physical_name": target_physical, + "storage_kind": storage_kind, + "declared_string_width": string_width, + "variable_label": None, + } + if mutation_journal is not None: + mutation_journal["added_columns"].append(target_physical) + mutation_journal["target_rows"].append(dict(created_target)) + _failure_boundary("schema") + relation = Table( + table_name, MetaData(), schema=dataset.get("physical_table_schema"), + autoload_with=connection, + ) + connection.execute(insert(core.variable).values(**created_target)) + variables.append(created_target) + by_name[str(created_target["source_name"]).casefold()] = created_target + _failure_boundary("catalog") + if isinstance(operation, CreateVariableOperation): + continue if isinstance(operation, RecodeOperation): source_variable = by_name[operation.source.casefold()] target_variable = by_name[operation.target.casefold()] @@ -571,18 +689,11 @@ def _apply_plan_on_connection( connection.execute(update(core.variable).where( core.variable.c.variable_id == variable["variable_id"] ).values(variable_label=operation.label)) - connection.execute(update(legacy_variable).where( - legacy_variable.c.dataset_id == legacy_dataset_id, - legacy_variable.c.ordinal == variable["source_ordinal"], - ).values(label=operation.label)) _failure_boundary("catalog") elif isinstance(operation, ReplaceValueLabelsOperation): _replace_value_labels( connection, core=core, - legacy_variable=legacy_variable, - legacy_labels=legacy_labels, - legacy_dataset_id=legacy_dataset_id, variable=by_name[operation.variable.casefold()], labels=operation.labels, ) @@ -599,26 +710,56 @@ def _apply_plan_on_connection( write_format_width=operation.width, write_format_decimals=operation.decimals, )) - encoded = json.dumps([5, operation.width, operation.decimals]) - connection.execute(update(legacy_variable).where( - legacy_variable.c.dataset_id == legacy_dataset_id, - legacy_variable.c.ordinal == variable["source_ordinal"], - ).values( - format=f"F{operation.width}.{operation.decimals}", - print_format=encoded, - write_format=encoded, - )) _failure_boundary("catalog") elif isinstance(operation, SetMeasurementLevelOperation): variable = by_name[operation.variable.casefold()] connection.execute(update(core.variable).where( core.variable.c.variable_id == variable["variable_id"] ).values(measurement_level=operation.level)) - connection.execute(update(legacy_variable).where( - legacy_variable.c.dataset_id == legacy_dataset_id, - legacy_variable.c.ordinal == variable["source_ordinal"], - ).values(measure=operation.level)) _failure_boundary("catalog") + elif isinstance(operation, DeleteVariableOperation): + variable = by_name[operation.variable.casefold()] + connection.exec_driver_sql( + f"ALTER TABLE {qualified_table} DROP COLUMN " + f"{quote(variable['physical_name'])}" + ) + _delete_variable_metadata(connection, core=core, variable=variable) + by_name.pop(operation.variable.casefold(), None) + used_physical.discard(str(variable["physical_name"]).casefold()) + variables = [ + row for row in variables + if row["variable_id"] != variable["variable_id"] + ] + _compact_variable_ordinals( + connection, core=core, variables=variables, + ) + canonical_physical = {"__case_ordinal"} + for remaining_variable in variables: + expected_physical = physical_name( + str(remaining_variable["source_name"]), canonical_physical, + ) + current_physical = str(remaining_variable["physical_name"]) + if current_physical == expected_physical: + continue + connection.exec_driver_sql( + f"ALTER TABLE {qualified_table} RENAME COLUMN " + f"{quote(current_physical)} TO {quote(expected_physical)}" + ) + connection.execute( + update(core.variable) + .where( + core.variable.c.variable_id + == remaining_variable["variable_id"] + ) + .values(physical_name=expected_physical) + ) + remaining_variable["physical_name"] = expected_physical + used_physical = canonical_physical + relation = Table( + table_name, MetaData(), schema=dataset.get("physical_table_schema"), + autoload_with=connection, + ) + _failure_boundary("schema") elif isinstance(operation, ExecuteOperation): continue else: # pragma: no cover @@ -757,12 +898,8 @@ def _compensate_failed_apply( with engine.begin() as connection: core = core_catalog(MetaData()) - legacy_metadata = MetaData() - _, legacy_variable, _, _ = legacy_catalog(legacy_metadata) - _, legacy_labels, _, _ = normalized_metadata_tables(legacy_metadata) target_rows = list(journal.get("target_rows") or ()) variable_ids = [str(row["variable_id"]) for row in target_rows] - ordinals = [int(row["source_ordinal"]) for row in target_rows] if variable_ids: label_set_ids = list(connection.execute( select(core.variable_value_label_set.c.value_label_set_id).where( @@ -782,16 +919,6 @@ def _compensate_failed_apply( connection.execute(delete(core.variable).where( core.variable.c.variable_id.in_(variable_ids) )) - legacy_dataset_id = journal.get("legacy_dataset_id") - if legacy_dataset_id is not None and ordinals: - connection.execute(delete(legacy_labels).where( - legacy_labels.c.dataset_id == legacy_dataset_id, - legacy_labels.c.variable_ordinal.in_(ordinals), - )) - connection.execute(delete(legacy_variable).where( - legacy_variable.c.dataset_id == legacy_dataset_id, - legacy_variable.c.ordinal.in_(ordinals), - )) apply_id = journal.get("apply_id") if apply_id: audit = apply_audit_catalog(MetaData()) @@ -845,9 +972,16 @@ def _run_in_place_submission( raise TransformationError( "actor_required", "A non-empty actor identity is mandatory.", ) - profile, _active = effective_profile( + profile, active = effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) + allow_delete_variable = True + if profile.name == "sqlite": + sqlite_version_parts = tuple( + int(part) for part in str(active["server_version"]).split(".") + ) + sqlite_version = (*sqlite_version_parts, 0, 0)[:3] + allow_delete_variable = sqlite_version >= (3, 35, 0) engine = create_engine(database_url) journal: dict[str, Any] = {} branch: str | None = None @@ -893,7 +1027,9 @@ def _run_in_place_submission( submission=submission, actor=actor, database_profile=profile.name, + target_profile=profile, allow_schema_change=profile.name in {"sqlite", "postgresql"}, + allow_delete_variable=allow_delete_variable, dolt_branch=branch, dolt_head=head, mutation_journal=journal, diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index e7715df..bd8488b 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -5,37 +5,34 @@ import math import re import sys -from contextlib import contextmanager from datetime import UTC, datetime -from decimal import Decimal +from contextlib import contextmanager from uuid import uuid4 +from decimal import Decimal from collections.abc import Iterable, Mapping from typing import Any -from sqlalchemy import delete, BigInteger, Boolean, Column, DateTime, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update -from sqlalchemy.dialects import mysql -from sqlalchemy.engine import make_url -from ..core import UnsupportedOperationError, safe_error_identity as _safe_error_identity -from .capabilities import ( - active_connection, dolt_operational_write_enabled, effective_profile, -) -from .dolt_conformance import DoltConformanceSource -from .profiles import ( - MYSQL_WIRE_PROFILES, preflight, preflight_identifier, - statement_payload_bytes, validate_connection_url, +from sqlalchemy import ( + BigInteger, Column, Float, MetaData, Table, Text, create_engine, insert, + inspect, or_, select, update, ) +from sqlalchemy.dialects import mysql, postgresql, sqlite +from ..core import UnsupportedOperationError +from .capabilities import active_connection, dolt_operational_write_enabled, effective_profile +from .profiles import preflight, statement_payload_bytes, validate_connection_url from .normative import ( - binary64_type, CATALOG_CONTRACT_ID, CATALOG_SCHEMA_VERSION, catalog as normative_catalog, create as create_normative_catalog, - dataset_id_for_name as normative_dataset_id_for_name, + delete_dataset_representation as delete_normative_dataset, finish_operation as finish_normative_operation, record_fidelity_events as record_normative_fidelity_events, record_operation as record_normative_operation, store_imported_dataset as store_normative_dataset, ) +from .catalog_verification import verify_catalog_relations +from .database_urls import require_persistent_database_url _IDENTIFIER = re.compile(r"[^a-zA-Z0-9_]+") @@ -49,15 +46,6 @@ def __init__(self, code: str, detail: str, *, details: Mapping[str, Any]) -> Non self.details = {"reason": code, **details} -class ImportRecoveryError(UnsupportedOperationError): - """Import recovery could not establish the promised terminal state.""" - - def __init__(self, code: str, detail: str, *, details: Mapping[str, Any]) -> None: - super().__init__(f"OpenStatSpec import recovery failed [{code}]: {detail}") - self.code = code - self.details = {"reason": code, **details} - - def _catalog_error(code: str, detail: str, **details: Any) -> CatalogPreflightError: return CatalogPreflightError(code, detail, details=details) @@ -68,531 +56,69 @@ def string_type(profile: Any) -> Text: def _wide_column_type(profile: Any, storage_kind: str) -> Any: - """Select the physical value type from the effective SQL profile.""" + """Return the strict physical type for one wide-table source column.""" return binary64_type() if storage_kind == "numeric" else string_type(profile) def _valid_wide_string_type(profile: Any, column_type: Any) -> bool: - """Require Dolt's declared LONGTEXT boundary during reflected validation.""" - return ( - isinstance(column_type, mysql.LONGTEXT) - if profile.name == "dolt" - else isinstance(column_type, Text) - ) - - -def _canonical_sha256(value: Any) -> str: - return hashlib.sha256( - json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") - ).hexdigest() - - -def _verification_fault_identity( - code: str, *, phase: str, evidence: Any, -) -> dict[str, Any]: - return { - "type": "InvariantVerificationError", - "code": code, - "phase": phase, - "message_sha256": _canonical_sha256(evidence), - } - - -def _normalized_dolt_rows( - rows: Iterable[Mapping[str, Any]], *, expected_keys: tuple[str, ...], -) -> list[dict[str, Any]]: - normalized = [] - for row in rows: - raw = dict(row) - if set(raw) != set(expected_keys): - raise UnsupportedOperationError( - "Dolt state probe returned an unexpected column shape." - ) - if expected_keys == ("table_name", "staged", "status"): - if ( - not isinstance(raw["table_name"], str) - or not raw["table_name"].strip() - or raw["staged"] not in {False, True, 0, 1} - or not isinstance(raw["status"], str) - or not raw["status"].strip() - ): - raise UnsupportedOperationError( - "Dolt status probe returned an invalid row value shape." - ) - raw["staged"] = bool(raw["staged"]) - else: - relation_names = (raw["from_table_name"], raw["to_table_name"]) - if ( - not any(isinstance(name, str) and name.strip() for name in relation_names) - or any(name is not None and not isinstance(name, str) for name in relation_names) - or not isinstance(raw["diff_type"], str) - or not raw["diff_type"].strip() - or raw["data_change"] not in {False, True, 0, 1} - or raw["schema_change"] not in {False, True, 0, 1} - ): - raise UnsupportedOperationError( - "Dolt diff-summary probe returned an invalid row value shape." - ) - raw["data_change"] = bool(raw["data_change"]) - raw["schema_change"] = bool(raw["schema_change"]) - normalized.append({key: raw[key] for key in expected_keys}) - return sorted(normalized, key=lambda row: json.dumps(row, sort_keys=True, default=str)) - - -def _dolt_evidence_block( - rows: Iterable[Mapping[str, Any]], *, audit_relations: set[str], - expected_keys: tuple[str, ...], -) -> dict[str, Any]: - normalized = _normalized_dolt_rows(rows, expected_keys=expected_keys) - - def is_audit_row(row: Mapping[str, Any]) -> bool: - if expected_keys == ("table_name", "staged", "status"): - return row["table_name"] in audit_relations and ( - str(row["status"]).strip().casefold() == "modified" - ) - from_name = row.get("from_table_name") - to_name = row.get("to_table_name") - return ( - isinstance(from_name, str) - and from_name == to_name - and from_name in audit_relations - and str(row.get("diff_type") or "").strip().casefold() == "modified" - and bool(row.get("data_change")) - and not bool(row.get("schema_change")) - ) - - audit = [row for row in normalized if is_audit_row(row)] - non_audit = [row for row in normalized if not is_audit_row(row)] - return { - "rows": normalized, - "sha256": _canonical_sha256(normalized), - "audit_catalog_rows": audit, - "audit_catalog_sha256": _canonical_sha256(audit), - "non_audit_rows": non_audit, - "non_audit_sha256": _canonical_sha256(non_audit), - } - - -def _capture_dolt_state( - connection: Any, *, profile_name: str, audit_relations: set[str], -) -> dict[str, Any] | None: - """Capture read-only Dolt version-control state without changing branches or HEAD.""" - if profile_name != "dolt": - return None - identity = connection.exec_driver_sql( - "SELECT DATABASE() AS database_name, ACTIVE_BRANCH() AS active_branch, " - "DOLT_HASHOF('HEAD') AS head_hash" - ).mappings().one() - summaries = {} - for label, left, right in ( - ("head_to_working", "HEAD", "WORKING"), - ("head_to_staged", "HEAD", "STAGED"), - ("staged_to_working", "STAGED", "WORKING"), - ): - rows = connection.exec_driver_sql( - "SELECT from_table_name, to_table_name, diff_type, " - "data_change, schema_change " - f"FROM DOLT_DIFF_SUMMARY('{left}', '{right}') " - "ORDER BY from_table_name, to_table_name, diff_type" - ).mappings().all() - summaries[label] = _dolt_evidence_block( - rows, audit_relations=audit_relations, - expected_keys=( - "from_table_name", "to_table_name", "diff_type", - "data_change", "schema_change", - ), - ) - status = _dolt_evidence_block( - connection.exec_driver_sql( - "SELECT table_name, staged, status FROM dolt_status " - "ORDER BY table_name, staged, status" - ).mappings().all(), - audit_relations=audit_relations, - expected_keys=("table_name", "staged", "status"), - ) - for key in ("database_name", "active_branch", "head_hash"): - if not isinstance(identity[key], str) or not identity[key].strip(): - raise UnsupportedOperationError( - f"Dolt state probe returned no non-empty {key}." - ) - result = { - "database": identity["database_name"].strip(), - "active_branch": identity["active_branch"].strip(), - "head": identity["head_hash"].strip(), - "status": status, - "diff_summaries": summaries, - } - result["snapshot_sha256"] = _canonical_sha256(result) - return result - - -def _require_dolt_working_set_binding( - snapshot: dict[str, Any] | None, active: Mapping[str, Any], *, phase: str, -) -> None: - if snapshot is None: - return - binding = active.get("working_set_binding") - if ( - not isinstance(binding, Mapping) - or snapshot["database"] != binding.get("database") - or snapshot["active_branch"] != binding.get("active_branch") - ): - raise UnsupportedOperationError( - f"Dolt database/branch working-set binding mismatch during {phase}." - ) - - -def _require_dolt_success_identity( - before: dict[str, Any] | None, after: dict[str, Any] | None, *, phase: str, -) -> None: - if before is None and after is None: - return - if before is None or after is None or any( - before[key] != after[key] for key in ("database", "active_branch", "head") - ): - raise UnsupportedOperationError( - f"Dolt database/branch/HEAD changed during {phase}." - ) - - -def _dolt_failure_boundary_evidence( - before: dict[str, Any] | None, after: dict[str, Any] | None, -) -> dict[str, Any]: - if before is None and after is None: - return {"applicable": False} - if before is None or after is None: - return {"applicable": True, "verified": False, "reason": "snapshot_missing"} - invariant_failures = [] - for key in ("database", "active_branch", "head"): - if before[key] != after[key]: - invariant_failures.append(f"{key}_changed") - if before["status"]["non_audit_sha256"] != after["status"]["non_audit_sha256"]: - invariant_failures.append("non_audit_status_changed") - for label in sorted(before["diff_summaries"]): - if ( - before["diff_summaries"][label]["non_audit_sha256"] - != after["diff_summaries"][label]["non_audit_sha256"] - ): - invariant_failures.append(f"non_audit_{label}_changed") - return { - "applicable": True, - "verified": not invariant_failures, - "invariant_failures": invariant_failures, - "before": before, - "after": after, - "permitted_delta": "failed-operation audit catalog relations only", - "prohibited_vc_actions": [ - "DOLT_ADD", "DOLT_COMMIT", "checkout", "reset", "branch_change", - ], - } - - -@contextmanager -def _bound_catalog_transaction( - *, engine: Any, profile_name: str, active: Mapping[str, Any], - audit_relations: set[str], phase: str, -) -> Iterable[Any]: - """Bind a write transaction to one Dolt database/branch/HEAD identity.""" - with engine.connect() as connection: - before = _capture_dolt_state( - connection, profile_name=profile_name, - audit_relations=audit_relations, - ) - _require_dolt_working_set_binding(before, active, phase=f"{phase} preflight") - connection.rollback() - with connection.begin(): - yield connection - after = _capture_dolt_state( - connection, profile_name=profile_name, - audit_relations=audit_relations, - ) - _require_dolt_working_set_binding(after, active, phase=f"{phase} completion") - _require_dolt_success_identity(before, after, phase=phase) - boundary = _dolt_failure_boundary_evidence(before, after) - if boundary.get("applicable") and not boundary.get("verified"): - raise UnsupportedOperationError( - f"Dolt non-audit working-set state changed during {phase}; " - "the transaction was rolled back." - ) - - -def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: - datasets = Table( - "dataset_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("data_table", String(255), nullable=False, unique=True), - Column("source_format", String(16), nullable=False), - Column("source_name", Text, nullable=False), - Column("source_table_name", Text), - Column("source_sha256", String(64), nullable=False), - Column("source_created_at", String(40)), - Column("source_modified_at", String(40)), - Column("imported_at", String(40), nullable=False), - Column("source_encoding", String(128)), - Column("case_count", BigInteger, nullable=False), - Column("file_label", Text, nullable=False, default=""), - Column("documents", Text, nullable=False, default="[]"), - Column("file_attributes", Text, nullable=False, default="{}"), - Column("case_weight_variable", String(255)), - Column("multiple_response_sets", Text, nullable=False, default="{}"), - ) - variables = Table( - "variable_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("ordinal", Integer, primary_key=True), - Column("source_name", String(255), nullable=False), - Column("physical_name", String(255), nullable=False), - Column("storage_kind", String(16), nullable=False), - Column("readstat_storage_type", String(32)), - Column("string_width", Integer), - Column("label", Text, nullable=False, default=""), - Column("format", String(64)), - # format remains the legacy print-format mirror; SPSS has a distinct write format. - Column("print_format", String(64)), - Column("write_format", String(64)), - Column("measure", String(32)), - Column("role", String(32)), - Column("alignment", String(32)), - Column("display_width", Integer), - Column("attributes", Text, nullable=False, default="{}"), - Column("compat_name", String(255)), - Column("value_labels", Text, nullable=False, default="{}"), - Column("missing_ranges", Text, nullable=False, default="[]"), - ) - fidelity_events = Table( - "fidelity_event_catalog", metadata, - Column("operation_id", String(36), primary_key=True), - Column("ordinal", Integer, primary_key=True), - Column("dataset_id", String(255)), - Column("direction", String(16), nullable=False), - Column("severity", String(16), nullable=False), - Column("detail", Text, nullable=False), - Column("details", Text, nullable=False, default="{}"), - Column("code", String(128), nullable=False), - ) - operations = Table( - "operation_catalog", metadata, - Column("operation_id", String(36), primary_key=True), - Column("direction", String(16), nullable=False), - Column("status", String(16), nullable=False), - Column("dataset_id", String(255)), - Column("source", Text), - Column("destination", Text), - Column("created_at", String(40), nullable=False), - Column("completed_at", String(40)), - Column("details", Text, nullable=False, default="{}"), - ) - return datasets, variables, fidelity_events, operations - + """Validate the profile-specific reflected string storage type.""" + if profile.name == "dolt": + return isinstance(column_type, mysql.LONGTEXT) + return isinstance(column_type, Text) -def _now() -> str: - return datetime.now(UTC).isoformat() -def _migrate_catalog_columns( - connection: Any, datasets: Table, variables: Table, multiple_response: Table, -) -> None: - """Add pyspssio metadata columns to catalogs created by earlier adapters. +def binary64_type() -> Float: + """Return the required IEEE-754 binary64 SQL type for every profile. - This additive migration is intentionally small and portable: no existing - source data or dictionary row is rewritten, while a later import can store - the newly observable metadata alongside it. + ``Float()`` is not adequate as a portable declaration: SQLAlchemy compiles + it to ``FLOAT`` for MySQL, which is single precision there. The strict + profile therefore declares the physical type explicitly for every target. """ - inspector = inspect(connection) - text_declaration = str( - Text().compile(dialect=connection.dialect) + return ( + Float(precision=53) + .with_variant(mysql.DOUBLE(asdecimal=False), "mysql") + .with_variant(mysql.DOUBLE(asdecimal=False), "mariadb") + .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql") + .with_variant(sqlite.REAL(), "sqlite") ) - additions = { - datasets.name: { - "file_attributes": f"{text_declaration} NOT NULL DEFAULT '{{}}'", - "case_weight_variable": "VARCHAR(255)", - }, - variables.name: { - "role": "VARCHAR(32)", - "attributes": f"{text_declaration} NOT NULL DEFAULT '{{}}'", - "compat_name": "VARCHAR(255)", - "print_format": "VARCHAR(64)", - "write_format": "VARCHAR(64)", - }, - multiple_response.name: { - "is_dichotomy": "BOOLEAN", - "use_category_labels": "BOOLEAN", - "use_first_var_label": "BOOLEAN", - "counted_value_type": "VARCHAR(16)", - "counted_numeric": "DOUBLE", - "counted_text": text_declaration, - }, - } - preparer = connection.dialect.identifier_preparer - for table_name, columns in additions.items(): - if not inspector.has_table(table_name): - continue - existing = {column["name"] for column in inspector.get_columns(table_name)} - for name, declaration in columns.items(): - if name not in existing: - connection.execute(text( - f"ALTER TABLE {preparer.quote(table_name)} ADD COLUMN " - f"{preparer.quote(name)} {declaration}" - )) - - -def _event_rows( - *, operation_id: str, dataset_id: str | None, direction: str, - fidelity_events: Iterable[Mapping[str, Any]], -) -> list[dict[str, Any]]: - """Normalize the public compact diagnostic shape into durable catalog rows.""" - rows: list[dict[str, Any]] = [] - for ordinal, event in enumerate(fidelity_events, start=1): - detail = str(event["detail"]) - details = event.get("details", {}) - rows.append({ - "operation_id": operation_id, "ordinal": ordinal, "dataset_id": dataset_id, - "direction": str(event.get("direction", direction)), - "severity": str(event.get("severity", "warning")), - "code": str(event["code"]), "detail": detail, - "details": json.dumps(details, default=str, sort_keys=True), - }) - return rows def _record_failed_preflight( - *, engine: Any, metadata: MetaData, datasets: Table, variable_catalog: Table, - multiple_response_catalog: Table, fidelity_event_catalog: Table, - operation_catalog: Table, normative: Any, operation_id: str, source_name: str, - source_format: str, variable_count: int, profile_name: str, error: Exception, - legacy: tuple[Table, ...], + *, engine: Any, normative: Any, operation_id: str, source_name: str, + source_format: str, variable_count: int, profile_name: str, + active: Mapping[str, Any], error: Exception, ) -> None: - """Persist a failed preflight without creating any source dataset state.""" - with engine.begin() as connection: - _require_verified_catalog(connection, normative, legacy) + """Persist a failed preflight in the bound normative audit catalog.""" + with _bound_catalog_transaction( + engine=engine, profile_name=profile_name, active=active, + audit_relations={normative.operation.name, normative.fidelity_event.name}, + phase="import preflight failure", + ) as connection: + require_verified_catalog(connection) failed_at = datetime.now(UTC).replace(tzinfo=None) record_normative_operation( connection, normative, operation_id=operation_id, operation_kind="import", status="failed", source_format=source_format, started_at=failed_at, completed_at=failed_at, ) - connection.execute(insert(operation_catalog).values( - operation_id=operation_id, direction="import", status="failed", dataset_id=None, - source=source_name, created_at=_now(), completed_at=_now(), - details=json.dumps({"reason": "preflight", "variable_count": variable_count, - "capability": getattr(error, "details", {})}, sort_keys=True), - )) - failed_events = ({ - "code": getattr(error, "code", "target_capability_exceeded"), - "detail": str(error), "severity": "error", "source_item": source_name, - "details": {"variable_count": variable_count, "profile": profile_name, - **getattr(error, "details", {})}, - },) - connection.execute(insert(fidelity_event_catalog), _event_rows( - operation_id=operation_id, dataset_id=None, direction="import", - fidelity_events=failed_events, - )) record_normative_fidelity_events( connection, normative, operation_id=operation_id, dataset_id=None, - direction="import", events=failed_events, + direction="import", events=({ + "code": str(getattr( + error, "code", "target_capability_exceeded")).replace("-", "_"), + "detail": str(error), "severity": "error", + "source_item": source_name, + "details": { + "variable_count": variable_count, + "profile": profile_name, + **getattr(error, "details", {}), + }, + },), ) -def multiple_response_set_catalog(metadata: MetaData) -> Table: - return Table( - "multiple_response_set_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("set_name", String(255), primary_key=True), - Column("member_ordinal", Integer, primary_key=True), - Column("kind", String(16)), Column("label", Text), - Column("is_dichotomy", Boolean), - Column("use_category_labels", Boolean), - Column("use_first_var_label", Boolean), - Column("counted_value", Text), - Column("counted_value_type", String(16)), - Column("counted_numeric", binary64_type()), - Column("counted_text", Text), - Column("variable_name", String(255)), - Column("definition", Text, nullable=False), - ) - - -def source_extension_catalog(metadata: MetaData) -> Table: - """Namespaced raw source semantics retained even when export is fail-closed.""" - return Table( - "source_extension_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("extension_key", String(255), primary_key=True), - Column("payload", Text, nullable=False), - ) - - -def source_extension_rows(dataset_id: str, extensions: Mapping[str, Any]) -> list[dict[str, Any]]: - return [ - {"dataset_id": dataset_id, "extension_key": str(key), - "payload": json.dumps(payload, default=str, ensure_ascii=False, sort_keys=True)} - for key, payload in sorted(extensions.items()) - ] - - -def document_catalog(metadata: MetaData) -> Table: - """Ordered file documents, normalized independently from the legacy JSON column.""" - return Table( - "document_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("ordinal", Integer, primary_key=True), - Column("text", Text, nullable=False), - ) - - -def value_label_catalog(metadata: MetaData) -> Table: - """Typed, ordered value labels; JSON on variable_catalog remains a read fallback.""" - return Table( - "value_label_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("variable_ordinal", Integer, primary_key=True), - Column("ordinal", Integer, primary_key=True), - Column("value_type", String(16), nullable=False), - Column("numeric_value", binary64_type()), - Column("text_value", Text), - Column("label", Text, nullable=False), - ) - - -def missing_rule_catalog(metadata: MetaData) -> Table: - """Typed inclusive SPSS user-missing intervals, including discrete values as lo == hi.""" - return Table( - "missing_rule_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("variable_ordinal", Integer, primary_key=True), - Column("ordinal", Integer, primary_key=True), - Column("kind", String(16), nullable=False), - Column("lower_type", String(16), nullable=False), - Column("lower_numeric", binary64_type()), - Column("lower_text", Text), - Column("upper_type", String(16), nullable=False), - Column("upper_numeric", binary64_type()), - Column("upper_text", Text), - Column("lower_inclusive", Boolean, nullable=False, default=True), - Column("upper_inclusive", Boolean, nullable=False, default=True), - ) - - -def attribute_catalog(metadata: MetaData) -> Table: - """Ordered SPSS custom-attribute values for files and variables. - - SPSS custom attributes are text-valued, but one attribute name can carry - an ordered array of values. ``scope`` is ``file`` for a file attribute - (with ``variable_ordinal == 0``) and ``variable`` for an attribute of one - source variable. This table is authoritative whenever it contains rows; - the JSON columns on older catalogs remain a migration fallback. - """ - return Table( - "attribute_catalog", metadata, - Column("dataset_id", String(255), primary_key=True), - Column("scope", String(16), primary_key=True), - Column("variable_ordinal", Integer, primary_key=True), - Column("attribute_ordinal", Integer, primary_key=True), - Column("value_ordinal", Integer, primary_key=True), - Column("attribute_name", String(255), nullable=False), - Column("attribute_value", Text, nullable=False), - ) def _attribute_values(value: Any) -> list[str]: @@ -631,26 +157,6 @@ def append(scope: str, variable_ordinal: int, attributes: Mapping[str, Any] | No return rows -def attributes_from_rows( - rows: Iterable[Mapping[str, Any]], *, variables: Iterable[Mapping[str, Any]], -) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: - """Rebuild ordered attributes; rows, not legacy JSON, are authoritative.""" - by_ordinal = {int(item["ordinal"]): str(item["source_name"]) for item in variables} - grouped: dict[tuple[str, int], dict[str, list[str]]] = {} - for row in rows: - target = grouped.setdefault((str(row["scope"]), int(row["variable_ordinal"])), {}) - target.setdefault(str(row["attribute_name"]), []).append(str(row["attribute_value"])) - - def collapse(values: Mapping[str, list[str]]) -> dict[str, Any]: - return {name: value[0] if len(value) == 1 else value for name, value in values.items()} - - file_attributes = collapse(grouped.get(("file", 0), {})) - variable_attributes = { - by_ordinal[ordinal]: collapse(attributes) - for (scope, ordinal), attributes in grouped.items() - if scope == "variable" and ordinal in by_ordinal - } - return file_attributes, variable_attributes def _typed_endpoint(value: Any) -> tuple[str, float | None, str | None]: @@ -718,11 +224,6 @@ def missing_rule_rows(dataset_id: str, variables: list[dict[str, Any]]) -> list[ return rows -def normalized_metadata_tables(metadata: MetaData) -> tuple[Table, Table, Table, Table]: - return ( - document_catalog(metadata), value_label_catalog(metadata), missing_rule_catalog(metadata), - attribute_catalog(metadata), - ) def _mr_counted_value(definition: Mapping[str, Any]) -> tuple[str | None, float | None, str | None]: value = definition.get("counted_value", definition.get("countedvalue")) @@ -944,44 +445,6 @@ def multiple_response_set_rows(dataset_id: str, definitions: str) -> list[dict[s return rows -def multiple_response_sets_from_rows(rows: Iterable[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: - """Reconstruct pyspssio MR metadata from normalized catalog rows. - - The new typed fields are authoritative. ``definition`` only supplies - backwards compatibility for catalogs written before those fields existed. - """ - result: dict[str, dict[str, Any]] = {} - for row in rows: - name = str(row["set_name"]) - if name not in result: - try: - legacy = json.loads(row.get("definition") or "{}") - except (TypeError, json.JSONDecodeError): - legacy = {} - definition = dict(legacy) if isinstance(legacy, Mapping) else {} - definition["variable_list"] = [] - result[name] = definition - definition = result[name] - if row.get("label") is not None: - definition["label"] = row["label"] - if row.get("is_dichotomy") is not None: - definition["is_dichotomy"] = bool(row["is_dichotomy"]) - elif row.get("kind") is not None: - definition["is_dichotomy"] = str(row["kind"]).upper() == "MD" - if row.get("use_category_labels") is not None: - definition["use_category_labels"] = bool(row["use_category_labels"]) - if row.get("use_first_var_label") is not None: - definition["use_first_var_label"] = bool(row["use_first_var_label"]) - value_type = row.get("counted_value_type") - if value_type == "numeric": - definition["counted_value"] = row.get("counted_numeric") - elif value_type == "text": - definition["counted_value"] = row.get("counted_text") - elif row.get("counted_value") not in (None, ""): - definition["counted_value"] = row["counted_value"] - if row.get("variable_name") is not None: - definition["variable_list"].append(row["variable_name"]) - return result def physical_name(source_name: str, used: set[str]) -> str: stem = _IDENTIFIER.sub("_", source_name).strip("_").lower() or "variable" @@ -994,606 +457,187 @@ def physical_name(source_name: str, used: set[str]) -> str: return candidate -def data_table_name(dataset_id: str) -> str: - stem = _IDENTIFIER.sub("_", dataset_id).strip("_").lower() or "dataset" - return f"data_{stem[:48]}" - +def _canonical_sha256(value: Any) -> str: + payload = json.dumps( + value, sort_keys=True, separators=(",", ":"), default=str, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() -def _catalog_layout(metadata: MetaData) -> tuple[tuple[Table, ...], Any]: - datasets, variables, fidelity_events, operations = catalog(metadata) - multiple_response = multiple_response_set_catalog(metadata) - source_extensions = source_extension_catalog(metadata) - documents, value_labels, missing_rules, attributes = normalized_metadata_tables(metadata) - return ( - datasets, variables, multiple_response, source_extensions, documents, - value_labels, missing_rules, attributes, fidelity_events, operations, - ), normative_catalog(metadata) -def _normalized_sql_type(inspector: Any, value: Any) -> str: - compiled = " ".join( - str(value.compile(dialect=inspector.bind.dialect)).strip().upper().split() - ) - if inspector.bind.dialect.name in {"mysql", "mariadb"}: - if compiled in {"BOOL", "BOOLEAN", "TINYINT(1)"}: - return "BOOLEAN/TINYINT(1)" - compiled = re.sub( - r"\b(TINYINT|SMALLINT|MEDIUMINT|INTEGER|INT|BIGINT)\(\d+\)", - r"\1", - compiled, - ) - return compiled +def _dolt_evidence_block( + rows: Iterable[Mapping[str, Any]], *, expected_keys: tuple[str, ...], +) -> dict[str, Any]: + normalized = [ + {key: dict(row)[key] for key in expected_keys} + for row in rows + ] + normalized.sort(key=lambda row: json.dumps(row, sort_keys=True, default=str)) + return {"rows": normalized, "sha256": _canonical_sha256(normalized)} -def _normalized_default(value: Any) -> str | None: - if value is None: +def _capture_dolt_state( + connection: Any, *, profile_name: str, audit_relations: set[str], +) -> dict[str, Any] | None: + """Capture and classify the Dolt working set without mutating it.""" + audit_relations = {str(name) for name in audit_relations} + if profile_name != "dolt": return None - result = " ".join(str(value).strip().split()) - while result.startswith("(") and result.endswith(")"): - result = result[1:-1].strip() - return result - - -def _expected_unique_constraints(table: Table) -> set[tuple[str, ...]]: - return { - tuple(column.name for column in constraint.columns) - for constraint in table.constraints - if getattr(constraint, "__visit_name__", "") == "unique_constraint" + identity = connection.exec_driver_sql( + "SELECT DATABASE() AS database_name, ACTIVE_BRANCH() AS active_branch, " + "DOLT_HASHOF('HEAD') AS head_hash" + ).mappings().one() + status_rows = connection.exec_driver_sql( + "SELECT table_name, staged, status FROM dolt_status " + "ORDER BY table_name, staged, status" + ).mappings().all() + state = { + "database": str(identity["database_name"]).strip(), + "active_branch": str(identity["active_branch"]).strip(), + "head": str(identity["head_hash"]).strip(), + "status": _dolt_evidence_block( + status_rows, expected_keys=("table_name", "staged", "status"), + ), + "diff_summaries": {}, } - - -def _actual_unique_constraints( - inspector: Any, table_name: str, -) -> set[tuple[str, ...]]: - constraints = { - tuple(str(name) for name in item.get("column_names") or ()) - for item in inspector.get_unique_constraints(table_name) + unrelated = { + "status": _dolt_evidence_block( + ( + row for row in status_rows + if str(dict(row).get("table_name") or "") not in audit_relations + ), + expected_keys=("table_name", "staged", "status"), + ), + "diff_summaries": {}, } - constraints.update( - tuple(str(name) for name in item.get("column_names") or ()) - for item in inspector.get_indexes(table_name) - if item.get("unique") - ) - constraints.discard(()) - return constraints - -def _expected_foreign_keys(table: Table) -> set[tuple[Any, ...]]: - return { - ( - tuple(column.name for column in constraint.columns), - next(iter(constraint.elements)).column.table.name, - tuple(element.column.name for element in constraint.elements), + def allowed_audit_data_diff(row: Mapping[str, Any]) -> bool: + values = dict(row) + from_name = str(values.get("from_table_name") or "") + to_name = str(values.get("to_table_name") or "") + data_change = str(values.get("data_change")).strip().lower() + schema_change = str(values.get("schema_change")).strip().lower() + return ( + from_name == to_name + and from_name in audit_relations + and str(values.get("diff_type") or "").strip().lower() == "modified" + and data_change in {"1", "true"} + and schema_change in {"0", "false"} ) - for constraint in table.foreign_key_constraints - } - -def _actual_foreign_keys( - inspector: Any, table_name: str, -) -> set[tuple[Any, ...]]: - return { - ( - tuple(str(name) for name in item.get("constrained_columns") or ()), - str(item.get("referred_table") or ""), - tuple(str(name) for name in item.get("referred_columns") or ()), + for label, left, right in ( + ("head_to_working", "HEAD", "WORKING"), + ("head_to_staged", "HEAD", "STAGED"), + ("staged_to_working", "STAGED", "WORKING"), + ): + rows = connection.exec_driver_sql( + "SELECT from_table_name, to_table_name, diff_type, " + "data_change, schema_change " + f"FROM DOLT_DIFF_SUMMARY('{left}', '{right}') " + "ORDER BY from_table_name, to_table_name, diff_type" + ).mappings().all() + expected_keys = ( + "from_table_name", "to_table_name", "diff_type", + "data_change", "schema_change", ) - for item in inspector.get_foreign_keys(table_name) - } + state["diff_summaries"][label] = _dolt_evidence_block( + rows, expected_keys=expected_keys, + ) + unrelated["diff_summaries"][label] = _dolt_evidence_block( + ( + row for row in rows + if not allowed_audit_data_diff(row) + ), + expected_keys=expected_keys, + ) + state["unrelated_working_set"] = unrelated + state["unrelated_sha256"] = _canonical_sha256(unrelated) + state["snapshot_sha256"] = _canonical_sha256(state) + return state -def _normalized_check_sql(value: Any) -> str: - result = " ".join(str(value).strip().casefold().split()) - result = result.replace("`", "").replace('"', "") - result = re.sub(r"\[([^]]+)\]", r"\1", result) - result = re.sub(r"(? None: + if snapshot is None: + return + binding = active.get("working_set_binding") + if ( + not isinstance(binding, Mapping) + or snapshot["database"] != binding.get("database") + or snapshot["active_branch"] != binding.get("active_branch") + ): + raise UnsupportedOperationError( + f"Dolt database/branch working-set binding mismatch during {phase}." + ) -def _expected_check_constraints(table: Table) -> set[str]: - return { - _normalized_check_sql(constraint.sqltext) - for constraint in table.constraints - if getattr(constraint, "__visit_name__", "") == "table_or_column_check_constraint" - } - - -def _actual_check_constraints(inspector: Any, table_name: str) -> set[str]: - return { - _normalized_check_sql(item.get("sqltext") or "") - for item in inspector.get_check_constraints(table_name) - } - - -_MIGRATED_SERVER_DEFAULTS = { - ("dataset_catalog", "file_attributes"): {None, "'{}'", '"{}"', "{}"}, - ("variable_catalog", "attributes"): {None, "'{}'", '"{}"', "{}"}, -} - - -def _catalog_table_shape_valid( - inspector: Any, table: Table, *, allow_missing: bool, -) -> bool: - actual = { - str(column["name"]): column - for column in inspector.get_columns(table.name) - } - expected = {column.name: column for column in table.columns} - if set(actual) - set(expected): - return False - if not allow_missing and set(actual) != set(expected): - return False - for name in set(actual) & set(expected): - expected_column = expected[name] - actual_column = actual[name] - if ( - _normalized_sql_type(inspector, expected_column.type) - != _normalized_sql_type(inspector, actual_column["type"]) - ): - return False - if bool(actual_column.get("nullable")) != bool(expected_column.nullable): - return False - actual_default = _normalized_default(actual_column.get("default")) - if (table.name, name) in _MIGRATED_SERVER_DEFAULTS: - if actual_default not in _MIGRATED_SERVER_DEFAULTS[(table.name, name)]: - return False - else: - expected_default = _normalized_default( - expected_column.server_default.arg - if expected_column.server_default is not None else None - ) - if actual_default != expected_default: - return False - if actual_column.get("identity") is not None or actual_column.get("computed") is not None: - return False - if expected_column.autoincrement is True and actual_column.get("autoincrement") is not True: - return False - if expected_column.autoincrement is False and actual_column.get("autoincrement") is True: - return False - expected_pk = tuple(column.name for column in table.primary_key.columns) - actual_pk = tuple( - str(name) for name in ( - inspector.get_pk_constraint(table.name).get("constrained_columns") or () - ) - ) - if actual_pk != expected_pk: - return False - if _actual_unique_constraints(inspector, table.name) != _expected_unique_constraints(table): - return False - if _actual_foreign_keys(inspector, table.name) != _expected_foreign_keys(table): - return False - if _actual_check_constraints(inspector, table.name) != _expected_check_constraints(table): - return False - return True - - -def _identity_shape_valid( - inspector: Any, table: Table, *, key_name: str, -) -> bool: - try: - return ( - [column.name for column in table.primary_key.columns] == [key_name] - and _catalog_table_shape_valid(inspector, table, allow_missing=False) - ) - except Exception: - return False - - -def _catalog_existing_shapes_valid( - inspector: Any, tables: Iterable[Table], -) -> bool: - """Require exact existing shape while allowing only absent migration columns.""" - try: - return all( - _catalog_table_shape_valid(inspector, table, allow_missing=True) - for table in tables - ) - except Exception: - return False - - -_MIGRATABLE_CATALOG_COLUMNS = { - "dataset_catalog": {"file_attributes", "case_weight_variable"}, - "variable_catalog": { - "role", "attributes", "compat_name", "print_format", "write_format", - }, - "multiple_response_set_catalog": { - "is_dichotomy", "use_category_labels", "use_first_var_label", - "counted_value_type", "counted_numeric", "counted_text", - }, -} - - -def _catalog_missing_columns( - inspector: Any, tables: Iterable[Table], -) -> dict[str, set[str]]: - missing = {} - for table in tables: - absent = {column.name for column in table.columns} - { - str(column["name"]) - for column in inspector.get_columns(table.name) - } - if absent: - missing[table.name] = absent - return missing - - -def _catalog_missing_columns_are_migratable( - missing: Mapping[str, set[str]], - *, allowed: Mapping[str, set[str]] | None = None, -) -> bool: - allowed_columns = _MIGRATABLE_CATALOG_COLUMNS if allowed is None else allowed - return bool(missing) and all( - columns <= allowed_columns.get(table_name, set()) - for table_name, columns in missing.items() - ) - - -def _registered_physical_relations( - connection: Any, *, existing_tables: set[str], normative: Any, - legacy: Iterable[Table], -) -> tuple[set[str], tuple[Table, ...], set[str], set[str], str]: - """Return owned relations and the optional workflow identity state.""" - legacy = tuple(legacy) - declared_tables = legacy + normative.all() - static_tables = {table.name for table in declared_tables} - physical_tables: set[str] = set() - physical_views: set[str] = set() - inspector = inspect(connection) - if ( - legacy[0].name in existing_tables - and "data_table" in { - str(column["name"]) for column in inspector.get_columns(legacy[0].name) - } - ): - physical_tables.update(str(name) for name in connection.execute( - select(legacy[0].c.data_table) - ).scalars() if name) - if ( - normative.dataset.name in existing_tables - and "physical_table_name" in { - str(column["name"]) for column in inspector.get_columns(normative.dataset.name) - } - ): - physical_tables.update(str(name) for name in connection.execute( - select(normative.dataset.c.physical_table_name) - ).scalars() if name) - - # The optional workflow is another OpenStatSpec-owned relation profile in - # the same dedicated namespace. Import locally to avoid its documented - # dependency on this module during module initialization. - from .workflow import ( # pylint: disable=import-outside-toplevel - PROFILE_ID, PROFILE_SCHEMA_VERSION, workflow_catalog, - ) - workflow = workflow_catalog(MetaData()) - workflow_tables = {table.name for table in workflow.all()} - workflow_identity = workflow.transformation_profile_identity - profile_present = workflow_identity.name in existing_tables - if profile_present: - declared_tables += workflow.all() - static_tables.update(workflow_tables) - if not workflow_tables <= existing_tables: - return static_tables, declared_tables, physical_tables, physical_views, "foreign" - if not _identity_shape_valid( - inspector, workflow_identity, key_name="profile_identity_key", - ): - return static_tables, declared_tables, physical_tables, physical_views, "foreign" - identities = connection.execute(select(workflow_identity)).mappings().all() - if len(identities) != 1: - return static_tables, declared_tables, physical_tables, physical_views, "ambiguous" - if ( - identities[0]["profile_identity_key"] != 1 - or identities[0]["contract_id"] != PROFILE_ID - or identities[0]["schema_version"] != PROFILE_SCHEMA_VERSION - or identities[0]["core_contract_id"] != CATALOG_CONTRACT_ID - ): - return static_tables, declared_tables, physical_tables, physical_views, "foreign" - for row in connection.execute(select( - workflow.derived_dataset.c.physical_relation_name, - workflow.derived_dataset.c.output_mode, - )).mappings(): - name = str(row["physical_relation_name"]) - if row["output_mode"] == "view": - physical_views.add(name) - else: - physical_tables.add(name) - elif workflow_tables & existing_tables: - return static_tables, declared_tables, physical_tables, physical_views, "foreign" - - # The compact in-place apply audit is optional but catalog-owned. Import - # locally because inplace_transform depends on this module at load time. - from .inplace_transform import ( # pylint: disable=import-outside-toplevel - apply_audit_catalog, - ) - apply_audit = apply_audit_catalog(MetaData()) - if apply_audit.name in existing_tables: - declared_tables += (apply_audit,) - static_tables.add(apply_audit.name) - - return static_tables, declared_tables, physical_tables, physical_views, "valid" - - -def _catalog_dataset_bijection_state( - connection: Any, *, normative: Any, legacy: Iterable[Table], -) -> str: - """Require one exact legacy-to-normative dataset/table mapping per row.""" - datasets = tuple(legacy)[0] - legacy_rows = [ - (row["dataset_id"], row["data_table"]) - for row in connection.execute(select( - datasets.c.dataset_id, datasets.c.data_table, - )).mappings() - ] - normative_rows = [ - (row["dataset_name"], row["physical_table_name"]) - for row in connection.execute(select( - normative.dataset.c.dataset_name, - normative.dataset.c.physical_table_name, - )).mappings() - ] - for rows in (legacy_rows, normative_rows): - if any( - not isinstance(dataset_name, str) or not dataset_name.strip() - or not isinstance(table_name, str) or not table_name.strip() - for dataset_name, table_name in rows - ): - return "unverified" - dataset_names = [dataset_name for dataset_name, _table_name in rows] - table_names = [table_name for _dataset_name, table_name in rows] - if ( - len(set(dataset_names)) != len(dataset_names) - or len(set(table_names)) != len(table_names) - or len(set(rows)) != len(rows) - ): - return "ambiguous" - return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" - - -def _catalog_variable_bijection_state( - connection: Any, *, normative: Any, legacy: Iterable[Table], -) -> str: - """Require exact legacy-to-normative variable identity mappings.""" - variables = tuple(legacy)[1] - legacy_rows = [ - ( - row["dataset_id"], row["ordinal"], row["source_name"], - row["physical_name"], row["storage_kind"], - ) - for row in connection.execute(select( - variables.c.dataset_id, variables.c.ordinal, - variables.c.source_name, variables.c.physical_name, - variables.c.storage_kind, - )).mappings() - ] - normative_rows = [ - ( - row["dataset_name"], row["source_ordinal"], row["source_name"], - row["physical_name"], row["storage_kind"], - ) - for row in connection.execute( - select( - normative.dataset.c.dataset_name, - normative.variable.c.source_ordinal, - normative.variable.c.source_name, - normative.variable.c.physical_name, - normative.variable.c.storage_kind, - ).join( - normative.variable, - normative.variable.c.dataset_id == normative.dataset.c.dataset_id, - ) - ).mappings() - ] - for rows in (legacy_rows, normative_rows): - if any( - not isinstance(dataset_name, str) or not dataset_name.strip() - or not isinstance(ordinal, int) or ordinal < 1 - or not isinstance(source_name, str) or not source_name.strip() - or not isinstance(physical_name, str) or not physical_name.strip() - or storage_kind not in {"numeric", "string"} - for dataset_name, ordinal, source_name, physical_name, storage_kind in rows - ): - return "unverified" - if len(set(rows)) != len(rows): - return "ambiguous" - return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" - - -def _catalog_state( - connection: Any, normative: Any, legacy: Iterable[Table], - *, allowed_migrations: Mapping[str, set[str]] | None = None, -) -> str: - inspector = inspect(connection) - existing_tables = set(inspector.get_table_names()) - existing_views = set(inspector.get_view_names()) - if existing_tables & existing_views: - return "ambiguous" - existing_relations = existing_tables | existing_views - if normative.catalog_identity.name not in existing_tables: - return "absent" if not existing_relations else "foreign" - try: - identities = connection.execute( - select(normative.catalog_identity) - ).mappings().all() - except Exception: - return "foreign" - if len(identities) != 1: - return "ambiguous" - if not _identity_shape_valid( - inspector, normative.catalog_identity, key_name="catalog_identity_key", - ): - return "foreign" - identity = identities[0] - if ( - identity["catalog_identity_key"] != 1 - or identity["contract_id"] != CATALOG_CONTRACT_ID - or identity["schema_version"] != CATALOG_SCHEMA_VERSION - ): - return "foreign" - static_tables, declared_tables, physical_tables, physical_views, profile_valid = ( - _registered_physical_relations( - connection, existing_tables=existing_tables, - normative=normative, legacy=legacy, - ) - ) - if profile_valid != "valid": - return profile_valid - if ( - physical_tables & physical_views - or static_tables & physical_tables - or static_tables & physical_views - ): - return "ambiguous" - owned_relations = static_tables | physical_tables | physical_views - if existing_relations - owned_relations: - return "foreign" - if physical_tables - existing_tables or physical_views - existing_views: - return "unverified" - if not static_tables <= existing_tables: - return "unverified" - if not _catalog_existing_shapes_valid(inspector, declared_tables): - return "foreign" - missing_columns = _catalog_missing_columns(inspector, declared_tables) - if missing_columns: - return ( - "migration_required" - if _catalog_missing_columns_are_migratable( - missing_columns, allowed=allowed_migrations, - ) - else "unverified" - ) - mapping_state = _catalog_dataset_bijection_state( - connection, normative=normative, legacy=legacy, - ) - if mapping_state != "valid": - return mapping_state - variable_mapping_state = _catalog_variable_bijection_state( - connection, normative=normative, legacy=legacy, - ) - if variable_mapping_state != "valid": - return variable_mapping_state - return "verified" - - -def _require_verified_catalog( - connection: Any, normative: Any, legacy: Iterable[Table], - *, allowed_migrations: Mapping[str, set[str]] | None = None, +def _require_dolt_success_identity( + before: Mapping[str, Any] | None, + after: Mapping[str, Any] | None, + *, + phase: str, ) -> None: - state = _catalog_state( - connection, normative, legacy, allowed_migrations=allowed_migrations, - ) - accepted_states = ( - {"verified", "migration_required"} - if allowed_migrations is not None - else {"verified"} - ) - if state not in accepted_states: + if before is None and after is None: + return + if before is None or after is None or any( + before[key] != after[key] for key in ("database", "active_branch", "head") + ) or ( + "unrelated_sha256" in before or "unrelated_sha256" in after + ) and before.get("unrelated_sha256") != after.get("unrelated_sha256"): raise UnsupportedOperationError( - f"The selected OpenStatSpec catalog is {state}; run explicit catalog initialization first." + f"Dolt identity or unrelated working-set state changed during {phase}." ) -def require_verified_catalog( - connection: Any, - *, allowed_migrations: Mapping[str, set[str]] | None = None, -) -> None: - """Require catalog ownership, shape, bijection, and only explicit migrations.""" - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - _require_verified_catalog( - connection, normative, legacy, allowed_migrations=allowed_migrations, - ) - - -def _catalog_snapshot( - connection: Any, -) -> tuple[set[str], dict[str, set[str]]]: - inspector = inspect(connection) - tables = set(inspector.get_table_names()) - return tables, { - table_name: { - str(column["name"]) for column in inspector.get_columns(table_name) - } - for table_name in tables +def _dolt_failure_boundary_evidence( + before: Mapping[str, Any] | None, + after: Mapping[str, Any] | None, +) -> dict[str, Any]: + return { + "applicable": before is not None or after is not None, + "before": before, + "after": after, } -def _compensate_catalog_initialization( - connection: Any, *, metadata: MetaData, before_tables: set[str], - before_columns: Mapping[str, set[str]], normative: Any, - legacy: Iterable[Table], -) -> None: - """Restore only when no concurrent initializer completed the catalog.""" - if _catalog_state(connection, normative, legacy) == "verified": - return - current_tables = set(inspect(connection).get_table_names()) - for table in reversed(metadata.sorted_tables): - if table.name in current_tables and table.name not in before_tables: - table.drop(connection, checkfirst=True) - inspector = inspect(connection) - preparer = connection.dialect.identifier_preparer - for table_name, original_columns in before_columns.items(): - if not inspector.has_table(table_name): - raise RuntimeError( - f"Pre-existing catalog table {table_name!r} disappeared during initialization." - ) - current_columns = { - str(column["name"]) for column in inspect(connection).get_columns(table_name) - } - for column_name in sorted(current_columns - original_columns): - connection.execute(text( - f"ALTER TABLE {preparer.quote(table_name)} " - f"DROP COLUMN {preparer.quote(column_name)}" - )) - - -def _requires_compensating_catalog_cleanup(profile_name: str) -> bool: - """Only non-transactional MySQL-wire DDL needs stale compensation.""" - return profile_name in MYSQL_WIRE_PROFILES - - -def _catalog_residual_inventory( - engine: Any, *, before_tables: set[str], - before_columns: Mapping[str, set[str]], -) -> dict[str, Any]: - try: - with engine.connect() as connection: - inspector = inspect(connection) - current_tables = set(inspector.get_table_names()) - added_columns = { - table_name: sorted( - { - str(column["name"]) - for column in inspector.get_columns(table_name) - } - original_columns +@contextmanager +def _bound_catalog_transaction( + *, engine: Any, profile_name: str, active: Mapping[str, Any], + audit_relations: set[str], phase: str, +): + """Bind a catalog mutation to one Dolt database, branch, and HEAD.""" + with engine.connect() as connection: + before = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding(before, active, phase=f"{phase} preflight") + if profile_name != "sqlite": + connection.rollback() + try: + with connection.begin(): + yield connection + after = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, ) - for table_name, original_columns in before_columns.items() - if table_name in current_tables - } - return { - "new_tables": sorted(current_tables - before_tables), - "missing_preexisting_tables": sorted(before_tables - current_tables), - "added_columns": { - name: columns for name, columns in added_columns.items() if columns - }, - "views": sorted(inspector.get_view_names()), - } - except Exception as inventory_error: - return {"inspection_error_type": type(inventory_error).__name__} + _require_dolt_success_identity(before, after, phase=phase) + except Exception: + after = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + _dolt_failure_boundary_evidence(before, after) + raise def dolt_state_snapshot( - *, - database_url: str, - dolt_conformance_source: DoltConformanceSource | None = None, + *, database_url: str, dolt_conformance_source: Any | None = None, ) -> dict[str, Any]: - """Return a read-only, digest-bound snapshot of one active Dolt database.""" + """Return read-only branch, HEAD, status, and diff evidence for Dolt.""" validate_connection_url(database_url) active = active_connection( database_url, dolt_conformance_source=dolt_conformance_source, @@ -1602,22 +646,54 @@ def dolt_state_snapshot( raise UnsupportedOperationError( "dolt_state_snapshot requires a positively identified Dolt connection." ) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - audit_relations = { - legacy[8].name, legacy[9].name, - normative.fidelity_event.name, normative.operation.name, - } engine = create_engine(database_url) with engine.connect() as connection: - state = _capture_dolt_state( - connection, profile_name="dolt", audit_relations=audit_relations, + identity = connection.exec_driver_sql( + "SELECT DATABASE() AS database_name, ACTIVE_BRANCH() AS active_branch, " + "DOLT_HASHOF('HEAD') AS head_hash" + ).mappings().one() + summaries = {} + for label, left, right in ( + ("head_to_working", "HEAD", "WORKING"), + ("head_to_staged", "HEAD", "STAGED"), + ("staged_to_working", "STAGED", "WORKING"), + ): + summaries[label] = _dolt_evidence_block( + connection.exec_driver_sql( + "SELECT from_table_name, to_table_name, diff_type, " + "data_change, schema_change " + f"FROM DOLT_DIFF_SUMMARY('{left}', '{right}') " + "ORDER BY from_table_name, to_table_name, diff_type" + ).mappings().all(), + expected_keys=( + "from_table_name", "to_table_name", "diff_type", + "data_change", "schema_change", + ), + ) + status = _dolt_evidence_block( + connection.exec_driver_sql( + "SELECT table_name, staged, status FROM dolt_status " + "ORDER BY table_name, staged, status" + ).mappings().all(), + expected_keys=("table_name", "staged", "status"), ) - assert state is not None - _require_dolt_working_set_binding( - state, active, phase="read-only state capture", - ) - binding = active["working_set_binding"] + state = { + "database": str(identity["database_name"]).strip(), + "active_branch": str(identity["active_branch"]).strip(), + "head": str(identity["head_hash"]).strip(), + "status": status, + "diff_summaries": summaries, + } + binding = active.get("working_set_binding") + if ( + not isinstance(binding, Mapping) + or state["database"] != binding.get("database") + or state["active_branch"] != binding.get("active_branch") + ): + raise UnsupportedOperationError( + "Dolt database/branch working-set binding mismatch during read-only state capture." + ) + state["snapshot_sha256"] = _canonical_sha256(state) return { "profile": "dolt", "server_version": active["server_version"], @@ -1630,1495 +706,722 @@ def dolt_state_snapshot( } -@contextmanager -def _catalog_initialization_serialization( - connection: Any, *, profile_name: str, -): - """Serialize catalog DDL across connections on one database server.""" - if profile_name == "sqlite": - connection.exec_driver_sql("BEGIN IMMEDIATE") - try: - yield - except Exception: - connection.rollback() - raise - else: - connection.commit() - return - if profile_name not in MYSQL_WIRE_PROFILES: - yield - return - lock_name = "openstatspec.catalog-initialize.v1" - acquired = connection.execute( - text("SELECT GET_LOCK(:lock_name, 30)"), {"lock_name": lock_name}, - ).scalar_one() - connection.commit() - if acquired != 1: - raise UnsupportedOperationError( - "Could not acquire the catalog initialization lock." - ) - try: - yield - finally: - try: - connection.execute( - text("SELECT RELEASE_LOCK(:lock_name)"), - {"lock_name": lock_name}, - ) - connection.commit() - except Exception: - connection.invalidate() - - def initialize_wide_catalog( - *, - database_url: str, - dolt_conformance_source: DoltConformanceSource | None = None, + *, database_url: str, dolt_conformance_source: Any | None = None, ) -> dict[str, Any]: - """Install or explicitly migrate a dedicated catalog after server preflight.""" + """Initialize or verify the singular normative OpenStatSpec catalog.""" validate_connection_url(database_url) - parsed_url = make_url(database_url) - sqlite_database = parsed_url.database or "" - sqlite_mode = str(parsed_url.query.get("mode", "")).lower() - if ( - parsed_url.get_backend_name() == "sqlite" - and ( - sqlite_database in {"", ":memory:"} - or sqlite_database.lower() == "file::memory:" - or sqlite_mode == "memory" - ) - ): - raise UnsupportedOperationError( - "Catalog initialization requires a persistent SQLite database URL." - ) + require_persistent_database_url(database_url) profile, active = effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - datasets, variables, multiple_response = legacy[:3] - with engine.connect() as connection: - with _catalog_initialization_serialization( - connection, profile_name=profile.name, - ): - state = _catalog_state(connection, normative, legacy) - if state not in {"absent", "verified", "migration_required"}: + normative = normative_catalog(MetaData()) + audit_relations = {table.name for table in normative.all()} + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="catalog initialization", + ) as connection: + inspector = inspect(connection) + tables = set(inspector.get_table_names()) + views = set(inspector.get_view_names()) + if tables or views: + if normative.catalog_identity.name not in tables: raise UnsupportedOperationError( - f"The selected database catalog is {state}; initialization is not permitted." + "The selected database catalog is foreign; " + "initialization is not permitted." ) - before_tables, before_columns = _catalog_snapshot(connection) - pre_dolt_state = _capture_dolt_state( - connection, profile_name=profile.name, audit_relations=set(), - ) - _require_dolt_working_set_binding( - pre_dolt_state, active, phase="catalog initialization preflight", - ) - if profile.name != "sqlite": - connection.rollback() + require_verified_catalog(connection) + else: try: - mutation = ( - connection.begin_nested() - if profile.name == "sqlite" - else connection.begin() - ) - with mutation: - create_normative_catalog(connection, normative) - metadata.create_all(connection, tables=list(legacy)) - _migrate_catalog_columns( - connection, datasets, variables, multiple_response, - ) - _require_verified_catalog(connection, normative, legacy) - post_dolt_state = _capture_dolt_state( - connection, profile_name=profile.name, audit_relations=set(), - ) - _require_dolt_working_set_binding( - post_dolt_state, active, phase="catalog initialization completion", - ) - _require_dolt_success_identity( - pre_dolt_state, post_dolt_state, phase="catalog initialization", - ) - except Exception as install_error: - cleanup_error = None - if _requires_compensating_catalog_cleanup(profile.name): - try: - with connection.begin(): - _compensate_catalog_initialization( - connection, metadata=metadata, before_tables=before_tables, - before_columns=before_columns, normative=normative, - legacy=legacy, - ) - except Exception as error: - cleanup_error = error - if cleanup_error is not None: - inventory = _catalog_residual_inventory( - engine, before_tables=before_tables, before_columns=before_columns, - ) - try: - after_dolt_state = _capture_dolt_state( - connection, profile_name=profile.name, audit_relations=set(), - ) - dolt_boundary = _dolt_failure_boundary_evidence( - pre_dolt_state, after_dolt_state, - ) - except Exception as snapshot_error: - dolt_boundary = { - "applicable": profile.name == "dolt", - "verified": False, - "snapshot_fault": _safe_error_identity( - snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", - ), - } - raise ImportRecoveryError( - "cleanup_failed", - "Catalog initialization failed and its DDL compensation also failed.", - details={ - "subcode": "catalog_install_cleanup_failed", - "original_cause": _safe_error_identity( - install_error, phase="catalog_initialization", - ), - "cleanup_fault": _safe_error_identity( - cleanup_error, phase="catalog_compensation", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.catalog-init-compensation.v1", - "action_id": _canonical_sha256({ - "namespace": active["catalog_binding"]["namespace"], - "before_tables": sorted(before_tables), - }), - "targets": { - "namespace": active["catalog_binding"]["namespace"], - "catalog_relations": sorted( - table.name for table in metadata.tables.values() - ), - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "cleanup_attempted": True, - "cleanup_succeeded": False, - "preexisting_unverified_catalog_mutation_forbidden": True, - "dolt_failure_boundary": dolt_boundary, - }, - "success_forbidden": True, - }, - ) from cleanup_error - try: - after_dolt_state = _capture_dolt_state( - connection, profile_name=profile.name, audit_relations=set(), - ) - dolt_boundary = _dolt_failure_boundary_evidence( - pre_dolt_state, after_dolt_state, - ) - except Exception as snapshot_error: - inventory = _catalog_residual_inventory( - engine, before_tables=before_tables, before_columns=before_columns, - ) - recovery = { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": _canonical_sha256({ - "namespace": active["catalog_binding"]["namespace"], - "before_tables": sorted(before_tables), - }), - "targets": { - "namespace": active["catalog_binding"]["namespace"], - "catalog_relations": sorted(metadata.tables), - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": { - "applicable": profile.name == "dolt", - "verified": False, - }, - } - raise ImportRecoveryError( - "cleanup_failed", - "Catalog compensation completed but Dolt state could not be verified.", - details={ - "subcode": "dolt_state_capture_failed", - "original_cause": _safe_error_identity( - install_error, phase="catalog_initialization", - ), - "cleanup_fault": _safe_error_identity( - snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": recovery, - "success_forbidden": True, - }, - ) from snapshot_error - if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): - inventory = _catalog_residual_inventory( - engine, before_tables=before_tables, before_columns=before_columns, - ) - recovery = { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": _canonical_sha256({ - "namespace": active["catalog_binding"]["namespace"], - "before_tables": sorted(before_tables), - }), - "targets": { - "namespace": active["catalog_binding"]["namespace"], - "catalog_relations": sorted(metadata.tables), - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": dolt_boundary, - } - raise ImportRecoveryError( - "cleanup_failed", - "Catalog compensation did not preserve Dolt failure-boundary invariants.", - details={ - "subcode": "dolt_state_invariant_failed", - "original_cause": _safe_error_identity( - install_error, phase="catalog_initialization", - ), - "cleanup_fault": _verification_fault_identity( - "dolt_state_invariant_failed", - phase="post_catalog_cleanup_dolt_state_verification", - evidence=dolt_boundary, - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": recovery, - "success_forbidden": True, - }, - ) from install_error - raise - return { - "profile": profile.name, - "server_version": active["server_version"], - "catalog": "verified", - } + create_normative_catalog(connection, normative) + except RuntimeError as error: + raise UnsupportedOperationError( + "The selected database catalog is foreign or incompatible; " + "initialization is not permitted." + ) from error + require_verified_catalog(connection) + return {"profile": profile.name, "catalog": "verified"} def _bounded_batches( - rows: list[dict[str, Any]], variables: list[dict[str, Any]], - maximum: int | None, -) -> Iterable[list[dict[str, Any]]]: - if maximum is None: - if rows: - yield rows - return - batch: list[dict[str, Any]] = [] - used = 0 + rows: Iterable[Mapping[str, Any]], variables: Iterable[Mapping[str, Any]], + maximum_statement_bytes: int | None, +) -> Iterable[list[Mapping[str, Any]]]: + """Partition rows without exceeding the profile's statement payload budget.""" + batch: list[Mapping[str, Any]] = [] + size = 0 for row in rows: - size = statement_payload_bytes(row, variables) - if size > maximum: - raise RuntimeError("A preflighted row exceeds the active statement payload limit.") - if batch and used + size > maximum: + row_size = statement_payload_bytes(row, variables) + if ( + batch and maximum_statement_bytes is not None + and size + row_size > maximum_statement_bytes + ): yield batch - batch, used = [], 0 + batch, size = [], 0 batch.append(row) - used += size + size += row_size if batch: yield batch -def _delete_normative_import_state( - connection: Any, normative: Any, *, dataset_name: str, - physical_table_name: str, normative_dataset_id: str | None, - normative_dataset_creation_attempted: bool, operation_id: str, -) -> None: - dataset_ids = [] - if normative_dataset_creation_attempted: - dataset_ids = list(connection.execute( - select(normative.dataset.c.dataset_id).where( - normative.dataset.c.dataset_name == dataset_name, - normative.dataset.c.physical_table_name == physical_table_name, - ) - ).scalars()) - if normative_dataset_id is not None and normative_dataset_id not in dataset_ids: - dataset_ids.append(normative_dataset_id) - connection.execute(delete(normative.fidelity_event).where( - normative.fidelity_event.c.operation_id == operation_id - )) - for normative_dataset_id in dataset_ids: - variable_ids = list(connection.execute( - select(normative.variable.c.variable_id) - .where(normative.variable.c.dataset_id == normative_dataset_id) - ).scalars()) - label_set_ids = list(connection.execute( - select(normative.value_label_set.c.value_label_set_id) - .where(normative.value_label_set.c.dataset_id == normative_dataset_id) - ).scalars()) - variable_set_ids = list(connection.execute( - select(normative.variable_set.c.variable_set_id) - .where(normative.variable_set.c.dataset_id == normative_dataset_id) - ).scalars()) - response_set_ids = list(connection.execute( - select(normative.multiple_response_set.c.multiple_response_set_id) - .where(normative.multiple_response_set.c.dataset_id == normative_dataset_id) - ).scalars()) - if variable_set_ids: - connection.execute(delete(normative.variable_set_member).where( - normative.variable_set_member.c.variable_set_id.in_(variable_set_ids) - )) - if response_set_ids: - connection.execute(delete(normative.multiple_response_member).where( - normative.multiple_response_member.c.multiple_response_set_id.in_(response_set_ids) - )) - if variable_ids: - for table in ( - normative.variable_value_label_set, normative.missing_rule, - normative.variable_attribute, - ): - connection.execute(delete(table).where(table.c.variable_id.in_(variable_ids))) - if label_set_ids: - connection.execute(delete(normative.value_label).where( - normative.value_label.c.value_label_set_id.in_(label_set_ids) - )) - for table in ( - normative.dataset_weight_variable, normative.dataset_attribute, - normative.document, normative.variable_set, normative.multiple_response_set, - normative.value_label_set, normative.fidelity_event, normative.variable, - ): - connection.execute(delete(table).where(table.c.dataset_id == normative_dataset_id)) - connection.execute(delete(normative.dataset).where( - normative.dataset.c.dataset_id == normative_dataset_id - )) - connection.execute(delete(normative.operation).where( - normative.operation.c.operation_id == operation_id - )) - - -def _create_operation_owned_data_table( - connection: Any, data_table: Table, state: dict[str, Any], -) -> None: - """Mark ownership only after this operation successfully creates the table.""" - data_table.create(connection) - state["data_table_created"] = True - - -def _cleanup_import_state( - connection: Any, *, dataset_id: str, operation_id: str, data_table: Table, - state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], -) -> None: - ( - datasets, variables, multiple_response, source_extensions, documents, - value_labels, missing_rules, attributes, fidelity_events, operations, - ) = legacy - _delete_normative_import_state( - connection, normative, dataset_name=dataset_id, - physical_table_name=data_table.name, - normative_dataset_id=state["normative_dataset_id"], - normative_dataset_creation_attempted=state["normative_dataset_creation_attempted"], - operation_id=operation_id, - ) - if state["legacy_dataset_created"]: - for table in ( - multiple_response, source_extensions, documents, value_labels, - missing_rules, attributes, - ): - connection.execute(delete(table).where(table.c.dataset_id == dataset_id)) - connection.execute(delete(variables).where(variables.c.dataset_id == dataset_id)) - connection.execute(delete(datasets).where(datasets.c.dataset_id == dataset_id)) - connection.execute(delete(fidelity_events).where( - fidelity_events.c.operation_id == operation_id - )) - connection.execute(delete(operations).where(operations.c.operation_id == operation_id)) - if state["data_table_created"]: - data_table.drop(connection, checkfirst=True) - - -def _record_failed_import_audit( - *, engine: Any, operation_id: str, source_name: str, source_format: str, - variable_count: int, profile_name: str, import_error: Exception, - normative: Any, legacy: tuple[Table, ...], -) -> None: - """Persist only a failed operation and NULL-dataset event after cleanup.""" - fidelity_events, operations = legacy[8:] - failed_event = { - "code": "import_failed", - "detail": "Import failed after mutation began; operation-owned state was removed.", - "severity": "error", - "source_item": source_name, - "details": { - "phase": "mutation", - "profile": profile_name, - "variable_count": variable_count, - "error_type": type(import_error).__name__, - }, - } - with engine.begin() as connection: - _require_verified_catalog(connection, normative, legacy) - failed_at = datetime.now(UTC).replace(tzinfo=None) - record_normative_operation( - connection, normative, operation_id=operation_id, - operation_kind="import", status="failed", source_format=source_format, - started_at=failed_at, completed_at=failed_at, - ) - connection.execute(insert(operations).values( - operation_id=operation_id, direction="import", status="failed", - dataset_id=None, source=source_name, created_at=_now(), - completed_at=_now(), details=json.dumps({ - "reason": "runtime_failure", - "variable_count": variable_count, - "error_type": type(import_error).__name__, - }, sort_keys=True), - )) - connection.execute(insert(fidelity_events), _event_rows( - operation_id=operation_id, dataset_id=None, direction="import", - fidelity_events=(failed_event,), - )) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, dataset_id=None, - direction="import", events=(failed_event,), - ) - - -def _record_import_cleanup_failure_audit( - *, engine: Any, operation_id: str, source_name: str, source_format: str, - profile_name: str, import_error: Exception, cleanup_error: Exception, - residual_object_inventory: Mapping[str, Any], - deterministic_recovery_evidence: Mapping[str, Any], - normative: Any, legacy: tuple[Table, ...], -) -> None: - """Best-effort immutable audit for verified-catalog cleanup failure.""" - fidelity_events, operations = legacy[8:] - original = _safe_error_identity(import_error, phase="import_mutation") - cleanup = _safe_error_identity(cleanup_error, phase="compensating_cleanup") - event_details = { - "original_cause": original, - "cleanup_fault": cleanup, - "residual_object_inventory": dict(residual_object_inventory), - "deterministic_recovery_evidence": dict( - deterministic_recovery_evidence - ), - } - event = { - "code": "cleanup_failed", - "detail": "Import cleanup failed; terminal recovery requires out-of-band review.", - "severity": "error", - "source_item": source_name, - "details": event_details, - } - with engine.begin() as connection: - _require_verified_catalog(connection, normative, legacy) - existing = connection.execute(select(operations).where( - operations.c.operation_id == operation_id - )).mappings().one_or_none() - normative_existing = connection.execute(select(normative.operation).where( - normative.operation.c.operation_id == operation_id - )).mappings().one_or_none() - if (existing is None) != (normative_existing is None): - raise UnsupportedOperationError( - "Import operation catalogs disagree about cleanup-failure state." - ) - if existing is not None: - if ( - existing["direction"] != "import" - or existing["status"] != "running" - or normative_existing["status"] != "started" - ): - raise UnsupportedOperationError( - "Existing import operation is not in an auditable running state." - ) - details = json.loads(existing["details"] or "{}") - details["cleanup_failure"] = event_details - connection.execute(update(operations).where( - operations.c.operation_id == operation_id - ).values( - status="failed", completed_at=_now(), - details=json.dumps(details, sort_keys=True), - )) - finish_normative_operation( - connection, normative, operation_id=operation_id, status="failed", - ) - ordinals = connection.execute(select(fidelity_events.c.ordinal).where( - fidelity_events.c.operation_id == operation_id - )).scalars().all() - event_row = _event_rows( - operation_id=operation_id, dataset_id=None, direction="import", - fidelity_events=(event,), - )[0] - event_row["ordinal"] = max(ordinals, default=0) + 1 - connection.execute(insert(fidelity_events).values(**event_row)) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, dataset_id=None, - direction="import", events=(event,), - ) - else: - failed_at = datetime.now(UTC).replace(tzinfo=None) - record_normative_operation( - connection, normative, operation_id=operation_id, - operation_kind="import", status="failed", source_format=source_format, - started_at=failed_at, completed_at=failed_at, - ) - connection.execute(insert(operations).values( - operation_id=operation_id, direction="import", status="failed", - dataset_id=None, source=source_name, created_at=_now(), - completed_at=_now(), details=json.dumps({ - "reason": "cleanup_failed", - "profile": profile_name, **event_details, - }, sort_keys=True), - )) - connection.execute(insert(fidelity_events), _event_rows( - operation_id=operation_id, dataset_id=None, direction="import", - fidelity_events=(event,), - )) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, dataset_id=None, - direction="import", events=(event,), - ) - - -def _import_residual_inventory( - engine: Any, *, dataset_id: str, operation_id: str, data_table: Table, - state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], -) -> dict[str, Any]: - try: - with engine.connect() as connection: - inspector = inspect(connection) - tables = set(inspector.get_table_names()) - - def count_rows(table: Table, condition: Any) -> int | None: - if table.name not in tables: - return None - return len(connection.execute(select(table).where(condition)).all()) - - return { - "data_table": { - "name": data_table.name, - "present": data_table.name in tables, - }, - "legacy_dataset_rows": count_rows( - legacy[0], legacy[0].c.dataset_id == dataset_id, - ), - "legacy_operation_rows": count_rows( - legacy[9], legacy[9].c.operation_id == operation_id, - ), - "legacy_fidelity_event_rows": count_rows( - legacy[8], legacy[8].c.operation_id == operation_id, - ), - "normative_dataset_rows": count_rows( - normative.dataset, - ( - normative.dataset.c.dataset_name == dataset_id - ) & ( - normative.dataset.c.physical_table_name == data_table.name - ), - ), - "normative_operation_rows": count_rows( - normative.operation, - normative.operation.c.operation_id == operation_id, - ), - "normative_fidelity_event_rows": count_rows( - normative.fidelity_event, - normative.fidelity_event.c.operation_id == operation_id, - ), - "mutation_markers": dict(state), - } - except Exception as inventory_error: - return {"inspection_error_type": type(inventory_error).__name__} - - -def _requires_compensating_import_cleanup(profile_name: str) -> bool: - """Transactional profiles rely on rollback, never stale cleanup markers.""" - return profile_name in MYSQL_WIRE_PROFILES - - -@contextmanager -def _import_cleanup_guard( - *, engine: Any, dataset_id: str, operation_id: str, data_table: Table, - source_name: str, source_format: str, variable_count: int, - profile_name: str, normative: Any, legacy: tuple[Table, ...], - snapshot_connection: Any, pre_dolt_state: dict[str, Any] | None, -) -> Iterable[dict[str, Any]]: - state: dict[str, Any] = { - "data_table_created": False, - "legacy_dataset_created": False, - "normative_dataset_creation_attempted": False, - "normative_dataset_id": None, - } - audit_relations = { - legacy[8].name, legacy[9].name, - normative.fidelity_event.name, normative.operation.name, +def _canonicalize_database_numeric_rows( + rows: Iterable[Mapping[str, Any]], + variables: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """Normalize finite DB-driver Decimal wrappers to binary64 values.""" + numeric_names = { + str(variable["physical_name"]) + for variable in variables + if variable.get("storage_kind") == "numeric" } + normalized = [] + for source in rows: + row = dict(source) + for name in numeric_names: + if isinstance(row.get(name), Decimal) and row[name].is_finite(): + row[name] = float(row[name]) + normalized.append(row) + return normalized - def capture_boundary() -> dict[str, Any]: - after = _capture_dolt_state( - snapshot_connection, profile_name=profile_name, - audit_relations=audit_relations, - ) - return _dolt_failure_boundary_evidence(pre_dolt_state, after) - try: - yield state - except Exception as import_error: - try: - if _requires_compensating_import_cleanup(profile_name): - with engine.begin() as cleanup_connection: - _cleanup_import_state( - cleanup_connection, dataset_id=dataset_id, - operation_id=operation_id, data_table=data_table, - state=state, normative=normative, legacy=legacy, - ) - except Exception as cleanup_error: - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=state, normative=normative, - legacy=legacy, - ) - try: - pre_audit_dolt_boundary = capture_boundary() - except Exception as snapshot_error: - pre_audit_dolt_boundary = { - "applicable": profile_name == "dolt", - "verified": False, - "snapshot_fault": _safe_error_identity( - snapshot_error, phase="post_cleanup_dolt_state_capture", - ), - } - audit_recovery = { - "procedure_id": "openstatspec.import-compensation.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "cleanup_attempted": True, - "cleanup_succeeded": False, - "operation_owned_state_targeted": True, - "dolt_failure_boundary": pre_audit_dolt_boundary, - } - cleanup_audit_fault = None - audit_permitted = not ( - pre_audit_dolt_boundary.get("applicable") - and not pre_audit_dolt_boundary.get("verified") - ) - if not audit_permitted: - cleanup_audit_fault = _verification_fault_identity( - "dolt_state_unverified_before_cleanup_failed_audit", - phase="pre_cleanup_failed_audit_boundary", - evidence=pre_audit_dolt_boundary, - ) - dolt_boundary = pre_audit_dolt_boundary - else: - try: - _record_import_cleanup_failure_audit( - engine=engine, operation_id=operation_id, - source_name=source_name, source_format=source_format, - profile_name=profile_name, import_error=import_error, - cleanup_error=cleanup_error, - residual_object_inventory=inventory, - deterministic_recovery_evidence=audit_recovery, - normative=normative, legacy=legacy, - ) - except Exception as audit_error: - cleanup_audit_fault = _safe_error_identity( - audit_error, phase="cleanup_failed_audit", - ) - try: - dolt_boundary = capture_boundary() - except Exception as snapshot_error: - dolt_boundary = { - "applicable": profile_name == "dolt", - "verified": False, - "snapshot_fault": _safe_error_identity( - snapshot_error, - phase="post_cleanup_audit_dolt_state_capture", - ), - } - raise ImportRecoveryError( - "cleanup_failed", - "Import failed and complete compensating cleanup also failed.", - details={ - "subcode": "import_cleanup_failed", - "original_cause": _safe_error_identity( - import_error, phase="import_mutation", - ), - "cleanup_fault": _safe_error_identity( - cleanup_error, phase="compensating_cleanup", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.import-compensation.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "cleanup_attempted": True, - "cleanup_succeeded": False, - "operation_owned_state_targeted": True, - "cleanup_failed_audit_persisted": cleanup_audit_fault is None, - "terminal_reporting": ( - "catalog_and_exception" if cleanup_audit_fault is None - else "out_of_band_exception" - ), - "dolt_failure_boundary": dolt_boundary, - }, - "audit_fault": cleanup_audit_fault, - "success_forbidden": True, - }, - ) from cleanup_error - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=state, normative=normative, - legacy=legacy, - ) - try: - pre_failed_audit_boundary = capture_boundary() - except Exception as snapshot_error: - raise ImportRecoveryError( - "cleanup_failed", - "Import cleanup completed but its pre-audit Dolt boundary could not be captured.", - details={ - "subcode": "pre_failed_audit_dolt_state_capture_failed", - "original_cause": _safe_error_identity( - import_error, phase="import_mutation", - ), - "cleanup_fault": _safe_error_identity( - snapshot_error, phase="pre_failed_audit_dolt_state_capture", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "failed_operation_audit_attempted": False, - "terminal_reporting": "out_of_band_exception", - "dolt_failure_boundary": { - "applicable": profile_name == "dolt", - "verified": False, - }, - }, - "audit_fault": None, - "success_forbidden": True, - }, - ) from snapshot_error - if ( - pre_failed_audit_boundary.get("applicable") - and not pre_failed_audit_boundary.get("verified") - ): - raise ImportRecoveryError( - "cleanup_failed", - "Import cleanup completed but its pre-audit Dolt boundary is unverified.", - details={ - "subcode": "pre_failed_audit_dolt_state_invariant_failed", - "original_cause": _safe_error_identity( - import_error, phase="import_mutation", - ), - "cleanup_fault": _verification_fault_identity( - "dolt_state_invariant_failed", - phase="pre_failed_audit_dolt_state_verification", - evidence=pre_failed_audit_boundary, - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "failed_operation_audit_attempted": False, - "terminal_reporting": "out_of_band_exception", - "dolt_failure_boundary": pre_failed_audit_boundary, - }, - "audit_fault": None, - "success_forbidden": True, - }, - ) from import_error - try: - _record_failed_import_audit( - engine=engine, operation_id=operation_id, - source_name=source_name, source_format=source_format, - variable_count=variable_count, profile_name=profile_name, - import_error=import_error, normative=normative, legacy=legacy, - ) - except Exception as audit_error: - raise ImportRecoveryError( - "failure_audit_failed", - "Import cleanup succeeded but its failed-operation audit could not be persisted.", - details={ - "original_cause": _safe_error_identity( - import_error, phase="import_mutation", - ), - "cleanup_fault": _safe_error_identity( - audit_error, phase="failed_operation_audit", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.failed-import-audit.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - }, - "success_forbidden": True, - }, - ) from audit_error - try: - dolt_boundary = capture_boundary() - except Exception as snapshot_error: - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=state, normative=normative, - legacy=legacy, - ) - raise ImportRecoveryError( - "cleanup_failed", - "Import cleanup completed but Dolt failure-boundary state could not be verified.", - details={ - "subcode": "dolt_state_capture_failed", - "original_cause": _safe_error_identity( - import_error, phase="import_mutation", - ), - "cleanup_fault": _safe_error_identity( - snapshot_error, phase="post_audit_dolt_state_capture", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": { - "applicable": profile_name == "dolt", - "verified": False, - }, - }, - "success_forbidden": True, - }, - ) from snapshot_error - if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=state, normative=normative, - legacy=legacy, - ) - raise ImportRecoveryError( - "cleanup_failed", - "Import cleanup did not preserve the Dolt failure-boundary invariants.", - details={ - "subcode": "dolt_state_invariant_failed", - "original_cause": _safe_error_identity( - import_error, phase="import_mutation", - ), - "cleanup_fault": _verification_fault_identity( - "dolt_state_invariant_failed", - phase="post_audit_dolt_state_verification", - evidence=dolt_boundary, - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": dolt_boundary, - }, - "success_forbidden": True, - }, - ) from import_error - raise +def data_table_name(dataset_id: str) -> str: + stem = _IDENTIFIER.sub("_", dataset_id).strip("_").lower() or "dataset" + return f"data_{stem[:48]}" def create_wide_dataset( *, database_url: str, dataset_id: str, source_name: str, source_format: str, - rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], file_label: str = "", - source_encoding: str | None = None, documents: str = "[]", - file_attributes: str = "{}", case_weight_variable: str | None = None, + rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], + file_label: str = "", documents: str = "[]", file_attributes: str = "{}", file_attribute_values: Mapping[str, Any] | None = None, variable_attribute_values: Mapping[str, Mapping[str, Any]] | None = None, - source_table_name: str | None = None, - source_sha256: str = "", + case_weight_variable: str | None = None, multiple_response_sets: str = "{}", + source_encoding: str | None = None, + source_table_name: str | None = None, source_sha256: str = "", source_created_at: str | None = None, source_modified_at: str | None = None, - imported_at: str = "", - multiple_response_sets: str = "{}", + imported_at: str | None = None, source_extensions: Mapping[str, Any] | None = None, fidelity_events: Iterable[Mapping[str, Any]] = (), operation_details: Mapping[str, Any] | None = None, - dolt_conformance_source: DoltConformanceSource | None = None, + dolt_conformance_source: Any | None = None, ) -> dict[str, Any]: + del source_table_name, source_created_at, source_modified_at + require_persistent_database_url(database_url) validate_connection_url(database_url) - profile, active = effective_profile( - database_url, dolt_conformance_source=dolt_conformance_source, + profile, active_connection = ( + effective_profile(database_url) + if dolt_conformance_source is None + else effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) ) engine = create_engine(database_url) - metadata = MetaData() - datasets, variable_catalog, fidelity_event_catalog, operation_catalog = catalog(metadata) - normative = normative_catalog(metadata) - multiple_response_catalog = multiple_response_set_catalog(metadata) - source_extensions_catalog = source_extension_catalog(metadata) - documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog = normalized_metadata_tables(metadata) - legacy = ( - datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, - documents_catalog, value_labels_catalog, missing_rules_catalog, - attributes_catalog, fidelity_event_catalog, operation_catalog, - ) + normative = normative_catalog(MetaData()) + audit_relations = {table.name for table in normative.all()} + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active_connection, + audit_relations=audit_relations, phase="catalog initialization", + ) as catalog_connection: + inspector = inspect(catalog_connection) + existing_tables = set(inspector.get_table_names()) + existing_views = set(inspector.get_view_names()) + if existing_tables or existing_views: + if normative.catalog_identity.name not in existing_tables: + raise UnsupportedOperationError( + "The selected database catalog is foreign; " + "import is not permitted." + ) + else: + create_normative_catalog(catalog_connection, normative) + require_verified_catalog(catalog_connection) operation_id = str(uuid4()) + normative_dataset_id = str(uuid4()) fidelity_events = tuple(fidelity_events) - source_rows = list(rows) - data_table = Table( - data_table_name(dataset_id), metadata, - Column("__case_ordinal", BigInteger, primary_key=True, nullable=False), - ) - audit_relations = { - fidelity_event_catalog.name, operation_catalog.name, - normative.fidelity_event.name, normative.operation.name, - } - preflight_state = { - "data_table_created": False, - "legacy_dataset_created": False, - "normative_dataset_creation_attempted": False, - "normative_dataset_id": None, - } - with engine.connect() as preflight_connection: - _require_verified_catalog(preflight_connection, normative, legacy) - preflight_dolt_state = _capture_dolt_state( - preflight_connection, profile_name=profile.name, - audit_relations=audit_relations, + source_rows = _canonicalize_database_numeric_rows(rows, variables) + try: + preflight(profile, variables, rows=source_rows) + validate_spss_catalog( + variables, + case_weight_variable=case_weight_variable, + multiple_response_sets=multiple_response_sets, ) - _require_dolt_working_set_binding( - preflight_dolt_state, active, phase="import preflight", + except Exception as error: + _record_failed_preflight( + engine=engine, normative=normative, operation_id=operation_id, + source_name=source_name, source_format=source_format, + variable_count=len(variables), profile_name=profile.name, + active=active_connection, error=error, ) - preflight_connection.rollback() - try: - preflight_identifier( - profile, data_table.name, role="physical data-table identifier", - ) - preflight(profile, variables, rows=source_rows) - validate_spss_catalog( - variables, - case_weight_variable=case_weight_variable, - multiple_response_sets=multiple_response_sets, + raise + + data_table = Table( + data_table_name(dataset_id), MetaData(), + Column("__case_ordinal", BigInteger, primary_key=True, nullable=False), + *( + Column( + item["physical_name"], + binary64_type() if item["storage_kind"] == "numeric" + else string_type(profile), + nullable=item["storage_kind"] == "numeric", ) - for item in variables: - data_table.append_column(Column( - item["physical_name"], - _wide_column_type(profile, item["storage_kind"]), - nullable=item["storage_kind"] == "numeric", - )) - except Exception as error: - try: - _record_failed_preflight( - engine=engine, metadata=metadata, datasets=datasets, - variable_catalog=variable_catalog, - multiple_response_catalog=multiple_response_catalog, - fidelity_event_catalog=fidelity_event_catalog, - operation_catalog=operation_catalog, operation_id=operation_id, - source_name=source_name, source_format=source_format, - variable_count=len(variables), profile_name=profile.name, - error=error, normative=normative, legacy=legacy, - ) - except Exception as audit_error: - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=preflight_state, - normative=normative, legacy=legacy, - ) - raise ImportRecoveryError( - "failure_audit_failed", - "Preflight failed and its failed-operation audit could not be persisted.", - details={ - "original_cause": _safe_error_identity( - error, phase="import_preflight", - ), - "cleanup_fault": _safe_error_identity( - audit_error, phase="failed_preflight_audit", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.failed-preflight-audit.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - }, - "success_forbidden": True, - }, - ) from audit_error - try: - post_preflight_dolt_state = _capture_dolt_state( - preflight_connection, profile_name=profile.name, - audit_relations=audit_relations, - ) - dolt_boundary = _dolt_failure_boundary_evidence( - preflight_dolt_state, post_preflight_dolt_state, - ) - except Exception as snapshot_error: - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=preflight_state, - normative=normative, legacy=legacy, - ) - raise ImportRecoveryError( - "cleanup_failed", - "Preflight audit completed but Dolt state could not be verified.", - details={ - "subcode": "dolt_state_capture_failed", - "original_cause": _safe_error_identity( - error, phase="import_preflight", - ), - "cleanup_fault": _safe_error_identity( - snapshot_error, phase="post_preflight_audit_dolt_state_capture", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": { - "applicable": profile.name == "dolt", - "verified": False, - }, - }, - "success_forbidden": True, - }, - ) from snapshot_error - if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): - inventory = _import_residual_inventory( - engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, state=preflight_state, - normative=normative, legacy=legacy, - ) - raise ImportRecoveryError( - "cleanup_failed", - "Preflight audit changed non-audit Dolt state.", - details={ - "subcode": "dolt_state_invariant_failed", - "original_cause": _safe_error_identity( - error, phase="import_preflight", - ), - "cleanup_fault": _verification_fault_identity( - "dolt_state_invariant_failed", - phase="post_preflight_audit_dolt_state_verification", - evidence=dolt_boundary, - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": operation_id, - "targets": { - "dataset_id": dataset_id, - "physical_table": data_table.name, - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": dolt_boundary, - }, - "success_forbidden": True, - }, - ) from error - raise - with engine.connect() as mutation_connection: - pre_dolt_state = _capture_dolt_state( - mutation_connection, profile_name=profile.name, - audit_relations=audit_relations, - ) - _require_dolt_working_set_binding( - pre_dolt_state, active, phase="import mutation preflight", - ) - mutation_connection.rollback() - with _import_cleanup_guard( - engine=engine, dataset_id=dataset_id, operation_id=operation_id, - data_table=data_table, source_name=source_name, - source_format=source_format, variable_count=len(variables), - profile_name=profile.name, normative=normative, legacy=legacy, - snapshot_connection=mutation_connection, - pre_dolt_state=pre_dolt_state, - ) as mutation: - with mutation_connection.begin(): - connection = mutation_connection - _require_verified_catalog(connection, normative, legacy) - record_normative_operation( - connection, normative, operation_id=operation_id, - operation_kind="import", status="started", source_format=source_format, - ) - connection.execute(insert(operation_catalog).values( - operation_id=operation_id, direction="import", status="running", dataset_id=dataset_id, - source=source_name, created_at=_now(), details=json.dumps({"variable_count": len(variables), **dict(operation_details or {})}, sort_keys=True), - )) - if connection.execute(select(datasets.c.dataset_id).where(datasets.c.dataset_id == dataset_id)).first(): - raise ValueError(f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset.") - if connection.execute(select(datasets.c.dataset_id).where(datasets.c.data_table == data_table.name)).first(): - raise ValueError(f"Dataset ID {dataset_id!r} collides with an existing physical data-table name; import was not started.") - _create_operation_owned_data_table( - connection, data_table, mutation, - ) - materialized = [ - {"__case_ordinal": ordinal, **row} - for ordinal, row in enumerate(source_rows, start=1) - ] - connection.execute(insert(datasets).values( - dataset_id=dataset_id, data_table=data_table.name, source_format=source_format, - source_name=source_name, source_encoding=source_encoding, case_count=len(materialized), - source_table_name=source_table_name, - source_sha256=source_sha256, - source_created_at=source_created_at, source_modified_at=source_modified_at, - imported_at=imported_at, - file_label=file_label, documents=documents, - file_attributes=file_attributes, case_weight_variable=case_weight_variable, - multiple_response_sets=multiple_response_sets, + for item in variables + ), + ) + materialized = [ + {"__case_ordinal": ordinal, **row} + for ordinal, row in enumerate(source_rows, start=1) + ] + docs_rows = document_rows(normative_dataset_id, documents) + labels_rows = value_label_rows(normative_dataset_id, variables) + missing_rows = missing_rule_rows(normative_dataset_id, variables) + attributes_rows = attribute_rows( + normative_dataset_id, variables, + file_attributes=file_attribute_values, + variable_attributes=variable_attribute_values, + ) + mrset_rows = multiple_response_set_rows( + normative_dataset_id, multiple_response_sets, + ) + namespace_owned = False + data_table_created = False + try: + with engine.begin() as setup: + require_verified_catalog(setup) + namespace_owned = True + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active_connection, + audit_relations={*audit_relations, data_table.name}, phase="import", + ) as connection: + require_verified_catalog(connection) + if connection.execute( + select(normative.dataset.c.dataset_id).where(or_( + normative.dataset.c.dataset_name == dataset_id, + normative.dataset.c.dataset_id == dataset_id, + normative.dataset.c.dataset_name == normative_dataset_id, )) - mutation["legacy_dataset_created"] = True - connection.execute(insert(variable_catalog), [dict(dataset_id=dataset_id, **item) for item in variables]) - docs_rows = document_rows(dataset_id, documents) - if docs_rows: - connection.execute(insert(documents_catalog), docs_rows) - labels_rows = value_label_rows(dataset_id, variables) - if labels_rows: - connection.execute(insert(value_labels_catalog), labels_rows) - missing_rows = missing_rule_rows(dataset_id, variables) - if missing_rows: - connection.execute(insert(missing_rules_catalog), missing_rows) - attributes_rows = attribute_rows( - dataset_id, variables, - file_attributes=file_attribute_values, - variable_attributes=variable_attribute_values, + ).first(): + raise ValueError( + f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset." ) - if attributes_rows: - connection.execute(insert(attributes_catalog), attributes_rows) - mrset_rows = multiple_response_set_rows(dataset_id, multiple_response_sets) - if mrset_rows: - connection.execute(insert(multiple_response_catalog), mrset_rows) - extension_rows = source_extension_rows(dataset_id, source_extensions or {}) - if extension_rows: - connection.execute(insert(source_extensions_catalog), extension_rows) - event_rows = _event_rows( - operation_id=operation_id, dataset_id=dataset_id, direction="import", fidelity_events=fidelity_events, + if connection.execute( + select(normative.dataset.c.dataset_id).where( + normative.dataset.c.physical_table_name == data_table.name, + normative.dataset.c.physical_table_schema.is_(None), ) - if event_rows: - connection.execute(insert(fidelity_event_catalog), event_rows) - mutation["normative_dataset_creation_attempted"] = True - normative_dataset_id = store_normative_dataset( - connection, normative, dataset_name=dataset_id, - source_format=source_format, physical_table_name=data_table.name, - dataset_label=file_label, source_encoding=source_encoding, - source_hash=source_sha256, source_case_count=len(materialized), - imported_at=imported_at or None, variables=variables, - documents=docs_rows, value_labels=labels_rows, - missing_rules=missing_rows, attributes=attributes_rows, - multiple_response_sets=mrset_rows, - source_extensions=source_extensions or {}, - case_weight_variable=case_weight_variable, + ).first(): + raise ValueError( + f"Dataset ID {dataset_id!r} collides with an existing physical " + "data-table name; import was not started." ) - mutation["normative_dataset_id"] = normative_dataset_id - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, - dataset_id=normative_dataset_id, direction="import", - events=fidelity_events, + if inspect(connection).has_table(data_table.name): + raise ValueError( + f"Physical data-table name {data_table.name!r} is already occupied." ) - if materialized: - for batch in _bounded_batches( - materialized, variables, profile.max_statement_bytes, + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="started", + source_format=source_format, + ) + data_table.create(connection) + data_table_created = True + store_normative_dataset( + connection, normative, dataset_name=dataset_id, + source_format=source_format, physical_table_name=data_table.name, + dataset_label=file_label, source_encoding=source_encoding, + source_hash=source_sha256, source_case_count=len(materialized), + imported_at=imported_at or None, variables=variables, + documents=docs_rows, value_labels=labels_rows, + missing_rules=missing_rows, attributes=attributes_rows, + multiple_response_sets=mrset_rows, + source_extensions=source_extensions or {}, + case_weight_variable=case_weight_variable, + dataset_id=normative_dataset_id, + ) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="import", + events=( + *fidelity_events, + *(({ + "code": "operation-engine-identity", + "detail": "Import engine identity recorded for audit.", + "severity": "info", + "source_item": source_name, + "details": dict(operation_details), + },) if operation_details else ()), + ), + ) + if materialized: + for batch in _bounded_batches( + materialized, variables, profile.max_statement_bytes, + ): + connection.execute(insert(data_table), batch) + finish_normative_operation( + connection, normative, operation_id=operation_id, + status="succeeded", + ) + except Exception as error: + if namespace_owned: + try: + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, + active=active_connection, + audit_relations={*audit_relations, data_table.name}, + phase="import cleanup", + ) as cleanup: + create_normative_catalog(cleanup, normative) + if cleanup.execute( + select(normative.dataset.c.dataset_id).where( + normative.dataset.c.dataset_id == normative_dataset_id + ) + ).first(): + delete_normative_dataset( + cleanup, normative, normative_dataset_id, + ) + table_is_owned = cleanup.execute( + select(normative.dataset.c.dataset_id).where( + normative.dataset.c.physical_table_name == data_table.name + ) + ).first() + if ( + data_table_created + and profile.name != "postgresql" + and table_is_owned is None + and inspect(cleanup).has_table(data_table.name) ): - connection.execute(insert(data_table), batch) - connection.execute(update(operation_catalog).where(operation_catalog.c.operation_id == operation_id).values( - status="succeeded", completed_at=_now(), - )) - finish_normative_operation( - connection, normative, operation_id=operation_id, status="succeeded", - ) - post_dolt_state = _capture_dolt_state( - mutation_connection, profile_name=profile.name, - audit_relations=audit_relations, - ) - _require_dolt_working_set_binding( - post_dolt_state, active, phase="import completion", - ) - _require_dolt_success_identity( - pre_dolt_state, post_dolt_state, phase="import", - ) - return {"dataset_id": dataset_id, "data_table": data_table.name, "case_count": len(materialized), "operation_id": operation_id} - - -def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any: - endpoint_type = row[f"{prefix}_type"] - if endpoint_type == "lowest": - return -sys.float_info.max - if endpoint_type == "highest": - return sys.float_info.max - return row[f"{prefix}_numeric"] if endpoint_type == "numeric" else row[f"{prefix}_text"] - + data_table.drop(cleanup, checkfirst=True) + operation_exists = cleanup.execute( + select(normative.operation.c.operation_id).where( + normative.operation.c.operation_id == operation_id + ) + ).first() + if operation_exists: + finish_normative_operation( + cleanup, normative, operation_id=operation_id, + status="failed", + ) + else: + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + cleanup, normative, operation_id=operation_id, + operation_kind="import", status="failed", + source_format=source_format, + started_at=failed_at, completed_at=failed_at, + ) + record_normative_fidelity_events( + cleanup, normative, operation_id=operation_id, + dataset_id=None, direction="import", events=({ + "code": "import_failed", + "detail": str(error), "severity": "error", + "source_item": source_name, + "details": {"reason": "post_preflight"}, + },), + ) + except Exception as cleanup_error: + raise RuntimeError( + f"OpenStatSpec compensating cleanup failed: {cleanup_error}" + ) from cleanup_error + raise + return { + "dataset_id": normative_dataset_id, + "dataset_name": dataset_id, + "data_table": data_table.name, + "case_count": len(materialized), + "operation_id": operation_id, + } -def _canonicalize_database_numeric_rows( - rows: Iterable[Mapping[str, Any]], - variables: Iterable[Mapping[str, Any]], -) -> list[dict[str, Any]]: - """Convert exact numeric driver wrappers at the SQL read boundary. - MySQL-family drivers may return DOUBLE columns as Decimal instances. The - public adapter contract remains binary64-only, so database-native decimal - wrappers are converted back to their declared physical representation - before strict preflight validation. Other unexpected values are preserved - so preflight can reject them with its machine-readable diagnostic. - """ - numeric_names = { - str(variable["physical_name"]) - for variable in variables - if variable.get("storage_kind") == "numeric" - } - normalized: list[dict[str, Any]] = [] - for source_row in rows: - row = dict(source_row) - for physical_name in numeric_names: - value = row.get(physical_name) - if isinstance(value, Decimal): - row[physical_name] = float(value) - normalized.append(row) - return normalized def read_wide_dataset( - *, - database_url: str, - dataset_id: str, - dolt_conformance_source: DoltConformanceSource | None = None, + *, database_url: str, dataset_id: str, profile: Any | None = None, + dolt_conformance_source: Any | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: - """Read a strict dataset, preferring normalized metadata with JSON compatibility fallback.""" - profile, _active = effective_profile( - database_url, dolt_conformance_source=dolt_conformance_source, - ) + """Read an export descriptor from the normative catalog without mutation.""" + if profile is None: + profile, _active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) - metadata = MetaData() - datasets, variable_catalog, fidelity_catalog, operation_catalog = catalog(metadata) - normative = normative_catalog(metadata) - multiple_response_catalog = multiple_response_set_catalog(metadata) - source_extensions_catalog = source_extension_catalog(metadata) - documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog = normalized_metadata_tables(metadata) - legacy = ( - datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, - documents_catalog, value_labels_catalog, missing_rules_catalog, - attributes_catalog, fidelity_catalog, operation_catalog, - ) + normative = normative_catalog(MetaData()) with engine.connect() as connection: - _require_verified_catalog(connection, normative, legacy) - dataset = dict(connection.execute(select(datasets).where(datasets.c.dataset_id == dataset_id)).mappings().one()) - data_table = Table(dataset["data_table"], MetaData(), autoload_with=connection) - variables = [dict(item) for item in connection.execute( - select(variable_catalog).where(variable_catalog.c.dataset_id == dataset_id).order_by(variable_catalog.c.ordinal) - ).mappings().all()] - rows = _canonicalize_database_numeric_rows( - connection.execute( - select(data_table).order_by(data_table.c.__case_ordinal) - ).mappings().all(), - variables, + require_verified_catalog(connection) + dataset_row = _resolve_normative_dataset(connection, normative, dataset_id) + core_id = str(dataset_row["dataset_id"]) + data_table = Table( + str(dataset_row["physical_table_name"]), MetaData(), + schema=dataset_row["physical_table_schema"], + autoload_with=connection, ) - document_rows_result = connection.execute( - select(documents_catalog).where(documents_catalog.c.dataset_id == dataset_id) - .order_by(documents_catalog.c.ordinal) + source_variables = connection.execute( + select(normative.variable) + .where(normative.variable.c.dataset_id == core_id) + .order_by(normative.variable.c.source_ordinal) ).mappings().all() - label_rows_result = connection.execute( - select(value_labels_catalog).where(value_labels_catalog.c.dataset_id == dataset_id) - .order_by(value_labels_catalog.c.variable_ordinal, value_labels_catalog.c.ordinal) + variables = [_export_variable(row) for row in source_variables] + variables_by_id = { + str(row["variable_id"]): variable + for row, variable in zip(source_variables, variables, strict=True) + } + variable_ids = tuple(variables_by_id) + rows = [dict(row) for row in connection.execute( + select(data_table).order_by(data_table.c.__case_ordinal) + ).mappings()] + rows = _canonicalize_database_numeric_rows(rows, variables) + documents = connection.execute( + select(normative.document) + .where(normative.document.c.dataset_id == core_id) + .order_by(normative.document.c.source_ordinal) + ).mappings().all() + dataset_attributes = connection.execute( + select(normative.dataset_attribute) + .where(normative.dataset_attribute.c.dataset_id == core_id) + .order_by( + normative.dataset_attribute.c.attribute_name, + normative.dataset_attribute.c.array_ordinal, + ) ).mappings().all() - missing_rows_result = connection.execute( - select(missing_rules_catalog).where(missing_rules_catalog.c.dataset_id == dataset_id) - .order_by(missing_rules_catalog.c.variable_ordinal, missing_rules_catalog.c.ordinal) + variable_attributes = connection.execute( + select(normative.variable_attribute) + .where(normative.variable_attribute.c.variable_id.in_(variable_ids)) + .order_by( + normative.variable_attribute.c.variable_id, + normative.variable_attribute.c.attribute_name, + normative.variable_attribute.c.array_ordinal, + ) + ).mappings().all() + labels = connection.execute( + select( + normative.variable_value_label_set.c.variable_id, + normative.value_label, + ) + .join( + normative.value_label, + normative.value_label.c.value_label_set_id + == normative.variable_value_label_set.c.value_label_set_id, + ) + .where( + normative.variable_value_label_set.c.variable_id.in_(variable_ids) + ) + .order_by( + normative.variable_value_label_set.c.variable_id, + normative.value_label.c.ordinal, + ) ).mappings().all() - attribute_rows_result = connection.execute( - select(attributes_catalog).where(attributes_catalog.c.dataset_id == dataset_id) + missing_rules = connection.execute( + select(normative.missing_rule) + .where(normative.missing_rule.c.variable_id.in_(variable_ids)) .order_by( - attributes_catalog.c.scope, attributes_catalog.c.variable_ordinal, - attributes_catalog.c.attribute_ordinal, attributes_catalog.c.value_ordinal, + normative.missing_rule.c.variable_id, + normative.missing_rule.c.ordinal, ) ).mappings().all() - mrset_rows_result = connection.execute( - select(multiple_response_catalog).where(multiple_response_catalog.c.dataset_id == dataset_id) - .order_by(multiple_response_catalog.c.set_name, multiple_response_catalog.c.member_ordinal) + variable_sets = connection.execute( + select(normative.variable_set) + .where(normative.variable_set.c.dataset_id == core_id) + .order_by(normative.variable_set.c.source_ordinal) + ).mappings().all() + variable_set_ids = tuple( + str(row["variable_set_id"]) for row in variable_sets + ) + variable_set_members = connection.execute( + select(normative.variable_set_member) + .where(normative.variable_set_member.c.variable_set_id.in_(variable_set_ids)) + .order_by( + normative.variable_set_member.c.variable_set_id, + normative.variable_set_member.c.source_ordinal, + ) ).mappings().all() - extension_rows_result = connection.execute( - select(source_extensions_catalog).where(source_extensions_catalog.c.dataset_id == dataset_id) - .order_by(source_extensions_catalog.c.extension_key) + response_sets = connection.execute( + select(normative.multiple_response_set) + .where(normative.multiple_response_set.c.dataset_id == core_id) + .order_by(normative.multiple_response_set.c.source_ordinal) ).mappings().all() - - if document_rows_result: - dataset["documents"] = json.dumps([item["text"] for item in document_rows_result], ensure_ascii=False) - if attribute_rows_result: - file_attributes, variable_attributes = attributes_from_rows( - attribute_rows_result, variables=variables, + response_set_ids = tuple( + str(row["multiple_response_set_id"]) for row in response_sets ) - dataset["file_attributes"] = json.dumps(file_attributes, ensure_ascii=False) - for variable in variables: - variable["attributes"] = json.dumps( - variable_attributes.get(variable["source_name"], {}), ensure_ascii=False, + response_members = connection.execute( + select(normative.multiple_response_member) + .where( + normative.multiple_response_member.c.multiple_response_set_id.in_( + response_set_ids + ) + ) + .order_by( + normative.multiple_response_member.c.multiple_response_set_id, + normative.multiple_response_member.c.source_ordinal, + ) + ).mappings().all() + weight_id = connection.execute( + select(normative.dataset_weight_variable.c.variable_id).where( + normative.dataset_weight_variable.c.dataset_id == core_id ) - if mrset_rows_result: - dataset["multiple_response_sets"] = json.dumps( - multiple_response_sets_from_rows(mrset_rows_result), ensure_ascii=False, default=str, + ).scalar_one_or_none() + + dataset = { + "dataset_id": core_id, + "data_table": str(dataset_row["physical_table_name"]), + "physical_table_schema": dataset_row["physical_table_schema"], + "source_format": dataset_row["source_format"], + "source_encoding": dataset_row["source_encoding"], + "case_count": int(dataset_row["source_case_count"]), + "file_label": dataset_row["dataset_label"] or "", + "documents": json.dumps( + [row["document_text"] for row in documents], ensure_ascii=False, + ), + "file_attributes": json.dumps( + _collapse_attributes(dataset_attributes), ensure_ascii=False, + ), + "case_weight_variable": ( + variables_by_id[str(weight_id)]["source_name"] + if weight_id is not None else None + ), + } + for row in labels: + variable = variables_by_id[str(row["variable_id"])] + values = json.loads(variable["value_labels"]) + code = ( + row["numeric_code"] + if row["code_kind"] == "numeric" else row["string_code"] + ) + values[str(code)] = row["label"] + variable["value_labels"] = json.dumps(values, ensure_ascii=False) + for row in missing_rules: + variable = variables_by_id[str(row["variable_id"])] + rules = json.loads(variable["missing_ranges"]) + rules.append(_export_missing_rule(row)) + variable["missing_ranges"] = json.dumps(rules, ensure_ascii=False) + grouped_attributes: dict[str, list[Mapping[str, Any]]] = {} + for row in variable_attributes: + grouped_attributes.setdefault(str(row["variable_id"]), []).append(row) + for variable_id, attribute_rows in grouped_attributes.items(): + variables_by_id[variable_id]["attributes"] = json.dumps( + _collapse_attributes(attribute_rows), ensure_ascii=False, + ) + names_by_id = { + variable_id: variable["source_name"] + for variable_id, variable in variables_by_id.items() + } + members_by_variable_set: dict[str, list[str]] = {} + for row in variable_set_members: + members_by_variable_set.setdefault(str(row["variable_set_id"]), []).append( + names_by_id[str(row["variable_id"])] ) - if extension_rows_result: - dataset["source_extensions"] = { - item["extension_key"]: json.loads(item["payload"]) - for item in extension_rows_result + dataset["source_extensions"] = { + "spss.variable_sets": { + str(row["set_name"]): members_by_variable_set.get( + str(row["variable_set_id"]), [] + ) + for row in variable_sets } - variables_by_ordinal = {item["ordinal"]: item for item in variables} - labels_by_variable: dict[int, dict[Any, str]] = {} - for item in label_rows_result: - labels_by_variable.setdefault(item["variable_ordinal"], {})[ - item["numeric_value"] if item["value_type"] == "numeric" else item["text_value"] - ] = item["label"] - for ordinal, labels in labels_by_variable.items(): - variables_by_ordinal[ordinal]["value_labels"] = json.dumps(labels, ensure_ascii=False) - rules_by_variable: dict[int, list[dict[str, Any]]] = {} - for item in missing_rows_result: - lower = _endpoint_from_row(item, prefix="lower") - upper = _endpoint_from_row(item, prefix="upper") - rules_by_variable.setdefault(item["variable_ordinal"], []).append({"lo": lower, "hi": upper}) - for ordinal, rules in rules_by_variable.items(): - variables_by_ordinal[ordinal]["missing_ranges"] = json.dumps(rules, ensure_ascii=False) + } if variable_sets else {} + members_by_response_set: dict[str, list[str]] = {} + for row in response_members: + set_id = str(row["multiple_response_set_id"]) + variable_id = str(row["variable_id"]) + if variable_id not in names_by_id: + raise _catalog_error( + "multiple-response-member-not-found", + "A multiple-response set references an unknown variable.", + multiple_response_set_id=set_id, + variable_id=variable_id, + ) + members_by_response_set.setdefault(set_id, []).append( + names_by_id[variable_id] + ) + dataset["multiple_response_sets"] = json.dumps({ + str(row["set_name"]): _export_response_set( + row, + members_by_response_set.get( + str(row["multiple_response_set_id"]), [] + ), + ) + for row in response_sets + }, ensure_ascii=False) preflight(profile, variables, rows=rows) return dataset, variables, rows -def read_fidelity_events( +def _verify_normative_catalog(connection: Any, tables: Any) -> None: + if not inspect(connection).has_table(tables.catalog_identity.name): + raise UnsupportedOperationError("The OpenStatSpec catalog is absent.") + identities = connection.execute(select(tables.catalog_identity)).mappings().all() + if len(identities) != 1 or ( + identities[0]["catalog_identity_key"] != 1 + or identities[0]["contract_id"] != CATALOG_CONTRACT_ID + or identities[0]["schema_version"] != CATALOG_SCHEMA_VERSION + ): + + raise RuntimeError("The core OpenStatSpec catalog identity is incompatible.") + +def require_verified_catalog( + connection: Any, *, - database_url: str, - dataset_id: str, - dolt_conformance_source: DoltConformanceSource | None = None, + allowed_migrations: Mapping[str, set[str]] | None = None, +) -> None: + """Verify strict core shape, optional profiles, and owned physical relations.""" + normative = normative_catalog(MetaData()) + _verify_normative_catalog(connection, normative) + verify_catalog_relations( + connection, normative, allowed_migrations=allowed_migrations, + ) + + +def _resolve_normative_dataset( + connection: Any, tables: Any, identifier: str, +) -> Mapping[str, Any]: + row = connection.execute( + select(tables.dataset).where(tables.dataset.c.dataset_id == identifier) + ).mappings().one_or_none() + if row is not None: + return row + return connection.execute( + select(tables.dataset).where(tables.dataset.c.dataset_name == identifier) + ).mappings().one() + + +def _format_json(family: Any, width: Any, decimals: Any) -> str | None: + if family is None: + return None + try: + numeric_family = int(family) + except (TypeError, ValueError): + suffix = f".{int(decimals or 0)}" if decimals else "" + return f"{family}{int(width)}{suffix}" + return json.dumps([numeric_family, width, decimals]) + + +def _export_variable(row: Mapping[str, Any]) -> dict[str, Any]: + return { + "ordinal": int(row["source_ordinal"]), + "source_name": row["source_name"], + "physical_name": row["physical_name"], + "storage_kind": row["storage_kind"], + "string_width": row["declared_string_width"], + "label": row["variable_label"] or "", + "print_format": _format_json( + row["print_format_family"], row["print_format_width"], + row["print_format_decimals"], + ), + "write_format": _format_json( + row["write_format_family"], row["write_format_width"], + row["write_format_decimals"], + ), + "measure": row["measurement_level"], + "role": row["variable_role"], + "alignment": row["display_alignment"], + "display_width": row["display_width"], + "attributes": "{}", + "compat_name": None, + "value_labels": "{}", + "missing_ranges": "[]", + } + + +def _collapse_attributes(rows: Iterable[Mapping[str, Any]]) -> dict[str, Any]: + grouped: dict[str, list[str]] = {} + for row in rows: + grouped.setdefault(str(row["attribute_name"]), []).append( + str(row["attribute_value"]) + ) + return { + name: values[0] if len(values) == 1 else values + for name, values in grouped.items() + } + + +def _export_missing_rule(row: Mapping[str, Any]) -> Any: + if row["rule_kind"] == "discrete": + return ( + row["numeric_value"] + if row["code_kind"] == "numeric" else row["string_value"] + ) + lower = ( + -sys.float_info.max + if row["lower_special"] == "LOWEST" else row["numeric_lower"] + ) + upper = ( + sys.float_info.max + if row["upper_special"] == "HIGHEST" else row["numeric_upper"] + ) + return {"lo": lower, "hi": upper} + + +def _export_response_set( + row: Mapping[str, Any], members: list[str], +) -> dict[str, Any]: + definition: dict[str, Any] = { + "variable_list": members, + "is_dichotomy": str(row["set_kind"]).upper() == "MD", + "use_category_labels": row["category_label_behavior"] == "counted_values", + "use_first_var_label": row["label_source"] == "variable_label", + } + if row["set_label"] is not None: + definition["label"] = row["set_label"] + if row["counted_value_kind"] == "numeric": + definition["counted_value"] = row["counted_numeric_value"] + elif row["counted_value_kind"] == "string": + definition["counted_value"] = row["counted_string_value"] + return definition + + +def read_fidelity_events( + *, database_url: str, dataset_id: str, + direction: str | None = None, + dolt_conformance_source: Any | None = None, ) -> tuple[dict[str, Any], ...]: - """Read import-time fidelity diagnostics for a catalogued dataset.""" + """Read fidelity diagnostics, optionally limited to one lifecycle direction.""" effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - fidelity_event_catalog = legacy[8] + normative = normative_catalog(MetaData()) with engine.connect() as connection: - _require_verified_catalog(connection, normative, legacy) - events = connection.execute( - select(fidelity_event_catalog) - .where( - fidelity_event_catalog.c.dataset_id == dataset_id, - fidelity_event_catalog.c.direction == "import", + require_verified_catalog(connection) + dataset = _resolve_normative_dataset(connection, normative, dataset_id) + statement = ( + select(normative.fidelity_event) + .where(normative.fidelity_event.c.dataset_id == dataset["dataset_id"]) + .where(normative.fidelity_event.c.severity != "info") + ) + if direction is not None: + statement = statement.where( + normative.fidelity_event.c.direction == direction ) - .order_by(fidelity_event_catalog.c.code) - ).mappings().all() - return tuple({ - "code": item["code"], "detail": item["detail"], - "details": json.loads(item["details"] or "{}"), - } for item in events) - + events = connection.execute(statement.order_by( + normative.fidelity_event.c.event_code + )).mappings().all() + result = [] + for item in events: + details = json.loads(item["detail_json"] or "{}") + result.append({ + "code": item["event_code"], + "detail": details.pop("message", ""), + "details": details, + }) + return tuple(result) -def record_export_cleanup_failure( - *, database_url: str, destination: str, original_error: Exception, - cleanup_error: Exception, - residual_object_inventory: Mapping[str, Any], - deterministic_recovery_evidence: Mapping[str, Any], - operation_id: str | None = None, - dolt_conformance_source: DoltConformanceSource | None = None, -) -> str: - """Best-effort immutable export cleanup-failure audit.""" +@contextmanager +def _bound_export_audit_transaction( + *, database_url: str, dolt_conformance_source: Any | None, phase: str, +): + """Open one verified transaction bound to the effective Dolt working set.""" + validate_connection_url(database_url) + require_persistent_database_url(database_url) profile, active = effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - fidelity_events, operations = legacy[8:] - requested_operation_id = operation_id - operation_id = operation_id or str(uuid4()) - original = _safe_error_identity(original_error, phase="export") - cleanup = _safe_error_identity(cleanup_error, phase="export_destination_restore") - event_details = { - "original_cause": original, - "cleanup_fault": cleanup, - "residual_object_inventory": dict(residual_object_inventory), - "deterministic_recovery_evidence": dict( - deterministic_recovery_evidence - ), - } - event = { - "code": "cleanup_failed", - "detail": "Export destination recovery failed; out-of-band review is required.", - "severity": "error", - "source_item": destination, - "details": event_details, - } - audit_relations = { - legacy[8].name, legacy[9].name, - normative.fidelity_event.name, normative.operation.name, - } + normative = normative_catalog(MetaData()) with _bound_catalog_transaction( engine=engine, profile_name=profile.name, active=active, - audit_relations=audit_relations, phase="record export cleanup failure", + audit_relations={normative.operation.name, normative.fidelity_event.name}, + phase=phase, ) as connection: - _require_verified_catalog(connection, normative, legacy) - if requested_operation_id is not None: - existing = connection.execute(select(operations).where( - operations.c.operation_id == operation_id - )).mappings().one_or_none() - normative_existing = connection.execute(select(normative.operation).where( - normative.operation.c.operation_id == operation_id - )).mappings().one_or_none() - if ( - existing is None or normative_existing is None - or existing["direction"] != "export" - or existing["status"] != "running" - or normative_existing["status"] != "started" - ): - raise UnsupportedOperationError( - "Existing export operation is not in an auditable terminal-transition state." - ) - details = json.loads(existing["details"] or "{}") - details["cleanup_failure"] = event_details - connection.execute(update(operations).where( - operations.c.operation_id == operation_id - ).values( - status="failed", completed_at=_now(), - details=json.dumps(details, sort_keys=True), - )) - finish_normative_operation( - connection, normative, operation_id=operation_id, status="failed", - ) - ordinals = connection.execute(select(fidelity_events.c.ordinal).where( - fidelity_events.c.operation_id == operation_id - )).scalars().all() - event_row = _event_rows( - operation_id=operation_id, dataset_id=None, direction="export", - fidelity_events=(event,), - )[0] - event_row["ordinal"] = max(ordinals, default=0) + 1 - connection.execute(insert(fidelity_events).values(**event_row)) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, dataset_id=None, - direction="export", events=(event,), - ) - else: - failed_at = datetime.now(UTC).replace(tzinfo=None) - record_normative_operation( - connection, normative, operation_id=operation_id, - operation_kind="export", status="failed", source_format=None, - started_at=failed_at, completed_at=failed_at, - ) - connection.execute(insert(operations).values( - operation_id=operation_id, direction="export", status="failed", - dataset_id=None, destination=destination, created_at=_now(), - completed_at=_now(), details=json.dumps({ - "reason": "cleanup_failed", **event_details, - }, sort_keys=True), - )) - connection.execute(insert(fidelity_events), _event_rows( - operation_id=operation_id, dataset_id=None, direction="export", - fidelity_events=(event,), - )) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, dataset_id=None, - direction="export", events=(event,), - ) - return operation_id + require_verified_catalog(connection) + yield connection, normative def record_export_operation( @@ -3126,325 +1429,283 @@ def record_export_operation( allowed_fidelity_events: Iterable[Mapping[str, Any]], operation_details: Mapping[str, Any] | None = None, terminal: bool = True, - dolt_conformance_source: DoltConformanceSource | None = None, + dolt_conformance_source: Any | None = None, ) -> str: - """Persist a completed export and the fidelity loss explicitly accepted by its caller.""" - profile, active = effective_profile( - database_url, dolt_conformance_source=dolt_conformance_source, - ) - engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - datasets, variables, multiple_response = legacy[:3] - fidelity_events, operations = legacy[8:] + """Persist a completed export only in the normative audit catalog.""" + operation_details = dict(operation_details or {}) operation_id = str(uuid4()) events = tuple(allowed_fidelity_events) - audit_relations = { - legacy[8].name, legacy[9].name, - normative.fidelity_event.name, normative.operation.name, - } - with _bound_catalog_transaction( - engine=engine, profile_name=profile.name, active=active, - audit_relations=audit_relations, phase="record export operation", - ) as connection: - _require_verified_catalog(connection, normative, legacy) - normative_dataset_id = normative_dataset_id_for_name(connection, normative, dataset_id) + with _bound_export_audit_transaction( + database_url=database_url, dolt_conformance_source=dolt_conformance_source, + phase="export audit creation", + ) as (connection, normative): + _verify_normative_catalog(connection, normative) + dataset = _resolve_normative_dataset(connection, normative, dataset_id) completed_at = datetime.now(UTC).replace(tzinfo=None) - normative_status = "succeeded" if terminal else "started" - legacy_status = "succeeded" if terminal else "running" + status = "succeeded" if terminal else "started" record_normative_operation( connection, normative, operation_id=operation_id, - operation_kind="export", status=normative_status, source_format=None, + operation_kind="export", status=status, source_format=None, started_at=completed_at, completed_at=completed_at if terminal else None, ) - connection.execute(insert(operations).values( - operation_id=operation_id, direction="export", status=legacy_status, dataset_id=dataset_id, - destination=destination, created_at=_now(), - completed_at=_now() if terminal else None, - details=json.dumps({"allow_loss": [event["code"] for event in events], **dict(operation_details or {})}, sort_keys=True), - )) - rows = _event_rows( - operation_id=operation_id, dataset_id=dataset_id, direction="export", - fidelity_events=({**event, "severity": event.get("severity", "warning"), - "details": {**event.get("details", {}), "accepted_by_user": True}} - for event in events), - ) - if rows: - connection.execute(insert(fidelity_events), rows) record_normative_fidelity_events( connection, normative, operation_id=operation_id, - dataset_id=normative_dataset_id, direction="export", - events=({**event, "severity": event.get("severity", "warning"), - "details": {**event.get("details", {}), "accepted_by_user": True}} - for event in events), + dataset_id=str(dataset["dataset_id"]), direction="export", + events=( + *( + { + **event, + "severity": event.get("severity", "warning"), + "details": { + **event.get("details", {}), + "accepted_by_user": True, + }, + } + for event in events + ), + *(({ + "code": "operation-engine-identity", + "detail": "Export engine identity recorded for audit.", + "severity": "info", + "source_item": destination, + "details": operation_details, + },) if operation_details else ()), + { + "code": "export-operation-dataset-binding", + "detail": "Export operation dataset identity recorded for audit.", + "severity": "info", + "source_item": destination, + "details": {}, + }, + ), ) return operation_id +def _export_operation_row( + connection: Any, normative: Any, operation_id: str, +) -> Mapping[str, Any]: + row = connection.execute( + select(normative.operation).where( + normative.operation.c.operation_id == operation_id + ) + ).mappings().one_or_none() + if row is None or row["operation_kind"] != "export": + raise UnsupportedOperationError("The export operation does not exist.") + return row + + +def _export_operation_dataset_id( + connection: Any, normative: Any, operation_id: str, +) -> str: + dataset_ids = set(connection.execute( + select(normative.fidelity_event.c.dataset_id) + .where(normative.fidelity_event.c.operation_id == operation_id) + .where(normative.fidelity_event.c.dataset_id.is_not(None)) + ).scalars()) + if len(dataset_ids) != 1: + raise UnsupportedOperationError( + "The export operation is not bound to exactly one dataset." + ) + dataset_id = str(dataset_ids.pop()) + if connection.execute( + select(normative.dataset.c.dataset_id).where( + normative.dataset.c.dataset_id == dataset_id + ) + ).scalar_one_or_none() is None: + raise UnsupportedOperationError( + "The export operation dataset no longer exists." + ) + return dataset_id + + def finish_export_operation( - *, - database_url: str, - operation_id: str, - dolt_conformance_source: DoltConformanceSource | None = None, + *, database_url: str, operation_id: str, + dolt_conformance_source: Any | None = None, ) -> None: - """Mark a published export successful only after filesystem finalization.""" - profile, active = effective_profile( - database_url, dolt_conformance_source=dolt_conformance_source, - ) - engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - operations = legacy[9] - audit_relations = { - legacy[8].name, legacy[9].name, - normative.fidelity_event.name, normative.operation.name, - } - with _bound_catalog_transaction( - engine=engine, profile_name=profile.name, active=active, - audit_relations=audit_relations, phase="finish export operation", - ) as connection: - _require_verified_catalog(connection, normative, legacy) - row = connection.execute(select(operations).where( - operations.c.operation_id == operation_id - )).mappings().one() - if row["direction"] != "export" or row["status"] != "running": + """Mark a started normative export operation as succeeded.""" + with _bound_export_audit_transaction( + database_url=database_url, dolt_conformance_source=dolt_conformance_source, + phase="export audit finalization", + ) as (connection, normative): + _verify_normative_catalog(connection, normative) + row = _export_operation_row(connection, normative, operation_id) + if row["status"] != "started": raise UnsupportedOperationError( - "Only a running export operation can be finalized." + "Only a started export operation can be finalized." ) - connection.execute(update(operations).where( - operations.c.operation_id == operation_id - ).values(status="succeeded", completed_at=_now())) finish_normative_operation( connection, normative, operation_id=operation_id, status="succeeded", ) def read_export_operation_state( - *, - database_url: str, - operation_id: str, - dolt_conformance_source: DoltConformanceSource | None = None, + *, database_url: str, operation_id: str, + dolt_conformance_source: Any | None = None, ) -> dict[str, Any]: - """Read both export-operation catalogs without changing either one.""" - validate_connection_url(database_url) + """Read the singular normative export-operation state.""" effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) - engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - operations = legacy[9] - with engine.connect() as connection: - _require_verified_catalog(connection, normative, legacy) - legacy_row = connection.execute(select( - operations.c.direction, - operations.c.status, - ).where( - operations.c.operation_id == operation_id - )).mappings().one_or_none() - normative_row = connection.execute(select( - normative.operation.c.operation_kind, - normative.operation.c.status, - ).where( - normative.operation.c.operation_id == operation_id - )).mappings().one_or_none() - - legacy_state = ( - None if legacy_row is None else { - "direction": legacy_row["direction"], - "status": legacy_row["status"], - } - ) - normative_state = ( - None if normative_row is None else { - "operation_kind": normative_row["operation_kind"], - "status": normative_row["status"], - } - ) - if ( - legacy_state == {"direction": "export", "status": "succeeded"} - and normative_state == { - "operation_kind": "export", "status": "succeeded", - } - ): - classification = "succeeded" - elif ( - legacy_state == {"direction": "export", "status": "running"} - and normative_state == { - "operation_kind": "export", "status": "started", - } - ): - classification = "running" - else: - classification = "ambiguous" + normative = normative_catalog(MetaData()) + with create_engine(database_url).connect() as connection: + require_verified_catalog(connection) + row = _export_operation_row(connection, normative, operation_id) + classification = { + "started": "running", + "succeeded": "succeeded", + "failed": "failed", + }.get(str(row["status"]), "ambiguous") return { "operation_id": operation_id, - "legacy": legacy_state, - "normative": normative_state, + "normative": { + "operation_kind": row["operation_kind"], + "status": row["status"], + }, "classification": classification, } def fail_export_operation( - *, - database_url: str, - operation_id: str, + *, database_url: str, operation_id: str, failure_details: Mapping[str, Any], - dolt_conformance_source: DoltConformanceSource | None = None, + dolt_conformance_source: Any | None = None, ) -> None: - """Close one running export after filesystem compensation succeeded.""" - profile, active = effective_profile( - database_url, dolt_conformance_source=dolt_conformance_source, - ) - engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - fidelity_events, operations = legacy[8:] - audit_relations = { - fidelity_events.name, operations.name, - normative.fidelity_event.name, normative.operation.name, + """Close a started normative export after filesystem compensation.""" + event = { + "code": "export_failed", + "detail": "Export publication or finalization failed after audit start.", + "severity": "error", + "details": dict(failure_details), } - with _bound_catalog_transaction( - engine=engine, profile_name=profile.name, active=active, - audit_relations=audit_relations, phase="fail export operation", - ) as connection: - _require_verified_catalog(connection, normative, legacy) - row = connection.execute(select(operations).where( - operations.c.operation_id == operation_id - )).mappings().one() - normative_row = connection.execute(select(normative.operation).where( - normative.operation.c.operation_id == operation_id - )).mappings().one() - if ( - row["direction"] != "export" or row["status"] != "running" - or normative_row["operation_kind"] != "export" - or normative_row["status"] != "started" - ): + with _bound_export_audit_transaction( + database_url=database_url, dolt_conformance_source=dolt_conformance_source, + phase="export audit failure", + ) as (connection, normative): + _verify_normative_catalog(connection, normative) + row = _export_operation_row(connection, normative, operation_id) + if row["status"] != "started": raise UnsupportedOperationError( - "Only matching running export operations can be failed." + "Only a started export operation can be failed." ) - details = json.loads(row["details"] or "{}") - details["failure"] = dict(failure_details) - connection.execute(update(operations).where( - operations.c.operation_id == operation_id - ).values( - status="failed", completed_at=_now(), - details=json.dumps(details, sort_keys=True), - )) + dataset_id = _export_operation_dataset_id( + connection, normative, operation_id, + ) finish_normative_operation( connection, normative, operation_id=operation_id, status="failed", ) - event = { - "code": "export_failed", - "detail": "Export publication or finalization failed after audit start.", - "severity": "error", - "source_item": row["destination"], - "details": dict(failure_details), - } - ordinals = connection.execute(select(fidelity_events.c.ordinal).where( - fidelity_events.c.operation_id == operation_id - )).scalars().all() - event_row = _event_rows( - operation_id=operation_id, dataset_id=row["dataset_id"], - direction="export", fidelity_events=(event,), - )[0] - event_row["ordinal"] = max(ordinals, default=0) + 1 - connection.execute(insert(fidelity_events).values(**event_row)) - normative_dataset_id = normative_dataset_id_for_name( - connection, normative, row["dataset_id"], - ) record_normative_fidelity_events( connection, normative, operation_id=operation_id, - dataset_id=normative_dataset_id, direction="export", events=(event,), + dataset_id=dataset_id, direction="export", events=(event,), ) def record_export_backup_retained( *, database_url: str, operation_id: str, destination: str, backup: str, cleanup_error: Exception, - dolt_conformance_source: DoltConformanceSource | None = None, + dolt_conformance_source: Any | None = None, ) -> None: - """Append a warning without rewriting a successfully finalized export.""" - profile, active = effective_profile( - database_url, dolt_conformance_source=dolt_conformance_source, - ) - engine = create_engine(database_url) - metadata = MetaData() - legacy, normative = _catalog_layout(metadata) - fidelity_events, operations = legacy[8:] - audit_relations = { - fidelity_events.name, operations.name, - normative.fidelity_event.name, normative.operation.name, - } - with _bound_catalog_transaction( - engine=engine, profile_name=profile.name, active=active, - audit_relations=audit_relations, phase="record retained export backup", - ) as connection: - _require_verified_catalog(connection, normative, legacy) - row = connection.execute(select(operations).where( - operations.c.operation_id == operation_id - )).mappings().one() - normative_row = connection.execute(select(normative.operation).where( - normative.operation.c.operation_id == operation_id - )).mappings().one() - if ( - row["direction"] != "export" or row["status"] != "succeeded" - or normative_row["operation_kind"] != "export" - or normative_row["status"] != "succeeded" - ): + """Append a warning to a successfully finalized normative export.""" + with _bound_export_audit_transaction( + database_url=database_url, dolt_conformance_source=dolt_conformance_source, + phase="export backup-retention audit", + ) as (connection, normative): + _verify_normative_catalog(connection, normative) + row = _export_operation_row(connection, normative, operation_id) + if row["status"] != "succeeded": raise UnsupportedOperationError( - "A retained backup warning requires a matching succeeded export." + "A retained backup warning requires a succeeded export." ) - details = json.loads(row["details"] or "{}") - details["backup_retained"] = { - "destination": destination, "durable_backup": backup, - "cleanup_error_type": type(cleanup_error).__name__, - } - connection.execute(update(operations).where( - operations.c.operation_id == operation_id - ).values(details=json.dumps(details, sort_keys=True))) - event = { - "code": "backup_retained", - "detail": "A successful export retained its durable prior-file backup.", - "severity": "warning", - "source_item": destination, - "details": { - "durable_backup": backup, - "cleanup_error_type": type(cleanup_error).__name__, - }, - } - ordinals = connection.execute(select(fidelity_events.c.ordinal).where( - fidelity_events.c.operation_id == operation_id - )).scalars().all() - event_row = _event_rows( - operation_id=operation_id, dataset_id=row["dataset_id"], - direction="export", fidelity_events=(event,), - )[0] - event_row["ordinal"] = max(ordinals, default=0) + 1 - connection.execute(insert(fidelity_events).values(**event_row)) - normative_dataset_id = normative_dataset_id_for_name( - connection, normative, row["dataset_id"], + dataset_id = _export_operation_dataset_id( + connection, normative, operation_id, + ) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=dataset_id, direction="export", events=({ + "code": "backup_retained", + "detail": "A successful export retained its durable prior-file backup.", + "severity": "warning", + "source_item": destination, + "details": { + "durable_backup": backup, + "cleanup_error_type": type(cleanup_error).__name__, + }, + },), ) + + +def record_export_cleanup_failure( + *, database_url: str, destination: str, original_error: Exception, + cleanup_error: Exception, + residual_object_inventory: Mapping[str, Any], + deterministic_recovery_evidence: Mapping[str, Any], + operation_id: str | None = None, + dolt_conformance_source: Any | None = None, +) -> str: + """Persist terminal cleanup failure in the normative audit catalog.""" + operation_id = operation_id or str(uuid4()) + event = { + "code": "cleanup_failed", + "detail": "Export destination recovery failed; out-of-band review is required.", + "severity": "error", + "source_item": destination, + "details": { + "original_error_type": type(original_error).__name__, + "cleanup_error_type": type(cleanup_error).__name__, + "residual_object_inventory": dict(residual_object_inventory), + "deterministic_recovery_evidence": dict( + deterministic_recovery_evidence + ), + }, + } + with _bound_export_audit_transaction( + database_url=database_url, dolt_conformance_source=dolt_conformance_source, + phase="export cleanup-failure audit", + ) as (connection, normative): + _verify_normative_catalog(connection, normative) + row = connection.execute( + select(normative.operation).where( + normative.operation.c.operation_id == operation_id + ) + ).mappings().one_or_none() + dataset_id = None + if row is None: + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="export", status="failed", source_format=None, + started_at=failed_at, completed_at=failed_at, + ) + else: + if row["operation_kind"] != "export" or row["status"] != "started": + raise UnsupportedOperationError( + "Existing export operation cannot transition to cleanup failure." + ) + dataset_id = _export_operation_dataset_id( + connection, normative, operation_id, + ) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) record_normative_fidelity_events( connection, normative, operation_id=operation_id, - dataset_id=normative_dataset_id, direction="export", events=(event,), + dataset_id=dataset_id, direction="export", events=(event,), ) + return operation_id def validate_wide_dataset( - *, - database_url: str, - dataset_id: str, - dolt_conformance_source: DoltConformanceSource | None = None, + *, database_url: str, dataset_id: str, + dolt_conformance_source: Any | None = None, ) -> dict[str, Any]: profile, _active = effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) dataset, variables, rows = read_wide_dataset( - database_url=database_url, - dataset_id=dataset_id, + database_url=database_url, dataset_id=dataset_id, profile=profile, dolt_conformance_source=dolt_conformance_source, ) + preflight(profile, variables, rows=rows) validate_spss_catalog( variables, case_weight_variable=dataset.get("case_weight_variable"), @@ -3453,7 +1714,12 @@ def validate_wide_dataset( if not variables: raise ValueError("A conforming dataset needs at least one source variable.") expected_columns = {"__case_ordinal", *(item["physical_name"] for item in variables)} - reflected_table = Table(dataset["data_table"], MetaData(), autoload_with=create_engine(database_url)) + engine = create_engine(database_url) + reflected_table = Table( + dataset["data_table"], MetaData(), + schema=dataset["physical_table_schema"], + autoload_with=engine, + ) reflected_columns = {column.name: column for column in reflected_table.columns} actual_columns = set(reflected_columns) if actual_columns != expected_columns: @@ -3470,7 +1736,12 @@ def validate_wide_dataset( if item["storage_kind"] == "numeric": if not isinstance(column.type, Float) or not column.nullable: raise ValueError(f"Numeric variable {item['source_name']!r} must be a nullable binary64 column.") - elif not _valid_wide_string_type(profile, column.type) or column.nullable: + elif profile.name == "dolt": + if not isinstance(column.type, mysql.LONGTEXT) or column.nullable: + raise ValueError( + f"String variable {item['source_name']!r} must be a non-null LONGTEXT column." + ) + elif not isinstance(column.type, Text) or column.nullable: raise ValueError(f"String variable {item['source_name']!r} must be a non-null text column.") if [row["__case_ordinal"] for row in rows] != list(range(1, len(rows) + 1)): raise ValueError("Case ordinals are not contiguous source order.") diff --git a/src/openstatspec/transform/__init__.py b/src/openstatspec/transform/__init__.py index bf89591..bf220bd 100644 --- a/src/openstatspec/transform/__init__.py +++ b/src/openstatspec/transform/__init__.py @@ -2,9 +2,11 @@ from .errors import SourcePosition, SourceSpan, TransformationFrontendError from .plan import ( + TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT, TRANSFORMATION_PLAN_CONTRACT, AssignOperation, BooleanExpression, ComparisonExpression, - ConditionalAssignOperation, ExecuteOperation, Operand, PredicateExpression, + ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation, + ExecuteOperation, Operand, PredicateExpression, RecodeMatch, RecodeOperation, RecodeResult, RecodeRule, ReplaceValueLabelsOperation, SetFormatOperation, SetMeasurementLevelOperation, SetVariableLabelOperation, TransformationPlan, @@ -17,15 +19,17 @@ from .validation import bind_transformation_plan __all__ = [ "AssignOperation", "BooleanExpression", "ComparisonExpression", - "ConditionalAssignOperation", "ExecuteOperation", "Operand", - "PredicateExpression", + "ConditionalAssignOperation", "CreateVariableOperation", "DeleteVariableOperation", + "ExecuteOperation", "Operand", "PredicateExpression", "BoundTransformation", "RecodeMatch", "RecodeOperation", "RecodeResult", "RecodeRule", "ReplaceValueLabelsOperation", "SPSS_FRONTEND_CONTRACT", + "SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT", "SetFormatOperation", "SetMeasurementLevelOperation", "SetVariableLabelOperation", "SourcePosition", "SourceSpan", "StorageKind", "SpssFrontendCompilation", - "SpssSyntaxProgram", "TRANSFORMATION_PLAN_CONTRACT", "TransformationFrontendError", + "SpssSyntaxProgram", "TRANSFORMATION_PLAN_CONTRACT", + "TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT", "TransformationFrontendError", "TransformationPlan", "TypedValue", "ValueLabel", "VariableDefinition", "VariableSchema", "bind_spss_syntax", "bind_transformation_plan", @@ -39,6 +43,7 @@ _SPSS_COMPAT_EXPORTS = { "SPSS_FRONTEND_CONTRACT", + "SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT", "SpssFrontendCompilation", "SpssSyntaxProgram", "bind_spss_syntax", diff --git a/src/openstatspec/transform/plan.py b/src/openstatspec/transform/plan.py index 97523d6..74bef7c 100644 --- a/src/openstatspec/transform/plan.py +++ b/src/openstatspec/transform/plan.py @@ -16,7 +16,8 @@ TRANSFORMATION_PLAN_V1_CONTRACT = "openstatspec-transformation-plan-v0.1" TRANSFORMATION_PLAN_CONTRACT = "openstatspec-transformation-plan-v0.2" -_TRANSFORMATION_PLAN_CONTRACTS = {TRANSFORMATION_PLAN_V1_CONTRACT, TRANSFORMATION_PLAN_CONTRACT} +TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT = "openstatspec-transformation-plan-v0.3" +_TRANSFORMATION_PLAN_CONTRACTS = {TRANSFORMATION_PLAN_V1_CONTRACT, TRANSFORMATION_PLAN_CONTRACT, TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT} _BINARY64 = re.compile(r"[0-9a-f]{16}") @@ -504,10 +505,77 @@ def as_dict(self) -> dict[str, str]: return {"op": self.op} + +@dataclass(frozen=True) +class CreateVariableOperation: + """Add one variable to the existing dataset and wide table.""" + + variable: str + storage_kind: Literal["numeric", "string"] + declared_string_width: int | None = None + op: Literal["create_variable"] = "create_variable" + + def __post_init__(self) -> None: + if self.op != "create_variable": + _invalid("Create-variable operation discriminator is invalid.") + if not isinstance(self.variable, str) or not self.variable: + _invalid("Created variable name must be non-empty text.") + if self.variable.startswith("__"): + raise frontend_error( + "reserved_target_name", + f"Target name {self.variable!r} is reserved.", + target=self.variable, + ) + if self.storage_kind not in {"numeric", "string"}: + _invalid("Created variable storage_kind must be numeric or string.") + if self.storage_kind == "string": + if ( + not isinstance(self.declared_string_width, int) + or isinstance(self.declared_string_width, bool) + or self.declared_string_width < 1 + ): + _invalid("String variables require a positive declared_string_width.") + elif self.declared_string_width is not None: + _invalid("Numeric variables cannot declare a string width.") + + def as_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "op": self.op, + "variable": self.variable, + "storage_kind": self.storage_kind, + } + if self.declared_string_width is not None: + result["declared_string_width"] = self.declared_string_width + return result + + +@dataclass(frozen=True) +class DeleteVariableOperation: + """Remove one variable and all of its normative metadata.""" + + variable: str + op: Literal["delete_variable"] = "delete_variable" + + def __post_init__(self) -> None: + if self.op != "delete_variable": + _invalid("Delete-variable operation discriminator is invalid.") + if not isinstance(self.variable, str) or not self.variable: + _invalid("Deleted variable name must be non-empty text.") + if self.variable.startswith("__"): + raise frontend_error( + "reserved_target_name", + f"Variable name {self.variable!r} is reserved.", + variable=self.variable, + ) + + def as_dict(self) -> dict[str, str]: + return {"op": self.op, "variable": self.variable} + PlanOperation = ( RecodeOperation | AssignOperation | ConditionalAssignOperation | SetVariableLabelOperation | ReplaceValueLabelsOperation | SetFormatOperation | SetMeasurementLevelOperation | ExecuteOperation + | CreateVariableOperation | DeleteVariableOperation ) @@ -520,10 +588,24 @@ class TransformationPlan: def __post_init__(self) -> None: if not isinstance(self.contract, str) or self.contract not in _TRANSFORMATION_PLAN_CONTRACTS: _invalid("Plan contract is not a supported transformation-plan contract.") + if ( + self.contract != TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT + and any( + isinstance(operation, ( + CreateVariableOperation, DeleteVariableOperation, + )) + for operation in self.operations + ) + ): + _invalid( + "Create/delete schema operations require " + "openstatspec-transformation-plan-v0.3." + ) if self.contract == TRANSFORMATION_PLAN_V1_CONTRACT and any( isinstance(operation, ( AssignOperation, ConditionalAssignOperation, SetFormatOperation, SetMeasurementLevelOperation, ExecuteOperation, + CreateVariableOperation, DeleteVariableOperation, )) for operation in self.operations ): @@ -539,6 +621,7 @@ def __post_init__(self) -> None: RecodeOperation, AssignOperation, ConditionalAssignOperation, SetVariableLabelOperation, ReplaceValueLabelsOperation, SetFormatOperation, SetMeasurementLevelOperation, ExecuteOperation, + CreateVariableOperation, DeleteVariableOperation, ), ) for operation in self.operations @@ -639,9 +722,8 @@ def _match(raw: Any) -> RecodeMatch: return RecodeMatch("system_missing") _invalid("Unknown recode match kind.") - def transformation_plan_from_dict(raw: Mapping[str, Any]) -> TransformationPlan: - """Strictly validate canonical v0.1 or additive v0.2 plan documents.""" + """Strictly validate canonical v0.1, v0.2, or schema-change v0.3 plans.""" if not isinstance(raw, Mapping): _invalid("Transformation plan must be an object.") _exact(raw, {"contract", "input_alias", "operations"}, "Transformation plan") @@ -733,6 +815,22 @@ def transformation_plan_from_dict(raw: Mapping[str, Any]) -> TransformationPlan: operations.append(ReplaceValueLabelsOperation( raw_operation["variable"], tuple(labels), )) + elif operation == "create_variable": + _exact( + raw_operation, + {"op", "variable", "storage_kind", "declared_string_width"} + if "declared_string_width" in raw_operation + else {"op", "variable", "storage_kind"}, + "Create-variable operation", + ) + operations.append(CreateVariableOperation( + variable=raw_operation["variable"], + storage_kind=raw_operation["storage_kind"], + declared_string_width=raw_operation.get("declared_string_width"), + )) + elif operation == "delete_variable": + _exact(raw_operation, {"op", "variable"}, "Delete-variable operation") + operations.append(DeleteVariableOperation(raw_operation["variable"])) else: _invalid(f"Unknown plan operation {operation!r}.") return TransformationPlan( diff --git a/src/openstatspec/transform/schema.py b/src/openstatspec/transform/schema.py index 5dace1d..4b66135 100644 --- a/src/openstatspec/transform/schema.py +++ b/src/openstatspec/transform/schema.py @@ -21,6 +21,7 @@ class VariableDefinition: format_width: int | None = None format_decimals: int | None = None measurement_level: Literal["nominal", "ordinal", "scale"] | None = None + declared_string_width: int | None = None def __post_init__(self) -> None: if not isinstance(self.name, str) or not self.name: @@ -28,6 +29,17 @@ def __post_init__(self) -> None: if self.storage_kind not in {"numeric", "string"}: raise ValueError("storage_kind must be numeric or string.") expected_type = "binary64" if self.storage_kind == "numeric" else "string" + if self.declared_string_width is not None: + if ( + not isinstance(self.declared_string_width, int) + or isinstance(self.declared_string_width, bool) + or self.declared_string_width < 1 + ): + raise ValueError("declared_string_width must be a positive integer.") + if self.storage_kind != "string": + raise ValueError( + "declared_string_width is only valid for string variables." + ) if any(label.value.type != expected_type for label in self.value_labels): raise ValueError( "Value-label types must match their variable storage kind." diff --git a/src/openstatspec/transform/validation.py b/src/openstatspec/transform/validation.py index c02733d..d3021c0 100644 --- a/src/openstatspec/transform/validation.py +++ b/src/openstatspec/transform/validation.py @@ -10,7 +10,9 @@ AssignOperation, BooleanExpression, ComparisonExpression, + CreateVariableOperation, ConditionalAssignOperation, + DeleteVariableOperation, ExecuteOperation, Operand, RecodeMatch, @@ -271,7 +273,6 @@ def _bind_conditional_assign( variable=target.name, ) - def bind_transformation_plan( plan: TransformationPlan, schema: VariableSchema ) -> BoundTransformation: @@ -282,6 +283,12 @@ def bind_transformation_plan( raise TypeError("schema must be a VariableSchema.") variables = list(schema.variables) for operation in plan.operations: + if isinstance(operation, CreateVariableOperation): + _bind_create(operation, variables) + continue + if isinstance(operation, DeleteVariableOperation): + _bind_delete(operation, variables) + continue if isinstance(operation, RecodeOperation): _bind_recode(operation, variables) continue @@ -305,6 +312,16 @@ def bind_transformation_plan( variable=variable.name, expected_type=expected, ) + if variable.storage_kind == "string" and variable.declared_string_width is not None: + for label in operation.labels: + assert label.value.value is not None + if len(label.value.value.encode("utf-8")) > variable.declared_string_width: + raise frontend_error( + "string_width_exceeded", + "A value-label code exceeds the variable's declared string width.", + variable=variable.name, + declared_string_width=variable.declared_string_width, + ) variables[index] = replace(variable, value_labels=operation.labels) continue if isinstance(operation, SetFormatOperation): @@ -330,3 +347,32 @@ def bind_transformation_plan( continue raise AssertionError(f"Unknown plan operation: {type(operation)!r}") return BoundTransformation(plan, VariableSchema(tuple(variables))) +def _bind_create( + operation: CreateVariableOperation, + variables: list[VariableDefinition], +) -> None: + if any(variable.name.casefold() == operation.variable.casefold() for variable in variables): + raise frontend_error( + "target_already_exists", + f"Target name {operation.variable!r} already exists.", + target=operation.variable, + ) + variables.append(VariableDefinition( + operation.variable, + operation.storage_kind, + declared_string_width=operation.declared_string_width, + )) + + +def _bind_delete( + operation: DeleteVariableOperation, + variables: list[VariableDefinition], +) -> None: + index, variable = _resolve(variables, operation.variable) + if len(variables) == 1: + raise frontend_error( + "cannot_delete_last_variable", + "A dataset must retain at least one variable.", + variable=variable.name, + ) + del variables[index] diff --git a/tests/conformance.py b/tests/conformance.py index 677f865..dbe1ae3 100755 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -143,7 +143,7 @@ def compare_sav_semantics(source: str | Path, exported: str | Path) -> dict[str, for attribute in ( "encoding", "file_label", "case_weight_var", "file_attributes", "mrsets", "var_types", "var_formats", "var_labels", "var_alignments", "var_column_widths", "var_measure_levels", "var_roles", - "var_value_labels", "var_attributes", "var_compat_names", + "var_value_labels", "var_attributes", ): if source_metadata.get(attribute) != exported_metadata.get(attribute): failures.append(attribute) diff --git a/tests/test_atomic_import.py b/tests/test_atomic_import.py index be54772..95bdb5a 100755 --- a/tests/test_atomic_import.py +++ b/tests/test_atomic_import.py @@ -1,16 +1,17 @@ import sqlite3 +from contextlib import contextmanager from dataclasses import replace import pytest + import openstatspec.sql.wide as wide -from openstatspec.sql.profiles import MYSQL, SQLITE, TargetCapabilityExceededError +from openstatspec.sql.profiles import DOLT, MYSQL, SQLITE, TargetCapabilityExceededError from openstatspec.sql.wide import create_wide_dataset -def test_invalid_string_row_is_rejected_before_dataset_or_data_table(tmp_path) -> None: +def test_invalid_string_row_leaves_no_dataset_or_data_table(tmp_path) -> None: database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string", "string_width": 8, "label": "", @@ -18,7 +19,7 @@ def test_invalid_string_row_is_rejected_before_dataset_or_data_table(tmp_path) - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", }] - with pytest.raises(TargetCapabilityExceededError, match="Target capability exceeded"): + with pytest.raises(TargetCapabilityExceededError): create_wide_dataset( database_url=database, dataset_id="broken", source_name="fixture.sav", source_format="SAV", rows=[{"name": None}], variables=variables, @@ -31,41 +32,114 @@ def test_invalid_string_row_is_rejected_before_dataset_or_data_table(tmp_path) - assert connection.execute("select count(*) from dataset").fetchone() == (0,) assert connection.execute("select count(*) from variable").fetchone() == (0,) +def test_failed_create_does_not_drop_a_concurrently_created_table( + tmp_path, monkeypatch, +) -> None: + database_path = tmp_path / "race.sqlite" + database = f"sqlite:///{database_path}" + variables = [{ + "ordinal": 1, "source_name": "name", "physical_name": "name", + "storage_kind": "string", "string_width": 8, "label": "", + "format": "A8", "measure": "nominal", "alignment": "left", + "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", + }] + real_inspect = wide.inspect + inspected_data_table = 0 + + def inspect_with_concurrent_table(bind): + actual = real_inspect(bind) + + class Inspector: + def __getattr__(self, name): + return getattr(actual, name) + + def has_table(self, table_name): + nonlocal inspected_data_table + if table_name == "data_race": + inspected_data_table += 1 + return inspected_data_table > 1 + return actual.has_table(table_name) + + return Inspector() + + real_create = wide.Table.create + real_drop = wide.Table.drop + dropped = [] + + def fail_data_table_create(table, bind, **kwargs): + if table.name == "data_race": + raise RuntimeError("concurrent create won") + return real_create(table, bind, **kwargs) + + def observe_drop(table, bind, **kwargs): + if table.name == "data_race": + dropped.append(table.name) + return real_drop(table, bind, **kwargs) + + monkeypatch.setattr(wide, "inspect", inspect_with_concurrent_table) + monkeypatch.setattr(wide.Table, "create", fail_data_table_create) + monkeypatch.setattr(wide.Table, "drop", observe_drop) + + with pytest.raises(RuntimeError, match="concurrent create won"): + create_wide_dataset( + database_url=database, dataset_id="race", source_name="race.sav", + source_format="SAV", rows=[{"name": "ok"}], variables=variables, + ) + + assert inspected_data_table == 1 + assert dropped == [] + + +def test_dataset_name_cannot_equal_an_existing_normative_uuid(tmp_path) -> None: + database = f"sqlite:///{tmp_path / 'namespace.sqlite'}" + variables = [{ + "ordinal": 1, "source_name": "name", "physical_name": "name", + "storage_kind": "string", "string_width": 8, "label": "", + "format": "A8", "measure": "nominal", "alignment": "left", + "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", + }] + first = create_wide_dataset( + database_url=database, dataset_id="first", source_name="first.sav", + source_format="SAV", rows=[{"name": "first"}], variables=variables, + ) -def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path, monkeypatch) -> None: + with pytest.raises(ValueError, match="already exists"): + create_wide_dataset( + database_url=database, dataset_id=first["dataset_id"], + source_name="second.sav", source_format="SAV", + rows=[{"name": "second"}], variables=variables, + ) + +def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path) -> None: database_path = tmp_path / "preflight.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) - too_many = SQLITE.max_source_variables + 1 - monkeypatch.setattr( - wide, "effective_profile", lambda _url, **_kwargs: (SQLITE, {}), - ) with pytest.raises(Exception, match="Target capability exceeded"): create_wide_dataset( database_url=database, dataset_id="too-wide", source_name="too-wide.sav", - source_format="SAV", rows=(), variables=[{}] * too_many, + source_format="SAV", rows=(), variables=[{}] * 2_001, ) connection = sqlite3.connect(database_path) - assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) + assert connection.execute("select count(*) from dataset").fetchone() == (0,) assert "data_too_wide" not in { row[0] for row in connection.execute("select name from sqlite_master where type = 'table'") } assert connection.execute( - "select direction, status, dataset_id from operation_catalog" - ).fetchall() == [("import", "failed", None)] + "select operation_kind, status from operation" + ).fetchall() == [("import", "failed")] direction, severity, code, details = connection.execute( - "select direction, severity, code, details from fidelity_event_catalog" + "select direction, severity, event_code, detail_json from fidelity_event" ).fetchone() - assert (direction, severity, code) == ("import", "error", "target_capability_exceeded") - assert f'"variable_count": {too_many}' in details + assert (direction, severity, code) == ( + "import", "error", "target_capability_exceeded", + ) + assert '"variable_count": 2001' in details def test_identifier_mapping_preflight_records_failure_before_dataset_creation(tmp_path) -> None: database_path = tmp_path / "identifier.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "wrong_name", "storage_kind": "string", "string_width": 8, "label": "", @@ -80,19 +154,22 @@ def test_identifier_mapping_preflight_records_failure_before_dataset_creation(tm ) connection = sqlite3.connect(database_path) - assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) - assert connection.execute("select dataset_id, status from operation_catalog").fetchall() == [(None, "failed")] - details = connection.execute("select details from fidelity_event_catalog").fetchone()[0] + assert connection.execute("select count(*) from dataset").fetchone() == (0,) + assert connection.execute( + "select status from operation" + ).fetchall() == [("failed",)] + details = connection.execute( + "select detail_json from fidelity_event" + ).fetchone()[0] assert '"reason": "physical_identifier_mapping_invalid"' in details def test_declared_string_width_preflight_is_atomic_and_diagnostic(tmp_path, monkeypatch) -> None: database_path = tmp_path / "string-width.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) monkeypatch.setattr( wide, "effective_profile", - lambda _url, **_kwargs: (replace(SQLITE, max_text_value_bytes=3), {}), + lambda _url: (replace(SQLITE, max_text_value_bytes=3), {}), ) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", @@ -108,23 +185,73 @@ def test_declared_string_width_preflight_is_atomic_and_diagnostic(tmp_path, monk ) connection = sqlite3.connect(database_path) - assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) + assert connection.execute("select count(*) from dataset").fetchone() == (0,) assert "data_too_wide_string" not in { row[0] for row in connection.execute("select name from sqlite_master where type = 'table'") } assert connection.execute( - "select dataset_id, status from operation_catalog" - ).fetchall() == [(None, "failed")] - details = connection.execute("select details from fidelity_event_catalog").fetchone()[0] + "select status from operation" + ).fetchall() == [("failed",)] + details = connection.execute( + "select detail_json from fidelity_event" + ).fetchone()[0] assert '"reason": "declared_string_width_limit"' in details assert '"string_width": 4' in details +def test_dolt_import_cleanup_uses_bound_catalog_transaction( + tmp_path, monkeypatch, +) -> None: + database_path = tmp_path / "dolt-cleanup.sqlite" + database = f"sqlite:///{database_path}" + active = { + "working_set_binding": { + "database": "test", + "active_branch": "main", + }, + } + monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, active)) + monkeypatch.setattr( + wide, "_capture_dolt_state", lambda *_args, **_kwargs: None, + ) + phases = [] + real_bound = wide._bound_catalog_transaction + + @contextmanager + def capture_bound(**kwargs): + phases.append(kwargs["phase"]) + with real_bound(**kwargs) as connection: + yield connection + + monkeypatch.setattr(wide, "_bound_catalog_transaction", capture_bound) + real_store = wide.store_normative_dataset + + def fail_after_normative_write(*args, **kwargs): + real_store(*args, **kwargs) + raise RuntimeError("fault after Dolt normative write") + + monkeypatch.setattr(wide, "store_normative_dataset", fail_after_normative_write) + variables = [{ + "ordinal": 1, "source_name": "score", "physical_name": "score", + "storage_kind": "numeric", "string_width": None, "label": "", + "format": "F8.0", "measure": "scale", "alignment": "right", + "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", + }] + + with pytest.raises(RuntimeError, match="fault after Dolt normative write"): + create_wide_dataset( + database_url=database, dataset_id="dolt-cleanup", + source_name="fixture.sav", source_format="SAV", + rows=[{"score": 1.0}], variables=variables, + ) + + assert phases[-2:] == ["import", "import cleanup"] + + def test_nonatomic_failure_after_normative_write_cleans_both_catalogs_and_data( tmp_path, monkeypatch, ) -> None: database_path = tmp_path / "nonatomic-cleanup.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string", "string_width": 8, "label": "", @@ -133,7 +260,7 @@ def test_nonatomic_failure_after_normative_write_cleans_both_catalogs_and_data( }] monkeypatch.setattr( wide, "effective_profile", - lambda _url, **_kwargs: (replace(MYSQL, name="mysql"), {}), + lambda _url: (replace(MYSQL, name="mysql"), {}), ) real_store = wide.store_normative_dataset @@ -160,26 +287,13 @@ def fail_after_normative_write(*args, **kwargs): "select name from sqlite_master where type = 'table'" ) } - if "dataset_catalog" in existing_tables: - assert connection.execute( - "select count(*) from dataset_catalog where dataset_id = 'cleanup'" - ).fetchone() == (0,) - if "variable_catalog" in existing_tables: - assert connection.execute( - "select count(*) from variable_catalog where dataset_id = 'cleanup'" - ).fetchone() == (0,) + assert not {name for name in existing_tables if name.endswith("_catalog")} assert connection.execute( "select count(*) from dataset where dataset_name = 'cleanup'" ).fetchone() == (0,) assert connection.execute( "select count(*) from variable" ).fetchone() == (0,) - assert connection.execute( - "select direction, status, dataset_id from operation_catalog" - ).fetchall() == [("import", "failed", None)] - assert connection.execute( - "select direction, severity, code, dataset_id from fidelity_event_catalog" - ).fetchall() == [("import", "error", "import_failed", None)] assert connection.execute( "select operation_kind, status from operation" ).fetchall() == [("import", "failed")] @@ -187,6 +301,39 @@ def fail_after_normative_write(*args, **kwargs): "select direction, severity, event_code, dataset_id from fidelity_event" ).fetchall() == [("import", "error", "import_failed", None)] +def test_occupied_foreign_view_namespace_fails_without_modification( + tmp_path, +) -> None: + database_path = tmp_path / "foreign-view.sqlite" + database = f"sqlite:///{database_path}" + connection = sqlite3.connect(database_path) + connection.execute("create view foreign_view as select 1 as value") + before_schema = connection.execute( + "select name, type, sql from sqlite_master order by name" + ).fetchall() + connection.close() + variables = [{ + "ordinal": 1, "source_name": "score", "physical_name": "score", + "storage_kind": "numeric", "string_width": None, "label": "", + "format": "F8.0", "measure": "scale", "alignment": "right", + "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", + }] + + with pytest.raises(wide.UnsupportedOperationError, match="foreign"): + create_wide_dataset( + database_url=database, dataset_id="foreign-view", + source_name="fixture.sav", source_format="SAV", + rows=[{"score": 1.0}], variables=variables, + ) + + connection = sqlite3.connect(database_path) + assert connection.execute( + "select name, type, sql from sqlite_master order by name" + ).fetchall() == before_schema + assert connection.execute("select value from foreign_view").fetchall() == [(1,)] + connection.close() + + def test_occupied_foreign_namespace_fails_without_modification(tmp_path) -> None: database_path = tmp_path / "foreign.sqlite" database = f"sqlite:///{database_path}" @@ -198,8 +345,17 @@ def test_occupied_foreign_namespace_fails_without_modification(tmp_path) -> None "select name, sql from sqlite_master where type = 'table' order by name" ).fetchall() + variables = [{ + "ordinal": 1, "source_name": "name", "physical_name": "name", + "storage_kind": "string", "string_width": 8, "label": "", + "format": "A8", "measure": "nominal", "alignment": "left", + "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", + }] with pytest.raises(RuntimeError, match="foreign"): - wide.initialize_wide_catalog(database_url=database) + create_wide_dataset( + database_url=database, dataset_id="foreign", source_name="fixture.sav", + source_format="SAV", rows=[{"name": "ok"}], variables=variables, + ) assert connection.execute( "select name, sql from sqlite_master where type = 'table' order by name" @@ -208,12 +364,15 @@ def test_occupied_foreign_namespace_fails_without_modification(tmp_path) -> None @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) -def test_nonfinite_preflight_creates_no_dataset_or_physical_table( - tmp_path, value, +def test_dolt_nonfinite_preflight_creates_no_dataset_or_physical_table( + tmp_path, monkeypatch, value, ) -> None: database_path = tmp_path / "dolt-nonfinite.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) + monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, {})) + monkeypatch.setattr( + wide, "_capture_dolt_state", lambda *_args, **_kwargs: None, + ) variables = [{ "ordinal": 1, "source_name": "value", "physical_name": "value", "storage_kind": "numeric", "string_width": None, "label": "", @@ -244,22 +403,21 @@ def test_nonfinite_preflight_creates_no_dataset_or_physical_table( ).fetchall() == [(None, "target_capability_exceeded")] -def test_limited_width_failure_preserves_initialized_identity_and_one_audit( +def test_empty_namespace_dolt_width_failure_initializes_identity_and_one_audit( tmp_path, monkeypatch, ) -> None: database_path = tmp_path / "dolt-preflight.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) + monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, {})) monkeypatch.setattr( - wide, "effective_profile", - lambda _url, **_kwargs: (replace(SQLITE, max_source_variables=1), {}), + wide, "_capture_dolt_state", lambda *_args, **_kwargs: None, ) with pytest.raises(Exception, match="Target capability exceeded"): create_wide_dataset( database_url=database, dataset_id="too-wide-dolt", source_name="too-wide-dolt.sav", source_format="SAV", - rows=(), variables=[{}, {}], + rows=(), variables=[{}] * 306, ) connection = sqlite3.connect(database_path) @@ -272,22 +430,14 @@ def test_limited_width_failure_preserves_initialized_identity_and_one_audit( assert connection.execute( "select dataset_id, event_code from fidelity_event" ).fetchall() == [(None, "target_capability_exceeded")] - assert connection.execute( - "select dataset_id, status from operation_catalog" - ).fetchall() == [(None, "failed")] - assert connection.execute( - "select dataset_id, code from fidelity_event_catalog" - ).fetchall() == [(None, "target_capability_exceeded")] -@pytest.mark.parametrize("failure_point", ["mirror_completion", "normative_completion"]) def test_nonatomic_failure_during_final_completion_still_cleans_dataset( - tmp_path, monkeypatch, failure_point, + tmp_path, monkeypatch, ) -> None: - database_path = tmp_path / f"{failure_point}.sqlite" + database_path = tmp_path / "normative-completion.sqlite" database = f"sqlite:///{database_path}" - wide.initialize_wide_catalog(database_url=database) - dataset_id = f"cleanup-{failure_point}" + dataset_name = "cleanup-normative-completion" variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string", "string_width": 8, "label": "", @@ -296,87 +446,43 @@ def test_nonatomic_failure_during_final_completion_still_cleans_dataset( }] monkeypatch.setattr( wide, "effective_profile", - lambda _url, **_kwargs: (replace(MYSQL, name="mysql"), {}), + lambda _url: (replace(MYSQL, name="mysql"), {}), ) + real_finish = wide.finish_normative_operation triggered = False - if failure_point == "mirror_completion": - real_update = wide.update - - def fail_first_operation_update(table): - nonlocal triggered - if table.name == "operation_catalog" and not triggered: - triggered = True - raise RuntimeError("fault after data insert") - return real_update(table) - - monkeypatch.setattr(wide, "update", fail_first_operation_update) - else: - real_finish = wide.finish_normative_operation - - def fail_first_normative_finish(*args, **kwargs): - nonlocal triggered - if not triggered: - triggered = True - raise RuntimeError("fault during final completion") - return real_finish(*args, **kwargs) - - monkeypatch.setattr( - wide, "finish_normative_operation", fail_first_normative_finish, - ) + def fail_first_normative_finish(*args, **kwargs): + nonlocal triggered + if not triggered: + triggered = True + raise RuntimeError("fault during final completion") + return real_finish(*args, **kwargs) + + monkeypatch.setattr( + wide, "finish_normative_operation", fail_first_normative_finish, + ) with pytest.raises(RuntimeError, match="fault"): create_wide_dataset( - database_url=database, dataset_id=dataset_id, - source_name=f"{dataset_id}.sav", source_format="SAV", + database_url=database, dataset_id=dataset_name, + source_name=f"{dataset_name}.sav", source_format="SAV", rows=[{"name": "ok"}], variables=variables, ) connection = sqlite3.connect(database_path) assert triggered is True - assert f"data_{dataset_id}" not in { - row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" - ) - } - existing_tables = { + tables = { row[0] for row in connection.execute( "select name from sqlite_master where type = 'table'" ) } - if "dataset_catalog" in existing_tables: - assert connection.execute( - "select count(*) from dataset_catalog where dataset_id = ?", (dataset_id,) - ).fetchone() == (0,) + assert f"data_{dataset_name}" not in tables + assert not {name for name in tables if name.endswith("_catalog")} assert connection.execute( - "select count(*) from dataset where dataset_name = ?", (dataset_id,) + "select count(*) from dataset where dataset_name = ?", (dataset_name,) ).fetchone() == (0,) - assert connection.execute( - "select status, dataset_id from operation_catalog" - ).fetchall() == [("failed", None)] assert connection.execute( "select status from operation" ).fetchall() == [("failed",)] assert connection.execute( - "select code, dataset_id from fidelity_event_catalog" + "select event_code, dataset_id from fidelity_event" ).fetchall() == [("import_failed", None)] - -def test_failed_table_create_never_claims_cleanup_ownership() -> None: - class ConcurrentTable: - def create(self, _connection): - raise RuntimeError("table already created by concurrent import") - - state = {"data_table_created": False} - - with pytest.raises(RuntimeError, match="concurrent import"): - wide._create_operation_owned_data_table( - object(), ConcurrentTable(), state, - ) - - assert state["data_table_created"] is False - -def test_transactional_profiles_never_run_stale_compensating_cleanup() -> None: - assert wide._requires_compensating_import_cleanup("sqlite") is False - assert wide._requires_compensating_import_cleanup("postgresql") is False - assert wide._requires_compensating_import_cleanup("mysql") is True - assert wide._requires_compensating_import_cleanup("mariadb") is True - assert wide._requires_compensating_import_cleanup("dolt") is True diff --git a/tests/test_attribute_catalog.py b/tests/test_attribute_catalog.py index d0e9c06..3f2740a 100644 --- a/tests/test_attribute_catalog.py +++ b/tests/test_attribute_catalog.py @@ -6,7 +6,6 @@ import pytest import openstatspec -from openstatspec.core import UnsupportedOperationError from openstatspec.sql.wide import create_wide_dataset, read_wide_dataset @@ -14,13 +13,12 @@ @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) -def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, suffix: str) -> None: +def test_normative_attributes_are_authoritative_for_sav_and_zsav_export(tmp_path, suffix: str) -> None: source = tmp_path / f"source{suffix}" destination = tmp_path / f"destination{suffix}" imported_again = tmp_path / f"again-{suffix[1:]}.sqlite" database_path = tmp_path / f"attributes-{suffix[1:]}.sqlite" database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) pyspssio.write_sav( str(source), pd.DataFrame({"answer": [1.0]}), metadata={ @@ -32,25 +30,26 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su openstatspec.import_sav(source, database_url=database, dataset_id="attributes") connection = sqlite3.connect(database_path) assert connection.execute( - "select scope, variable_ordinal, attribute_ordinal, value_ordinal, attribute_name, attribute_value " - "from attribute_catalog where dataset_id = 'attributes' " - "order by scope, variable_ordinal, attribute_ordinal, value_ordinal" + "select attribute_name, array_ordinal, attribute_value " + "from dataset_attribute order by rowid" ).fetchall() == [ - ("file", 0, 1, 1, "Source", "source-file"), - ("file", 0, 2, 1, "Order", "second"), - ("variable", 1, 1, 1, "Source", "source-variable"), - ("variable", 1, 2, 1, "Flag", "yes"), + ("Source", 1, "source-file"), + ("Order", 1, "second"), + ] + assert connection.execute( + "select a.attribute_name, a.array_ordinal, a.attribute_value " + "from variable_attribute a order by a.rowid" + ).fetchall() == [ + ("Source", 1, "source-variable"), + ("Flag", 1, "yes"), ] - # Deliberately corrupt the legacy copies. Export must use the normalized rows. - connection.execute("update dataset_catalog set file_attributes = ? where dataset_id = 'attributes'", (json.dumps({"Source": "legacy-file"}),)) - connection.execute("update variable_catalog set attributes = ? where dataset_id = 'attributes'", (json.dumps({"Source": "legacy-variable"}),)) connection.execute( - "update attribute_catalog set attribute_value = 'catalog-file' " - "where dataset_id = 'attributes' and scope = 'file' and attribute_name = 'Source'" + "update dataset_attribute set attribute_value = 'catalog-file' " + "where attribute_name = 'Source'" ) connection.execute( - "update attribute_catalog set attribute_value = 'catalog-variable' " - "where dataset_id = 'attributes' and scope = 'variable' and attribute_name = 'Source'" + "update variable_attribute set attribute_value = 'catalog-variable' " + "where attribute_name = 'Source'" ) connection.commit() @@ -63,53 +62,18 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su assert exported["var_attributes"] == { "answer": {"Source": "catalog-variable", "Flag": "yes"}, } - imported_again_database = f"sqlite:///{imported_again}" - openstatspec.initialize_catalog(database_url=imported_again_database) - openstatspec.import_sav(destination, database_url=imported_again_database, dataset_id="again") + openstatspec.import_sav(destination, database_url=f"sqlite:///{imported_again}", dataset_id="again") reimported = sqlite3.connect(imported_again) assert reimported.execute( - "select attribute_name, attribute_value from attribute_catalog " - "where dataset_id = 'again' and scope = 'file' order by attribute_ordinal, value_ordinal" - ).fetchall() == [("Source", "catalog-file"), ("Order", "second")] + "select attribute_name, attribute_value from dataset_attribute order by attribute_name" + ).fetchall() == [("Order", "second"), ("Source", "catalog-file")] assert reimported.execute( - "select attribute_name, attribute_value from attribute_catalog " - "where dataset_id = 'again' and scope = 'variable' order by attribute_ordinal, value_ordinal" - ).fetchall() == [("Source", "catalog-variable"), ("Flag", "yes")] - - -def test_attribute_catalog_falls_back_to_legacy_json_without_rewriting_it(tmp_path) -> None: - source = tmp_path / "legacy.sav" - destination = tmp_path / "legacy-out.sav" - database_path = tmp_path / "legacy.sqlite" - database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) - pyspssio.write_sav( - str(source), pd.DataFrame({"answer": [1.0]}), - metadata={"file_attributes": {"File": "legacy"}, "var_attributes": {"answer": {"Var": "legacy"}}}, - ) - openstatspec.import_sav(source, database_url=database, dataset_id="legacy") - connection = sqlite3.connect(database_path) - connection.execute("delete from attribute_catalog") - connection.commit() - - # An existing verified catalog with no normalized attribute rows falls back - # to legacy JSON without mutating the normalized catalog. - dataset, variables, _ = read_wide_dataset(database_url=database, dataset_id="legacy") - assert json.loads(dataset["file_attributes"]) == {"File": "legacy"} - assert json.loads(variables[0]["attributes"]) == {"Var": "legacy"} - assert connection.execute( - "select count(*) from attribute_catalog" - ).fetchone() == (0,) - openstatspec.export_sav( - database_url=database, dataset_id="legacy", destination=destination, - allow_loss=_REQUIRED_ENGINE_LOSS, - ) - assert pyspssio.read_metadata(str(destination))["file_attributes"] == {"File": "legacy"} + "select attribute_name, attribute_value from variable_attribute order by attribute_name" + ).fetchall() == [("Flag", "yes"), ("Source", "catalog-variable")] -def test_attribute_catalog_preserves_ordered_arrays_through_raw_pyspssio_bridge(tmp_path) -> None: +def test_normative_attributes_preserve_ordered_arrays_through_raw_pyspssio_bridge(tmp_path) -> None: database = f"sqlite:///{tmp_path / 'array.sqlite'}" - openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="array", source_name="array.sav", source_format="SAV", rows=[{"answer": 1.0}], diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 07fb41e..1c34809 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -1,26 +1,29 @@ -from decimal import Decimal import sqlite3 -import threading import pytest -from sqlalchemy import MetaData, Table import openstatspec -import openstatspec.sql.wide as wide from openstatspec.core import UnsupportedOperationError -from openstatspec.sql.profiles import ( - MYSQL, POSTGRESQL, TargetCapabilityExceededError, preflight, -) +from openstatspec.sql import wide from openstatspec.sql.wide import ( - ImportRecoveryError, _bounded_batches, create_wide_dataset, - read_wide_dataset, + finish_export_operation, + read_export_operation_state, + record_export_cleanup_failure, record_export_operation, - validate_wide_dataset, ) +NORMATIVE_TABLES = { + "catalog_identity", "dataset", "operation", "variable", + "dataset_weight_variable", "value_label_set", "value_label", + "variable_value_label_set", "missing_rule", "dataset_attribute", + "variable_attribute", "document", "variable_set", "variable_set_member", + "multiple_response_set", "multiple_response_member", "fidelity_event", +} + + def _variables(): return [{ "ordinal": 1, @@ -43,73 +46,65 @@ def _table_names(path): try: return { row[0] for row in connection.execute( - "select name from sqlite_master where type = 'table'" + "select name from sqlite_master " + "where type = 'table' and name not like 'sqlite_%'" ) } finally: connection.close() -def test_import_requires_explicit_catalog_and_creates_no_relations(tmp_path): +def _create_dataset(database): + return create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + +def test_import_initializes_only_normative_catalog(tmp_path): path = tmp_path / "absent.sqlite" database = f"sqlite:///{path}" - with pytest.raises(UnsupportedOperationError, match="catalog is absent"): - create_wide_dataset( - database_url=database, - dataset_id="sample", - source_name="fixture.sav", - source_format="SAV", - rows=[{"name": "ok"}], - variables=_variables(), - ) - - assert _table_names(path) == set() + _create_dataset(database) + assert _table_names(path) == NORMATIVE_TABLES | {"data_sample"} -@pytest.mark.parametrize( - "operation", - [ - lambda database: read_wide_dataset(database_url=database, dataset_id="missing"), - lambda database: validate_wide_dataset(database_url=database, dataset_id="missing"), - lambda database: record_export_operation( - database_url=database, - dataset_id="missing", - destination="output.sav", - allowed_fidelity_events=(), - ), - ], -) -def test_read_validate_and_export_require_catalog_without_mutation(tmp_path, operation): - path = tmp_path / "absent.sqlite" +def test_initializer_creates_only_normative_catalog_and_is_idempotent(tmp_path): + path = tmp_path / "strict.sqlite" database = f"sqlite:///{path}" - with pytest.raises(UnsupportedOperationError, match="catalog is absent"): - operation(database) - - assert _table_names(path) == set() + initialized = openstatspec.initialize_catalog(database_url=database) + assert initialized["profile"] == "sqlite" + assert initialized["catalog"] == "verified" + assert _table_names(path) == NORMATIVE_TABLES + assert openstatspec.initialize_catalog(database_url=database)["catalog"] == "verified" + assert _table_names(path) == NORMATIVE_TABLES @pytest.mark.parametrize( "foreign_sql", [ "create table foreign_relation (value integer)", - "create table view_source (value integer); create view foreign_view as select value from view_source", + "create view foreign_view as select 1 as value", ], ) -def test_initializer_rejects_foreign_tables_and_views_without_modification( +def test_initializer_rejects_foreign_relations_without_modification( tmp_path, foreign_sql, ): path = tmp_path / "foreign.sqlite" connection = sqlite3.connect(path) - connection.executescript(foreign_sql) + connection.execute(foreign_sql) before = connection.execute( "select type, name, sql from sqlite_master " "where name not like 'sqlite_%' order by type, name" ).fetchall() connection.close() - with pytest.raises(UnsupportedOperationError, match="catalog is foreign"): + with pytest.raises(UnsupportedOperationError, match="foreign"): openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") connection = sqlite3.connect(path) @@ -121,530 +116,93 @@ def test_initializer_rejects_foreign_tables_and_views_without_modification( assert after == before -def test_initializer_compensates_partial_catalog_install(tmp_path, monkeypatch): - path = tmp_path / "partial.sqlite" - - def fail_migration(*_args, **_kwargs): - raise RuntimeError("injected catalog migration failure") - - monkeypatch.setattr(wide, "_migrate_catalog_columns", fail_migration) - with pytest.raises(RuntimeError, match="injected catalog migration failure"): - openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") - - assert _table_names(path) == set() - - -def test_post_ddl_failure_removes_dataset_state_and_persists_null_dataset_audit( - tmp_path, monkeypatch, -): - path = tmp_path / "runtime.sqlite" - database = f"sqlite:///{path}" - openstatspec.initialize_catalog(database_url=database) - original = wide.store_normative_dataset - - def store_then_fail(*args, **kwargs): - original(*args, **kwargs) - raise RuntimeError("injected normative failure") - - monkeypatch.setattr(wide, "store_normative_dataset", store_then_fail) - with pytest.raises(RuntimeError, match="injected normative failure"): - create_wide_dataset( - database_url=database, - dataset_id="sample", - source_name="fixture.sav", - source_format="SAV", - rows=[{"name": "ok"}], - variables=_variables(), - ) - - assert "data_sample" not in _table_names(path) - connection = sqlite3.connect(path) - assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) - assert connection.execute("select count(*) from dataset").fetchone() == (0,) - assert connection.execute( - "select status, dataset_id from operation_catalog" - ).fetchall() == [("failed", None)] - assert connection.execute( - "select code, dataset_id from fidelity_event_catalog" - ).fetchall() == [("import_failed", None)] - assert connection.execute( - "select status from operation" - ).fetchall() == [("failed",)] - assert connection.execute( - "select event_code, dataset_id from fidelity_event" - ).fetchall() == [("import_failed", None)] - connection.close() - - -def test_cleanup_failure_has_machine_readable_error(tmp_path, monkeypatch): - path = tmp_path / "cleanup.sqlite" +def test_initializer_rejects_obsolete_relation_until_manual_remediation(tmp_path): + path = tmp_path / "obsolete.sqlite" database = f"sqlite:///{path}" openstatspec.initialize_catalog(database_url=database) - - def fail_cleanup(*_args, **_kwargs): - raise RuntimeError("injected cleanup failure") - - def fail_mutation(*_args, **_kwargs): - raise RuntimeError("injected mutation failure") - - # Exercise the non-transactional MySQL-wire compensation path while - # retaining SQLite as the dependency-free test transport. - monkeypatch.setattr( - wide, "effective_profile", lambda _url, **_kwargs: (MYSQL, {}), - ) - monkeypatch.setattr(wide, "_cleanup_import_state", fail_cleanup) - monkeypatch.setattr(wide, "store_normative_dataset", fail_mutation) - with pytest.raises(ImportRecoveryError) as error: - create_wide_dataset( - database_url=database, - dataset_id="sample", - source_name="fixture.sav", - source_format="SAV", - rows=[{"name": "ok"}], - variables=_variables(), - ) - - assert error.value.code == "cleanup_failed" - assert error.value.details["original_cause"]["type"] == "RuntimeError" - assert error.value.details["cleanup_fault"]["type"] == "RuntimeError" - assert error.value.details["success_forbidden"] is True - evidence = error.value.details["deterministic_recovery_evidence"] - assert evidence["procedure_id"] == "openstatspec.import-compensation.v1" - assert evidence["cleanup_attempted"] is True - assert evidence["cleanup_succeeded"] is False - assert evidence["operation_owned_state_targeted"] is True - assert evidence["cleanup_failed_audit_persisted"] is True - assert evidence["terminal_reporting"] == "catalog_and_exception" - assert len(evidence["residual_inventory_sha256"]) == 64 connection = sqlite3.connect(path) - assert connection.execute( - "select status, dataset_id from operation_catalog" - ).fetchall() == [("failed", None)] - assert connection.execute( - "select code, dataset_id from fidelity_event_catalog" - ).fetchall() == [("cleanup_failed", None)] + connection.execute("create table dataset_catalog (dataset_id text primary key)") + connection.commit() connection.close() - -def test_database_decimal_numeric_wrappers_are_restored_to_binary64(): - variables = [{ - "physical_name": "score", - "storage_kind": "numeric", - }] - rows = wide._canonicalize_database_numeric_rows( - [ - {"score": Decimal("1.5000000000"), "name": "alpha"}, - {"score": None, "name": "missing"}, - ], - variables, - ) - - assert rows == [ - {"score": 1.5, "name": "alpha"}, - {"score": None, "name": "missing"}, - ] - assert isinstance(rows[0]["score"], float) - - -@pytest.mark.parametrize( - "value", [Decimal("NaN"), Decimal("Infinity"), Decimal("-Infinity")], -) -def test_database_decimal_nonfinite_wrappers_remain_rejected(value): - variables = [{ - "ordinal": 1, - "source_name": "score", - "physical_name": "score", - "storage_kind": "numeric", - }] - rows = wide._canonicalize_database_numeric_rows([{"score": value}], variables) - - with pytest.raises(TargetCapabilityExceededError) as error: - preflight(MYSQL, variables, rows=rows) - - assert error.value.details["reason"] == "nonfinite_numeric_value" + with pytest.raises(UnsupportedOperationError, match="obsolete"): + openstatspec.initialize_catalog(database_url=database) def test_bounded_batches_never_exceed_statement_payload_limit(): variables = _variables() - rows = [ - {"name": "aaaa"}, - {"name": "bbbb"}, - {"name": "cccc"}, - ] + rows = [{"name": "aaaa"}, {"name": "bbbb"}, {"name": "cccc"}] single = wide.statement_payload_bytes(rows[0], variables) - batches = list(_bounded_batches(rows, variables, single * 2)) - - assert batches == [rows[:2], rows[2:]] - - -def test_bounded_batches_budget_numeric_pymysql_wire_literals(): - variables = [{ - "source_name": "value", - "physical_name": "value", - "storage_kind": "numeric", - }] - rows = [{"value": -1.7976931348623157e+308}] * 2 - - assert wide.statement_payload_bytes(rows[0], variables) == 56 - assert list(_bounded_batches(rows, variables, 111)) == [[rows[0]], [rows[1]]] - assert list(_bounded_batches(rows, variables, 112)) == [rows] + assert list(_bounded_batches(rows, variables, single * 2)) == [ + rows[:2], rows[2:], + ] -def test_duplicate_import_failure_preserves_existing_dataset(tmp_path): +def test_duplicate_import_preserves_existing_dataset(tmp_path): path = tmp_path / "duplicate.sqlite" database = f"sqlite:///{path}" openstatspec.initialize_catalog(database_url=database) - create_wide_dataset( - database_url=database, - dataset_id="sample", - source_name="first.sav", - source_format="SAV", - rows=[{"name": "first"}], - variables=_variables(), - ) + _create_dataset(database) with pytest.raises(ValueError, match="already exists"): - create_wide_dataset( - database_url=database, - dataset_id="sample", - source_name="second.sav", - source_format="SAV", - rows=[{"name": "second"}], - variables=_variables(), - ) + _create_dataset(database) connection = sqlite3.connect(path) - assert connection.execute( - "select dataset_id, case_count from dataset_catalog" - ).fetchall() == [("sample", 1)] assert connection.execute( "select dataset_name, source_case_count from dataset" ).fetchall() == [("sample", 1)] - assert connection.execute( - "select name from data_sample" - ).fetchall() == [("first",)] + assert connection.execute("select name from data_sample").fetchall() == [("ok",)] connection.close() - -def test_variable_bijection_rejects_storage_kind_mismatch(tmp_path): - path = tmp_path / "variable-storage-mismatch.sqlite" +def test_normative_export_operation_transitions_to_success(tmp_path): + path = tmp_path / "export.sqlite" database = f"sqlite:///{path}" openstatspec.initialize_catalog(database_url=database) - variables = _variables() - variables[0].update({ - "storage_kind": "numeric", - "string_width": None, - }) - create_wide_dataset( + _create_dataset(database) + + operation_id = record_export_operation( database_url=database, dataset_id="sample", - source_name="fixture.sav", - source_format="SAV", - rows=[{"name": 1.0}], - variables=variables, - ) - connection = sqlite3.connect(path) - connection.execute( - "update variable set storage_kind = 'string' where source_name = 'name'" + destination="output.sav", + allowed_fidelity_events=(), + terminal=False, ) - connection.commit() - connection.close() + assert read_export_operation_state( + database_url=database, operation_id=operation_id, + )["classification"] == "running" - with pytest.raises(UnsupportedOperationError, match="catalog is unverified"): - read_wide_dataset(database_url=database, dataset_id="sample") + finish_export_operation( + database_url=database, operation_id=operation_id, + ) + assert read_export_operation_state( + database_url=database, operation_id=operation_id, + )["classification"] == "succeeded" -def test_missing_additive_column_requires_explicit_migration_without_audit_mutation( - tmp_path, -): - path = tmp_path / "migration-required.sqlite" +def test_cleanup_failure_is_recorded_without_compatibility_catalog(tmp_path): + path = tmp_path / "cleanup.sqlite" database = f"sqlite:///{path}" openstatspec.initialize_catalog(database_url=database) - connection = sqlite3.connect(path) - connection.execute("alter table variable_catalog drop column compat_name") - connection.commit() - connection.close() - - with pytest.raises(UnsupportedOperationError, match="catalog is migration_required"): - create_wide_dataset( - database_url=database, dataset_id="sample", - source_name="fixture.sav", source_format="SAV", - rows=[{"name": "ok"}], variables=_variables(), - ) - connection = sqlite3.connect(path) - assert "compat_name" not in { - row[1] for row in connection.execute("pragma table_info(variable_catalog)") - } - assert connection.execute("select count(*) from operation_catalog").fetchone() == (0,) - connection.close() - - result = openstatspec.initialize_catalog(database_url=database) - assert result["catalog"] == "verified" - connection = sqlite3.connect(path) - assert "compat_name" in { - row[1] for row in connection.execute("pragma table_info(variable_catalog)") - } - connection.close() - - -def test_multiple_catalog_identities_are_ambiguous_and_never_mutated(tmp_path): - path = tmp_path / "ambiguous.sqlite" - connection = sqlite3.connect(path) - connection.executescript( - "create table catalog_identity (" - "catalog_identity_key integer primary key, " - "contract_id varchar(128) not null unique, " - "schema_version integer not null, created_at datetime not null);" - "insert into catalog_identity values " - "(1, 'openstatspec-strict-wide-table-v1', 1, '2026-01-01')," - "(2, 'conflicting-contract', 1, '2026-01-01');" + operation_id = record_export_cleanup_failure( + database_url=database, + destination="output.sav", + original_error=RuntimeError("export failed"), + cleanup_error=RuntimeError("restore failed"), + residual_object_inventory={"backup": True}, + deterministic_recovery_evidence={"procedure_id": "test"}, ) - before = connection.execute( - "select * from catalog_identity order by catalog_identity_key" - ).fetchall() - connection.close() - - database = f"sqlite:///{path}" - with pytest.raises(UnsupportedOperationError, match="catalog is ambiguous"): - create_wide_dataset( - database_url=database, dataset_id="sample", - source_name="fixture.sav", source_format="SAV", - rows=[{"name": "ok"}], variables=_variables(), - ) - with pytest.raises(UnsupportedOperationError, match="catalog is ambiguous"): - openstatspec.initialize_catalog(database_url=database) connection = sqlite3.connect(path) assert connection.execute( - "select * from catalog_identity order by catalog_identity_key" - ).fetchall() == before - assert _table_names(path) == {"catalog_identity"} - connection.close() - - -@pytest.mark.parametrize( - ("reflected", "expected"), - [ - ( - "((contract_id)::text = " - "'openstatspec-strict-wide-table-v1'::text)", - "contract_id = 'openstatspec-strict-wide-table-v1'", - ), - ( - "(`contract_id` = _utf8mb4'openstatspec-strict-wide-table-v1')", - "contract_id = 'openstatspec-strict-wide-table-v1'", - ), - ("((catalog_identity_key)::integer = 1)", "catalog_identity_key = 1"), - ], -) -def test_reflected_check_normalization_removes_only_noop_dialect_syntax( - reflected, expected, -): - assert wide._normalized_check_sql(reflected) == expected - - -def test_reflected_mysql_integer_display_width_is_not_semantic(): - from types import SimpleNamespace - from sqlalchemy.dialects import mysql - - inspector = SimpleNamespace(bind=SimpleNamespace(dialect=mysql.dialect())) - assert wide._normalized_sql_type( - inspector, mysql.INTEGER(display_width=11), - ) == "INTEGER" - assert wide._normalized_sql_type( - inspector, mysql.BIGINT(display_width=20), - ) == "BIGINT" - assert wide._normalized_sql_type( - inspector, mysql.VARCHAR(length=255), - ) == "VARCHAR(255)" - -@pytest.mark.parametrize("database_url", [ - "sqlite://", - "sqlite:///:memory:", - "sqlite:///file:memdb1?mode=memory&cache=shared&uri=true", - "sqlite:///file::memory:?cache=shared&uri=true", -]) -def test_catalog_initialization_rejects_ephemeral_sqlite_url(database_url): - with pytest.raises( - UnsupportedOperationError, - match="requires a persistent SQLite database URL", - ): - openstatspec.initialize_catalog(database_url=database_url) - -def test_catalog_initialization_allows_file_backed_sqlite_uri(tmp_path): - path = tmp_path / "uri-catalog.sqlite" - database = f"sqlite:///file:{path}?mode=rwc&uri=true" - - assert openstatspec.initialize_catalog(database_url=database)["catalog"] == "verified" - assert path.is_file() - - -def test_import_fidelity_reader_excludes_export_operational_events(tmp_path): - path = tmp_path / "fidelity-directions.sqlite" - database = f"sqlite:///{path}" - openstatspec.initialize_catalog(database_url=database) - create_wide_dataset( - database_url=database, - dataset_id="sample", - source_name="fixture.sav", - source_format="SAV", - rows=[{"name": "ok"}], - variables=_variables(), - ) - connection = sqlite3.connect(path) - connection.executemany( - "INSERT INTO fidelity_event_catalog " - "(operation_id, ordinal, dataset_id, direction, severity, detail, " - "details, code) VALUES (?, 1, 'sample', ?, 'error', ?, '{}', ?)", - [ - ("import-event", "import", "source fidelity", "source_loss"), - ("export-event", "export", "transient export failure", "export_failed"), - ], - ) - connection.commit() + "select status from operation where operation_id = ?", + (operation_id,), + ).fetchone() == ("failed",) + assert connection.execute( + "select event_code from fidelity_event where operation_id = ?", + (operation_id,), + ).fetchone() == ("cleanup_failed",) + assert "dataset_catalog" not in _table_names(path) connection.close() - - assert wide.read_fidelity_events( - database_url=database, dataset_id="sample", - ) == ({ - "code": "source_loss", - "detail": "source fidelity", - "details": {}, - },) - -def test_stale_initializer_compensation_preserves_concurrent_verified_catalog( - tmp_path, -): - path = tmp_path / "concurrent-init.sqlite" - database = f"sqlite:///{path}" - openstatspec.initialize_catalog(database_url=database) - before = _table_names(path) - metadata = MetaData() - legacy, normative = wide._catalog_layout(metadata) - engine = wide.create_engine(database) - with engine.begin() as connection: - wide._compensate_catalog_initialization( - connection, - metadata=metadata, - before_tables=set(), - before_columns={}, - normative=normative, - legacy=legacy, - ) - wide.require_verified_catalog(connection) - engine.dispose() - - assert _table_names(path) == before - -def test_sqlite_initializer_serializes_failure_before_concurrent_winner( - tmp_path, monkeypatch, -): - path = tmp_path / "serialized-concurrent-init.sqlite" - database = f"sqlite:///{path}" - loser_inside_migration = threading.Event() - winner_started = threading.Event() - release_loser = threading.Event() - original_migrate = wide._migrate_catalog_columns - failures = {} - - def migrate(connection, datasets, variables, multiple_response): - if threading.current_thread().name == "losing-initializer": - loser_inside_migration.set() - assert winner_started.wait(5) - assert release_loser.wait(5) - raise RuntimeError("injected catalog migration failure") - return original_migrate(connection, datasets, variables, multiple_response) - - def compensation_must_not_run(*_args, **_kwargs): - raise AssertionError("SQLite must roll back DDL without stale compensation") - - def initialize(name): - if name == "winner": - winner_started.set() - try: - openstatspec.initialize_catalog(database_url=database) - except Exception as error: - failures[name] = error - - monkeypatch.setattr(wide, "_migrate_catalog_columns", migrate) - monkeypatch.setattr( - wide, "_compensate_catalog_initialization", compensation_must_not_run, - ) - - loser = threading.Thread( - target=initialize, args=("loser",), name="losing-initializer", - ) - winner = threading.Thread( - target=initialize, args=("winner",), name="winning-initializer", - ) - loser.start() - assert loser_inside_migration.wait(5) - winner.start() - assert winner_started.wait(5) - release_loser.set() - loser.join(10) - winner.join(10) - - assert not loser.is_alive() - assert not winner.is_alive() - assert type(failures.get("loser")) is RuntimeError - assert str(failures["loser"]) == "injected catalog migration failure" - assert "winner" not in failures - assert openstatspec.initialize_catalog(database_url=database)["catalog"] == "verified" - - -def test_mysql_catalog_initialization_lock_spans_mutation_boundary(): - events = [] - - class Result: - @staticmethod - def scalar_one(): - return 1 - - class Connection: - def execute(self, statement, parameters): - events.append((str(statement), parameters["lock_name"])) - return Result() - - def commit(self): - events.append(("commit", None)) - - def invalidate(self): - raise AssertionError("healthy lock connection must not be invalidated") - - with wide._catalog_initialization_serialization( - Connection(), profile_name="mysql", - ): - events.append(("mutation", None)) - - assert [event[0] for event in events] == [ - "SELECT GET_LOCK(:lock_name, 30)", - "commit", - "mutation", - "SELECT RELEASE_LOCK(:lock_name)", - "commit", - ] - -def test_postgresql_initializer_relies_on_transaction_rollback(tmp_path, monkeypatch): - path = tmp_path / "postgresql-rollback.sqlite" - - def fail_migration(*_args, **_kwargs): - raise RuntimeError("injected catalog migration failure") - - def compensation_must_not_run(*_args, **_kwargs): - raise AssertionError("PostgreSQL must not use stale DDL compensation") - - monkeypatch.setattr( - wide, "effective_profile", lambda _url, **_kwargs: (POSTGRESQL, {}), - ) - monkeypatch.setattr(wide, "_migrate_catalog_columns", fail_migration) - monkeypatch.setattr( - wide, "_compensate_catalog_initialization", compensation_must_not_run, - ) - - with pytest.raises(RuntimeError, match="injected catalog migration failure"): - openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") - diff --git a/tests/test_catalog_persistence_review.py b/tests/test_catalog_persistence_review.py new file mode 100644 index 0000000..c1d50f5 --- /dev/null +++ b/tests/test_catalog_persistence_review.py @@ -0,0 +1,253 @@ +"""Regression tests for persistent URLs and operation-bound audit writes.""" + +import json +import sqlite3 +from contextlib import contextmanager + +import pytest + +from openstatspec.core import UnsupportedOperationError +from openstatspec.sql import wide +from openstatspec.sql.profiles import TargetCapabilityExceededError + + +def _variables(): + return [{ + "ordinal": 1, + "source_name": "name", + "physical_name": "name", + "storage_kind": "string", + "string_width": 8, + "label": "", + "format": "A8", + "measure": "nominal", + "alignment": "left", + "display_width": 8, + "value_labels": "{}", + "missing_ranges": "[]", + }] + + +def _create(database_url, dataset_id="sample"): + return wide.create_wide_dataset( + database_url=database_url, + dataset_id=dataset_id, + source_name=f"{dataset_id}.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + +@pytest.mark.parametrize( + "database_url", + [ + "sqlite://", + "sqlite:///:memory:", + "sqlite:///file:shared-memory?mode=memory&uri=true", + ], +) +def test_import_rejects_ephemeral_sqlite_catalogs(database_url): + with pytest.raises(UnsupportedOperationError, match="persistent SQLite"): + _create(database_url) + + +def test_initialization_does_not_modify_an_existing_unverified_catalog(tmp_path): + path = tmp_path / "partial.sqlite" + connection = sqlite3.connect(path) + connection.execute("create table dataset_catalog (legacy_name text)") + connection.commit() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="foreign"): + wide.initialize_wide_catalog(database_url=f"sqlite:///{path}") + + connection = sqlite3.connect(path) + tables = { + row[0] + for row in connection.execute( + "select name from sqlite_master where type = 'table'" + ) + } + connection.close() + assert tables == {"dataset_catalog"} + + +def test_import_routes_catalog_and_dataset_mutations_through_binding_guard( + tmp_path, monkeypatch, +): + database_url = f"sqlite:///{tmp_path / 'bound.sqlite'}" + real_bound_transaction = wide._bound_catalog_transaction + phases = [] + + @contextmanager + def observe(**kwargs): + phases.append(kwargs["phase"]) + with real_bound_transaction(**kwargs) as connection: + yield connection + + monkeypatch.setattr(wide, "_bound_catalog_transaction", observe) + _create(database_url) + + assert phases == ["catalog initialization", "import"] + + +def test_failed_import_preflight_routes_audit_through_binding_guard( + tmp_path, monkeypatch, +): + database_url = f"sqlite:///{tmp_path / 'failed-preflight.sqlite'}" + real_bound_transaction = wide._bound_catalog_transaction + phases = [] + + @contextmanager + def observe(**kwargs): + phases.append(kwargs["phase"]) + with real_bound_transaction(**kwargs) as connection: + yield connection + + monkeypatch.setattr(wide, "_bound_catalog_transaction", observe) + variables = _variables() + variables[0].update({ + "storage_kind": "numeric", "string_width": None, + "format": "F8.2", "alignment": "right", + }) + with pytest.raises(TargetCapabilityExceededError): + wide.create_wide_dataset( + database_url=database_url, + dataset_id="invalid", + source_name="invalid.sav", + source_format="SAV", + rows=[{"name": "not-a-number"}], + variables=variables, + ) + + assert phases == ["catalog initialization", "import preflight failure"] + + +def test_failed_import_preflight_rejects_catalog_drift_before_audit( + tmp_path, monkeypatch, +): + path = tmp_path / "failed-preflight-drift.sqlite" + database_url = f"sqlite:///{path}" + real_bound_transaction = wide._bound_catalog_transaction + + @contextmanager + def inject_drift(**kwargs): + if kwargs["phase"] == "import preflight failure": + connection = sqlite3.connect(path) + connection.execute("create view foreign_view as select 1 as value") + connection.commit() + connection.close() + with real_bound_transaction(**kwargs) as connection: + yield connection + + monkeypatch.setattr(wide, "_bound_catalog_transaction", inject_drift) + variables = _variables() + variables[0].update({ + "storage_kind": "numeric", "string_width": None, + "format": "F8.2", "alignment": "right", + }) + + with pytest.raises(UnsupportedOperationError, match="foreign"): + wide.create_wide_dataset( + database_url=database_url, + dataset_id="invalid", + source_name="invalid.sav", + source_format="SAV", + rows=[{"name": "not-a-number"}], + variables=variables, + ) + + connection = sqlite3.connect(path) + assert connection.execute("select count(*) from operation").fetchone() == (0,) + assert connection.execute("select count(*) from fidelity_event").fetchone() == (0,) + connection.close() + + +def test_export_audit_mutations_route_through_binding_guard(tmp_path, monkeypatch): + database_url = f"sqlite:///{tmp_path / 'bound-export.sqlite'}" + _create(database_url) + real_bound_transaction = wide._bound_catalog_transaction + phases = [] + + @contextmanager + def observe(**kwargs): + phases.append(kwargs["phase"]) + with real_bound_transaction(**kwargs) as connection: + yield connection + + monkeypatch.setattr(wide, "_bound_catalog_transaction", observe) + failed_id = wide.record_export_operation( + database_url=database_url, + dataset_id="sample", + destination="failed.sav", + allowed_fidelity_events=(), + terminal=False, + ) + wide.fail_export_operation( + database_url=database_url, + operation_id=failed_id, + failure_details={"reason": "test"}, + ) + succeeded_id = wide.record_export_operation( + database_url=database_url, + dataset_id="sample", + destination="succeeded.sav", + allowed_fidelity_events=(), + terminal=False, + ) + wide.finish_export_operation( + database_url=database_url, operation_id=succeeded_id, + ) + wide.record_export_backup_retained( + database_url=database_url, + operation_id=succeeded_id, + destination="succeeded.sav", + backup="succeeded.sav.backup", + cleanup_error=RuntimeError("test"), + ) + wide.record_export_cleanup_failure( + database_url=database_url, + destination="cleanup.sav", + original_error=RuntimeError("export"), + cleanup_error=RuntimeError("cleanup"), + residual_object_inventory={}, + deterministic_recovery_evidence={}, + ) + + assert phases == [ + "export audit creation", + "export audit failure", + "export audit creation", + "export audit finalization", + "export backup-retention audit", + "export cleanup-failure audit", + ] + + +def test_export_engine_identity_is_persisted_as_non_loss_audit_metadata(tmp_path): + path = tmp_path / "export-audit.sqlite" + database_url = f"sqlite:///{path}" + _create(database_url) + engine_identity = {"name": "pyspssio", "commit": "export-commit"} + + operation_id = wide.record_export_operation( + database_url=database_url, + dataset_id="sample", + destination="out.sav", + allowed_fidelity_events=(), + operation_details={"engine": engine_identity}, + ) + + connection = sqlite3.connect(path) + row = connection.execute( + "select severity, event_code, source_item, detail_json " + "from fidelity_event where operation_id = ?", + (operation_id,), + ).fetchone() + connection.close() + assert row[:3] == ("info", "operation-engine-identity", "out.sav") + assert json.loads(row[3])["engine"] == engine_identity + assert wide.read_fidelity_events( + database_url=database_url, dataset_id="sample", + ) == () diff --git a/tests/test_cli.py b/tests/test_cli.py index 6f85938..e10b865 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -67,7 +67,7 @@ def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> "multiple_response_sets": "supported", "variable_alignment": "supported", "variable_sets": "supported", - "compatible_variable_names": "supported", + "compatible_variable_names": "requires-explicit-loss-consent", "custom_attributes": { "scalar_values": "supported", "ordered_value_arrays": "supported", diff --git a/tests/test_conditional_inplace_transform.py b/tests/test_conditional_inplace_transform.py index c37e45e..f4c62f3 100644 --- a/tests/test_conditional_inplace_transform.py +++ b/tests/test_conditional_inplace_transform.py @@ -2,11 +2,11 @@ import sqlite3 -from types import SimpleNamespace import pytest import openstatspec import openstatspec.sql.inplace_transform as inplace_transform +from openstatspec.sql.profiles import DOLT from openstatspec.sql.wide import create_wide_dataset @@ -115,10 +115,6 @@ def test_exact_bounded_program_applies_data_and_both_catalogs( "write_format_decimals, measurement_level FROM variable " "WHERE source_name = 'target'" ).fetchone() == ("Example label", "F", 1, 0, "F", 1, 0, "nominal") - assert connection.execute( - "SELECT label, format, print_format, write_format, measure " - "FROM variable_catalog WHERE source_name = 'target'" - ).fetchone() == ("Example label", "F1.0", "[5, 1, 0]", "[5, 1, 0]", "nominal") assert connection.execute( "SELECT numeric_code, label FROM value_label ORDER BY ordinal" ).fetchall() == [(0.0, "No"), (1.0, "Yes")] @@ -168,9 +164,6 @@ def fail(selected: str) -> None: assert connection.execute( "SELECT COUNT(*) FROM variable WHERE source_name = 'target'" ).fetchone() == (0,) - assert connection.execute( - "SELECT COUNT(*) FROM variable_catalog WHERE source_name = 'target'" - ).fetchone() == (0,) assert connection.execute( "SELECT COUNT(*) FROM transformation_apply" ).fetchone() == (0,) @@ -261,7 +254,7 @@ def test_dolt_mock_applies_exact_program_to_preexisting_target_without_schema_dd monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (DOLT, {}), ) states = iter([ ("main", "abc123", 0), @@ -298,7 +291,7 @@ def test_dolt_mock_rejects_create_target_before_schema_mutation( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (DOLT, {}), ) monkeypatch.setattr( inplace_transform, @@ -341,7 +334,7 @@ def test_dolt_mock_rechecks_clean_state_after_dataset_lock( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (DOLT, {}), ) states = iter([ ("main", "abc123", 0), diff --git a/tests/test_document_round_trip.py b/tests/test_document_round_trip.py index 1b1bae1..a047bc8 100644 --- a/tests/test_document_round_trip.py +++ b/tests/test_document_round_trip.py @@ -10,7 +10,6 @@ import openstatspec from openstatspec.spss.raw_dictionary import ( read_document_lines, - write_compatible_names, write_document_lines, ) @@ -35,7 +34,7 @@ def test_document_lines_round_trip_through_sqlite(destination_suffix: str, tmp_p assert imported.diagnostics == () connection = sqlite3.connect(tmp_path / "documents.sqlite") assert connection.execute( - "select ordinal, text from document_catalog order by ordinal" + "select source_ordinal, document_text from document order by source_ordinal" ).fetchall() == [(1, expected[0]), (2, expected[1])] openstatspec.export_sav( @@ -75,7 +74,7 @@ def test_document_lines_import_from_zsav_and_export_to_sav(tmp_path: Path) -> No assert pyspssio.read_sav(str(destination))[0]["answer"].tolist() == [1.0, 2.0] -def test_document_and_compatible_name_round_trip_to_zsav(tmp_path: Path) -> None: +def test_document_and_long_variable_name_round_trip_to_zsav(tmp_path: Path) -> None: source = tmp_path / "source.sav" destination = tmp_path / "destination.zsav" database = f"sqlite:///{tmp_path / 'combined.sqlite'}" @@ -83,13 +82,14 @@ def test_document_and_compatible_name_round_trip_to_zsav(tmp_path: Path) -> None source_name = "long_variable_name" pyspssio.write_sav(str(source), pd.DataFrame({source_name: [7.0]})) write_document_lines(source, ["Combined dictionary fixture."], encoding="UTF-8") - write_compatible_names(source, {source_name: "ANSWER"}, encoding="UTF-8") openstatspec.import_sav(source, database_url=database, dataset_id="combined") - openstatspec.export_sav(database_url=database, dataset_id="combined", destination=destination) + openstatspec.export_sav( + database_url=database, dataset_id="combined", destination=destination, + allow_loss=["compatible-variable-names-not-preserved"], + ) assert read_document_lines(destination, encoding="UTF-8") == ["Combined dictionary fixture."] - assert pyspssio.read_metadata(str(destination))["var_compat_names"][source_name] == "ANSWER" assert pyspssio.read_sav(str(destination))[0][source_name].tolist() == [7.0] diff --git a/tests/test_dolt_conformance.py b/tests/test_dolt_conformance.py index e70d164..b4e0330 100644 --- a/tests/test_dolt_conformance.py +++ b/tests/test_dolt_conformance.py @@ -380,4 +380,3 @@ def capture_failure(**kwargs: object) -> None: dolt_conformance_source=sentinel, ) assert calls == [("cleanup", sentinel), ("failure", sentinel)] - diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index deded49..cd97771 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -1,6 +1,7 @@ from __future__ import annotations import sqlite3 +from dataclasses import replace from types import SimpleNamespace import pytest @@ -12,7 +13,12 @@ InPlacePlanSubmission, _apply_plan_on_connection, ) -from openstatspec.sql.wide import create_wide_dataset +from openstatspec.sql.profiles import ( + DOLT, SQLITE, TargetCapabilityExceededError, +) +from openstatspec.sql.wide import ( + create_wide_dataset, read_wide_dataset, validate_wide_dataset, +) def _variables() -> list[dict[str, object]]: @@ -102,7 +108,9 @@ def test_plan_applies_to_same_dataset_and_physical_table_without_copy( ), actor="test-agent", database_profile="sqlite", + target_profile=inplace_transform.effective_profile(url)[0], allow_schema_change=True, + allow_delete_variable=True, dolt_branch=None, dolt_head=None, ) @@ -142,9 +150,7 @@ def test_plan_applies_to_same_dataset_and_physical_table_without_copy( assert connection.execute( "SELECT numeric_code, label FROM value_label ORDER BY ordinal" ).fetchall() == [(0.0, "Lower"), (1.0, "Upper")] - assert connection.execute( - "SELECT label FROM variable_catalog WHERE source_name = 'score_band'" - ).fetchone() == ("Score band",) + assert "variable_catalog" not in tables assert connection.execute( "SELECT database_profile, dolt_branch, dolt_head_before, " "dolt_head_after, actor, status " @@ -155,6 +161,268 @@ def test_plan_applies_to_same_dataset_and_physical_table_without_copy( ) +def test_string_declaration_creates_column_and_catalog_variable(catalog) -> None: + url, path, dataset_id, table_name = catalog + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="STRING note (A8).", + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select source_name, storage_kind, declared_string_width from variable " + "where dataset_id = ? order by source_ordinal", + (dataset_id,), + ).fetchall() == [("score", "numeric", None), ("note", "string", 8)] + assert connection.execute( + f'SELECT note FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == [("",), ("",), ("",)] + note_column = next( + row for row in connection.execute(f'PRAGMA table_info("{table_name}")') + if row[1] == "note" + ) + assert note_column[2:5] == ("TEXT", 1, "''") + connection.close() + assert validate_wide_dataset( + database_url=url, dataset_id=dataset_id, + )["valid"] is True + + +def test_delete_recanonicalizes_surviving_collision_columns(tmp_path) -> None: + path = tmp_path / "collision-delete.sqlite" + url = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=url) + base = _variables()[0] + variables = [ + { + **base, + "ordinal": ordinal, + "source_name": source_name, + "physical_name": physical, + } + for ordinal, (source_name, physical) in enumerate(( + ("a-b", "a_b"), ("a_b", "a_b_2"), ("keep", "keep"), + ), start=1) + ] + create_wide_dataset( + database_url=url, + dataset_id="collision_source", + source_name="collision.sav", + source_format="SAV", + source_sha256="c" * 64, + rows=[ + {"a_b": 1.0, "a_b_2": 2.0, "keep": 3.0}, + {"a_b": 4.0, "a_b_2": 5.0, "keep": 6.0}, + ], + variables=variables, + ) + openstatspec.install_in_place_transformation_schema(database_url=url) + connection = sqlite3.connect(path) + dataset_id = connection.execute( + "select dataset_id from dataset where dataset_name = 'collision_source'" + ).fetchone()[0] + connection.close() + plan = openstatspec.TransformationPlan( + (openstatspec.DeleteVariableOperation("a-b"),), + contract="openstatspec-transformation-plan-v0.3", + ) + + openstatspec.apply_transformation_plan_in_place( + database_url=url, + dataset_id=dataset_id, + plan=plan, + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select source_name, physical_name, source_ordinal from variable " + "where dataset_id = ? order by source_ordinal", + (dataset_id,), + ).fetchall() == [("a_b", "a_b", 1), ("keep", "keep", 2)] + assert connection.execute( + "select a_b, keep from data_collision_source order by __case_ordinal" + ).fetchall() == [(2.0, 3.0), (5.0, 6.0)] + connection.close() + assert validate_wide_dataset( + database_url=url, dataset_id=dataset_id, + )["valid"] is True + + +def test_delete_prunes_an_empty_spss_variable_set(catalog) -> None: + url, path, dataset_id, _table_name = catalog + connection = sqlite3.connect(path) + variable_id = connection.execute( + "select variable_id from variable where dataset_id = ?", + (dataset_id,), + ).fetchone()[0] + variable_set_id = "00000000-0000-0000-0000-000000000002" + connection.execute( + "insert into variable_set " + "(variable_set_id, dataset_id, source_ordinal, set_name) " + "values (?, ?, 1, 'scores')", + (variable_set_id, dataset_id), + ) + connection.execute( + "insert into variable_set_member " + "(variable_set_id, variable_id, source_ordinal) values (?, ?, 1)", + (variable_set_id, variable_id), + ) + connection.commit() + connection.close() + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="COMPUTE other = score. DELETE VARIABLES score.", + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select count(*) from variable_set where dataset_id = ?", + (dataset_id,), + ).fetchone() == (0,) + assert connection.execute( + "select count(*) from variable_set_member" + ).fetchone() == (0,) + connection.close() + dataset, _variables, _rows = read_wide_dataset( + database_url=url, dataset_id=dataset_id, + ) + assert dataset["source_extensions"] == {} + + +def test_delete_prunes_an_empty_multiple_response_set(catalog) -> None: + url, path, dataset_id, _table_name = catalog + connection = sqlite3.connect(path) + variable_id = connection.execute( + "select variable_id from variable where dataset_id = ?", + (dataset_id,), + ).fetchone()[0] + response_set_id = "00000000-0000-0000-0000-000000000001" + connection.execute( + "insert into multiple_response_set " + "(multiple_response_set_id, dataset_id, source_ordinal, set_name, " + "set_kind, counted_value_kind, counted_numeric_value) " + "values (?, ?, 1, '$scores', 'MD', 'numeric', 1.0)", + (response_set_id, dataset_id), + ) + connection.execute( + "insert into multiple_response_member " + "(multiple_response_set_id, variable_id, source_ordinal) " + "values (?, ?, 1)", + (response_set_id, variable_id), + ) + connection.commit() + connection.close() + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="COMPUTE other = score. DELETE VARIABLES score.", + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select count(*) from multiple_response_set where dataset_id = ?", + (dataset_id,), + ).fetchone() == (0,) + assert connection.execute( + "select count(*) from multiple_response_member" + ).fetchone() == (0,) + connection.close() + assert validate_wide_dataset( + database_url=url, dataset_id=dataset_id, + )["valid"] is True + + +def test_delete_then_recreate_same_name_resolves_operations_in_order(catalog) -> None: + url, path, dataset_id, table_name = catalog + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text=( + "COMPUTE other = score. DELETE VARIABLES score. " + "RECODE other (2 = 9) (ELSE = COPY) INTO score." + ), + actor="test-agent", + ) + + connection = sqlite3.connect(path) + variables = connection.execute( + "select source_name, physical_name, source_ordinal from variable " + "where dataset_id = ? order by source_ordinal", + (dataset_id,), + ).fetchall() + assert [(row[0], row[2]) for row in variables] == [ + ("other", 1), ("score", 2), + ] + physical = {row[0]: row[1] for row in variables} + assert connection.execute( + f'SELECT "{physical["other"]}", "{physical["score"]}" ' + f'FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == [(1.0, 1.0), (2.0, 9.0), (3.0, 3.0)] + connection.close() + assert validate_wide_dataset( + database_url=url, dataset_id=dataset_id, + )["valid"] is True + + +def test_temporary_created_target_can_be_deleted_before_final_schema(catalog) -> None: + url, path, dataset_id, table_name = catalog + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="COMPUTE tmp = score. DELETE VARIABLES tmp.", + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select source_name, source_ordinal from variable " + "where dataset_id = ? order by source_ordinal", + (dataset_id,), + ).fetchall() == [("score", 1)] + assert [ + row[1] for row in connection.execute(f'PRAGMA table_info("{table_name}")') + ] == ["__case_ordinal", "score"] + connection.close() + + +def test_temporary_target_type_is_not_taken_from_same_name_recreation(catalog) -> None: + url, path, dataset_id, table_name = catalog + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text=( + "COMPUTE tmp = score. DELETE VARIABLES tmp. STRING tmp (A4)." + ), + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select source_name, storage_kind, declared_string_width " + "from variable where dataset_id = ? order by source_ordinal", + (dataset_id,), + ).fetchall() == [("score", "numeric", None), ("tmp", "string", 4)] + assert connection.execute( + f'SELECT score, tmp FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == [(1.0, ""), (2.0, ""), (3.0, "")] + connection.close() + assert validate_wide_dataset( + database_url=url, dataset_id=dataset_id, + )["valid"] is True + + def test_public_apply_supports_non_dolt_without_building_undo(catalog) -> None: url, path, dataset_id, table_name = catalog plan = _plan("RECODE score (1 = 0).") @@ -179,6 +447,25 @@ def test_public_apply_supports_non_dolt_without_building_undo(catalog) -> None: ) +def test_schema_commands_record_the_v03_frontend_contract(catalog) -> None: + url, path, dataset_id, _table_name = catalog + + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="STRING note (A4).", + actor="test-agent", + ) + + audit = sqlite3.connect(path).execute( + "SELECT source_kind, frontend_contract FROM transformation_apply" + ).fetchone() + assert audit == ( + "spss_syntax", + "openstatspec-spss-syntax-frontend-v0.3", + ) + + @pytest.mark.parametrize("as_mapping", [False, True]) def test_public_generic_plan_apply_accepts_object_and_mapping( catalog, as_mapping, @@ -211,6 +498,45 @@ def test_public_generic_plan_apply_accepts_object_and_mapping( ) +def test_generic_string_width_is_rejected_before_ddl( + catalog, monkeypatch, +) -> None: + url, path, dataset_id, table_name = catalog + monkeypatch.setattr( + inplace_transform, + "effective_profile", + lambda _url, **_kwargs: ( + replace(SQLITE, max_text_value_bytes=3), + {"server_version": "3.35.0"}, + ), + ) + plan = openstatspec.TransformationPlan( + (openstatspec.CreateVariableOperation("note", "string", 4),), + contract="openstatspec-transformation-plan-v0.3", + ) + + with pytest.raises(TargetCapabilityExceededError, match="permits 3"): + openstatspec.apply_transformation_plan_in_place( + database_url=url, + dataset_id=dataset_id, + plan=plan, + actor="test-agent", + ) + + connection = sqlite3.connect(path) + assert [ + row[1] for row in connection.execute(f'PRAGMA table_info("{table_name}")') + ] == ["__case_ordinal", "score"] + assert connection.execute( + "select source_name from variable where dataset_id = ?", + (dataset_id,), + ).fetchall() == [("score",)] + assert connection.execute( + "select count(*) from transformation_apply" + ).fetchone() == (0,) + connection.close() + + def test_generic_plan_is_bound_to_live_schema_before_mutation(catalog) -> None: url, path, dataset_id, table_name = catalog plan = openstatspec.TransformationPlan(( @@ -366,6 +692,38 @@ def test_missing_audit_schema_fails_before_mutation(catalog) -> None: ).fetchall() == [(1.0,), (2.0,), (3.0,)] +def test_delete_is_rejected_when_drop_column_is_unavailable( + catalog, monkeypatch, +) -> None: + url, path, dataset_id, table_name = catalog + monkeypatch.setattr( + inplace_transform, + "effective_profile", + lambda _url, **_kwargs: ( + SQLITE, + {"server_version": "3.34.0"}, + ), + ) + + with pytest.raises(openstatspec.TransformationError) as caught: + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="COMPUTE other = score. DELETE VARIABLES score.", + actor="test-agent", + ) + + assert caught.value.code == "delete_variable_not_supported" + connection = sqlite3.connect(path) + assert connection.execute( + f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == [(1.0,), (2.0,), (3.0,)] + assert [ + row[1] for row in connection.execute(f'PRAGMA table_info("{table_name}")') + ] == ["__case_ordinal", "score"] + connection.close() + + def test_nontransactional_ddl_profile_rejects_create_before_mutation( catalog, ) -> None: @@ -382,7 +740,9 @@ def test_nontransactional_ddl_profile_rejects_create_before_mutation( ), actor="test-agent", database_profile="mysql", + target_profile=inplace_transform.effective_profile(url)[0], allow_schema_change=False, + allow_delete_variable=True, dolt_branch=None, dolt_head=None, ) @@ -406,7 +766,7 @@ def test_public_apply_binds_expected_dolt_branch_and_head( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (DOLT, {}), ) states = iter([ ("feature/recode", "abc123", 0), @@ -437,7 +797,7 @@ def test_public_apply_rejects_dirty_dolt_working_set_before_mutation( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (DOLT, {}), ) monkeypatch.setattr( inplace_transform, @@ -473,7 +833,7 @@ def test_public_apply_rejects_dolt_context_mismatch_before_mutation( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (DOLT, {}), ) monkeypatch.setattr( inplace_transform, "_dolt_state", lambda _connection: state @@ -591,7 +951,7 @@ def test_schema_install_requires_initialized_verified_catalog(tmp_path) -> None: with pytest.raises( openstatspec.UnsupportedOperationError, - match="explicit catalog initialization", + match="catalog is absent", ): openstatspec.install_in_place_transformation_schema(database_url=url) @@ -604,16 +964,15 @@ def test_schema_install_requires_initialized_verified_catalog(tmp_path) -> None: def test_public_apply_rejects_divergent_catalog_before_mutation(catalog) -> None: url, path, dataset_id, table_name = catalog connection = sqlite3.connect(path) - deleted = connection.execute("DELETE FROM dataset_catalog") - assert deleted.rowcount == 1 + connection.execute("UPDATE catalog_identity SET schema_version = 999") connection.commit() before = connection.execute( f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' ).fetchall() with pytest.raises( - openstatspec.UnsupportedOperationError, - match="explicit catalog initialization", + RuntimeError, + match="identity is incompatible", ): openstatspec.apply_spss_in_place( database_url=url, @@ -660,16 +1019,16 @@ def test_public_apply_rejects_divergent_variable_mapping_before_mutation( ) -> None: url, path, dataset_id, table_name = catalog connection = sqlite3.connect(path) - deleted = connection.execute("DELETE FROM variable_catalog") - assert deleted.rowcount == 1 + deleted = connection.execute("DELETE FROM variable") + assert deleted.rowcount > 0 connection.commit() before = connection.execute( f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' ).fetchall() with pytest.raises( - openstatspec.UnsupportedOperationError, - match="explicit catalog initialization", + openstatspec.TransformationError, + match="no variables", ): openstatspec.apply_spss_in_place( database_url=url, @@ -695,7 +1054,10 @@ def test_transactional_ddl_rollback_skips_unlocked_compensation( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url, **_kwargs: (SimpleNamespace(name=profile_name), {}), + lambda _url, **_kwargs: ( + SimpleNamespace(name=profile_name), + {"server_version": "3.35.0"}, + ), ) def fail_after_schema_change( @@ -724,4 +1086,3 @@ def fail_after_schema_change( "RECODE score (1 = 0) INTO score_band." ), ) - diff --git a/tests/test_loss_reports.py b/tests/test_loss_reports.py index 2e4100f..d30b32c 100755 --- a/tests/test_loss_reports.py +++ b/tests/test_loss_reports.py @@ -1,4 +1,3 @@ -import json import sqlite3 import pandas as pd @@ -8,8 +7,8 @@ import openstatspec from openstatspec.core import UnsupportedOperationError from openstatspec.sql.wide import create_wide_dataset -from openstatspec.spss.raw_dictionary import write_compatible_names from openstatspec.spss import sav as sav_module +from openstatspec.spss.raw_dictionary import write_compatible_names _REQUIRED_ENGINE_LOSS = [] @@ -19,7 +18,6 @@ def test_persisted_import_fidelity_events_require_consent_after_reopen(tmp_path) source = tmp_path / "source.sav" database_path = tmp_path / "persisted.sqlite" database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) blocked = tmp_path / "blocked.sav" approved = tmp_path / "approved.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) @@ -27,25 +25,29 @@ def test_persisted_import_fidelity_events_require_consent_after_reopen(tmp_path) imported = openstatspec.import_sav(source, database_url=database, dataset_id="persisted") assert {diagnostic.code for diagnostic in imported.diagnostics} == set(_REQUIRED_ENGINE_LOSS) connection = sqlite3.connect(database_path) - assert {row[0] for row in connection.execute("select code from fidelity_event_catalog")} == set(_REQUIRED_ENGINE_LOSS) - import_details = json.loads(connection.execute("select details from operation_catalog order by created_at limit 1").fetchone()[0]) - assert import_details["engine"]["package"] == "openstatspec-pyspssio" - assert import_details["engine"]["pinned_commit"] == "e069adf33c70bcd9e8e6ee495106479463a84fa2" + assert { + row[0] for row in connection.execute( + "select event_code from fidelity_event where severity != 'info'" + ) + } == set(_REQUIRED_ENGINE_LOSS) + assert connection.execute( + "select operation_kind, status from operation order by started_at limit 1" + ).fetchone() == ("import", "succeeded") openstatspec.export_sav(database_url=database, dataset_id="persisted", destination=blocked) assert blocked.exists() exported = openstatspec.export_sav(database_url=database, dataset_id="persisted", destination=approved, allow_loss=_REQUIRED_ENGINE_LOSS) assert approved.exists() - export_details = json.loads(connection.execute("select details from operation_catalog where operation_id = ?", (exported["operation_id"],)).fetchone()[0]) - assert export_details["engine"]["installed_version"] == pyspssio.__version__ + assert connection.execute( + "select operation_kind, status from operation where operation_id = ?", (exported["operation_id"],) + ).fetchone() == ("export", "succeeded") assert {diagnostic.code for diagnostic in exported.diagnostics} == set(_REQUIRED_ENGINE_LOSS) def test_loss_allowed_export_persists_accepted_diagnostics(tmp_path) -> None: database_path = tmp_path / "accepted-loss.sqlite" database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) source = tmp_path / "source.sav" destination = tmp_path / "accepted.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) @@ -53,7 +55,7 @@ def test_loss_allowed_export_persists_accepted_diagnostics(tmp_path) -> None: result = openstatspec.export_sav(database_url=database, dataset_id="accepted", destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS) connection = sqlite3.connect(database_path) - rows = connection.execute("select direction, severity, code, details from fidelity_event_catalog where operation_id = ? order by code", (result["operation_id"],)).fetchall() + rows = connection.execute("select direction, severity, event_code, detail_json from fidelity_event where operation_id = ? and severity != 'info' order by event_code", (result["operation_id"],)).fetchall() assert [(row[0], row[1], row[2]) for row in rows] == [("export", "warning", code) for code in _REQUIRED_ENGINE_LOSS] assert all('"accepted_by_user": true' in row[3] for row in rows) @@ -61,7 +63,6 @@ def test_loss_allowed_export_persists_accepted_diagnostics(tmp_path) -> None: def test_non_utf8_source_encoding_is_explicit_export_loss(tmp_path) -> None: database_path = tmp_path / "legacy-encoding.sqlite" database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) destination = tmp_path / "legacy-encoding.sav" create_wide_dataset( database_url=database, dataset_id="legacy-encoding", source_name="legacy.sav", @@ -84,7 +85,6 @@ def test_non_utf8_source_encoding_is_explicit_export_loss(tmp_path) -> None: def test_explicit_legacy_locale_selects_the_single_engine_route(tmp_path, monkeypatch) -> None: database_path = tmp_path / "legacy-locale.sqlite" database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) destination = tmp_path / "legacy-locale.sav" create_wide_dataset( database_url=database, dataset_id="legacy-locale", source_name="legacy.sav", @@ -125,38 +125,35 @@ def test_legacy_locale_must_emit_the_exact_source_encoding() -> None: with pytest.raises(UnsupportedOperationError, match="instead of required"): sav_module._require_matching_legacy_encoding("WINDOWS-1252", "UTF-8", True) -@pytest.mark.parametrize("suffix", [".sav", ".zsav"]) -def test_compatible_variable_name_round_trips_from_current_sql_catalog(tmp_path, suffix: str) -> None: - source = tmp_path / f"compat-source{suffix}" - database_path = tmp_path / f"compat-{suffix[1:]}.sqlite" - database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) - destination = tmp_path / f"compat-destination{suffix}" + +def test_compatible_variable_names_are_explicit_imported_loss(tmp_path) -> None: + source = tmp_path / "compatible-name.sav" + destination = tmp_path / "compatible-name-out.sav" + database = f"sqlite:///{tmp_path / 'compatible-name.sqlite'}" source_name = "long_variable_name" + loss_code = "compatible-variable-names-not-preserved" pyspssio.write_sav(str(source), pd.DataFrame({source_name: [1.0]})) write_compatible_names(source, {source_name: "ANSWER"}, encoding="UTF-8") - assert pyspssio.read_metadata(str(source))["var_compat_names"][source_name] == "ANSWER" - imported = openstatspec.import_sav(source, database_url=database, dataset_id=f"compat-{suffix[1:]}") - assert imported.diagnostics == () - connection = sqlite3.connect(database_path) - assert connection.execute( - "select compat_name from variable_catalog where dataset_id = ? and source_name = ?", - (f"compat-{suffix[1:]}", source_name), - ).fetchone() == ("ANSWER",) - - # The normalized catalog is authoritative: a legitimate 8-byte short name - # edited there must be written into both SAV dictionary locations. - connection.execute( - "update variable_catalog set compat_name = ? where dataset_id = ? and source_name = ?", - ("EXAMPLE", f"compat-{suffix[1:]}", source_name), + imported = openstatspec.import_sav( + source, database_url=database, dataset_id="compatible-name", ) - connection.commit() - result = openstatspec.export_sav( - database_url=database, dataset_id=f"compat-{suffix[1:]}", destination=destination, + assert {diagnostic.code for diagnostic in imported.diagnostics} == {loss_code} + assert imported.diagnostics[0].details == {"variable_names": [source_name]} + + with pytest.raises(UnsupportedOperationError, match=loss_code): + openstatspec.export_sav( + database_url=database, dataset_id="compatible-name", + destination=destination, + ) + + exported = openstatspec.export_sav( + database_url=database, dataset_id="compatible-name", + destination=destination, allow_loss=[loss_code], + ) + assert {diagnostic.code for diagnostic in exported.diagnostics} == {loss_code} + assert ( + pyspssio.read_metadata(str(destination))["var_compat_names"][source_name] + != "ANSWER" ) - assert result.diagnostics == () - metadata = pyspssio.read_metadata(str(destination)) - assert metadata["var_compat_names"][source_name] == "EXAMPLE" - assert pyspssio.read_sav(str(destination))[0][source_name].tolist() == [1.0] diff --git a/tests/test_official_conformance.py b/tests/test_official_conformance.py index 57c2d90..48e95a6 100644 --- a/tests/test_official_conformance.py +++ b/tests/test_official_conformance.py @@ -236,7 +236,9 @@ def _assert_round_trip( imported = openstatspec.import_sav( source, database_url=database_url, dataset_id=dataset_id, ) - assert imported.diagnostics == () + assert {diagnostic.code for diagnostic in imported.diagnostics} <= { + "compatible-variable-names-not-preserved", + } assert openstatspec.validate( database_url=database_url, dataset_id=dataset_id, )["valid"] is True @@ -244,8 +246,11 @@ def _assert_round_trip( exported = openstatspec.export_sav( database_url=database_url, dataset_id=dataset_id, destination=destination, + allow_loss=["compatible-variable-names-not-preserved"], ) - assert exported.diagnostics == () + assert {diagnostic.code for diagnostic in exported.diagnostics} <= { + "compatible-variable-names-not-preserved", + } assert compare_sav_semantics(source, destination) == { "equivalent": True, "differences": [], diff --git a/tests/test_pyspssio_catalog_authority.py b/tests/test_pyspssio_catalog_authority.py index 6093c0d..591ee0b 100644 --- a/tests/test_pyspssio_catalog_authority.py +++ b/tests/test_pyspssio_catalog_authority.py @@ -1,4 +1,3 @@ -import json import sqlite3 import pandas as pd @@ -31,8 +30,11 @@ def dictionary_with_variable_set(path): assert {diagnostic.code for diagnostic in imported.diagnostics} == set() connection = sqlite3.connect(database_path) assert connection.execute( - "select extension_key, payload from source_extension_catalog where dataset_id = 'variables'" - ).fetchone() == ("spss.variable_sets", json.dumps({"Analysis": ["answer"]})) + "select vs.set_name, v.source_name from variable_set vs " + "join variable_set_member vsm on vsm.variable_set_id = vs.variable_set_id " + "join variable v on v.variable_id = vsm.variable_id " + "order by vs.source_ordinal, vsm.source_ordinal" + ).fetchall() == [("Analysis", "answer")] def test_normalized_mr_catalog_is_authoritative_for_export(tmp_path) -> None: @@ -60,20 +62,24 @@ def test_normalized_mr_catalog_is_authoritative_for_export(tmp_path) -> None: openstatspec.import_sav(source, database_url=database, dataset_id="mr") connection = sqlite3.connect(database_path) rows = connection.execute( - "select set_name, kind, is_dichotomy, use_category_labels, use_first_var_label, counted_value_type, counted_numeric, variable_name " - "from multiple_response_set_catalog order by set_name, member_ordinal" + "select mrs.set_name, mrs.set_kind, " + "mrs.category_label_behavior, mrs.label_source, " + "mrs.counted_value_kind, mrs.counted_numeric_value, v.source_name " + "from multiple_response_set mrs " + "join multiple_response_member mrm " + "on mrm.multiple_response_set_id = mrs.multiple_response_set_id " + "join variable v on v.variable_id = mrm.variable_id " + "order by mrs.set_name, mrm.source_ordinal" ).fetchall() assert rows == [ - ("$extended", "MD", 1, 1, 1, "numeric", 1.0, "ex_a"), - ("$extended", "MD", 1, 1, 1, "numeric", 1.0, "ex_b"), - ("$mc", "MC", 0, 0, 0, None, None, "mc_a"), - ("$mc", "MC", 0, 0, 0, None, None, "mc_b"), - ("$md", "MD", 1, 0, 0, "numeric", 1.0, "md_a"), - ("$md", "MD", 1, 0, 0, "numeric", 1.0, "md_b"), + ("$extended", "MD", "counted_values", "variable_label", "numeric", 1.0, "ex_a"), + ("$extended", "MD", "counted_values", "variable_label", "numeric", 1.0, "ex_b"), + ("$mc", "MC", "variable_labels", "set_label", None, None, "mc_a"), + ("$mc", "MC", "variable_labels", "set_label", None, None, "mc_b"), + ("$md", "MD", "variable_labels", "set_label", "numeric", 1.0, "md_a"), + ("$md", "MD", "variable_labels", "set_label", "numeric", 1.0, "md_b"), ] - # The JSON is only legacy compatibility now; normalized rows must drive the writer. - connection.execute("update dataset_catalog set multiple_response_sets = '{}' where dataset_id = 'mr'") - connection.execute("update multiple_response_set_catalog set label = 'MD catalog' where dataset_id = 'mr' and set_name = '$md'") + connection.execute("update multiple_response_set set set_label = 'MD catalog' where set_name = '$md'") connection.commit() openstatspec.export_sav(database_url=database, dataset_id="mr", destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS) exported = pyspssio.read_metadata(str(destination))["mrsets"] diff --git a/tests/test_review_transaction_boundary.py b/tests/test_review_transaction_boundary.py new file mode 100644 index 0000000..2fe5165 --- /dev/null +++ b/tests/test_review_transaction_boundary.py @@ -0,0 +1,209 @@ +"""Regression tests for bound commit checks and export dataset linkage.""" + +import sqlite3 + +from sqlalchemy import create_engine + +from openstatspec.sql import wide + + +def _variables(): + return [{ + "ordinal": 1, + "source_name": "name", + "physical_name": "name", + "storage_kind": "string", + "string_width": 8, + "label": "", + "format": "A8", + "measure": "nominal", + "alignment": "left", + "display_width": 8, + "value_labels": "{}", + "missing_ranges": "[]", + }] + + +def test_dolt_completion_identity_is_checked_before_transaction_commit( + monkeypatch, +): + engine = create_engine("sqlite://") + state = { + "database": "catalog", + "active_branch": "main", + "head": "abc123", + } + transaction_states = [] + + def capture(connection, **_kwargs): + transaction_states.append(connection.in_transaction()) + return state + + monkeypatch.setattr(wide, "_capture_dolt_state", capture) + with wide._bound_catalog_transaction( + engine=engine, + profile_name="dolt", + active={ + "working_set_binding": { + "database": "catalog", + "active_branch": "main", + }, + }, + audit_relations=set(), + phase="test", + ): + pass + + assert transaction_states == [False, True] + + +class _DoltResult: + def __init__(self, rows): + self.rows = rows + + def mappings(self): + return self + + def one(self): + return self.rows[0] + + def all(self): + return self.rows + + +class _DoltDiffConnection: + def __init__(self, summary): + self.summary = summary + + def exec_driver_sql(self, statement): + if "ACTIVE_BRANCH" in statement: + return _DoltResult([{ + "database_name": "catalog", + "active_branch": "main", + "head_hash": "abc123", + }]) + if "dolt_status" in statement: + return _DoltResult([]) + return _DoltResult([self.summary]) + + +def _dolt_summary(**changes): + result = { + "from_table_name": "operation", + "to_table_name": "operation", + "diff_type": "modified", + "data_change": 1, + "schema_change": 0, + } + result.update(changes) + return result + + +def test_dolt_classifier_allows_only_same_table_audit_data_changes(): + allowed = wide._capture_dolt_state( + _DoltDiffConnection(_dolt_summary()), + profile_name="dolt", + audit_relations={"operation", "fidelity_event"}, + ) + assert allowed is not None + assert all( + not evidence["rows"] + for evidence in allowed["unrelated_working_set"]["diff_summaries"].values() + ) + + for unsafe in ( + _dolt_summary(schema_change=1), + _dolt_summary(diff_type="dropped", to_table_name=None, schema_change=1), + _dolt_summary( + diff_type="renamed", to_table_name="fidelity_event", schema_change=1, + ), + ): + classified = wide._capture_dolt_state( + _DoltDiffConnection(unsafe), + profile_name="dolt", + audit_relations={"operation", "fidelity_event"}, + ) + assert classified is not None + assert all( + evidence["rows"] == [unsafe] + for evidence in classified[ + "unrelated_working_set" + ]["diff_summaries"].values() + ) + + +def test_dolt_completion_rejects_unrelated_working_set_changes(): + before = { + "database": "catalog", "active_branch": "main", "head": "abc123", + "unrelated_sha256": "before", + } + after = {**before, "unrelated_sha256": "after"} + + import pytest + from openstatspec.core import UnsupportedOperationError + + with pytest.raises(UnsupportedOperationError, match="unrelated working-set"): + wide._require_dolt_success_identity(before, after, phase="export audit") + + +def test_export_lifecycle_events_remain_linked_to_the_dataset(tmp_path): + path = tmp_path / "export-events.sqlite" + database_url = f"sqlite:///{path}" + wide.create_wide_dataset( + database_url=database_url, + dataset_id="sample", + source_name="sample.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + failed_id = wide.record_export_operation( + database_url=database_url, + dataset_id="sample", + destination="failed.sav", + allowed_fidelity_events=(), + terminal=False, + ) + wide.fail_export_operation( + database_url=database_url, + operation_id=failed_id, + failure_details={"reason": "test"}, + ) + succeeded_id = wide.record_export_operation( + database_url=database_url, + dataset_id="sample", + destination="succeeded.sav", + allowed_fidelity_events=(), + terminal=False, + ) + wide.finish_export_operation( + database_url=database_url, + operation_id=succeeded_id, + ) + wide.record_export_backup_retained( + database_url=database_url, + operation_id=succeeded_id, + destination="succeeded.sav", + backup="succeeded.sav.backup", + cleanup_error=RuntimeError("test"), + ) + + events = wide.read_fidelity_events( + database_url=database_url, + dataset_id="sample", + ) + assert {event["code"] for event in events} == { + "backup_retained", + "export_failed", + } + assert wide.read_fidelity_events( + database_url=database_url, dataset_id="sample", direction="import", + ) == () + connection = sqlite3.connect(path) + assert connection.execute( + "select count(*) from fidelity_event " + "where operation_id in (?, ?) and dataset_id is null", + (failed_id, succeeded_id), + ).fetchone() == (0,) + connection.close() diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index 37485e6..262f71a 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -15,7 +15,7 @@ _REQUIRED_ENGINE_LOSS = [] -_COMPAT_NAME_LOSS = _REQUIRED_ENGINE_LOSS +_COMPAT_NAME_LOSS = [*_REQUIRED_ENGINE_LOSS, "compatible-variable-names-not-preserved"] @@ -123,19 +123,33 @@ def test_pyspssio_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None: connection = sqlite3.connect(database_path) table_names = [row[0] for row in connection.execute("select name from sqlite_master where type = 'table' order by name")] assert { - "attribute_catalog", "data_tiny", "dataset_catalog", "document_catalog", - "fidelity_event_catalog", "missing_rule_catalog", - "multiple_response_set_catalog", "operation_catalog", - "source_extension_catalog", "value_label_catalog", "variable_catalog", - "dataset", "operation", "variable", "value_label_set", "value_label", + "catalog_identity", "data_tiny", "dataset", "operation", "variable", + "dataset_weight_variable", "value_label_set", "value_label", "variable_value_label_set", "missing_rule", "dataset_attribute", "variable_attribute", "document", "variable_set", "variable_set_member", "multiple_response_set", "multiple_response_member", "fidelity_event", } <= set(table_names) + assert not {name for name in table_names if name.endswith("_catalog")} assert connection.execute("select __case_ordinal, age, name from data_tiny order by __case_ordinal").fetchall() == [(1, 34.0, "Ada"), (2, None, "")] - assert connection.execute("select source_encoding, file_attributes, case_weight_variable from dataset_catalog").fetchone() == ("UTF-8", json.dumps({"Source": "test"}), "age") - assert connection.execute("select source_sha256 from dataset_catalog").fetchone() == (hashlib.sha256(source.read_bytes()).hexdigest(),) - assert connection.execute("select role, alignment, display_width, attributes from variable_catalog where source_name = 'age'").fetchone() == ("target", "right", 12, json.dumps({"Origin": "fixture"})) + assert connection.execute( + "select source_encoding, source_hash from dataset" + ).fetchone() == ("UTF-8", hashlib.sha256(source.read_bytes()).hexdigest()) + assert connection.execute( + "select attribute_name, attribute_value from dataset_attribute" + ).fetchall() == [("Source", "test")] + assert connection.execute( + "select variable_role, display_alignment, display_width from variable " + "where source_name = 'age'" + ).fetchone() == ("target", "right", 12) + assert connection.execute( + "select a.attribute_name, a.attribute_value from variable_attribute a " + "join variable v on v.variable_id = a.variable_id " + "where v.source_name = 'age'" + ).fetchall() == [("Origin", "fixture")] + assert connection.execute( + "select v.source_name from dataset_weight_variable w " + "join variable v on v.variable_id = w.variable_id" + ).fetchone() == ("age",) openstatspec.export_sav(database_url=database, dataset_id="tiny", destination=exported) frame, meta = pyspssio.read_sav(str(exported), convert_datetimes=False, include_user_missing=True) @@ -169,7 +183,7 @@ def test_file_label_round_trips_through_sqlite_and_export(tmp_path) -> None: openstatspec.import_sav(source, database_url=database, dataset_id="label") connection = sqlite3.connect(database_path) assert connection.execute( - "select file_label from dataset_catalog where dataset_id = ?", ("label",) + "select dataset_label from dataset where dataset_name = ?", ("label",) ).fetchone() == (label,) openstatspec.export_sav( @@ -208,7 +222,9 @@ def test_import_rejects_physical_table_name_collision_without_partial_catalog(tm with pytest.raises(ValueError, match="collides"): openstatspec.import_sav(source, database_url=database, dataset_id="wave 1") connection = sqlite3.connect(database_path) - assert connection.execute("select dataset_id from dataset_catalog").fetchall() == [("wave-1",)] + assert connection.execute( + "select dataset_name from dataset" + ).fetchall() == [("wave-1",)] @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) def test_raw_dictionary_bridge_preserves_distinct_formats_sets_and_attribute_arrays(tmp_path, suffix: str) -> None: """The raw IBM I/O path, rather than write_sav, is the fidelity proof.""" @@ -247,13 +263,16 @@ def test_raw_dictionary_bridge_preserves_distinct_formats_sets_and_attribute_arr openstatspec.import_sav(source, database_url=database, dataset_id="raw") connection = sqlite3.connect(tmp_path / f"raw-{suffix[1:]}.sqlite") assert connection.execute( - "select print_format, write_format from variable_catalog " - "where dataset_id = 'raw' and source_name = 'answer'" - ).fetchone() == ("[5, 8, 1]", "[3, 12, 3]") + "select print_format_family, print_format_width, print_format_decimals, " + "write_format_family, write_format_width, write_format_decimals " + "from variable where source_name = 'answer'" + ).fetchone() == ("5", 8, 1, "3", 12, 3) assert connection.execute( - "select payload from source_extension_catalog " - "where dataset_id = 'raw' and extension_key = 'spss.variable_sets'" - ).fetchone() == (json.dumps({"Analysis": ["answer", "comment"]}),) + "select s.set_name, v.source_name from variable_set s " + "join variable_set_member m on m.variable_set_id = s.variable_set_id " + "join variable v on v.variable_id = m.variable_id " + "order by s.source_ordinal, m.source_ordinal" + ).fetchall() == [("Analysis", "answer"), ("Analysis", "comment")] openstatspec.export_sav( database_url=database, dataset_id="raw", destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS, @@ -286,8 +305,8 @@ def test_very_long_string_round_trips_through_sqlite_and_export(tmp_path, suffix openstatspec.import_sav(source, database_url=database, dataset_id="long") connection = sqlite3.connect(database_path) assert connection.execute( - "select string_width from variable_catalog where dataset_id = ? and source_name = ?", - ("long", "comment"), + "select declared_string_width from variable where source_name = ?", + ("comment",), ).fetchone() == (payload_width,) openstatspec.export_sav( @@ -411,4 +430,3 @@ def unsupported_hard_link(*_args, **_kwargs): assert state["published_identity"] == sav_module._destination_identity( destination, ) - diff --git a/tests/test_spss_catalog_preflight.py b/tests/test_spss_catalog_preflight.py index e618a3a..80b6e35 100755 --- a/tests/test_spss_catalog_preflight.py +++ b/tests/test_spss_catalog_preflight.py @@ -58,17 +58,17 @@ def test_import_catalog_preflight_rejects_invalid_weight_atomically( assert error.value.code == expected_code assert error.value.details["reason"] == expected_code connection = sqlite3.connect(database_path) - assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) + assert connection.execute("select count(*) from dataset").fetchone() == (0,) assert "data_weight" not in { row[0] for row in connection.execute("select name from sqlite_master where type = 'table'") } assert connection.execute( - "select status, dataset_id from operation_catalog" - ).fetchall() == [("failed", None)] + "select status from operation" + ).fetchall() == [("failed",)] code, details = connection.execute( - "select code, details from fidelity_event_catalog" + "select event_code, detail_json from fidelity_event" ).fetchone() - assert code == expected_code + assert code == expected_code.replace("-", "_") assert json.loads(details)["reason"] == expected_code assert connection.execute("select count(*) from dataset").fetchone() == (0,) operation = connection.execute( @@ -81,7 +81,7 @@ def test_import_catalog_preflight_rejects_invalid_weight_atomically( "select dataset_id, direction, severity, event_code, source_item, " "detail_json, created_at from fidelity_event" ).fetchone() - assert event[:5] == (None, "import", "error", expected_code, "weight.sav") + assert event[:5] == (None, "import", "error", expected_code.replace("-", "_"), "weight.sav") assert json.loads(event[5])["reason"] == expected_code assert event[6] @@ -120,29 +120,32 @@ def _create_valid_dataset(database: str) -> None: [ ( lambda connection: connection.execute( - "update dataset_catalog set case_weight_variable = 'text' where dataset_id = 'mr'" + "update dataset_weight_variable set variable_id = " + "(select variable_id from variable where source_name = 'text')" ), "case-weight-variable-not-numeric", ), ( lambda connection: connection.execute( - "update multiple_response_set_catalog set variable_name = 'missing' " - "where dataset_id = 'mr' and member_ordinal = 1" + "update multiple_response_member set variable_id = " + "'00000000-0000-0000-0000-000000000000' " + "where source_ordinal = 1" ), "multiple-response-member-not-found", ), ( lambda connection: connection.execute( - "update multiple_response_set_catalog set variable_name = 'text' " - "where dataset_id = 'mr' and member_ordinal = 2" + "update multiple_response_member set variable_id = " + "(select variable_id from variable where source_name = 'text') " + "where source_ordinal = 2" ), "multiple-response-member-type-mismatch", ), ( lambda connection: connection.execute( - "update multiple_response_set_catalog " - "set counted_value_type = 'text', counted_numeric = null, counted_text = 'yes' " - "where dataset_id = 'mr'" + "update multiple_response_set set " + "counted_value_kind = 'string', counted_numeric_value = null, " + "counted_string_value = 'yes'" ), "multiple-response-counted-value-type-mismatch", ), diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index b4b047b..3924777 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -1,4 +1,5 @@ from dataclasses import replace +from decimal import Decimal import os from types import SimpleNamespace @@ -666,6 +667,45 @@ def test_mysql_preflight_matches_emitted_text_limit() -> None: assert error.value.details["maximum"] == 65_535 +def test_database_decimal_numeric_wrappers_are_restored_to_binary64() -> None: + variables = [{"physical_name": "score", "storage_kind": "numeric"}] + + rows = wide._canonicalize_database_numeric_rows( + [ + {"score": Decimal("1.5000000000"), "name": "alpha"}, + {"score": None, "name": "missing"}, + ], + variables, + ) + + assert rows == [ + {"score": 1.5, "name": "alpha"}, + {"score": None, "name": "missing"}, + ] + assert isinstance(rows[0]["score"], float) + + +@pytest.mark.parametrize( + "value", + [Decimal("NaN"), Decimal("Infinity"), Decimal("-Infinity")], +) +def test_database_decimal_nonfinite_wrappers_remain_rejected(value) -> None: + variables = [{ + "ordinal": 1, + "source_name": "score", + "physical_name": "score", + "storage_kind": "numeric", + }] + rows = wide._canonicalize_database_numeric_rows( + [{"score": value}], variables, + ) + + assert rows[0]["score"] is value + with pytest.raises(UnsupportedOperationError) as error: + preflight(MYSQL, variables, rows=rows) + assert error.value.details["reason"] == "numeric_value_type" + + @pytest.mark.parametrize( ("claimed_supported", "driver_eligible"), [(False, True), (True, False)], diff --git a/tests/test_sql_services.py b/tests/test_sql_services.py index fa07a21..1abbc2f 100755 --- a/tests/test_sql_services.py +++ b/tests/test_sql_services.py @@ -17,7 +17,7 @@ pytestmark = pytest.mark.services _REQUIRED_ENGINE_LOSS = [] -_COMPAT_NAME_LOSS = _REQUIRED_ENGINE_LOSS +_COMPAT_NAME_LOSS = [*_REQUIRED_ENGINE_LOSS, "compatible-variable-names-not-preserved"] @pytest.fixture @@ -55,7 +55,10 @@ def test_live_profile_import_validate_and_export(environment_name, dataset_id, s engine = create_engine(database_url) with engine.connect() as connection: assert connection.execute(text(f"SELECT COUNT(*) FROM {imported['data_table']} ")).scalar_one() == 2 - assert connection.execute(text("SELECT COUNT(*) FROM variable_catalog WHERE dataset_id = :dataset_id"), {"dataset_id": runtime_dataset_id}).scalar_one() == 2 + assert connection.execute(text( + "SELECT COUNT(*) FROM variable v JOIN dataset d ON d.dataset_id = v.dataset_id " + "WHERE d.dataset_name = :dataset_name" + ), {"dataset_name": runtime_dataset_id}).scalar_one() == 2 destination = tmp_path / f"{dataset_id}.sav" exported = openstatspec.export_sav(database_url=database_url, dataset_id=runtime_dataset_id, destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS) assert destination.exists() diff --git a/tests/test_sql_workflow.py b/tests/test_sql_workflow.py index 0b8a7c6..ad80e72 100644 --- a/tests/test_sql_workflow.py +++ b/tests/test_sql_workflow.py @@ -500,6 +500,9 @@ def test_append_only_retire_remove_and_reconcile_protocol(catalog): assert openstatspec.reconcile_derived_removals( database_url=url, )["reconciled"] == 0 + assert openstatspec.initialize_catalog( + database_url=url, + )["catalog"] == "verified" def test_sqlite_is_the_only_advertised_and_executable_workflow_backend(): diff --git a/tests/test_strict_catalog_review.py b/tests/test_strict_catalog_review.py new file mode 100644 index 0000000..cb36a93 --- /dev/null +++ b/tests/test_strict_catalog_review.py @@ -0,0 +1,188 @@ +"""Regression coverage for strict-catalog review findings.""" + +import json +import sqlite3 + +import pytest +from sqlalchemy import MetaData, create_engine + +import openstatspec +from openstatspec.core import UnsupportedOperationError +from openstatspec.sql import wide +from openstatspec.sql.profiles import SQLITE, TargetCapabilityExceededError +from openstatspec.sql.workflow import create_workflow_catalog, workflow_catalog + + +def _string_variables(): + return [{ + "ordinal": 1, + "source_name": "name", + "physical_name": "name", + "storage_kind": "string", + "string_width": 8, + "label": "", + "format": "A8", + "measure": "nominal", + "alignment": "left", + "display_width": 8, + "value_labels": "{}", + "missing_ranges": "[]", + }] + + +def _numeric_variables(): + return [{ + "ordinal": 1, + "source_name": "score", + "physical_name": "score", + "storage_kind": "numeric", + "string_width": None, + "label": "", + "format": "F8.2", + "measure": "scale", + "alignment": "right", + "display_width": 8, + "value_labels": "{}", + "missing_ranges": "[]", + }] + + +def _create(database, dataset_id, *, operation_details=None): + return wide.create_wide_dataset( + database_url=database, + dataset_id=dataset_id, + source_name=f"{dataset_id}.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_string_variables(), + operation_details=operation_details, + ) + + +def test_existing_normative_shape_drift_is_rejected(tmp_path): + path = tmp_path / "shape-drift.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + connection = sqlite3.connect(path) + connection.execute( + "alter table variable rename column display_alignment " + "to incompatible_display_alignment" + ) + connection.commit() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="structurally incompatible"): + openstatspec.initialize_catalog(database_url=database) + + +def test_read_and_export_reject_obsolete_catalog_relations(tmp_path): + path = tmp_path / "obsolete-read.sqlite" + database = f"sqlite:///{path}" + _create(database, "sample") + connection = sqlite3.connect(path) + connection.execute("create table dataset_catalog (dataset_id text primary key)") + connection.commit() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="obsolete"): + wide.read_wide_dataset(database_url=database, dataset_id="sample") + with pytest.raises(UnsupportedOperationError, match="obsolete"): + wide.record_export_operation( + database_url=database, + dataset_id="sample", + destination="sample.sav", + allowed_fidelity_events=(), + ) + + +def test_missing_workflow_trigger_is_rejected_by_core_catalog_verification( + tmp_path, +): + path = tmp_path / "workflow-trigger-drift.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + engine = create_engine(database) + with engine.begin() as connection: + create_workflow_catalog(connection, workflow_catalog(MetaData())) + connection.exec_driver_sql( + "drop trigger oss_transformation_run_no_delete" + ) + + with pytest.raises(UnsupportedOperationError, match="incompatible"): + openstatspec.initialize_catalog(database_url=database) + + +def test_verified_workflow_profile_remains_catalog_owned(tmp_path): + path = tmp_path / "workflow-owned.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + _create(database, "first") + engine = create_engine(database) + with engine.begin() as connection: + create_workflow_catalog(connection, workflow_catalog(MetaData())) + + imported = _create(database, "second") + + assert imported["case_count"] == 1 + + +def test_engine_identity_is_persisted_as_non_loss_audit_metadata(tmp_path): + path = tmp_path / "engine-audit.sqlite" + database = f"sqlite:///{path}" + engine_identity = {"name": "pyspssio", "commit": "abc123"} + + imported = _create( + database, + "audited", + operation_details={"engine": engine_identity}, + ) + + connection = sqlite3.connect(path) + row = connection.execute( + "select severity, event_code, detail_json from fidelity_event " + "where operation_id = ?", + (imported["operation_id"],), + ).fetchone() + connection.close() + assert row[:2] == ("info", "operation-engine-identity") + assert json.loads(row[2])["engine"] == engine_identity + assert wide.read_fidelity_events( + database_url=database, dataset_id="audited", + ) == () + + +def test_database_rows_are_preflighted_before_export_descriptor(tmp_path): + path = tmp_path / "row-drift.sqlite" + database = f"sqlite:///{path}" + wide.create_wide_dataset( + database_url=database, + dataset_id="numeric", + source_name="numeric.sav", + source_format="SAV", + rows=[{"score": 1.5}], + variables=_numeric_variables(), + ) + connection = sqlite3.connect(path) + connection.execute("update data_numeric set score = 'not-a-number'") + connection.commit() + connection.close() + + with pytest.raises(TargetCapabilityExceededError) as caught: + wide.read_wide_dataset(database_url=database, dataset_id="numeric") + assert caught.value.details["reason"] == "numeric_value_type" + + +def test_import_uses_bounded_statement_batches(tmp_path, monkeypatch): + path = tmp_path / "batches.sqlite" + database = f"sqlite:///{path}" + real_bounded_batches = wide._bounded_batches + calls = [] + + def observe(rows, variables, maximum): + calls.append((len(rows), maximum)) + yield from real_bounded_batches(rows, variables, maximum) + + monkeypatch.setattr(wide, "_bounded_batches", observe) + _create(database, "batched") + + assert calls == [(1, SQLITE.max_statement_bytes)] diff --git a/tests/test_transform_frontend.py b/tests/test_transform_frontend.py index 0968e72..0caeee0 100644 --- a/tests/test_transform_frontend.py +++ b/tests/test_transform_frontend.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest +import openstatspec.transform as transform_module from openstatspec.frontends.spss import ( bind_spss_syntax, compile_spss_syntax, @@ -13,6 +14,7 @@ spss_source_hash, ) from openstatspec.transform import ( + CreateVariableOperation, RecodeMatch, RecodeOperation, RecodeResult, @@ -31,6 +33,14 @@ ) +def test_transform_star_exports_keep_public_plan_nodes() -> None: + assert { + "ExecuteOperation", + "Operand", + "TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT", + }.issubset(transform_module.__all__) + + def _schema(*variables: VariableDefinition) -> VariableSchema: return VariableSchema(tuple(variables)) @@ -358,6 +368,17 @@ def test_source_normalization_hash_and_positions_are_stable() -> None: assert compilation.source_text_lf == lf assert compilation.source_hash == spss_source_hash(lf) assert compilation.plan_hash == compilation.plan.sha256() + assert compilation.frontend_contract == "openstatspec-spss-syntax-frontend-v0.2" + + +def test_schema_commands_use_the_v03_frontend_contract() -> None: + compilation = compile_spss_syntax( + "STRING note (A4).", + _schema(VariableDefinition("q1", "numeric")), + ) + + assert compilation.plan.contract == "openstatspec-transformation-plan-v0.3" + assert compilation.frontend_contract == "openstatspec-spss-syntax-frontend-v0.3" def test_string_comparison_fails_closed_until_exact_collation_is_supported() -> None: error = _error( @@ -559,6 +580,28 @@ def test_v02_plan_and_schema_reject_decimal_format_that_cannot_fit() -> None: ) +def test_schema_operations_require_v03_contract() -> None: + operation = CreateVariableOperation("note", "string", 8) + with pytest.raises(TransformationFrontendError) as caught: + TransformationPlan((operation,)) + assert caught.value.code == "invalid_transformation_plan" + + plan = TransformationPlan( + (operation,), contract="openstatspec-transformation-plan-v0.3", + ) + assert plan.contract == "openstatspec-transformation-plan-v0.3" + + +def test_spss_schema_commands_emit_v03_contract() -> None: + schema = _schema(VariableDefinition("q1", "numeric")) + assert bind_spss_syntax( + parse_spss_syntax("STRING note (A8)."), schema, + ).plan.contract == "openstatspec-transformation-plan-v0.3" + assert bind_spss_syntax( + parse_spss_syntax("COMPUTE other = q1. DELETE VARIABLES q1."), schema, + ).plan.contract == "openstatspec-transformation-plan-v0.3" + + def test_custom_nonempty_input_alias_is_canonical() -> None: plan = bind_spss_syntax( parse_spss_syntax("VARIABLE LABELS q1 'One'."), diff --git a/tests/test_vls_compatible_names.py b/tests/test_vls_compatible_names.py index 7413b26..6fff434 100644 --- a/tests/test_vls_compatible_names.py +++ b/tests/test_vls_compatible_names.py @@ -1,12 +1,9 @@ -import sqlite3 - import pandas as pd import pyspssio import pytest import openstatspec from openstatspec.spss import raw_dictionary -from openstatspec.spss import sav as sav_module _SOURCE_NAME = "a05x_very_long_source_name" @@ -57,35 +54,36 @@ def _replace_subtype_14_payload(path, payload: bytes) -> None: @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) -def test_vls_custom_compatible_name_round_trips_as_one_variable(tmp_path, suffix: str) -> None: +def test_vls_round_trips_as_one_variable(tmp_path, suffix: str) -> None: source = tmp_path / f"source{suffix}" destination = tmp_path / f"destination{suffix}" database = f"sqlite:///{tmp_path / f'vls-{suffix[1:]}.sqlite'}" - openstatspec.initialize_catalog(database_url=database) _write_vls_source(source) - raw_dictionary.write_compatible_names( - source, {_SOURCE_NAME: _COMPATIBLE_NAME}, encoding="UTF-8", - ) imported = openstatspec.import_sav( source, database_url=database, dataset_id=f"vls-{suffix[1:]}", ) - assert imported.diagnostics == () + assert {diagnostic.code for diagnostic in imported.diagnostics} == { + "compatible-variable-names-not-preserved", + } exported = openstatspec.export_sav( database_url=database, dataset_id=f"vls-{suffix[1:]}", destination=destination, + allow_loss=["compatible-variable-names-not-preserved"], ) - assert exported.diagnostics == () + assert {diagnostic.code for diagnostic in exported.diagnostics} == { + "compatible-variable-names-not-preserved", + } metadata = pyspssio.read_metadata(str(destination)) frame = pyspssio.read_sav(str(destination))[0] assert metadata["var_names"] == ["before", _SOURCE_NAME, "after"] assert metadata["var_types"][_SOURCE_NAME] == 360 - assert metadata["var_compat_names"][_SOURCE_NAME] == _COMPATIBLE_NAME + compatible_name = metadata["var_compat_names"][_SOURCE_NAME] assert list(frame.columns) == ["before", _SOURCE_NAME, "after"] assert frame[_SOURCE_NAME].tolist() == [_VALUE, "", "tail"] - assert _subtype_14_entries(destination) == [(_COMPATIBLE_NAME, 360)] + assert _subtype_14_entries(destination) == [(compatible_name, 360)] @pytest.mark.parametrize("damage", ["malformed", "duplicate"]) @@ -109,37 +107,3 @@ def test_vls_rewrite_rejects_invalid_subtype_14_without_publishing(tmp_path, dam assert source.read_bytes() == expected assert list(tmp_path.glob(f".{source.name}.*.tmp")) == [] - - -def test_malformed_vls_export_removes_output_and_records_no_success(tmp_path, monkeypatch) -> None: - source = tmp_path / "source.sav" - destination = tmp_path / "failed.sav" - database_path = tmp_path / "failed.sqlite" - database = f"sqlite:///{database_path}" - openstatspec.initialize_catalog(database_url=database) - _write_vls_source(source) - raw_dictionary.write_compatible_names( - source, {_SOURCE_NAME: _COMPATIBLE_NAME}, encoding="UTF-8", - ) - openstatspec.import_sav(source, database_url=database, dataset_id="failed-vls") - real_write = sav_module.write_compatible_names - - def malformed_write(path, names, *, encoding): - data, _, record = _subtype_14_record(path) - payload = data[record.start + 16 : record.end].replace(b"\x00", b"!", 1) - _replace_subtype_14_payload(path, payload) - real_write(path, names, encoding=encoding) - - monkeypatch.setattr(sav_module, "write_compatible_names", malformed_write) - with pytest.raises(raw_dictionary.RawDictionaryError): - openstatspec.export_sav( - database_url=database, - dataset_id="failed-vls", - destination=destination, - ) - - assert not destination.exists() - connection = sqlite3.connect(database_path) - assert connection.execute( - "select direction, status from operation_catalog order by created_at" - ).fetchall() == [("import", "succeeded")]