From 695a2ab804e266be4706d08d0572f99836a6a8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 19:12:44 +0300 Subject: [PATCH 01/10] Add canonical in-place transformations --- src/Core/CapabilityDeclaration.php | 13 +- .../Canonical/CanonicalJson.php | 34 + .../Execution/DatasetBinding.php | 16 + src/Transformation/Execution/DoltEvidence.php | 48 ++ .../Execution/DoltEvidenceReader.php | 10 + src/Transformation/Execution/DoltGuard.php | 47 ++ .../Execution/ExecutionResult.php | 47 ++ .../InPlaceTransformationExecutor.php | 629 ++++++++++++++++++ .../Execution/PdoDoltEvidenceReader.php | 47 ++ .../Execution/VariableBinding.php | 19 + .../Model/Action/AssignValueAction.php | 29 + .../Model/Action/CopySourceAction.php | 22 + .../Model/Action/SetMissingAction.php | 22 + src/Transformation/Model/RecodeAction.php | 13 + src/Transformation/Model/RecodeOperation.php | 50 ++ src/Transformation/Model/RecodeRule.php | 32 + src/Transformation/Model/RecodeSelector.php | 13 + src/Transformation/Model/ScalarValue.php | 80 +++ .../Model/Selector/ElseSelector.php | 22 + .../Model/Selector/ExactValueSelector.php | 29 + .../Model/Selector/MissingValueSelector.php | 22 + .../Model/Selector/NumericRangeSelector.php | 43 ++ .../Model/SetValueLabelsOperation.php | 51 ++ .../Model/SetVariableLabelOperation.php | 45 ++ .../Model/TransformationOperation.php | 18 + .../Model/TransformationPlan.php | 60 ++ src/Transformation/Model/ValueLabel.php | 29 + .../Validation/InvalidTransformationPlan.php | 28 + .../Validation/PlanValidator.php | 298 +++++++++ .../Validation/ValidationResult.php | 29 + .../Validation/ValidationViolation.php | 20 + tests/Core/CapabilityDeclarationTest.php | 16 +- .../Canonical/TransformationPlanTest.php | 182 +++++ .../Execution/DoltGuardTest.php | 77 +++ .../Execution/DoltHeadGuardTest.php | 34 + .../InPlaceTransformationExecutorTest.php | 284 ++++++++ 36 files changed, 2456 insertions(+), 2 deletions(-) create mode 100644 src/Transformation/Canonical/CanonicalJson.php create mode 100644 src/Transformation/Execution/DatasetBinding.php create mode 100644 src/Transformation/Execution/DoltEvidence.php create mode 100644 src/Transformation/Execution/DoltEvidenceReader.php create mode 100644 src/Transformation/Execution/DoltGuard.php create mode 100644 src/Transformation/Execution/ExecutionResult.php create mode 100644 src/Transformation/Execution/InPlaceTransformationExecutor.php create mode 100644 src/Transformation/Execution/PdoDoltEvidenceReader.php create mode 100644 src/Transformation/Execution/VariableBinding.php create mode 100644 src/Transformation/Model/Action/AssignValueAction.php create mode 100644 src/Transformation/Model/Action/CopySourceAction.php create mode 100644 src/Transformation/Model/Action/SetMissingAction.php create mode 100644 src/Transformation/Model/RecodeAction.php create mode 100644 src/Transformation/Model/RecodeOperation.php create mode 100644 src/Transformation/Model/RecodeRule.php create mode 100644 src/Transformation/Model/RecodeSelector.php create mode 100644 src/Transformation/Model/ScalarValue.php create mode 100644 src/Transformation/Model/Selector/ElseSelector.php create mode 100644 src/Transformation/Model/Selector/ExactValueSelector.php create mode 100644 src/Transformation/Model/Selector/MissingValueSelector.php create mode 100644 src/Transformation/Model/Selector/NumericRangeSelector.php create mode 100644 src/Transformation/Model/SetValueLabelsOperation.php create mode 100644 src/Transformation/Model/SetVariableLabelOperation.php create mode 100644 src/Transformation/Model/TransformationOperation.php create mode 100644 src/Transformation/Model/TransformationPlan.php create mode 100644 src/Transformation/Model/ValueLabel.php create mode 100644 src/Transformation/Validation/InvalidTransformationPlan.php create mode 100644 src/Transformation/Validation/PlanValidator.php create mode 100644 src/Transformation/Validation/ValidationResult.php create mode 100644 src/Transformation/Validation/ValidationViolation.php create mode 100644 tests/Transformation/Canonical/TransformationPlanTest.php create mode 100644 tests/Transformation/Execution/DoltGuardTest.php create mode 100644 tests/Transformation/Execution/DoltHeadGuardTest.php create mode 100644 tests/Transformation/Execution/InPlaceTransformationExecutorTest.php diff --git a/src/Core/CapabilityDeclaration.php b/src/Core/CapabilityDeclaration.php index 5cf186f..35055a5 100644 --- a/src/Core/CapabilityDeclaration.php +++ b/src/Core/CapabilityDeclaration.php @@ -195,7 +195,18 @@ private function profile(string $name, PdoSqlProfile $profile, bool $active): ar 'unit' => 'bytes', ], ] : null, - 'transformation_workflow' => $name === 'dolt' ? 'unsupported' : null, + 'transformation_workflow' => 'supported', + 'in_place_transformations' => [ + 'status' => 'supported', + 'existing_target' => 'supported', + 'new_numeric_target' => $profile->ddlAtomic() + ? 'supported_in_native_transaction' + : 'preexisting_target_required', + 'dolt_repository_guard' => $name === 'dolt' + ? 'clean_working_set_and_stable_branch_head' + : null, + 'persistent_rollback_artifacts' => false, + ], 'physical_table_mapping' => 'dataset.physical_table_schema + dataset.physical_table_name', 'identifier_policy' => 'deterministic ASCII mapping; source name remains authoritative', ]; diff --git a/src/Transformation/Canonical/CanonicalJson.php b/src/Transformation/Canonical/CanonicalJson.php new file mode 100644 index 0000000..a633bc3 --- /dev/null +++ b/src/Transformation/Canonical/CanonicalJson.php @@ -0,0 +1,34 @@ + $item) { + $value[$key] = self::sortObjects($item); + } + + return $value; + } +} diff --git a/src/Transformation/Execution/DatasetBinding.php b/src/Transformation/Execution/DatasetBinding.php new file mode 100644 index 0000000..0da07bb --- /dev/null +++ b/src/Transformation/Execution/DatasetBinding.php @@ -0,0 +1,16 @@ + $dirtyTables */ + public function __construct( + private string $branch, + private string $head, + private array $dirtyTables, + ) {} + + public function branch(): string + { + return $this->branch; + } + + public function head(): string + { + return $this->head; + } + + /** @return list */ + public function dirtyTables(): array + { + return $this->dirtyTables; + } + + public function isClean(): bool + { + return $this->dirtyTables === []; + } + + /** @return array{branch: string, head: string, clean: bool, dirty_tables: list} */ + public function toArray(): array + { + return [ + 'branch' => $this->branch, + 'head' => $this->head, + 'clean' => $this->isClean(), + 'dirty_tables' => $this->dirtyTables, + ]; + } +} diff --git a/src/Transformation/Execution/DoltEvidenceReader.php b/src/Transformation/Execution/DoltEvidenceReader.php new file mode 100644 index 0000000..5e2adce --- /dev/null +++ b/src/Transformation/Execution/DoltEvidenceReader.php @@ -0,0 +1,10 @@ +reader->read(); + if (!$evidence->isClean()) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'Dolt transformations require a clean working set before execution; dirty tables: ' + . implode(', ', $evidence->dirtyTables()) . '.', + ); + } + + return $evidence; + } + + public function afterExecution(DoltEvidence $before): DoltEvidence + { + $after = $this->reader->read(); + if ($after->branch() !== $before->branch()) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'The active Dolt branch changed during transformation execution.', + ); + } + if ($after->head() !== $before->head()) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'Dolt HEAD changed during transformation execution; the executor never commits.', + ); + } + + return $after; + } +} diff --git a/src/Transformation/Execution/ExecutionResult.php b/src/Transformation/Execution/ExecutionResult.php new file mode 100644 index 0000000..c3b5790 --- /dev/null +++ b/src/Transformation/Execution/ExecutionResult.php @@ -0,0 +1,47 @@ +datasetId; + } + + public function planHash(): string + { + return $this->planHash; + } + + public function operationCount(): int + { + return $this->operationCount; + } + + public function auditOperationId(): ?string + { + return $this->auditOperationId; + } + + public function doltBefore(): ?DoltEvidence + { + return $this->doltBefore; + } + + public function doltAfter(): ?DoltEvidence + { + return $this->doltAfter; + } +} diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php new file mode 100644 index 0000000..9381f43 --- /dev/null +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -0,0 +1,629 @@ +validator = $validator ?? new PlanValidator(); + } + + public function execute(TransformationPlan $plan): ExecutionResult + { + $this->validator->assertValid($plan); + $this->connection->assertClaimedSupported(); + CatalogOwnership::assertReadyForUse($this->connection->pdo); + + $dataset = $this->resolveDataset($plan->datasetId()); + $variables = $this->resolveVariables($dataset); + $plannedVariables = $this->preflight($plan, $variables); + + $guard = $this->doltGuard(); + $doltBefore = $guard?->beforeExecution(); + $journal = $this->existingJournal(); + $operationId = $journal?->start( + 'transform', + $dataset->datasetName, + null, + [], + $this->auditDetails($plan, $dataset, $doltBefore), + ); + $doltAfter = null; + + try { + if (!$this->connection->pdo->beginTransaction()) { + throw new PDOException('The SQL driver did not start the transformation transaction.'); + } + $created = []; + foreach ($plan->operations() as $operation) { + $this->applyOperation($operation, $dataset, $plannedVariables, $created); + } + $doltAfter = $doltBefore === null + ? null + : $guard->afterExecution($doltBefore); + if ($journal !== null && $operationId !== null) { + $journal->succeed($operationId, $dataset->datasetName, []); + } + if (!$this->connection->pdo->commit()) { + throw new PDOException('The SQL driver did not commit the transformation transaction.'); + } + } catch (Throwable $exception) { + if ($this->connection->pdo->inTransaction()) { + $this->connection->pdo->rollBack(); + } + if ($journal !== null && $operationId !== null) { + try { + $journal->fail($operationId, $dataset->datasetName, $exception); + } catch (Throwable) { + // Preserve the transformation failure; audit is best-effort after rollback. + } + } + throw $exception; + } + + return new ExecutionResult( + $dataset->datasetId, + $plan->hash(), + count($plan->operations()), + $operationId, + $doltBefore, + $doltAfter, + ); + } + + private function resolveDataset(string $datasetId): DatasetBinding + { + $statement = $this->statement( + 'SELECT dataset_id, dataset_name, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + ); + $statement->execute([$datasetId]); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) !== 1) { + throw $this->invalidCatalog('The transformation dataset_id resolves to exactly one normative dataset row.'); + } + $row = $rows[0]; + $table = $row['physical_table_name'] ?? null; + $schema = $row['physical_table_schema'] ?? null; + $name = $row['dataset_name'] ?? null; + if (($row['dataset_id'] ?? null) !== $datasetId + || !is_string($table) + || $table === '' + || ($schema !== null && (!is_string($schema) || $schema === '')) + || ($name !== null && !is_string($name)) + ) { + throw $this->invalidCatalog('The normative dataset row has malformed physical table metadata.'); + } + + return new DatasetBinding($datasetId, $name, $schema, $table); + } + + /** @return array */ + private function resolveVariables(DatasetBinding $dataset): array + { + $statement = $this->statement( + 'SELECT variable_id, source_name, physical_name, storage_kind, source_ordinal, declared_string_width ' + . 'FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', + ); + $statement->execute([$dataset->datasetId]); + $variables = []; + $physicalNames = []; + foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { + $id = $row['variable_id'] ?? null; + $source = $row['source_name'] ?? null; + $physical = $row['physical_name'] ?? null; + $kind = $row['storage_kind'] ?? null; + $ordinal = filter_var($row['source_ordinal'] ?? null, FILTER_VALIDATE_INT); + $declaredStringWidth = $row['declared_string_width'] === null + ? null + : filter_var($row['declared_string_width'], FILTER_VALIDATE_INT); + if (!is_string($id) || $id === '' + || !is_string($source) || $source === '' + || !is_string($physical) || $physical === '' + || !in_array($kind, ['numeric', 'string'], true) + || !is_int($ordinal) || $ordinal < 1 + || ($kind === 'string' && (!is_int($declaredStringWidth) || $declaredStringWidth < 1)) + || isset($variables[$source]) || isset($physicalNames[$physical]) + ) { + throw $this->invalidCatalog('The normative variable mapping is malformed or ambiguous.'); + } + $variables[$source] = new VariableBinding( + $id, + $source, + $physical, + $kind, + $ordinal, + is_int($declaredStringWidth) ? $declaredStringWidth : null, + ); + $physicalNames[$physical] = true; + } + if ($variables === []) { + throw $this->invalidCatalog('The registered dataset has no normative variable mappings.'); + } + + $columns = implode(', ', array_map( + fn(VariableBinding $variable): string => $this->quote($variable->physicalName), + array_values($variables), + )); + try { + $this->connection->pdo->query( + 'SELECT ' . $columns . ' FROM ' . $this->qualifiedTable($dataset) . ' WHERE 1 = 0', + ); + } catch (PDOException $exception) { + throw $this->invalidCatalog('The catalogued physical wide table or variable mapping is not readable: ' . $exception->getMessage()); + } + + return $variables; + } + + /** + * @param array $variables + * @return array + */ + private function preflight(TransformationPlan $plan, array $variables): array + { + $used = []; + $nextOrdinal = 1; + foreach ($variables as $variable) { + $used[$variable->physicalName] = true; + $nextOrdinal = max($nextOrdinal, $variable->sourceOrdinal + 1); + } + + foreach ($plan->operations() as $operation) { + if ($operation instanceof RecodeOperation) { + $source = $variables[$operation->sourceVariable()] ?? null; + if ($source === null) { + throw $this->invalidCatalog(sprintf( + 'Recode source variable "%s" is not registered for dataset_id %s.', + $operation->sourceVariable(), + $plan->datasetId(), + )); + } + $target = $variables[$operation->targetVariable()] ?? null; + if ($target === null) { + $this->assertCanCreateTarget($source); + $physical = $this->connection->profile->physicalIdentifier($operation->targetVariable(), $used); + $target = new VariableBinding( + NormativeCatalog::uuid(), + $operation->targetVariable(), + $physical, + $source->storageKind, + $nextOrdinal++, + null, + false, + ); + $variables[$operation->targetVariable()] = $target; + $used[$physical] = true; + } + $this->assertRecodeKinds($operation, $source, $target); + continue; + } + + $target = $variables[$operation->targetVariable()] ?? null; + if ($target === null) { + throw $this->invalidCatalog(sprintf( + 'Metadata target variable "%s" is not registered for dataset_id %s.', + $operation->targetVariable(), + $plan->datasetId(), + )); + } + if ($operation instanceof SetValueLabelsOperation) { + foreach ($operation->labels() as $label) { + $this->assertScalarKind($label->value(), $target, 'value label'); + } + } + } + + return $variables; + } + + private function assertCanCreateTarget(VariableBinding $source): void + { + if ($source->storageKind === 'string') { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + 'A new string target requires an explicit declared_string_width; register the target variable first.', + ); + } + if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) + || !$this->connection->profile->ddlAtomic() + ) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + sprintf( + '%s cannot atomically add a new INTO target to an existing wide table; register the target variable first.', + $this->connection->profileName, + ), + ); + } + } + + private function assertRecodeKinds( + RecodeOperation $operation, + VariableBinding $source, + VariableBinding $target, + ): void { + foreach ($operation->rules() as $rule) { + $selector = $rule->selector(); + if ($selector instanceof ExactValueSelector) { + $this->assertScalarKind($selector->value(), $source, 'exact recode selector'); + } elseif ($selector instanceof NumericRangeSelector && $source->storageKind !== 'numeric') { + throw $this->invalidCatalog('A numeric range selector cannot read a string variable.'); + } + + $action = $rule->action(); + if ($action instanceof AssignValueAction) { + $this->assertScalarKind($action->value(), $target, 'recode assignment'); + } elseif ($action instanceof CopySourceAction && $source->storageKind !== $target->storageKind) { + throw $this->invalidCatalog('Copy-source recoding requires source and target variables with the same storage kind.'); + } elseif ($action instanceof CopySourceAction + && $source->storageKind === 'string' + && $this->stringWidth($source) > $this->stringWidth($target) + ) { + throw $this->invalidCatalog(sprintf( + 'Copy-source recoding cannot copy string width %d into variable "%s" width %d.', + $this->stringWidth($source), + $target->sourceName, + $this->stringWidth($target), + )); + } + } + } + + private function assertScalarKind(ScalarValue $value, VariableBinding $variable, string $context): void + { + $kind = $value->type() === 'number' ? 'numeric' : 'string'; + if ($kind !== $variable->storageKind) { + throw $this->invalidCatalog(sprintf( + 'The %s type does not match variable "%s" storage kind %s.', + $context, + $variable->sourceName, + $variable->storageKind, + )); + } + if ($kind === 'string' && strlen($value->stringValue()) > $this->stringWidth($variable)) { + throw $this->invalidCatalog(sprintf( + 'The %s exceeds variable "%s" declared string width %d bytes.', + $context, + $variable->sourceName, + $this->stringWidth($variable), + )); + } + } + + private function stringWidth(VariableBinding $variable): int + { + if ($variable->storageKind !== 'string' + || $variable->declaredStringWidth === null + || $variable->declaredStringWidth < 1 + ) { + throw $this->invalidCatalog(sprintf( + 'String variable "%s" requires a positive declared_string_width.', + $variable->sourceName, + )); + } + + return $variable->declaredStringWidth; + } + + /** + * @param array $variables + * @param array $created + */ + private function applyOperation( + TransformationOperation $operation, + DatasetBinding $dataset, + array $variables, + array &$created, + ): void { + $target = $variables[$operation->targetVariable()]; + if ($operation instanceof RecodeOperation) { + $this->ensureTargetExists($dataset, $target, $created); + $this->applyRecode($operation, $dataset, $variables[$operation->sourceVariable()], $target); + } elseif ($operation instanceof SetVariableLabelOperation) { + $this->applyVariableLabel($operation, $dataset, $target); + } elseif ($operation instanceof SetValueLabelsOperation) { + $this->applyValueLabels($operation, $dataset, $target); + } + } + + /** @param array $created */ + private function ensureTargetExists(DatasetBinding $dataset, VariableBinding $target, array &$created): void + { + if ($target->persisted || isset($created[$target->sourceName])) { + return; + } + $type = $target->storageKind === 'numeric' + ? $this->connection->profile->numericType() + : $this->connection->profile->textType(); + $this->connection->pdo->exec(sprintf( + 'ALTER TABLE %s ADD COLUMN %s %s NULL', + $this->qualifiedTable($dataset), + $this->quote($target->physicalName), + $type, + )); + $this->statement( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + )->execute([ + $target->variableId, + $dataset->datasetId, + $target->sourceOrdinal, + $target->sourceName, + $target->physicalName, + $target->storageKind, + ]); + $created[$target->sourceName] = true; + } + + private function applyRecode( + RecodeOperation $operation, + DatasetBinding $dataset, + VariableBinding $source, + VariableBinding $target, + ): void { + $sourceSql = $this->quote($source->physicalName); + $when = []; + $parameters = []; + $else = null; + foreach ($operation->rules() as $rule) { + $selector = $rule->selector(); + if ($selector instanceof ElseSelector) { + $else = $this->actionSql($rule->action(), $sourceSql, $parameters); + continue; + } + $condition = match (true) { + $selector instanceof ExactValueSelector => $this->exactCondition($sourceSql, $selector, $parameters), + $selector instanceof NumericRangeSelector => $this->rangeCondition($sourceSql, $selector, $parameters), + $selector instanceof MissingValueSelector => $sourceSql . ' IS NULL', + default => throw $this->invalidCatalog('The recode selector is not executable.'), + }; + $actionSql = $this->actionSql($rule->action(), $sourceSql, $parameters); + $when[] = 'WHEN ' . $condition . ' THEN ' . $actionSql; + } + if ($else === null) { + throw $this->invalidCatalog('A recode requires one explicit final else action.'); + } + $sql = sprintf( + 'UPDATE %s SET %s = CASE %s ELSE %s END', + $this->qualifiedTable($dataset), + $this->quote($target->physicalName), + implode(' ', $when), + $else, + ); + $this->statement($sql)->execute($parameters); + } + + /** @param list $parameters */ + private function exactCondition(string $sourceSql, ExactValueSelector $selector, array &$parameters): string + { + $parameters[] = $this->boundScalar($selector->value()); + + return $sourceSql . ' = ?'; + } + + /** @param list $parameters */ + private function rangeCondition(string $sourceSql, NumericRangeSelector $selector, array &$parameters): string + { + $parts = []; + if ($selector->lower() !== null) { + $parts[] = $sourceSql . ' >= ?'; + $parameters[] = Binary64::encode($selector->lower()); + } + if ($selector->upper() !== null) { + $parts[] = $sourceSql . ' <= ?'; + $parameters[] = Binary64::encode($selector->upper()); + } + + return implode(' AND ', $parts); + } + + /** @param list $parameters */ + private function actionSql(RecodeAction $action, string $sourceSql, array &$parameters): string + { + if ($action instanceof AssignValueAction) { + $parameters[] = $this->boundScalar($action->value()); + + return '?'; + } + if ($action instanceof SetMissingAction) { + return 'NULL'; + } + if ($action instanceof CopySourceAction) { + return $sourceSql; + } + + throw $this->invalidCatalog('The recode action is not executable.'); + } + + private function applyVariableLabel( + SetVariableLabelOperation $operation, + DatasetBinding $dataset, + VariableBinding $target, + ): void { + $this->statement('UPDATE variable SET variable_label = ? WHERE variable_id = ? AND dataset_id = ?')->execute([ + $operation->label(), + $target->variableId, + $dataset->datasetId, + ]); + } + + private function applyValueLabels( + SetValueLabelsOperation $operation, + DatasetBinding $dataset, + VariableBinding $target, + ): void { + $statement = $this->statement( + 'SELECT vvls.value_label_set_id, vls.dataset_id FROM variable_value_label_set vvls ' + . 'JOIN value_label_set vls ON vls.value_label_set_id = vvls.value_label_set_id ' + . 'WHERE vvls.variable_id = ?', + ); + $statement->execute([$target->variableId]); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { + throw $this->invalidCatalog('The variable value-label association is malformed or crosses dataset identity.'); + } + + $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) + ? $rows[0]['value_label_set_id'] + : null; + if ($setId !== null) { + $references = $this->statement( + 'SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?', + ); + $references->execute([$setId]); + $referenceCount = (int) $references->fetchColumn(); + if ($referenceCount > 1) { + $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); + $setId = null; + } else { + $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); + } + } + + if ($operation->labels() === []) { + if ($setId !== null) { + $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); + $this->statement('DELETE FROM value_label_set WHERE value_label_set_id = ?')->execute([$setId]); + } + + return; + } + + if ($setId === null) { + $setId = NormativeCatalog::uuid(); + $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)')->execute([ + $setId, + $dataset->datasetId, + null, + ]); + $this->statement( + 'INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)', + )->execute([$target->variableId, $setId]); + } + + $insert = $this->statement( + 'INSERT INTO value_label ' + . '(value_label_id, value_label_set_id, ordinal, code_kind, numeric_code, string_code, label) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + foreach ($operation->labels() as $index => $label) { + $value = $label->value(); + $insert->execute([ + NormativeCatalog::uuid(), + $setId, + $index + 1, + $value->type() === 'number' ? 'numeric' : 'string', + $value->type() === 'number' ? Binary64::encode($value->numberValue()) : null, + $value->type() === 'string' ? $value->stringValue() : null, + $label->label(), + ]); + } + } + + private function boundScalar(ScalarValue $value): string + { + return $value->type() === 'number' + ? Binary64::encode($value->numberValue()) + : $value->stringValue(); + } + + private function qualifiedTable(DatasetBinding $dataset): string + { + $table = $this->quote($dataset->table); + + return $dataset->schema === null ? $table : $this->quote($dataset->schema) . '.' . $table; + } + + private function quote(string $identifier): string + { + return $this->connection->profile->quoteIdentifier($identifier); + } + + private function statement(string $sql): PDOStatement + { + $statement = $this->connection->pdo->prepare($sql); + if ($statement === false) { + throw new PDOException('The transformation SQL statement could not be prepared.'); + } + + return $statement; + } + + private function doltGuard(): ?DoltGuard + { + if ($this->connection->profileName !== 'dolt') { + return null; + } + + return new DoltGuard($this->doltEvidenceReader ?? new PdoDoltEvidenceReader($this->connection->pdo)); + } + + private function existingJournal(): ?OperationJournal + { + try { + $this->connection->pdo->query('SELECT operation_id FROM operation_catalog WHERE 1 = 0'); + $this->connection->pdo->query('SELECT operation_id FROM fidelity_event_catalog WHERE 1 = 0'); + $this->connection->pdo->query('SELECT operation_id FROM operation WHERE 1 = 0'); + } catch (PDOException) { + return null; + } + + return new OperationJournal($this->connection->pdo); + } + + /** @return array */ + private function auditDetails( + TransformationPlan $plan, + DatasetBinding $dataset, + ?DoltEvidence $doltBefore, + ): array { + return [ + 'plan_hash' => $plan->hash(), + 'dataset_id' => $dataset->datasetId, + 'mode' => 'in_place', + 'dolt_branch_before' => $doltBefore?->branch(), + 'dolt_head_before' => $doltBefore?->head(), + ]; + } + + private function invalidCatalog(string $message): UnsupportedOperation + { + return new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $message); + } +} diff --git a/src/Transformation/Execution/PdoDoltEvidenceReader.php b/src/Transformation/Execution/PdoDoltEvidenceReader.php new file mode 100644 index 0000000..e6eba55 --- /dev/null +++ b/src/Transformation/Execution/PdoDoltEvidenceReader.php @@ -0,0 +1,47 @@ +pdo->query("SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash"); + $row = $identity === false ? false : $identity->fetch(PDO::FETCH_ASSOC); + $status = $this->pdo->query('SELECT table_name FROM dolt_status ORDER BY table_name'); + $tables = $status === false ? false : $status->fetchAll(PDO::FETCH_COLUMN); + } catch (PDOException $exception) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'The Dolt branch, HEAD, and working-set evidence could not be read: ' . $exception->getMessage(), + ); + } + + $branch = is_array($row) ? $row['branch_name'] ?? null : null; + $head = is_array($row) ? $row['head_hash'] ?? null : null; + if (!is_string($branch) || $branch === '' || !is_string($head) || $head === '' || !is_array($tables)) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'The Dolt branch, HEAD, or working-set evidence was malformed.', + ); + } + + $dirtyTables = array_values(array_unique(array_map( + static fn(mixed $table): string => (string) $table, + $tables, + ))); + + return new DoltEvidence($branch, $head, $dirtyTables); + } +} diff --git a/src/Transformation/Execution/VariableBinding.php b/src/Transformation/Execution/VariableBinding.php new file mode 100644 index 0000000..521283f --- /dev/null +++ b/src/Transformation/Execution/VariableBinding.php @@ -0,0 +1,19 @@ +value; + } + + /** @return array{type: string, value: array} */ + public function canonicalArray(): array + { + return ['type' => $this->type(), 'value' => $this->value->canonicalArray()]; + } +} diff --git a/src/Transformation/Model/Action/CopySourceAction.php b/src/Transformation/Model/Action/CopySourceAction.php new file mode 100644 index 0000000..64fd47b --- /dev/null +++ b/src/Transformation/Model/Action/CopySourceAction.php @@ -0,0 +1,22 @@ + $this->type()]; + } +} diff --git a/src/Transformation/Model/Action/SetMissingAction.php b/src/Transformation/Model/Action/SetMissingAction.php new file mode 100644 index 0000000..50444e4 --- /dev/null +++ b/src/Transformation/Model/Action/SetMissingAction.php @@ -0,0 +1,22 @@ + $this->type()]; + } +} diff --git a/src/Transformation/Model/RecodeAction.php b/src/Transformation/Model/RecodeAction.php new file mode 100644 index 0000000..b495cbd --- /dev/null +++ b/src/Transformation/Model/RecodeAction.php @@ -0,0 +1,13 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Model/RecodeOperation.php b/src/Transformation/Model/RecodeOperation.php new file mode 100644 index 0000000..27b2b72 --- /dev/null +++ b/src/Transformation/Model/RecodeOperation.php @@ -0,0 +1,50 @@ + $rules */ + public function __construct( + private string $sourceVariable, + private string $targetVariable, + private array $rules, + ) {} + + public function type(): string + { + return 'recode'; + } + + public function sourceVariable(): string + { + return $this->sourceVariable; + } + + public function targetVariable(): string + { + return $this->targetVariable; + } + + /** @return list */ + public function rules(): array + { + return $this->rules; + } + + /** @return array{type: string, source_variable: string, target_variable: string, rules: list>} */ + public function canonicalArray(): array + { + return [ + 'type' => $this->type(), + 'source_variable' => $this->sourceVariable, + 'target_variable' => $this->targetVariable, + 'rules' => array_map( + static fn(RecodeRule $rule): array => $rule->canonicalArray(), + $this->rules, + ), + ]; + } +} diff --git a/src/Transformation/Model/RecodeRule.php b/src/Transformation/Model/RecodeRule.php new file mode 100644 index 0000000..905b10b --- /dev/null +++ b/src/Transformation/Model/RecodeRule.php @@ -0,0 +1,32 @@ +selector; + } + + public function action(): RecodeAction + { + return $this->action; + } + + /** @return array{selector: array, action: array} */ + public function canonicalArray(): array + { + return [ + 'selector' => $this->selector->canonicalArray(), + 'action' => $this->action->canonicalArray(), + ]; + } +} diff --git a/src/Transformation/Model/RecodeSelector.php b/src/Transformation/Model/RecodeSelector.php new file mode 100644 index 0000000..ba45a19 --- /dev/null +++ b/src/Transformation/Model/RecodeSelector.php @@ -0,0 +1,13 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Model/ScalarValue.php b/src/Transformation/Model/ScalarValue.php new file mode 100644 index 0000000..a068e19 --- /dev/null +++ b/src/Transformation/Model/ScalarValue.php @@ -0,0 +1,80 @@ +type; + } + + public function value(): float|string + { + return $this->value; + } + + public function numberValue(): float + { + if (!is_float($this->value)) { + throw new InvalidArgumentException('A string scalar has no numeric value.'); + } + + return $this->value; + } + + public function stringValue(): string + { + if (!is_string($this->value)) { + throw new InvalidArgumentException('A numeric scalar has no string value.'); + } + + return $this->value; + } + + /** @return array{type: string, binary64?: string, value?: string} */ + public function canonicalArray(): array + { + if (is_string($this->value)) { + return ['type' => 'string', 'value' => $this->value]; + } + + return ['type' => 'number', 'binary64' => bin2hex(pack('E', $this->normalisedNumber()))]; + } + + public function identity(): string + { + return $this->type . ':' . ($this->canonicalArray()['binary64'] ?? $this->value); + } + + private function normalisedNumber(): float + { + $value = $this->value; + if (!is_float($value)) { + throw new InvalidArgumentException('A string scalar has no numeric representation.'); + } + + // IEEE-754 considers both zero representations equal for recoding. + return $value === 0.0 ? 0.0 : $value; + } +} diff --git a/src/Transformation/Model/Selector/ElseSelector.php b/src/Transformation/Model/Selector/ElseSelector.php new file mode 100644 index 0000000..9bfbd57 --- /dev/null +++ b/src/Transformation/Model/Selector/ElseSelector.php @@ -0,0 +1,22 @@ + $this->type()]; + } +} diff --git a/src/Transformation/Model/Selector/ExactValueSelector.php b/src/Transformation/Model/Selector/ExactValueSelector.php new file mode 100644 index 0000000..4444f98 --- /dev/null +++ b/src/Transformation/Model/Selector/ExactValueSelector.php @@ -0,0 +1,29 @@ +value; + } + + /** @return array{type: string, value: array} */ + public function canonicalArray(): array + { + return ['type' => $this->type(), 'value' => $this->value->canonicalArray()]; + } +} diff --git a/src/Transformation/Model/Selector/MissingValueSelector.php b/src/Transformation/Model/Selector/MissingValueSelector.php new file mode 100644 index 0000000..c52bb2a --- /dev/null +++ b/src/Transformation/Model/Selector/MissingValueSelector.php @@ -0,0 +1,22 @@ + $this->type()]; + } +} diff --git a/src/Transformation/Model/Selector/NumericRangeSelector.php b/src/Transformation/Model/Selector/NumericRangeSelector.php new file mode 100644 index 0000000..73af548 --- /dev/null +++ b/src/Transformation/Model/Selector/NumericRangeSelector.php @@ -0,0 +1,43 @@ +lower; + } + + public function upper(): ?float + { + return $this->upper; + } + + /** @return array{type: string, lower: ?array, upper: ?array, bounds: string} */ + public function canonicalArray(): array + { + return [ + 'type' => $this->type(), + 'lower' => $this->lower === null ? null : ScalarValue::number($this->lower)->canonicalArray(), + 'upper' => $this->upper === null ? null : ScalarValue::number($this->upper)->canonicalArray(), + 'bounds' => 'inclusive', + ]; + } +} diff --git a/src/Transformation/Model/SetValueLabelsOperation.php b/src/Transformation/Model/SetValueLabelsOperation.php new file mode 100644 index 0000000..ab6f4d9 --- /dev/null +++ b/src/Transformation/Model/SetValueLabelsOperation.php @@ -0,0 +1,51 @@ + $labels */ + public function __construct( + private string $targetVariable, + private array $labels, + ) {} + + public function type(): string + { + return 'set_value_labels'; + } + + public function sourceVariable(): string + { + return $this->targetVariable; + } + + public function targetVariable(): string + { + return $this->targetVariable; + } + + /** @return list */ + public function labels(): array + { + return $this->labels; + } + + /** @return array{type: string, source_variable: string, target_variable: string, replacement: string, labels: list>} */ + public function canonicalArray(): array + { + return [ + 'type' => $this->type(), + 'source_variable' => $this->targetVariable, + 'target_variable' => $this->targetVariable, + 'replacement' => 'complete', + 'labels' => array_map( + static fn(ValueLabel $label): array => $label->canonicalArray(), + $this->labels, + ), + ]; + } +} diff --git a/src/Transformation/Model/SetVariableLabelOperation.php b/src/Transformation/Model/SetVariableLabelOperation.php new file mode 100644 index 0000000..1293b5e --- /dev/null +++ b/src/Transformation/Model/SetVariableLabelOperation.php @@ -0,0 +1,45 @@ +targetVariable; + } + + public function targetVariable(): string + { + return $this->targetVariable; + } + + public function label(): ?string + { + return $this->label; + } + + /** @return array{type: string, source_variable: string, target_variable: string, label: ?string} */ + public function canonicalArray(): array + { + return [ + 'type' => $this->type(), + 'source_variable' => $this->targetVariable, + 'target_variable' => $this->targetVariable, + 'label' => $this->label, + ]; + } +} diff --git a/src/Transformation/Model/TransformationOperation.php b/src/Transformation/Model/TransformationOperation.php new file mode 100644 index 0000000..8d63194 --- /dev/null +++ b/src/Transformation/Model/TransformationOperation.php @@ -0,0 +1,18 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Model/TransformationPlan.php b/src/Transformation/Model/TransformationPlan.php new file mode 100644 index 0000000..b063098 --- /dev/null +++ b/src/Transformation/Model/TransformationPlan.php @@ -0,0 +1,60 @@ + $operations */ + public function __construct( + private string $datasetId, + private array $operations, + ) {} + + public function datasetId(): string + { + return $this->datasetId; + } + + /** @return list */ + public function operations(): array + { + return $this->operations; + } + + /** @return array{contract: string, mode: string, dataset_id: string, operations: list>} */ + public function canonicalArray(): array + { + return [ + 'contract' => self::CONTRACT, + 'mode' => 'in_place', + 'dataset_id' => $this->datasetId, + 'operations' => array_map( + static fn(TransformationOperation $operation): array => $operation->canonicalArray(), + $this->operations, + ), + ]; + } + + public function canonicalJson(): string + { + return CanonicalJson::encode($this->canonicalArray()); + } + + public function hash(): string + { + return hash('sha256', $this->canonicalJson()); + } +} diff --git a/src/Transformation/Model/ValueLabel.php b/src/Transformation/Model/ValueLabel.php new file mode 100644 index 0000000..9828281 --- /dev/null +++ b/src/Transformation/Model/ValueLabel.php @@ -0,0 +1,29 @@ +value; + } + + public function label(): string + { + return $this->label; + } + + /** @return array{value: array, label: string} */ + public function canonicalArray(): array + { + return ['value' => $this->value->canonicalArray(), 'label' => $this->label]; + } +} diff --git a/src/Transformation/Validation/InvalidTransformationPlan.php b/src/Transformation/Validation/InvalidTransformationPlan.php new file mode 100644 index 0000000..61d9834 --- /dev/null +++ b/src/Transformation/Validation/InvalidTransformationPlan.php @@ -0,0 +1,28 @@ + $violations */ + public function __construct(private readonly array $violations) + { + $first = $violations[0]; + parent::__construct(sprintf( + 'Invalid transformation plan (%s at %s): %s', + $first->code, + $first->path, + $first->message, + )); + } + + /** @return non-empty-list */ + public function violations(): array + { + return $this->violations; + } +} diff --git a/src/Transformation/Validation/PlanValidator.php b/src/Transformation/Validation/PlanValidator.php new file mode 100644 index 0000000..77cf268 --- /dev/null +++ b/src/Transformation/Validation/PlanValidator.php @@ -0,0 +1,298 @@ + */ + private array $violations = []; + + public function validate(TransformationPlan $plan): ValidationResult + { + $this->violations = []; + if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $plan->datasetId()) !== 1) { + $this->violation( + 'dataset_id.invalid_uuid', + '$.dataset_id', + 'The dataset identity must be a canonical lowercase RFC 9562 UUID string.', + ); + } + + if ($plan->operations() === []) { + $this->violation('operations.empty', '$.operations', 'A transformation plan must contain at least one operation.'); + } + + /** @var array $targets */ + $targets = []; + foreach ($plan->operations() as $index => $operation) { + $path = '$.operations[' . $index . ']'; + $this->validateOperation($operation, $path); + $targetKey = $operation::class . "\0" . $operation->targetVariable(); + if (isset($targets[$targetKey])) { + $this->violation( + 'operation.duplicate_target', + $path . '.target_variable', + sprintf('Operation type %s targets variable %s more than once.', $operation->type(), $operation->targetVariable()), + ); + } else { + $targets[$targetKey] = $index; + } + } + + return new ValidationResult($this->violations); + } + + public function assertValid(TransformationPlan $plan): void + { + $this->validate($plan)->throwIfInvalid(); + } + + private function validateOperation(TransformationOperation $operation, string $path): void + { + if (!in_array($operation::class, [ + RecodeOperation::class, + SetVariableLabelOperation::class, + SetValueLabelsOperation::class, + ], true)) { + $this->violation( + 'operation.unsupported_type', + $path . '.type', + 'Only canonical, source-language-neutral operation types are permitted.', + ); + + return; + } + + $this->validateVariableName($operation->sourceVariable(), $path . '.source_variable'); + $this->validateVariableName($operation->targetVariable(), $path . '.target_variable'); + + if ($operation instanceof RecodeOperation) { + $this->validateRecode($operation, $path); + } elseif ($operation instanceof SetVariableLabelOperation) { + $this->validateText($operation->label(), $path . '.label'); + } elseif ($operation instanceof SetValueLabelsOperation) { + $this->validateValueLabels($operation, $path); + } + } + + private function validateRecode(RecodeOperation $operation, string $path): void + { + if ($operation->rules() === []) { + $this->violation('recode.rules_empty', $path . '.rules', 'A recode operation must contain ordered rules.'); + + return; + } + + /** @var array $exactValues */ + $exactValues = []; + /** @var list $ranges */ + $ranges = []; + $missingIndex = null; + $elseIndexes = []; + foreach ($operation->rules() as $index => $rule) { + $rulePath = $path . '.rules[' . $index . ']'; + $selector = $rule->selector(); + $this->validateSelector($selector, $rulePath . '.selector'); + $this->validateAction($rule->action(), $rulePath . '.action'); + + if ($selector instanceof ExactValueSelector) { + $identity = $selector->value()->identity(); + if (isset($exactValues[$identity])) { + $this->violation('recode.ambiguous_mapping', $rulePath . '.selector', 'An exact source value is mapped more than once.'); + } else { + $exactValues[$identity] = $index; + } + } elseif ($selector instanceof NumericRangeSelector) { + $ranges[] = ['index' => $index, 'lower' => $selector->lower(), 'upper' => $selector->upper()]; + } elseif ($selector instanceof MissingValueSelector) { + if ($missingIndex !== null) { + $this->violation('recode.ambiguous_mapping', $rulePath . '.selector', 'The system-missing value is mapped more than once.'); + } + $missingIndex = $index; + } elseif ($selector instanceof ElseSelector) { + $elseIndexes[] = $index; + } + } + + if (count($elseIndexes) !== 1) { + $this->violation('recode.else_required_once', $path . '.rules', 'A recode must contain exactly one explicit else rule.'); + } elseif ($elseIndexes[0] !== array_key_last($operation->rules())) { + $this->violation('recode.else_not_last', $path . '.rules[' . $elseIndexes[0] . ']', 'The else rule must be last.'); + } + + $this->validateRangeAmbiguity($ranges, $exactValues, $operation, $path); + } + + private function validateSelector(RecodeSelector $selector, string $path): void + { + if (!in_array($selector::class, [ + ExactValueSelector::class, + NumericRangeSelector::class, + MissingValueSelector::class, + ElseSelector::class, + ], true)) { + $this->violation('selector.unsupported_type', $path . '.type', 'The selector is not part of the canonical source-neutral contract.'); + + return; + } + + if ($selector instanceof ExactValueSelector) { + $this->validateScalar($selector->value(), $path . '.value'); + } elseif ($selector instanceof NumericRangeSelector) { + $lower = $selector->lower(); + $upper = $selector->upper(); + if ($lower === null && $upper === null) { + $this->violation('selector.range_unbounded', $path, 'A numeric range must have at least one finite bound.'); + } + if (($lower !== null && !is_finite($lower)) || ($upper !== null && !is_finite($upper))) { + $this->violation('selector.range_non_finite', $path, 'Numeric range bounds must be finite.'); + } elseif ($lower !== null && $upper !== null && $lower > $upper) { + $this->violation('selector.range_reversed', $path, 'A numeric range lower bound must not exceed its upper bound.'); + } + } + } + + private function validateAction(RecodeAction $action, string $path): void + { + if (!in_array($action::class, [ + AssignValueAction::class, + SetMissingAction::class, + CopySourceAction::class, + ], true)) { + $this->violation('action.unsupported_type', $path . '.type', 'The action is not part of the canonical source-neutral contract.'); + + return; + } + + if ($action instanceof AssignValueAction) { + $this->validateScalar($action->value(), $path . '.value'); + } + } + + private function validateValueLabels(SetValueLabelsOperation $operation, string $path): void + { + /** @var array $values */ + $values = []; + foreach ($operation->labels() as $index => $label) { + $labelPath = $path . '.labels[' . $index . ']'; + $this->validateScalar($label->value(), $labelPath . '.value'); + $this->validateText($label->label(), $labelPath . '.label'); + $identity = $label->value()->identity(); + if (isset($values[$identity])) { + $this->violation('value_labels.duplicate_value', $labelPath . '.value', 'A value may have only one label in a replacement set.'); + } + $values[$identity] = true; + } + } + + private function validateScalar(ScalarValue $value, string $path): void + { + if ($value->type() === 'number') { + if (!is_finite($value->numberValue())) { + $this->violation('value.non_finite', $path, 'Canonical numeric values must be finite binary64 values.'); + } + + return; + } + + $this->validateText($value->stringValue(), $path); + } + + private function validateVariableName(string $name, string $path): void + { + if (strlen($name) > 255 || preg_match('/\A[\p{L}_][\p{L}\p{N}_]*\z/uD', $name) !== 1) { + $this->violation( + 'variable.invalid_name', + $path, + 'Variable names must be 1-255 UTF-8 bytes and contain letters, numbers, or underscore, without a leading number.', + ); + } + } + + private function validateText(?string $text, string $path): void + { + if ($text !== null && (str_contains($text, "\0") || preg_match('//u', $text) !== 1)) { + $this->violation('text.invalid_unicode', $path, 'Text must contain valid UTF-8 scalar text without NUL.'); + } + } + + /** + * @param list $ranges + * @param array $exactValues + */ + private function validateRangeAmbiguity(array $ranges, array $exactValues, RecodeOperation $operation, string $path): void + { + foreach ($ranges as $leftOffset => $left) { + if (($left['lower'] !== null && !is_finite($left['lower'])) || ($left['upper'] !== null && !is_finite($left['upper']))) { + continue; + } + foreach (array_slice($ranges, $leftOffset + 1) as $right) { + if (($right['lower'] !== null && !is_finite($right['lower'])) || ($right['upper'] !== null && !is_finite($right['upper']))) { + continue; + } + if ($this->rangesOverlap($left, $right)) { + $this->violation( + 'recode.ambiguous_mapping', + $path . '.rules[' . $right['index'] . '].selector', + 'Inclusive numeric ranges must not overlap.', + ); + } + } + + foreach ($exactValues as $identity => $exactIndex) { + if (!str_starts_with($identity, 'number:')) { + continue; + } + $selector = $operation->rules()[$exactIndex]->selector(); + if ($selector instanceof ExactValueSelector && $this->rangeContains($left, $selector->value()->numberValue())) { + $this->violation( + 'recode.ambiguous_mapping', + $path . '.rules[' . $exactIndex . '].selector', + 'An exact numeric selector must not overlap a numeric range.', + ); + } + } + } + } + + /** + * @param array{lower: ?float, upper: ?float} $left + * @param array{lower: ?float, upper: ?float} $right + */ + private function rangesOverlap(array $left, array $right): bool + { + return ($left['upper'] === null || $right['lower'] === null || $left['upper'] >= $right['lower']) + && ($right['upper'] === null || $left['lower'] === null || $right['upper'] >= $left['lower']); + } + + /** @param array{lower: ?float, upper: ?float} $range */ + private function rangeContains(array $range, float $value): bool + { + return ($range['lower'] === null || $value >= $range['lower']) + && ($range['upper'] === null || $value <= $range['upper']); + } + + private function violation(string $code, string $path, string $message): void + { + $this->violations[] = new ValidationViolation($code, $path, $message); + } +} diff --git a/src/Transformation/Validation/ValidationResult.php b/src/Transformation/Validation/ValidationResult.php new file mode 100644 index 0000000..349ec1d --- /dev/null +++ b/src/Transformation/Validation/ValidationResult.php @@ -0,0 +1,29 @@ + $violations */ + public function __construct(private array $violations) {} + + public function isValid(): bool + { + return $this->violations === []; + } + + /** @return list */ + public function violations(): array + { + return $this->violations; + } + + public function throwIfInvalid(): void + { + if ($this->violations !== []) { + throw new InvalidTransformationPlan($this->violations); + } + } +} diff --git a/src/Transformation/Validation/ValidationViolation.php b/src/Transformation/Validation/ValidationViolation.php new file mode 100644 index 0000000..37ef75c --- /dev/null +++ b/src/Transformation/Validation/ValidationViolation.php @@ -0,0 +1,20 @@ + $this->code, 'path' => $this->path, 'message' => $this->message]; + } +} diff --git a/tests/Core/CapabilityDeclarationTest.php b/tests/Core/CapabilityDeclarationTest.php index 8796c83..60c39d4 100644 --- a/tests/Core/CapabilityDeclarationTest.php +++ b/tests/Core/CapabilityDeclarationTest.php @@ -69,6 +69,16 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): $profile['ci_tested_server_versions'], ); self::assertNotSame('', $profile['physical_table_mapping']); + self::assertSame('supported', $profile['transformation_workflow']); + self::assertSame('supported', $profile['in_place_transformations']['status']); + self::assertSame('supported', $profile['in_place_transformations']['existing_target']); + self::assertSame( + in_array($name, ['sqlite', 'postgresql'], true) + ? 'supported_in_native_transaction' + : 'preexisting_target_required', + $profile['in_place_transformations']['new_numeric_target'], + ); + self::assertFalse($profile['in_place_transformations']['persistent_rollback_artifacts']); if ($name === 'dolt') { self::assertSame(['maximum_value_bytes'], array_keys($profile['theoretical_limits'])); self::assertSame(306, $profile['proposed_adapter_limits']['maximum_physical_columns']); @@ -89,7 +99,10 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): 'system_missing' => 'sql_null', ], $profile['numeric_exception_policy']); self::assertSame(65_504, $profile['storage_evidence']['text']['observed_value_bytes']); - self::assertSame('unsupported', $profile['transformation_workflow']); + self::assertSame( + 'clean_working_set_and_stable_branch_head', + $profile['in_place_transformations']['dolt_repository_guard'], + ); self::assertSame('mysql_compatible', $profile['transport']); self::assertSame(['2.2.2', '2.2.3'], $profile['exact_ci_tested_versions']); self::assertSame( @@ -98,6 +111,7 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): ); self::assertNull($profile['identity']['active_probe_results']); } else { + self::assertNull($profile['in_place_transformations']['dolt_repository_guard']); self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_physical_columns']); self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_row_bytes']); self::assertArrayNotHasKey('maximum_identifier_bytes', $profile['theoretical_limits']); diff --git a/tests/Transformation/Canonical/TransformationPlanTest.php b/tests/Transformation/Canonical/TransformationPlanTest.php new file mode 100644 index 0000000..dfe5229 --- /dev/null +++ b/tests/Transformation/Canonical/TransformationPlanTest.php @@ -0,0 +1,182 @@ +validPlan(); + (new PlanValidator())->assertValid($plan); + + self::assertSame(self::DATASET_ID, $plan->datasetId()); + self::assertCount(3, $plan->operations()); + self::assertSame('age', $plan->operations()[0]->sourceVariable()); + self::assertSame('age_group', $plan->operations()[0]->targetVariable()); + self::assertSame(TransformationPlan::CONTRACT, $plan->canonicalArray()['contract']); + self::assertSame('in_place', $plan->canonicalArray()['mode']); + self::assertArrayNotHasKey('output_dataset_id', $plan->canonicalArray()); + self::assertArrayNotHasKey('output_table', $plan->canonicalArray()); + self::assertSame($plan->canonicalJson(), $plan->canonicalJson()); + self::assertMatchesRegularExpression('/\A[0-9a-f]{64}\z/D', $plan->hash()); + self::assertSame(hash('sha256', $plan->canonicalJson()), $plan->hash()); + self::assertStringNotContainsString('spss', strtolower($plan->canonicalJson())); + self::assertStringNotContainsString('stata', strtolower($plan->canonicalJson())); + self::assertStringNotContainsString('sas', strtolower($plan->canonicalJson())); + } + + public function testCanonicalJsonHasStableObjectKeyOrderingAndTypedNumbers(): void + { + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('score', 'score', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::number(1)), + new AssignValueAction(ScalarValue::number(-0.0)), + ), + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + ]); + + self::assertSame( + '{"contract":"openstatspec-transformation-plan-v1","dataset_id":"123e4567-e89b-42d3-a456-426614174000","mode":"in_place","operations":[{"rules":[{"action":{"type":"assign","value":{"binary64":"0000000000000000","type":"number"}},"selector":{"type":"exact","value":{"binary64":"3ff0000000000000","type":"number"}}},{"action":{"type":"copy_source"},"selector":{"type":"else"}}],"source_variable":"score","target_variable":"score","type":"recode"}]}', + $plan->canonicalJson(), + ); + self::assertSame('d562adfb994ddad015bd0fee06dc56026c6fa405476ca980e92190c00162ba52', $plan->hash()); + } + + public function testValidatorCollectsIdentityNameDuplicateTargetAndLabelViolations(): void + { + $duplicateLabels = [ + new ValueLabel(ScalarValue::number(1), 'One'), + new ValueLabel(ScalarValue::number(1.0), 'Still one'), + ]; + $plan = new TransformationPlan('NOT-A-UUID', [ + new SetVariableLabelOperation('1 invalid', "bad\0label"), + new SetVariableLabelOperation('1 invalid', 'again'), + new SetValueLabelsOperation('status', $duplicateLabels), + ]); + + $result = (new PlanValidator())->validate($plan); + + self::assertFalse($result->isValid()); + self::assertSame([ + 'dataset_id.invalid_uuid', + 'variable.invalid_name', + 'variable.invalid_name', + 'text.invalid_unicode', + 'variable.invalid_name', + 'variable.invalid_name', + 'operation.duplicate_target', + 'value_labels.duplicate_value', + ], $this->codes($result->violations())); + + $this->expectException(InvalidTransformationPlan::class); + $result->throwIfInvalid(); + } + + public function testValidatorRejectsAmbiguousAndIncompleteRecodeMappings(): void + { + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('score', 'score', [ + new RecodeRule(new ExactValueSelector(ScalarValue::number(2)), new SetMissingAction()), + new RecodeRule(new ExactValueSelector(ScalarValue::number(2.0)), new CopySourceAction()), + new RecodeRule(new NumericRangeSelector(0.0, 10.0), new CopySourceAction()), + new RecodeRule(new NumericRangeSelector(10.0, null), new CopySourceAction()), + new RecodeRule(new MissingValueSelector(), new SetMissingAction()), + new RecodeRule(new MissingValueSelector(), new CopySourceAction()), + ]), + ]); + + $codes = $this->codes((new PlanValidator())->validate($plan)->violations()); + + self::assertContains('recode.else_required_once', $codes); + self::assertContains('recode.ambiguous_mapping', $codes); + self::assertGreaterThanOrEqual(4, count(array_filter( + $codes, + static fn(string $code): bool => $code === 'recode.ambiguous_mapping', + ))); + } + + public function testValidatorRejectsMalformedRangesElseOrderingAndNonFiniteValues(): void + { + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('score', 'score', [ + new RecodeRule(new NumericRangeSelector(null, null), new CopySourceAction()), + new RecodeRule(new NumericRangeSelector(5.0, 4.0), new CopySourceAction()), + new RecodeRule(new NumericRangeSelector(INF, null), new CopySourceAction()), + new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(NAN))), + new RecodeRule(new ExactValueSelector(ScalarValue::string('late')), new CopySourceAction()), + ]), + ]); + + $codes = $this->codes((new PlanValidator())->validate($plan)->violations()); + + self::assertContains('selector.range_unbounded', $codes); + self::assertContains('selector.range_reversed', $codes); + self::assertContains('selector.range_non_finite', $codes); + self::assertContains('value.non_finite', $codes); + self::assertContains('recode.else_not_last', $codes); + } + + public function testEmptyPlanAndEmptyRecodeAreRejected(): void + { + $empty = (new PlanValidator())->validate(new TransformationPlan(self::DATASET_ID, [])); + self::assertSame(['operations.empty'], $this->codes($empty->violations())); + + $emptyRecode = (new PlanValidator())->validate(new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('score', 'score', []), + ])); + self::assertSame(['recode.rules_empty'], $this->codes($emptyRecode->violations())); + } + + private function validPlan(): TransformationPlan + { + return new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('age', 'age_group', [ + new RecodeRule(new NumericRangeSelector(null, 17.0), new AssignValueAction(ScalarValue::number(1))), + new RecodeRule(new NumericRangeSelector(18.0, null), new AssignValueAction(ScalarValue::number(2))), + new RecodeRule(new MissingValueSelector(), new SetMissingAction()), + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + new SetVariableLabelOperation('age_group', 'Age group'), + new SetValueLabelsOperation('age_group', [ + new ValueLabel(ScalarValue::number(1), 'Child'), + new ValueLabel(ScalarValue::number(2), 'Adult'), + ]), + ]); + } + + /** + * @param list<\OpenStatSpec\Transformation\Validation\ValidationViolation> $violations + * @return list + */ + private function codes(array $violations): array + { + return array_map( + static fn(\OpenStatSpec\Transformation\Validation\ValidationViolation $violation): string => $violation->code, + $violations, + ); + } +} diff --git a/tests/Transformation/Execution/DoltGuardTest.php b/tests/Transformation/Execution/DoltGuardTest.php new file mode 100644 index 0000000..52631eb --- /dev/null +++ b/tests/Transformation/Execution/DoltGuardTest.php @@ -0,0 +1,77 @@ +reader( + new DoltEvidence('main', 'abc123', ['respondents']), + )); + + try { + $guard->beforeExecution(); + self::fail('A dirty Dolt working set was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $exception->diagnosticCode); + self::assertStringContainsString('clean working set', $exception->getMessage()); + self::assertStringContainsString('respondents', $exception->getMessage()); + } + } + + public function testItCapturesPostEvidenceWithoutRequiringTheExpectedEditToBeClean(): void + { + $guard = new DoltGuard($this->reader( + new DoltEvidence('main', 'abc123', []), + new DoltEvidence('main', 'abc123', ['respondents', 'variable']), + )); + + $before = $guard->beforeExecution(); + $after = $guard->afterExecution($before); + + self::assertTrue($before->isClean()); + self::assertFalse($after->isClean()); + self::assertSame(['respondents', 'variable'], $after->dirtyTables()); + } + + public function testItRejectsBranchOrHeadMutation(): void + { + $guard = new DoltGuard($this->reader( + new DoltEvidence('main', 'abc123', []), + new DoltEvidence('other', 'abc123', ['respondents']), + )); + $before = $guard->beforeExecution(); + + $this->expectException(UnsupportedOperation::class); + $this->expectExceptionMessage('branch changed'); + $guard->afterExecution($before); + } + + private function reader(DoltEvidence ...$evidence): DoltEvidenceReader + { + return new class (array_values($evidence)) implements DoltEvidenceReader { + /** @param list $evidence */ + public function __construct(private array $evidence) {} + + public function read(): DoltEvidence + { + $next = array_shift($this->evidence); + if ($next === null) { + throw new \LogicException('The test evidence queue is empty.'); + } + + return $next; + } + }; + } +} diff --git a/tests/Transformation/Execution/DoltHeadGuardTest.php b/tests/Transformation/Execution/DoltHeadGuardTest.php new file mode 100644 index 0000000..1def689 --- /dev/null +++ b/tests/Transformation/Execution/DoltHeadGuardTest.php @@ -0,0 +1,34 @@ +reads === 1 + ? new DoltEvidence('main', 'before123', []) + : new DoltEvidence('main', 'after456', ['respondents']); + } + }; + $guard = new DoltGuard($reader); + $before = $guard->beforeExecution(); + + $this->expectException(UnsupportedOperation::class); + $this->expectExceptionMessage('HEAD changed'); + $guard->afterExecution($before); + } +} diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php new file mode 100644 index 0000000..d2818e8 --- /dev/null +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -0,0 +1,284 @@ +pdo = new PDO('sqlite::memory:', options: [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_STRINGIFY_FETCHES => false, + ]); + $this->pdo->exec('PRAGMA foreign_keys = ON'); + (new NormativeCatalog($this->pdo))->createTables(); + $this->pdo->exec( + 'CREATE TABLE respondents (__case_ordinal INTEGER NOT NULL PRIMARY KEY, source_value REAL NULL, destination REAL NULL)', + ); + $this->pdo->prepare( + 'INSERT INTO dataset ' + . '(dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'respondents', 'survey', 5, '2026-07-31 00:00:00']); + $insertVariable = $this->pdo->prepare( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + ); + $insertVariable->execute(['018f47f2-8b6a-7c3d-9e1f-123456789abd', self::DATASET_ID, 1, 'SourceValue', 'source_value', 'numeric']); + $insertVariable->execute(['018f47f2-8b6a-7c3d-9e1f-123456789abe', self::DATASET_ID, 2, 'Destination', 'destination', 'numeric']); + $insertCase = $this->pdo->prepare( + 'INSERT INTO respondents (__case_ordinal, source_value, destination) VALUES (?, ?, ?)', + ); + foreach ([[1, 1.0, -1.0], [2, 2.0, -1.0], [3, 3.0, -1.0], [4, 9.0, -1.0], [5, null, -1.0]] as $case) { + $insertCase->execute($case); + } + CatalogOwnership::markCurrentVersion($this->pdo); + } + + public function testCanonicalOperationsMutateDataAndMetadataInPlace(): void + { + $tablesBefore = $this->tableNames(); + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceValue', 'Destination', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::number(1)), + new AssignValueAction(ScalarValue::number(10)), + ), + new RecodeRule( + new NumericRangeSelector(2.0, 3.0), + new AssignValueAction(ScalarValue::number(20)), + ), + new RecodeRule( + new MissingValueSelector(), + new AssignValueAction(ScalarValue::number(99)), + ), + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + new SetVariableLabelOperation('Destination', 'Recoded destination'), + new SetValueLabelsOperation('Destination', [ + new ValueLabel(ScalarValue::number(10), 'Ten'), + new ValueLabel(ScalarValue::number(20), 'Twenty'), + new ValueLabel(ScalarValue::number(99), 'Missing source'), + ]), + ]); + + $result = (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + + self::assertSame(self::DATASET_ID, $result->datasetId()); + self::assertSame($plan->hash(), $result->planHash()); + self::assertSame(3, $result->operationCount()); + self::assertNull($result->auditOperationId(), 'Execution must not create a missing journal schema.'); + self::assertSame([10.0, 20.0, 20.0, 9.0, 99.0], $this->query( + 'SELECT destination FROM respondents ORDER BY __case_ordinal', + )->fetchAll(PDO::FETCH_COLUMN)); + self::assertSame('Recoded destination', $this->query( + "SELECT variable_label FROM variable WHERE source_name = 'Destination'", + )->fetchColumn()); + self::assertSame( + [['10.0', 'Ten'], ['20.0', 'Twenty'], ['99.0', 'Missing source']], + $this->query( + 'SELECT CAST(vl.numeric_code AS TEXT), vl.label FROM value_label vl ORDER BY vl.ordinal', + )->fetchAll(PDO::FETCH_NUM), + ); + self::assertSame($tablesBefore, $this->tableNames()); + self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM dataset')->fetchColumn()); + self::assertFalse(in_array('operation_catalog', $this->tableNames(), true)); + } + + public function testSqliteAddsANewTargetInsideTheSameWideTableAndTransaction(): void + { + $tablesBefore = $this->tableNames(); + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceValue', 'CreatedTarget', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::number(1)), + new AssignValueAction(ScalarValue::number(100)), + ), + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + + self::assertSame($tablesBefore, $this->tableNames(), 'A transformation must not create a table or snapshot.'); + self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM dataset')->fetchColumn()); + self::assertSame(3, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); + self::assertSame('createdtarget', $this->query( + "SELECT physical_name FROM variable WHERE source_name = 'CreatedTarget'", + )->fetchColumn()); + self::assertSame([100.0, 2.0, 3.0, 9.0, null], $this->query( + 'SELECT createdtarget FROM respondents ORDER BY __case_ordinal', + )->fetchAll(PDO::FETCH_COLUMN)); + } + + public function testNewStringTargetRequiresExplicitCatalogWidth(): void + { + $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); + $this->pdo->prepare( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + )->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789abf', + self::DATASET_ID, + 3, + 'SourceText', + 'source_text', + 'string', + 8, + ]); + $tablesBefore = $this->tableNames(); + $variablesBefore = (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn(); + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceText', 'CreatedText', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::string('a')), + new AssignValueAction(ScalarValue::string('b')), + ), + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + ]); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + self::fail('A string target without declared_string_width unexpectedly executed.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + } + + self::assertSame($tablesBefore, $this->tableNames()); + self::assertSame($variablesBefore, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); + self::assertFalse(in_array('createdtext', $this->tableColumns(), true)); + } + + public function testExistingStringTargetEnforcesNormativeByteWidthBeforeMutation(): void + { + $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); + $this->pdo->exec('ALTER TABLE respondents ADD COLUMN destination_text TEXT NULL'); + $insertVariable = $this->pdo->prepare( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + $insertVariable->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789abf', + self::DATASET_ID, + 3, + 'SourceText', + 'source_text', + 'string', + 4, + ]); + $insertVariable->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789ac0', + self::DATASET_ID, + 4, + 'DestinationText', + 'destination_text', + 'string', + 1, + ]); + + $plans = [ + new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceText', 'DestinationText', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::string('a')), + new AssignValueAction(ScalarValue::string('long')), + ), + new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::string('x'))), + ]), + ]), + new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceText', 'DestinationText', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::string('a')), + new AssignValueAction(ScalarValue::string('x')), + ), + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + ]), + new TransformationPlan(self::DATASET_ID, [ + new SetValueLabelsOperation('DestinationText', [ + new ValueLabel(ScalarValue::string('long'), 'Too wide'), + ]), + ]), + ]; + + foreach ($plans as $plan) { + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + self::fail('A string operation wider than declared_string_width unexpectedly executed.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + } + } + self::assertSame( + [null, null, null, null, null], + $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') + ->fetchAll(PDO::FETCH_COLUMN), + ); + } + + /** @return list */ + private function tableNames(): array + { + $names = $this->query( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", + )->fetchAll(PDO::FETCH_COLUMN); + + return array_values(array_map(static fn(mixed $name): string => (string) $name, $names)); + } + + /** @return list */ + private function tableColumns(): array + { + $names = $this->query('PRAGMA table_info(respondents)')->fetchAll(PDO::FETCH_ASSOC); + + return array_values(array_map( + static fn(array $column): string => (string) $column['name'], + $names, + )); + } + + private function query(string $sql): PDOStatement + { + $statement = $this->pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + + return $statement; + } +} From e6900c25ffa78fc8bd3f1daded9fa226b96456c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 19:13:25 +0300 Subject: [PATCH 02/10] Add SPSS transformation frontend --- src/Frontend/Sas/README.md | 6 + src/Frontend/Spss/Ast/ElseInput.php | 7 + src/Frontend/Spss/Ast/ExecuteStatement.php | 15 + src/Frontend/Spss/Ast/MissingInput.php | 7 + src/Frontend/Spss/Ast/Program.php | 11 + src/Frontend/Spss/Ast/RangeInput.php | 13 + src/Frontend/Spss/Ast/RecodeInput.php | 7 + src/Frontend/Spss/Ast/RecodeOutput.php | 13 + src/Frontend/Spss/Ast/RecodeOutputKind.php | 12 + src/Frontend/Spss/Ast/RecodeRule.php | 13 + src/Frontend/Spss/Ast/RecodeStatement.php | 25 ++ src/Frontend/Spss/Ast/ScalarValue.php | 10 + src/Frontend/Spss/Ast/Statement.php | 10 + src/Frontend/Spss/Ast/SystemMissingInput.php | 7 + src/Frontend/Spss/Ast/ValueInput.php | 10 + src/Frontend/Spss/Ast/ValueLabel.php | 10 + src/Frontend/Spss/Ast/ValueLabelGroup.php | 14 + .../Spss/Ast/ValueLabelsStatement.php | 16 ++ .../Spss/Ast/VariableLabelsStatement.php | 16 ++ src/Frontend/Spss/Binder.php | 96 +++++++ src/Frontend/Spss/Binding/BoundProgram.php | 11 + src/Frontend/Spss/Binding/BoundRecode.php | 17 ++ src/Frontend/Spss/Binding/BoundStatement.php | 7 + .../Spss/Binding/BoundValueLabels.php | 13 + .../Spss/Binding/BoundVariableLabel.php | 10 + src/Frontend/Spss/Compiler.php | 119 ++++++++ src/Frontend/Spss/Diagnostic.php | 14 + src/Frontend/Spss/Lexer.php | 172 +++++++++++ src/Frontend/Spss/Parser.php | 267 ++++++++++++++++++ src/Frontend/Spss/SpssCompiler.php | 39 +++ src/Frontend/Spss/SpssSyntaxException.php | 19 ++ src/Frontend/Spss/Token.php | 20 ++ src/Frontend/Spss/TokenType.php | 19 ++ src/Frontend/Stata/README.md | 6 + tests/Frontend/Spss/LexerTest.php | 46 +++ tests/Frontend/Spss/ParserTest.php | 82 ++++++ tests/Frontend/Spss/SpssCompilerTest.php | 124 ++++++++ 37 files changed, 1303 insertions(+) create mode 100644 src/Frontend/Sas/README.md create mode 100644 src/Frontend/Spss/Ast/ElseInput.php create mode 100644 src/Frontend/Spss/Ast/ExecuteStatement.php create mode 100644 src/Frontend/Spss/Ast/MissingInput.php create mode 100644 src/Frontend/Spss/Ast/Program.php create mode 100644 src/Frontend/Spss/Ast/RangeInput.php create mode 100644 src/Frontend/Spss/Ast/RecodeInput.php create mode 100644 src/Frontend/Spss/Ast/RecodeOutput.php create mode 100644 src/Frontend/Spss/Ast/RecodeOutputKind.php create mode 100644 src/Frontend/Spss/Ast/RecodeRule.php create mode 100644 src/Frontend/Spss/Ast/RecodeStatement.php create mode 100644 src/Frontend/Spss/Ast/ScalarValue.php create mode 100644 src/Frontend/Spss/Ast/Statement.php create mode 100644 src/Frontend/Spss/Ast/SystemMissingInput.php create mode 100644 src/Frontend/Spss/Ast/ValueInput.php create mode 100644 src/Frontend/Spss/Ast/ValueLabel.php create mode 100644 src/Frontend/Spss/Ast/ValueLabelGroup.php create mode 100644 src/Frontend/Spss/Ast/ValueLabelsStatement.php create mode 100644 src/Frontend/Spss/Ast/VariableLabelsStatement.php create mode 100644 src/Frontend/Spss/Binder.php create mode 100644 src/Frontend/Spss/Binding/BoundProgram.php create mode 100644 src/Frontend/Spss/Binding/BoundRecode.php create mode 100644 src/Frontend/Spss/Binding/BoundStatement.php create mode 100644 src/Frontend/Spss/Binding/BoundValueLabels.php create mode 100644 src/Frontend/Spss/Binding/BoundVariableLabel.php create mode 100644 src/Frontend/Spss/Compiler.php create mode 100644 src/Frontend/Spss/Diagnostic.php create mode 100644 src/Frontend/Spss/Lexer.php create mode 100644 src/Frontend/Spss/Parser.php create mode 100644 src/Frontend/Spss/SpssCompiler.php create mode 100644 src/Frontend/Spss/SpssSyntaxException.php create mode 100644 src/Frontend/Spss/Token.php create mode 100644 src/Frontend/Spss/TokenType.php create mode 100644 src/Frontend/Stata/README.md create mode 100644 tests/Frontend/Spss/LexerTest.php create mode 100644 tests/Frontend/Spss/ParserTest.php create mode 100644 tests/Frontend/Spss/SpssCompilerTest.php diff --git a/src/Frontend/Sas/README.md b/src/Frontend/Sas/README.md new file mode 100644 index 0000000..b9a1343 --- /dev/null +++ b/src/Frontend/Sas/README.md @@ -0,0 +1,6 @@ +# SAS transformation frontend + +This namespace is reserved for a future SAS frontend. No SAS lexer, parser, +binder, compiler, conformance claim, or runtime support is currently provided. + +Do not treat this placeholder as an accepted-syntax list or roadmap commitment. diff --git a/src/Frontend/Spss/Ast/ElseInput.php b/src/Frontend/Spss/Ast/ElseInput.php new file mode 100644 index 0000000..344da43 --- /dev/null +++ b/src/Frontend/Spss/Ast/ElseInput.php @@ -0,0 +1,7 @@ +sourceLine; + } +} diff --git a/src/Frontend/Spss/Ast/MissingInput.php b/src/Frontend/Spss/Ast/MissingInput.php new file mode 100644 index 0000000..91eef12 --- /dev/null +++ b/src/Frontend/Spss/Ast/MissingInput.php @@ -0,0 +1,7 @@ + $statements */ + public function __construct(public array $statements) {} +} diff --git a/src/Frontend/Spss/Ast/RangeInput.php b/src/Frontend/Spss/Ast/RangeInput.php new file mode 100644 index 0000000..ab44e1e --- /dev/null +++ b/src/Frontend/Spss/Ast/RangeInput.php @@ -0,0 +1,13 @@ + $sources + * @param non-empty-list $rules + * @param list $targets + */ + public function __construct( + public int $sourceLine, + public array $sources, + public array $rules, + public array $targets, + ) {} + + public function line(): int + { + return $this->sourceLine; + } +} diff --git a/src/Frontend/Spss/Ast/ScalarValue.php b/src/Frontend/Spss/Ast/ScalarValue.php new file mode 100644 index 0000000..1612fe2 --- /dev/null +++ b/src/Frontend/Spss/Ast/ScalarValue.php @@ -0,0 +1,10 @@ + $variables + * @param non-empty-list $labels + */ + public function __construct(public array $variables, public array $labels) {} +} diff --git a/src/Frontend/Spss/Ast/ValueLabelsStatement.php b/src/Frontend/Spss/Ast/ValueLabelsStatement.php new file mode 100644 index 0000000..1e4fe20 --- /dev/null +++ b/src/Frontend/Spss/Ast/ValueLabelsStatement.php @@ -0,0 +1,16 @@ + $groups */ + public function __construct(public int $sourceLine, public array $groups) {} + + public function line(): int + { + return $this->sourceLine; + } +} diff --git a/src/Frontend/Spss/Ast/VariableLabelsStatement.php b/src/Frontend/Spss/Ast/VariableLabelsStatement.php new file mode 100644 index 0000000..bc7506b --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableLabelsStatement.php @@ -0,0 +1,16 @@ + $labels */ + public function __construct(public int $sourceLine, public array $labels) {} + + public function line(): int + { + return $this->sourceLine; + } +} diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php new file mode 100644 index 0000000..7196637 --- /dev/null +++ b/src/Frontend/Spss/Binder.php @@ -0,0 +1,96 @@ +statements as $statement) { + if ($statement instanceof ExecuteStatement) { + continue; + } + if ($statement instanceof RecodeStatement) { + $targets = $statement->targets === [] ? $statement->sources : $statement->targets; + if (count($statement->sources) !== count($targets)) { + $this->fail($statement->line(), 'RECODE INTO must have exactly one target for each source variable.'); + } + $elseSeen = false; + foreach ($statement->rules as $index => $rule) { + if ($rule->input instanceof MissingInput) { + $this->fail( + $statement->line(), + 'MISSING includes user-missing values, which require dataset metadata; use SYSMIS or explicit values.', + ); + } + if ($rule->input instanceof ElseInput) { + if ($elseSeen || $index !== array_key_last($statement->rules)) { + $this->fail($statement->line(), 'ELSE may occur only once and must be the final RECODE rule.'); + } + $elseSeen = true; + } + if ($rule->input instanceof RangeInput) { + $lower = $rule->input->lower?->value; + $upper = $rule->input->upper?->value; + if ($lower === null && $upper === null) { + $this->fail($statement->line(), 'LOWEST THRU HIGHEST is not a finite canonical range; use ELSE instead.'); + } + if (is_string($lower) || is_string($upper)) { + $this->fail($statement->line(), 'RECODE ranges must use numeric bounds.'); + } + if ($lower !== null && $upper !== null && $lower > $upper) { + $this->fail($statement->line(), 'RECODE range lower bound must not exceed its upper bound.'); + } + } + } + foreach ($statement->sources as $index => $source) { + $bound[] = new BoundRecode($source, $targets[$index], $statement->rules); + } + continue; + } + if ($statement instanceof VariableLabelsStatement) { + foreach ($statement->labels as $variable => $label) { + $bound[] = new BoundVariableLabel($variable, $label); + } + continue; + } + if ($statement instanceof ValueLabelsStatement) { + foreach ($statement->groups as $group) { + foreach ($group->variables as $variable) { + $bound[] = new BoundValueLabels($variable, $group->labels); + } + } + continue; + } + + $this->fail($statement->line(), sprintf('Unsupported AST statement %s.', $statement::class)); + } + + return new BoundProgram($datasetId, $bound); + } + + private function fail(int $line, string $message): never + { + throw new SpssSyntaxException([new Diagnostic($line, 1, $message)]); + } +} diff --git a/src/Frontend/Spss/Binding/BoundProgram.php b/src/Frontend/Spss/Binding/BoundProgram.php new file mode 100644 index 0000000..822799f --- /dev/null +++ b/src/Frontend/Spss/Binding/BoundProgram.php @@ -0,0 +1,11 @@ + $statements */ + public function __construct(public string $datasetId, public array $statements) {} +} diff --git a/src/Frontend/Spss/Binding/BoundRecode.php b/src/Frontend/Spss/Binding/BoundRecode.php new file mode 100644 index 0000000..186885f --- /dev/null +++ b/src/Frontend/Spss/Binding/BoundRecode.php @@ -0,0 +1,17 @@ + $rules */ + public function __construct( + public string $sourceVariable, + public string $targetVariable, + public array $rules, + ) {} +} diff --git a/src/Frontend/Spss/Binding/BoundStatement.php b/src/Frontend/Spss/Binding/BoundStatement.php new file mode 100644 index 0000000..e4cc42f --- /dev/null +++ b/src/Frontend/Spss/Binding/BoundStatement.php @@ -0,0 +1,7 @@ + $labels */ + public function __construct(public string $variable, public array $labels) {} +} diff --git a/src/Frontend/Spss/Binding/BoundVariableLabel.php b/src/Frontend/Spss/Binding/BoundVariableLabel.php new file mode 100644 index 0000000..65e76d0 --- /dev/null +++ b/src/Frontend/Spss/Binding/BoundVariableLabel.php @@ -0,0 +1,10 @@ +statements as $statement) { + if ($statement instanceof BoundRecode) { + $rules = []; + $hasElse = false; + foreach ($statement->rules as $rule) { + $selector = $this->selector($rule->input); + $hasElse = $hasElse || $selector instanceof ElseSelector; + $rules[] = new RecodeRule($selector, $this->action($rule->output)); + } + if (!$hasElse) { + $defaultAction = $statement->sourceVariable === $statement->targetVariable + ? new CopySourceAction() + : new SetMissingAction(); + $rules[] = new RecodeRule(new ElseSelector(), $defaultAction); + } + $operations[] = new RecodeOperation($statement->sourceVariable, $statement->targetVariable, $rules); + continue; + } + if ($statement instanceof BoundVariableLabel) { + $operations[] = new SetVariableLabelOperation($statement->variable, $statement->label); + continue; + } + if ($statement instanceof BoundValueLabels) { + $labels = array_map( + fn(AstValueLabel $label): ValueLabel => new ValueLabel( + $this->scalar($label->value), + $label->label, + ), + $statement->labels, + ); + $operations[] = new SetValueLabelsOperation($statement->variable, $labels); + continue; + } + + throw new LogicException(sprintf('Unsupported bound SPSS statement %s.', $statement::class)); + } + + $plan = new TransformationPlan($program->datasetId, $operations); + $this->validator->assertValid($plan); + + return $plan; + } + + private function selector(RecodeInput $input): RecodeSelector + { + return match (true) { + $input instanceof ValueInput => new ExactValueSelector($this->scalar($input->value)), + $input instanceof RangeInput => new NumericRangeSelector( + $input->lower === null ? null : (float) $input->lower->value, + $input->upper === null ? null : (float) $input->upper->value, + ), + $input instanceof SystemMissingInput => new MissingValueSelector(), + $input instanceof ElseInput => new ElseSelector(), + default => throw new LogicException(sprintf('Unsupported SPSS recode input %s.', $input::class)), + }; + } + + private function action(RecodeOutput $output): RecodeAction + { + return match ($output->kind) { + RecodeOutputKind::Value => new AssignValueAction($this->scalar( + $output->value ?? throw new LogicException('A value output must contain a scalar.'), + )), + RecodeOutputKind::Copy => new CopySourceAction(), + RecodeOutputKind::SystemMissing => new SetMissingAction(), + }; + } + + private function scalar(AstScalarValue $value): ScalarValue + { + return is_string($value->value) ? ScalarValue::string($value->value) : ScalarValue::number($value->value); + } +} diff --git a/src/Frontend/Spss/Diagnostic.php b/src/Frontend/Spss/Diagnostic.php new file mode 100644 index 0000000..e613571 --- /dev/null +++ b/src/Frontend/Spss/Diagnostic.php @@ -0,0 +1,14 @@ + */ + public function tokenize(string $source): array + { + $source = preg_replace('/^\xEF\xBB\xBF/', '', $source) ?? $source; + $source = str_replace(["\r\n", "\r"], "\n", $source); + $tokens = []; + $offset = 0; + $line = 1; + $column = 1; + $atStatementStart = true; + $length = strlen($source); + + while ($offset < $length) { + $character = $source[$offset]; + if (ctype_space($character)) { + $this->advance($character, $offset, $line, $column); + continue; + } + + if ($atStatementStart && $character === '*') { + while ($offset < $length && $source[$offset] !== '.') { + $this->advance($source[$offset], $offset, $line, $column); + } + if ($offset === $length) { + $this->fail($line, $column, 'Comment is missing its period terminator.'); + } + $this->advance('.', $offset, $line, $column); + continue; + } + + $tokenLine = $line; + $tokenColumn = $column; + $atStatementStart = false; + + if ($this->startsNumber($source, $offset)) { + $tokens[] = $this->number($source, $offset, $line, $column); + continue; + } + + $punctuation = match ($character) { + '(' => TokenType::LeftParenthesis, + ')' => TokenType::RightParenthesis, + '=' => TokenType::Equals, + ',' => TokenType::Comma, + '/' => TokenType::Slash, + '.' => TokenType::Terminator, + default => null, + }; + if ($punctuation !== null) { + $tokens[] = new Token($punctuation, $character, $tokenLine, $tokenColumn); + $this->advance($character, $offset, $line, $column); + if ($punctuation === TokenType::Terminator) { + $atStatementStart = true; + } + continue; + } + + if ($character === '\'' || $character === '"') { + $tokens[] = $this->string($source, $offset, $line, $column); + continue; + } + + if ($this->isIdentifierStart($character)) { + $start = $offset; + while ($offset < $length && $this->isIdentifierPart($source[$offset])) { + $this->advance($source[$offset], $offset, $line, $column); + } + $tokens[] = new Token(TokenType::Identifier, substr($source, $start, $offset - $start), $tokenLine, $tokenColumn); + continue; + } + + $this->fail($line, $column, sprintf('Unexpected character %s.', json_encode($character, JSON_THROW_ON_ERROR))); + } + + $tokens[] = new Token(TokenType::EndOfFile, '', $line, $column); + + return $tokens; + } + + private function string(string $source, int &$offset, int &$line, int &$column): Token + { + $quote = $source[$offset]; + $tokenLine = $line; + $tokenColumn = $column; + $this->advance($quote, $offset, $line, $column); + $value = ''; + $length = strlen($source); + + while ($offset < $length) { + $character = $source[$offset]; + if ($character !== $quote) { + $value .= $character; + $this->advance($character, $offset, $line, $column); + continue; + } + $this->advance($character, $offset, $line, $column); + if ($offset < $length && $source[$offset] === $quote) { + $value .= $quote; + $this->advance($quote, $offset, $line, $column); + continue; + } + + return new Token(TokenType::String, $value, $tokenLine, $tokenColumn); + } + + $this->fail($tokenLine, $tokenColumn, 'String literal is not closed.'); + } + + private function number(string $source, int &$offset, int &$line, int &$column): Token + { + $tokenLine = $line; + $tokenColumn = $column; + $remaining = substr($source, $offset); + if (preg_match('/^[+-]?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[Ee][+-]?\d+)?/', $remaining, $matches) !== 1) { + $this->fail($line, $column, 'Invalid numeric literal.'); + } + $lexeme = $matches[0]; + foreach (str_split($lexeme) as $character) { + $this->advance($character, $offset, $line, $column); + } + + return new Token(TokenType::Number, $lexeme, $tokenLine, $tokenColumn); + } + + private function startsNumber(string $source, int $offset): bool + { + $character = $source[$offset]; + if (ctype_digit($character)) { + return true; + } + $next = $source[$offset + 1] ?? ''; + if ($character === '.' && ctype_digit($next)) { + return true; + } + + return ($character === '+' || $character === '-') && (ctype_digit($next) || $next === '.'); + } + + private function isIdentifierStart(string $character): bool + { + return ctype_alpha($character) || str_contains('_@#$', $character); + } + + private function isIdentifierPart(string $character): bool + { + return ctype_alnum($character) || str_contains('_@#$', $character); + } + + private function advance(string $character, int &$offset, int &$line, int &$column): void + { + ++$offset; + if ($character === "\n") { + ++$line; + $column = 1; + } else { + ++$column; + } + } + + private function fail(int $line, int $column, string $message): never + { + throw new SpssSyntaxException([new Diagnostic($line, $column, $message)]); + } +} diff --git a/src/Frontend/Spss/Parser.php b/src/Frontend/Spss/Parser.php new file mode 100644 index 0000000..63a0875 --- /dev/null +++ b/src/Frontend/Spss/Parser.php @@ -0,0 +1,267 @@ + */ + private array $tokens = []; + private int $position = 0; + + public function __construct(private readonly Lexer $lexer = new Lexer()) {} + + public function parse(string $source): Program + { + return $this->parseTokens($this->lexer->tokenize($source)); + } + + /** @param list $tokens */ + public function parseTokens(array $tokens): Program + { + $this->tokens = $tokens; + $this->position = 0; + $statements = []; + + while (!$this->check(TokenType::EndOfFile)) { + $command = $this->current(); + if ($this->matchKeyword('RECODE')) { + $statements[] = $this->recode($command); + } elseif ($this->matchKeyword('VARIABLE')) { + $this->consumeKeyword('LABELS', 'Expected LABELS after VARIABLE.'); + $statements[] = $this->variableLabels($command); + } elseif ($this->matchKeyword('VALUE')) { + $this->consumeKeyword('LABELS', 'Expected LABELS after VALUE.'); + $statements[] = $this->valueLabels($command); + } elseif ($this->matchKeyword('VAR')) { + $this->consumeKeyword('LAB', 'Expected LAB after VAR.'); + $statements[] = $this->variableLabels($command); + } elseif ($this->matchKeyword('VAL')) { + $this->consumeKeyword('LAB', 'Expected LAB after VAL.'); + $statements[] = $this->valueLabels($command); + } elseif ($this->matchKeyword('EXECUTE')) { + $statements[] = new ExecuteStatement($command->line); + } else { + $this->fail($command, sprintf('Unsupported SPSS command %s.', $command->lexeme === '' ? '' : $command->lexeme)); + } + + $this->consume(TokenType::Terminator, 'Expected a period after the SPSS command.'); + } + + return new Program($statements); + } + + private function recode(Token $command): RecodeStatement + { + $sources = []; + do { + $sources[] = $this->consumeIdentifier('Expected a source variable after RECODE.')->lexeme; + } while ($this->check(TokenType::Identifier) && !$this->current()->isKeyword('INTO')); + + $rules = []; + while ($this->match(TokenType::LeftParenthesis)) { + $input = $this->recodeInput(); + $this->consume(TokenType::Equals, 'Expected = in a RECODE rule.'); + $output = $this->recodeOutput(); + $this->consume(TokenType::RightParenthesis, 'Expected ) after a RECODE rule.'); + $rules[] = new RecodeRule($input, $output); + } + if ($rules === []) { + $this->fail($this->current(), 'RECODE requires at least one parenthesized rule.'); + } + + $targets = []; + if ($this->matchKeyword('INTO')) { + do { + $targets[] = $this->consumeIdentifier('Expected a target variable after INTO.')->lexeme; + } while ($this->check(TokenType::Identifier)); + } + + return new RecodeStatement($command->line, $sources, $rules, $targets); + } + + private function recodeInput(): RecodeInput + { + if ($this->matchKeyword('SYSMIS')) { + return new SystemMissingInput(); + } + if ($this->matchKeyword('MISSING')) { + return new MissingInput(); + } + if ($this->matchKeyword('ELSE')) { + return new ElseInput(); + } + if ($this->matchKeyword('LOWEST')) { + $this->consumeKeyword('THRU', 'LOWEST must be followed by THRU.'); + $upper = $this->matchKeyword('HIGHEST') ? null : $this->scalar('Expected an upper bound after LOWEST THRU.'); + + return new RangeInput(null, $upper); + } + + $lower = $this->scalar('Expected a value or selector in a RECODE rule.'); + if (!$this->matchKeyword('THRU')) { + return new ValueInput($lower); + } + $upper = $this->matchKeyword('HIGHEST') ? null : $this->scalar('Expected an upper bound after THRU.'); + + return new RangeInput($lower, $upper); + } + + private function recodeOutput(): RecodeOutput + { + if ($this->matchKeyword('COPY')) { + return new RecodeOutput(RecodeOutputKind::Copy); + } + if ($this->matchKeyword('SYSMIS')) { + return new RecodeOutput(RecodeOutputKind::SystemMissing); + } + + return new RecodeOutput(RecodeOutputKind::Value, $this->scalar('Expected a value, COPY, or SYSMIS after =.')); + } + + private function variableLabels(Token $command): VariableLabelsStatement + { + $labels = []; + while (!$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { + $variable = $this->consumeIdentifier('Expected a variable name in VARIABLE LABELS.')->lexeme; + $label = $this->consume(TokenType::String, 'Expected a quoted variable label.')->lexeme; + $labels[$variable] = $label; + } + if ($labels === []) { + $this->fail($this->current(), 'VARIABLE LABELS requires at least one variable and label.'); + } + + return new VariableLabelsStatement($command->line, $labels); + } + + private function valueLabels(Token $command): ValueLabelsStatement + { + $groups = []; + do { + $variables = []; + while ($this->check(TokenType::Identifier)) { + $variables[] = $this->advance()->lexeme; + } + if ($variables === []) { + $this->fail($this->current(), 'VALUE LABELS requires at least one variable before its value-label pairs.'); + } + + $labels = []; + while (!$this->check(TokenType::Slash) && !$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { + $value = $this->scalar('Expected a value in VALUE LABELS.'); + $label = $this->consume(TokenType::String, 'Expected a quoted label after the value.')->lexeme; + $labels[] = new ValueLabel($value, $label); + } + if ($labels === []) { + $this->fail($this->current(), 'VALUE LABELS requires at least one value-label pair.'); + } + $groups[] = new ValueLabelGroup($variables, $labels); + } while ($this->match(TokenType::Slash)); + + return new ValueLabelsStatement($command->line, $groups); + } + + private function scalar(string $message): ScalarValue + { + if ($this->match(TokenType::String)) { + return new ScalarValue($this->previous()->lexeme); + } + if (!$this->match(TokenType::Number)) { + $this->fail($this->current(), $message); + } + $lexeme = $this->previous()->lexeme; + $value = (float) $lexeme; + if (!is_finite($value)) { + $this->fail($this->previous(), 'Numeric literals must be finite IEEE-754 binary64 values.'); + } + + return new ScalarValue($value); + } + + private function consumeIdentifier(string $message): Token + { + return $this->consume(TokenType::Identifier, $message); + } + + private function consumeKeyword(string $keyword, string $message): Token + { + if (!$this->current()->isKeyword($keyword)) { + $this->fail($this->current(), $message); + } + + return $this->advance(); + } + + private function consume(TokenType $type, string $message): Token + { + if (!$this->check($type)) { + $this->fail($this->current(), $message); + } + + return $this->advance(); + } + + private function match(TokenType $type): bool + { + if (!$this->check($type)) { + return false; + } + $this->advance(); + + return true; + } + + private function matchKeyword(string $keyword): bool + { + if (!$this->current()->isKeyword($keyword)) { + return false; + } + $this->advance(); + + return true; + } + + private function check(TokenType $type): bool + { + return $this->current()->type === $type; + } + + private function advance(): Token + { + return $this->tokens[$this->position++]; + } + + private function current(): Token + { + return $this->tokens[$this->position]; + } + + private function previous(): Token + { + return $this->tokens[$this->position - 1]; + } + + private function fail(Token $token, string $message): never + { + throw new SpssSyntaxException([new Diagnostic($token->line, $token->column, $message)]); + } +} diff --git a/src/Frontend/Spss/SpssCompiler.php b/src/Frontend/Spss/SpssCompiler.php new file mode 100644 index 0000000..1000fd7 --- /dev/null +++ b/src/Frontend/Spss/SpssCompiler.php @@ -0,0 +1,39 @@ +parser->parse($source); + } + + public function bind(string $datasetId, Program $program): BoundProgram + { + return $this->binder->bind($datasetId, $program); + } + + public function compile(string $source, string $datasetId): TransformationPlan + { + return $this->compiler->compile($this->binder->bind($datasetId, $this->parser->parse($source))); + } + + public function compileForDataset(string $datasetId, string $source): TransformationPlan + { + return $this->compile($source, $datasetId); + } +} diff --git a/src/Frontend/Spss/SpssSyntaxException.php b/src/Frontend/Spss/SpssSyntaxException.php new file mode 100644 index 0000000..58a9c63 --- /dev/null +++ b/src/Frontend/Spss/SpssSyntaxException.php @@ -0,0 +1,19 @@ + $diagnostics + */ + public function __construct(public readonly array $diagnostics) + { + $first = $diagnostics[0]; + parent::__construct(sprintf('SPSS syntax error at %d:%d: %s', $first->line, $first->column, $first->message)); + } +} diff --git a/src/Frontend/Spss/Token.php b/src/Frontend/Spss/Token.php new file mode 100644 index 0000000..5a68650 --- /dev/null +++ b/src/Frontend/Spss/Token.php @@ -0,0 +1,20 @@ +type === TokenType::Identifier && strcasecmp($this->lexeme, $keyword) === 0; + } +} diff --git a/src/Frontend/Spss/TokenType.php b/src/Frontend/Spss/TokenType.php new file mode 100644 index 0000000..46beb41 --- /dev/null +++ b/src/Frontend/Spss/TokenType.php @@ -0,0 +1,19 @@ +tokenize("* generated comment.\nVARIABLE LABELS score 'A score''s label'.\n"); + + self::assertSame( + [TokenType::Identifier, TokenType::Identifier, TokenType::Identifier, TokenType::String, TokenType::Terminator, TokenType::EndOfFile], + array_column($tokens, 'type'), + ); + self::assertSame("A score's label", $tokens[3]->lexeme); + self::assertSame(2, $tokens[0]->line); + } + + public function testDecimalPointIsNotMistakenForAStatementTerminator(): void + { + $tokens = (new Lexer())->tokenize('RECODE score (.5=2).'); + + self::assertSame('.5', $tokens[3]->lexeme); + self::assertSame(TokenType::Number, $tokens[3]->type); + self::assertSame(1, count(array_filter($tokens, static fn($token): bool => $token->type === TokenType::Terminator))); + } + + public function testRejectsUnknownCharactersWithPositionedDiagnostic(): void + { + try { + (new Lexer())->tokenize('RECODE score (`=1).'); + self::fail('Unknown syntax unexpectedly tokenized.'); + } catch (SpssSyntaxException $exception) { + self::assertSame(1, $exception->diagnostics[0]->line); + self::assertSame(15, $exception->diagnostics[0]->column); + self::assertStringContainsString('Unexpected character', $exception->diagnostics[0]->message); + } + } +} diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php new file mode 100644 index 0000000..6bdef05 --- /dev/null +++ b/tests/Frontend/Spss/ParserTest.php @@ -0,0 +1,82 @@ +parse(<<<'SPSS' + RECODE score + (1=10) + (2 THRU 4=20) + (LOWEST THRU 0=SYSMIS) + (5 THRU HIGHEST=COPY) + (SYSMIS=99) + (MISSING=98) + (ELSE=COPY) + INTO band. + SPSS); + + self::assertCount(1, $program->statements); + $statement = $program->statements[0]; + self::assertInstanceOf(RecodeStatement::class, $statement); + self::assertSame(['score'], $statement->sources); + self::assertSame(['band'], $statement->targets); + self::assertInstanceOf(ValueInput::class, $statement->rules[0]->input); + self::assertInstanceOf(RangeInput::class, $statement->rules[1]->input); + $lowest = $statement->rules[2]->input; + self::assertInstanceOf(RangeInput::class, $lowest); + self::assertNull($lowest->lower); + $highest = $statement->rules[3]->input; + self::assertInstanceOf(RangeInput::class, $highest); + self::assertNull($highest->upper); + self::assertInstanceOf(SystemMissingInput::class, $statement->rules[4]->input); + self::assertInstanceOf(MissingInput::class, $statement->rules[5]->input); + self::assertInstanceOf(ElseInput::class, $statement->rules[6]->input); + self::assertSame(RecodeOutputKind::SystemMissing, $statement->rules[2]->output->kind); + self::assertSame(RecodeOutputKind::Copy, $statement->rules[3]->output->kind); + } + + public function testParsesVariableAndValueLabelGroups(): void + { + $program = (new Parser())->parse(<<<'SPSS' + VARIABLE LABELS score 'Overall score' band 'Score band'. + VALUE LABELS score 1 'One' 2 'Two' / band 'L' 'Low' 'H' 'High'. + SPSS); + + self::assertInstanceOf(VariableLabelsStatement::class, $program->statements[0]); + self::assertSame(['score' => 'Overall score', 'band' => 'Score band'], $program->statements[0]->labels); + self::assertInstanceOf(ValueLabelsStatement::class, $program->statements[1]); + self::assertCount(2, $program->statements[1]->groups); + self::assertSame(['score'], $program->statements[1]->groups[0]->variables); + self::assertSame(['band'], $program->statements[1]->groups[1]->variables); + } + + public function testFailsClosedForUnknownOrUnterminatedCommands(): void + { + foreach (['COMPUTE score=1.', 'RECODE score (1=2)'] as $syntax) { + try { + (new Parser())->parse($syntax); + self::fail('Unsupported or unterminated syntax unexpectedly parsed.'); + } catch (SpssSyntaxException $exception) { + self::assertNotEmpty($exception->diagnostics); + } + } + } +} diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php new file mode 100644 index 0000000..c67e02d --- /dev/null +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -0,0 +1,124 @@ +compile(<<<'SPSS' + RECODE score (1=10) (2 THRU 4=20) (SYSMIS=99) (ELSE=COPY). + VARIABLE LABELS score 'Overall score'. + VALUE LABELS score 10 'Low' 20 'High'. + EXECUTE. + SPSS, self::DATASET_ID); + + self::assertSame(self::DATASET_ID, $plan->datasetId()); + self::assertCount(3, $plan->operations()); + /** @var RecodeOperation $recode */ + $recode = $plan->operations()[0]; + self::assertInstanceOf(RecodeOperation::class, $recode); + self::assertInstanceOf(MissingValueSelector::class, $recode->rules()[2]->selector()); + self::assertInstanceOf(ElseSelector::class, $recode->rules()[3]->selector()); + self::assertInstanceOf(CopySourceAction::class, $recode->rules()[3]->action()); + self::assertInstanceOf(SetVariableLabelOperation::class, $plan->operations()[1]); + self::assertInstanceOf(SetValueLabelsOperation::class, $plan->operations()[2]); + self::assertSame('openstatspec-transformation-plan-v1', $plan->canonicalArray()['contract']); + self::assertSame('in_place', $plan->canonicalArray()['mode']); + } + + public function testMakesSpssDefaultElseSemanticsExplicit(): void + { + $inPlace = (new SpssCompiler())->compile('RECODE score (1=2).', self::DATASET_ID); + $into = (new SpssCompiler())->compile('RECODE score (1=2) INTO band.', self::DATASET_ID); + /** @var RecodeOperation $inPlaceRecode */ + $inPlaceRecode = $inPlace->operations()[0]; + /** @var RecodeOperation $intoRecode */ + $intoRecode = $into->operations()[0]; + + self::assertInstanceOf(CopySourceAction::class, $inPlaceRecode->rules()[1]->action()); + self::assertInstanceOf(SetMissingAction::class, $intoRecode->rules()[1]->action()); + self::assertInstanceOf(ElseSelector::class, $intoRecode->rules()[1]->selector()); + } + + public function testExpandsParallelSourceAndIntoLists(): void + { + $plan = (new SpssCompiler())->compile('RECODE first second (1=2) INTO new_first new_second.', self::DATASET_ID); + + self::assertCount(2, $plan->operations()); + self::assertSame('first', $plan->operations()[0]->sourceVariable()); + self::assertSame('new_first', $plan->operations()[0]->targetVariable()); + self::assertSame('second', $plan->operations()[1]->sourceVariable()); + self::assertSame('new_second', $plan->operations()[1]->targetVariable()); + } + + public function testBinderRejectsInvalidElsePositionAndIntoArity(): void + { + foreach ([ + 'RECODE score (ELSE=COPY) (1=2).', + 'RECODE first second (1=2) INTO only_one.', + "RECODE score ('a' THRU 'z'=1).", + ] as $syntax) { + $this->expectCompileFailure($syntax); + } + } + + public function testFailsClosedForUserMissingSelectorWithoutDatasetMetadata(): void + { + try { + (new SpssCompiler())->compile('RECODE score (MISSING=0) (ELSE=COPY).', self::DATASET_ID); + self::fail('MISSING unexpectedly compiled as system missing.'); + } catch (SpssSyntaxException $exception) { + self::assertStringContainsString('user-missing', $exception->diagnostics[0]->message); + } + } + + public function testLargeIntegerLiteralIsParsedDirectlyAsBinary64(): void + { + $plan = (new SpssCompiler())->compile( + 'RECODE score (999999999999999999999=1) (ELSE=COPY).', + self::DATASET_ID, + ); + + /** @var RecodeOperation $recode */ + $recode = $plan->operations()[0]; + /** @var \OpenStatSpec\Transformation\Model\Selector\ExactValueSelector $selector */ + $selector = $recode->rules()[0]->selector(); + self::assertInstanceOf( + \OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, + $selector, + ); + self::assertSame((float) '999999999999999999999', $selector->value()->numberValue()); + self::assertNotSame((float) PHP_INT_MAX, $selector->value()->numberValue()); + } + + public function testRejectsNonFiniteNumericLiteral(): void + { + $this->expectCompileFailure('RECODE score (1e999=1) (ELSE=COPY).'); + } + + private function expectCompileFailure(string $syntax): void + { + try { + (new SpssCompiler())->compile($syntax, self::DATASET_ID); + self::fail('Semantically invalid syntax unexpectedly compiled.'); + } catch (SpssSyntaxException $exception) { + self::assertNotEmpty($exception->diagnostics); + } + } +} From 906d5c93c3aef6d4f52542f1aedbeb9bc997a5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 19:14:33 +0300 Subject: [PATCH 03/10] Document transformation architecture --- README.md | 3 + docs/architecture.md | 20 +++++ docs/transformations.md | 182 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 docs/transformations.md diff --git a/README.md b/README.md index 0bf78d3..bbbad15 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,9 @@ Pass only loss codes consciously accepted for that conversion. `operation_catalo - `src/Spss` - SAV/ZSAV gating, typed V3 engine bridge and public adapter API. See [docs/architecture.md](docs/architecture.md) for the complete relational contract. +See [docs/transformations.md](docs/transformations.md) for the canonical +transformation plan, frontend boundaries, in-place guarantees, supported +syntax, and development commands. ## Upgrading an existing catalogue diff --git a/docs/architecture.md b/docs/architecture.md index af3744e..b44edd5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,6 +82,26 @@ Encrypted files, Portable (`.por`) files and arbitrary external-engine formats a `src/Spss` owns SAV/ZSAV extension gating, external-engine normalization, typed php-spss V3 bridging and the public `SpssAdapter` API. +### Transformations + +The src/Transformation package owns the statistics-package-neutral canonical +plan, validation, deterministic plan identity, and PDO in-place executor. An +apply preserves the canonical dataset UUID and registered wide-table identity. +It does not publish a derived dataset or create a persistent data copy, +snapshot, rollback table, or OpenStatSpec-managed version. + +The src/Frontend/Spss package is a separate language boundary that turns the +documented SPSS subset into a canonical plan. The executor accepts that plan +and does not import or invoke the SPSS frontend. The Stata and SAS directories +are documented placeholders only; they contain no implementation and make no +support claim. + +All implemented PDO profiles remain eligible transformation targets. Dolt adds +active-branch, HEAD, and clean-working-set guards; it is not a gateway for the +feature. Dolt owns history and rollback. See the +[transformation manual](transformations.md) for the complete contract and +operational guidance. + ## External engine The selected engine is [TonisOrmisson/php-spss](https://github.com/TonisOrmisson/php-spss), Composer dependency `tiamo/spss` 3.x. It is external. If a compatible reader or writer is unavailable, the adapter produces an explicit `external_engine_unavailable` diagnostic rather than pretending conversion succeeded. diff --git a/docs/transformations.md b/docs/transformations.md new file mode 100644 index 0000000..b95a3b4 --- /dev/null +++ b/docs/transformations.md @@ -0,0 +1,182 @@ +# Transformations + +## Purpose and boundary + +The transformation layer applies small, deterministic edits to an existing +OpenStatSpec dataset. Its canonical `TransformationPlan`, validation, and SQL +executor do not depend on SPSS, Stata, SAS, or another statistics package. +Language-specific syntax belongs to a frontend that compiles into the same +canonical plan. + +The initial frontend implements a documented subset of SPSS transformation +syntax. The Stata and SAS directories are placeholders only. Their presence is +an architectural reservation, not a support claim. + +## In-place contract + +Every successful apply preserves both identities: + +- the existing `dataset.dataset_id`; and +- the existing `dataset.physical_table_schema` plus + `dataset.physical_table_name`. + +The executor updates that wide table and its existing normative metadata +catalog in place. It does not create a derived dataset, persistent output or +staging table, full-table copy, snapshot table, hidden rollback table, or a +parallel OpenStatSpec version. A successful edit therefore does not increase +the persistent dataset count or physical data-table count. + +The database engine's native transaction is used where it can make the +operation atomic. OpenStatSpec does not add a durable undo or recovery-version +layer around engines whose DDL commits implicitly. Dolt remains the history, +diff, branch, and rollback layer when Dolt is the selected SQL server. + +## Architecture + +The package separates four responsibilities: + +1. `OpenStatSpec\Transformation\Model` defines the canonical, typed plan and + operations. +2. `OpenStatSpec\Transformation\Validation` validates plans without knowing + their source language or SQL dialect. +3. `OpenStatSpec\Frontend\Spss` lexes, parses, binds, and compiles the supported + SPSS subset into a canonical plan. +4. `OpenStatSpec\Transformation\Execution` resolves catalog identities and + applies a validated plan through the active PDO profile. + +The SQL executor accepts a completed plan. It never invokes the SPSS parser. +Likewise, the SPSS frontend does not issue SQL or select a database profile. +This is the package boundary a future real frontend must use. + +Canonical serialization is deterministic. The plan hash identifies the exact +validated operation sequence; source text and language provenance stay outside +the source-neutral plan. + +## Supported operations + +The canonical layer supports: + +- ordered recode rules with exact values, numeric ranges, missing values, and + exactly one explicit final else rule; +- assigning a scalar value, copying the source value, or assigning system + missing; +- variable-label replacement; and +- complete value-label replacement for one variable. + +Recode rules use first-match semantics. Validation rejects overlapping, +duplicate, or ill-typed rules before SQL mutation. An SPSS frontend plan always +meets the canonical explicit-else contract: when source syntax omits `ELSE`, +the compiler adds SPSS's context-appropriate default action. Variables are +resolved through the normative `variable` catalog and physical identifiers +are quoted by the active PDO SQL profile; callers cannot supply raw table or +column SQL. + +## SPSS frontend scope + +The SPSS frontend recognizes the documented transformation subset: + +- `RECODE ... INTO ...` with exact values, `THRU` ranges, + `LOWEST`, `HIGHEST`, `SYSMIS`, `ELSE`, `COPY`, and `SYSMIS` + outputs; +- `VARIABLE LABELS`; and +- `VALUE LABELS`. + +Keywords are case-insensitive. Dataset variable references currently must +match the normative `variable.source_name` spelling exactly; this documented +subset does not claim SPSS's case-insensitive symbol binding. The `MISSING` +selector fails closed because SPSS user-missing semantics require binding the +dataset's `missing_rule` metadata; use `SYSMIS` for system missing or list +supported explicit values. Statements end with a period. Unsupported SPSS +commands fail closed with a frontend diagnostic; they are not silently skipped +or passed to an external statistics engine. This package does not claim full +SPSS syntax compatibility. + +## SQL profiles and Dolt + +Transformations are not restricted to Dolt. The executor uses every SQL +connection profile implemented by this package: SQLite, PostgreSQL, +MySQL/MariaDB, and Dolt. + +Dolt adds safety evidence rather than acting as a gateway. Before mutation the +executor checks the active branch, resolves `HEAD`, and requires a clean Dolt +working set. After mutation it verifies that branch and `HEAD` did not change +under the operation. The executor does not switch branches or create a Dolt +commit. The caller owns the later review and commit policy. + +MySQL-family DDL commits implicitly. A recode into a new physical target column +can only be part of one native atomic apply on a profile with transactional DDL. +On MySQL, MariaDB, and Dolt, create and catalog the intended target variable in +the deployment workflow before applying a recode to it. Existing-column +recodes and metadata edits remain supported. This capability boundary avoids +pretending that a compensating copy or OpenStatSpec rollback layer is atomic. + +SQLite and PostgreSQL may create a new numeric target column inside their +native transaction. A new string target must be registered on every profile +before execution so its normative `declared_string_width` is explicit. + +The machine-readable capability declaration reports in-place transformations +as supported for every implemented SQL profile and states these target-creation +boundaries. Dolt additionally reports its clean-working-set and stable +branch/HEAD guard. + +## Minimal PHP flow + +```php +use OpenStatSpec\Frontend\Spss\SpssCompiler; +use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; + +$datasetId = '018f47a2-4c10-7d34-8f11-93b1c3efc321'; +$syntax = 'RECODE score (1=10) (ELSE=COPY).'; + +$plan = (new SpssCompiler())->compile($syntax, $datasetId); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($plan); +``` + +`$pdo` must already point to the dedicated OpenStatSpec catalog namespace. +The executor verifies the catalog ownership marker and resolves the same +`dataset_id` and physical wide table before mutation. + +## Development commands + +Install dependencies and run the complete local gate from the PHP repository: + +```bash +composer install +composer check +``` + +Run only transformation tests while developing the layer: + +```bash +vendor/bin/phpunit tests/Transformation tests/Frontend/Spss +``` + +Apply the formatter, then rerun the complete gate: + +```bash +composer fix +composer check +``` + +Database integration checks require the corresponding PDO driver and server. +They must use a dedicated OpenStatSpec namespace, as described in the +[architecture guide](architecture.md#deployment-namespace-and-connection-isolation). + +## Operational checklist + +Before applying a plan: + +1. verify that the connection uses the intended dedicated OpenStatSpec + namespace; +2. select the existing dataset by its canonical UUID; +3. compile source syntax explicitly with the intended frontend, or construct a + canonical plan directly; +4. validate the plan before any mutation; +5. on Dolt, start from the expected branch and a clean working set; and +6. after success, inspect the data and metadata diff and decide separately + whether to create a Dolt commit. + +OpenStatSpec stores only compact operation evidence such as the plan identity +and relevant Dolt state. It never stores copied row state as transformation +audit data. From 05be2a62cd0085bfa1cf57ea294a312260d9e062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 19:36:04 +0300 Subject: [PATCH 04/10] Address transformation review findings --- .../InPlaceTransformationExecutor.php | 18 +++++- .../InPlaceTransformationExecutorTest.php | 57 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index 9381f43..badfffc 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -301,6 +301,11 @@ private function assertRecodeKinds( $target->sourceName, $this->stringWidth($target), )); + } elseif ($action instanceof SetMissingAction && $target->storageKind === 'string') { + throw $this->invalidCatalog(sprintf( + 'System-missing recoding is not representable for string variable "%s".', + $target->sourceName, + )); } } } @@ -378,8 +383,11 @@ private function ensureTargetExists(DatasetBinding $dataset, VariableBinding $ta $type, )); $this->statement( - 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, ' + . 'print_format_family, print_format_width, print_format_decimals, ' + . 'write_format_family, write_format_width, write_format_decimals) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', )->execute([ $target->variableId, $dataset->datasetId, @@ -387,6 +395,12 @@ private function ensureTargetExists(DatasetBinding $dataset, VariableBinding $ta $target->sourceName, $target->physicalName, $target->storageKind, + 5, + 8, + 0, + 5, + 8, + 0, ]); $created[$target->sourceName] = true; } diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index d2818e8..dad7811 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -12,6 +12,7 @@ use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Model\Action\AssignValueAction; use OpenStatSpec\Transformation\Model\Action\CopySourceAction; +use OpenStatSpec\Transformation\Model\Action\SetMissingAction; use OpenStatSpec\Transformation\Model\RecodeOperation; use OpenStatSpec\Transformation\Model\RecodeRule; use OpenStatSpec\Transformation\Model\ScalarValue; @@ -139,11 +140,67 @@ public function testSqliteAddsANewTargetInsideTheSameWideTableAndTransaction(): self::assertSame('createdtarget', $this->query( "SELECT physical_name FROM variable WHERE source_name = 'CreatedTarget'", )->fetchColumn()); + self::assertSame( + ['5', 8, 0, '5', 8, 0], + $this->query( + "SELECT print_format_family, print_format_width, print_format_decimals, " + . "write_format_family, write_format_width, write_format_decimals " + . "FROM variable WHERE source_name = 'CreatedTarget'", + )->fetch(PDO::FETCH_NUM), + ); self::assertSame([100.0, 2.0, 3.0, 9.0, null], $this->query( 'SELECT createdtarget FROM respondents ORDER BY __case_ordinal', )->fetchAll(PDO::FETCH_COLUMN)); } + public function testSystemMissingActionForStringTargetFailsBeforeMutation(): void + { + $this->pdo->exec("ALTER TABLE respondents ADD COLUMN source_text TEXT NOT NULL DEFAULT ''"); + $this->pdo->exec("ALTER TABLE respondents ADD COLUMN destination_text TEXT NOT NULL DEFAULT 'original'"); + $insertVariable = $this->pdo->prepare( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + $insertVariable->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789abf', + self::DATASET_ID, + 3, + 'SourceText', + 'source_text', + 'string', + 8, + ]); + $insertVariable->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789ac0', + self::DATASET_ID, + 4, + 'DestinationText', + 'destination_text', + 'string', + 8, + ]); + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceText', 'DestinationText', [ + new RecodeRule(new ElseSelector(), new SetMissingAction()), + ]), + ]); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + self::fail('System-missing recoding into a string target unexpectedly executed.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertStringContainsString('not representable', $exception->getMessage()); + } + + self::assertSame( + ['original', 'original', 'original', 'original', 'original'], + $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') + ->fetchAll(PDO::FETCH_COLUMN), + ); + } + public function testNewStringTargetRequiresExplicitCatalogWidth(): void { $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); From 83492f49b79a103c8fb4ef5a2b59e54ef725750c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 19:43:41 +0300 Subject: [PATCH 05/10] Handle sequential and else-only recodes --- .../InPlaceTransformationExecutor.php | 10 ++++++++++ .../Validation/PlanValidator.php | 12 ----------- tests/Frontend/Spss/SpssCompilerTest.php | 16 +++++++++++++++ .../Canonical/TransformationPlanTest.php | 19 ++++++++++++++++-- .../InPlaceTransformationExecutorTest.php | 20 +++++++++++++++++++ 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index badfffc..05a0c32 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -433,6 +433,16 @@ private function applyRecode( if ($else === null) { throw $this->invalidCatalog('A recode requires one explicit final else action.'); } + if ($when === []) { + $this->statement(sprintf( + 'UPDATE %s SET %s = %s', + $this->qualifiedTable($dataset), + $this->quote($target->physicalName), + $else, + ))->execute($parameters); + + return; + } $sql = sprintf( 'UPDATE %s SET %s = CASE %s ELSE %s END', $this->qualifiedTable($dataset), diff --git a/src/Transformation/Validation/PlanValidator.php b/src/Transformation/Validation/PlanValidator.php index 77cf268..0697906 100644 --- a/src/Transformation/Validation/PlanValidator.php +++ b/src/Transformation/Validation/PlanValidator.php @@ -41,21 +41,9 @@ public function validate(TransformationPlan $plan): ValidationResult $this->violation('operations.empty', '$.operations', 'A transformation plan must contain at least one operation.'); } - /** @var array $targets */ - $targets = []; foreach ($plan->operations() as $index => $operation) { $path = '$.operations[' . $index . ']'; $this->validateOperation($operation, $path); - $targetKey = $operation::class . "\0" . $operation->targetVariable(); - if (isset($targets[$targetKey])) { - $this->violation( - 'operation.duplicate_target', - $path . '.target_variable', - sprintf('Operation type %s targets variable %s more than once.', $operation->type(), $operation->targetVariable()), - ); - } else { - $targets[$targetKey] = $index; - } } return new ValidationResult($this->violations); diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index c67e02d..19c3279 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -67,6 +67,22 @@ public function testExpandsParallelSourceAndIntoLists(): void self::assertSame('new_second', $plan->operations()[1]->targetVariable()); } + public function testPreservesSequentialOperationsOnTheSameTarget(): void + { + $plan = (new SpssCompiler())->compile(<<<'SPSS' + RECODE score (1=2). + RECODE score (2=3). + VARIABLE LABELS score 'First label'. + VARIABLE LABELS score 'Replacement label'. + SPSS, self::DATASET_ID); + + self::assertCount(4, $plan->operations()); + self::assertSame( + ['recode', 'recode', 'set_variable_label', 'set_variable_label'], + array_map(static fn($operation): string => $operation->type(), $plan->operations()), + ); + } + public function testBinderRejectsInvalidElsePositionAndIntoArity(): void { foreach ([ diff --git a/tests/Transformation/Canonical/TransformationPlanTest.php b/tests/Transformation/Canonical/TransformationPlanTest.php index dfe5229..b7de90e 100644 --- a/tests/Transformation/Canonical/TransformationPlanTest.php +++ b/tests/Transformation/Canonical/TransformationPlanTest.php @@ -66,7 +66,7 @@ public function testCanonicalJsonHasStableObjectKeyOrderingAndTypedNumbers(): vo self::assertSame('d562adfb994ddad015bd0fee06dc56026c6fa405476ca980e92190c00162ba52', $plan->hash()); } - public function testValidatorCollectsIdentityNameDuplicateTargetAndLabelViolations(): void + public function testValidatorCollectsIdentityNameAndLabelViolations(): void { $duplicateLabels = [ new ValueLabel(ScalarValue::number(1), 'One'), @@ -88,7 +88,6 @@ public function testValidatorCollectsIdentityNameDuplicateTargetAndLabelViolatio 'text.invalid_unicode', 'variable.invalid_name', 'variable.invalid_name', - 'operation.duplicate_target', 'value_labels.duplicate_value', ], $this->codes($result->violations())); @@ -96,6 +95,22 @@ public function testValidatorCollectsIdentityNameDuplicateTargetAndLabelViolatio $result->throwIfInvalid(); } + public function testValidatorAllowsSequentialOperationsOnTheSameTarget(): void + { + $plan = new TransformationPlan(self::DATASET_ID, [ + new SetVariableLabelOperation('status', 'First label'), + new SetVariableLabelOperation('status', 'Replacement label'), + new RecodeOperation('status', 'status', [ + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + new RecodeOperation('status', 'status', [ + new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(1))), + ]), + ]); + + self::assertTrue((new PlanValidator())->validate($plan)->isValid()); + } + public function testValidatorRejectsAmbiguousAndIncompleteRecodeMappings(): void { $plan = new TransformationPlan(self::DATASET_ID, [ diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index dad7811..0b9419e 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -153,6 +153,26 @@ public function testSqliteAddsANewTargetInsideTheSameWideTableAndTransaction(): )->fetchAll(PDO::FETCH_COLUMN)); } + public function testRecodeWithOnlyElseAssignsTheExpressionDirectly(): void + { + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceValue', 'Destination', [ + new RecodeRule( + new ElseSelector(), + new AssignValueAction(ScalarValue::number(7)), + ), + ]), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + + self::assertSame( + [7.0, 7.0, 7.0, 7.0, 7.0], + $this->query('SELECT destination FROM respondents ORDER BY __case_ordinal') + ->fetchAll(PDO::FETCH_COLUMN), + ); + } + public function testSystemMissingActionForStringTargetFailsBeforeMutation(): void { $this->pdo->exec("ALTER TABLE respondents ADD COLUMN source_text TEXT NOT NULL DEFAULT ''"); From 684a3b8ac56151772ef5fba15c1937b3244564bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 19:52:30 +0300 Subject: [PATCH 06/10] Reject dependency-unsafe parallel recodes --- docs/transformations.md | 6 ++++++ src/Frontend/Spss/Binder.php | 10 ++++++++++ tests/Frontend/Spss/SpssCompilerTest.php | 16 ++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/docs/transformations.md b/docs/transformations.md index b95a3b4..ee41622 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -91,6 +91,12 @@ commands fail closed with a frontend diagnostic; they are not silently skipped or passed to an external statistics engine. This package does not claim full SPSS syntax compatibility. +A multi-variable RECODE with INTO targets is expanded into ordered canonical +operations only when an earlier target does not overwrite a source needed by a +later pair in the same statement. Dependency-overlapping lists fail closed +because preserving SPSS simultaneous-input semantics would otherwise require +hidden row snapshots. + ## SQL profiles and Dolt Transformations are not restricted to Dolt. The executor uses every SQL diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index 7196637..57da311 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -35,6 +35,16 @@ public function bind(string $datasetId, Program $program): BoundProgram if (count($statement->sources) !== count($targets)) { $this->fail($statement->line(), 'RECODE INTO must have exactly one target for each source variable.'); } + foreach ($targets as $targetIndex => $target) { + foreach (array_slice($statement->sources, $targetIndex + 1) as $laterSource) { + if ($target === $laterSource) { + $this->fail( + $statement->line(), + 'RECODE INTO targets must not overwrite a source used by a later pair in the same statement.', + ); + } + } + } $elseSeen = false; foreach ($statement->rules as $index => $rule) { if ($rule->input instanceof MissingInput) { diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 19c3279..b68fbca 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -83,6 +83,22 @@ public function testPreservesSequentialOperationsOnTheSameTarget(): void ); } + public function testRejectsParallelRecodeThatWouldOverwriteALaterSource(): void + { + try { + (new SpssCompiler())->compile( + 'RECODE a b (ELSE=COPY) INTO b c.', + self::DATASET_ID, + ); + self::fail('A dependency-unsafe parallel RECODE unexpectedly compiled.'); + } catch (SpssSyntaxException $exception) { + self::assertStringContainsString( + 'overwrite a source used by a later pair', + $exception->diagnostics[0]->message, + ); + } + } + public function testBinderRejectsInvalidElsePositionAndIntoArity(): void { foreach ([ From 7c793476af01fdc370e5790b7352679905ba6120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 20:09:22 +0300 Subject: [PATCH 07/10] Accept catalog variable names in transformation plans --- CHANGELOG.md | 17 ++++++++++++++++- src/Transformation/Validation/PlanValidator.php | 4 ++-- .../Canonical/TransformationPlanTest.php | 16 ++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63dcfad..fd2f28e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,22 @@ ## [Unreleased] +## [0.4.0] - 2026-07-31 + +### Added + +- Added a source-neutral canonical transformation plan, validation, provenance, + and in-place execution layer for recodes, variable labels, and value labels. +- Added an SPSS syntax frontend and documented extension points for future + statistical-language frontends, with explicit SAS and Stata placeholders. + ### Changed +- Transformations now mutate the existing logical dataset and physical wide + table without creating copied datasets, persistent staging tables, or an + OpenStatSpec-managed undo/version history; Dolt identity can be recorded for + audit without making Dolt mandatory for other supported connections. + - Distinguished conservative MySQL 8.4.x/9.7.x, MariaDB 11.4.x/11.8.x/12.3.x, and PostgreSQL 17.x/18.x runtime claims from exact CI evidence at MySQL 8.4.11/9.7.2, MariaDB 11.4.12/11.8.8/12.3.2, and @@ -17,7 +31,8 @@ - Pinned active conformance fixtures and capabilities to released OpenStatSpec specification v0.1.0 at commit `d287c2cde9ade71f04e27dd012caec876901aed5`. -[Unreleased]: https://github.com/OpenStatSpec/php/compare/v0.3.0...HEAD +[Unreleased]: https://github.com/OpenStatSpec/php/compare/v0.4.0...HEAD +[0.4.0]: https://github.com/OpenStatSpec/php/compare/v0.3.0...v0.4.0 ## [0.3.0] - 2026-07-30 diff --git a/src/Transformation/Validation/PlanValidator.php b/src/Transformation/Validation/PlanValidator.php index 0697906..6ff6544 100644 --- a/src/Transformation/Validation/PlanValidator.php +++ b/src/Transformation/Validation/PlanValidator.php @@ -207,11 +207,11 @@ private function validateScalar(ScalarValue $value, string $path): void private function validateVariableName(string $name, string $path): void { - if (strlen($name) > 255 || preg_match('/\A[\p{L}_][\p{L}\p{N}_]*\z/uD', $name) !== 1) { + if ($name === '' || strlen($name) > 255 || str_contains($name, "\0") || preg_match('//u', $name) !== 1) { $this->violation( 'variable.invalid_name', $path, - 'Variable names must be 1-255 UTF-8 bytes and contain letters, numbers, or underscore, without a leading number.', + 'Variable names must contain 1-255 bytes of valid UTF-8 scalar text without NUL.', ); } } diff --git a/tests/Transformation/Canonical/TransformationPlanTest.php b/tests/Transformation/Canonical/TransformationPlanTest.php index b7de90e..bf1724e 100644 --- a/tests/Transformation/Canonical/TransformationPlanTest.php +++ b/tests/Transformation/Canonical/TransformationPlanTest.php @@ -73,8 +73,8 @@ public function testValidatorCollectsIdentityNameAndLabelViolations(): void new ValueLabel(ScalarValue::number(1.0), 'Still one'), ]; $plan = new TransformationPlan('NOT-A-UUID', [ - new SetVariableLabelOperation('1 invalid', "bad\0label"), - new SetVariableLabelOperation('1 invalid', 'again'), + new SetVariableLabelOperation("invalid\0name", "bad\0label"), + new SetVariableLabelOperation("invalid\0name", 'again'), new SetValueLabelsOperation('status', $duplicateLabels), ]); @@ -111,6 +111,18 @@ public function testValidatorAllowsSequentialOperationsOnTheSameTarget(): void self::assertTrue((new PlanValidator())->validate($plan)->isValid()); } + public function testValidatorAllowsSourceNeutralCatalogVariableNames(): void + { + $plan = new TransformationPlan(self::DATASET_ID, [ + new SetVariableLabelOperation('@score', 'At-prefixed variable'), + new SetVariableLabelOperation('#temp', 'Scratch variable'), + new SetVariableLabelOperation('$weight', 'System variable'), + new SetVariableLabelOperation('wave-1 score', 'Catalog name outside SPSS syntax'), + ]); + + self::assertTrue((new PlanValidator())->validate($plan)->isValid()); + } + public function testValidatorRejectsAmbiguousAndIncompleteRecodeMappings(): void { $plan = new TransformationPlan(self::DATASET_ID, [ From 96836e2e076be000992bb977c8bf5b8933aab1c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 20:21:14 +0300 Subject: [PATCH 08/10] Preflight transformation target column capacity --- .../InPlaceTransformationExecutor.php | 16 +++++- .../InPlaceTransformationExecutorTest.php | 50 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index 05a0c32..c82c531 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -216,7 +216,7 @@ private function preflight(TransformationPlan $plan, array $variables): array } $target = $variables[$operation->targetVariable()] ?? null; if ($target === null) { - $this->assertCanCreateTarget($source); + $this->assertCanCreateTarget($source, count($variables)); $physical = $this->connection->profile->physicalIdentifier($operation->targetVariable(), $used); $target = new VariableBinding( NormativeCatalog::uuid(), @@ -252,7 +252,7 @@ private function preflight(TransformationPlan $plan, array $variables): array return $variables; } - private function assertCanCreateTarget(VariableBinding $source): void + private function assertCanCreateTarget(VariableBinding $source, int $registeredVariableCount): void { if ($source->storageKind === 'string') { throw new UnsupportedOperation( @@ -271,6 +271,18 @@ private function assertCanCreateTarget(VariableBinding $source): void ), ); } + + $maximum = $this->connection->profile->effectiveMaximumSourceVariables($this->connection->pdo); + if ($registeredVariableCount >= $maximum) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + sprintf( + '%s supports at most %d source variables in one OpenStatSpec wide table.', + $this->connection->profileName, + $maximum, + ), + ); + } } private function assertRecodeKinds( diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 0b9419e..04a310a 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -153,6 +153,56 @@ public function testSqliteAddsANewTargetInsideTheSameWideTableAndTransaction(): )->fetchAll(PDO::FETCH_COLUMN)); } + public function testNewTargetIsRejectedBeforeAlterAtTheEffectiveColumnLimit(): void + { + $connection = new Connection($this->pdo); + $maximum = $connection->profile->effectiveMaximumSourceVariables($this->pdo); + $columns = ['__case_ordinal INTEGER NOT NULL PRIMARY KEY']; + for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { + $columns[] = 'v' . $ordinal . ' REAL NULL'; + } + + $this->pdo->exec('DROP TABLE respondents'); + $this->pdo->exec('CREATE TABLE respondents (' . implode(', ', $columns) . ')'); + $this->pdo->exec('DELETE FROM variable'); + $insert = $this->pdo->prepare( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + ); + $this->pdo->beginTransaction(); + for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { + $name = 'V' . $ordinal; + $insert->execute([ + sprintf('00000000-0000-4000-8000-%012d', $ordinal), + self::DATASET_ID, + $ordinal, + $name, + strtolower($name), + 'numeric', + ]); + } + $this->pdo->commit(); + $columnsBefore = count($this->query('PRAGMA table_info(respondents)')->fetchAll()); + + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('V1', 'OverflowTarget', [ + new RecodeRule(new ElseSelector(), new CopySourceAction()), + ]), + ]); + + try { + (new InPlaceTransformationExecutor($connection))->execute($plan); + self::fail('A target beyond the effective source-variable limit was created.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('at most ' . $maximum . ' source variables', $exception->getMessage()); + } + + self::assertSame($maximum, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); + self::assertSame($columnsBefore, count($this->query('PRAGMA table_info(respondents)')->fetchAll())); + } + public function testRecodeWithOnlyElseAssignsTheExpressionDirectly(): void { $plan = new TransformationPlan(self::DATASET_ID, [ From 7e8772e70a35a7cfe00e18677b65ab4b6378c7a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 20:35:07 +0300 Subject: [PATCH 09/10] Preserve transaction and string comparison semantics --- src/Sql/AbstractPdoSqlProfile.php | 5 ++ src/Sql/MySqlProfile.php | 6 ++ src/Sql/PdoSqlProfile.php | 3 + src/Sql/PostgreSqlProfile.php | 6 ++ src/Sql/SqliteProfile.php | 6 ++ .../InPlaceTransformationExecutor.php | 11 ++- tests/Sql/PdoSqlProfileTest.php | 5 ++ .../InPlaceTransformationExecutorTest.php | 75 +++++++++++++++++++ 8 files changed, 116 insertions(+), 1 deletion(-) diff --git a/src/Sql/AbstractPdoSqlProfile.php b/src/Sql/AbstractPdoSqlProfile.php index 52a5a5a..dd6ea9e 100644 --- a/src/Sql/AbstractPdoSqlProfile.php +++ b/src/Sql/AbstractPdoSqlProfile.php @@ -10,6 +10,11 @@ abstract class AbstractPdoSqlProfile implements PdoSqlProfile { + public function exactValueCondition(string $expression, bool $stringValue): string + { + return $expression . ' = ?'; + } + public function physicalIdentifier(string $source, array $used = []): string { $base = trim(strtolower((string) preg_replace('/[^a-zA-Z0-9_]+/', '_', $source)), '_'); diff --git a/src/Sql/MySqlProfile.php b/src/Sql/MySqlProfile.php index ec55951..77b81a8 100644 --- a/src/Sql/MySqlProfile.php +++ b/src/Sql/MySqlProfile.php @@ -62,6 +62,12 @@ public function textType(): string { return 'LONGTEXT'; } + public function exactValueCondition(string $expression, bool $stringValue): string + { + return $stringValue + ? 'BINARY ' . $expression . ' = BINARY ?' + : parent::exactValueCondition($expression, false); + } public function effectiveMaximumValueBytes(PDO $pdo): int { $packet = $this->packetPayloadBytes($pdo); diff --git a/src/Sql/PdoSqlProfile.php b/src/Sql/PdoSqlProfile.php index 7586f0c..23b464d 100644 --- a/src/Sql/PdoSqlProfile.php +++ b/src/Sql/PdoSqlProfile.php @@ -45,6 +45,9 @@ public function numericType(): string; public function textType(): string; + /** Returns an exact-value predicate containing exactly one value placeholder. */ + public function exactValueCondition(string $expression, bool $stringValue): string; + /** * Creates a deterministic, dialect-safe physical identifier. The source name * itself remains authoritative in the variables catalogue. diff --git a/src/Sql/PostgreSqlProfile.php b/src/Sql/PostgreSqlProfile.php index 73c1d0e..d7c33e7 100644 --- a/src/Sql/PostgreSqlProfile.php +++ b/src/Sql/PostgreSqlProfile.php @@ -52,4 +52,10 @@ public function textType(): string { return 'TEXT'; } + public function exactValueCondition(string $expression, bool $stringValue): string + { + return $stringValue + ? $expression . ' COLLATE "C" = ? COLLATE "C"' + : parent::exactValueCondition($expression, false); + } } diff --git a/src/Sql/SqliteProfile.php b/src/Sql/SqliteProfile.php index 49a1082..55322ad 100644 --- a/src/Sql/SqliteProfile.php +++ b/src/Sql/SqliteProfile.php @@ -53,6 +53,12 @@ public function textType(): string { return 'TEXT'; } + public function exactValueCondition(string $expression, bool $stringValue): string + { + return $stringValue + ? $expression . ' COLLATE BINARY = ? COLLATE BINARY' + : parent::exactValueCondition($expression, false); + } public function effectiveMaximumSourceVariables(PDO $pdo): int { $maximum = $this->compileOption($pdo, 'MAX_COLUMN'); diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index c82c531..ed1998a 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -48,6 +48,12 @@ public function execute(TransformationPlan $plan): ExecutionResult { $this->validator->assertValid($plan); $this->connection->assertClaimedSupported(); + if ($this->connection->pdo->inTransaction()) { + throw new UnsupportedOperation( + DiagnosticCode::UnsupportedOperation, + 'An in-place transformation cannot start inside a caller-owned active transaction.', + ); + } CatalogOwnership::assertReadyForUse($this->connection->pdo); $dataset = $this->resolveDataset($plan->datasetId()); @@ -470,7 +476,10 @@ private function exactCondition(string $sourceSql, ExactValueSelector $selector, { $parameters[] = $this->boundScalar($selector->value()); - return $sourceSql . ' = ?'; + return $this->connection->profile->exactValueCondition( + $sourceSql, + $selector->value()->type() === 'string', + ); } /** @param list $parameters */ diff --git a/tests/Sql/PdoSqlProfileTest.php b/tests/Sql/PdoSqlProfileTest.php index c6ddb7c..02cdebb 100644 --- a/tests/Sql/PdoSqlProfileTest.php +++ b/tests/Sql/PdoSqlProfileTest.php @@ -31,8 +31,13 @@ public function testProfilesDeclarePortableSqlRulesWithoutServerConnections(): v self::assertSame("`name`", $mysql->quoteIdentifier('name')); self::assertSame(1599, $postgres->maximumSourceVariables()); self::assertSame(1016, $mysql->maximumSourceVariables()); + self::assertSame('column_name = ?', $postgres->exactValueCondition('column_name', false)); + self::assertSame('column_name COLLATE "C" = ? COLLATE "C"', $postgres->exactValueCondition('column_name', true)); + self::assertSame('column_name COLLATE BINARY = ? COLLATE BINARY', $sqlite->exactValueCondition('column_name', true)); + self::assertSame('BINARY column_name = BINARY ?', $mysql->exactValueCondition('column_name', true)); $dolt = new DoltProfile(); + self::assertSame('BINARY column_name = BINARY ?', $dolt->exactValueCondition('column_name', true)); self::assertSame(305, $dolt->maximumSourceVariables()); self::assertSame(65_504, $dolt->maximumRowBytes()); self::assertSame('bytes', $dolt->identifierLimitUnit()); diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 04a310a..dbcbdc1 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -223,6 +223,44 @@ public function testRecodeWithOnlyElseAssignsTheExpressionDirectly(): void ); } + public function testCallerOwnedTransactionIsRejectedWithoutRollbackOrJournalWork(): void + { + $tablesBefore = $this->tableNames(); + $this->pdo->beginTransaction(); + $this->pdo->exec('UPDATE respondents SET source_value = 42 WHERE __case_ordinal = 1'); + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceValue', 'Destination', [ + new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(7))), + ]), + ]); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + self::fail('Execution unexpectedly joined a caller-owned transaction.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::UnsupportedOperation, $exception->diagnosticCode); + self::assertStringContainsString('caller-owned active transaction', $exception->getMessage()); + } + + self::assertTrue($this->pdo->inTransaction()); + self::assertSame(42.0, $this->query( + 'SELECT source_value FROM respondents WHERE __case_ordinal = 1', + )->fetchColumn()); + self::assertSame($tablesBefore, $this->tableNames()); + self::assertFalse(in_array('operation_catalog', $this->tableNames(), true)); + self::assertSame( + [-1.0, -1.0, -1.0, -1.0, -1.0], + $this->query('SELECT destination FROM respondents ORDER BY __case_ordinal') + ->fetchAll(PDO::FETCH_COLUMN), + ); + + self::assertTrue($this->pdo->commit()); + self::assertFalse($this->pdo->inTransaction()); + self::assertSame(42.0, $this->query( + 'SELECT source_value FROM respondents WHERE __case_ordinal = 1', + )->fetchColumn()); + } + public function testSystemMissingActionForStringTargetFailsBeforeMutation(): void { $this->pdo->exec("ALTER TABLE respondents ADD COLUMN source_text TEXT NOT NULL DEFAULT ''"); @@ -271,6 +309,43 @@ public function testSystemMissingActionForStringTargetFailsBeforeMutation(): voi ); } + public function testExactStringSelectorIgnoresCaseInsensitiveColumnCollation(): void + { + $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT COLLATE NOCASE NULL'); + $this->pdo->exec('ALTER TABLE respondents ADD COLUMN destination_text TEXT NULL'); + $insertVariable = $this->pdo->prepare( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + $insertVariable->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789abf', self::DATASET_ID, 3, + 'SourceText', 'source_text', 'string', 8, + ]); + $insertVariable->execute([ + '018f47f2-8b6a-7c3d-9e1f-123456789ac0', self::DATASET_ID, 4, + 'DestinationText', 'destination_text', 'string', 8, + ]); + $this->pdo->exec("UPDATE respondents SET source_text = CASE __case_ordinal WHEN 1 THEN 'Match' WHEN 2 THEN 'match' ELSE 'other' END"); + $plan = new TransformationPlan(self::DATASET_ID, [ + new RecodeOperation('SourceText', 'DestinationText', [ + new RecodeRule( + new ExactValueSelector(ScalarValue::string('Match')), + new AssignValueAction(ScalarValue::string('exact')), + ), + new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::string('else'))), + ]), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + + self::assertSame( + ['exact', 'else', 'else', 'else', 'else'], + $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') + ->fetchAll(PDO::FETCH_COLUMN), + ); + } + public function testNewStringTargetRequiresExplicitCatalogWidth(): void { $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); From 136e09905e238568d73ec79851b42a061fcd478e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 31 Jul 2026 20:54:42 +0300 Subject: [PATCH 10/10] Harden SPSS frontend edge cases --- src/Frontend/Spss/Binder.php | 6 +++ src/Frontend/Spss/Lexer.php | 55 ++++++++++++++++++------ tests/Frontend/Spss/LexerTest.php | 19 ++++++++ tests/Frontend/Spss/SpssCompilerTest.php | 16 +++++++ 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index 57da311..f3c901b 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -35,6 +35,12 @@ public function bind(string $datasetId, Program $program): BoundProgram if (count($statement->sources) !== count($targets)) { $this->fail($statement->line(), 'RECODE INTO must have exactly one target for each source variable.'); } + if ($statement->targets !== [] && count(array_unique($targets)) !== count($targets)) { + $this->fail( + $statement->line(), + 'RECODE INTO must not contain duplicate target variables in the same statement.', + ); + } foreach ($targets as $targetIndex => $target) { foreach (array_slice($statement->sources, $targetIndex + 1) as $laterSource) { if ($target === $laterSource) { diff --git a/src/Frontend/Spss/Lexer.php b/src/Frontend/Spss/Lexer.php index b80b89f..cb55b51 100644 --- a/src/Frontend/Spss/Lexer.php +++ b/src/Frontend/Spss/Lexer.php @@ -19,15 +19,15 @@ public function tokenize(string $source): array $length = strlen($source); while ($offset < $length) { - $character = $source[$offset]; + $character = $this->characterAt($source, $offset, $line, $column); if (ctype_space($character)) { $this->advance($character, $offset, $line, $column); continue; } if ($atStatementStart && $character === '*') { - while ($offset < $length && $source[$offset] !== '.') { - $this->advance($source[$offset], $offset, $line, $column); + while ($offset < $length && $this->characterAt($source, $offset, $line, $column) !== '.') { + $this->advance($this->characterAt($source, $offset, $line, $column), $offset, $line, $column); } if ($offset === $length) { $this->fail($line, $column, 'Comment is missing its period terminator.'); @@ -70,14 +70,19 @@ public function tokenize(string $source): array if ($this->isIdentifierStart($character)) { $start = $offset; - while ($offset < $length && $this->isIdentifierPart($source[$offset])) { - $this->advance($source[$offset], $offset, $line, $column); + while ($offset < $length) { + $identifierCharacter = $this->characterAt($source, $offset, $line, $column); + if (!$this->isIdentifierPart($identifierCharacter)) { + break; + } + $this->advance($identifierCharacter, $offset, $line, $column); } $tokens[] = new Token(TokenType::Identifier, substr($source, $start, $offset - $start), $tokenLine, $tokenColumn); continue; } - $this->fail($line, $column, sprintf('Unexpected character %s.', json_encode($character, JSON_THROW_ON_ERROR))); + $encoded = json_encode($character, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE); + $this->fail($line, $column, sprintf('Unexpected character %s.', $encoded === false ? '?' : $encoded)); } $tokens[] = new Token(TokenType::EndOfFile, '', $line, $column); @@ -87,7 +92,7 @@ public function tokenize(string $source): array private function string(string $source, int &$offset, int &$line, int &$column): Token { - $quote = $source[$offset]; + $quote = $this->characterAt($source, $offset, $line, $column); $tokenLine = $line; $tokenColumn = $column; $this->advance($quote, $offset, $line, $column); @@ -95,14 +100,14 @@ private function string(string $source, int &$offset, int &$line, int &$column): $length = strlen($source); while ($offset < $length) { - $character = $source[$offset]; + $character = $this->characterAt($source, $offset, $line, $column); if ($character !== $quote) { $value .= $character; $this->advance($character, $offset, $line, $column); continue; } $this->advance($character, $offset, $line, $column); - if ($offset < $length && $source[$offset] === $quote) { + if ($offset < $length && $this->characterAt($source, $offset, $line, $column) === $quote) { $value .= $quote; $this->advance($quote, $offset, $line, $column); continue; @@ -146,17 +151,43 @@ private function startsNumber(string $source, int $offset): bool private function isIdentifierStart(string $character): bool { - return ctype_alpha($character) || str_contains('_@#$', $character); + return preg_match('/\A\p{L}\z/uD', $character) === 1 || str_contains('_@#$', $character); } private function isIdentifierPart(string $character): bool { - return ctype_alnum($character) || str_contains('_@#$', $character); + return preg_match('/\A[\p{L}\p{M}\p{N}]\z/uD', $character) === 1 || str_contains('_@#$', $character); + } + + private function characterAt(string $source, int $offset, int $line, int $column): string + { + $firstByte = ord($source[$offset]); + $byteLength = match (true) { + $firstByte <= 0x7F => 1, + $firstByte >= 0xC2 && $firstByte <= 0xDF => 2, + $firstByte >= 0xE0 && $firstByte <= 0xEF => 3, + $firstByte >= 0xF0 && $firstByte <= 0xF4 => 4, + default => null, + }; + $character = $byteLength === null ? '' : substr($source, $offset, $byteLength); + if ( + $byteLength === null + || strlen($character) !== $byteLength + || preg_match('/\A.\z/usD', $character) !== 1 + ) { + $this->fail( + $line, + $column, + sprintf('Invalid UTF-8 sequence beginning with byte 0x%02X.', $firstByte), + ); + } + + return $character; } private function advance(string $character, int &$offset, int &$line, int &$column): void { - ++$offset; + $offset += strlen($character); if ($character === "\n") { ++$line; $column = 1; diff --git a/tests/Frontend/Spss/LexerTest.php b/tests/Frontend/Spss/LexerTest.php index 3e70b93..71b437e 100644 --- a/tests/Frontend/Spss/LexerTest.php +++ b/tests/Frontend/Spss/LexerTest.php @@ -32,6 +32,25 @@ public function testDecimalPointIsNotMistakenForAStatementTerminator(): void self::assertSame(1, count(array_filter($tokens, static fn($token): bool => $token->type === TokenType::Terminator))); } + public function testTokenizesPrecomposedAndDecomposedUnicodeIdentifiers(): void + { + $precomposed = "\u{00E9}chelle"; + $decomposed = "e\u{0301}chelle"; + $tokens = (new Lexer())->tokenize($precomposed . ' ' . $decomposed . '.'); + + self::assertSame(TokenType::Identifier, $tokens[0]->type); + self::assertSame($precomposed, $tokens[0]->lexeme); + self::assertSame(TokenType::Identifier, $tokens[1]->type); + self::assertSame($decomposed, $tokens[1]->lexeme); + } + + public function testRejectsInvalidTrailingUtf8Byte(): void + { + $this->expectException(SpssSyntaxException::class); + + (new Lexer())->tokenize("RECODE score (1=2).\xC3"); + } + public function testRejectsUnknownCharactersWithPositionedDiagnostic(): void { try { diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index b68fbca..246b468 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -99,6 +99,22 @@ public function testRejectsParallelRecodeThatWouldOverwriteALaterSource(): void } } + public function testRejectsDuplicateIntoTargetsWithinOneParallelRecode(): void + { + try { + (new SpssCompiler())->compile( + 'RECODE first second (1=2) INTO result result.', + self::DATASET_ID, + ); + self::fail('A parallel RECODE with duplicate INTO targets unexpectedly compiled.'); + } catch (SpssSyntaxException $exception) { + self::assertStringContainsString( + 'duplicate target variables', + $exception->diagnostics[0]->message, + ); + } + } + public function testBinderRejectsInvalidElsePositionAndIntoArity(): void { foreach ([