diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29e736a..5a33edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v7 with: repository: OpenStatSpec/specification - ref: 6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045 + ref: 34141dda023d9e0217c37c232e39f436edfb0746 path: openstatspec-specification - uses: shivammathur/setup-php@v2 with: @@ -62,7 +62,7 @@ jobs: uses: actions/checkout@v7 with: repository: OpenStatSpec/specification - ref: 6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045 + ref: 34141dda023d9e0217c37c232e39f436edfb0746 path: openstatspec-specification - uses: shivammathur/setup-php@v2 with: @@ -105,7 +105,7 @@ jobs: uses: actions/checkout@v7 with: repository: OpenStatSpec/specification - ref: 6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045 + ref: 34141dda023d9e0217c37c232e39f436edfb0746 path: openstatspec-specification - uses: shivammathur/setup-php@v2 with: @@ -148,7 +148,7 @@ jobs: uses: actions/checkout@v7 with: repository: OpenStatSpec/specification - ref: 6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045 + ref: 34141dda023d9e0217c37c232e39f436edfb0746 path: openstatspec-specification - uses: shivammathur/setup-php@v2 with: @@ -157,3 +157,49 @@ jobs: extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - run: vendor/bin/phpunit --filter "MariaDbSpssRoundTripTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest" + + dolt-integration: + name: Dolt 2.2.2 integration / PHP ${{ matrix.php }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + php: ['8.4', '8.5'] + services: + dolt: + image: dolthub/dolt-sql-server:2.2.2@sha256:eb59cbb2bd5e1860f192a88f68c89697578256216ff848acf4bdc60ab5fc7ffc + env: + DOLT_DATABASE: openstatspec + DOLT_USER: openstatspec + DOLT_PASSWORD: openstatspec + DOLT_USER_HOST: "%" + DOLT_ROOT_PASSWORD: root + DOLT_ROOT_HOST: "%" + ports: + - 3306:3306 + env: + OPENSTATSPEC_DOLT_DSN: mysql:host=127.0.0.1;port=3306;dbname=openstatspec;charset=utf8mb4 + OPENSTATSPEC_DOLT_USER: openstatspec + OPENSTATSPEC_DOLT_PASSWORD: openstatspec + steps: + - uses: actions/checkout@v7 + - name: Checkout OpenStatSpec specification fixtures + uses: actions/checkout@v7 + with: + repository: OpenStatSpec/specification + ref: 34141dda023d9e0217c37c232e39f436edfb0746 + path: openstatspec-specification + - uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + coverage: none + extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib + - name: Wait for Dolt 2.2.2 + run: | + for attempt in {1..120}; do + php -r 'try { $pdo = new PDO(getenv("OPENSTATSPEC_DOLT_DSN"), getenv("OPENSTATSPEC_DOLT_USER"), getenv("OPENSTATSPEC_DOLT_PASSWORD")); exit($pdo->query("SELECT DOLT_VERSION()")->fetchColumn() === "2.2.2" ? 0 : 1); } catch (Throwable) { exit(1); }' && exit 0 + sleep 1 + done + exit 1 + - run: composer install --no-interaction --prefer-dist + - run: vendor/bin/phpunit --filter "DoltSpssRoundTripTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest" diff --git a/README.md b/README.md index 571ab75..158aa22 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ It imports an unencrypted SPSS `.sav` or `.zsav` dataset into a relational datab This is an early reference implementation. Its round-trip contract is **semantic**, not byte-identical: supported cases, order, variables, values, dictionary metadata and technical metadata are preserved; compression layout, timestamps and other writer-specific bytes are not promised. -SQLite, PostgreSQL 17/18, MySQL 8.4/9.7 and MariaDB 11.4/11.8/12.3 are implemented PDO profiles. Each follows one strict-wide contract: +SQLite, PostgreSQL 17/18, MySQL 8.4/9.7, MariaDB 11.4/11.8/12.3 and Dolt 2.2.2 are implemented PDO profiles. Each follows one strict-wide contract: 1. One source dataset becomes one dedicated SQL data table. 2. One SPSS case becomes one SQL row. @@ -32,6 +32,7 @@ Composer resolves dependencies against PHP 8.4.1, the package minimum. ## API ```php +use OpenStatSpec\Spss\GuardedImportSpssEngine; use OpenStatSpec\Spss\SpssAdapter; $pdo = new PDO('pgsql:host=localhost;dbname=statistics', $user, $password); @@ -44,6 +45,35 @@ $export = $adapter->export('survey_2026', '/data/survey-export.sav'); // SpssExportResult: operationId, datasetName, caseCount, diagnostics, allowLoss ``` +Use `GuardedImportSpssEngine` when an engine must read from an ephemeral +descriptor while the adapter and database receive only a logical source path: + +```php +$engine = new GuardedImportSpssEngine($innerEngine, $procFdPath, 'sav'); +$adapter = new SpssAdapter($pdo, $engine); +$import = $adapter->import( + $engine->logicalPath(), + 'survey_2026', + verifiedSourceSha256: $verifiedSourceSha256, +); +``` + +`verifiedSourceSha256` must be exactly 64 lowercase hexadecimal characters. +The adapter persists it as `dataset.source_hash`, but validates only its shape: +the caller is responsible for proving that it hashes the exact bytes read by +the engine. Keep any physical guarded path, such as `/proc/self/fd/...`, +internal to the engine; `SpssAdapter::import()` rejects exact Linux +descriptor paths under `/proc/*/fd/` and `/dev/fd/` before any database +mutation. `GuardedImportSpssEngine` also recursively rejects descriptor +paths in inner-engine identity keys or values and replaces every inner read +exception with a neutral logical-source error. Sanitized errors do not chain +the original exception, so descriptor paths cannot enter operation or fidelity +journals through identity metadata or read failures. The adapter and catalogue +need only the logical `.sav`/`.zsav` path +and the verified hash. Omitting the argument preserves the +existing behavior: a readable source file is hashed by pathname, otherwise +`dataset.source_hash` is `NULL`. + ### Fidelity policy Export is fail-closed. If an exporter reports a known fidelity diagnostic, it does **not** write a file until the caller explicitly accepts its code: @@ -86,7 +116,7 @@ unrelated application code while an adapter operation is running: - PostgreSQL: create a dedicated schema and use a dedicated connection with a fixed `search_path` containing that schema only. -- MySQL/MariaDB: select a dedicated database in the adapter DSN. +- MySQL/MariaDB/Dolt: select a dedicated database in the adapter DSN. - SQLite: use a dedicated database file and connection. The machine-readable capability declaration must expose the active namespace @@ -134,7 +164,7 @@ composer check `composer check` validates Composer configuration, lints PHP, checks style, runs PHPStan and runs PHPUnit. Use `composer fix` for safe style fixes, then rerun `composer check`. -GitHub Actions runs the regular suite on PHP 8.4 and 8.5. It also runs real SAV and ZSAV integration round trips against PostgreSQL 17 and 18, MySQL 8.4 and 9.7, and MariaDB 11.4, 11.8 and 12.3. Those profile checks use their PDO drivers and php-spss V3 read/write paths, not only DDL snapshots. +GitHub Actions runs the regular suite on PHP 8.4 and 8.5. It also runs real SAV and ZSAV integration round trips against PostgreSQL 17 and 18, MySQL 8.4 and 9.7, MariaDB 11.4, 11.8 and 12.3, and Dolt 2.2.2. Those profile checks use their PDO drivers and php-spss V3 read/write paths, not only DDL snapshots. ## Contributing diff --git a/docs/architecture.md b/docs/architecture.md index 84cd2f0..3d3310c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,6 +30,7 @@ The public `SpssAdapter` chooses a profile from the PDO driver: | `sqlite` | SQLite | in-memory unit and round-trip suite | | `pgsql` | PostgreSQL | live PostgreSQL 17 and 18 SAV/ZSAV CI round trips | | `mysql` | MySQL/MariaDB | live MySQL 8.4/9.7 and MariaDB 11.4/11.8/12.3 SAV/ZSAV CI round trips | +| `mysql` | Dolt | live Dolt 2.2.2 SAV/ZSAV CI round trips; detected by `@@version_comment` plus `DOLT_VERSION()` | Every profile creates the same logical strict-wide layout and metadata catalogue. Physical SQL types, identifier limits and capability preflight are profile-specific. A source that cannot be represented must be rejected before an incomplete substitute is created. @@ -101,7 +102,7 @@ $adapter->migrateCatalog(); The command creates and versions the canonical catalogue through `openstatspec_schema_migration`; it also applies the write-format migration to -SQLite, MySQL/MariaDB and PostgreSQL compatibility catalogues, then backfills +SQLite, MySQL/MariaDB/Dolt and PostgreSQL compatibility catalogues, then backfills each exportable legacy dataset into the singular standard tables. A completely empty dedicated namespace is initialized automatically on its @@ -146,7 +147,7 @@ Verify `current_schema()` and `current_schemas(false)` on the exact connection passed to `SpssAdapter`. Do not change `search_path` while an import, export, or catalogue migration is running. -### MySQL and MariaDB +### MySQL, MariaDB and Dolt Create and select a dedicated database in the PDO DSN, for example `dbname=openstatspec`. Grant the adapter principal only the required privileges diff --git a/src/Core/CapabilityDeclaration.php b/src/Core/CapabilityDeclaration.php index cc144aa..e6bd703 100644 --- a/src/Core/CapabilityDeclaration.php +++ b/src/Core/CapabilityDeclaration.php @@ -7,6 +7,8 @@ use JsonSerializable; use OpenStatSpec\Spss\SpssEngine; use OpenStatSpec\Sql\CatalogOwnership; +use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Sql\DoltProfile; use OpenStatSpec\Sql\MySqlProfile; use OpenStatSpec\Sql\PdoSqlProfile; use OpenStatSpec\Sql\PostgreSqlProfile; @@ -17,16 +19,22 @@ final readonly class CapabilityDeclaration implements JsonSerializable { public const SPECIFICATION_RELEASE = null; - public const SPECIFICATION_COMMIT = '6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045'; + public const SPECIFICATION_COMMIT = '34141dda023d9e0217c37c232e39f436edfb0746'; - public function __construct(private PDO $pdo, private SpssEngine $engine) {} + private Connection $connection; + + public function __construct(PDO|Connection $connection, private SpssEngine $engine) + { + $this->connection = $connection instanceof Connection ? $connection : new Connection($connection); + } /** @return array */ public function toArray(): array { $mysql = new MySqlProfile(); - $serverVersion = (string) $this->pdo->getAttribute(PDO::ATTR_SERVER_VERSION); - $activeProfile = $this->activeProfile($serverVersion); + $pdo = $this->connection->pdo; + $serverVersion = $this->connection->serverVersion; + $activeProfile = $this->connection->profileName; $serverAssessment = ServerVersionPolicy::assess($activeProfile, $serverVersion); return [ @@ -49,14 +57,18 @@ public function toArray(): array 'active_connection' => [ 'profile' => $activeProfile, 'server_version' => $serverVersion, + 'raw_server_version' => $this->connection->rawServerVersion, + 'identity_source' => $this->connection->identitySource, + 'identity_probe_results' => $this->connection->identityProbeResults, 'claimed_supported' => $serverAssessment['claimed_supported'], 'matched_claim' => $serverAssessment['matched_claim'], - 'catalog_binding' => CatalogOwnership::binding($this->pdo), + 'catalog_binding' => CatalogOwnership::binding($pdo), ], 'sql_profiles' => [ 'sqlite' => $this->profile('sqlite', new SqliteProfile(), $activeProfile === 'sqlite'), 'mysql' => $this->profile('mysql', $mysql, $activeProfile === 'mysql'), 'mariadb' => $this->profile('mariadb', $mysql, $activeProfile === 'mariadb'), + 'dolt' => $this->profile('dolt', new DoltProfile(), $activeProfile === 'dolt'), 'postgresql' => $this->profile('postgresql', new PostgreSqlProfile(), $activeProfile === 'postgresql'), ], ]; @@ -72,29 +84,43 @@ public function jsonSerialize(): array private function profile(string $name, PdoSqlProfile $profile, bool $active): array { $identifierLimit = $this->identifierLimit($profile); - $theoretical = [ + $declared = [ 'maximum_physical_columns' => $profile->maximumSourceVariables() + 1, 'maximum_source_variables' => $profile->maximumSourceVariables(), 'identifier_limit' => $identifierLimit, 'maximum_value_bytes' => $profile->maximumValueBytes(), 'maximum_row_bytes' => $profile->maximumRowBytes(), ]; + $theoretical = $name === 'dolt' + ? ['maximum_value_bytes' => $profile->maximumValueBytes()] + : $declared; + $proposed = $name === 'dolt' ? [ + 'maximum_physical_columns' => $declared['maximum_physical_columns'], + 'maximum_source_variables' => $declared['maximum_source_variables'], + 'maximum_row_bytes' => $declared['maximum_row_bytes'], + ] : null; + $observedLimits = $name === 'dolt' ? [ + 'minimum_observed_physical_columns' => 307, + 'identifier_limit' => $identifierLimit, + 'rejected_identifier_bytes' => 65, + ] : null; + $effective = null; if ($active) { - $effectiveVariables = $profile->effectiveMaximumSourceVariables($this->pdo); + $effectiveVariables = $profile->effectiveMaximumSourceVariables($this->connection->pdo); $effective = [ 'maximum_physical_columns' => $effectiveVariables + 1, 'maximum_source_variables' => $effectiveVariables, 'identifier_limit' => $identifierLimit, - 'maximum_value_bytes' => $profile->effectiveMaximumValueBytes($this->pdo), - 'maximum_row_bytes' => $profile->effectiveMaximumRowBytes($this->pdo), - 'maximum_statement_bytes' => $profile->effectiveMaximumStatementBytes($this->pdo), - 'sources' => $profile->effectiveLimitSources($this->pdo), + 'maximum_value_bytes' => $profile->effectiveMaximumValueBytes($this->connection->pdo), + 'maximum_row_bytes' => $profile->effectiveMaximumRowBytes($this->connection->pdo), + 'maximum_statement_bytes' => $profile->effectiveMaximumStatementBytes($this->connection->pdo), + 'sources' => $profile->effectiveLimitSources($this->connection->pdo), ]; } $sources = $effective['sources'] ?? []; - $observed = array_filter( + $hasActiveObservation = array_filter( $sources, static fn(string $source): bool => str_contains($source, 'active '), ) !== []; @@ -104,25 +130,80 @@ private function profile(string $name, PdoSqlProfile $profile, bool $active): ar ) !== []; return [ + 'profile' => $name, + 'engine' => $name, + 'dialect' => $name === 'dolt' ? 'mysql' : $name, + 'transport' => $name === 'dolt' ? 'mysql_compatible' : $name, + 'specification_commit' => self::SPECIFICATION_COMMIT, + 'specification_status' => 'release_candidate', + 'specification_release' => self::SPECIFICATION_RELEASE, 'driver' => $name === 'postgresql' ? 'pgsql' : ($name === 'mariadb' ? 'mysql' : $profile->driverName()), + 'identity' => $name === 'dolt' ? [ + 'required_probes' => ['@@version', '@@version_comment', 'DOLT_VERSION()'], + 'version_comment_normalized_equals' => 'dolt', + 'signals_must_be_mutually_consistent' => true, + 'failure_policy' => 'fail_before_catalog_or_dataset_mutation', + 'active_probe_results' => $active ? $this->connection->identityProbeResults : null, + ] : null, 'claimed_server_versions' => ServerVersionPolicy::claim($name), + 'claimed_version_range' => $name === 'dolt' ? [ + 'minimum_inclusive' => '2.2.2', + 'maximum_inclusive' => '2.2.2', + ] : null, 'ci_tested_server_versions' => match ($name) { 'mysql' => ['MySQL 8.4.x', 'MySQL 9.7.x'], 'mariadb' => ['MariaDB 11.4.x', 'MariaDB 11.8.x', 'MariaDB 12.3.x'], + 'dolt' => ['Dolt 2.2.2'], 'postgresql' => ['PostgreSQL 17.x', 'PostgreSQL 18.x'], default => ['active PDO SQLite version reported by CI'], }, + 'exact_ci_tested_versions' => $name === 'dolt' ? ['2.2.2'] : null, 'theoretical_limits' => $theoretical, + 'proposed_adapter_limits' => $proposed, + 'observed_limits' => $observedLimits, 'effective_limits' => $effective, 'effective_limits_status' => !$active ? 'not_connected' - : ($observed ? 'active_connection_mixed' : ($compileTimeCeiling ? 'compile_time_ceiling' : 'profile_theoretical_fallback')), + : ($hasActiveObservation ? 'active_connection_mixed' : ($compileTimeCeiling ? 'compile_time_ceiling' : 'profile_theoretical_fallback')), 'numeric_type' => $profile->numericType(), 'text_type' => $profile->textType(), 'ddl_atomic' => $profile->ddlAtomic(), 'failure_cleanup' => $profile->ddlAtomic() ? 'transaction_rollback' : 'compensating_cleanup', + 'limit_bases' => [ + 'maximum_physical_columns' => $name === 'dolt' ? 'proposed_adapter_envelope' : 'theoretical_engine_limit', + 'maximum_source_variables' => $name === 'dolt' ? 'proposed_adapter_envelope' : 'theoretical_engine_limit', + 'identifier_limit' => $name === 'dolt' ? 'observed_exact_version' : 'theoretical_engine_limit', + 'maximum_value_bytes' => 'theoretical_engine_limit', + 'maximum_row_bytes' => $name === 'dolt' ? 'proposed_adapter_envelope' : 'theoretical_engine_limit', + 'maximum_statement_bytes' => 'active_connection_observation', + ], + 'numeric_exception_policy' => $name === 'dolt' ? [ + 'nan' => 'reject_before_mutation', + 'positive_infinity' => 'reject_before_mutation', + 'negative_infinity' => 'reject_before_mutation', + 'system_missing' => 'sql_null', + ] : null, + 'storage_evidence' => $name === 'dolt' ? [ + 'binary64' => [ + 'type' => 'DOUBLE', + 'classification' => 'observed_exact_version', + 'source' => 'Dolt 2.2.2 interoperability verification', + 'version' => '2.2.2', + 'maximum_finite_round_trip_exact' => true, + 'non_finite_policy' => 'reject_before_mutation', + ], + 'text' => [ + 'type' => 'LONGTEXT NOT NULL', + 'classification' => 'observed_exact_version', + 'source' => 'Dolt 2.2.2 interoperability verification', + 'version' => '2.2.2', + 'observed_value_bytes' => 65_504, + 'unit' => 'bytes', + ], + ] : null, + 'transformation_workflow' => $name === 'dolt' ? 'unsupported' : null, 'physical_table_mapping' => 'dataset.physical_table_schema + dataset.physical_table_name', - 'identifier_policy' => 'deterministic_safe_mapping; source name remains authoritative', + 'identifier_policy' => 'deterministic ASCII mapping; source name remains authoritative', ]; } @@ -136,15 +217,4 @@ private function identifierLimit(PdoSqlProfile $profile): array 'repertoire' => $profile->generatedIdentifierRepertoire(), ]; } - - private function activeProfile(string $serverVersion): string - { - $driver = (string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME); - return match ($driver) { - 'sqlite' => 'sqlite', - 'pgsql' => 'postgresql', - 'mysql' => stripos($serverVersion, 'mariadb') !== false ? 'mariadb' : 'mysql', - default => throw new UnsupportedOperation(DiagnosticCode::UnsupportedSqlDriver, 'The active connection has no capability profile.'), - }; - } } diff --git a/src/Core/ServerVersionPolicy.php b/src/Core/ServerVersionPolicy.php index 676dad9..63632bb 100644 --- a/src/Core/ServerVersionPolicy.php +++ b/src/Core/ServerVersionPolicy.php @@ -14,6 +14,7 @@ public static function assess(string $profile, string $serverVersion): array $supported = match ($profile) { 'mysql' => preg_match('/^(?:MySQL\s+)?(?:8\.4|9\.7)(?:\.|$)/i', trim($serverVersion)) === 1, 'mariadb' => preg_match('/^(?:MariaDB\s+)?(?:11\.4|11\.8|12\.3)(?:\.|$)/i', trim($serverVersion)) === 1, + 'dolt' => trim($serverVersion) === '2.2.2', 'postgresql' => preg_match('/^(?:PostgreSQL\s+)?(?:17|18)(?:\.|$)/i', trim($serverVersion)) === 1, 'sqlite' => self::supportsSqlite($serverVersion), default => false, @@ -39,6 +40,7 @@ public static function claim(string $profile): string return match ($profile) { 'mysql' => 'MySQL 8.4.x or 9.7.x', 'mariadb' => 'MariaDB 11.4.x, 11.8.x or 12.3.x', + 'dolt' => 'Dolt 2.2.2', 'postgresql' => 'PostgreSQL 17.x or 18.x', 'sqlite' => 'SQLite >=3.24.0 <4.0.0', default => 'unsupported', diff --git a/src/Spss/GuardedImportSpssEngine.php b/src/Spss/GuardedImportSpssEngine.php new file mode 100644 index 0000000..a93eb1a --- /dev/null +++ b/src/Spss/GuardedImportSpssEngine.php @@ -0,0 +1,153 @@ +format = $format; + $this->logicalPath = 'source.' . $format; + } + + /** @return array */ + public function identity(): array + { + try { + $identity = $this->inner->identity(); + $this->assertIdentitySafe($identity); + + return $identity; + } catch (Throwable) { + throw $this->sanitizedIdentityFailure(); + } + } + + /** @return array */ + public function capabilities(): array + { + return $this->inner->capabilities(); + } + + public function read(string $sourcePath): Dataset + { + if (!hash_equals($this->logicalPath, $sourcePath)) { + throw $this->sanitizedReadFailure(); + } + + try { + $dataset = $this->inner->read($this->physicalDescriptorPath); + if (!in_array($dataset->technicalMetadata->sourceFormat, ['sav', 'zsav'], true) + || !hash_equals($this->format, $dataset->technicalMetadata->sourceFormat) + ) { + throw $this->sanitizedReadFailure(); + } + + return $dataset; + } catch (Throwable) { + throw $this->sanitizedReadFailure(); + } + } + + public function write(string $targetPath, Dataset $dataset): void + { + $this->inner->write($targetPath, $dataset); + } + + public function logicalPath(): string + { + return $this->logicalPath; + } + + private function assertIdentitySafe(mixed $value, int $depth = 0): void + { + if ($depth > 32) { + throw $this->sanitizedIdentityFailure(); + } + if (is_string($value)) { + if (preg_match('//u', $value) !== 1 + || str_contains($value, $this->physicalDescriptorPath) + || preg_match( + '~(?:/proc/(?:self|thread-self|[0-9]+)/fd/[0-9]+|/dev/fd/[0-9]+)~', + $value, + ) === 1 + ) { + throw $this->sanitizedIdentityFailure(); + } + + return; + } + if (is_float($value)) { + if (!is_finite($value)) { + throw $this->sanitizedIdentityFailure(); + } + + return; + } + if (is_array($value)) { + foreach ($value as $key => $item) { + if (is_string($key)) { + $this->assertIdentitySafe($key, $depth + 1); + } + $this->assertIdentitySafe($item, $depth + 1); + } + + return; + } + if ($value !== null && !is_bool($value) && !is_int($value)) { + throw $this->sanitizedIdentityFailure(); + } + } + + private function sanitizedIdentityFailure(): UnsupportedOperation + { + return new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'The guarded SPSS engine identity is not safe for journaling.', + ); + } + + private function sanitizedReadFailure(): UnsupportedOperation + { + return new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'The guarded SPSS source could not be read for logical path ' . $this->logicalPath . '.', + ); + } +} diff --git a/src/Spss/SpssAdapter.php b/src/Spss/SpssAdapter.php index 01346b0..aec05a5 100644 --- a/src/Spss/SpssAdapter.php +++ b/src/Spss/SpssAdapter.php @@ -11,7 +11,9 @@ use OpenStatSpec\Sql\CanonicalCatalogProjection; use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Sql\MySqlWideTableDefinition; use OpenStatSpec\Sql\MySqlWideTableExporter; +use OpenStatSpec\Sql\MySqlProfile; use OpenStatSpec\Sql\MySqlSchema; use OpenStatSpec\Sql\MySqlWideTableImporter; use OpenStatSpec\Sql\NormativeCatalog; @@ -28,17 +30,24 @@ { private Connection $connection; private SpssEngine $engine; + private ?\Closure $beforeImportFinalization; - public function __construct(PDO $pdo, ?SpssEngine $engine = null) - { + public function __construct( + PDO $pdo, + ?SpssEngine $engine = null, + ?callable $beforeImportFinalization = null, + ) { $this->connection = new Connection($pdo); $this->engine = $engine ?? new PhpSpssEngine(); + $this->beforeImportFinalization = $beforeImportFinalization === null + ? null + : \Closure::fromCallable($beforeImportFinalization); } /** @return array */ public function capabilities(): array { - return (new CapabilityDeclaration($this->connection->pdo, $this->engine))->toArray(); + return (new CapabilityDeclaration($this->connection, $this->engine))->toArray(); } public function pdo(): PDO @@ -53,7 +62,7 @@ public function migrateCatalog(): void CatalogOwnership::ensure($this->connection->pdo); match ($this->connection->profile->driverName()) { 'pgsql' => (new PostgreSqlSchema($this->connection->pdo))->createCatalog(), - 'mysql' => (new MySqlSchema($this->connection->pdo))->createCatalog(), + 'mysql' => (new MySqlSchema($this->connection->pdo, $this->mySqlProfile()))->createCatalog(), default => (new SqliteWideTableImporter($this->connection->pdo))->migrateCatalog(), }; $catalog = new NormativeCatalog($this->connection->pdo); @@ -117,13 +126,20 @@ private function legacySourceFormat(string $datasetName): string return in_array($format, ['sav', 'zsav'], true) ? $format : 'sav'; } - public function import(string $sourcePath, string $datasetName): SpssImportResult - { + /** The optional provenance hash must already verify the bytes read by the engine. */ + public function import( + string $sourcePath, + string $datasetName, + ?string $verifiedSourceSha256 = null, + ): SpssImportResult { + $this->assertLogicalSourcePath($sourcePath); + $verifiedSourceSha256 = NormativeCatalog::validateSourceSha256($verifiedSourceSha256); $this->connection->assertClaimedSupported(); $this->ensureCatalogReady(); $sourceFormat = $this->spssFormat($sourcePath); $journal = new OperationJournal($this->connection->pdo); $operationId = $journal->start('import', null, $sourcePath, engineDetails: $this->engine->identity(), sourceFormat: $sourceFormat); + $mySqlDefinition = null; try { if (!in_array($sourceFormat, ['sav', 'zsav'], true)) { throw new UnsupportedOperation( @@ -132,18 +148,60 @@ public function import(string $sourcePath, string $datasetName): SpssImportResul ); } $source = SpssSourceNormalizer::normalize($this->engine->read($sourcePath)); - match ($this->connection->profile->driverName()) { - 'pgsql' => (new PostgreSqlWideTableImporter($this->connection->pdo))->import($source, $datasetName, $sourcePath), - 'mysql' => (new MySqlWideTableImporter($this->connection->pdo))->import($source, $datasetName, $sourcePath), - default => (new SqliteWideTableImporter($this->connection->pdo))->import($source, $datasetName, $sourcePath), - }; + if ($this->connection->profile->driverName() === 'pgsql') { + (new PostgreSqlWideTableImporter($this->connection->pdo))->import( + $source, + $datasetName, + $sourcePath, + $verifiedSourceSha256, + ); + } elseif ($this->connection->profile->driverName() === 'mysql') { + $mySqlDefinition = (new MySqlWideTableImporter( + $this->connection->pdo, + $this->mySqlProfile(), + ))->import($source, $datasetName, $sourcePath, $verifiedSourceSha256); + } else { + (new SqliteWideTableImporter($this->connection->pdo))->import( + $source, + $datasetName, + $sourcePath, + $verifiedSourceSha256, + ); + } + if ($this->beforeImportFinalization !== null) { + ($this->beforeImportFinalization)(); + } $diagnostics = []; $journal->succeed($operationId, $datasetName, $diagnostics); return new SpssImportResult($operationId, $datasetName, count($source['data']), $diagnostics); } catch (Throwable $exception) { - $journal->fail($operationId, null, $exception, sourceItem: $sourcePath); - throw $exception; + $failure = $exception; + if ($mySqlDefinition instanceof MySqlWideTableDefinition) { + try { + (new MySqlWideTableImporter( + $this->connection->pdo, + $this->mySqlProfile(), + ))->compensateFailure($datasetName, $mySqlDefinition); + } catch (Throwable $cleanupFailure) { + $failure = $cleanupFailure; + } + } + $journal->fail($operationId, null, $failure, sourceItem: $sourcePath); + throw $failure; + } + } + + private function assertLogicalSourcePath(string $sourcePath): void + { + if (preg_match( + '~\A(?:/proc/(?:self|thread-self|[0-9]+)/fd/[0-9]+|/dev/fd/[0-9]+)\z~', + $sourcePath, + ) === 1) { + throw new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'Ephemeral descriptor paths must remain internal to an injected SPSS engine; import requires a logical SAV or ZSAV source path.', + ); } } @@ -185,4 +243,17 @@ private function spssFormat(string $path): string { return strtolower(pathinfo($path, PATHINFO_EXTENSION)); } + + private function mySqlProfile(): MySqlProfile + { + $profile = $this->connection->profile; + if (!$profile instanceof MySqlProfile) { + throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + 'The active MySQL-family connection did not provide a MySQL-compatible SQL profile.', + ); + } + + return $profile; + } } diff --git a/src/Sql/AbstractPdoSqlProfile.php b/src/Sql/AbstractPdoSqlProfile.php index 1e51de6..52a5a5a 100644 --- a/src/Sql/AbstractPdoSqlProfile.php +++ b/src/Sql/AbstractPdoSqlProfile.php @@ -68,8 +68,10 @@ public function assertDataset(array $variables, array $rows, ?PDO $pdo = null): $maximumValueBytes = $pdo === null ? $this->maximumValueBytes() : $this->effectiveMaximumValueBytes($pdo); $maximumRowBytes = $pdo === null ? $this->maximumRowBytes() : $this->effectiveMaximumRowBytes($pdo); $declaredRowBytes = 8; + $storageKinds = []; foreach ($variables as $variable) { $kind = is_string($variable['type'] ?? null) && str_contains(strtolower($variable['type']), 'string') ? 'string' : 'numeric'; + $storageKinds[] = $kind; $width = $kind === 'string' && is_int($variable['width'] ?? null) ? $variable['width'] : 0; if ($width < 0 || $width > $maximumValueBytes) { $this->capabilityExceeded('declared string width', $width, $maximumValueBytes); @@ -80,19 +82,46 @@ public function assertDataset(array $variables, array $rows, ?PDO $pdo = null): $this->capabilityExceeded('declared row size', $declaredRowBytes, $maximumRowBytes); } $maximumStatementBytes = $pdo === null ? $this->maximumRowBytes() : $this->effectiveMaximumStatementBytes($pdo); + $maximumEncodedCaseBytes = min($maximumRowBytes, $maximumStatementBytes); foreach ($rows as $row) { if (count($row) !== count($variables)) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every source case must contain exactly one value per source variable.'); } - $encodedRowBytes = 8; - foreach ($row as $value) { + $encodedRowBytes = 0; + foreach ($row as $index => $value) { + $kind = $storageKinds[$index]; + if ($kind === 'string' && !is_string($value)) { + throw new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'SPSS string values must be non-null strings.', + ); + } + if ($kind === 'numeric' + && $value !== null + && !is_int($value) + && !is_float($value) + ) { + throw new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'SPSS numeric values must be binary64 numbers or system-missing NULL.', + ); + } + if ($kind === 'numeric' + && (is_int($value) || is_float($value)) + && !is_finite((float) $value) + ) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + $this->driverName() . ' rejects non-finite SPSS numeric values before mutation.', + ); + } if (is_string($value) && strlen($value) > $maximumValueBytes) { $this->capabilityExceeded('encoded string value', strlen($value), $maximumValueBytes); } $encodedRowBytes += is_string($value) ? strlen($value) : 8; } - if ($encodedRowBytes > $maximumStatementBytes) { - $this->capabilityExceeded('encoded case payload', $encodedRowBytes, $maximumStatementBytes); + if ($encodedRowBytes > $maximumEncodedCaseBytes) { + $this->capabilityExceeded('encoded case payload', $encodedRowBytes, $maximumEncodedCaseBytes); } } } diff --git a/src/Sql/CanonicalCatalogProjection.php b/src/Sql/CanonicalCatalogProjection.php index 2890dcb..92f235c 100644 --- a/src/Sql/CanonicalCatalogProjection.php +++ b/src/Sql/CanonicalCatalogProjection.php @@ -174,7 +174,7 @@ private function projectAttributes(string $datasetId, string $datasetName): void $file->execute([$datasetName, $row['attribute_name'], (int) $row['array_ordinal'], $row['attribute_value']]); } $variable = $this->statement('INSERT INTO variable_attributes (dataset_name, variable_ordinal, attribute_name, ordinal, value) VALUES (?, ?, ?, ?, ?)'); - foreach ($this->all('SELECT source.source_ordinal, attribute.attribute_name, attribute.array_ordinal, attribute.attribute_value FROM variable_attribute attribute JOIN variable source ON source.variable_id = attribute.variable_id WHERE source.dataset_id = ? ORDER BY source.source_ordinal, attribute.attribute_name, attribute.array_ordinal', [$datasetId]) as $row) { + foreach ($this->all('SELECT v.source_ordinal, va.attribute_name, va.array_ordinal, va.attribute_value FROM variable_attribute va JOIN variable v ON v.variable_id = va.variable_id WHERE v.dataset_id = ? ORDER BY v.source_ordinal, va.attribute_name, va.array_ordinal', [$datasetId]) as $row) { $variable->execute([$datasetName, (int) $row['source_ordinal'], $row['attribute_name'], (int) $row['array_ordinal'], $row['attribute_value']]); } } diff --git a/src/Sql/Connection.php b/src/Sql/Connection.php index 67fd717..0fd659f 100644 --- a/src/Sql/Connection.php +++ b/src/Sql/Connection.php @@ -14,33 +14,30 @@ public PdoSqlProfile $profile; public string $profileName; public string $serverVersion; + public string $rawServerVersion; + public string $identitySource; + /** @var array */ + public array $identityProbeResults; public bool $claimedSupported; public ?string $matchedClaim; public function __construct(public PDO $pdo) { - $driver = $pdo->getAttribute(PDO::ATTR_DRIVER_NAME); - if (!is_string($driver)) { - throw new UnsupportedOperation( - DiagnosticCode::UnsupportedSqlDriver, - 'The PDO connection did not report a usable SQL driver name.', - ); - } - - $this->serverVersion = (string) $pdo->getAttribute(PDO::ATTR_SERVER_VERSION); - $this->profileName = match ($driver) { - 'sqlite' => 'sqlite', - 'pgsql' => 'postgresql', - 'mysql' => stripos($this->serverVersion, 'mariadb') !== false ? 'mariadb' : 'mysql', - default => throw new UnsupportedOperation( - DiagnosticCode::UnsupportedSqlDriver, - sprintf('The PDO driver "%s" has no OpenStatSpec SQL profile.', $driver), - ), - }; + $identity = ServerIdentity::detect($pdo); + $this->profileName = $identity->profileName; + $this->serverVersion = $identity->serverVersion; + $this->rawServerVersion = $identity->rawServerVersion; + $this->identitySource = $identity->identitySource; + $this->identityProbeResults = $identity->probeResults; $this->profile = match ($this->profileName) { 'sqlite' => new SqliteProfile(), 'postgresql' => new PostgreSqlProfile(), 'mysql', 'mariadb' => new MySqlProfile(), + 'dolt' => new DoltProfile(), + default => throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + sprintf('The SQL profile "%s" has no OpenStatSpec implementation.', $this->profileName), + ), }; $assessment = ServerVersionPolicy::assess($this->profileName, $this->serverVersion); $this->claimedSupported = $assessment['claimed_supported']; diff --git a/src/Sql/DoltProfile.php b/src/Sql/DoltProfile.php new file mode 100644 index 0000000..5706c7b --- /dev/null +++ b/src/Sql/DoltProfile.php @@ -0,0 +1,52 @@ + 'Dolt 2.2.2 validated adapter envelope: 305 source variables (306 physical columns including __case_ordinal)', + 'maximum_value_bytes' => 'Dolt 2.2.2 LONGTEXT type limit constrained by ' . $packetSource, + 'maximum_row_bytes' => 'Dolt 2.2.2 65,504-byte tuple-data preflight; LONGTEXT uses a 20-byte out-of-band address', + 'maximum_statement_bytes' => $packetSource, + 'identifier_limit' => $this->identifierLimitSource(), + ]; + } +} diff --git a/src/Sql/MySqlIndexIntrospection.php b/src/Sql/MySqlIndexIntrospection.php new file mode 100644 index 0000000..a5fa9ae --- /dev/null +++ b/src/Sql/MySqlIndexIntrospection.php @@ -0,0 +1,30 @@ +> $rows + * @return array> + */ + public static function uniqueColumnLists(iterable $rows): array + { + $uniqueIndexes = []; + foreach ($rows as $row) { + $row = array_change_key_case($row, CASE_LOWER); + if ((int) ($row['non_unique'] ?? 1) !== 0 + || !is_string($row['index_name'] ?? null) + || !is_string($row['column_name'] ?? null) + ) { + continue; + } + $uniqueIndexes[$row['index_name']][] = $row['column_name']; + } + + return $uniqueIndexes; + } +} diff --git a/src/Sql/MySqlProfile.php b/src/Sql/MySqlProfile.php index 4a25ad1..b863c0b 100644 --- a/src/Sql/MySqlProfile.php +++ b/src/Sql/MySqlProfile.php @@ -6,7 +6,7 @@ use PDO; -final class MySqlProfile extends AbstractPdoSqlProfile +class MySqlProfile extends AbstractPdoSqlProfile { public function driverName(): string { diff --git a/src/Sql/MySqlSchema.php b/src/Sql/MySqlSchema.php index 4ff1a2b..dc6fd58 100644 --- a/src/Sql/MySqlSchema.php +++ b/src/Sql/MySqlSchema.php @@ -8,14 +8,14 @@ use OpenStatSpec\Core\UnsupportedOperation; use PDO; -/** MySQL/MariaDB DDL only; it does not claim SAV/ZSAV import/export support. */ +/** MySQL-family DDL only; it does not claim SAV/ZSAV import/export support. */ final readonly class MySqlSchema { private MySqlProfile $profile; - public function __construct(private PDO $pdo) + public function __construct(private PDO $pdo, ?MySqlProfile $profile = null) { - $this->profile = new MySqlProfile(); + $this->profile = $profile ?? new MySqlProfile(); } /** @return list */ diff --git a/src/Sql/MySqlWideTableDefinition.php b/src/Sql/MySqlWideTableDefinition.php index 6413cba..83a2df2 100644 --- a/src/Sql/MySqlWideTableDefinition.php +++ b/src/Sql/MySqlWideTableDefinition.php @@ -4,7 +4,7 @@ namespace OpenStatSpec\Sql; -/** A deterministic MySQL/MariaDB native-table DDL plan. */ +/** A deterministic MySQL-family native-table DDL plan. */ final readonly class MySqlWideTableDefinition { /** @@ -14,5 +14,6 @@ public function __construct( public string $tableName, public string $createSql, public array $columns, + public ?string $normativeDatasetId = null, ) {} } diff --git a/src/Sql/MySqlWideTableExporter.php b/src/Sql/MySqlWideTableExporter.php index 63c478b..24621dd 100644 --- a/src/Sql/MySqlWideTableExporter.php +++ b/src/Sql/MySqlWideTableExporter.php @@ -30,7 +30,7 @@ use SPSS\Sav\VariableSet; use SPSS\Sav\VariableType; -/** Reconstructs a complete V3 Dataset from the MySQL/MariaDB strict-wide catalogue. */ +/** Reconstructs a complete V3 Dataset from the MySQL-family strict-wide catalogue. */ final readonly class MySqlWideTableExporter { public function __construct(private PDO $pdo) {} @@ -41,7 +41,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array if (!in_array($targetFormat, ['sav', 'zsav'], true)) { throw new UnsupportedOperation( DiagnosticCode::UnsupportedSourceFormat, - 'The MySQL/MariaDB profile can only construct SAV or ZSAV datasets.', + 'The MySQL-family profile can only construct SAV or ZSAV datasets.', ); } @@ -57,7 +57,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array $rows = []; while (($row = $cases->fetch(PDO::FETCH_ASSOC)) !== false) { if (!is_array($row)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB wide-table reader returned an invalid case row.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family wide-table reader returned an invalid case row.'); } $values = []; foreach ($variables as $variable) { @@ -117,7 +117,7 @@ private function dataset(string $datasetName): array $statement->execute([$datasetName]); $dataset = $statement->fetch(PDO::FETCH_ASSOC); if (!is_array($dataset) || !is_string($dataset['table_name'] ?? null) || $dataset['table_name'] === '') { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The requested dataset is not present in the MySQL/MariaDB catalogue.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The requested dataset is not present in the MySQL-family catalogue.'); } return ['table_name' => $dataset['table_name']]; @@ -157,7 +157,7 @@ private function variables(string $datasetName): array || (!is_string($label) && $label !== null) || ($storageKind === 'string' && ($sourceWidth < 1 || $sourceWidth > 32767)) ) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB variable catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family variable catalogue is malformed.'); } $variables[] = [ @@ -208,7 +208,7 @@ private function dictionary(string $datasetName, int $ordinal): array $typedLabels = []; while (($row = $labels->fetch(PDO::FETCH_ASSOC)) !== false) { if (!is_array($row) || !is_string($row['label'] ?? null)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB value-label catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family value-label catalogue is malformed.'); } $typedLabels[] = new ValueLabel($this->dictionaryValue($row), $row['label']); } @@ -220,7 +220,7 @@ private function dictionary(string $datasetName, int $ordinal): array return ['labels' => $typedLabels, 'missing' => MissingValues::none()]; } if ($format < -3 || ($format < 0 && $format !== -2 && $format !== -3) || $format > 3) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB user-missing rule has an unsupported SPSS missing format.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing rule has an unsupported SPSS missing format.'); } $valuesStatement = $this->statement('SELECT value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); @@ -228,14 +228,14 @@ private function dictionary(string $datasetName, int $ordinal): array $values = []; while (($row = $valuesStatement->fetch(PDO::FETCH_ASSOC)) !== false) { if (!is_array($row)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB user-missing value catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing value catalogue is malformed.'); } $values[] = $this->dictionaryValue($row); } if ($format === -2) { if (count($values) !== 2) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB user-missing rule has an incomplete ordered value list.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing rule has an incomplete ordered value list.'); } return ['labels' => $typedLabels, 'missing' => MissingValues::range( @@ -245,7 +245,7 @@ private function dictionary(string $datasetName, int $ordinal): array } if ($format === -3) { if (count($values) !== 3) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB user-missing rule has an incomplete ordered value list.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing rule has an incomplete ordered value list.'); } return ['labels' => $typedLabels, 'missing' => MissingValues::rangeAndValue( @@ -255,7 +255,7 @@ private function dictionary(string $datasetName, int $ordinal): array )]; } if (count($values) !== $format) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB user-missing rule has an incomplete ordered value list.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing rule has an incomplete ordered value list.'); } return ['labels' => $typedLabels, 'missing' => MissingValues::discrete(...$values)]; @@ -279,7 +279,7 @@ private function dictionaryValue(array $row): int|float|string } } - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB dictionary contains an invalid typed value.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family dictionary contains an invalid typed value.'); } private function numeric(int|float|string $value): int|float @@ -300,7 +300,7 @@ private function role(string $datasetName, int $ordinal): VariableRole $statement->execute([$datasetName, $ordinal]); $role = $this->integer($statement->fetchColumn()); if ($role === null || ($typed = VariableRole::tryFrom($role)) === null) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB variable role catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family variable role catalogue is malformed.'); } return $typed; @@ -343,7 +343,7 @@ private function attributeValues(PDOStatement $statement): array $name = $row['attribute_name'] ?? null; $value = $row['value'] ?? null; if (!is_string($name) || $name === '' || !is_string($value)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB attribute catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family attribute catalogue is malformed.'); } $grouped[$name][] = $value; } @@ -362,14 +362,14 @@ private function variableSets(string $datasetName): array $ordinal = $this->integer($set['set_ordinal'] ?? null); $name = $set['name'] ?? null; if ($ordinal === null || !is_string($name) || $name === '') { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB variable-set catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family variable-set catalogue is malformed.'); } $members->execute([$datasetName, $ordinal]); $names = []; while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL/MariaDB variable set references an unknown variable.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL-family variable set references an unknown variable.'); } $names[] = $sourceName; } @@ -394,7 +394,7 @@ private function multipleResponseSets(string $datasetName): array $labelSource = is_string($set['label_source'] ?? null) ? MultipleResponseLabelSource::tryFrom($set['label_source']) : null; $label = $set['label'] ?? null; if ($ordinal === null || !is_string($name) || $name === '' || $type === null || $categoryLabels === null || $labelSource === null || ($label !== null && !is_string($label))) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB multiple-response-set catalogue is malformed.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family multiple-response-set catalogue is malformed.'); } $countedValue = $this->countedValue($set); $members->execute([$datasetName, $ordinal]); @@ -402,14 +402,14 @@ private function multipleResponseSets(string $datasetName): array while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL/MariaDB multiple-response set references an unknown variable.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL-family multiple-response set references an unknown variable.'); } $names[] = $sourceName; } try { $result[] = new MultipleResponseSet($name, $type, $names, $label, $countedValue, $categoryLabels, $labelSource); } catch (\InvalidArgumentException $exception) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB multiple-response-set catalogue is inconsistent: ' . $exception->getMessage()); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family multiple-response-set catalogue is inconsistent: ' . $exception->getMessage()); } } @@ -431,7 +431,7 @@ private function countedValue(array $set): int|string|null return (int) $value; } - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL/MariaDB multiple-response set has an invalid counted value.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL-family multiple-response set has an invalid counted value.'); } private function fileLabel(string $datasetName): ?string @@ -455,7 +455,7 @@ private function weightVariableName(string $datasetName): ?string return null; } if (!is_string($name) || $name === '') { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB weight-variable catalogue references an unknown source variable.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family weight-variable catalogue references an unknown source variable.'); } return $name; @@ -496,7 +496,7 @@ private function caseValue(mixed $value, string $storageKind): int|float|string| { if ($storageKind === 'string') { if (!is_string($value)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB wide table contains a null or non-string SPSS string value.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family wide table contains a null or non-string SPSS string value.'); } return $value; @@ -508,7 +508,7 @@ private function caseValue(mixed $value, string $storageKind): int|float|string| return (float) $value; } - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB wide table contains a non-numeric SPSS numeric value.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family wide table contains a non-numeric SPSS numeric value.'); } private function integer(mixed $value): ?int @@ -520,7 +520,7 @@ private function statement(string $sql): PDOStatement { $statement = $this->pdo->prepare($sql); if ($statement === false) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL/MariaDB profile could not prepare a required catalogue query.'); + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family profile could not prepare a required catalogue query.'); } return $statement; diff --git a/src/Sql/MySqlWideTableImporter.php b/src/Sql/MySqlWideTableImporter.php index ab10537..d43be05 100644 --- a/src/Sql/MySqlWideTableImporter.php +++ b/src/Sql/MySqlWideTableImporter.php @@ -12,19 +12,29 @@ use Throwable; /** - * MySQL/MariaDB importer for the strict one-source-dataset/one-wide-table + * MySQL-family importer for the strict one-source-dataset/one-wide-table * contract. MySQL DDL commits implicitly, so a failure after DDL is handled * with best-effort compensating cleanup rather than a false atomicity claim. */ final readonly class MySqlWideTableImporter { - public function __construct(private PDO $pdo) {} + private MySqlProfile $profile; + + public function __construct(private PDO $pdo, ?MySqlProfile $profile = null) + { + $this->profile = $profile ?? new MySqlProfile(); + } /** * @param array $source */ - public function import(array $source, string $datasetName, string $sourcePath = ""): MySqlWideTableDefinition - { + public function import( + array $source, + string $datasetName, + string $sourcePath = "", + ?string $verifiedSourceSha256 = null, + ): MySqlWideTableDefinition { + $verifiedSourceSha256 = NormativeCatalog::validateSourceSha256($verifiedSourceSha256); $variables = $source['variables'] ?? null; $rows = $source['data'] ?? null; if (!is_array($variables) || !array_is_list($variables) || $variables === []) { @@ -40,15 +50,17 @@ public function import(array $source, string $datasetName, string $sourcePath = ); } - (new MySqlProfile())->assertDataset($variables, $rows, $this->pdo); + $this->profile->assertDataset($variables, $rows, $this->pdo); - $schema = new MySqlSchema($this->pdo); - // Complete physical-name and width preflight happens before any DDL. + $schema = new MySqlSchema($this->pdo, $this->profile); + // Complete source, physical-name, and width preflight happens before any DDL. $definition = $schema->wideTableDefinition($datasetName, $variables); + $v3Metadata = $this->assertSourceMetadata($source, $variables, $definition); $schema->createCatalog(); $this->pdo->exec($definition->createSql); + $ownedDefinition = $definition; try { $this->pdo->beginTransaction(); $this->storeDatasetMetadata($datasetName, $source); @@ -57,12 +69,23 @@ public function import(array $source, string $datasetName, string $sourcePath = $this->storeWeightVariable($datasetName, $source['weightVariableName'] ?? null, $definition); $this->storeDisplayMetadata($datasetName, $source['displayParameters'] ?? []); $this->storeDictionaryMetadata($datasetName, $variables, $source['valueLabels'] ?? []); - if (is_array($variables[0] ?? null) && array_key_exists('role', $variables[0])) { - (new SqliteV3MetadataImporter($this->pdo))->store($datasetName, $source); + if ($v3Metadata !== null) { + (new SqliteV3MetadataImporter($this->pdo))->storeValidated($datasetName, $v3Metadata); } $this->insertCases($definition, $rows); - if ($sourcePath !== "") { - (new NormativeCatalog($this->pdo))->storeImportedDataset($datasetName, $sourcePath, $source); + if ($sourcePath !== "" || $verifiedSourceSha256 !== null) { + $datasetId = (new NormativeCatalog($this->pdo))->storeImportedDataset( + $datasetName, + $sourcePath, + $source, + $verifiedSourceSha256, + ); + $ownedDefinition = new MySqlWideTableDefinition( + $definition->tableName, + $definition->createSql, + $definition->columns, + $datasetId, + ); } $this->pdo->commit(); } catch (Throwable $exception) { @@ -70,13 +93,92 @@ public function import(array $source, string $datasetName, string $sourcePath = $this->pdo->rollBack(); } - // CREATE TABLE has already committed on MySQL/MariaDB. Remove the - // target-specific artefacts that this invocation could have left. - $this->compensateFailure($datasetName, $definition); + // Catalogue DML was rolled back. Only the physical DDL survives. + $this->dropPhysicalTable($definition); throw $exception; } - return $definition; + return $ownedDefinition; + } + + /** + * @param array $source + * @param list> $variables + * @return V3MetadataPlan|null + */ + private function assertSourceMetadata( + array $source, + array $variables, + MySqlWideTableDefinition $definition, + ): ?V3MetadataPlan { + $weightVariableName = $source['weightVariableName'] ?? null; + if ($weightVariableName !== null) { + if (!is_string($weightVariableName) || $weightVariableName === '') { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The SPSS weight-variable reference must be a non-empty source variable name.'); + } + if (!in_array($weightVariableName, array_column($definition->columns, 'sourceName'), true)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The SPSS weight-variable reference must name a source variable.'); + } + } + + $technical = $source['technicalMetadata'] ?? null; + if ($technical !== null + && (!is_array($technical) + || !is_string($technical['sourceFormat'] ?? null) + || $technical['sourceFormat'] === '' + || !is_string($technical['encoding'] ?? null) + || $technical['encoding'] === '') + ) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'V3 technical metadata requires a non-empty source format and encoding.'); + } + + foreach ($variables as $variable) { + $this->writeFormatField($variable, 'writeFormatFamily', 5); + $this->writeFormatField($variable, 'writeFormatWidth', 8); + $this->writeFormatField($variable, 'writeFormatDecimals', 0); + $format = $variable['missingFormat'] ?? 0; + if (!is_int($format)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS missing-value format must be an integer.'); + } + if ($format === 0) { + continue; + } + $values = $variable['missingValues'] ?? []; + if (!is_array($values) || !array_is_list($values)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS user-missing values must be an ordered list.'); + } + foreach ($values as $value) { + $this->dictionaryValue($value); + } + } + + $records = $source['valueLabels'] ?? []; + if (!is_array($records) || !array_is_list($records)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS value-label records must be an ordered list.'); + } + foreach ($records as $record) { + if (!is_array($record) + || !is_array($record['indexes'] ?? null) + || !array_is_list($record['indexes']) + || !is_array($record['labels'] ?? null) + || !array_is_list($record['labels']) + ) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS value-label records must contain ordered indexes and labels.'); + } + foreach ($record['indexes'] as $index) { + if (!is_int($index) || !array_key_exists($index, $variables)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS value-label indexes must refer to a source variable.'); + } + } + foreach ($record['labels'] as $label) { + if (!is_array($label) || !is_string($label['label'] ?? null) || !array_key_exists('value', $label)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every SPSS value label must contain a typed value and string label.'); + } + $this->dictionaryValue($label['value']); + } + } + + return V3MetadataPlan::fromSourceIfPresent($source); } /** @@ -334,6 +436,13 @@ private function dictionaryValue(mixed $value): array return ['text', null, $value]; } if (is_int($value) || is_float($value)) { + if (!is_finite((float) $value)) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + 'The Dolt profile rejects non-finite SPSS numeric values before mutation.', + ); + } + return ['numeric', Binary64::encode($value), null]; } throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS dictionary values must be strings or binary64 numbers.'); @@ -354,12 +463,18 @@ private function technicalFloat(mixed $value): ?float return is_float($value) || is_int($value) ? (float) $value : null; } - private function compensateFailure(string $datasetName, MySqlWideTableDefinition $definition): void + public function compensateFailure(string $datasetName, MySqlWideTableDefinition $definition): void { try { - // MySQL DDL implicitly commits. Delete all catalogue rows that may - // have been inserted before the failing statement, in dependency - // order, then remove the dedicated wide table. + $datasetId = $definition->normativeDatasetId; + if ($datasetId === null) { + throw new \RuntimeException('The import attempt has no normative dataset ownership token.'); + } + $this->assertOwnedNormativeDataset($datasetId, $datasetName, $definition->tableName); + $this->removeNormativeDataset($datasetId); + + // This exact normative dataset ID proves that this attempt created + // the matching compatibility rows and physical relation. foreach ([ 'multiple_response_set_members', 'multiple_response_sets', @@ -379,30 +494,89 @@ private function compensateFailure(string $datasetName, MySqlWideTableDefinition 'variables', 'datasets', ] as $table) { - $statement = $this->pdo->prepare('DELETE FROM ' . $table . ' WHERE dataset_name = ?'); + $sql = 'DELETE FROM ' . $table . ' WHERE dataset_name = ?'; + if ($table === 'datasets') { + $sql .= ' AND table_name = ?'; + } + $statement = $this->pdo->prepare($sql); if ($statement !== false) { - $statement->execute([$datasetName]); + $statement->execute($table === 'datasets' + ? [$datasetName, $definition->tableName] + : [$datasetName]); } } - $quote = chr(96); - $this->pdo->exec( - 'DROP TABLE IF EXISTS ' . $quote . str_replace($quote, $quote . $quote, $definition->tableName) . $quote, - ); + $this->dropPhysicalTable($definition); } catch (Throwable $cleanupFailure) { throw new \RuntimeException( - "MySQL/MariaDB import cleanup failed; the target may require manual inspection: " . $definition->tableName, + 'MySQL-family import cleanup failed; the target may require manual inspection: ' . $definition->tableName, previous: $cleanupFailure, ); } } + private function assertOwnedNormativeDataset( + string $datasetId, + string $datasetName, + string $tableName, + ): void { + $dataset = $this->pdo->prepare( + 'SELECT dataset_name, physical_table_name FROM dataset WHERE dataset_id = ?', + ); + if ($dataset === false) { + throw new \RuntimeException('Could not verify normative dataset ownership.'); + } + $dataset->execute([$datasetId]); + $rows = $dataset->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) !== 1 + || ($rows[0]['dataset_name'] ?? null) !== $datasetName + || ($rows[0]['physical_table_name'] ?? null) !== $tableName + ) { + throw new \RuntimeException('The normative dataset ownership token does not match this import attempt.'); + } + } + + private function removeNormativeDataset(string $datasetId): void + { + foreach ([ + 'UPDATE fidelity_event SET dataset_id = NULL WHERE dataset_id = ?', + 'DELETE FROM multiple_response_member WHERE multiple_response_set_id IN (SELECT multiple_response_set_id FROM multiple_response_set WHERE dataset_id = ?)', + 'DELETE FROM multiple_response_set WHERE dataset_id = ?', + 'DELETE FROM variable_set_member WHERE variable_set_id IN (SELECT variable_set_id FROM variable_set WHERE dataset_id = ?)', + 'DELETE FROM variable_set WHERE dataset_id = ?', + 'DELETE FROM missing_rule WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)', + 'DELETE FROM variable_value_label_set WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)', + 'DELETE FROM value_label WHERE value_label_set_id IN (SELECT value_label_set_id FROM value_label_set WHERE dataset_id = ?)', + 'DELETE FROM value_label_set WHERE dataset_id = ?', + 'DELETE FROM variable_attribute WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)', + 'DELETE FROM dataset_attribute WHERE dataset_id = ?', + 'DELETE FROM document WHERE dataset_id = ?', + 'DELETE FROM dataset_weight_variable WHERE dataset_id = ?', + 'DELETE FROM variable WHERE dataset_id = ?', + 'DELETE FROM dataset WHERE dataset_id = ?', + ] as $sql) { + $statement = $this->pdo->prepare($sql); + if ($statement === false) { + throw new \RuntimeException('Could not prepare normative Dolt cleanup.'); + } + $statement->execute([$datasetId]); + } + } + + private function dropPhysicalTable(MySqlWideTableDefinition $definition): void + { + $quote = chr(96); + $this->pdo->exec( + 'DROP TABLE IF EXISTS ' . $quote . str_replace($quote, $quote . $quote, $definition->tableName) . $quote, + ); + } + private function requiredStatement(string $sql, string $description): PDOStatement { $statement = $this->pdo->prepare($sql); if ($statement === false) { throw new UnsupportedOperation( DiagnosticCode::InvalidSourceDataset, - 'The MySQL/MariaDB profile could not prepare the ' . $description . ' statement.', + 'The MySQL-family profile could not prepare the ' . $description . ' statement.', ); } @@ -459,6 +633,12 @@ private function caseValue(mixed $value, string $storageKind): int|string|null 'SPSS numeric values must be binary64 numbers or system-missing NULL.', ); } + if (!is_finite($value)) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + 'The Dolt profile rejects non-finite SPSS numeric values before mutation.', + ); + } // A 17-digit decimal representation round-trips every IEEE-754 binary64. return sprintf('%.17g', $value); diff --git a/src/Sql/NormativeCatalog.php b/src/Sql/NormativeCatalog.php index d000cd1..6fdd8ed 100644 --- a/src/Sql/NormativeCatalog.php +++ b/src/Sql/NormativeCatalog.php @@ -95,10 +95,47 @@ public function hasDataset(string $datasetName): bool return $statement->fetchColumn() !== false; } + public static function validateSourceSha256(?string $verifiedSourceSha256): ?string + { + if ($verifiedSourceSha256 !== null + && preg_match('/\A[0-9a-f]{64}\z/', $verifiedSourceSha256) !== 1 + ) { + throw new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'The verified source SHA-256 must be exactly 64 lowercase hexadecimal characters.', + ); + } - /** @param array $source */ - public function storeImportedDataset(string $datasetName, string $sourcePath, array $source): void + return $verifiedSourceSha256; + } + + private function sourceHash(string $sourcePath, ?string $verifiedSourceSha256): ?string { + if ($verifiedSourceSha256 !== null) { + return self::validateSourceSha256($verifiedSourceSha256); + } + if (!is_file($sourcePath)) { + return null; + } + $sourceSha256 = hash_file('sha256', $sourcePath); + if (!is_string($sourceSha256)) { + throw new UnsupportedOperation( + DiagnosticCode::InvalidSourceDataset, + 'The source file SHA-256 could not be calculated.', + ); + } + + return self::validateSourceSha256($sourceSha256); + } + + /** @param array $source */ + public function storeImportedDataset( + string $datasetName, + string $sourcePath, + array $source, + ?string $verifiedSourceSha256 = null, + ): string { + $verifiedSourceSha256 = self::validateSourceSha256($verifiedSourceSha256); $this->createTables(); $mappings = $this->physicalVariables($datasetName); /** @var list $variables */ @@ -123,7 +160,7 @@ public function storeImportedDataset(string $datasetName, string $sourcePath, ar $datasetName, is_string($source['fileLabel'] ?? null) ? $source['fileLabel'] : null, is_string($technical['encoding'] ?? null) ? $technical['encoding'] : null, - is_file($sourcePath) ? hash_file('sha256', $sourcePath) : null, + $this->sourceHash($sourcePath, $verifiedSourceSha256), count($this->list($source['data'] ?? null, 'Source cases')), self::timestamp(), ]); @@ -165,6 +202,8 @@ public function storeImportedDataset(string $datasetName, string $sourcePath, ar $this->storeDocuments($datasetId, $this->list($source['documents'] ?? [], 'Documents')); $this->storeVariableSets($datasetId, $variableIds, $this->list($source['variableSets'] ?? [], 'Variable sets'), $variables); $this->storeMultipleResponseSets($datasetId, $variableIds, $this->list($source['multipleResponseSets'] ?? [], 'Multiple-response sets'), $variables); + + return $datasetId; } private function physicalTableName(string $datasetName): string @@ -483,9 +522,9 @@ private function enforceSetOrdinalConstraints(string $driver): void return; } if ($driver === 'mysql') { - // MySQL/MariaDB DDL is not transactional. Every statement is - // idempotent and migration version 3 is recorded only after all - // constraints have been restored successfully. + // The MySQL-family compatibility path treats DDL as non-transactional. + // Every statement is idempotent and migration version 3 is recorded only + // after all constraints have been restored successfully. $this->pdo->exec('ALTER TABLE variable_set MODIFY source_ordinal INTEGER NOT NULL'); $this->pdo->exec('ALTER TABLE multiple_response_set MODIFY source_ordinal INTEGER NOT NULL'); $this->ensureMySqlUniqueIndex('variable_set', 'uq_variable_set_dataset_ordinal'); @@ -583,11 +622,14 @@ private function ensurePostgreSqlUniqueIndex(string $table, string $index): void private function ensureMySqlUniqueIndex(string $table, string $index): void { - $statement = $this->statement("SELECT index_name FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? GROUP BY index_name HAVING MIN(non_unique) = 0 AND GROUP_CONCAT(column_name ORDER BY seq_in_index SEPARATOR ',') = 'dataset_id,source_ordinal' LIMIT 1"); + $statement = $this->statement('SELECT index_name, non_unique, column_name FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? ORDER BY index_name, seq_in_index'); $statement->execute([$table]); - if ($statement->fetchColumn() === false) { - $this->pdo->exec('CREATE UNIQUE INDEX ' . $index . ' ON ' . $table . ' (dataset_id, source_ordinal)'); + $uniqueIndexes = MySqlIndexIntrospection::uniqueColumnLists($statement->fetchAll(PDO::FETCH_ASSOC)); + if (in_array(['dataset_id', 'source_ordinal'], $uniqueIndexes, true)) { + return; } + + $this->pdo->exec('CREATE UNIQUE INDEX ' . $index . ' ON ' . $table . ' (dataset_id, source_ordinal)'); } private function ensureColumn(string $table, string $column, string $definition): void diff --git a/src/Sql/PostgreSqlWideTableImporter.php b/src/Sql/PostgreSqlWideTableImporter.php index 36beff7..0ea9f7a 100644 --- a/src/Sql/PostgreSqlWideTableImporter.php +++ b/src/Sql/PostgreSqlWideTableImporter.php @@ -32,6 +32,8 @@ public function createTables(array $source, string $datasetName): PostgreSqlWide ); } + V3MetadataPlan::fromSourceIfPresent($source); + $schema = new PostgreSqlSchema($this->pdo); // Validate the complete physical mapping before any DDL starts. $definition = $schema->wideTableDefinition($datasetName, $variables); @@ -57,8 +59,13 @@ public function createTables(array $source, string $datasetName): PostgreSqlWide * * @param array $source */ - public function import(array $source, string $datasetName, string $sourcePath = ""): PostgreSqlWideTableDefinition - { + public function import( + array $source, + string $datasetName, + string $sourcePath = "", + ?string $verifiedSourceSha256 = null, + ): PostgreSqlWideTableDefinition { + $verifiedSourceSha256 = NormativeCatalog::validateSourceSha256($verifiedSourceSha256); $variables = $source['variables'] ?? null; $rows = $source['data'] ?? null; if (!is_array($variables) || !array_is_list($variables) || $variables === []) { @@ -69,6 +76,7 @@ public function import(array $source, string $datasetName, string $sourcePath = } (new PostgreSqlProfile())->assertDataset($variables, $rows, $this->pdo); + $v3Metadata = V3MetadataPlan::fromSourceIfPresent($source); $schema = new PostgreSqlSchema($this->pdo); // Preflight the entire physical-name mapping before changing the target. @@ -84,12 +92,17 @@ public function import(array $source, string $datasetName, string $sourcePath = $this->storeWeightVariable($datasetName, $source['weightVariableName'] ?? null, $definition); $this->storeDisplayMetadata($datasetName, $source['displayParameters'] ?? []); $this->storeDictionaryMetadata($datasetName, $variables, $source['valueLabels'] ?? []); - if (is_array($variables[0] ?? null) && array_key_exists('role', $variables[0])) { - (new SqliteV3MetadataImporter($this->pdo))->store($datasetName, $source); + if ($v3Metadata !== null) { + (new SqliteV3MetadataImporter($this->pdo))->storeValidated($datasetName, $v3Metadata); } $this->insertCases($definition, $rows); - if ($sourcePath !== "") { - (new NormativeCatalog($this->pdo))->storeImportedDataset($datasetName, $sourcePath, $source); + if ($sourcePath !== "" || $verifiedSourceSha256 !== null) { + (new NormativeCatalog($this->pdo))->storeImportedDataset( + $datasetName, + $sourcePath, + $source, + $verifiedSourceSha256, + ); } $this->pdo->commit(); } catch (Throwable $exception) { diff --git a/src/Sql/ServerIdentity.php b/src/Sql/ServerIdentity.php new file mode 100644 index 0000000..66e6b15 --- /dev/null +++ b/src/Sql/ServerIdentity.php @@ -0,0 +1,156 @@ + $probeResults */ + private function __construct( + public string $driverName, + public string $profileName, + public string $serverVersion, + public string $rawServerVersion, + public string $identitySource, + public array $probeResults, + ) {} + + public static function detect(PDO $pdo): self + { + $driver = $pdo->getAttribute(PDO::ATTR_DRIVER_NAME); + if (!is_string($driver)) { + throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + 'The PDO connection did not report a usable SQL driver name.', + ); + } + + $rawServerVersion = (string) $pdo->getAttribute(PDO::ATTR_SERVER_VERSION); + if ($driver === 'mysql') { + return self::detectMySqlFamily($pdo, $rawServerVersion); + } + + $profileName = match ($driver) { + 'sqlite' => 'sqlite', + 'pgsql' => 'postgresql', + default => throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + sprintf('The PDO driver "%s" has no OpenStatSpec SQL profile.', $driver), + ), + }; + + return new self( + $driver, + $profileName, + $rawServerVersion, + $rawServerVersion, + 'PDO::ATTR_SERVER_VERSION', + ['PDO::ATTR_SERVER_VERSION' => $rawServerVersion], + ); + } + + private static function detectMySqlFamily(PDO $pdo, string $rawServerVersion): self + { + $version = self::requiredScalar($pdo, 'SELECT @@version', '@@version'); + $versionComment = self::scalar($pdo, 'SELECT @@version_comment'); + if ($versionComment === null || trim($versionComment) === '') { + throw self::identityFailure('The MySQL-compatible server did not provide a non-empty @@version_comment.'); + } + + $version = trim($version); + $versionComment = trim($versionComment); + $normalizedVersion = strtolower($version); + $normalizedComment = strtolower($versionComment); + if ($normalizedComment === 'dolt') { + $doltVersion = self::scalar($pdo, 'SELECT DOLT_VERSION()'); + if ($doltVersion === null || trim($doltVersion) === '') { + throw self::identityFailure( + 'Dolt identity requires exact @@version_comment = Dolt and a non-empty DOLT_VERSION() result.', + ); + } + + return new self( + 'mysql', + 'dolt', + trim($doltVersion), + $rawServerVersion, + '@@version + @@version_comment + DOLT_VERSION()', + [ + 'PDO::ATTR_SERVER_VERSION' => $rawServerVersion, + '@@version' => $version, + '@@version_comment' => $versionComment, + 'DOLT_VERSION()' => trim($doltVersion), + ], + ); + } + + $versionClaimsMariaDb = str_contains($normalizedVersion, 'mariadb'); + $commentClaimsMariaDb = str_contains($normalizedComment, 'mariadb'); + $commentClaimsMySql = str_contains($normalizedComment, 'mysql'); + if ($versionClaimsMariaDb !== $commentClaimsMariaDb + || ($commentClaimsMariaDb && $commentClaimsMySql) + ) { + throw self::identityFailure( + 'The MySQL-compatible server returned conflicting product identity signals.', + ); + } + + $profileName = match (true) { + $versionClaimsMariaDb && $commentClaimsMariaDb => 'mariadb', + !$versionClaimsMariaDb && $commentClaimsMySql => 'mysql', + default => throw self::identityFailure( + 'The MySQL-compatible server product is unknown; only claimed MySQL, MariaDB, and Dolt identities are supported.', + ), + }; + + return new self( + 'mysql', + $profileName, + $version, + $rawServerVersion, + '@@version + @@version_comment', + [ + 'PDO::ATTR_SERVER_VERSION' => $rawServerVersion, + '@@version' => $version, + '@@version_comment' => $versionComment, + 'DOLT_VERSION()' => null, + ], + ); + } + + private static function requiredScalar(PDO $pdo, string $sql, string $probeName): string + { + $value = self::scalar($pdo, $sql); + if ($value === null || trim($value) === '') { + throw self::identityFailure( + sprintf('The MySQL-compatible server did not provide a non-empty %s.', $probeName), + ); + } + + return $value; + } + + private static function identityFailure(string $message): UnsupportedOperation + { + return new UnsupportedOperation(DiagnosticCode::TargetCapabilityExceeded, $message); + } + + private static function scalar(PDO $pdo, string $sql): ?string + { + try { + $statement = $pdo->query($sql); + $value = $statement === false ? false : $statement->fetchColumn(); + } catch (PDOException) { + return null; + } + + return is_string($value) || is_int($value) ? (string) $value : null; + } +} diff --git a/src/Sql/SqliteV3MetadataImporter.php b/src/Sql/SqliteV3MetadataImporter.php index 697c8f2..f1222db 100644 --- a/src/Sql/SqliteV3MetadataImporter.php +++ b/src/Sql/SqliteV3MetadataImporter.php @@ -4,11 +4,9 @@ namespace OpenStatSpec\Sql; -use OpenStatSpec\Core\DiagnosticCode; -use OpenStatSpec\Core\UnsupportedOperation; use PDO; -/** Persists V3 SPSS metadata that belongs beside the strict wide data table. */ +/** Persists canonically validated V3 SPSS metadata beside a strict wide data table. */ final readonly class SqliteV3MetadataImporter { public function __construct(private PDO $pdo) {} @@ -27,169 +25,84 @@ public static function createTables(PDO $pdo): void /** @param array $source */ public function store(string $datasetName, array $source): void { - $variables = $source['variables'] ?? null; - if (!is_array($variables)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'V3 metadata requires a source variable list.'); - } + $this->storeValidated($datasetName, V3MetadataPlan::fromSource($source)); + } - $this->storeRoles($datasetName, $variables); - $this->storeVariableAttributes($datasetName, $variables); - $this->storeFileAttributes($datasetName, $this->list($source['fileAttributes'] ?? [], 'File attributes')); - $ordinals = $this->variableOrdinals($variables); - $this->storeVariableSets($datasetName, $this->list($source['variableSets'] ?? [], 'Variable sets'), $ordinals); - $this->storeMultipleResponseSets($datasetName, $this->list($source['multipleResponseSets'] ?? [], 'Multiple-response sets'), $ordinals); + public function storeValidated(string $datasetName, V3MetadataPlan $metadata): void + { + $this->storeRoles($datasetName, $metadata->roles); + $this->storeVariableAttributes($datasetName, $metadata->variableAttributes); + $this->storeFileAttributes($datasetName, $metadata->fileAttributes); + $this->storeVariableSets($datasetName, $metadata->variableSets); + $this->storeMultipleResponseSets($datasetName, $metadata->multipleResponseSets); } - /** @param array $variables */ - private function storeRoles(string $datasetName, array $variables): void + /** @param list $roles */ + private function storeRoles(string $datasetName, array $roles): void { $statement = $this->pdo->prepare('INSERT INTO variable_roles (dataset_name, variable_ordinal, role) VALUES (?, ?, ?)'); - foreach ($variables as $ordinal => $variable) { - $role = $this->field($variable, 'role'); - if (!is_int($role) || $role < 0 || $role > 5) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every variable must have a valid SPSS role.'); - } + foreach ($roles as $ordinal => $role) { $statement->execute([$datasetName, $ordinal + 1, $role]); } } - /** @param array $variables */ - private function storeVariableAttributes(string $datasetName, array $variables): void + /** @param list}> $attributes */ + private function storeVariableAttributes(string $datasetName, array $attributes): void { $statement = $this->pdo->prepare('INSERT INTO variable_attributes (dataset_name, variable_ordinal, attribute_name, ordinal, value) VALUES (?, ?, ?, ?, ?)'); - foreach ($variables as $variableOrdinal => $variable) { - $attributes = $this->list($this->field($variable, 'attributes') ?? [], 'Variable attributes'); - foreach ($attributes as $attribute) { - $name = $this->field($attribute, 'name'); - $values = $this->field($attribute, 'values'); - if (!is_string($name) || $name === '' || !is_array($values) || $values === []) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every variable attribute must have a name and values.'); - } - foreach ($values as $ordinal => $value) { - if (!is_string($value)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Variable attribute values must be strings.'); - } - $statement->execute([$datasetName, $variableOrdinal + 1, $name, $ordinal + 1, $value]); - } + foreach ($attributes as $attribute) { + foreach ($attribute['values'] as $ordinal => $value) { + $statement->execute([$datasetName, $attribute['variableOrdinal'], $attribute['name'], $ordinal + 1, $value]); } } } - /** @param array $attributes */ + /** @param list}> $attributes */ private function storeFileAttributes(string $datasetName, array $attributes): void { $statement = $this->pdo->prepare('INSERT INTO file_attributes (dataset_name, attribute_name, ordinal, value) VALUES (?, ?, ?, ?)'); foreach ($attributes as $attribute) { - $name = $this->field($attribute, 'name'); - $values = $this->field($attribute, 'values'); - if (!is_string($name) || $name === '' || !is_array($values) || $values === []) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every file attribute must have a name and values.'); - } - foreach ($values as $ordinal => $value) { - if (!is_string($value)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'File attribute values must be strings.'); - } - $statement->execute([$datasetName, $name, $ordinal + 1, $value]); + foreach ($attribute['values'] as $ordinal => $value) { + $statement->execute([$datasetName, $attribute['name'], $ordinal + 1, $value]); } } } - /** - * @param array $sets - * @param array $variableOrdinals - */ - private function storeVariableSets(string $datasetName, array $sets, array $variableOrdinals): void + /** @param list}> $sets */ + private function storeVariableSets(string $datasetName, array $sets): void { - $set = $this->pdo->prepare('INSERT INTO variable_sets (dataset_name, set_ordinal, name) VALUES (?, ?, ?)'); - $member = $this->pdo->prepare('INSERT INTO variable_set_members (dataset_name, set_ordinal, member_ordinal, variable_ordinal) VALUES (?, ?, ?, ?)'); - foreach ($sets as $setOrdinal => $source) { - $name = $this->field($source, 'name'); - $members = $this->field($source, 'variableNames'); - if (!is_string($name) || $name === '' || !is_array($members)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every variable set must have a name and ordered members.'); - } - $set->execute([$datasetName, $setOrdinal + 1, $name]); - foreach ($members as $memberOrdinal => $variableName) { - if (!is_string($variableName) || !isset($variableOrdinals[$variableName])) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A variable set references an unknown source variable.'); - } - $member->execute([$datasetName, $setOrdinal + 1, $memberOrdinal + 1, $variableOrdinals[$variableName]]); + $setStatement = $this->pdo->prepare('INSERT INTO variable_sets (dataset_name, set_ordinal, name) VALUES (?, ?, ?)'); + $memberStatement = $this->pdo->prepare('INSERT INTO variable_set_members (dataset_name, set_ordinal, member_ordinal, variable_ordinal) VALUES (?, ?, ?, ?)'); + foreach ($sets as $setOrdinal => $set) { + $setStatement->execute([$datasetName, $setOrdinal + 1, $set['name']]); + foreach ($set['members'] as $memberOrdinal => $variableOrdinal) { + $memberStatement->execute([$datasetName, $setOrdinal + 1, $memberOrdinal + 1, $variableOrdinal]); } } } - /** - * @param array $sets - * @param array $variableOrdinals - */ - private function storeMultipleResponseSets(string $datasetName, array $sets, array $variableOrdinals): void + /** @param list}> $sets */ + private function storeMultipleResponseSets(string $datasetName, array $sets): void { - $set = $this->pdo->prepare('INSERT INTO multiple_response_sets (dataset_name, set_ordinal, name, set_type, label, counted_value_kind, counted_numeric_value, counted_text_value, category_labels, label_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); - $member = $this->pdo->prepare('INSERT INTO multiple_response_set_members (dataset_name, set_ordinal, member_ordinal, variable_ordinal) VALUES (?, ?, ?, ?)'); - foreach ($sets as $setOrdinal => $source) { - $name = $this->field($source, 'name'); - $type = $this->field($source, 'type'); - $members = $this->field($source, 'variableNames'); - $label = $this->field($source, 'label'); - $countedValue = $this->field($source, 'countedValue'); - $categoryLabels = $this->field($source, 'categoryLabels'); - $labelSource = $this->field($source, 'labelSource'); - if (!is_string($name) || $name === '' || !in_array($type, ['category', 'dichotomy'], true) || !is_array($members) || ($label !== null && !is_string($label)) || !in_array($categoryLabels, ['variable_labels', 'counted_values'], true) || !in_array($labelSource, ['set_label', 'variable_label'], true) || ($countedValue !== null && !is_int($countedValue) && !is_float($countedValue) && !is_string($countedValue))) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A multiple-response set is malformed.'); - } - if (($type === 'category' && $countedValue !== null) || ($type === 'dichotomy' && ($countedValue === null || $countedValue === ''))) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A multiple-response set has an invalid counted value.'); - } - $set->execute([ + $setStatement = $this->pdo->prepare('INSERT INTO multiple_response_sets (dataset_name, set_ordinal, name, set_type, label, counted_value_kind, counted_numeric_value, counted_text_value, category_labels, label_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); + $memberStatement = $this->pdo->prepare('INSERT INTO multiple_response_set_members (dataset_name, set_ordinal, member_ordinal, variable_ordinal) VALUES (?, ?, ?, ?)'); + foreach ($sets as $setOrdinal => $set) { + $countedValue = $set['countedValue']; + $setStatement->execute([ $datasetName, $setOrdinal + 1, - $name, - $type, - $label, + $set['name'], + $set['type'], + $set['label'], $countedValue === null ? null : (is_string($countedValue) ? 'text' : 'numeric'), is_int($countedValue) || is_float($countedValue) ? (float) $countedValue : null, is_string($countedValue) ? $countedValue : null, - $categoryLabels, - $labelSource, + $set['categoryLabels'], + $set['labelSource'], ]); - foreach ($members as $memberOrdinal => $variableName) { - if (!is_string($variableName) || !isset($variableOrdinals[$variableName])) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A multiple-response set references an unknown source variable.'); - } - $member->execute([$datasetName, $setOrdinal + 1, $memberOrdinal + 1, $variableOrdinals[$variableName]]); + foreach ($set['members'] as $memberOrdinal => $variableOrdinal) { + $memberStatement->execute([$datasetName, $setOrdinal + 1, $memberOrdinal + 1, $variableOrdinal]); } } } - - /** @param array $variables - * @return array - */ - private function variableOrdinals(array $variables): array - { - $result = []; - foreach ($variables as $ordinal => $variable) { - $name = $this->field($variable, 'name'); - if (!is_string($name) || $name === '' || isset($result[$name])) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Source variable names must be unique for metadata references.'); - } - $result[$name] = $ordinal + 1; - } - - return $result; - } - - /** @return array */ - private function list(mixed $value, string $description): array - { - if (!is_array($value) || !array_is_list($value)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $description . ' must be a list.'); - } - - return $value; - } - - private function field(mixed $source, string $name): mixed - { - return is_array($source) ? ($source[$name] ?? null) : null; - } } diff --git a/src/Sql/SqliteWideTableImporter.php b/src/Sql/SqliteWideTableImporter.php index d5d22f3..f88b9b4 100644 --- a/src/Sql/SqliteWideTableImporter.php +++ b/src/Sql/SqliteWideTableImporter.php @@ -21,14 +21,20 @@ public function __construct(private PDO $pdo) } /** @param array $source */ - public function import(array $source, string $datasetName, string $sourcePath = ""): void - { + public function import( + array $source, + string $datasetName, + string $sourcePath = "", + ?string $verifiedSourceSha256 = null, + ): void { + $verifiedSourceSha256 = NormativeCatalog::validateSourceSha256($verifiedSourceSha256); $variables = $this->variables($source['variables']); $sourceRows = $source['data'] ?? null; if (!is_array($sourceRows) || !array_is_list($sourceRows)) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The source dataset must contain an ordered case list.'); } $this->profile->assertDataset($source['variables'], $sourceRows, $this->pdo); + $v3Metadata = V3MetadataPlan::fromSource($source); $tableName = 'dataset_' . $this->identifier($datasetName); $this->pdo->beginTransaction(); @@ -37,7 +43,7 @@ public function import(array $source, string $datasetName, string $sourcePath = $this->storeDatasetMetadata($datasetName, $source); $this->storeTechnicalMetadata($datasetName, $source); $this->storeDictionaryMetadata($datasetName, $source['variables'], $source['valueLabels'] ?? []); - (new SqliteV3MetadataImporter($this->pdo))->store($datasetName, $source); + (new SqliteV3MetadataImporter($this->pdo))->storeValidated($datasetName, $v3Metadata); $this->storeDisplayMetadata($datasetName, is_array($source['displayParameters'] ?? null) ? $source['displayParameters'] : []); $this->createDataTable($tableName, $variables); $this->pdo->prepare('INSERT INTO datasets (dataset_name, table_name) VALUES (?, ?)')->execute([$datasetName, $tableName]); @@ -47,8 +53,13 @@ public function import(array $source, string $datasetName, string $sourcePath = } $this->storeWeightVariable($datasetName, $source['weightVariableName'] ?? null, $variables); $this->insertCases($tableName, $variables, $sourceRows); - if ($sourcePath !== "") { - (new NormativeCatalog($this->pdo))->storeImportedDataset($datasetName, $sourcePath, $source); + if ($sourcePath !== "" || $verifiedSourceSha256 !== null) { + (new NormativeCatalog($this->pdo))->storeImportedDataset( + $datasetName, + $sourcePath, + $source, + $verifiedSourceSha256, + ); } $this->pdo->commit(); } catch (Throwable $exception) { diff --git a/src/Sql/V3MetadataPlan.php b/src/Sql/V3MetadataPlan.php new file mode 100644 index 0000000..bf45e80 --- /dev/null +++ b/src/Sql/V3MetadataPlan.php @@ -0,0 +1,217 @@ + $roles + * @param list}> $variableAttributes + * @param list}> $fileAttributes + * @param list}> $variableSets + * @param list}> $multipleResponseSets + */ + private function __construct( + public array $roles, + public array $variableAttributes, + public array $fileAttributes, + public array $variableSets, + public array $multipleResponseSets, + ) {} + + /** @param array $source */ + public static function fromSourceIfPresent(array $source): ?self + { + $present = array_key_exists('fileAttributes', $source) + || array_key_exists('variableSets', $source) + || array_key_exists('multipleResponseSets', $source); + $variables = $source['variables'] ?? null; + if (is_array($variables)) { + foreach ($variables as $variable) { + if (is_array($variable) && (array_key_exists('role', $variable) || array_key_exists('attributes', $variable))) { + $present = true; + break; + } + } + } + + return $present ? self::fromSource($source) : null; + } + + /** @param array $source */ + public static function fromSource(array $source): self + { + $variables = self::list($source['variables'] ?? null, 'V3 source variables'); + if ($variables === []) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'V3 metadata requires a source variable list.'); + } + + $roles = []; + $variableAttributes = []; + $variableOrdinals = []; + foreach ($variables as $ordinal => $variable) { + $name = self::field($variable, 'name'); + if (!is_string($name) || $name === '' || isset($variableOrdinals[$name])) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Source variable names must be unique for metadata references.'); + } + $variableOrdinals[$name] = $ordinal + 1; + + $role = self::field($variable, 'role'); + if (!is_int($role) || $role < 0 || $role > 5) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every variable must have a valid SPSS role.'); + } + $roles[] = $role; + + foreach (self::attributes(self::field($variable, 'attributes') ?? [], 'Variable') as $attribute) { + $variableAttributes[] = [ + 'variableOrdinal' => $ordinal + 1, + 'name' => $attribute['name'], + 'values' => $attribute['values'], + ]; + } + } + + return new self( + $roles, + $variableAttributes, + self::attributes($source['fileAttributes'] ?? [], 'File'), + self::variableSets($source['variableSets'] ?? [], $variableOrdinals), + self::multipleResponseSets($source['multipleResponseSets'] ?? [], $variableOrdinals), + ); + } + + /** + * @param array $variableOrdinals + * @return list}> + */ + private static function variableSets(mixed $sourceSets, array $variableOrdinals): array + { + $result = []; + $names = []; + foreach (self::list($sourceSets, 'Variable sets') as $source) { + $name = self::field($source, 'name'); + $members = self::field($source, 'variableNames'); + if (!is_string($name) || $name === '' || isset($names[$name]) || !is_array($members) || !array_is_list($members)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every variable set must have a unique name and ordered members.'); + } + $names[$name] = true; + $result[] = ['name' => $name, 'members' => self::members($members, $variableOrdinals, 'A variable set')]; + } + + return $result; + } + + /** + * @param array $variableOrdinals + * @return list}> + */ + private static function multipleResponseSets(mixed $sourceSets, array $variableOrdinals): array + { + $result = []; + $names = []; + foreach (self::list($sourceSets, 'Multiple-response sets') as $source) { + $name = self::field($source, 'name'); + $type = self::field($source, 'type'); + $members = self::field($source, 'variableNames'); + $label = self::field($source, 'label'); + $countedValue = self::field($source, 'countedValue'); + $categoryLabels = self::field($source, 'categoryLabels'); + $labelSource = self::field($source, 'labelSource'); + if (!is_string($name) || $name === '' || isset($names[$name]) + || !in_array($type, ['category', 'dichotomy'], true) + || !is_array($members) || !array_is_list($members) + || ($label !== null && !is_string($label)) + || !in_array($categoryLabels, ['variable_labels', 'counted_values'], true) + || !in_array($labelSource, ['set_label', 'variable_label'], true) + || ($countedValue !== null && !is_int($countedValue) && !is_float($countedValue) && !is_string($countedValue)) + ) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A multiple-response set is malformed.'); + } + if ((is_int($countedValue) || is_float($countedValue)) && !is_finite((float) $countedValue)) { + throw new UnsupportedOperation(DiagnosticCode::TargetCapabilityExceeded, 'SQL targets reject non-finite multiple-response counted values before mutation.'); + } + if (($type === 'category' && $countedValue !== null) || ($type === 'dichotomy' && ($countedValue === null || $countedValue === ''))) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A multiple-response set has an invalid counted value.'); + } + $names[$name] = true; + $result[] = [ + 'name' => $name, + 'type' => $type, + 'label' => $label, + 'countedValue' => $countedValue, + 'categoryLabels' => $categoryLabels, + 'labelSource' => $labelSource, + 'members' => self::members($members, $variableOrdinals, 'A multiple-response set'), + ]; + } + + return $result; + } + + /** + * @param list $sourceMembers + * @param array $variableOrdinals + * @return list + */ + private static function members(array $sourceMembers, array $variableOrdinals, string $description): array + { + $members = []; + $seen = []; + foreach ($sourceMembers as $variableName) { + if (!is_string($variableName) || !isset($variableOrdinals[$variableName])) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $description . ' references an unknown source variable.'); + } + if (isset($seen[$variableName])) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $description . ' must not contain duplicate source variables.'); + } + $seen[$variableName] = true; + $members[] = $variableOrdinals[$variableName]; + } + + return $members; + } + + /** @return list}> */ + private static function attributes(mixed $sourceAttributes, string $description): array + { + $result = []; + $names = []; + foreach (self::list($sourceAttributes, $description . ' attributes') as $attribute) { + $name = self::field($attribute, 'name'); + $values = self::field($attribute, 'values'); + if (!is_string($name) || $name === '' || isset($names[$name]) || !is_array($values) || !array_is_list($values) || $values === []) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'Every ' . strtolower($description) . ' attribute must have a unique name and ordered values.'); + } + foreach ($values as $value) { + if (!is_string($value)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $description . ' attribute values must be strings.'); + } + } + $names[$name] = true; + $result[] = ['name' => $name, 'values' => $values]; + } + + return $result; + } + + /** @return list */ + private static function list(mixed $value, string $description): array + { + if (!is_array($value) || !array_is_list($value)) { + throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $description . ' must be a list.'); + } + + return $value; + } + + private static function field(mixed $source, string $name): mixed + { + return is_array($source) ? ($source[$name] ?? null) : null; + } +} diff --git a/tests/Core/CapabilityDeclarationTest.php b/tests/Core/CapabilityDeclarationTest.php index 6118fda..c5d7623 100644 --- a/tests/Core/CapabilityDeclarationTest.php +++ b/tests/Core/CapabilityDeclarationTest.php @@ -17,12 +17,18 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): self::assertSame('release_candidate', $declaration['specification_status']); self::assertNull($declaration['specification_release']); self::assertSame(CapabilityDeclaration::SPECIFICATION_RELEASE, $declaration['specification_release']); - self::assertSame('6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045', $declaration['specification_commit']); + self::assertSame('34141dda023d9e0217c37c232e39f436edfb0746', $declaration['specification_commit']); self::assertSame(CapabilityDeclaration::SPECIFICATION_COMMIT, $declaration['specification_commit']); self::assertMatchesRegularExpression('/^[0-9a-f]{40}$/', $declaration['specification_commit']); self::assertSame(['import', 'export', 'semantic_round_trip'], $declaration['directions']); self::assertSame('sqlite', $declaration['active_connection']['profile']); self::assertNotSame('', $declaration['active_connection']['server_version']); + self::assertSame($declaration['active_connection']['server_version'], $declaration['active_connection']['raw_server_version']); + self::assertSame('PDO::ATTR_SERVER_VERSION', $declaration['active_connection']['identity_source']); + self::assertSame( + ['PDO::ATTR_SERVER_VERSION' => $declaration['active_connection']['server_version']], + $declaration['active_connection']['identity_probe_results'], + ); self::assertTrue($declaration['active_connection']['claimed_supported']); self::assertSame('SQLite >=3.24.0 <4.0.0', $declaration['active_connection']['matched_claim']); self::assertTrue($declaration['active_connection']['catalog_binding']['exclusive_namespace_verified']); @@ -47,22 +53,52 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): foreach ($declaration['required_capabilities'] as $supported) { self::assertTrue($supported); } - foreach (['sqlite', 'mysql', 'mariadb', 'postgresql'] as $name) { + foreach (['sqlite', 'mysql', 'mariadb', 'dolt', 'postgresql'] as $name) { $profile = $declaration['sql_profiles'][$name]; - self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_physical_columns']); + self::assertSame($name, $profile['profile']); + self::assertSame(CapabilityDeclaration::SPECIFICATION_COMMIT, $profile['specification_commit']); self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_value_bytes']); - self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_row_bytes']); - self::assertArrayNotHasKey('maximum_identifier_bytes', $profile['theoretical_limits']); - self::assertGreaterThan(0, $profile['theoretical_limits']['identifier_limit']['value']); - self::assertSame( - in_array($name, ['mysql', 'mariadb'], true) ? 'characters' : 'bytes', - $profile['theoretical_limits']['identifier_limit']['unit'], - ); - self::assertNotSame('', $profile['theoretical_limits']['identifier_limit']['source']); - self::assertSame('ASCII lowercase letters, digits, and underscore', $profile['theoretical_limits']['identifier_limit']['repertoire']); self::assertNotSame('', $profile['claimed_server_versions']); self::assertNotEmpty($profile['ci_tested_server_versions']); self::assertNotSame('', $profile['physical_table_mapping']); + if ($name === 'dolt') { + self::assertSame(['maximum_value_bytes'], array_keys($profile['theoretical_limits'])); + self::assertSame(306, $profile['proposed_adapter_limits']['maximum_physical_columns']); + self::assertSame(305, $profile['proposed_adapter_limits']['maximum_source_variables']); + self::assertSame(65_504, $profile['proposed_adapter_limits']['maximum_row_bytes']); + self::assertSame(307, $profile['observed_limits']['minimum_observed_physical_columns']); + self::assertSame(64, $profile['observed_limits']['identifier_limit']['value']); + self::assertSame(65, $profile['observed_limits']['rejected_identifier_bytes']); + self::assertSame('observed_exact_version', $profile['limit_bases']['identifier_limit']); + self::assertTrue($profile['storage_evidence']['binary64']['maximum_finite_round_trip_exact']); + self::assertSame('reject_before_mutation', $profile['storage_evidence']['binary64']['non_finite_policy']); + self::assertSame([ + 'nan' => 'reject_before_mutation', + 'positive_infinity' => 'reject_before_mutation', + 'negative_infinity' => 'reject_before_mutation', + 'system_missing' => 'sql_null', + ], $profile['numeric_exception_policy']); + self::assertSame(65_504, $profile['storage_evidence']['text']['observed_value_bytes']); + self::assertSame('unsupported', $profile['transformation_workflow']); + self::assertSame('mysql_compatible', $profile['transport']); + self::assertSame(['2.2.2'], $profile['exact_ci_tested_versions']); + self::assertSame( + ['@@version', '@@version_comment', 'DOLT_VERSION()'], + $profile['identity']['required_probes'], + ); + self::assertNull($profile['identity']['active_probe_results']); + } else { + self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_physical_columns']); + self::assertGreaterThan(0, $profile['theoretical_limits']['maximum_row_bytes']); + self::assertArrayNotHasKey('maximum_identifier_bytes', $profile['theoretical_limits']); + self::assertGreaterThan(0, $profile['theoretical_limits']['identifier_limit']['value']); + self::assertSame( + in_array($name, ['mysql', 'mariadb'], true) ? 'characters' : 'bytes', + $profile['theoretical_limits']['identifier_limit']['unit'], + ); + self::assertNotSame('', $profile['theoretical_limits']['identifier_limit']['source']); + self::assertSame('ASCII lowercase letters, digits, and underscore', $profile['theoretical_limits']['identifier_limit']['repertoire']); + } if ($name === 'sqlite') { self::assertSame('compile_time_ceiling', $profile['effective_limits_status']); self::assertGreaterThan(0, $profile['effective_limits']['maximum_source_variables']); diff --git a/tests/Core/ServerVersionPolicyTest.php b/tests/Core/ServerVersionPolicyTest.php index 97cbca4..4517d9b 100644 --- a/tests/Core/ServerVersionPolicyTest.php +++ b/tests/Core/ServerVersionPolicyTest.php @@ -9,6 +9,7 @@ use OpenStatSpec\Core\UnsupportedOperation; use OpenStatSpec\Spss\SpssAdapter; use PDO; +use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -24,6 +25,9 @@ public static function versions(): iterable yield 'MariaDB LTS supported' => ['mariadb', '11.8.8-MariaDB-ubu2404', true, 'MariaDB 11.4.x, 11.8.x or 12.3.x']; yield 'MariaDB latest supported' => ['mariadb', '12.3.2-MariaDB-ubu2404', true, 'MariaDB 11.4.x, 11.8.x or 12.3.x']; yield 'MariaDB old' => ['mariadb', '10.11.11-MariaDB', false, null]; + yield 'Dolt supported exact release' => ['dolt', '2.2.2', true, 'Dolt 2.2.2']; + yield 'Dolt unvalidated patch release' => ['dolt', '2.2.3', false, null]; + yield 'Dolt malformed' => ['dolt', 'not-a-version', false, null]; yield 'PostgreSQL previous major supported' => ['postgresql', '17.5 (Ubuntu 17.5-1)', true, 'PostgreSQL 17.x or 18.x']; yield 'PostgreSQL latest supported' => ['postgresql', '18.4 (Debian 18.4-1)', true, 'PostgreSQL 17.x or 18.x']; yield 'PostgreSQL old' => ['postgresql', '16.9', false, null]; @@ -52,6 +56,17 @@ public function testUnsupportedServerFailsBeforeDdl(): void PDO::ATTR_SERVER_VERSION => '8.0.40', default => '', }); + $pdo->method('query')->willReturnCallback(function (string $query): PDOStatement { + $value = match ($query) { + 'SELECT @@version' => '8.0.40', + 'SELECT @@version_comment' => 'MySQL Community Server - GPL', + default => throw new \LogicException('Unexpected identity probe: ' . $query), + }; + $statement = $this->createMock(PDOStatement::class); + $statement->method('fetchColumn')->willReturn($value); + + return $statement; + }); $pdo->expects(self::never())->method('exec'); $adapter = new SpssAdapter($pdo); diff --git a/tests/Integration/DoltSpssRoundTripTest.php b/tests/Integration/DoltSpssRoundTripTest.php new file mode 100644 index 0000000..fc0a000 --- /dev/null +++ b/tests/Integration/DoltSpssRoundTripTest.php @@ -0,0 +1,443 @@ +mysql(); + $engine = new PhpSpssEngine(); + + foreach (['sav' => '$FL2', 'zsav' => '$FL3'] as $format => $header) { + $token = bin2hex(random_bytes(6)); + $datasetName = 'dolt envelope ' . $format . ' ' . $token; + $tooWideName = 'dolt too wide ' . $format . ' ' . $token; + $sourcePath = sys_get_temp_dir() . '/openstatspec-dolt-envelope-source-' . $token . '.' . $format; + $targetPath = sys_get_temp_dir() . '/openstatspec-dolt-envelope-target-' . $token . '.' . $format; + $tooWidePath = sys_get_temp_dir() . '/openstatspec-dolt-too-wide-' . $token . '.' . $format; + $tableName = null; + $tooWideOperationId = null; + + try { + $fixture = $this->envelopeFixture($format, 305); + $engine->write($sourcePath, $fixture); + self::assertSame($header, $this->fileHeader($sourcePath)); + + $adapter = new SpssAdapter($pdo, $engine); + $adapter->import($sourcePath, $datasetName); + $tableName = $this->tableName($pdo, $datasetName); + + self::assertSame(306, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$tableName], + )); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM ' . $this->quote($tableName), [])); + + $result = $adapter->export($datasetName, $targetPath); + self::assertSame([], $result->diagnostics); + self::assertSame(1, $result->caseCount); + self::assertSame($header, $this->fileHeader($targetPath)); + + $roundTrip = $engine->read($targetPath); + self::assertSame($fixture->rows(), $roundTrip->rows()); + self::assertCount(305, $roundTrip->variables()); + self::assertSame('Weight', $roundTrip->metadata->weightVariableName); + self::assertSame('short value', $roundTrip->rows()[0][1]); + self::assertSame(340, strlen((string) $roundTrip->rows()[0][2])); + self::assertSame(400, $roundTrip->variables()[2]->width); + + $engine->write($tooWidePath, $this->envelopeFixture($format, 306)); + $tooWideTable = (new DoltProfile())->physicalIdentifier('dataset_' . $tooWideName); + try { + $adapter->import($tooWidePath, $tooWideName); + self::fail('Expected 306 Dolt source variables to fail before DDL.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + } + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$tooWideTable], + )); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM datasets WHERE dataset_name = ?', [$tooWideName])); + $tooWideOperationId = $this->scalar( + $pdo, + 'SELECT operation_id FROM operation_catalog WHERE target_path = ? ORDER BY started_at DESC LIMIT 1', + [$tooWidePath], + ); + self::assertIsString($tooWideOperationId); + self::assertSame('failed', $this->scalar( + $pdo, + 'SELECT status FROM operation WHERE operation_id = ?', + [$tooWideOperationId], + )); + self::assertSame(1, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM fidelity_event WHERE operation_id = ? AND dataset_id IS NULL AND event_code = ?', + [$tooWideOperationId, DiagnosticCode::TargetCapabilityExceeded->value], + )); + } finally { + $this->cleanup($pdo, $datasetName, $tableName); + $this->cleanup($pdo, $tooWideName, null); + $this->cleanupOperation($pdo, $tooWideOperationId); + @unlink($sourcePath); + @unlink($targetPath); + @unlink($tooWidePath); + } + } + } + + public function testInvalidWeightReferenceRejectsBeforeDdlAndPreservesFailureAudit(): void + { + $pdo = $this->mysql(); + $token = bin2hex(random_bytes(6)); + $datasetName = 'dolt post ddl failure ' . $token; + $sourcePath = 'dolt-post-ddl-failure-' . $token . '.sav'; + $tableName = (new DoltProfile())->physicalIdentifier('dataset_' . $datasetName); + $operationId = null; + $fixture = new Dataset( + new VariableDictionary([ + new VariableMetadata( + name: 'OnlyVariable', + type: VariableType::NUMERIC, + width: 0, + printFormat: new VariableFormat(5, 8, 2), + writeFormat: new VariableFormat(5, 8, 2), + dictionaryIndex: 1, + ), + ]), + [[1.0]], + new FileMetadata('Dolt pre-DDL invalid-weight fixture', weightVariableName: 'MissingWeight'), + new FileTechnicalMetadata(sourceFormat: 'sav', compression: 1), + ); + $adapter = new SpssAdapter($pdo, new FakeSpssEngine($fixture)); + + try { + $adapter->migrateCatalog(); + try { + $adapter->import($sourcePath, $datasetName); + self::fail('Expected the invalid weight reference to fail before wide-table DDL.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertStringContainsString('weight-variable reference', $exception->getMessage()); + } + + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$tableName], + )); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM datasets WHERE dataset_name = ?', [$datasetName])); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM variables WHERE dataset_name = ?', [$datasetName])); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset WHERE dataset_name = ?', [$datasetName])); + + $operationId = $this->scalar( + $pdo, + 'SELECT operation_id FROM operation_catalog WHERE target_path = ? ORDER BY started_at DESC LIMIT 1', + [$sourcePath], + ); + self::assertIsString($operationId); + self::assertNotSame('', $operationId); + self::assertSame('failed', $this->scalar($pdo, 'SELECT status FROM operation WHERE operation_id = ?', [$operationId])); + self::assertSame(1, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM fidelity_event WHERE operation_id = ? AND dataset_id IS NULL AND direction = ? AND severity = ? AND event_code = ? AND source_item = ?', + [$operationId, 'import', 'error', DiagnosticCode::InvalidSourceDataset->value, $sourcePath], + )); + } finally { + $this->cleanup($pdo, $datasetName, $tableName); + if (is_string($operationId) && $operationId !== '') { + $pdo->prepare('DELETE FROM fidelity_event WHERE operation_id = ?')->execute([$operationId]); + $pdo->prepare('DELETE FROM fidelity_event_catalog WHERE operation_id = ?')->execute([$operationId]); + $pdo->prepare('DELETE FROM operation_catalog WHERE operation_id = ?')->execute([$operationId]); + $pdo->prepare('DELETE FROM operation WHERE operation_id = ?')->execute([$operationId]); + } + } + } + + public function testObservedDoltStorageAndIdentifierBoundaries(): void + { + $pdo = $this->mysql(); + $token = bin2hex(random_bytes(6)); + $identifier64 = 'i' . str_repeat('a', 63); + $identifier65 = 'i' . str_repeat('a', 64); + $boundaryTable = 'oss_dolt_boundary_' . $token; + $columnsTable = 'oss_dolt_columns_' . $token; + $tooLargeDataset = 'dolt too large row ' . $token; + $tooLargeTable = (new DoltProfile())->physicalIdentifier('dataset_' . $tooLargeDataset); + $text = str_repeat("\xC3\xB5", 32_752); + self::assertSame(65_504, strlen($text)); + + try { + $pdo->exec('CREATE TABLE ' . $this->quote($identifier64) . ' (id BIGINT NOT NULL PRIMARY KEY)'); + self::assertSame(1, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$identifier64], + )); + + try { + $pdo->exec('CREATE TABLE ' . $this->quote($identifier65) . ' (id BIGINT NOT NULL PRIMARY KEY)'); + self::fail('Dolt accepted a 65-byte ASCII table identifier.'); + } catch (PDOException) { + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$identifier65], + )); + } + + $columns = ['__case_ordinal BIGINT NOT NULL PRIMARY KEY']; + for ($index = 1; $index <= 306; ++$index) { + $columns[] = $this->quote(sprintf('c%03d', $index)) . ' DOUBLE NULL'; + } + $pdo->exec('CREATE TABLE ' . $this->quote($columnsTable) . ' (' . implode(', ', $columns) . ')'); + self::assertSame(307, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$columnsTable], + )); + + $pdo->exec( + 'CREATE TABLE ' . $this->quote($boundaryTable) + . ' (__case_ordinal BIGINT NOT NULL PRIMARY KEY, numeric_value DOUBLE NOT NULL, text_value LONGTEXT NOT NULL)', + ); + $statement = $pdo->prepare( + 'INSERT INTO ' . $this->quote($boundaryTable) + . ' (__case_ordinal, numeric_value, text_value) VALUES (?, ?, ?)', + ); + if ($statement === false) { + throw new RuntimeException('Could not prepare Dolt boundary insert.'); + } + $statement->execute([1, sprintf('%.17g', PHP_FLOAT_MAX), $text]); + + $query = $pdo->query( + 'SELECT numeric_value, text_value FROM ' . $this->quote($boundaryTable) + . ' WHERE __case_ordinal = 1', + ); + if ($query === false) { + throw new RuntimeException('Could not read the Dolt boundary row.'); + } + $stored = $query->fetch(PDO::FETCH_ASSOC); + self::assertIsArray($stored); + self::assertSame(Binary64::encode(PHP_FLOAT_MAX), Binary64::encode((float) $stored['numeric_value'])); + self::assertSame($text, $stored['text_value']); + self::assertSame(65_504, strlen((string) $stored['text_value'])); + + try { + (new MySqlWideTableImporter($pdo, new DoltProfile()))->import([ + 'variables' => [['name' => 'payload', 'type' => 'string', 'width' => 65_505]], + 'data' => [[str_repeat('a', 65_505)]], + ], $tooLargeDataset); + self::fail('Dolt accepted an encoded row above the 65,504-byte ceiling.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + } + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$tooLargeTable], + )); + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM datasets WHERE dataset_name = ?', + [$tooLargeDataset], + )); + } finally { + $this->cleanup($pdo, $tooLargeDataset, $tooLargeTable); + foreach ([$identifier65, $identifier64, $columnsTable, $boundaryTable] as $table) { + $pdo->exec('DROP TABLE IF EXISTS ' . $this->quote($table)); + } + } + } + + public function testFinalizationFailureCompensatesCommittedDoltDatasetAndPreservesNullDatasetAudit(): void + { + $pdo = $this->mysql(); + $token = bin2hex(random_bytes(6)); + $datasetName = 'dolt finalization failure ' . $token; + $sourcePath = 'dolt-finalization-failure-' . $token . '.sav'; + $tableName = (new DoltProfile())->physicalIdentifier('dataset_' . $datasetName); + $operationId = null; + $priorDatasetId = NormativeCatalog::uuid(); + $priorOperationId = NormativeCatalog::uuid(); + $priorFidelityEventId = NormativeCatalog::uuid(); + $priorPhysicalTable = 'prior_owned_' . $token; + $adapter = new SpssAdapter( + $pdo, + new FakeSpssEngine($this->envelopeFixture('sav', 3)), + beforeImportFinalization: static function (): void { + throw new RuntimeException('Injected finalization failure.'); + }, + ); + + try { + $adapter->migrateCatalog(); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?)', + )->execute([ + $priorDatasetId, '1.0', 'sav', + $priorPhysicalTable, $datasetName, 0, '2026-07-30 00:00:00', + ]); + $pdo->prepare( + 'INSERT INTO operation (operation_id, operation_kind, status, source_format, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?)', + )->execute([ + $priorOperationId, 'import', 'failed', 'sav', + '2026-07-30 00:00:00', '2026-07-30 00:00:01', + ]); + $pdo->prepare( + 'INSERT INTO fidelity_event (fidelity_event_id, operation_id, dataset_id, direction, severity, event_code, source_item, detail_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([ + $priorFidelityEventId, $priorOperationId, $priorDatasetId, + 'import', 'error', 'prior_failure', 'prior.sav', '{}', + '2026-07-30 00:00:01', + ]); + + try { + $adapter->import($sourcePath, $datasetName); + self::fail('The injected finalization failure was not reported.'); + } catch (RuntimeException $exception) { + self::assertSame('Injected finalization failure.', $exception->getMessage()); + } + + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [$tableName], + )); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM datasets WHERE dataset_name = ?', [$datasetName])); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset WHERE dataset_name = ?', [$datasetName])); + self::assertSame($priorDatasetId, $this->scalar( + $pdo, + 'SELECT dataset_id FROM dataset WHERE dataset_name = ?', + [$datasetName], + )); + self::assertSame(1, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM fidelity_event WHERE fidelity_event_id = ? AND dataset_id = ? AND event_code = ?', + [$priorFidelityEventId, $priorDatasetId, 'prior_failure'], + )); + $operationId = $this->scalar( + $pdo, + 'SELECT operation_id FROM operation_catalog WHERE target_path = ? ORDER BY started_at DESC LIMIT 1', + [$sourcePath], + ); + self::assertIsString($operationId); + self::assertSame('failed', $this->scalar( + $pdo, + 'SELECT status FROM operation WHERE operation_id = ?', + [$operationId], + )); + self::assertSame(1, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM fidelity_event WHERE operation_id = ? AND dataset_id IS NULL AND event_code = ?', + [$operationId, 'operation_failed'], + )); + } finally { + $this->cleanup($pdo, $datasetName, $tableName); + $this->cleanupOperation($pdo, $operationId); + $pdo->prepare('DELETE FROM fidelity_event WHERE fidelity_event_id = ?')->execute([$priorFidelityEventId]); + $pdo->prepare('DELETE FROM operation WHERE operation_id = ?')->execute([$priorOperationId]); + $pdo->prepare('DELETE FROM dataset WHERE dataset_id = ?')->execute([$priorDatasetId]); + } + } + + private function cleanupOperation(PDO $pdo, mixed $operationId): void + { + if (!is_string($operationId) || $operationId === '') { + return; + } + foreach (['fidelity_event', 'fidelity_event_catalog', 'operation_catalog', 'operation'] as $table) { + $statement = $pdo->prepare('DELETE FROM ' . $table . ' WHERE operation_id = ?'); + if ($statement !== false) { + $statement->execute([$operationId]); + } + } + } + + private function envelopeFixture(string $format, int $variableCount): Dataset + { + $variables = [ + new VariableMetadata( + name: 'Weight', + type: VariableType::NUMERIC, + width: 0, + printFormat: new VariableFormat(5, 8, 2), + writeFormat: new VariableFormat(5, 8, 2), + dictionaryIndex: 1, + ), + new VariableMetadata( + name: 'ShortText', + type: VariableType::STRING, + width: 16, + printFormat: new VariableFormat(1, 16), + writeFormat: new VariableFormat(1, 16), + dictionaryIndex: 2, + ), + new VariableMetadata( + name: 'LongText', + type: VariableType::STRING, + width: 400, + printFormat: new VariableFormat(1, 255), + writeFormat: new VariableFormat(1, 255), + dictionaryIndex: 3, + ), + ]; + $row = [1.25, 'short value', str_repeat("\xC3\xB5", 170)]; + for ($index = 4; $index <= $variableCount; ++$index) { + $variables[] = new VariableMetadata( + name: sprintf('V%03d', $index), + type: VariableType::NUMERIC, + width: 0, + printFormat: new VariableFormat(5, 8, 2), + writeFormat: new VariableFormat(5, 8, 2), + dictionaryIndex: $index, + ); + $row[] = (float) $index; + } + + return new Dataset( + new VariableDictionary($variables), + [$row], + new FileMetadata('Dolt 305-variable envelope fixture', weightVariableName: 'Weight'), + new FileTechnicalMetadata(sourceFormat: $format, compression: $format === 'zsav' ? 2 : 1), + ); + } + + protected function serviceName(): string + { + return 'dolt'; + } + + protected function environmentPrefix(): string + { + return 'OPENSTATSPEC_DOLT'; + } +} diff --git a/tests/Integration/MySqlFamilySpssRoundTripTestCase.php b/tests/Integration/MySqlFamilySpssRoundTripTestCase.php index 42a9499..839d0aa 100644 --- a/tests/Integration/MySqlFamilySpssRoundTripTestCase.php +++ b/tests/Integration/MySqlFamilySpssRoundTripTestCase.php @@ -32,7 +32,7 @@ use SPSS\Sav\VariableType; /** - * Runs only against an explicitly configured MySQL/MariaDB instance. + * Runs only against an explicitly configured MySQL-family instance. * * GitHub Actions supplies the service. Developers can opt in locally with: * The concrete MySQL and MariaDB subclasses each supply their own DSN variables. @@ -64,7 +64,7 @@ public function testRealEngineRoundTripsSavAndZsavThroughMySqlFamily(): void self::assertMatchesRegularExpression('/^dataset_/', $tableName); $caseCount = $pdo->query('SELECT COUNT(*) FROM ' . $this->quote($tableName)); if ($caseCount === false) { - throw new RuntimeException('Could not count imported MySQL/MariaDB cases.'); + throw new RuntimeException('Could not count imported MySQL-family cases.'); } self::assertSame(2, (int) $caseCount->fetchColumn()); $this->assertVariableCatalog( @@ -117,7 +117,7 @@ public function testRealEngineRoundTripsSavAndZsavThroughMySqlFamily(): void self::assertSame($fixture->rows(), $roundTrip->rows()); self::assertSame($format, $roundTrip->technicalMetadata->sourceFormat); self::assertSame($compression, $roundTrip->technicalMetadata->compression); - self::assertSame('MySQL/MariaDB integration fixture', $roundTrip->metadata->label); + self::assertSame('MySQL-family integration fixture', $roundTrip->metadata->label); self::assertSame('Score', $roundTrip->metadata->weightVariableName); self::assertSame(['First document line', 'Second document line'], $roundTrip->metadata->documents()); self::assertCount(3, $roundTrip->variables()); @@ -133,7 +133,7 @@ public function testRealEngineRoundTripsSavAndZsavThroughMySqlFamily(): void self::assertEquals(MissingValues::discrete('MISSING'), $roundTrip->variables()[1]->missingValues); self::assertSame(400, $roundTrip->variables()[2]->width); self::assertSame(340, strlen((string) $roundTrip->rows()[0][2])); - self::assertEquals([new FileAttribute('Source', ['MySQL/MariaDB integration'])], $roundTrip->metadata->attributes()); + self::assertEquals([new FileAttribute('Source', ['MySQL-family integration'])], $roundTrip->metadata->attributes()); self::assertEquals([new VariableSet('Core', ['Score', 'Reason'])], $roundTrip->metadata->variableSets()); self::assertCount(1, $roundTrip->metadata->multipleResponseSets()); self::assertSame(MultipleResponseSetType::DICHOTOMY, $roundTrip->metadata->multipleResponseSets()[0]->type); @@ -150,7 +150,7 @@ abstract protected function serviceName(): string; abstract protected function environmentPrefix(): string; - private function mysql(): PDO + protected function mysql(): PDO { $prefix = $this->environmentPrefix(); $dsn = getenv($prefix . '_DSN'); @@ -218,10 +218,10 @@ private function fixture(string $format): Dataset ]), [[7.5, 'present', $longText], [null, 'MISSING', '']], new FileMetadata( - 'MySQL/MariaDB integration fixture', + 'MySQL-family integration fixture', weightVariableName: 'Score', documents: ['First document line', 'Second document line'], - attributes: [new FileAttribute('Source', ['MySQL/MariaDB integration'])], + attributes: [new FileAttribute('Source', ['MySQL-family integration'])], variableSets: [new VariableSet('Core', ['Score', 'Reason'])], multipleResponseSets: [ new MultipleResponseSet( @@ -239,13 +239,13 @@ private function fixture(string $format): Dataset ); } - private function tableName(PDO $pdo, string $datasetName): string + protected function tableName(PDO $pdo, string $datasetName): string { $statement = $pdo->prepare('SELECT table_name FROM datasets WHERE dataset_name = ?'); $statement->execute([$datasetName]); $tableName = $statement->fetchColumn(); if (!is_string($tableName) || $tableName === '') { - throw new RuntimeException('The MySQL/MariaDB dataset catalogue entry was not created.'); + throw new RuntimeException('The MySQL-family dataset catalogue entry was not created.'); } return $tableName; @@ -263,7 +263,7 @@ private function rows(PDO $pdo, string $sql, array $parameters): array } /** @param list $parameters */ - private function scalar(PDO $pdo, string $sql, array $parameters): mixed + protected function scalar(PDO $pdo, string $sql, array $parameters): mixed { $statement = $pdo->prepare($sql); $statement->execute($parameters); @@ -271,12 +271,13 @@ private function scalar(PDO $pdo, string $sql, array $parameters): mixed return $statement->fetchColumn(); } - private function cleanup(PDO $pdo, string $datasetName, ?string $tableName): void + protected function cleanup(PDO $pdo, string $datasetName, ?string $tableName): void { if ($tableName !== null) { $pdo->exec('DROP TABLE IF EXISTS ' . $this->quote($tableName)); } foreach ([ + 'dataset_weight_variables', 'multiple_response_set_members', 'multiple_response_sets', 'variable_set_members', @@ -314,12 +315,12 @@ private function catalogueTableExists(PDO $pdo, string $catalogue): bool } } - private function quote(string $identifier): string + protected function quote(string $identifier): string { return chr(96) . str_replace(chr(96), chr(96) . chr(96), $identifier) . chr(96); } - private function fileHeader(string $path): string + protected function fileHeader(string $path): string { $header = file_get_contents($path, false, null, 0, 4); if (!is_string($header)) { diff --git a/tests/Integration/OfficialSpssConformanceManifestTest.php b/tests/Integration/OfficialSpssConformanceManifestTest.php index dc583ba..75ee5e4 100644 --- a/tests/Integration/OfficialSpssConformanceManifestTest.php +++ b/tests/Integration/OfficialSpssConformanceManifestTest.php @@ -182,7 +182,7 @@ private function assertExpectedCatalog(PDO $pdo, string $datasetName, array $exp if (array_key_exists('variable_attributes', $expected)) { $actual = []; - foreach ($this->rows($pdo, 'SELECT variable.source_name, attribute.attribute_name, attribute.array_ordinal, attribute.attribute_value FROM variable_attribute attribute JOIN variable ON variable.variable_id = attribute.variable_id WHERE variable.dataset_id = ? ORDER BY variable.source_ordinal, attribute.attribute_name, attribute.array_ordinal', [$datasetId]) as $row) { + foreach ($this->rows($pdo, 'SELECT variable.source_name, attribute_row.attribute_name, attribute_row.array_ordinal, attribute_row.attribute_value FROM variable_attribute attribute_row JOIN variable ON variable.variable_id = attribute_row.variable_id WHERE variable.dataset_id = ? ORDER BY variable.source_ordinal, attribute_row.attribute_name, attribute_row.array_ordinal', [$datasetId]) as $row) { $key = (string) $row['source_name'] . "\0" . (string) $row['attribute_name']; if (!isset($actual[$key])) { $actual[$key] = ['variable' => (string) $row['source_name'], 'name' => (string) $row['attribute_name'], 'values' => []]; @@ -290,7 +290,7 @@ private function assertCanonicalExpectation(PDO $pdo, string $datasetName, strin 'utf8_source_encoding' => self::assertSame('UTF-8', strtoupper((string) $this->scalar($pdo, 'SELECT source_encoding FROM dataset WHERE dataset_id = ?', [$datasetId]))), 'string_over_255_bytes' => self::assertGreaterThan(0, $count("SELECT COUNT(*) FROM variable WHERE dataset_id = ? AND storage_kind = 'string' AND declared_string_width > 255")), 'dataset_attribute_arrays' => self::assertGreaterThan(1, $count('SELECT COUNT(*) FROM dataset_attribute WHERE dataset_id = ?')), - 'variable_attribute_arrays' => self::assertGreaterThan(1, $count('SELECT COUNT(*) FROM variable_attribute attribute JOIN variable ON variable.variable_id = attribute.variable_id WHERE variable.dataset_id = ?')), + 'variable_attribute_arrays' => self::assertGreaterThan(1, $count('SELECT COUNT(*) FROM variable_attribute attribute_row JOIN variable ON variable.variable_id = attribute_row.variable_id WHERE variable.dataset_id = ?')), 'variable_sets_ordered' => self::assertGreaterThan(0, $count('SELECT COUNT(*) FROM variable_set WHERE dataset_id = ? AND source_ordinal IS NOT NULL')), 'multiple_response_md' => self::assertGreaterThan(0, $count("SELECT COUNT(*) FROM multiple_response_set WHERE dataset_id = ? AND set_kind = 'MD'")), 'multiple_response_mc' => self::assertGreaterThan(0, $count("SELECT COUNT(*) FROM multiple_response_set WHERE dataset_id = ? AND set_kind = 'MC'")), @@ -425,7 +425,7 @@ private function profiles(): array if (in_array('sqlite', PDO::getAvailableDrivers(), true)) { $profiles['sqlite'] = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); } - foreach (['mysql' => 'OPENSTATSPEC_MYSQL', 'mariadb' => 'OPENSTATSPEC_MARIADB', 'postgresql' => 'OPENSTATSPEC_PG'] as $name => $prefix) { + foreach (['mysql' => 'OPENSTATSPEC_MYSQL', 'mariadb' => 'OPENSTATSPEC_MARIADB', 'dolt' => 'OPENSTATSPEC_DOLT', 'postgresql' => 'OPENSTATSPEC_PG'] as $name => $prefix) { $dsn = getenv($prefix . '_DSN'); $driver = $name === 'postgresql' ? 'pgsql' : 'mysql'; if (!is_string($dsn) || $dsn === '' || !in_array($driver, PDO::getAvailableDrivers(), true)) { diff --git a/tests/Integration/ServerCatalogMigrationTest.php b/tests/Integration/ServerCatalogMigrationTest.php index f454b98..11834d5 100644 --- a/tests/Integration/ServerCatalogMigrationTest.php +++ b/tests/Integration/ServerCatalogMigrationTest.php @@ -5,6 +5,7 @@ namespace OpenStatSpec\Tests\Integration; use OpenStatSpec\Spss\SpssAdapter; +use OpenStatSpec\Sql\MySqlIndexIntrospection; use PDO; use PDOException; use PHPUnit\Framework\TestCase; @@ -106,9 +107,10 @@ private function dropSetOrdinalUniqueKey(PDO $pdo, string $table): void return; } - $statement = $pdo->prepare("SELECT index_name FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? GROUP BY index_name HAVING MIN(non_unique) = 0 AND GROUP_CONCAT(column_name ORDER BY seq_in_index SEPARATOR ',') = 'dataset_id,source_ordinal' LIMIT 1"); + $statement = $pdo->prepare('SELECT index_name, non_unique, column_name FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? ORDER BY index_name, seq_in_index'); $statement->execute([$table]); - $index = $statement->fetchColumn(); + $uniqueIndexes = MySqlIndexIntrospection::uniqueColumnLists($statement->fetchAll(PDO::FETCH_ASSOC)); + $index = array_search(['dataset_id', 'source_ordinal'], $uniqueIndexes, true); if (is_string($index)) { $pdo->exec('ALTER TABLE `' . str_replace('`', '``', $table) . '` DROP INDEX `' . str_replace('`', '``', $index) . '`'); } @@ -138,7 +140,7 @@ private function assertNotNullAndUnique(PDO $pdo, string $name): void private function connections(): array { $connections = []; - foreach (['mysql' => 'OPENSTATSPEC_MYSQL', 'mariadb' => 'OPENSTATSPEC_MARIADB', 'postgresql' => 'OPENSTATSPEC_PG'] as $name => $prefix) { + foreach (['mysql' => 'OPENSTATSPEC_MYSQL', 'mariadb' => 'OPENSTATSPEC_MARIADB', 'dolt' => 'OPENSTATSPEC_DOLT', 'postgresql' => 'OPENSTATSPEC_PG'] as $name => $prefix) { $dsn = getenv($prefix . '_DSN'); $driver = $name === 'postgresql' ? 'pgsql' : 'mysql'; if (!is_string($dsn) || $dsn === '' || !in_array($driver, PDO::getAvailableDrivers(), true)) { diff --git a/tests/Integration/StrictSpssDictionaryConformanceTest.php b/tests/Integration/StrictSpssDictionaryConformanceTest.php index e89930a..1b014bc 100644 --- a/tests/Integration/StrictSpssDictionaryConformanceTest.php +++ b/tests/Integration/StrictSpssDictionaryConformanceTest.php @@ -86,6 +86,7 @@ private function profiles(): array foreach ([ 'mysql' => 'OPENSTATSPEC_MYSQL', 'mariadb' => 'OPENSTATSPEC_MARIADB', + 'dolt' => 'OPENSTATSPEC_DOLT', 'postgresql' => 'OPENSTATSPEC_PG', ] as $name => $prefix) { $dsn = getenv($prefix . '_DSN'); diff --git a/tests/Spss/SpssAdapterTest.php b/tests/Spss/SpssAdapterTest.php index 10f0815..c5aa9b1 100644 --- a/tests/Spss/SpssAdapterTest.php +++ b/tests/Spss/SpssAdapterTest.php @@ -6,6 +6,7 @@ use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Spss\GuardedImportSpssEngine; use OpenStatSpec\Spss\PhpSpssEngine; use OpenStatSpec\Spss\SpssAdapter; use OpenStatSpec\Spss\SpssSourceNormalizer; @@ -290,6 +291,360 @@ public function testTypedDatasetImportCreatesWideTableAndExportPreservesSupporte self::assertNull($multipleResponseSets[1]->countedValue); } + public function testGuardedEngineKeepsPhysicalDescriptorPathOutOfDatabaseProvenance(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $physicalDescriptorPath = '/proc/self/fd/17'; + $innerEngine = new FakeSpssEngine($this->fixture('sav')); + $engine = new GuardedImportSpssEngine($innerEngine, $physicalDescriptorPath, 'sav'); + $adapter = new SpssAdapter($pdo, $engine); + $verifiedSourceSha256 = str_repeat('a', 64); + + $adapter->import( + $engine->logicalPath(), + 'Guarded import', + verifiedSourceSha256: $verifiedSourceSha256, + ); + + self::assertSame('source.sav', $engine->logicalPath()); + self::assertSame($physicalDescriptorPath, $innerEngine->lastReadPath()); + self::assertSame( + [['source_hash' => $verifiedSourceSha256]], + self::rows($pdo, 'SELECT source_hash FROM dataset WHERE dataset_name = "Guarded import"'), + ); + $journal = self::rows( + $pdo, + 'SELECT target_path, engine_details FROM operation_catalog WHERE direction = "import"', + ); + self::assertSame('source.sav', $journal[0]['target_path']); + self::assertStringNotContainsString($physicalDescriptorPath, $journal[0]['engine_details']); + self::assertSame([], self::rows($pdo, 'SELECT source_item FROM fidelity_event_catalog')); + self::assertSame([], self::rows($pdo, 'SELECT source_item FROM fidelity_event')); + } + + public function testGuardedEngineSanitizesPhysicalDescriptorReadFailureBeforeJournaling(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $physicalDescriptorPath = '/proc/self/fd/17'; + $innerEngine = new FakeSpssEngine( + $this->fixture('sav'), + new RuntimeException('Failed to read ' . $physicalDescriptorPath), + ); + $engine = new GuardedImportSpssEngine($innerEngine, $physicalDescriptorPath, 'sav'); + $adapter = new SpssAdapter($pdo, $engine); + + try { + $adapter->import( + $engine->logicalPath(), + 'Rejected guarded import', + verifiedSourceSha256: str_repeat('a', 64), + ); + self::fail('A guarded inner read failure was not reported.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertStringContainsString('source.sav', $exception->getMessage()); + self::assertStringNotContainsString($physicalDescriptorPath, $exception->getMessage()); + self::assertNull($exception->getPrevious()); + } + + self::assertSame($physicalDescriptorPath, $innerEngine->lastReadPath()); + $persisted = json_encode([ + 'operation_catalog' => self::rows($pdo, 'SELECT * FROM operation_catalog'), + 'fidelity_event_catalog' => self::rows($pdo, 'SELECT * FROM fidelity_event_catalog'), + 'operation' => self::rows($pdo, 'SELECT * FROM operation'), + 'fidelity_event' => self::rows($pdo, 'SELECT * FROM fidelity_event'), + 'dataset' => self::rows($pdo, 'SELECT * FROM dataset'), + ], JSON_THROW_ON_ERROR); + self::assertStringContainsString('source.sav', $persisted); + self::assertStringNotContainsString($physicalDescriptorPath, $persisted); + } + + public function testGuardedEngineRejectsReturnedTechnicalFormatMismatchWithoutJournalLeak(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $physicalDescriptorPath = '/proc/self/fd/17'; + $innerEngine = new FakeSpssEngine($this->fixture('zsav')); + $engine = new GuardedImportSpssEngine($innerEngine, $physicalDescriptorPath, 'sav'); + $adapter = new SpssAdapter($pdo, $engine); + + try { + $adapter->import( + $engine->logicalPath(), + 'Rejected guarded format mismatch', + verifiedSourceSha256: str_repeat('a', 64), + ); + self::fail('A guarded dataset with mismatched technical source format was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertSame( + 'The guarded SPSS source could not be read for logical path source.sav.', + $exception->getMessage(), + ); + self::assertStringNotContainsString($physicalDescriptorPath, $exception->getMessage()); + self::assertNull($exception->getPrevious()); + } + + self::assertSame($physicalDescriptorPath, $innerEngine->lastReadPath()); + self::assertSame( + [[ + 'target_path' => 'source.sav', + 'source_format' => 'sav', + 'status' => 'failed', + 'failure_code' => 'invalid_source_dataset', + ]], + self::rows( + $pdo, + 'SELECT target_path, source_format, status, failure_code FROM operation_catalog', + ), + ); + $persisted = json_encode([ + 'operation_catalog' => self::rows($pdo, 'SELECT * FROM operation_catalog'), + 'fidelity_event_catalog' => self::rows($pdo, 'SELECT * FROM fidelity_event_catalog'), + 'operation' => self::rows($pdo, 'SELECT * FROM operation'), + 'fidelity_event' => self::rows($pdo, 'SELECT * FROM fidelity_event'), + 'dataset' => self::rows($pdo, 'SELECT * FROM dataset'), + ], JSON_THROW_ON_ERROR); + self::assertStringContainsString('source.sav', $persisted); + self::assertStringNotContainsString($physicalDescriptorPath, $persisted); + self::assertSame([], self::rows($pdo, 'SELECT * FROM dataset')); + } + + public function testGuardedEngineRejectsDescriptorBearingNestedIdentityWithoutJournaling(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $physicalDescriptorPath = '/proc/self/fd/17'; + $maliciousIdentities = [ + ['package' => 'malicious', 'nested' => ['detail' => $physicalDescriptorPath]], + ['package' => 'malicious', 'nested' => ['detail' => 'opened /dev/fd/22']], + ['package' => 'malicious', 'nested' => ['descriptor-/proc/123/fd/8' => true]], + ]; + + foreach ($maliciousIdentities as $maliciousIdentity) { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $innerEngine = new FakeSpssEngine( + $this->fixture(), + identityOverride: $maliciousIdentity, + ); + $engine = new GuardedImportSpssEngine($innerEngine, $physicalDescriptorPath, 'sav'); + $adapter = new SpssAdapter($pdo, $engine); + + try { + $adapter->import( + $engine->logicalPath(), + 'Rejected guarded identity', + verifiedSourceSha256: str_repeat('a', 64), + ); + self::fail('A descriptor-bearing engine identity was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertSame( + 'The guarded SPSS engine identity is not safe for journaling.', + $exception->getMessage(), + ); + self::assertDoesNotMatchRegularExpression( + '~(?:/proc/(?:self|thread-self|[0-9]+)/fd/[0-9]+|/dev/fd/[0-9]+)~', + $exception->getMessage(), + ); + self::assertNull($exception->getPrevious()); + } + + self::assertSame( + [], + self::rows( + $pdo, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('operation_catalog', 'fidelity_event_catalog')", + ), + ); + self::assertSame([], self::rows($pdo, 'SELECT * FROM operation')); + self::assertSame([], self::rows($pdo, 'SELECT * FROM fidelity_event')); + self::assertSame([], self::rows($pdo, 'SELECT * FROM dataset')); + } + } + + public function testGuardedEngineSupportsZsavLogicalPathWithDevFdSource(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $physicalDescriptorPath = '/dev/fd/23'; + $innerEngine = new FakeSpssEngine($this->fixture('zsav')); + $engine = new GuardedImportSpssEngine($innerEngine, $physicalDescriptorPath, 'zsav'); + $adapter = new SpssAdapter($pdo, $engine); + $verifiedSourceSha256 = str_repeat('b', 64); + + $adapter->import( + $engine->logicalPath(), + 'Guarded ZSAV import', + verifiedSourceSha256: $verifiedSourceSha256, + ); + + self::assertSame('source.zsav', $engine->logicalPath()); + self::assertSame($physicalDescriptorPath, $innerEngine->lastReadPath()); + self::assertSame( + [['target_path' => 'source.zsav', 'source_format' => 'zsav']], + self::rows( + $pdo, + 'SELECT target_path, source_format FROM operation_catalog WHERE direction = "import"', + ), + ); + self::assertSame( + [['source_format' => 'zsav', 'source_hash' => $verifiedSourceSha256]], + self::rows( + $pdo, + 'SELECT source_format, source_hash FROM dataset WHERE dataset_name = "Guarded ZSAV import"', + ), + ); + } + + public function testGuardedEngineRejectsNonJsonSafeIdentityWithoutJournaling(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $physicalDescriptorPath = '/proc/self/fd/17'; + $unsafeIdentities = [ + ['package' => 'malicious', 'detail' => INF], + ['package' => 'malicious', 'detail' => NAN], + ['package' => 'malicious', 'detail' => "\xC3\x28"], + ['package' => 'malicious', "invalid-key-\xC3\x28" => true], + ]; + + foreach ($unsafeIdentities as $unsafeIdentity) { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $innerEngine = new FakeSpssEngine( + $this->fixture(), + identityOverride: $unsafeIdentity, + ); + $engine = new GuardedImportSpssEngine($innerEngine, $physicalDescriptorPath, 'sav'); + $adapter = new SpssAdapter($pdo, $engine); + + try { + $adapter->import( + $engine->logicalPath(), + 'Rejected non-JSON-safe identity', + verifiedSourceSha256: str_repeat('a', 64), + ); + self::fail('A non-JSON-safe engine identity was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertSame( + 'The guarded SPSS engine identity is not safe for journaling.', + $exception->getMessage(), + ); + self::assertNull($exception->getPrevious()); + } + + self::assertSame( + [], + self::rows( + $pdo, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('operation_catalog', 'fidelity_event_catalog')", + ), + ); + self::assertSame([], self::rows($pdo, 'SELECT * FROM operation')); + self::assertSame([], self::rows($pdo, 'SELECT * FROM fidelity_event')); + self::assertSame([], self::rows($pdo, 'SELECT * FROM dataset')); + } + } + + public function testImportRejectsEphemeralDescriptorPathsBeforeDatabaseMutation(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + foreach (['/proc/self/fd/7', '/proc/thread-self/fd/7', '/proc/123/fd/7', '/dev/fd/7'] as $sourcePath) { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $adapter = new SpssAdapter($pdo, new FakeSpssEngine($this->fixture())); + + try { + $adapter->import( + $sourcePath, + 'Rejected descriptor import', + verifiedSourceSha256: str_repeat('a', 64), + ); + self::fail('An ephemeral descriptor path was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + } + + self::assertSame( + [], + self::rows($pdo, "SELECT name FROM sqlite_master WHERE type = 'table'"), + ); + } + } + + public function testImportWithoutExplicitHashRetainsReadablePathHashing(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + $sourcePath = sys_get_temp_dir() . '/openstatspec-source-hash-' . uniqid('', true) . '.sav'; + try { + file_put_contents($sourcePath, 'source bytes'); + $expectedSourceSha256 = hash_file('sha256', $sourcePath); + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $adapter = new SpssAdapter($pdo, new FakeSpssEngine($this->fixture())); + + $adapter->import($sourcePath, 'Readable source'); + + self::assertSame( + [['source_hash' => $expectedSourceSha256]], + self::rows($pdo, 'SELECT source_hash FROM dataset WHERE dataset_name = "Readable source"'), + ); + } finally { + @unlink($sourcePath); + } + } + + public function testImportRejectsInvalidVerifiedSha256BeforeDatabaseMutation(): void + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + } + + foreach (['abc', str_repeat('A', 64)] as $invalidSourceSha256) { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $adapter = new SpssAdapter($pdo, new FakeSpssEngine($this->fixture())); + + try { + $adapter->import( + 'source.sav', + 'Rejected import', + verifiedSourceSha256: $invalidSourceSha256, + ); + self::fail('An invalid verified source SHA-256 was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + } + + self::assertSame( + [], + self::rows($pdo, "SELECT name FROM sqlite_master WHERE type = 'table'"), + ); + } + } + public function testPhpSpssEngineWritesAndReadsTypedDataset(): void { $target = sys_get_temp_dir() . '/openstatspec-v3-' . uniqid('', true) . '.sav'; @@ -640,7 +995,7 @@ public function testV3MigrationRestoresSetOrdinalConstraints(): void $pdo->exec("INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES ('vs-duplicate', 'dataset-v3', 1, 'Duplicate')"); } - private function fixture(): Dataset + private function fixture(string $sourceFormat = 'zsav'): Dataset { return new Dataset( new VariableDictionary([ @@ -696,7 +1051,7 @@ private function fixture(): Dataset ], ), new FileTechnicalMetadata( - sourceFormat: 'zsav', + sourceFormat: $sourceFormat, recordType: '$FL3', sourceVersion: 'OpenStatSpec 0.1', provenance: 'Päritolu: küsitlus', diff --git a/tests/Sql/MySqlIndexIntrospectionTest.php b/tests/Sql/MySqlIndexIntrospectionTest.php new file mode 100644 index 0000000..7793c65 --- /dev/null +++ b/tests/Sql/MySqlIndexIntrospectionTest.php @@ -0,0 +1,22 @@ + ['dataset_id', 'source_ordinal'], + ], MySqlIndexIntrospection::uniqueColumnLists([ + ['INDEX_NAME' => 'uq_dataset_ordinal', 'NON_UNIQUE' => 0, 'COLUMN_NAME' => 'dataset_id'], + ['Index_Name' => 'uq_dataset_ordinal', 'Non_Unique' => '0', 'Column_Name' => 'source_ordinal'], + ['INDEX_NAME' => 'ignored_non_unique', 'NON_UNIQUE' => 1, 'COLUMN_NAME' => 'dataset_id'], + ])); + } +} diff --git a/tests/Sql/MySqlSchemaTest.php b/tests/Sql/MySqlSchemaTest.php index 3b2921c..548203e 100644 --- a/tests/Sql/MySqlSchemaTest.php +++ b/tests/Sql/MySqlSchemaTest.php @@ -6,6 +6,7 @@ use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Sql\DoltProfile; use OpenStatSpec\Sql\MySqlSchema; use PDO; use PHPUnit\Framework\TestCase; @@ -63,6 +64,26 @@ public function testWideTableRejectsColumnsBeyondMysqlInnoDbCapability(): void } } + public function testInjectedDoltProfileAppliesThe305VariableEnvelope(): void + { + $schema = new MySqlSchema($this->createMock(PDO::class), new DoltProfile()); + $variables = []; + for ($index = 1; $index <= 305; ++$index) { + $variables[] = ['name' => 'v' . $index, 'type' => 'numeric']; + } + + $definition = $schema->wideTableDefinition('dolt fixture', $variables); + self::assertCount(305, $definition->columns); + + $variables[] = ['name' => 'v306', 'type' => 'numeric']; + try { + $schema->wideTableDefinition('too wide', $variables); + self::fail('Expected the injected Dolt profile to reject 306 source variables.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + } + } + public function testDdlCanBeExecutedThroughThePdoAbstractionWithoutAServer(): void { $pdo = $this->createMock(PDO::class); diff --git a/tests/Sql/MySqlWideTableImporterTest.php b/tests/Sql/MySqlWideTableImporterTest.php index 725ec63..7f3c9d6 100644 --- a/tests/Sql/MySqlWideTableImporterTest.php +++ b/tests/Sql/MySqlWideTableImporterTest.php @@ -4,9 +4,14 @@ namespace OpenStatSpec\Tests\Sql; +use OpenStatSpec\Core\DiagnosticCode; +use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Sql\DoltProfile; use OpenStatSpec\Sql\MySqlWideTableImporter; use PDO; use PDOStatement; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; use RuntimeException; @@ -263,15 +268,34 @@ public function testImportsV3ExtensionMetadataAndPreservesOrderedMembers(): void ], $multipleResponseMemberRows); } - public function testCompensatesForImplicitDdlCommitWhenCaseInsertFails(): void + public function testInjectedDoltProfileRejects306VariablesBeforeDdl(): void + { + $pdo = $this->createMock(PDO::class); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + $variables = []; + for ($index = 1; $index <= 306; ++$index) { + $variables[] = ['name' => 'v' . $index, 'type' => 'numeric']; + } + + try { + (new MySqlWideTableImporter($pdo, new DoltProfile()))->import([ + 'variables' => $variables, + 'data' => [], + ], 'too wide'); + self::fail('Expected Dolt preflight to reject 306 source variables.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + } + } + + public function testRolledBackCaseInsertFailureDropsOnlyAttemptPhysicalTable(): void { $pdo = $this->createMock(PDO::class); $dataset = $this->createMock(PDOStatement::class); $variables = $this->createMock(PDOStatement::class); $cases = $this->createMock(PDOStatement::class); - $cleanup = $this->createMock(PDOStatement::class); $executedSql = []; - $cleanupSql = []; $pdo->expects(self::exactly(22))->method('exec')->willReturnCallback(function (string $sql) use (&$executedSql): int { $executedSql[] = $sql; @@ -282,26 +306,15 @@ public function testCompensatesForImplicitDdlCommitWhenCaseInsertFails(): void $pdo->expects(self::once())->method('inTransaction')->willReturn(true); $pdo->expects(self::once())->method('rollBack')->willReturn(true); $pdo->expects(self::never())->method('commit'); - $pdo->expects(self::exactly(20))->method('prepare')->willReturnCallback( - static function (string $sql) use ($dataset, $variables, $cases, $cleanup, &$cleanupSql): PDOStatement { - if (str_starts_with($sql, 'DELETE FROM ')) { - $cleanupSql[] = $sql; - - return $cleanup; - } - - return match (true) { - str_starts_with($sql, 'INSERT INTO datasets') => $dataset, - str_starts_with($sql, 'INSERT INTO variables') => $variables, - default => $cases, - }; - }, + $pdo->expects(self::exactly(3))->method('prepare')->willReturnOnConsecutiveCalls( + $dataset, + $variables, + $cases, ); $dataset->expects(self::once())->method('execute')->willReturn(true); $variables->expects(self::once())->method('execute')->willReturn(true); $cases->expects(self::once())->method('execute')->willThrowException(new RuntimeException('insert failed')); - $cleanup->expects(self::exactly(17))->method('execute')->with(['customer survey'])->willReturn(true); $this->expectExceptionMessage('insert failed'); try { @@ -312,15 +325,139 @@ static function (string $sql) use ($dataset, $variables, $cases, $cleanup, &$cle } finally { self::assertStringStartsWith('DROP TABLE IF EXISTS ', $executedSql[21] ?? ''); self::assertStringContainsString('dataset_customer_survey', $executedSql[21] ?? ''); - self::assertSame([ - 'DELETE FROM multiple_response_set_members WHERE dataset_name = ?', - 'DELETE FROM multiple_response_sets WHERE dataset_name = ?', - 'DELETE FROM variable_set_members WHERE dataset_name = ?', - 'DELETE FROM variable_sets WHERE dataset_name = ?', - 'DELETE FROM variable_attributes WHERE dataset_name = ?', - 'DELETE FROM file_attributes WHERE dataset_name = ?', - 'DELETE FROM variable_roles WHERE dataset_name = ?', - ], array_slice($cleanupSql, 0, 7)); } } + + #[DataProvider('nonFiniteValues')] + public function testDoltRejectsEveryNonFiniteValueBeforeDdl(float $value): void + { + $pdo = $this->doltPreflightPdo(); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + + try { + (new MySqlWideTableImporter($pdo, new DoltProfile()))->import([ + 'variables' => [['name' => 'Score', 'type' => 'numeric']], + 'data' => [[$value]], + ], 'non finite'); + self::fail('Dolt accepted a non-finite value.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('non-finite', $exception->getMessage()); + } + } + + public function testDoltRejectsMissingWeightAndNonFiniteDictionaryBeforeDdl(): void + { + foreach ([ + [ + 'variables' => [['name' => 'Score', 'type' => 'numeric']], + 'weightVariableName' => 'Missing', + 'data' => [[1.0]], + ], + [ + 'variables' => [[ + 'name' => 'Score', + 'type' => 'numeric', + 'missingFormat' => 1, + 'missingValues' => [NAN], + ]], + 'data' => [[1.0]], + ], + ] as $source) { + $pdo = $this->doltPreflightPdo(); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + + try { + (new MySqlWideTableImporter($pdo, new DoltProfile()))->import($source, 'invalid metadata'); + self::fail('Dolt accepted invalid source metadata.'); + } catch (UnsupportedOperation) { + } + } + } + + /** @param array $metadata */ + #[DataProvider('invalidV3Metadata')] + public function testDoltRejectsInvalidV3MetadataBeforeDdl(array $metadata, DiagnosticCode $diagnosticCode): void + { + $pdo = $this->doltPreflightPdo(); + $pdo->expects(self::never())->method('beginTransaction'); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + $source = array_merge([ + 'variables' => [[ + 'name' => 'Score', + 'type' => 'numeric', + 'role' => 0, + 'attributes' => [], + ]], + 'data' => [[1.0]], + ], $metadata); + + try { + (new MySqlWideTableImporter($pdo, new DoltProfile()))->import($source, 'invalid v3 metadata'); + self::fail('Dolt accepted invalid V3 metadata.'); + } catch (UnsupportedOperation $exception) { + self::assertSame($diagnosticCode, $exception->diagnosticCode); + } + } + + /** @return iterable, DiagnosticCode}> */ + public static function invalidV3Metadata(): iterable + { + yield 'malformed file attributes' => [ + ['fileAttributes' => [['name' => 'Broken', 'values' => 'not-a-list']]], + DiagnosticCode::InvalidSourceDataset, + ]; + yield 'unknown variable-set member' => [ + ['variableSets' => [['name' => 'Core', 'variableNames' => ['Missing']]]], + DiagnosticCode::InvalidSourceDataset, + ]; + yield 'duplicate variable-set member' => [ + ['variableSets' => [['name' => 'Core', 'variableNames' => ['Score', 'Score']]]], + DiagnosticCode::InvalidSourceDataset, + ]; + yield 'duplicate multiple-response-set member' => [[ + 'multipleResponseSets' => [[ + 'name' => '$Set', + 'type' => 'dichotomy', + 'variableNames' => ['Score', 'Score'], + 'label' => null, + 'countedValue' => 1.0, + 'categoryLabels' => 'counted_values', + 'labelSource' => 'variable_label', + ]], + ], DiagnosticCode::InvalidSourceDataset]; + yield 'non-finite multiple-response counted value' => [[ + 'multipleResponseSets' => [[ + 'name' => '$Set', + 'type' => 'dichotomy', + 'variableNames' => ['Score'], + 'label' => null, + 'countedValue' => INF, + 'categoryLabels' => 'counted_values', + 'labelSource' => 'variable_label', + ]], + ], DiagnosticCode::TargetCapabilityExceeded]; + } + + /** @return iterable */ + public static function nonFiniteValues(): iterable + { + yield 'NaN' => [NAN]; + yield 'positive infinity' => [INF]; + yield 'negative infinity' => [-INF]; + } + + /** @return PDO&MockObject */ + private function doltPreflightPdo(): PDO + { + $pdo = $this->createMock(PDO::class); + $statement = $this->createMock(PDOStatement::class); + $pdo->method('query')->with('SELECT @@max_allowed_packet')->willReturn($statement); + $statement->method('fetchColumn')->willReturn('1073741824'); + + return $pdo; + } } diff --git a/tests/Sql/PdoSqlProfileTest.php b/tests/Sql/PdoSqlProfileTest.php index b0542fd..2cf2e27 100644 --- a/tests/Sql/PdoSqlProfileTest.php +++ b/tests/Sql/PdoSqlProfileTest.php @@ -6,6 +6,7 @@ use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Sql\DoltProfile; use OpenStatSpec\Sql\MySqlProfile; use OpenStatSpec\Sql\PostgreSqlProfile; use OpenStatSpec\Sql\SqliteProfile; @@ -29,6 +30,57 @@ public function testProfilesDeclarePortableSqlRulesWithoutServerConnections(): v self::assertSame("`name`", $mysql->quoteIdentifier('name')); self::assertSame(1599, $postgres->maximumSourceVariables()); self::assertSame(1016, $mysql->maximumSourceVariables()); + + $dolt = new DoltProfile(); + self::assertSame(305, $dolt->maximumSourceVariables()); + self::assertSame(65_504, $dolt->maximumRowBytes()); + self::assertSame('bytes', $dolt->identifierLimitUnit()); + } + + public function testDoltEnvelopeAccepts305VariablesAndRejects306(): void + { + $profile = new DoltProfile(); + $profile->assertCanRepresent(305); + self::assertSame(305, $profile->maximumSourceVariables()); + + $this->expectException(UnsupportedOperation::class); + $this->expectExceptionMessage('at most 305 source variables'); + $profile->assertCanRepresent(306); + } + + public function testDoltActivePreflightAccepts65504BytesAndRejects65505(): void + { + $pdo = $this->createMock(PDO::class); + $statement = $this->createMock(PDOStatement::class); + $pdo->method('query')->with('SELECT @@max_allowed_packet')->willReturn($statement); + $statement->method('fetchColumn')->willReturn('1073741824'); + $profile = new DoltProfile(); + $variables = [['name' => 'payload', 'type' => 'string', 'width' => 65_505]]; + + $profile->assertDataset($variables, [[str_repeat('a', 65_504)]], $pdo); + + try { + $profile->assertDataset($variables, [[str_repeat('a', 65_505)]], $pdo); + self::fail('Dolt accepted an encoded case beyond its 65,504-byte envelope.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('encoded case payload is 65505 bytes', $exception->getMessage()); + self::assertStringContainsString('limit is 65504 bytes', $exception->getMessage()); + } + } + + public function testDoltLimitSourcesDoNotClaimInnoDbStorageLimits(): void + { + $pdo = $this->createMock(PDO::class); + $statement = $this->createMock(PDOStatement::class); + $pdo->expects(self::once())->method('query')->with('SELECT @@max_allowed_packet')->willReturn($statement); + $statement->expects(self::once())->method('fetchColumn')->willReturn('1073741824'); + + $sources = (new DoltProfile())->effectiveLimitSources($pdo); + self::assertStringNotContainsString('InnoDB', implode("\n", $sources)); + self::assertStringContainsString('305 source variables', $sources['maximum_source_variables']); + self::assertStringContainsString('65,504-byte', $sources['maximum_row_bytes']); + self::assertStringContainsString('active @@max_allowed_packet', $sources['maximum_statement_bytes']); } public function testProfilesRejectWideTablesBeyondDeclaredCapabilities(): void diff --git a/tests/Sql/PostgreSqlWideTableImporterTest.php b/tests/Sql/PostgreSqlWideTableImporterTest.php index b98e8c4..465264f 100644 --- a/tests/Sql/PostgreSqlWideTableImporterTest.php +++ b/tests/Sql/PostgreSqlWideTableImporterTest.php @@ -236,19 +236,12 @@ public function testImportsVariableDisplayMetadataThroughPdoTransaction(): void public function testRejectsNullStringBeforeCommitAndRollsBack(): void { $pdo = $this->createMock(PDO::class); - $dataset = $this->createMock(PDOStatement::class); - $variables = $this->createMock(PDOStatement::class); - $cases = $this->createMock(PDOStatement::class); - - $pdo->expects(self::once())->method('beginTransaction')->willReturn(true); + $pdo->expects(self::never())->method('beginTransaction'); $pdo->expects(self::never())->method('commit'); - $pdo->expects(self::once())->method('inTransaction')->willReturn(true); - $pdo->expects(self::once())->method('rollBack')->willReturn(true); - $pdo->expects(self::atLeast(18))->method('exec')->willReturn(0); - $pdo->expects(self::exactly(3))->method('prepare')->willReturnOnConsecutiveCalls($dataset, $variables, $cases); - $dataset->method('execute')->willReturn(true); - $variables->method('execute')->willReturn(true); - $cases->expects(self::never())->method('execute'); + $pdo->expects(self::never())->method('inTransaction'); + $pdo->expects(self::never())->method('rollBack'); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); $this->expectExceptionMessage('SPSS string values must be non-null strings.'); (new PostgreSqlWideTableImporter($pdo))->import([ diff --git a/tests/Sql/ServerIdentityTest.php b/tests/Sql/ServerIdentityTest.php new file mode 100644 index 0000000..f0da13c --- /dev/null +++ b/tests/Sql/ServerIdentityTest.php @@ -0,0 +1,206 @@ +mysqlPdo('8.0.33', [ + 'SELECT @@version' => '8.0.33', + 'SELECT @@version_comment' => ' Dolt ', + 'SELECT DOLT_VERSION()' => '2.2.2', + ]); + + $identity = ServerIdentity::detect($pdo); + self::assertSame('dolt', $identity->profileName); + self::assertSame('2.2.2', $identity->serverVersion); + self::assertSame('8.0.33', $identity->rawServerVersion); + self::assertSame('@@version + @@version_comment + DOLT_VERSION()', $identity->identitySource); + self::assertSame([ + 'PDO::ATTR_SERVER_VERSION' => '8.0.33', + '@@version' => '8.0.33', + '@@version_comment' => 'Dolt', + 'DOLT_VERSION()' => '2.2.2', + ], $identity->probeResults); + + $connection = new Connection($pdo); + self::assertTrue($connection->claimedSupported); + self::assertSame('Dolt 2.2.2', $connection->matchedClaim); + self::assertInstanceOf(DoltProfile::class, $connection->profile); + } + + public function testDoltCommentWithoutProductVersionFailsClosedEvenWithSupportedLookingWireVersion(): void + { + $pdo = $this->mysqlPdo('8.4.6', [ + 'SELECT @@version' => '8.4.6', + 'SELECT @@version_comment' => 'Dolt', + 'SELECT DOLT_VERSION()' => null, + ]); + + try { + new Connection($pdo); + self::fail('A Dolt identity without DOLT_VERSION() must fail closed.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('DOLT_VERSION()', $exception->getMessage()); + } + } + + public function testNonDoltCommentDoesNotProbeDoltFunctionAndUsesMySqlProfile(): void + { + $pdo = $this->nonDoltPdo('8.4.6', 'MySQL Community Server - GPL'); + + $connection = new Connection($pdo); + self::assertSame('mysql', $connection->profileName); + self::assertSame('8.4.6', $connection->serverVersion); + self::assertSame('@@version + @@version_comment', $connection->identitySource); + self::assertTrue($connection->claimedSupported); + self::assertSame('MySQL 8.4.x or 9.7.x', $connection->matchedClaim); + } + + public function testNonDoltCommentDoesNotProbeDoltFunctionAndUsesMariaDbProfile(): void + { + $version = '11.4.5-MariaDB-ubu2404'; + $pdo = $this->nonDoltPdo($version, 'mariadb.org binary distribution'); + + $connection = new Connection($pdo); + self::assertSame('mariadb', $connection->profileName); + self::assertSame($version, $connection->serverVersion); + self::assertSame('@@version + @@version_comment', $connection->identitySource); + self::assertTrue($connection->claimedSupported); + self::assertSame('MariaDB 11.4.x, 11.8.x or 12.3.x', $connection->matchedClaim); + } + + public function testUnvalidatedDoltVersionIsDetectedButNotClaimed(): void + { + $pdo = $this->mysqlPdo('8.0.33', [ + 'SELECT @@version' => '8.0.33', + 'SELECT @@version_comment' => 'Dolt', + 'SELECT DOLT_VERSION()' => '2.2.3', + ]); + + $connection = new Connection($pdo); + self::assertSame('dolt', $connection->profileName); + self::assertSame('2.2.3', $connection->serverVersion); + self::assertFalse($connection->claimedSupported); + self::assertNull($connection->matchedClaim); + } + + public function testUnknownMySqlWireProductFailsBeforeAnyMutation(): void + { + $pdo = $this->mysqlPdo('8.4.6', [ + 'SELECT @@version' => '8.4.6', + 'SELECT @@version_comment' => 'Acme compatible database', + ]); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + + try { + new Connection($pdo); + self::fail('An unknown MySQL-wire product was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('product is unknown', $exception->getMessage()); + } + } + + public function testConflictingMySqlAndMariaDbSignalsFailBeforeAnyMutation(): void + { + $pdo = $this->mysqlPdo('11.4.5-MariaDB', [ + 'SELECT @@version' => '11.4.5-MariaDB', + 'SELECT @@version_comment' => 'MySQL Community Server - GPL', + ]); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + + try { + new Connection($pdo); + self::fail('Conflicting MySQL-family identity signals were accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('conflicting product identity', $exception->getMessage()); + } + } + + public function testMissingVersionProbeFailsBeforeDoltProbeOrMutation(): void + { + $pdo = $this->mysqlPdo('8.0.33', [ + 'SELECT @@version' => null, + 'SELECT @@version_comment' => 'Dolt', + 'SELECT DOLT_VERSION()' => '2.2.2', + ]); + $pdo->expects(self::never())->method('exec'); + $pdo->expects(self::never())->method('prepare'); + + try { + new Connection($pdo); + self::fail('A missing @@version probe was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + self::assertStringContainsString('@@version', $exception->getMessage()); + } + } + + private function nonDoltPdo(string $wireVersion, string $versionComment): PDO + { + $pdo = $this->createMock(PDO::class); + $pdo->method('getAttribute')->willReturnCallback(static fn(int $attribute): string => match ($attribute) { + PDO::ATTR_DRIVER_NAME => 'mysql', + PDO::ATTR_SERVER_VERSION => $wireVersion, + default => '', + }); + $pdo->expects(self::exactly(2)) + ->method('query') + ->willReturnCallback(function (string $query) use ($wireVersion, $versionComment): PDOStatement { + $value = match ($query) { + 'SELECT @@version' => $wireVersion, + 'SELECT @@version_comment' => $versionComment, + default => throw new \LogicException('Unexpected identity probe: ' . $query), + }; + $statement = $this->createMock(PDOStatement::class); + $statement->method('fetchColumn')->willReturn($value); + + return $statement; + }); + + return $pdo; + } + + /** + * @param array $queryValues + * @return PDO&MockObject + */ + private function mysqlPdo(string $wireVersion, array $queryValues): PDO + { + $pdo = $this->createMock(PDO::class); + $pdo->method('getAttribute')->willReturnCallback(static fn(int $attribute): string => match ($attribute) { + PDO::ATTR_DRIVER_NAME => 'mysql', + PDO::ATTR_SERVER_VERSION => $wireVersion, + default => '', + }); + $pdo->method('query')->willReturnCallback(function (string $query) use ($queryValues): PDOStatement|false { + if (!array_key_exists($query, $queryValues)) { + return false; + } + $statement = $this->createMock(PDOStatement::class); + $statement->method('fetchColumn')->willReturn($queryValues[$query]); + + return $statement; + }); + + return $pdo; + } +} diff --git a/tests/Support/FakeSpssEngine.php b/tests/Support/FakeSpssEngine.php index 48fcfad..d029fe3 100644 --- a/tests/Support/FakeSpssEngine.php +++ b/tests/Support/FakeSpssEngine.php @@ -6,16 +6,27 @@ use OpenStatSpec\Spss\SpssEngine; use SPSS\Sav\Dataset; +use Throwable; final class FakeSpssEngine implements SpssEngine { /** @var list */ private array $writes = []; + private ?string $lastReadPath = null; - public function __construct(private Dataset $dataset) {} + /** @param ?array $identityOverride */ + public function __construct( + private Dataset $dataset, + private ?Throwable $readFailure = null, + private ?array $identityOverride = null, + ) {} public function identity(): array { + if ($this->identityOverride !== null) { + return $this->identityOverride; + } + return [ 'package' => 'fake-spss-engine', 'version' => 'test', @@ -52,9 +63,23 @@ public function capabilities(): array public function read(string $sourcePath): Dataset { + $this->lastReadPath = $sourcePath; + if ($this->readFailure !== null) { + throw $this->readFailure; + } + return $this->dataset; } + public function lastReadPath(): string + { + if ($this->lastReadPath === null) { + throw new \LogicException('No SAV dataset was read.'); + } + + return $this->lastReadPath; + } + public function write(string $targetPath, Dataset $dataset): void { $this->writes[] = ['targetPath' => $targetPath, 'dataset' => $dataset];