From 03e8526a52c2d7d67e2fce15676d763969d7c3e7 Mon Sep 17 00:00:00 2001 From: SimonTaurus Date: Sun, 30 Aug 2026 04:39:35 +0200 Subject: [PATCH] test(parity): compare against oold-js instead of oold-schema's scripts The reference implementation moved to OO-LD/oold-js, so the corpus and the implementation checking it are now separate checkouts: OOLD_SCHEMA_DIR for the examples and meta-schemas, OOLD_JS_DIR for the reference. CI pins the reference by tag and leaves the corpus tracking main, so a reference change is adopted deliberately while a specification change is still noticed on the next run. Closes #136 --- .github/workflows/main.yml | 30 ++++---- CLAUDE.md | 4 +- docs/architecture.md | 6 +- docs/how-to/validation.md | 18 ++--- tests/test_validation/conftest.py | 15 ++++ tests/test_validation/test_parity_live.py | 91 +++++++++++++---------- 6 files changed, 95 insertions(+), 69 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 41cd4d5..d9af30b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,23 +72,25 @@ jobs: run: make validate - name: Check parity against the reference harness + env: + # The reference is pinned, so a change to it is adopted by editing this line rather + # than arriving with the next push. The corpus is not: it tracks oold-schema's main, + # which is what lets this notice a specification change the fixture snapshot cannot. + OOLD_JS_REF: v0.1.0 run: | - # oold-schema owns the reference implementation. Comparing against its current main - # catches drift that the committed fixture snapshot cannot see. - # - # This gates, as of oold-schema switching its own CI to this validator. Until then the - # reference was still what checked that repository, so parity was informative and a - # disagreement had a second opinion behind it. Now nothing else validates the - # specification's own examples, which makes agreement between the two implementations - # the evidence the switch rests on - and evidence that cannot fail is not evidence. + # Two repositories: oold-schema owns the corpus and the meta-schemas, oold-js owns the + # reference implementation. Comparing against the corpus's current main catches drift + # that the committed fixture snapshot cannot see. # - # It stays valuable after the migration rather than becoming scaffolding: validate.mjs - # is being extracted to oold-js, so this becomes a standing cross-implementation - # conformance check, which is what catches an ambiguity in the specification rather - # than a bug in one port. + # This gates. Nothing else validates the specification's own examples since oold-schema + # switched its CI to this validator, which makes agreement between the two + # implementations the evidence that switch rests on - and evidence that cannot fail is + # not evidence. It is a standing cross-implementation conformance check, which is what + # catches an ambiguity in the specification rather than a bug in one port. git clone --depth 1 https://github.com/OO-LD/oold-schema.git /tmp/oold-schema - npm --prefix /tmp/oold-schema install --no-audit --no-fund - OOLD_SCHEMA_DIR=/tmp/oold-schema uv run python -m pytest \ + git clone --depth 1 --branch "$OOLD_JS_REF" https://github.com/OO-LD/oold-js.git /tmp/oold-js + npm --prefix /tmp/oold-js ci --no-audit --no-fund + OOLD_SCHEMA_DIR=/tmp/oold-schema OOLD_JS_DIR=/tmp/oold-js uv run python -m pytest \ tests/test_validation/test_parity_live.py -v --no-cov check-docs: diff --git a/CLAUDE.md b/CLAUDE.md index 21e6864..a4a8a8e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,10 +11,10 @@ make test # pytest with coverage make validate # run the validator over the committed fixtures make docs-test # strict docs build, fails on any warning -OOLD_SCHEMA_DIR=../oold-schema uv run pytest -m parity # compare against oold-schema's own validator +OOLD_SCHEMA_DIR=../oold-schema OOLD_JS_DIR=../oold-js uv run pytest -m parity # compare against the reference ``` -The parity tests skip silently without `OOLD_SCHEMA_DIR`, so a green `make test` does not mean +The parity tests skip silently without those variables, so a green `make test` does not mean parity holds. Run them explicitly when touching `src/oold/validation/`. ## Where the details live diff --git a/docs/architecture.md b/docs/architecture.md index 9f1ee84..cf7184b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -109,10 +109,8 @@ Backends are registered per IRI prefix via `set_resolver` / `set_backend`, so mu ### Validation Layer (optional) `oold.validation` checks that a schema is well formed and that its `@context` actually carries -every declared property into RDF. It is a native port of the reference harness in -[oold-schema](https://github.com/OO-LD/oold-schema)'s -[`scripts/validate.mjs`](https://github.com/OO-LD/oold-schema/blob/v1.0.0-rc.2/scripts/validate.mjs) -(pinned at `v1.0.0-rc.2`, since upstream intends to replace that script with this implementation), +every declared property into RDF. It is a native port of the reference harness +[oold-js](https://github.com/OO-LD/oold-js) (pinned at [`v0.1.0`](https://github.com/OO-LD/oold-js/tree/v0.1.0)), and reuses `pyld` from the serialization layer, so the JSON-LD half of it adds no dependencies. One pipeline backs three surfaces - the library API, the `oold validate` CLI, and an MCP server - diff --git a/docs/how-to/validation.md b/docs/how-to/validation.md index 01cb739..74d9e72 100644 --- a/docs/how-to/validation.md +++ b/docs/how-to/validation.md @@ -2,7 +2,7 @@ `oold` can check that an OO-LD schema is well formed and that an instance document conforms to the schema it names. It is a native Python port of the reference harness in -[oold-schema](https://github.com/OO-LD/oold-schema) (`scripts/validate.mjs`), so the two agree on +[oold-js](https://github.com/OO-LD/oold-js), so the two agree on verdicts, and it is available three ways: as a library, as a CLI, and as an MCP server. Install the extra: @@ -240,9 +240,8 @@ Transport is stdio. Tools: `validate_oold_schema`, `validate_oold_instance`, ## Differences from the reference harness The two are intended to agree on verdicts. Where they differ, it is deliberate. The comparison is -pinned to [`scripts/validate.mjs` at `v1.0.0-rc.2`](https://github.com/OO-LD/oold-schema/blob/v1.0.0-rc.2/scripts/validate.mjs), -a tag rather than a moving path, because upstream intends to replace that script with this -implementation: +pinned to [oold-js `v0.1.0`](https://github.com/OO-LD/oold-js/tree/v0.1.0), a tag rather than a +moving branch, so a change to the reference is adopted deliberately: | Difference | Why | |---|---| @@ -261,15 +260,16 @@ checkout. ## Testing against the upstream repository ```bash -OOLD_SCHEMA_DIR=../oold-schema uv run pytest tests/test_validation -q +OOLD_SCHEMA_DIR=../oold-schema OOLD_JS_DIR=../oold-js uv run pytest tests/test_validation -q ``` -Without that variable the parity tests skip and the suite stays self-contained. With it, the full -validator runs over oold-schema's own `examples/` and `examples/compliance/`, and the overall -verdict is compared against `node scripts/validate.mjs`. +Without those variables the parity tests skip and the suite stays self-contained. With them, the +full validator runs over oold-schema's own `examples/` and `examples/compliance/`, and the overall +verdict is compared against the oold-js reference run over the same corpus and the same +meta-schemas. The `format` assertions are pinned separately: `tests/data/format_parity.json` holds 98 outcomes captured from ajv as the reference configures it (`ajv-formats` in *full* mode, plus the -`iri`/`iri-reference` override `validate.mjs` applies), and the suite asserts Python agrees on +`iri`/`iri-reference` override the reference applies), and the suite asserts Python agrees on every one. Two are easy to get wrong: in full mode `time` requires an offset and `email` requires a dotted domain. diff --git a/tests/test_validation/conftest.py b/tests/test_validation/conftest.py index 637e58b..1bb4ec7 100644 --- a/tests/test_validation/conftest.py +++ b/tests/test_validation/conftest.py @@ -93,3 +93,18 @@ def upstream_dir() -> Path | None: return None path = Path(raw) return path if path.is_dir() else None + + +@pytest.fixture +def reference_dir() -> Path | None: + """A local oold-js checkout, when OOLD_JS_DIR points at one. + + Separate from :func:`upstream_dir` because the corpus and the implementation that checks it + now live in different repositories: parity compares this package against `oold-js` over + `oold-schema`'s examples, so a run needs both and can have either without the other. + """ + raw = os.environ.get("OOLD_JS_DIR") + if not raw: + return None + path = Path(raw) + return path if path.is_dir() else None diff --git a/tests/test_validation/test_parity_live.py b/tests/test_validation/test_parity_live.py index dfff0bb..086c67f 100644 --- a/tests/test_validation/test_parity_live.py +++ b/tests/test_validation/test_parity_live.py @@ -1,18 +1,21 @@ -"""Opt-in parity tests against a real oold-schema checkout. +"""Opt-in parity tests against a real oold-schema corpus and the oold-js reference. -These are the tests that catch drift from the reference implementation. They are skipped unless -``OOLD_SCHEMA_DIR`` points at a local checkout:: +These are the tests that catch drift from the reference implementation. Two checkouts are +needed, because the corpus and the reference are separate repositories:: - OOLD_SCHEMA_DIR=../oold-schema uv run pytest tests/test_validation -q + OOLD_SCHEMA_DIR=../oold-schema OOLD_JS_DIR=../oold-js uv run pytest tests/test_validation -q -The committed fixture slice is a snapshot and cannot notice upstream changes; this can. CI does -not depend on it, so the suite stays self-contained by default. +``OOLD_SCHEMA_DIR`` alone still runs the checks that need only the corpus; the ones that compare +verdicts skip without ``OOLD_JS_DIR``. The committed fixture slice is a snapshot and cannot +notice upstream changes; this can. CI pins the reference by tag, so a change there is adopted +deliberately rather than arriving with the next push. """ from __future__ import annotations import shutil import subprocess +from pathlib import Path import pytest @@ -30,6 +33,42 @@ def upstream(upstream_dir): return upstream_dir +@pytest.fixture +def reference(upstream, reference_dir): + """Run the oold-js validator over a directory, against the same meta-schemas this port uses. + + `--meta` points at the corpus checkout rather than anything `oold-js` ships: the reference + vendors no meta-schemas, so both implementations read the one file set and a disagreement + can only come from the code. + """ + node = shutil.which("node") + if node is None: + pytest.skip("node is not available") + if reference_dir is None: + pytest.skip("set OOLD_JS_DIR to a local oold-js checkout to compare verdicts") + script = reference_dir / "src" / "validate.mjs" + if not script.is_file() or not (reference_dir / "node_modules").is_dir(): + pytest.skip("the reference harness is not installed (run npm install in oold-js)") + + def run(target: Path) -> subprocess.CompletedProcess: + # S603: a fixed script inside a checkout the developer pointed us at. + return subprocess.run( # noqa: S603 + [ + node, + str(script), + str(Path(target).resolve()), + "--meta", + str((upstream / "meta").resolve()), + ], + cwd=str(reference_dir), + capture_output=True, + text=True, + timeout=600, + ) + + return run + + def _options(upstream_dir): """Validate against the meta-schemas the upstream checkout actually declares. @@ -56,27 +95,13 @@ def test_vocabulary_coverage_matches_upstream(upstream): assert coverage and all(c.status == "ok" for c in coverage), [c.message for c in coverage] -def test_verdict_agrees_with_the_reference_harness(upstream): - """Run `node scripts/validate.mjs` and require the same overall verdict. +def test_verdict_agrees_with_the_reference_harness(upstream, reference): + """Run the oold-js validator and require the same overall verdict. Check *counts* legitimately differ: this port splits some of the reference's combined sections and adds `context.predicates`. The verdict must not. """ - node = shutil.which("node") - if node is None: - pytest.skip("node is not available") - script = upstream / "scripts" / "validate.mjs" - if not script.is_file() or not (upstream / "node_modules").is_dir(): - pytest.skip("the reference harness is not installed (run npm install in oold-schema)") - - # S603: a fixed script inside a checkout the developer pointed us at. - completed = subprocess.run( # noqa: S603 - [node, str(script)], - cwd=str(upstream), - capture_output=True, - text=True, - timeout=600, - ) + completed = reference(upstream / "examples") reference_passed = completed.returncode == 0 ours = validate_directory(upstream / "examples", _options(upstream)) @@ -90,26 +115,12 @@ def test_verdict_agrees_with_the_reference_harness(upstream): ) -def test_the_reference_cannot_resolve_a_context_leaving_the_directory(upstream, remote_context_dir): +def test_the_reference_cannot_resolve_a_context_leaving_the_directory(reference, remote_context_dir): """Documents the one capability this port adds, by demonstrating the difference. - If upstream ever gains this ability, the divergence note in the docs is stale. + If the reference ever gains this ability, the divergence note in the docs is stale. """ - node = shutil.which("node") - if node is None: - pytest.skip("node is not available") - script = upstream / "scripts" / "validate.mjs" - if not script.is_file() or not (upstream / "node_modules").is_dir(): - pytest.skip("the reference harness is not installed") - - # S603: a fixed script inside a checkout the developer pointed us at. - completed = subprocess.run( # noqa: S603 - [node, str(script), str(remote_context_dir.resolve())], - cwd=str(upstream), - capture_output=True, - text=True, - timeout=600, - ) + completed = reference(remote_context_dir) ours = validate_directory(remote_context_dir, Options(meta=("remote",), offline=False)) assert ours.passed, "this port is expected to resolve a ../ context reference"