Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@ Per in-toto convention, `0.X` versions are major: fields may change until 1.0.
chat-completions shape, hosted or local, with `baseUrl` and an optional `apiKeyEnv`; standard
library only, no SDK.

### Changed

- `statement.validate()` now prefixes every schema error with its JSON-pointer-style path
(`predicate/attributes/3/environmentSensitive: ...`), matching the convention `run/manifest.py`
already used; a root-level error gets the stable `(root)` prefix instead of an empty one. A
`const` mismatch also names the expected and offending values in the same line. The two
`validate()` functions now share one formatter, `context_report.schema_errors`.

## [0.1.0] - 2026-09-06

First public release: the format, a reference producer and verifier, a run manifest for
Expand Down
8 changes: 3 additions & 5 deletions src/context_report/run/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

from jsonschema import Draft202012Validator

from context_report.schema_errors import format_errors

MANIFEST_VERSION = "v0.1"
MODE_ISOLATED = "isolated"
MODE_LEAVE_ONE_OUT = "leave-one-out"
Expand Down Expand Up @@ -102,11 +104,7 @@ def schema() -> dict[str, Any]:

def validate(doc: dict[str, Any]) -> list[str]:
"""Schema errors as `path: message` strings; empty means well-formed."""
validator = Draft202012Validator(schema())
return [
"/".join(str(p) for p in e.absolute_path) + ": " + e.message
for e in sorted(validator.iter_errors(doc), key=lambda e: list(e.absolute_path))
]
return format_errors(Draft202012Validator(schema()).iter_errors(doc))


def _resolve(base: Path, raw: str) -> Path:
Expand Down
27 changes: 27 additions & 0 deletions src/context_report/schema_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
"""JSON Schema error formatting shared by every `validate()` in this package."""

from __future__ import annotations

from collections.abc import Iterable

from jsonschema.exceptions import ValidationError

ROOT = "(root)"


def error_path(error: ValidationError) -> str:
"""JSON-pointer-style path to the failing field; the stable `ROOT` marker at the top level."""
return "/".join(str(p) for p in error.absolute_path) or ROOT


def format_error(error: ValidationError) -> str:
"""`path: message`; a `const` failure also names the expected and offending values."""
message = error.message
if error.validator == "const":
message = f"expected {error.validator_value!r}, got {error.instance!r}"
return f"{error_path(error)}: {message}"


def format_errors(errors: Iterable[ValidationError]) -> list[str]:
"""Sort by path and format every error into the `path: message` shape."""
return [format_error(e) for e in sorted(errors, key=lambda e: list(e.absolute_path))]
9 changes: 7 additions & 2 deletions src/context_report/statement.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from jsonschema import Draft202012Validator

from context_report.rows import PREDICATE_TYPE, STATEMENT_TYPE, Row
from context_report.schema_errors import format_errors

SUBJECT_KINDS = ("plugin", "instruction-file", "skill", "hook", "mcp-server", "subagent")

Expand Down Expand Up @@ -129,5 +130,9 @@ def schema() -> dict[str, Any]:


def validate(stmt: dict[str, Any]) -> list[str]:
"""Schema errors for a statement, empty when it conforms. Never raises on a bad instance."""
return sorted(e.message for e in Draft202012Validator(schema()).iter_errors(stmt))
"""Schema errors for a statement as `path: message` strings, empty when it conforms.

Never raises on a bad instance. See `schema_errors.format_error` for the path convention,
including the `(root)` marker used for a top-level error.
"""
return format_errors(Draft202012Validator(schema()).iter_errors(stmt))
42 changes: 42 additions & 0 deletions tests/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import pytest
from jsonschema import Draft202012Validator

from context_report.statement import validate

ROOT = Path(__file__).resolve().parents[1]
V01 = ROOT / "spec" / "attestation" / "v0.1"
SCHEMA = json.loads((V01 / "schema.json").read_text(encoding="utf-8"))
Expand Down Expand Up @@ -131,6 +133,46 @@ def test_measured_latency_must_record_its_environment() -> None:
assert errors(bad), "latency is environmentSensitive by definition"


def test_const_mismatch_names_the_path_and_the_expected_and_actual_values() -> None:
"""A const failure (here: the cost.latency_ms row's environmentSensitive) names both sides."""
bad = copy.deepcopy(EXAMPLE)
idx, r = next(
(i, a)
for i, a in enumerate(bad["predicate"]["attributes"])
if a["attribute"] == "cost.latency_ms"
)
r["environmentSensitive"] = False
errs = validate(bad)
assert any(
e == f"predicate/attributes/{idx}/environmentSensitive: expected True, got False"
for e in errs
)


def test_nested_if_then_failure_under_a_subject_kind_block_is_path_prefixed() -> None:
"""An instruction-file's cost.latency_ms row must be NotApplicable; the path names the row."""
bad = copy.deepcopy(EXAMPLE)
bad["predicate"]["subjectKind"] = "instruction-file"
idx = next(
i
for i, a in enumerate(bad["predicate"]["attributes"])
if a["attribute"] == "cost.latency_ms"
)
errs = validate(bad)
assert any(
e == f"predicate/attributes/{idx}/result: expected 'NotApplicable', got 'PASSED'"
for e in errs
)


def test_root_level_error_keeps_a_stable_non_empty_prefix() -> None:
"""A missing top-level field has no JSON path segment of its own; it still gets a prefix."""
bad = copy.deepcopy(EXAMPLE)
del bad["_type"]
errs = validate(bad)
assert any(e.startswith("(root): ") and "_type" in e for e in errs)


def test_attribute_that_does_not_apply_to_the_kind_must_be_not_applicable() -> None:
"""A hook script is not injected into context, so cost.context_tokens cannot PASS for a hook."""
bad = copy.deepcopy(EXAMPLE)
Expand Down