diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b9ebc0..8118bf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,31 @@ jobs: - run: python -m ruff format --check . - run: python -m pytest -q + # Proves the README's own quick start still works: pytest (see tests/test_quickstart_block.py) + # already proves the extractor fails a broken block, so this job just has to run the real one. + quickstart: + name: Quick start verified + runs-on: ubuntu-latest + # Reads the checkout; writes nothing. + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + # Installed from this checkout, not PyPI, so the README's own `pip install context-report` + # line finds itself already satisfied and does not fetch a different released version. + - run: pip install . + - name: Extract and run the README's quickstart block + run: | + workdir=$(mktemp -d) + python3 tools/quickstart_block.py README.md > "$workdir/quickstart.sh" + cd "$workdir" + bash -euo pipefail quickstart.sh + # This repo is itself a chock adopter (see AGENTS.md, .agents/policies/): it governs # its own contributions the same way chock and chock-catalog do. Checking it here # proves the adoption on this repo rather than only in chock's own tests, and closes diff --git a/.github/workflows/render-demo.yml b/.github/workflows/render-demo.yml new file mode 100644 index 0000000..3da7144 --- /dev/null +++ b/.github/workflows/render-demo.yml @@ -0,0 +1,45 @@ +name: Render demo GIF +# Dispatch-only: this never commits to main (protected), it just proves docs/assets/demo.gif is +# reproducible from docs/assets/demo.tape. Download the demo-gif artifact and commit it by hand. +on: + workflow_dispatch: +permissions: {} +jobs: + render: + name: Render docs/assets/demo.tape + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + - run: pip install . + # No verified pin for charmbracelet/vhs-action was available to this change (see the PR + # description's Deviations); this reproduces the same tape with the plain VHS binary + # instead, using the release URL already pinned in docs/README-refresh's render recipe. + - name: Install ttyd, ffmpeg and vhs + # Checksum from charmbracelet/vhs's own v0.10.0 release checksums.txt -- verified before + # this pin was added; a corrupted or substituted download fails the sha256sum check below. + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq ttyd ffmpeg + curl -sSL -o /tmp/vhs.tgz https://github.com/charmbracelet/vhs/releases/download/v0.10.0/vhs_0.10.0_Linux_x86_64.tar.gz + echo "b552c3870aca101dcafe533cfef32dceb7b783400ad32642e728775c9f125407 /tmp/vhs.tgz" | sha256sum -c - + tar xzf /tmp/vhs.tgz -C /tmp + sudo install -m 0755 /tmp/vhs_0.10.0_Linux_x86_64/vhs /usr/local/bin/vhs + - name: Render as a non-root user (Chromium refuses to run as root) + run: | + sudo useradd -m vhsuser + sudo chmod -R a+rX "$PWD" + sudo chmod 777 docs/assets + sudo su vhsuser -c "cd $PWD && vhs docs/assets/demo.tape" + sudo chown "$(id -u)":"$(id -g)" docs/assets/demo.gif + sudo chmod 755 docs/assets + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: demo-gif + path: docs/assets/demo.gif diff --git a/CHANGELOG.md b/CHANGELOG.md index a5cba07..95af0a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ Per in-toto convention, `0.X` versions are major: fields may change until 1.0. ## Unreleased +### Docs + +- README: a rendered `produce`/`verify` demo GIF (`docs/assets/demo.tape`, reproducible via + `.github/workflows/render-demo.yml`), a quickstart with real `report.json` rows verified in CI + (`tools/quickstart_block.py`, `.github/workflows/ci.yml`'s `quickstart` job), and a Supported + agents table. + ### Added - `openai-compatible` provider for `models[]` and `judge`: any server speaking the OpenAI diff --git a/README.md b/README.md index 22126ce..04e9b14 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ -# context-report +
+ +

context-report

+ +

An open, signed report format for whether an agent context artifact actually works.

[![CI](https://github.com/open-coder-ai/context-report/actions/workflows/ci.yml/badge.svg)](https://github.com/open-coder-ai/context-report/actions/workflows/ci.yml) -[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) +[![PyPI](https://img.shields.io/pypi/v/context-report)](https://pypi.org/project/context-report/) [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/open-coder-ai/context-report/badge)](https://scorecard.dev/viewer/?uri=github.com/open-coder-ai/context-report) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) + +
+ +

+ Terminal recording: context-report produce measures my-plugin, a two-file Claude Code plugin bundle (a PreToolUse hook that denies a command containing "destructive-pattern"), for reachability, cost and fault, writing report.json; context-report verify then reprints every row's re-derivable or claimed basis and ends with the line 'well-formed and bound: True'. +

`context-report` is an open, signed report format for one question: **does this agent context artifact actually work?** @@ -22,23 +33,72 @@ whole (the consumer sets its own thresholds). ## 30-second quickstart ```bash -pip install context-report # 0.1.0 on PyPI; pip install -e ".[dev]" from a checkout to hack on it -context-report produce --subject ./my-plugin --kind plugin --target claude_code --n 20 \ - --out report.json # one statement: reachability, cost and fault rows for one target -context-report run run.json # a whole manifest: subjects x models x tasks in one shot -context-report compare out --history # every run of that manifest side by side +pip install context-report +mkdir -p my-plugin/hooks +cat > my-plugin/hooks/guard.py <<'PY' +#!/usr/bin/env python3 +import json, sys +event = json.load(sys.stdin) +command = event.get("tool_input", {}).get("command", "") +if "destructive-pattern" in command: + print(json.dumps({"decision": "deny", "reason": "blocked destructive command"})) +sys.exit(0) +PY +cat > my-plugin/hooks/hooks.json <<'JSON' +{ + "hooks": { + "PreToolUse": [ + {"hooks": [{"command": "python3", "args": ["${CLAUDE_PLUGIN_ROOT}/hooks/guard.py"]}]} + ] + } +} +JSON +context-report produce --subject ./my-plugin --kind plugin --target claude_code --n 20 --out report.json +context-report verify report.json --subject ./my-plugin +``` + +`verify` reprints each row's `basis` and `result`, then ends with `well-formed and bound: True` — +never a "pass" for the artifact as a whole. `report.json` is one row per fact; each row's `basis` +is **re-derivable** (anyone can recompute it from the subject) or **claimed** (the author asserts +it). Three real rows from the `report.json` this exact block just produced: + +```json +[ + { + "attribute": "reachability", + "basis": "re-derivable", + "result": "FAILED", + "inputHash": "sha256:fc126ed825ddfeb440a0d1d5bf133780e4b275b8c4c18d1556ea5ab251405bc9", + "conditions": {"cwdTested": ["root", "nested", "parent", "outside"]}, + "values": {"reachable_from": ["parent"], "unreachable_from": ["root", "nested", "outside"]} + }, + { + "attribute": "fault.malformedOutput", + "basis": "re-derivable", + "result": "PASSED", + "inputHash": "sha256:85fab0cf63d9db25ab5aba6ada7d396feceb0474f3018f83c24ce542441c1a88", + "conditions": {"cases": ["on_malformed_json", "on_empty_stdin", "on_null_tool_input", "control_benign"]}, + "reasoning": "exit 0 on malformed input is how a Claude Code hook fails open; whether that is acceptable is the consumer's threshold." + }, + { + "attribute": "cost.context_tokens", + "basis": "re-derivable", + "result": "PASSED", + "inputHash": "sha256:8d2af315178e658732a9e48147395296f8b33277bfb7413d55ad9722053b9026", + "conditions": {"tokenizer": "approx-regex-v1", "files": 1}, + "measurement": {"unit": "tokens", "n": 1, "min": 50, "max": 50, "mean": 50} + } +] ``` -The optional `context-report[efficacy]` extra pulls in the `anthropic` client for the `efficacy` -row (`context-report efficacy --help`). `context-report run` reads a JSON manifest matching -[`spec/run/v0.1/schema.json`](spec/run/v0.1/schema.json) — see -[`spec/run/v0.1/examples/run.json`](spec/run/v0.1/examples/run.json) for a worked one (two -subjects, four models across the three providers, three tasks) — and supports `--dry-run` (rules and call budget, no model -touched), `--n` (override `arms.nPerArm` for a smoke run), and `--resume` (continue the latest run, -reusing every existing statement and matching transcript, calling only for the rest). Two providers -have a backend: `anthropic` (the API) and `claude-cli` (the local `claude` CLI, so one manifest can -compare `opus`/`sonnet`/`fable`); any other subject model gets an honest `NotAvailable` efficacy -row instead of a guess. +`reachability` `FAILED` here is not a bug in the example: `${CLAUDE_PLUGIN_ROOT}` resolves to the +relative path `./my-plugin` you passed, so the hook only starts from the one cwd where that path +still points at the plugin — the exact failure mode the measurement below calls out. + +Beyond one statement at a time, `context-report run` drives a whole manifest — subjects × models × +tasks — and `context-report compare` puts every run of that manifest side by side; see +[`docs/cli.md`](docs/cli.md) for the manifest schema, `--dry-run`/`--n`/`--resume`, and which +model providers a manifest can reach. ## What a report looks like @@ -102,19 +162,11 @@ a prompt-injection rule moved nothing on any model. See ## Who it's for -- **An artifact author** wants a report their own CI can produce before anyone else asks for one. -- **A catalog maintainer** wants a submission format their existing verifier can check without - adopting anyone else's test suite, and a `re-derivable`/`claimed` split to build a policy on. -- **A researcher or reviewer** wants a re-derivable record of what was actually measured, not a - vendor's prose description of it. - -## Every model you can reach - -Subject models come from the manifest, never from code: `anthropic`, `claude-cli`, or -`openai-compatible` with a `baseUrl`, which is any server speaking the chat-completions shape, -hosted (OpenAI, Gemini, Mistral, Groq) or local (Ollama, vLLM, LM Studio). One manifest lines up -every model you can reach; the API-shaped ones answer without tools or a checkout, which the run -spec states. +| Who | What they want | +| :--- | :--- | +| An artifact author | a report their own CI can produce before anyone else asks for one | +| A catalog maintainer | a submission format their existing verifier can check without adopting anyone else's test suite, and a `re-derivable`/`claimed` split to build a policy on | +| A researcher or reviewer | a re-derivable record of what was actually measured, not a vendor's prose description of it | ## Two models, not one @@ -123,6 +175,19 @@ without it; the **judge model** never performs the task, only reads the transcri whether that arm met the rule's criterion, held fixed across every subject model so a comparison across models is fair. A machine-checkable criterion is graded by code instead, never guessed at. +Subject models come from the manifest, never from code — one manifest lines up every model you can +reach, the API-shaped ones answering without tools or a checkout: + +| Provider | What it reaches | +| :--- | :--- | +| `anthropic` | the Anthropic API | +| `claude-cli` | the local `claude` CLI — compares `opus`, `sonnet` and `fable` under one account login | +| `openai-compatible` | any server speaking the chat-completions shape, given a `baseUrl`: hosted (OpenAI, Gemini, Mistral, Groq) or local (Ollama, vLLM, LM Studio) | +| anything else | an honest `NotAvailable` efficacy row, never a guess | + +See [`docs/cli.md`](docs/cli.md#every-model-you-can-reach) for the full picture of what a manifest +can reach. + ## Use as a library Beyond the CLI, `context_report` exposes a small stable API for a catalog or CI job to import @@ -139,6 +204,22 @@ result = verify(stmt, subject_path="clone/") # bound + schema check, never a ve See [`docs/library.md`](docs/library.md) for a full catalog-verification and CI-production example. +## Supported agents + +`produce` drives the subject's own hook command through recorded per-target payloads — no live +agent required. `reachability`, `cost.latency_ms` and `fault.malformedOutput` are re-derivable for +every target below; the three fault rows that need a live client (`fault.scriptMissing`, +`fault.interpreterMissing`, `fault.timeout`) are always `NotAvailable` in v0.1 and cite a vendor-docs +oracle where one is on file (none yet for `codex_cli` — see +[Good first contributions](CONTRIBUTING.md#good-first-contributions)). + +| Agent | What is measured | Config file | +| :--- | :--- | :--- | +| `claude_code` | reachability · cost · fault (malformedOutput measured; scriptMissing/interpreterMissing/timeout `NotAvailable`, oracle on file) | `hooks/hooks.json` (+ `.claude-plugin/plugin.json`) | +| `codex_cli` | reachability · cost · fault (malformedOutput measured; scriptMissing/interpreterMissing/timeout `NotAvailable`, no oracle on file) | `hooks/hooks.json` | +| `copilot` | reachability · cost · fault (malformedOutput measured; scriptMissing/interpreterMissing/timeout `NotAvailable`, oracle on file) | `com.github.copilot/hooks/hooks.json` | +| `cursor` | reachability · cost · fault (malformedOutput measured; scriptMissing/interpreterMissing/timeout `NotAvailable`, oracle on file) | `hooks/hooks.json` | + ## Contributing Bug reports, spec feedback, and PRs are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for the @@ -147,6 +228,10 @@ of your own runs happen in [GitHub Discussions](https://github.com/open-coder-ai/context-report/discussions). See [SECURITY.md](SECURITY.md) to report a vulnerability privately. +```bash +python -m ruff check . && python -m ruff format --check . && python -m pytest -q +``` + Scoped starting points, each naming the file it lives in, are listed under [Good first contributions](CONTRIBUTING.md#good-first-contributions): another target agent's payload shape, `codex_cli`'s documented fault behaviour, a producer that drives a live client @@ -157,19 +242,17 @@ measurements. Comment on a keep the `Co-Authored-By` trailer if an agent helped — every diff is read in full before merge either way. -## Part of the open-coder-ai family - -Everything under [open-coder-ai](https://github.com/open-coder-ai) is built on one rule: a claim must match a -mechanism. Where this repository sits among the others: - -| Repository | What it is | -| :--- | :--- | -| [chock](https://github.com/open-coder-ai/chock) | The framework: write a policy once, enforce it on git hooks, CI, and every agent | -| [chock-catalog](https://github.com/open-coder-ai/chock-catalog) | The policies, each graded by what it actually enforces | -| [agentseam](https://github.com/open-coder-ai/agentseam) | The primitives layer under chock: one handler API over every agent's hooks, with a capability matrix that carries its provenance | -| [chock-threat-intel](https://github.com/open-coder-ai/chock-threat-intel) | A weekly, human-reviewed threat digest scored against the catalog | -| [chock-claude-plugins](https://github.com/open-coder-ai/chock-claude-plugins) · [copilot](https://github.com/open-coder-ai/chock-copilot-plugins) · [cursor](https://github.com/open-coder-ai/chock-cursor-plugins) · [codex](https://github.com/open-coder-ai/chock-codex-plugins) | The catalog compiled into each client's native plugin format; generated only, rebuilt and diffed in CI | -| [chock-quickstart](https://github.com/open-coder-ai/chock-quickstart) · [chock-example](https://github.com/open-coder-ai/chock-example) | Template repositories: exactly what `chock init` leaves behind, and a working adoption with one policy per layer | +## Part of open-coder-ai + +| | | +|---|---| +| [agentseam](https://github.com/open-coder-ai/agentseam) | the primitives — one handler API and a verified capability matrix across 16 agents | +| [chock](https://github.com/open-coder-ai/chock) | the compiler — one policy into git hooks, CI gates and native pre-tool hooks | +| [chock-catalog](https://github.com/open-coder-ai/chock-catalog) | the policies — 39, each labelled enforced or advisory, with replayed evals | +| [context-report](https://github.com/open-coder-ai/context-report) | the evidence — a signed report of whether an agent artifact actually works | +| [chock-threat-intel](https://github.com/open-coder-ai/chock-threat-intel) | the threat ledger the catalog's policies answer to | +| chock-{claude,cursor,copilot,codex}-plugins | the catalog, packaged for each agent's plugin format (generated) | +| chock-quickstart · chock-example | template repos: what `chock init` leaves behind, and a full adoption | ## License diff --git a/docs/assets/demo.gif b/docs/assets/demo.gif new file mode 100644 index 0000000..80ca3d4 Binary files /dev/null and b/docs/assets/demo.gif differ diff --git a/docs/assets/demo.tape b/docs/assets/demo.tape new file mode 100644 index 0000000..a10dad4 --- /dev/null +++ b/docs/assets/demo.tape @@ -0,0 +1,62 @@ +Output docs/assets/demo.gif +Set Shell "bash" +Set FontSize 16 +Set Width 1200 +Set Height 640 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 40ms +Set Padding 24 + +Hide +Type `export HOME=$(mktemp -d) && cd $(mktemp -d) && mkdir -p my-plugin/hooks` +Enter +Type `cat > my-plugin/hooks/guard.py <<'PY'` +Enter +Type `#!/usr/bin/env python3` +Enter +Type `import json, sys` +Enter +Type `event = json.load(sys.stdin)` +Enter +Type `command = event.get("tool_input", {}).get("command", "")` +Enter +Type `if "destructive-pattern" in command:` +Enter +Type ` print(json.dumps({"decision": "deny", "reason": "blocked destructive command"}))` +Enter +Type `sys.exit(0)` +Enter +Type `PY` +Enter +Type `cat > my-plugin/hooks/hooks.json <<'JSON'` +Enter +Type `{` +Enter +Type ` "hooks": {` +Enter +Type ` "PreToolUse": [` +Enter +Type ` {"hooks": [{"command": "python3", "args": ["${CLAUDE_PLUGIN_ROOT}/hooks/guard.py"]}]}` +Enter +Type ` ]` +Enter +Type ` }` +Enter +Type `}` +Enter +Type `JSON` +Enter +Type `clear` +Enter +Show + +# The subject: my-plugin, a two-file Claude Code plugin bundle (my-plugin/hooks/{guard.py, +# hooks.json}) declaring one PreToolUse hook that denies a command containing "destructive-pattern" +# -- the same commands as the README's own quickstart block. +Type `context-report produce --subject ./my-plugin --kind plugin --target claude_code --n 20 --out report.json` +Enter +Sleep 4s + +Type `context-report verify report.json --subject ./my-plugin` +Enter +Sleep 4s diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..acc52aa --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,25 @@ +# CLI: `run` and `compare` + +Beyond `produce` and `verify` (see the README's quickstart), `context-report` drives whole +measurement manifests and compares runs over time. + +## Running a manifest + +The optional `context-report[efficacy]` extra pulls in the `anthropic` client for the `efficacy` +row (`context-report efficacy --help`). `context-report run` reads a JSON manifest matching +[`spec/run/v0.1/schema.json`](../spec/run/v0.1/schema.json) — see +[`spec/run/v0.1/examples/run.json`](../spec/run/v0.1/examples/run.json) for a worked one (two +subjects, four models across the three providers, three tasks) — and supports `--dry-run` (rules and call budget, no model +touched), `--n` (override `arms.nPerArm` for a smoke run), and `--resume` (continue the latest run, +reusing every existing statement and matching transcript, calling only for the rest). Two providers +have a backend: `anthropic` (the API) and `claude-cli` (the local `claude` CLI, so one manifest can +compare `opus`/`sonnet`/`fable`); any other subject model gets an honest `NotAvailable` efficacy +row instead of a guess. + +## Every model you can reach + +Subject models come from the manifest, never from code: `anthropic`, `claude-cli`, or +`openai-compatible` with a `baseUrl`, which is any server speaking the chat-completions shape, +hosted (OpenAI, Gemini, Mistral, Groq) or local (Ollama, vLLM, LM Studio). One manifest lines up +every model you can reach; the API-shaped ones answer without tools or a checkout, which the run +spec states. diff --git a/tests/test_quickstart_block.py b/tests/test_quickstart_block.py new file mode 100644 index 0000000..f2fb933 --- /dev/null +++ b/tests/test_quickstart_block.py @@ -0,0 +1,66 @@ +"""tools/quickstart_block.py pulls the README's first fenced bash block, and only that one.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) +from quickstart_block import extract_quickstart_block + + +def test_extracts_the_bash_block_under_the_heading() -> None: + markdown = """ +## 30-second quickstart + +```bash +echo one +echo two +``` + +```json +{"not": "this"} +``` + +## Next section +""" + assert extract_quickstart_block(markdown) == "echo one\necho two\n" + + +def test_missing_heading_raises() -> None: + with pytest.raises(ValueError, match="heading"): + extract_quickstart_block("# nothing here") + + +def test_missing_bash_fence_raises() -> None: + markdown = "## 30-second quickstart\n\nno code block here\n" + with pytest.raises(ValueError, match="no closed fenced bash block"): + extract_quickstart_block(markdown) + + +def test_a_broken_extracted_block_fails_under_bash_dash_e() -> None: + """Proves the CI job actually fails when the README's quickstart block is broken.""" + markdown = """ +## 30-second quickstart + +```bash +false +echo unreachable +``` +""" + script = extract_quickstart_block(markdown) + result = subprocess.run( # noqa: S603 -- fixed argv, no shell, test-only + ["bash", "-euo", "pipefail", "-c", script], # noqa: S607 -- resolved via PATH, test-only + capture_output=True, + check=False, + ) + assert result.returncode != 0 + + +def test_readme_quickstart_block_is_extractable() -> None: + readme = Path(__file__).resolve().parents[1] / "README.md" + block = extract_quickstart_block(readme.read_text(encoding="utf-8")) + assert "context-report produce" in block diff --git a/tools/quickstart_block.py b/tools/quickstart_block.py new file mode 100644 index 0000000..e17195e --- /dev/null +++ b/tools/quickstart_block.py @@ -0,0 +1,40 @@ +"""Extract the first fenced ```bash block under the README's quickstart heading, stdlib only.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +HEADING = "## 30-second quickstart" + + +def extract_quickstart_block(markdown: str, heading: str = HEADING) -> str: + """The body of the first ```bash fence after `heading`, up to the next `##` heading.""" + lines = markdown.splitlines() + try: + start = lines.index(heading) + except ValueError: + raise ValueError(f"heading {heading!r} not found") from None + in_block = False + block: list[str] = [] + for line in lines[start + 1 :]: + if line.startswith("## ") and not in_block: + break + if not in_block: + if line.strip() == "```bash": + in_block = True + continue + if line.strip() == "```": + return "\n".join(block) + "\n" + block.append(line) + raise ValueError(f"no closed fenced bash block found under {heading!r}") + + +def main(argv: list[str]) -> int: + path = Path(argv[1]) if len(argv) > 1 else Path("README.md") + sys.stdout.write(extract_quickstart_block(path.read_text(encoding="utf-8"))) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))