diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8118bf1..2ce3a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,3 +127,27 @@ jobs: run: | chock check chock sync --repo . --check + + # A stale generated figure fails the build: regenerate every docs/figures/*.svg and diff against + # what is committed. make_row_basis.py and make_social_card.py both shell out to the README's + # own quickstart block (see tools/quickstart_block.py) to draw their numbers from a real, + # freshly produced report.json, so context-report itself must be installed first. + figures: + name: Generated figures are not stale + runs-on: ubuntu-latest + # Reads the checkout; writes nothing (the regenerated files are diffed, never pushed). + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + - run: pip install --require-hashes -r requirements/ci.txt + - run: pip install --no-deps -e . + - name: Regenerate figures + run: cd docs/figures && for g in make_*.py; do python "$g"; done + - name: Fail on drift + run: git diff --exit-code -- docs/figures diff --git a/README.md b/README.md index 04e9b14..e5bf4b4 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,14 @@ it). Three real rows from the `report.json` this exact block just produced: relative path `./my-plugin` you passed, so the hook only starts from the one cwd where that path still points at the plugin — the exact failure mode the measurement below calls out. +The full statement has 11 rows, not 3 — every row's basis, from this exact quickstart block run +fresh: + + + + Of 11 rows in the report.json this quickstart block produces, 3 are recomputed from the artifact, 1 is the author's unverified claim, and 7 have not been measured this run — shown as absence of evidence, not a weak score. + + Beyond one statement at a time, `context-report run` drives a whole manifest — subjects × models × tasks — and `context-report compare` puts every run of that manifest side by side; see [`docs/cli.md`](docs/cli.md) for the manifest schema, `--dry-run`/`--n`/`--resume`, and which @@ -143,7 +151,10 @@ skills. Three findings from that run: `FAILED, 0 of 4`, exit 126. Every hook that runs, across both samples, allows on malformed input. See [§5.2](paper/context-report.md#52-top-n-catalog-plugins). -![Eighteen plugins by four measured attributes](paper/figures/fig-catalog-status.svg) + + + Grid of 18 catalog plugins by 4 measured attributes: three plugins fail reachability outright, every plugin that measures malformed-input handling passes it, and not-applicable cells are shown as a neutral dash, never a low score. + **Cost spans two orders of magnitude.** Hooks that shell out to `npx` cost 916.8–941.5 ms p50; a local script costs 7.3–53.9 ms. Context weight varies about a hundredfold across the sample, @@ -244,6 +255,11 @@ merge either way. ## Part of open-coder-ai + + + The open-coder-ai family: agentseam is the foundation, chock sits on it, chock-catalog feeds chock and generates the four plugin repositories, chock-threat-intel feeds the catalog, and context-report runs as a verification arm measuring all four. + + | | | |---|---| | [agentseam](https://github.com/open-coder-ai/agentseam) | the primitives — one handler API and a verified capability matrix across 16 agents | diff --git a/docs/figures/family-dark.svg b/docs/figures/family-dark.svg new file mode 100644 index 0000000..af55724 --- /dev/null +++ b/docs/figures/family-dark.svg @@ -0,0 +1,46 @@ + + The open-coder-ai family + A layered diagram. agentseam is the foundation across the bottom; chock sits on it; chock-catalog feeds chock and generates the four plugin repositories; chock-threat-intel feeds the catalog; context-report runs as a verification arm beside all of them. + + + chock-threat-intel + the threat ledger the catalog's + policies answer to + + plugin repositories + the catalog, packaged for each + agent's plugin format (generated) + + + + + + chock-catalog + the policies — 39, each labelled enforced or advisory, with replayed evals + + + + chock + the compiler — one policy into git hooks, CI gates and native pre-tool + hooks + + + + agentseam + the primitives — one handler API and a verified capability matrix across + 16 agents + + context-report + the evidence — a + signed report of + whether an agent + artifact actually + works + measures all four + + + + + + + diff --git a/docs/figures/family-light.svg b/docs/figures/family-light.svg new file mode 100644 index 0000000..cb95f50 --- /dev/null +++ b/docs/figures/family-light.svg @@ -0,0 +1,46 @@ + + The open-coder-ai family + A layered diagram. agentseam is the foundation across the bottom; chock sits on it; chock-catalog feeds chock and generates the four plugin repositories; chock-threat-intel feeds the catalog; context-report runs as a verification arm beside all of them. + + + chock-threat-intel + the threat ledger the catalog's + policies answer to + + plugin repositories + the catalog, packaged for each + agent's plugin format (generated) + + + + + + chock-catalog + the policies — 39, each labelled enforced or advisory, with replayed evals + + + + chock + the compiler — one policy into git hooks, CI gates and native pre-tool + hooks + + + + agentseam + the primitives — one handler API and a verified capability matrix across + 16 agents + + context-report + the evidence — a + signed report of + whether an agent + artifact actually + works + measures all four + + + + + + + diff --git a/docs/figures/fig-catalog-status-dark.svg b/docs/figures/fig-catalog-status-dark.svg new file mode 100644 index 0000000..6588b6a --- /dev/null +++ b/docs/figures/fig-catalog-status-dark.svg @@ -0,0 +1,186 @@ + + Catalog plugin status: 18 plugins by 4 measured attributes + A grid of 18 public Claude Code plugins by reachability, malformed-input handling, latency and context tokens, one cell per statement's result. Three plugins fail reachability outright; every plugin that measures malformed-input handling passes it; cells marked not-applicable (dash, neutral grey) mean the row does not apply to that plugin, not that it scored low. + + one cell per statement's result, from the 18-plugin catalog sample + reachability + malformed + input + latency + context + tokens + agentforce-adlc + + P + + P + + P + + P + ai-plugins + + P + + P + + P + + P + altimate-code + + - + + - + + - + + P + aws-core + + P + + P + + P + + P + aws-serverless + + - + + - + + - + + P + aws-startup-advisor + + - + + - + + - + + P + azure + + - + + - + + - + + P + carta-cap-table + + F + + P + + E + + P + carta-crm + + F + + P + + E + + P + carta-investors + + F + + P + + E + + P + protect-mcp + + P + + P + + P + + P + review-agent-governance + + P + + P + + P + + P + planning-with-files + + P + + P + + P + + P + 42crunch-api-security-testing + + - + + - + + - + + P + activecampaign + + - + + - + + - + + P + adobe-for-creativity + + - + + - + + - + + P + agent-sdk-dev + + - + + - + + - + + - + aikido + + - + + - + + - + + P + + P + PASSED + + E + Error + + F + FAILED + + - + n/a (not applicable) + diff --git a/docs/figures/fig-catalog-status-light.svg b/docs/figures/fig-catalog-status-light.svg new file mode 100644 index 0000000..c75b3b1 --- /dev/null +++ b/docs/figures/fig-catalog-status-light.svg @@ -0,0 +1,186 @@ + + Catalog plugin status: 18 plugins by 4 measured attributes + A grid of 18 public Claude Code plugins by reachability, malformed-input handling, latency and context tokens, one cell per statement's result. Three plugins fail reachability outright; every plugin that measures malformed-input handling passes it; cells marked not-applicable (dash, neutral grey) mean the row does not apply to that plugin, not that it scored low. + + one cell per statement's result, from the 18-plugin catalog sample + reachability + malformed + input + latency + context + tokens + agentforce-adlc + + P + + P + + P + + P + ai-plugins + + P + + P + + P + + P + altimate-code + + - + + - + + - + + P + aws-core + + P + + P + + P + + P + aws-serverless + + - + + - + + - + + P + aws-startup-advisor + + - + + - + + - + + P + azure + + - + + - + + - + + P + carta-cap-table + + F + + P + + E + + P + carta-crm + + F + + P + + E + + P + carta-investors + + F + + P + + E + + P + protect-mcp + + P + + P + + P + + P + review-agent-governance + + P + + P + + P + + P + planning-with-files + + P + + P + + P + + P + 42crunch-api-security-testing + + - + + - + + - + + P + activecampaign + + - + + - + + - + + P + adobe-for-creativity + + - + + - + + - + + P + agent-sdk-dev + + - + + - + + - + + - + aikido + + - + + - + + - + + P + + P + PASSED + + E + Error + + F + FAILED + + - + n/a (not applicable) + diff --git a/docs/figures/fig-row-basis-dark.svg b/docs/figures/fig-row-basis-dark.svg new file mode 100644 index 0000000..9ab7e02 --- /dev/null +++ b/docs/figures/fig-row-basis-dark.svg @@ -0,0 +1,23 @@ + + Row bases in one context-report statement + Of 11 rows in the report.json the README's own quickstart block produces fresh, 3 are recomputed from the artifact, 1 is the author's unverified claim, and 7 have not been measured this run -- shown as absence of evidence, not a weak score. + + + + + + report.json from the README's quickstart, run fresh -- 11 rows total + recomputed from the artifact + re-derivable, and a real result was measured + + 3 + the author's claim + basis: claimed -- not independently checked this run + + 1 + not measured this run + re-derivable in principle; NotAvailable/Error here + + 7 + ordered by strength of evidence: recomputed > claimed > not measured (absence, not a lower grade) + diff --git a/docs/figures/fig-row-basis-light.svg b/docs/figures/fig-row-basis-light.svg new file mode 100644 index 0000000..1207823 --- /dev/null +++ b/docs/figures/fig-row-basis-light.svg @@ -0,0 +1,23 @@ + + Row bases in one context-report statement + Of 11 rows in the report.json the README's own quickstart block produces fresh, 3 are recomputed from the artifact, 1 is the author's unverified claim, and 7 have not been measured this run -- shown as absence of evidence, not a weak score. + + + + + + report.json from the README's quickstart, run fresh -- 11 rows total + recomputed from the artifact + re-derivable, and a real result was measured + + 3 + the author's claim + basis: claimed -- not independently checked this run + + 1 + not measured this run + re-derivable in principle; NotAvailable/Error here + + 7 + ordered by strength of evidence: recomputed > claimed > not measured (absence, not a lower grade) + diff --git a/docs/figures/make_catalog_status.py b/docs/figures/make_catalog_status.py new file mode 100644 index 0000000..d58de6b --- /dev/null +++ b/docs/figures/make_catalog_status.py @@ -0,0 +1,171 @@ +"""Restyles fig-catalog-status into the shared open-coder-ai visual language. + +Same measurement as `paper/figures/make_figures.py`'s `fig_catalog_status` (18 public catalog +plugins x 4 measured attributes, one cell per statement's `result`) and the same data files; +only the rendering changes, from matplotlib to a light/dark SVG pair drawn with `palette.py`. + +Run from this directory: `python make_catalog_status.py`. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import palette as p + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +CATALOG_DIR = REPO / "paper" / "measurements" / "catalog-sample" +ORDER_FILE = REPO / "paper" / "figures" / "data" / "catalog_plugin_order.json" + +# The same four attributes paper/figures/make_figures.py's STATUS_ATTRS puts in this grid. +ATTRS = [ + ("reachability", "reachability"), + ("fault.malformedOutput", "malformed\ninput"), + ("cost.latency_ms", "latency"), + ("cost.context_tokens", "context\ntokens"), +] + +# Every result this grid actually contains (verified against the committed statements): PASSED, +# FAILED and Error are three distinct, evidence-bearing outcomes -- the ENFORCEMENT ramp's three +# steps, ordered here from least to most cause for concern. NotApplicable is not a fourth grade of +# that ramp: it means the row does not apply (e.g. "declares no hooks"), which is absence of +# evidence, so it is always NEUTRAL, never a ramp colour. +GLYPHS = {"PASSED": "P", "Error": "E", "FAILED": "F", "NotApplicable": "-"} +RESULT_ORDER = ["PASSED", "Error", "FAILED", "NotApplicable"] + +W = 700 +MARGIN = 16 +NAME_COL_W = 220 +COL_W = 108 +HEADER_TOP = 40 +HEADER_H = 34 +ROW_H = 22 +GRID_X0 = MARGIN + NAME_COL_W +GRID_TOP = HEADER_TOP + HEADER_H + 6 + + +def _load_plugins() -> list[dict]: + """One entry per plugin, in `catalog_plugin_order.json`'s order (mirrors make_figures.py's + `_load_catalog_plugins`, reimplemented here without the matplotlib import that module carries, + since this generator must stay stdlib-only).""" + order = json.loads(ORDER_FILE.read_text(encoding="utf-8")) + plugins = [] + for entry in order: + stmt = json.loads( + (CATALOG_DIR / entry["marketplace"] / f"{entry['plugin']}.json").read_text( + encoding="utf-8" + ) + ) + rows = {row["attribute"]: row for row in stmt["predicate"]["attributes"]} + plugins.append({"plugin": entry["plugin"], "rows": rows}) + return plugins + + +def _cell_colour(result: str, t: dict) -> str: + if result == "NotApplicable": + return t["neutral"] + return t["enforcement"][RESULT_ORDER.index(result)] + + +def _cell_text_colour(result: str, t: dict) -> str: + """Dark glyph on a light fill, light glyph on a dark fill; NEUTRAL always reads as text.""" + if result in ("Error", "FAILED"): + return t["surface"] + return t["text"] + + +def render(t: dict, name: str) -> str: # noqa: ARG001 -- write_pair's render(t, name) contract + plugins = _load_plugins() + n = len(plugins) + grid_bottom = GRID_TOP + n * ROW_H + legend_y = grid_bottom + 28 + height = legend_y + 24 + + svg = p.open_svg( + W, + height, + t, + "Catalog plugin status: 18 plugins by 4 measured attributes", + "A grid of 18 public Claude Code plugins by reachability, malformed-input handling, " + "latency and context tokens, one cell per statement's result. Three plugins fail " + "reachability outright; every plugin that measures malformed-input handling passes it; " + "cells marked not-applicable (dash, neutral grey) mean the row does not apply to that " + "plugin, not that it scored low.", + ) + + svg += p.text( + MARGIN, + 26, + "one cell per statement's result, from the 18-plugin catalog sample", + t["secondary"], + 12, + ) + + for col, (_attr, label) in enumerate(ATTRS): + cx = GRID_X0 + col * COL_W + COL_W / 2 + lines = label.split("\n") + y0 = HEADER_TOP + 14 if len(lines) == 1 else HEADER_TOP + 6 + for i, line in enumerate(lines): + svg += p.text(cx, y0 + i * 14, line, t["text"], 11, weight="600", anchor="middle") + + for row, plugin in enumerate(plugins): + y = GRID_TOP + row * ROW_H + svg += p.text( + MARGIN, + y + ROW_H / 2 + 4, + plugin["plugin"], + t["text"], + 10, + p.MONO, + ) + for col, (attr, _label) in enumerate(ATTRS): + result = plugin["rows"][attr]["result"] + x = GRID_X0 + col * COL_W + p.GAP + fill = _cell_colour(result, t) + svg += p.box(x, y + p.GAP, COL_W - 2 * p.GAP, ROW_H - 2 * p.GAP, fill, rx=3) + svg += p.text( + x + (COL_W - 2 * p.GAP) / 2, + y + ROW_H / 2 + 4, + GLYPHS[result], + _cell_text_colour(result, t), + 10, + weight="600", + anchor="middle", + ) + + entries = [ + ( + r, + { + "PASSED": "PASSED", + "Error": "Error", + "FAILED": "FAILED", + "NotApplicable": "n/a (not applicable)", + }[r], + ) + for r in RESULT_ORDER + ] + entry_w = (W - 2 * MARGIN) / len(entries) + for i, (result, label) in enumerate(entries): + x = MARGIN + i * entry_w + fill = _cell_colour(result, t) + svg += p.box(x, legend_y, 16, 16, fill, rx=3) + svg += p.text( + x + 8, + legend_y + 12, + GLYPHS[result], + _cell_text_colour(result, t), + 9, + weight="600", + anchor="middle", + ) + svg += p.text(x + 22, legend_y + 12, label, t["secondary"], 11) + + return svg + p.close_svg() + + +if __name__ == "__main__": + for path in p.write_pair("fig-catalog-status", render): + print("wrote", path) diff --git a/docs/figures/make_family.py b/docs/figures/make_family.py new file mode 100644 index 0000000..6a70d5e --- /dev/null +++ b/docs/figures/make_family.py @@ -0,0 +1,101 @@ +"""The open-coder-ai family: what each repository is, and what feeds what. + +Carried byte-identically by four repositories; see palette.py on why every +statement stays on one line under 88 characters. +""" + +import palette as p + +W, H = 800, 420 +NAME, ROLE = 13, 11 +WIDE, NARROW, ARM = 74, 34, 22 + +# Split into fragments joined with a space: a magic trailing comma keeps each list +# exploded, so `ruff format` leaves this alone at 88, 100 and 120 alike. +ROLES = { + "agentseam": [ + "the primitives — one handler API and a verified capability", + "matrix across 16 agents", + ], + "chock": [ + "the compiler — one policy into git hooks, CI gates and", + "native pre-tool hooks", + ], + "chock-catalog": [ + "the policies — 39, each labelled enforced or advisory,", + "with replayed evals", + ], + "context-report": [ + "the evidence — a signed report of whether an agent", + "artifact actually works", + ], + "chock-threat-intel": ["the threat ledger the catalog's policies answer to"], + "plugins": ["the catalog, packaged for each agent's plugin format (generated)"], +} + + +def role(key): + """One repository's one-line description, reassembled from its fragments.""" + return " ".join(ROLES[key]) + + +def _block(rect, name, role, t, chars, *, filled=False): + """A repository: monospace identifier, wrapped role beneath it.""" + x, y, w, h = rect + accent = t["enforcement"][1] + fill = accent if filled else t["surface"] + out = p.box(x, y, w, h, fill, None if filled else accent) + label = t["surface"] if filled else t["text"] + body = t["surface"] if filled else t["secondary"] + out += p.text(x + 12, y + 22, name, label, NAME, p.MONO, "600") + for i, line in enumerate(p.wrap(role, chars)): + out += p.text(x + 12, y + 40 + i * 15, line, body, ROLE) + return out + + +DESC = ( + "A layered diagram. agentseam is the foundation across the bottom; chock sits " + "on it; chock-catalog feeds chock and generates the four plugin repositories; " + "chock-threat-intel feeds the catalog; context-report runs as a verification " + "arm beside all of them." +) + + +def render(t, _name): + """One theme's copy of the family diagram.""" + a = t["enforcement"][1] + svg = p.open_svg(W, H, t, "The open-coder-ai family", DESC) + + intel = role("chock-threat-intel") + svg += _block((24, 24, 262, 72), "chock-threat-intel", intel, t, NARROW) + + svg += p.box(314, 24, 262, 72, t["surface"], t["neutral"]) + svg += p.text(326, 44, "plugin repositories", t["text"], NAME, p.MONO, "600") + for i, line in enumerate(p.wrap(role("plugins"), NARROW)): + svg += p.text(326, 62 + i * 15, line, t["secondary"], ROLE) + + svg += p.arrow(155, 96, 155, 122, a) + svg += p.arrow(445, 122, 445, 96, a) + + cat = role("chock-catalog") + svg += _block((24, 124, 552, 74), "chock-catalog", cat, t, WIDE, filled=True) + svg += p.arrow(300, 198, 300, 224, a) + svg += _block((24, 226, 552, 74), "chock", role("chock"), t, WIDE, filled=True) + svg += p.arrow(300, 328, 300, 302, a) + seam = role("agentseam") + svg += _block((24, 330, 552, 74), "agentseam", seam, t, WIDE, filled=True) + + svg += p.box(596, 24, 180, 380, t["surface"], a) + svg += p.text(608, 46, "context-report", t["text"], NAME, p.MONO, "600") + for i, line in enumerate(p.wrap(role("context-report"), ARM)): + svg += p.text(608, 68 + i * 15, line, t["secondary"], ROLE) + svg += p.text(608, 390, "measures all four", t["secondary"], ROLE) + for y in (140, 242, 346): + svg += p.arrow(592, y, 580, y, a) + + return svg + p.close_svg() + + +if __name__ == "__main__": + for path in p.write_pair("family", render): + print("wrote", path) diff --git a/docs/figures/make_row_basis.py b/docs/figures/make_row_basis.py new file mode 100644 index 0000000..abfafc0 --- /dev/null +++ b/docs/figures/make_row_basis.py @@ -0,0 +1,198 @@ +"""The row-basis figure: what each row of a real report.json was concluded from. + +Runs the README's own 30-second quickstart block fresh (the same mechanism the README's "Three +real rows" JSON block is pulled from -- see tools/quickstart_block.py and +tests/test_quickstart_block.py) and counts every row's basis in the resulting report.json, so this +figure can never disagree with the README: regenerating either regenerates from the same command. + +Run from this directory: `python make_row_basis.py`. Needs `context-report` importable/installed +(as the repo's own CI already ensures before this job) so the extracted quickstart block's +`context-report produce` call resolves. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import palette as p + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +README = REPO / "README.md" + +sys.path.insert(0, str(REPO / "tools")) +from quickstart_block import extract_quickstart_block # noqa: E402 -- needs REPO on sys.path first + +CLAIMED = "claimed" +# Mirrors context_report.rows.NOT_MEASURED: a result of NotAvailable, Error or NotApplicable +# means nothing was actually recomputed this run. Kept as a literal (not imported) so this +# generator stays stdlib-only -- importing context_report here would pull in its third-party +# dependencies (jsonschema, pyyaml) at figure-generation time. +NOT_MEASURED = frozenset({"NotAvailable", "Error", "NotApplicable"}) + +W, H = 700, 320 +MARGIN = 16 +BAR_MAX_W = W - 2 * MARGIN +MIN_INSIDE_LABEL_W = 24 # narrower than this, the count goes outside the bar instead + + +def _produce_fresh_report() -> dict: + """Run the README's quickstart block in a scratch dir and return the report.json it writes. + + PYTHONPATH is pinned to this checkout's src/ so the block's `context-report` invocations + import *this* commit's code even if another context-report checkout is also installed + somewhere on this machine -- the figure must describe the commit it ships with, never a + stray sibling checkout. + """ + block = extract_quickstart_block(README.read_text(encoding="utf-8")) + env = dict(os.environ) + repo_src = str(REPO / "src") + env["PYTHONPATH"] = ( + repo_src + os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else repo_src + ) + with tempfile.TemporaryDirectory() as workdir: + subprocess.run( # noqa: S602 -- fixed, repo-owned script; same command the README documents + block, + shell=True, + cwd=workdir, + check=True, + capture_output=True, + text=True, + executable="/bin/bash", + env=env, + ) + return json.loads((Path(workdir) / "report.json").read_text(encoding="utf-8")) + + +def _row_bases(stmt: dict) -> list[tuple[str, str, str]]: + """(attribute, basis, result) for every row, in statement order.""" + return [ + (row["attribute"], row["basis"], row["result"]) for row in stmt["predicate"]["attributes"] + ] + + +def _categorise(rows: list[tuple[str, str, str]]) -> dict[str, list[str]]: + """Partition every row into exactly one of three mutually exclusive evidentiary bases. + + `claimed` (an author's assertion, schema basis "claimed") and `recomputed` (schema basis + "re-derivable" AND a real, non-absent result) are the two evidence-bearing bases here -- only + two, because that is what this real report actually contains; a third ramp colour is not used + for a distinction the data does not carry. `absence` (re-derivable in principle, but this run's + result is NotAvailable/Error/NotApplicable -- nothing was actually recomputed) is NEUTRAL: not + a weaker grade of "recomputed", a different thing. + """ + buckets: dict[str, list[str]] = {"recomputed": [], "claimed": [], "absence": []} + for attribute, basis, result in rows: + if basis == CLAIMED: + buckets["claimed"].append(attribute) + elif result in NOT_MEASURED: + buckets["absence"].append(attribute) + else: + buckets["recomputed"].append(attribute) + return buckets + + +BARS = [ + ("recomputed", "recomputed from the artifact", "re-derivable, and a real result was measured"), + ("claimed", "the author's claim", "basis: claimed -- not independently checked this run"), + ("absence", "not measured this run", "re-derivable in principle; NotAvailable/Error here"), +] + + +def _bar_colour(key: str, t: dict) -> str: + return {"recomputed": t["enforcement"][2], "claimed": t["enforcement"][0]}.get( + key, t["neutral"] + ) + + +def _hatch_id(name: str) -> str: + return f"hatch-{name}" + + +def render(t: dict, name: str, buckets: dict[str, list[str]]) -> str: + total = sum(len(v) for v in buckets.values()) + max_n = max(len(v) for v in buckets.values()) + row_block = 72 + bar_h = 20 + + svg = p.open_svg( + W, + H, + t, + "Row bases in one context-report statement", + f"Of {total} rows in the report.json the README's own quickstart block produces fresh, " + f"{len(buckets['recomputed'])} are recomputed from the artifact, " + f"{len(buckets['claimed'])} is the author's unverified claim, and " + f"{len(buckets['absence'])} have not been measured this run -- shown as absence of " + "evidence, not a weak score.", + ) + + svg += ( + f' \n' + f' \n' + f' \n' + " \n" + ) + + svg += p.text( + MARGIN, + 26, + f"report.json from the README's quickstart, run fresh -- {total} rows total", + t["secondary"], + 12, + ) + + y = 48 + for key, label, sublabel in BARS: + n = len(buckets[key]) + bar_w = (n / max_n) * BAR_MAX_W if max_n else 0 + svg += p.text(MARGIN, y, label, t["text"], 12, weight="600") + svg += p.text(MARGIN, y + 15, sublabel, t["secondary"], 10) + by = y + 22 + fill = f"url(#{_hatch_id(name)})" if key == "absence" else _bar_colour(key, t) + if bar_w > 0: + svg += p.box(MARGIN, by, bar_w, bar_h, fill, rx=3) + inside = bar_w > MIN_INSIDE_LABEL_W + if inside: + text_colour = t["surface"] if key == "recomputed" else t["text"] + svg += p.text( + MARGIN + bar_w - 8, + by + bar_h / 2 + 4, + str(n), + text_colour, + 12, + weight="600", + anchor="end", + ) + else: + svg += p.text( + MARGIN + bar_w + 8, by + bar_h / 2 + 4, str(n), t["text"], 12, weight="600" + ) + y += row_block + + svg += p.text( + MARGIN, + H - 14, + "ordered by strength of evidence: recomputed > claimed > not measured " + "(absence, not a lower grade)", + t["secondary"], + 10, + ) + + return svg + p.close_svg() + + +if __name__ == "__main__": + statement = _produce_fresh_report() + rows = _row_bases(statement) + row_buckets = _categorise(rows) + for k, v in row_buckets.items(): + print(k, len(v), v) + for path in p.write_pair("fig-row-basis", lambda t, n: render(t, n, row_buckets)): + print("wrote", path) diff --git a/docs/figures/make_social_card.py b/docs/figures/make_social_card.py new file mode 100644 index 0000000..edac589 --- /dev/null +++ b/docs/figures/make_social_card.py @@ -0,0 +1,66 @@ +"""GitHub social-preview card: repo name, one line of what it is, one real number. Light only -- +GitHub's social preview has no dark variant. No logo: this repo has none in docs/assets/. + +Run from this directory: `python make_social_card.py`. Rendering the PNG needs `cairosvg`; if it +is not installed (as in CI, which never needs it -- see docs/figures/make_row_basis.py) the SVG is +still written and the PNG step is skipped rather than failing the regeneration. +""" + +from __future__ import annotations + +from pathlib import Path + +import palette as p +from make_row_basis import _categorise, _produce_fresh_report, _row_bases + +HERE = Path(__file__).resolve().parent +W, H = 1280, 640 + +TAGLINE = "an open, signed report of whether an agent context artifact actually works" + + +def _headline_number() -> str: + """The same real report make_row_basis.py draws from, reduced to one number for the card.""" + buckets = _categorise(_row_bases(_produce_fresh_report())) + total = sum(len(v) for v in buckets.values()) + return f"{len(buckets['recomputed'])} of {total} rows recomputed from the artifact, fresh" + + +def render(number: str) -> str: + t = p.theme("light") + svg = p.open_svg( + W, + H, + t, + "context-report", + "context-report: an open, signed report of whether an agent context artifact actually " + f"works. {number}.", + ) + accent = t["enforcement"][2] + svg += p.box(0, 0, 14, H, accent, rx=0) + svg += p.text(72, 260, "context-report", t["text"], 64, p.MONO, "600") + svg += p.text(72, 320, TAGLINE, t["secondary"], 26) + svg += p.box(72, 380, 760, 3, t["neutral"], rx=0) + svg += p.text(72, 440, number, accent, 30, p.MONO, "600") + return svg + p.close_svg() + + +def main() -> None: + number = _headline_number() + print(number) + svg_path = HERE / "social-card.svg" + svg_path.write_text(render(number).strip() + "\n", encoding="utf-8") + print("wrote", svg_path) + + try: + import cairosvg + except ImportError: + print("cairosvg not installed -- skipping PNG render (not needed in CI)") + return + png_path = HERE / "social-card.png" + cairosvg.svg2png(url=str(svg_path), write_to=str(png_path), output_width=W, output_height=H) + print("wrote", png_path) + + +if __name__ == "__main__": + main() diff --git a/docs/figures/palette.py b/docs/figures/palette.py new file mode 100644 index 0000000..e5ff671 --- /dev/null +++ b/docs/figures/palette.py @@ -0,0 +1,131 @@ +"""The one visual language every open-coder-ai figure is drawn in. + +Four repositories carry this file byte-identically while configuring `ruff format` at +three different line lengths (88, 100 and 120), so no formatted output could satisfy all +of them. Every statement here therefore fits on a single line under 88 characters and +nothing is split across lines: a formatter at any width finds nothing to join or wrap. +Keep it that way when editing. +""" + +# Enforcement is ordinal: advisory < in-agent < enforced. Never reorder these. +ENFORCEMENT = { + "light": ["#86b6ef", "#2a78d6", "#104281"], + "dark": ["#9ec5f4", "#3987e5", "#184f95"], +} + +# agentseam grades five levels; the ramp is the same hue walked further. +LEVELS = { + "light": ["#86b6ef", "#5598e7", "#2a78d6", "#1c5cab", "#104281"], + "dark": ["#9ec5f4", "#6da7ec", "#3987e5", "#256abf", "#184f95"], +} + +# Absence is not a low score, so it is never a ramp colour. +NEUTRAL = {"light": "#d8d7d2", "dark": "#383835"} + +SURFACE = {"light": "#fcfcfb", "dark": "#1a1a19"} +TEXT = {"light": "#0b0b0b", "dark": "#ffffff"} +TEXT_SECONDARY = {"light": "#52514e", "dark": "#c3c2b7"} + +STROKE_WIDTH = 2 +CORNER = 4 +GAP = 2 # surface showing between adjacent fills +GRID_OPACITY = 0.3 + +SANS = "-apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif" +MONO = "ui-monospace, SFMono-Regular, 'SF Mono', Menlo, Consolas, monospace" + +THEMES = ("light", "dark") + +_XML = (("&", "&"), ("<", "<"), (">", ">"), ('"', """)) + + +def theme(name): + """Every colour of one theme, resolved.""" + return { + "enforcement": ENFORCEMENT[name], + "levels": LEVELS[name], + "neutral": NEUTRAL[name], + "surface": SURFACE[name], + "text": TEXT[name], + "secondary": TEXT_SECONDARY[name], + } + + +def esc(s): + """XML-escape a label.""" + out = str(s) + for old, new in _XML: + out = out.replace(old, new) + return out + + +def open_svg(width, height, t, title, desc): + """An accessible root element: the title and description are the alt text.""" + ns = 'xmlns="http://www.w3.org/2000/svg"' + size = f'width="{width:d}" height="{height:d}"' + view = f'viewBox="0 0 {width:d} {height:d}"' + root = f'\n' + alt = f' {esc(title)}\n' + alt += f' {esc(desc)}\n' + return root + alt + f' \n' + + +def close_svg(): + return "\n" + + +def box(x, y, w, h, fill, stroke=None, rx=CORNER): + """A rounded rectangle, filled and optionally outlined.""" + edge = f' stroke="{stroke}" stroke-width="{STROKE_WIDTH:d}"' if stroke else "" + geom = f'x="{x:g}" y="{y:g}" width="{w:g}" height="{h:g}"' + return f' \n' + + +def text(x, y, s, fill, size=13, family=SANS, weight="400", anchor="start"): + """One line of type, anchored at (x, y).""" + font = f'font-family="{family}" font-size="{size:g}" font-weight="{weight}"' + place = f'x="{x:g}" y="{y:g}"' + paint = f'fill="{fill}" text-anchor="{anchor}"' + return f" {esc(s)}\n" + + +def arrow(x1, y1, x2, y2, colour, head=6): + """A straight connector with a solid head, drawn direction-aware.""" + ends = f'x1="{x1:g}" y1="{y1:g}" x2="{x2:g}" y2="{y2:g}"' + paint = f'stroke="{colour}" stroke-width="{STROKE_WIDTH:d}"' + line = f' \n' + if x1 == x2: # vertical: the head's base sits back along the travel + base = y2 - head if y2 > y1 else y2 + head + wings = ((x2 - head, base), (x2 + head, base)) + else: + base = x2 - head if x2 > x1 else x2 + head + wings = ((base, y2 - head), (base, y2 + head)) + corners = ((x2, y2), *wings) + pts = " ".join(f"{px:g},{py:g}" for px, py in corners) + return line + f' \n' + + +def write_pair(stem, render): + """`render(theme_colours, theme_name) -> svg string`, once per theme.""" + written = [] + for name in THEMES: + path = f"{stem}-{name}.svg" + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write(render(theme(name), name)) + written.append(path) + return written + + +def wrap(s, width): + """Greedy wrap to `width` characters, so a label fits without a font metric.""" + words, lines, line = s.split(), [], "" + for w in words: + candidate = (line + " " + w).strip() + if len(candidate) > width and line: + lines.append(line) + line = w + else: + line = candidate + if line: + lines.append(line) + return lines diff --git a/docs/figures/social-card.png b/docs/figures/social-card.png new file mode 100644 index 0000000..a22543c Binary files /dev/null and b/docs/figures/social-card.png differ diff --git a/docs/figures/social-card.svg b/docs/figures/social-card.svg new file mode 100644 index 0000000..61227ea --- /dev/null +++ b/docs/figures/social-card.svg @@ -0,0 +1,10 @@ + + context-report + context-report: an open, signed report of whether an agent context artifact actually works. 3 of 11 rows recomputed from the artifact, fresh. + + + context-report + an open, signed report of whether an agent context artifact actually works + + 3 of 11 rows recomputed from the artifact, fresh + diff --git a/pyproject.toml b/pyproject.toml index 0e90fb0..455fe8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,8 +80,13 @@ extend-exclude = [".chock/bin/"] [tool.ruff.lint] select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C90", "S", "BLE", "FBT", "T20", "SIM", "RET", "ARG", "ERA", "PIE", "PL", "TRY", "RUF"] ignore = ["TRY003", "S101"] - [tool.ruff.lint.per-file-ignores] +# The open-coder-ai family's shared visual language, carried byte-identical by four +# repositories and never edited here. Its drawing primitives take many arguments because +# they take coordinates, and a generator prints what it wrote. Ignored by name rather than +# excluded: a blanket exclude would also hide an undefined name in a file nobody edits here. +"docs/figures/palette.py" = ["PLR0913", "PLR0917"] +"docs/figures/make_family.py" = ["PLR0913"] "tests/*" = ["PLR2004"] # Test doubles implement Runner/Judge Protocols by contract; unused args and terse asserts are fine. "tests/efficacy/*" = ["ARG002", "ARG005", "E702", "FBT002"] @@ -96,3 +101,9 @@ ignore = ["TRY003", "S101"] # Chock policy-authoring skill template: a runnable stub scaffolded into new policies, # printing is its whole point. Matches chock's own per-file-ignore for the same file. ".agents/skills/policy-init/assets/templates/scripts-stub.py" = ["T201"] +# Figure generators (block 2, BRIEF.md): run by hand or in CI's figures job, never imported; +# each prints what it wrote/computed, same pattern as the shared make_family.py above (excluded +# rather than ignored, since that file cannot be edited here). make_social_card.py's cairosvg +# import is deferred so the CI figures job -- which never installs cairosvg, see its docstring -- +# can still regenerate the SVG and skip only the PNG step. +"docs/figures/make_*.py" = ["T201", "PLC0415"]