From bf646686140f34b2eced43e629a6bb8fb3954fcb Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 25 Jul 2026 19:51:48 +0100 Subject: [PATCH 1/2] docs: use final AsDecided repository paths --- README.md | 21 ++++++++++--------- autonomousqa/EXPLAINER.md | 2 +- autonomousqa/README.md | 7 ++++--- autonomousqa/schema/run_record.schema.json | 2 +- .../docs/AGENTIC_BENCHMARK_RUN_HANDOFF.md | 8 +++---- .../rac/roadmaps/publish-swe-decisionbench.md | 4 ++-- 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f01929a..ca8f12b 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# rac-benchmarks +# AsDecided Benchmarks -Evaluation suites for [RAC](https://github.com/itsthelore/rac-core) +Evaluation suites for [AsDecided](https://github.com/asdecided/core) (requirements-as-code) — one subdir per benchmark. Per ADR-092 (one repo per concern, subdir per member) this is the single home for RAC's benchmarks; future suites land as sibling subdirs rather than new repositories. -Each benchmark consumes `rac` only as an **external CLI on `PATH`** and imports +Each benchmark consumes the engine only as an **external CLI on `PATH`** and imports no engine code (DG-ADR-0001), so the suites stay decoupled from the engine's internals. Scoring is deterministic and offline — no embeddings, no LLM judge, no network, no randomness, no clock in the scored path (ADR-066): the @@ -28,7 +28,7 @@ corpus. ## Shared harness [`harness/`](harness/) is the shared package the per-tool benchmarks consume: -a subprocess runner (rac as an external CLI), a deterministic scorer, a +a subprocess runner (the engine as an external CLI), a deterministic scorer, a `{metrics, metadata, per_query}` scorecard writer, and a CI gate. Each benchmark subdir is a thin `run.py` over it, with the `rac eval` flag surface: @@ -47,7 +47,8 @@ committed `baseline.json`. Baselines change only through a human-reviewed ## Running the battery ``` -pip install "git+https://github.com/itsthelore/rac-core.git" # rac on PATH +brew install asdecided/tap/asdecided-core +export RAC_BIN=decided # temporary benchmark adapter variable pip install -e ".[dev]" python -m pytest -q ``` @@ -55,11 +56,11 @@ python -m pytest -q ## History `decisiongrounding/` is the former **`itsthelore/decisiongrounding`** repository, -moved here with its history preserved (ADR-092 convergence). The benchmark runs -unchanged against the published `rac` CLI; its deterministic scoring contract -(ADR-066) is untouched. Its port onto the shared harness is tracked as a -`tool-benchmarks` roadmap initiative in rac-core and deliberately does not -expand the frozen restructure item's scope. +moved here with its history preserved (ADR-092 convergence). Its deterministic +scoring contract (ADR-066) is untouched. The benchmark currently names its +grounding arm and adapter variable `rac`/`RAC_BIN`; point that adapter at the +native `decided` executable as shown above. Renaming the benchmark vocabulary +is separate from this repository-topology cutover. ## License diff --git a/autonomousqa/EXPLAINER.md b/autonomousqa/EXPLAINER.md index 325d3e2..28e0a48 100644 --- a/autonomousqa/EXPLAINER.md +++ b/autonomousqa/EXPLAINER.md @@ -18,7 +18,7 @@ because it launders uncertainty into false confidence. This benchmark exists to make those claims comparable and honest. It is **agent-agnostic**: the apps, the requirement lists, and the scoring are fixed, and any agent that can be run from a command line against a URL can -be measured. [Proofkeeper](https://github.com/itsthelore/proofkeeper) is the +be measured. [Proofkeeper](https://github.com/asdecided/proofkeeper) is the reference agent, but a benchmark the reference agent merely *wins* carries more weight than a marketing page — and if another agent beats it here, the results page will say so. diff --git a/autonomousqa/README.md b/autonomousqa/README.md index 95ae7ce..dce93f1 100644 --- a/autonomousqa/README.md +++ b/autonomousqa/README.md @@ -11,7 +11,7 @@ scoring that re-derives from recorded evidence, never from a model's opinion. "Our agent verifies more of your product for fewer tokens" is marketing until anyone can re-run it. This benchmark makes it an argument: a property of the **corpora, sample apps, and deterministic scoring** — runnable against any -autonomous-QA agent — with [Proofkeeper](https://github.com/itsthelore/proofkeeper) +autonomous-QA agent — with [Proofkeeper](https://github.com/asdecided/proofkeeper) as the reference agent. An agent is measured on: - **Verified rate** — capabilities whose compiled test passed the agent's @@ -95,8 +95,9 @@ make test # the offline pytest battery (no browser, no agen make fixtures # every scripted flow across all four apps ``` -`smoke` and `fixtures` need `rac` on PATH (`pip install rac-core`, or set -`RAC_BIN`) because the corpus reaches the agent over `rac export --graph`. +`smoke` and `fixtures` need the native engine on PATH. Install it with +`brew install asdecided/tap/asdecided-core`, then set `RAC_BIN=decided` while +the benchmark adapter still uses its legacy variable name. ## Run it for real (BYOK) diff --git a/autonomousqa/schema/run_record.schema.json b/autonomousqa/schema/run_record.schema.json index 12baa20..2221ece 100644 --- a/autonomousqa/schema/run_record.schema.json +++ b/autonomousqa/schema/run_record.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/itsthelore/rac-benchmarks/autonomousqa/schema/run_record.schema.json", + "$id": "https://github.com/asdecided/benchmarks/autonomousqa/schema/run_record.schema.json", "title": "autonomousqa run record", "description": "One agent run over one capability of one frozen sample app. Carries the exact harness configuration and the raw evidence (stdout + exit code) so verdicts re-derive deterministically without re-running the agent or re-spending tokens.", "type": "object", diff --git a/decisiongrounding/docs/AGENTIC_BENCHMARK_RUN_HANDOFF.md b/decisiongrounding/docs/AGENTIC_BENCHMARK_RUN_HANDOFF.md index 6bf6708..40af537 100644 --- a/decisiongrounding/docs/AGENTIC_BENCHMARK_RUN_HANDOFF.md +++ b/decisiongrounding/docs/AGENTIC_BENCHMARK_RUN_HANDOFF.md @@ -7,7 +7,7 @@ session. ## How to use -1. Start a Claude Code session with the `itsthelore/rac-benchmarks` repository +1. Start a Claude Code session with the `asdecided/benchmarks` repository in scope. The benchmark lives in its `decisiongrounding/` subdirectory (the standalone `itsthelore/decisiongrounding` repo was archived into it, history preserved); every command below runs from that subdirectory. @@ -31,7 +31,7 @@ clone), and the `paper/` LaTeX scaffold + `scripts/paper_figs.py`. --- ```text -You are Claude Code working on the itsthelore/rac-benchmarks repository. The +You are Claude Code working on the asdecided/benchmarks repository. The benchmark is its decisiongrounding/ subdirectory — treat that directory as your working directory for every command below. It is a reproducible benchmark testing whether deterministic, supersession-aware @@ -76,8 +76,8 @@ Either way: - Work on a NEW branch off fresh origin/main: claude/decision-grounding-results-. Never push to main. - Install: `pip install -e ".[real,schema,chart]"`. The `rac` arm needs the `rac` - CLI on PATH (pip install "git+https://github.com/itsthelore/rac-core.git" if - absent). Alternatively `./scripts/from_source.sh` does venv + install + loads + CLI on PATH (`brew install asdecided/tap/asdecided-core`, then + `export RAC_BIN=decided`). Alternatively `./scripts/from_source.sh` does venv + install + loads keys from .env + probe + run in one go; the granular steps below are recommended for the first real run so you can inspect each stage. diff --git a/decisiongrounding/rac/roadmaps/publish-swe-decisionbench.md b/decisiongrounding/rac/roadmaps/publish-swe-decisionbench.md index 1cd7019..9af9e17 100644 --- a/decisiongrounding/rac/roadmaps/publish-swe-decisionbench.md +++ b/decisiongrounding/rac/roadmaps/publish-swe-decisionbench.md @@ -94,5 +94,5 @@ tags: [publication, paper, arxiv] ## Related Tickets -- itsthelore/rac-benchmarks#11 -- itsthelore/rac-core#295 +- asdecided/benchmarks#11 +- asdecided/core#295 From 0e539b065422fae6178c2fe3a481f424d4286d8f Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 25 Jul 2026 21:17:52 +0100 Subject: [PATCH 2/2] ci: cut benchmarks over to native AsDecided --- .github/scripts/install-asdecided.sh | 23 ++++++++++++++++++++++ .github/workflows/autonomousqa.yml | 16 ++++++++------- .github/workflows/ci.yml | 10 ++++------ .github/workflows/decisiongrounding-ci.yml | 12 +++++------ autonomousqa/runner/cli.py | 4 ++-- decisiongrounding/.github/workflows/ci.yml | 12 ++++------- gitchameleon/build_corpus.py | 1 - harness/cli.py | 2 +- harness/runner.py | 4 ++-- tests/conftest.py | 4 ++-- tests/test_benchmarks.py | 2 +- tests/test_gitchameleon_scaffold.py | 2 +- 12 files changed, 54 insertions(+), 38 deletions(-) create mode 100755 .github/scripts/install-asdecided.sh diff --git a/.github/scripts/install-asdecided.sh b/.github/scripts/install-asdecided.sh new file mode 100755 index 0000000..3826e37 --- /dev/null +++ b/.github/scripts/install-asdecided.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="${1:-0.23.1}" +version="${version#v}" +if [[ "$version" != "0.23.1" ]]; then + echo "::error::No verified checksum is recorded for asdecided-core $version" + exit 1 +fi + +archive="asdecided-x86_64-unknown-linux-gnu.tar.gz" +digest="51cce8025a7cb2f8b2caea93a8ea71be0ad8c5c316fd0ecced688267bf97b8ac" +install_dir="${RUNNER_TEMP}/asdecided-${version}" +download="${RUNNER_TEMP}/${archive}" +url="https://github.com/asdecided/core/releases/download/v${version}/${archive}" + +mkdir -p "$install_dir" +curl --fail --location --retry 3 --silent --show-error "$url" --output "$download" +echo "$digest $download" | sha256sum --check - +tar -xzf "$download" -C "$install_dir" + +echo "$install_dir" >> "$GITHUB_PATH" +"$install_dir/decided" --version diff --git a/.github/workflows/autonomousqa.yml b/.github/workflows/autonomousqa.yml index af996b1..c8c13a6 100644 --- a/.github/workflows/autonomousqa.yml +++ b/.github/workflows/autonomousqa.yml @@ -13,10 +13,12 @@ on: paths: - "autonomousqa/**" - ".github/workflows/autonomousqa.yml" + - ".github/scripts/install-asdecided.sh" pull_request: paths: - "autonomousqa/**" - ".github/workflows/autonomousqa.yml" + - ".github/scripts/install-asdecided.sh" workflow_dispatch: concurrency: @@ -58,15 +60,15 @@ jobs: with: python-version: "3.12" cache: pip - - name: Install rac - run: pip install "rac-core @ git+https://github.com/itsthelore/rac-core.git" + - name: Install AsDecided + run: bash ../.github/scripts/install-asdecided.sh 0.23.1 - name: Gate the member corpus - run: rac gate rac + run: decided gate rac - name: Validate every seeded app corpus run: | for corpus in apps/*/rac; do echo "== $corpus" - rac validate "$corpus" + decided validate "$corpus" done smoke: @@ -89,9 +91,9 @@ jobs: - name: Install the playwright browser run: npx playwright install --with-deps chromium working-directory: autonomousqa/workspace - - name: Install the harness and rac + - name: Install the harness and AsDecided run: | pip install -e . - pip install "rac-core @ git+https://github.com/itsthelore/rac-core.git" + bash ../.github/scripts/install-asdecided.sh 0.23.1 - name: Smoke — drive, compile, fidelity-gate, meter, re-score - run: python -m runner.cli smoke + run: RAC_BIN=decided python -m runner.cli smoke diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7533da..d321d18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI # Gate every per-tool benchmark on every push to main and every pull request. -# rac is installed from its public source and consumed strictly as an external +# AsDecided is installed from its checksum-verified native release and consumed as an external # CLI on PATH (DG-ADR-0001) — the suite imports no engine code. Re-baselining # is human-gated: no step here may ever pass --update-baseline (a test # enforces this). @@ -31,10 +31,8 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - - name: Install rac (external CLI on PATH, never an import) - run: | - python -m pip install --upgrade pip - pip install "git+https://github.com/itsthelore/rac-core.git" + - name: Install AsDecided (external CLI on PATH, never an import) + run: bash .github/scripts/install-asdecided.sh 0.23.1 - name: Install the harness run: pip install -e ".[dev]" @@ -42,7 +40,7 @@ jobs: - name: Validate every fixture corpus run: | for bench in search-artifacts find-decisions get-artifact get-related get-summary; do - rac validate "$bench/corpus" + decided validate "$bench/corpus" done - name: Run the benchmark battery diff --git a/.github/workflows/decisiongrounding-ci.yml b/.github/workflows/decisiongrounding-ci.yml index b81bb84..5bdc771 100644 --- a/.github/workflows/decisiongrounding-ci.yml +++ b/.github/workflows/decisiongrounding-ci.yml @@ -9,9 +9,9 @@ name: decisiongrounding CI on: push: branches: [main] - paths: ["decisiongrounding/**"] + paths: ["decisiongrounding/**", ".github/scripts/install-asdecided.sh"] pull_request: - paths: ["decisiongrounding/**"] + paths: ["decisiongrounding/**", ".github/scripts/install-asdecided.sh"] permissions: contents: read @@ -62,10 +62,8 @@ jobs: python-version: "3.12" cache: pip - - name: Install rac - run: | - python -m pip install --upgrade pip - pip install "git+https://github.com/itsthelore/rac-core.git" + - name: Install AsDecided + run: bash ../.github/scripts/install-asdecided.sh 0.23.1 - name: Gate the RAC corpus (validate + relationships + review) - run: rac gate rac + run: decided gate rac diff --git a/autonomousqa/runner/cli.py b/autonomousqa/runner/cli.py index 35f2e3d..e4c092e 100644 --- a/autonomousqa/runner/cli.py +++ b/autonomousqa/runner/cli.py @@ -40,10 +40,10 @@ def export_graph(corpus: Path, out_path: Path) -> None: """Export the corpus graph over the published `rac export --graph` contract.""" - rac = os.environ.get("RAC_BIN", "rac") + rac = os.environ.get("RAC_BIN", "decided") if shutil.which(rac) is None: raise FileNotFoundError( - f"'{rac}' not on PATH — install rac-core or set RAC_BIN") + f"'{rac}' not on PATH — install asdecided-core or set RAC_BIN") result = subprocess.run([rac, "export", str(corpus), "--graph"], capture_output=True, text=True) if result.returncode != 0: diff --git a/decisiongrounding/.github/workflows/ci.yml b/decisiongrounding/.github/workflows/ci.yml index 0e130b2..6ca6780 100644 --- a/decisiongrounding/.github/workflows/ci.yml +++ b/decisiongrounding/.github/workflows/ci.yml @@ -43,9 +43,7 @@ jobs: corpus: # Dogfood RAC: enforce the project's own knowledge corpus the same way the - # benchmark studies — validation, relationships, and review must pass. `rac` - # is installed from its public source (it is the external tool the rac arm - # also uses; not a Python dependency of the benchmark). + # benchmark studies — validation, relationships, and review must pass. name: rac corpus gate runs-on: ubuntu-latest steps: @@ -57,10 +55,8 @@ jobs: python-version: "3.12" cache: pip - - name: Install rac - run: | - python -m pip install --upgrade pip - pip install "git+https://github.com/itsthelore/rac-core.git" + - name: Install AsDecided + run: bash .github/scripts/install-asdecided.sh 0.23.1 - name: Gate the RAC corpus (validate + relationships + review) - run: rac gate rac + run: decided gate rac diff --git a/gitchameleon/build_corpus.py b/gitchameleon/build_corpus.py index b1ea79b..3b12fc2 100644 --- a/gitchameleon/build_corpus.py +++ b/gitchameleon/build_corpus.py @@ -67,7 +67,6 @@ def decision_markdown(row: dict) -> str: "schema_version: 1", f"id: {artifact_id(row['example_id'])}", "type: decision", - f"tags: [dependencies, versions, {library}]", "---", f"# Library Version Pin: {library} {version}", "", diff --git a/harness/cli.py b/harness/cli.py index 2be5047..15174a5 100644 --- a/harness/cli.py +++ b/harness/cli.py @@ -79,7 +79,7 @@ def run_benchmark( queries_path: str, *, generated_at: str | None = None, - executable: str = "rac", + executable: str = "decided", ) -> card_mod.Scorecard: """Score every committed case and assemble the scorecard. diff --git a/harness/runner.py b/harness/runner.py index 5d0a907..72bad00 100644 --- a/harness/runner.py +++ b/harness/runner.py @@ -42,11 +42,11 @@ def payload(self) -> dict: class RacRunner: """Runs ``rac`` subcommands and hands back their JSON contracts.""" - def __init__(self, executable: str = "rac") -> None: + def __init__(self, executable: str = "decided") -> None: if shutil.which(executable) is None: raise UsageError( f"'{executable}' not found on PATH — the suite consumes rac as an " - "external CLI (DG-ADR-0001); install rac-core first" + "external CLI (DG-ADR-0001); install asdecided-core first" ) self.executable = executable diff --git a/tests/conftest.py b/tests/conftest.py index 30c66e8..76614ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,8 +64,8 @@ def copy_corpus(bench: str, tmp_path: Path) -> Path: @pytest.fixture(scope="session", autouse=True) def rac_on_path(): - if shutil.which("rac") is None: - pytest.fail("rac is not on PATH — the suite consumes rac as an external CLI") + if shutil.which("decided") is None: + pytest.fail("decided is not on PATH — the suite consumes AsDecided as an external CLI") @pytest.fixture(scope="session") diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 00c2206..088fd8b 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -57,7 +57,7 @@ def test_scorecard_has_exactly_three_top_level_objects(bench, scorecards): @pytest.mark.parametrize("bench", BENCHMARKS) def test_metadata_records_version_hashes_and_count(bench, scorecards): meta = scorecards[bench]["metadata"] - assert meta["rac_version"].startswith("rac ") + assert meta["rac_version"].startswith("decided ") assert meta["corpus_hash"].startswith("sha256:") assert meta["query_set_hash"].startswith("sha256:") assert meta["n_queries"] == len(scorecards[bench]["per_query"]) diff --git a/tests/test_gitchameleon_scaffold.py b/tests/test_gitchameleon_scaffold.py index 0c70caa..97b879d 100644 --- a/tests/test_gitchameleon_scaffold.py +++ b/tests/test_gitchameleon_scaffold.py @@ -57,7 +57,7 @@ def test_corpus_builder_is_deterministic(tmp_path): def test_built_corpora_are_schema_valid(tmp_path): corpus = _build(tmp_path) completed = subprocess.run( - ["rac", "validate", str(corpus)], capture_output=True, text=True, check=False + ["decided", "validate", str(corpus)], capture_output=True, text=True, check=False ) assert completed.returncode == 0, completed.stdout + completed.stderr