feat(benchmarks): reproducible runner for four benchmarks, with the multi-round decider made visible

.gitignore

@@ -260,8 +260,16 @@ evaluation/src/adapters/*/prompts/profile/*.json
 
 # Locomo source dataset (downloadable, not source code)
 data/locomo10.json
+# Benchmark inputs live here (benchmarks/configs/*.toml default to it), and they are
+# large, redistributable only by their own authors, and not ours to ship. The directory
+# itself is kept so a fresh clone has somewhere obvious to put them.
+benchmarks/data/*
+!benchmarks/data/.gitkeep
 evaluation/data/locomo/locomo10.json
 evaluation/locomo_evaluation/data/locomo10.json
+# Benchmark inputs live here (benchmarks/configs/*.toml default to it), and they are
+# large, redistributable only by their own authors, and not ours to ship. The directory
+# itself is kept so a fresh clone has somewhere obvious to put them.
 
 # Legacy src kept locally for migration reference; not under version control.
 src_old/
@@ -280,3 +288,62 @@ benchmarks/.env
 
 # Local everos runtime data (memory root, indexes, OME state)
 .everos*/
+
+# ---------------------------------------------------------------------------
+# Internal-only benchmark scaffolding — kept in the working tree, never shipped
+# ---------------------------------------------------------------------------
+# These exist to run ablations on this machine, not to let anyone reproduce a
+# published number. Two things make them unshippable rather than merely
+# unpolished: they hardcode absolute paths into a private workspace
+# (/Evermind/..., /root/fullrun, /root/.everos_bench), and the audit set compares
+# against reference harnesses that live outside this repository and are not
+# published — so a user who cloned this would get code that cannot run at all.
+#
+# The shippable harness is what remains: run.py + config.py + adapters/ +
+# configs/ + metrics/ + scripts/reproduce.sh + README.md + .env.example.
+
+# Everything that measures something OTHER than a benchmark's published number:
+# extraction-backbone arms, retrieval-policy arms, and the two store helpers that
+# only those flows use (rewriting an existing store's episodes with a different
+# extractor, and waiting for the index to catch up afterwards). Both sweep drivers
+# read checkpoints out of a private path, and one needs a train/test split file
+# from another repository, so neither can run anywhere else.
+#
+# The shipped surface is the reproduction and nothing else: reproduce.sh + run.py +
+# config.py + adapters/ + configs/ + metrics/ + README.md + .env.example.
+benchmarks/ablations/
+
+# Migration audit. Every file here resolves paths under Evaluation/ to diff this
+# harness against the reference implementations it was ported from; without that
+# checkout none of it executes. MIGRATION*.md are the internal record of that
+# port, not user documentation.
+benchmarks/audit/
+
+# Tests that require the same absent checkout. benchmark_parity_env.py says it
+# plainly: 269 of the 395 benchmark tests skip when the references are missing,
+# and `make test` still reports success — so shipping them hands users a suite
+# that is permanently green and permanently vacuous.
+tests/unit/benchmark_parity_env.py
+tests/unit/test_benchmark_differential.py
+tests/unit/test_benchmark_differential_audit.py
+tests/unit/test_benchmark_differential_lme_emb.py
+tests/unit/test_benchmark_differential_phases.py
+tests/unit/test_benchmark_differential_pool.py
+tests/unit/test_benchmark_differential_surface.py
+tests/unit/test_benchmark_parity.py
+tests/unit/test_benchmark_parity_gate.py
+tests/unit/test_benchmark_parity_reported.py
+# Also diffs against LoCoMo/EverOS/test_locomo.py (line 27), so it belongs here
+# rather than in the shipped suite.
+tests/unit/test_benchmark_client_and_gate.py
+
+# Pins the ablation tooling above, so it goes with it. A test whose subject is not
+# shipped would fail at import on a fresh clone, not skip.
+tests/unit/test_benchmark_reextract_sampling.py
+tests/unit/test_benchmark_metrics_and_reextract.py
+tests/unit/test_benchmark_store_tools.py
+tests/unit/test_benchmark_ablation_tools.py
+
+# Hardcodes four store paths on this machine (/root/smoke_*). Marked `slow` so
+# it is deselected by default, but the paths would still be published.
+tests/unit/test_multiround_per_store.py

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install install-deps lint docs-check check-commits check-pr-title check-assets check-file-sizes check-deprecated-names check-github-docs check-cjk check-datetime openapi check-openapi format test integration package cov ci clean
+.PHONY: help install install-deps lint docs-check check-commits check-pr-title check-assets check-file-sizes check-deprecated-names check-github-docs check-cjk check-datetime openapi check-openapi format test integration package cov ci verify-parity clean
 
 help:
 	@echo "Targets:"
@@ -36,8 +36,8 @@ install: install-deps
 	uv run pre-commit install --hook-type commit-msg
 
 lint:
-	uv run ruff check src tests
-	uv run ruff format --check src tests
+	uv run ruff check src tests benchmarks
+	uv run ruff format --check src tests benchmarks
 	uv run lint-imports
 	uv run python scripts/check_repo_assets.py
 	uv run python scripts/check_file_sizes.py
@@ -105,12 +105,23 @@ check-openapi:
 	uv run python scripts/dump_openapi.py --check
 
 format:
-	uv run ruff check --fix src tests
-	uv run ruff format src tests
+	uv run ruff check --fix src tests benchmarks
+	uv run ruff format src tests benchmarks
 
 test:
 	uv run pytest tests/unit -v
 
+# The differential tests compare against a checkout outside this repository, so all of
+# them skip when it is absent -- 269 of the 395 benchmark tests, and `make test` still
+# reported success. This target refuses to skip: run it before any evaluation, because a
+# number produced by an unverified harness is worse than no number.
+verify-parity:
+	BENCHMARK_PARITY_STRICT=1 uv run pytest tests/unit -q -k benchmark
+	uv run python benchmarks/audit/check_parity.py
+	uv run python benchmarks/audit/check_protocol.py
+	uv run python benchmarks/audit/coverage.py
+	uv run python benchmarks/audit/partition.py
+
 integration:
 	uv run pytest tests/integration -v
 

benchmarks/.env.example

@@ -4,3 +4,69 @@ ANSWER_API_KEY=sk-...
 ANSWER_BASE_URL=https://openrouter.ai/api/v1
 JUDGE_API_KEY=sk-...
 JUDGE_BASE_URL=https://openrouter.ai/api/v1
+
+# --- Infrastructure the servers need. Model choice does NOT belong here:
+# it lives in configs/<benchmark>.toml, which is the single source of the
+# experiment's identity. A model name in this file was silently overridden by
+# the config at launch while still being what the run record reported. ---
+# Required when using --servers: the fleet inherits this process's environment, and a
+# server whose LLM is unconfigured aborts at startup ("LLM api_key and base_url is not
+# configured"). `everos init` only writes a blank template, so these belong here rather
+# than in each generated everos.toml.
+EVEROS_LLM__BASE_URL=https://openrouter.ai/api/v1
+EVEROS_LLM__API_KEY=
+EVEROS_LLM__TIMEOUT_SECONDS=300
+EVEROS_EMBEDDING__BASE_URL=http://127.0.0.1:9200/v1
+EVEROS_EMBEDDING__MODEL=Qwen3-Embedding-4B
+EVEROS_EMBEDDING__API_KEY=
+# Engine-wide cap on concurrent extractions. The default of 20 throttles a bulk ingest
+# long before the machine does -- each slot spends its time waiting on a remote call.
+EVEROS_OME_MAX_CONCURRENT_RUNS=64
+
+# ---------------------------------------------------------------------------
+# Paths (all optional — every one has a working default)
+# ---------------------------------------------------------------------------
+# LEFT EMPTY ON PURPOSE. Each of these has a default that works out of the box, and
+# an empty value falls through to it. A placeholder like `/path/to/Evaluation` would
+# NOT: it is a set variable, so it wins over the default, and the run then writes to
+# a directory literally called `/path/to/...`. A value that looks configured but is
+# not is worse than no value at all.
+#
+# Fill one in only when the real path is somewhere else.
+
+# Datasets. Default: benchmarks/data/<the file named below>.
+#   locomo        -> benchmarks/data/locomo10.json
+#   longmemeval   -> benchmarks/data/longmemeval_s.json
+#   subtlememory  -> benchmarks/data/subtlememory/      (a directory)
+#   evermembench  -> benchmarks/data/evermembench.json
+BENCH_DATA_LOCOMO=
+BENCH_DATA_LONGMEMEVAL=
+BENCH_DATA_SUBTLEMEMORY=
+BENCH_DATA_EVERMEMBENCH=
+
+# EverMemBench only: its raw release, read to recover session names the converted
+# file does not carry. Default: benchmarks/data/raw/EverMemBench-Dynamic.
+EVERMEMBENCH_RAW_ROOT=
+
+# Where runs are written. Default: benchmarks/results/<Dataset>/. Point this at a
+# directory outside the repository when the filesystem holding it is near capacity —
+# a full disk surfaces as [Errno 28] on every answer write, which is retried and then
+# recorded as a failed question rather than as an outage.
+BENCH_EVAL_ROOT=
+
+# ---------------------------------------------------------------------------
+# The multi-round retrieval decider (optional)
+# ---------------------------------------------------------------------------
+# Leave BOTH empty and the decider runs the model from [llm] — the single-model
+# setup, which needs no extra configuration and is what a first run should use.
+#
+# Set them TOGETHER to give the decider its own model. A name without its endpoint
+# returns 404 on every call, and the retrieval loop then falls back to a fixed core
+# and still reports a complete result. run.py probes the decider at startup and
+# refuses to run rather than let that happen quietly.
+#
+# Our published numbers used BENCH_DECIDER_MODEL=qwen3.6-27B.
+BENCH_DECIDER_MODEL=
+BENCH_DECIDER_BASE_URL=
+# Any non-empty value for a self-hosted server; a hosted API needs its real key.
+BENCH_DECIDER_API_KEY=EMPTY