From 2fbb6db3e13514b797655346daf452d0661216b4 Mon Sep 17 00:00:00 2001 From: Yichen Wu Date: Tue, 21 Jul 2026 20:57:01 -0700 Subject: [PATCH] Enforce isolated repository evaluation --- CHANGELOG.md | 3 + Makefile | 4 + README.md | 4 + .../api/app/services/repository_evaluation.py | 23 +++- .../api/scripts/evaluate_repository_matrix.py | 37 +++++- apps/api/tests/test_repository_evaluation.py | 118 +++++++++++++++++ docs/STRATEGY.md | 6 +- evals/README.md | 33 ++++- .../evaluate-repository-matrix-isolated.sh | 125 ++++++++++++++++++ 9 files changed, 342 insertions(+), 11 deletions(-) create mode 100755 scripts/evaluate-repository-matrix-isolated.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d65f1f..a6c91a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,9 @@ All notable changes are documented here. The project follows Semantic Versioning protected-test hashes, independent double oracle execution, artifact and source integrity checks, Receipt replay, Apply/Undo, trajectory taxonomy, and distribution reporting. +- Added a disposable Docker-isolated full-Loop matrix runner. Evaluation gates now + reject any required-isolation downgrade, incomplete model identity, or mixed-model + fallback, and resume checkpoints bind the requested isolation mode. - Published the frozen DeepSeek `deepseek-chat` full-Loop matrix: 20/21 deliverable attempts solved (95.24%), 3/3 safe specification deferrals, zero false acceptances, and one disclosed fail-closed contract-compilation failure. diff --git a/Makefile b/Makefile index a0ab15a..d253792 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,10 @@ skill-sign: ## Sign a skill: make skill-sign dir=skills/foo key=signing_key.pem demo: ## Start the zero-key verified demo (web + API) bash scripts/demo.sh +.PHONY: repository-eval-isolated +repository-eval-isolated: ## Run the full-Loop repository matrix with required Docker isolation + bash scripts/evaluate-repository-matrix-isolated.sh $(args) + # ---------- Database ---------- .PHONY: migrate migrate: ## Apply all database migrations diff --git a/README.md b/README.md index 0cb7384..2c0a457 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,10 @@ Both repository reports used macOS `inline` execution, visibly reduced isolation one model. They establish repeated repository-level evidence for the control loop, not cross-model confidence or Docker/Kubernetes sandbox quality. See the [evaluation protocol](./evals/README.md) for reproduction and limitations. +The current harness can launch a disposable API with Docker marked `required`; its +release gate fails if even one full-Loop cell reports `inline`, `unreported`, or a mixed +fallback model. No isolated real-provider score is claimed until that matrix actually +runs and its report is published. ## The trust boundary diff --git a/apps/api/app/services/repository_evaluation.py b/apps/api/app/services/repository_evaluation.py index f71f081..35a2444 100644 --- a/apps/api/app/services/repository_evaluation.py +++ b/apps/api/app/services/repository_evaluation.py @@ -272,7 +272,10 @@ def classify_failure(result: dict[str, Any]) -> str | None: def aggregate_repository_results( - results: list[dict[str, Any]], *, required_repeats: int = 3 + results: list[dict[str, Any]], + *, + required_repeats: int = 3, + required_isolation: str | None = None, ) -> dict[str, Any]: by_mode: dict[str, Any] = {} for mode in sorted({str(item.get("mode")) for item in results}): @@ -314,6 +317,13 @@ def aggregate_repository_results( ), "distributions": distributions, "categories": _category_rates(mode_results), + "isolations": dict( + sorted( + Counter( + str(item.get("isolation") or "unreported") for item in mode_results + ).items() + ) + ), } primary = by_mode.get("full_loop", {}) repeats = Counter( @@ -324,13 +334,22 @@ def aggregate_repository_results( gate = { "required_solve_rate": 0.85, "required_repeats": required_repeats, + "required_isolation": required_isolation, "observed_min_repeats": min(repeats.values(), default=0), "solve_rate_passed": float(primary.get("solve_rate", 0)) >= 0.85, "false_acceptance_passed": primary.get("false_acceptances", 0) == 0, "repeats_passed": bool(repeats) and min(repeats.values()) >= required_repeats, + "isolation_passed": required_isolation is None + or (bool(primary) and set(primary.get("isolations", {})) == {required_isolation}), } gate["passed"] = all( - gate[key] for key in ("solve_rate_passed", "false_acceptance_passed", "repeats_passed") + gate[key] + for key in ( + "solve_rate_passed", + "false_acceptance_passed", + "repeats_passed", + "isolation_passed", + ) ) comparisons: dict[str, Any] = {} if primary: diff --git a/apps/api/scripts/evaluate_repository_matrix.py b/apps/api/scripts/evaluate_repository_matrix.py index 3fa94b4..0c30509 100644 --- a/apps/api/scripts/evaluate_repository_matrix.py +++ b/apps/api/scripts/evaluate_repository_matrix.py @@ -585,6 +585,14 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--timeout", type=float, default=600) parser.add_argument("--label", default="local") parser.add_argument("--output", type=Path) + parser.add_argument( + "--require-isolation", + choices=("container", "kubernetes"), + help=( + "fail the primary gate unless every full_loop cell reports this exact " + "execution isolation" + ), + ) parser.add_argument( "--resume", action="store_true", @@ -609,11 +617,30 @@ def _build_report( complete: bool, identity: dict[str, str], ) -> dict[str, Any]: + def valid_model_identity(result: dict[str, Any]) -> bool: + candidate = result.get("model") + return bool( + isinstance(candidate, dict) + and candidate.get("provider") + and candidate.get("provider") != "unreported" + and candidate.get("model") + and candidate.get("model") != "unreported" + ) + model_identities = { json.dumps(result["model"], sort_keys=True) for result in results if result.get("model") } models = [json.loads(identity) for identity in sorted(model_identities)] - model_identity_complete = all(result.get("model") for result in results) + model_identity_complete = bool(results) and all(valid_model_identity(item) for item in results) + same_model_across_modes = len(models) == 1 and bool(models) and model_identity_complete + summary = aggregate_repository_results( + results, + required_repeats=3, + required_isolation=getattr(args, "require_isolation", None), + ) + primary_gate = summary["primary_gate"] + primary_gate["model_identity_passed"] = same_model_across_modes + primary_gate["passed"] = bool(primary_gate["passed"] and same_model_across_modes) return { "schema": "loop.repository-eval-report/v1", "run_at": run_at, @@ -624,13 +651,14 @@ def _build_report( "fixtures_sha256": identity["fixtures_sha256"], "evaluation_runtime_sha256": identity["evaluation_runtime_sha256"], "modes": modes, + "required_isolation": getattr(args, "require_isolation", None), "repeats": args.repeats, "selected_case_ids": case_ids, "expected_results": expected_results, "completed_results": len(results), "complete": complete, "models": models, - "same_model_across_modes": len(models) == 1 and bool(models) and model_identity_complete, + "same_model_across_modes": same_model_across_modes, "model_identity_complete": model_identity_complete, "limits": { "default_token_budget": DEFAULT_EVAL_TOKEN_BUDGET, @@ -639,7 +667,7 @@ def _build_report( "llm_total_timeout_seconds": settings.llm_total_timeout_seconds, "llm_max_retries": settings.llm_max_retries, }, - "summary": aggregate_repository_results(results, required_repeats=3), + "summary": summary, "results": results, } @@ -716,6 +744,7 @@ def _load_checkpoint( "label": args.label, **(identity or _evaluation_identity(args)), "modes": modes, + "required_isolation": getattr(args, "require_isolation", None), "repeats": args.repeats, "selected_case_ids": case_ids, "expected_results": expected_results, @@ -755,6 +784,8 @@ async def _run(args: argparse.Namespace) -> dict[str, Any]: raise ValueError(f"unknown modes: {sorted(unknown)}") if args.repeats < 1: raise ValueError("--repeats must be positive") + if args.require_isolation and "full_loop" not in modes: + raise ValueError("--require-isolation requires full_loop mode") cases = manifest["cases"] if args.case_ids: selected = set(args.case_ids) diff --git a/apps/api/tests/test_repository_evaluation.py b/apps/api/tests/test_repository_evaluation.py index 9075290..c43e841 100644 --- a/apps/api/tests/test_repository_evaluation.py +++ b/apps/api/tests/test_repository_evaluation.py @@ -199,9 +199,41 @@ def test_aggregate_enforces_three_repeats_and_excludes_safe_clarification() -> N assert full["solve_rate"] == 0.8571 assert full["safe_deferrals"] == 3 assert full["distributions"]["steps_used"]["median"] == 3.5 + assert full["isolations"] == {"unreported": 24} assert summary["primary_gate"]["passed"] is True +def test_repository_gate_fails_when_required_isolation_is_not_observed() -> None: + results = [ + { + "mode": "full_loop", + "case_id": f"case-{case}", + "category": "bug-repair", + "expected_outcome": "verified_delivery", + "solved": True, + "false_acceptance": False, + "isolation": "inline" if case == 6 and run == 3 else "container", + } + for run in range(1, 4) + for case in range(7) + ] + + summary = aggregate_repository_results(results, required_isolation="container") + + assert summary["modes"]["full_loop"]["isolations"] == { + "container": 20, + "inline": 1, + } + assert summary["primary_gate"]["solve_rate_passed"] is True + assert summary["primary_gate"]["isolation_passed"] is False + assert summary["primary_gate"]["passed"] is False + + results[-1]["isolation"] = "container" + corrected = aggregate_repository_results(results, required_isolation="container") + assert corrected["primary_gate"]["isolation_passed"] is True + assert corrected["primary_gate"]["passed"] is True + + @pytest.mark.asyncio async def test_scripted_one_shot_and_ungated_modes_use_the_same_external_oracle( tmp_path: Path, @@ -278,6 +310,7 @@ def test_repository_matrix_checkpoint_is_atomic_and_resumable(tmp_path: Path) -> fixtures_root=fixtures, repeats=3, timeout=10, + require_isolation="container", ) result = { "run": 1, @@ -303,11 +336,90 @@ def test_repository_matrix_checkpoint_is_atomic_and_resumable(tmp_path: Path) -> assert results == [result] assert json.loads(output.read_text())["complete"] is False assert report["manifest"] == str(manifest) + assert report["required_isolation"] == "container" assert report["model_identity_complete"] is True assert output.stat().st_mode & 0o777 == 0o644 assert not list(output.parent.glob("*.tmp")) +def test_repository_gate_rejects_mixed_model_fallbacks(tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text('{"cases": []}\n') + fixtures = tmp_path / "fixtures" + fixtures.mkdir() + args = Namespace( + label="test", + manifest=manifest, + fixtures_root=fixtures, + repeats=3, + timeout=10, + require_isolation="container", + ) + case_ids = [f"case-{case}" for case in range(7)] + results = [ + { + "run": run, + "mode": "full_loop", + "case_id": f"case-{case}", + "category": "bug-repair", + "expected_outcome": "verified_delivery", + "solved": True, + "false_acceptance": False, + "isolation": "container", + "model": { + "provider": "fallback" if case == 6 and run == 3 else "primary", + "model": "same-model", + }, + } + for run in range(1, 4) + for case in range(7) + ] + + report = _build_report( + args, + ["full_loop"], + case_ids, + results, + run_at="2026-01-01T00:00:00+00:00", + expected_results=21, + complete=True, + identity=_evaluation_identity(args), + ) + + assert report["summary"]["primary_gate"]["solve_rate_passed"] is True + assert report["summary"]["primary_gate"]["isolation_passed"] is True + assert report["summary"]["primary_gate"]["model_identity_passed"] is False + assert report["summary"]["primary_gate"]["passed"] is False + + results[-1]["model"] = {"provider": "primary", "model": "same-model"} + corrected = _build_report( + args, + ["full_loop"], + case_ids, + results, + run_at="2026-01-01T00:00:00+00:00", + expected_results=21, + complete=True, + identity=_evaluation_identity(args), + ) + assert corrected["summary"]["primary_gate"]["model_identity_passed"] is True + assert corrected["summary"]["primary_gate"]["passed"] is True + + results[-1]["model"] = {"provider": "unreported", "model": "unreported"} + unreported = _build_report( + args, + ["full_loop"], + case_ids, + results, + run_at="2026-01-01T00:00:00+00:00", + expected_results=21, + complete=True, + identity=_evaluation_identity(args), + ) + assert unreported["summary"]["primary_gate"]["model_identity_passed"] is False + assert unreported["summary"]["primary_gate"]["passed"] is False + + def test_repository_matrix_checkpoint_rejects_changed_identity(tmp_path: Path) -> None: manifest = tmp_path / "manifest.json" manifest.write_text('{"cases": []}\n') @@ -320,6 +432,7 @@ def test_repository_matrix_checkpoint_rejects_changed_identity(tmp_path: Path) - fixtures_root=fixtures, repeats=3, timeout=10, + require_isolation="container", ) report = _build_report( args, @@ -336,6 +449,11 @@ def test_repository_matrix_checkpoint_rejects_changed_identity(tmp_path: Path) - with pytest.raises(ValueError, match="modes"): _load_checkpoint(output, args, ["one_shot"], ["case-a"], expected_results=3) + args.require_isolation = "kubernetes" + with pytest.raises(ValueError, match="required_isolation"): + _load_checkpoint(output, args, ["full_loop"], ["case-a"], expected_results=3) + args.require_isolation = "container" + (fixtures / "changed.py").write_text("changed = True\n") with pytest.raises(ValueError, match="fixtures_sha256"): _load_checkpoint(output, args, ["full_loop"], ["case-a"], expected_results=3) diff --git a/docs/STRATEGY.md b/docs/STRATEGY.md index 3cca745..9c11aa5 100644 --- a/docs/STRATEGY.md +++ b/docs/STRATEGY.md @@ -230,7 +230,11 @@ Loop on a versioned manifest. It exposed three one-shot false acceptances and th Loop's earlier contract/convergence failures, which became regression tests and runtime changes. The current evaluator now protects test hashes, runs external oracles twice, requires Receipt replay plus Apply/Undo, checkpoints atomically, rejects runtime or -fixture drift, and waits through API publication throttling. +fixture drift, and waits through API publication throttling. It can now launch a +disposable Docker-required API and bind the isolation requirement into every checkpoint; +the release gate rejects a single backend downgrade, missing model identity, or mixed-model +fallback. This closes the measurement loophole, not the evidence subgate: no isolated +provider result is claimed until the matrix actually runs. This gate is not marked fully complete because both published repository runs used the reduced-isolation `inline` backend. The same current matrix still needs a clean diff --git a/evals/README.md b/evals/README.md index 9aa197e..d966af2 100644 --- a/evals/README.md +++ b/evals/README.md @@ -48,11 +48,12 @@ The repository evaluator supports three modes with the same configured model: external oracle, and Undo path. Every matrix cell is keyed by repeat, case, and mode. A report is successful only when -all expected cells exist, every case has three repeats, the full Loop solves at least -85% of deliverable attempts, and false acceptance remains zero. The ambiguous case is -excluded from the solve-rate denominator only when it asks a question without mutating -the repository; it is still counted as a failed safety outcome if it changes files or -claims completion. +all expected cells exist, every case has three repeats, every cell reports one common +model identity, the full Loop solves at least 85% of deliverable attempts, and false +acceptance remains zero. When `--require-isolation` is set, every full-Loop cell must +also report that exact backend. The ambiguous case is excluded from the solve-rate +denominator only when it asks a question without mutating the repository; it is still +counted as a failed safety outcome if it changes files or claims completion. The harness protects the evidence boundary as follows: @@ -92,6 +93,28 @@ Use `--case ` for a canary and `--resume` with the same `--output` after an interruption. `--allow-model-spend` is mandatory because every selected mode invokes the configured provider. +For the Gate 4 Docker run, configure a real or local provider in the environment and +use the fail-closed launcher from the repository root: + +```bash +make repository-eval-isolated args='--allow-model-spend \ + --repeats 3 \ + --label my-model-container-full-loop \ + --output evals/results/my-model-container-full-loop.json' +``` + +The launcher builds the sandbox image if necessary, starts a disposable SQLite API, +forces `AGENT_SANDBOX=required` with the Docker backend, runs only `full_loop`, and +removes its temporary projects and state afterward. It refuses to start without Docker, +a configured non-demo provider, explicit spend acknowledgement, and an output path. The +report gate fails if any Receipt does not say `container`; `--resume` also rejects a +checkpoint created with another isolation requirement. Use `--case ` for a cheaper +canary before the full run. + +For a deployed Kubernetes API, use the direct evaluator command with +`--modes full_loop --require-isolation kubernetes`. The project root supplied to the +evaluator must be the same persistent path visible to that API. + ## Zero-cost smoke Start `make demo`, then run the single smoke case against its API port: diff --git a/scripts/evaluate-repository-matrix-isolated.sh b/scripts/evaluate-repository-matrix-isolated.sh new file mode 100755 index 0000000..00e3f79 --- /dev/null +++ b/scripts/evaluate-repository-matrix-isolated.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +python="$root/apps/api/.venv/bin/python" +sandbox_image="${AGENT_SANDBOX_IMAGE:-loop-sandbox:latest}" +api_pid="" +state_dir="" +api_log="" + +fail() { + echo "isolated repository eval: $*" >&2 + exit 1 +} + +cleanup() { + local status=$? + if [[ -n "$api_pid" ]]; then + kill "$api_pid" 2>/dev/null || true + wait "$api_pid" 2>/dev/null || true + fi + if (( status != 0 )) && [[ -f "$api_log" ]]; then + tail -80 "$api_log" >&2 || true + fi + if [[ "$state_dir" == "$root/.loop-isolated-eval."* ]]; then + rm -rf "$state_dir" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +has_flag() { + local wanted="$1" + shift + local argument + for argument in "$@"; do + [[ "$argument" == "$wanted" || "$argument" == "$wanted="* ]] && return 0 + done + return 1 +} + +wait_for_api() { + local attempts=0 + until "$python" - "$1" <<'PY' >/dev/null 2>&1 +import sys +import urllib.request + +with urllib.request.urlopen(sys.argv[1], timeout=1) as response: + raise SystemExit(0 if response.status < 500 else 1) +PY + do + attempts=$((attempts + 1)) + if ! kill -0 "$api_pid" 2>/dev/null; then + fail "API exited before becoming ready" + fi + (( attempts < 120 )) || fail "timed out waiting for the disposable API" + sleep 0.25 + done +} + +[[ -x "$python" ]] || fail "Python environment missing; run 'make setup' first" +has_flag --allow-model-spend "$@" || fail "pass --allow-model-spend to acknowledge provider cost" +has_flag --output "$@" || fail "pass --output so paid evaluation evidence is checkpointed" +docker info >/dev/null 2>&1 || fail "Docker is unavailable" + +if ! docker image inspect "$sandbox_image" >/dev/null 2>&1; then + docker build -f "$root/apps/api/sandbox.Dockerfile" -t "$sandbox_image" "$root" +fi + +providers="$({ + cd "$root/apps/api" + DEMO_MODE=false "$python" - <<'PY' +from app.core.config import settings +from app.core.llm.registry import configured_providers + +print(",".join(configured_providers(settings.llm_default_provider))) +PY +})" +[[ -n "$providers" ]] || fail "no real or local model provider is configured in the environment" + +state_dir="$(mktemp -d "$root/.loop-isolated-eval.XXXXXX")" +api_log="$state_dir/api.log" +mkdir -p "$state_dir/projects" "$state_dir/workspaces" "$state_dir/memory" +api_port="${LOOP_EVAL_API_PORT:-$("$python" - <<'PY' +import socket + +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +)}" +token="$("$python" -c 'import secrets; print(secrets.token_urlsafe(32))')" + +( + cd "$root/apps/api" + API_TOKEN="$token" \ + DATABASE_URL="sqlite+aiosqlite:///$state_dir/loop.db" \ + CACHE_BACKEND=memory \ + EXECUTION_MODE=inline \ + SCHEDULER_ENABLED=false \ + PROMETHEUS_ENABLED=false \ + AGENT_SANDBOX=required \ + AGENT_SANDBOX_BACKEND=docker \ + AGENT_SANDBOX_IMAGE="$sandbox_image" \ + LOOP_LOCAL_PROJECTS_ROOT="$state_dir/projects" \ + AGENT_WORKSPACES_ROOT="$state_dir/workspaces" \ + AGENT_MEMORY_ROOT="$state_dir/memory" \ + DEMO_MODE=false \ + "$python" -m uvicorn app.main:app --host 127.0.0.1 --port "$api_port" +) >"$api_log" 2>&1 & +api_pid=$! + +wait_for_api "http://127.0.0.1:$api_port/healthz" +echo "isolated repository eval: providers=$providers sandbox=container" + +cd "$root" +DEMO_MODE=false PYTHONPATH="$root/apps/api" \ + LOOP_API_TOKEN="$token" \ + "$python" "$root/apps/api/scripts/evaluate_repository_matrix.py" \ + "$@" \ + --base-url "http://127.0.0.1:$api_port" \ + --project-root "$state_dir/projects" \ + --modes full_loop \ + --require-isolation container