From 0c7165debfad53b7eaa4c41acd07522a591cc6e4 Mon Sep 17 00:00:00 2001 From: jon-devlapaz Date: Sun, 16 Aug 2026 18:50:41 -0500 Subject: [PATCH 1/6] feat: prepare blinded promotion reviews --- .../scripts/skill_eval_loop.py | 154 ++++++++++++++++++ tests/test_skill_eval_loop.py | 42 ++++- 2 files changed, 195 insertions(+), 1 deletion(-) diff --git a/skills/skill-eval-loop/scripts/skill_eval_loop.py b/skills/skill-eval-loop/scripts/skill_eval_loop.py index 2e4ece0..808667d 100644 --- a/skills/skill-eval-loop/scripts/skill_eval_loop.py +++ b/skills/skill-eval-loop/scripts/skill_eval_loop.py @@ -478,6 +478,28 @@ def write_json(path: Path, value: dict[str, Any]) -> None: path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") +def load_json_object(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"{label}: invalid JSON: {exc.msg}") from exc + if not isinstance(value, dict): + raise ValueError(f"{label}: must be an object") + return value + + +def retained_file(root: Path, relative: Any, label: str) -> Path: + value = relative_workspace_path(relative, label) + path = (root / value).resolve() + try: + path.relative_to(root.resolve()) + except ValueError as exc: + raise ValueError(f"{label}: must stay inside the retained run") from exc + if not path.is_file(): + raise ValueError(f"{label}: file does not exist") + return path + + def safe_task_id(task_id: str) -> None: if not task_id or task_id in {".", ".."} or not task_id[0].isalnum(): raise ValueError(f'task "{task_id}" field id: must be path-safe') @@ -1807,6 +1829,132 @@ def calibrate(arguments: argparse.Namespace) -> int: return calibration_exit_code(result) +def prepare_review(arguments: argparse.Namespace) -> int: + run_dir = absolute_path(arguments.run_dir, "run-dir") + output = absolute_path(arguments.output, "output") + run_path = run_dir / "run.json" + run = load_json_object(run_path, "run") + configuration = run.get("configuration") + if not isinstance(configuration, dict) or configuration.get("evaluation_role") != "promotion": + raise ValueError("run: must be a promotion run") + if not run.get("valid") or run.get("quality_status") == "unknown": + raise ValueError("run: promotion evidence must be runner-valid and quality-complete") + if configuration.get("trials", 0) < 3: + raise ValueError("run: promotion evidence must contain at least 3 trials") + tasks_path = retained_file(run_dir, "tasks.jsonl", "run tasks") + if hash_file(tasks_path) != configuration.get("tasks_sha256"): + raise ValueError("run: retained tasks hash does not match the promotion plan") + if output.exists(): + raise ValueError(f"output directory already exists: {output}") + + items: list[dict[str, Any]] = [] + copied: list[tuple[Path, Path]] = [] + for pair in run.get("pairs", []): + if not isinstance(pair, dict): + raise ValueError("run field pairs: must contain objects") + task_id = required_string(pair.get("task_id"), "run pair field task_id") + safe_task_id(task_id) + trial = pair.get("trial") + if not isinstance(trial, int) or trial < 1: + raise ValueError("run pair field trial: must be a positive integer") + report_path = retained_file(run_dir, pair.get("report_json"), "run pair report_json") + report = load_json_object(report_path, "pair report") + if not report.get("runner_valid"): + raise ValueError(f"run pair {task_id} trial {trial}: runner is invalid") + pairwise = report.get("pairwise") + if not isinstance(pairwise, list) or not pairwise: + raise ValueError(f"run pair {task_id} trial {trial}: missing pairwise evidence") + for index, judgment in enumerate(pairwise, start=1): + if not isinstance(judgment, dict) or judgment.get("status") == "unknown": + raise ValueError( + f"run pair {task_id} trial {trial} rubric {index}: judgment is incomplete" + ) + artifacts = judgment.get("artifacts") + if not isinstance(artifacts, dict): + raise ValueError( + f"run pair {task_id} trial {trial} rubric {index}: missing artifacts" + ) + prompt_path = retained_file(report_path.parent, artifacts.get("prompt"), "judge prompt") + dimensions = judgment.get("dimensions") + if not isinstance(dimensions, list) or not dimensions: + raise ValueError( + f"run pair {task_id} trial {trial} rubric {index}: missing dimensions" + ) + dimension_names = [ + required_string(dimension.get("name"), "judge dimension field name") + for dimension in dimensions + if isinstance(dimension, dict) + ] + if len(dimension_names) != len(dimensions): + raise ValueError("judge dimensions: must contain objects") + item_id = f"{task_id}.trial-{trial:03d}.rubric-{index:03d}" + destination = Path("items") / f"{item_id}.txt" + copied.append((prompt_path, destination)) + items.append( + { + "id": item_id, + "task_id": task_id, + "trial": trial, + "rubric_index": index, + "prompt": destination.as_posix(), + "prompt_sha256": hash_file(prompt_path), + "dimensions": dimension_names, + "source_report": str( + report_path.resolve().relative_to(run_dir.resolve()).as_posix() + ), + } + ) + if not items: + raise ValueError("run: no pairwise evidence is available for human review") + + output.mkdir(parents=True) + for source, relative in copied: + destination = output / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + manifest = { + "version": 1, + "run_sha256": hash_file(run_path), + "tasks_sha256": configuration["tasks_sha256"], + "required_reviewers": 2, + "items": items, + } + manifest_path = output / "manifest.json" + write_json(manifest_path, manifest) + write_json( + output / "labels-template.json", + { + "version": 1, + "manifest_sha256": hash_file(manifest_path), + "reviewer_id": "", + "labels": [ + { + "item_id": item["id"], + "prompt_sha256": item["prompt_sha256"], + "winner": "", + "rationale": "", + "transcript_reviewed": False, + "dimensions": [ + {"name": name, "winner": "", "rationale": ""} + for name in item["dimensions"] + ], + } + for item in items + ], + }, + ) + print_json( + { + "valid": True, + "mode": "prepare_review", + "output_dir": str(output), + "items": len(items), + "required_reviewers": 2, + } + ) + return 0 + + def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser(prog="skill-eval-loop") commands = result.add_subparsers(dest="command", required=True) @@ -1843,6 +1991,12 @@ def parser() -> argparse.ArgumentParser: calibrate_parser.add_argument("--timeout-seconds", type=int, default=120) calibrate_parser.add_argument("--dry-run", action="store_true") calibrate_parser.set_defaults(handler=calibrate) + review_parser = commands.add_parser( + "prepare-review", help="create a blinded human-review packet from a promotion run" + ) + review_parser.add_argument("--run-dir", required=True) + review_parser.add_argument("--output", required=True) + review_parser.set_defaults(handler=prepare_review) return result diff --git a/tests/test_skill_eval_loop.py b/tests/test_skill_eval_loop.py index e875002..7b102ce 100644 --- a/tests/test_skill_eval_loop.py +++ b/tests/test_skill_eval_loop.py @@ -50,6 +50,8 @@ def run_live_rubric( control_response: str = "Blue", calibration: Path | str | None = None, use_calibration: bool = True, + trials: int = 1, + promotion: bool = False, ) -> tuple[subprocess.CompletedProcess[str], Path, Path]: skill = self.make_skill(root) tasks = root / "tasks.jsonl" @@ -114,7 +116,11 @@ def run_live_rubric( judge_model, "--timeout-seconds", "1", - ] + (["--calibration", str(calibration)] if calibration is not None else []), + "--trials", + str(trials), + ] + + (["--calibration", str(calibration)] if calibration is not None else []) + + (["--promotion"] if promotion else []), cwd=ROOT, text=True, capture_output=True, @@ -475,6 +481,40 @@ def test_promotion_plan_records_role_and_requires_rubric_calibration(self) -> No self.assertEqual(uncalibrated.returncode, 1) self.assertIn("require accepted calibration", uncalibrated.stderr) + def test_prepare_review_creates_a_blinded_packet_bound_to_a_promotion_run(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + result, run_dir, _ = self.run_live_rubric(root, trials=3, promotion=True) + self.assertEqual(result.returncode, 0, result.stderr) + packet = root / "review-packet" + + prepared = self.run_cli( + "prepare-review", + "--run-dir", + str(run_dir), + "--output", + str(packet), + ) + + self.assertEqual(prepared.returncode, 0, prepared.stderr) + manifest = json.loads((packet / "manifest.json").read_text(encoding="utf-8")) + template = json.loads((packet / "labels-template.json").read_text(encoding="utf-8")) + self.assertEqual(manifest["version"], 1) + self.assertEqual(manifest["required_reviewers"], 2) + self.assertEqual(len(manifest["items"]), 3) + self.assertEqual(template["manifest_sha256"], self.hash_file(packet / "manifest.json")) + for item in manifest["items"]: + prompt = (packet / item["prompt"]).read_text(encoding="utf-8") + self.assertNotIn("control", prompt.casefold()) + self.assertNotIn("treatment", prompt.casefold()) + self.assertEqual(item["prompt_sha256"], self.hash_file(packet / item["prompt"])) + + @staticmethod + def hash_file(path: Path) -> str: + import hashlib + + return hashlib.sha256(path.read_bytes()).hexdigest() + def test_dry_run_requires_explicit_or_target_owned_tasks_before_harness_resolution(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) From a38752103125f6ca5e117c479b1f1d7df459bd5e Mon Sep 17 00:00:00 2001 From: jon-devlapaz Date: Sun, 16 Aug 2026 18:55:33 -0500 Subject: [PATCH 2/6] feat: finalize human-grounded promotion evidence --- .../scripts/skill_eval_loop.py | 373 +++++++++++++++++- tests/test_skill_eval_loop.py | 155 +++++++- 2 files changed, 523 insertions(+), 5 deletions(-) diff --git a/skills/skill-eval-loop/scripts/skill_eval_loop.py b/skills/skill-eval-loop/scripts/skill_eval_loop.py index 808667d..1d4339b 100644 --- a/skills/skill-eval-loop/scripts/skill_eval_loop.py +++ b/skills/skill-eval-loop/scripts/skill_eval_loop.py @@ -6,6 +6,7 @@ import argparse import hashlib import json +import math import os from pathlib import Path, PurePath import random @@ -1802,7 +1803,13 @@ def healthcheck(arguments: argparse.Namespace) -> int: { "valid": not missing, "skill_dir": str(root), - "commands": ["healthcheck", "run", "calibrate"], + "commands": [ + "healthcheck", + "run", + "calibrate", + "prepare-review", + "finalize-review", + ], "errors": [f"{relative} is missing" for relative in missing], } ) @@ -1829,9 +1836,7 @@ def calibrate(arguments: argparse.Namespace) -> int: return calibration_exit_code(result) -def prepare_review(arguments: argparse.Namespace) -> int: - run_dir = absolute_path(arguments.run_dir, "run-dir") - output = absolute_path(arguments.output, "output") +def load_reviewable_run(run_dir: Path) -> tuple[dict[str, Any], dict[str, Any], Path]: run_path = run_dir / "run.json" run = load_json_object(run_path, "run") configuration = run.get("configuration") @@ -1844,6 +1849,13 @@ def prepare_review(arguments: argparse.Namespace) -> int: tasks_path = retained_file(run_dir, "tasks.jsonl", "run tasks") if hash_file(tasks_path) != configuration.get("tasks_sha256"): raise ValueError("run: retained tasks hash does not match the promotion plan") + return run, configuration, run_path + + +def prepare_review(arguments: argparse.Namespace) -> int: + run_dir = absolute_path(arguments.run_dir, "run-dir") + output = absolute_path(arguments.output, "output") + run, configuration, run_path = load_reviewable_run(run_dir) if output.exists(): raise ValueError(f"output directory already exists: {output}") @@ -1943,6 +1955,24 @@ def prepare_review(arguments: argparse.Namespace) -> int: ], }, ) + write_json( + output / "holdout-attestation-template.json", + { + "version": 1, + "tasks_sha256": configuration["tasks_sha256"], + "custodian_id": "", + "independent_of_skill_authoring": False, + "unseen_during_development": False, + "coverage": { + "positive": False, + "negative": False, + "ambiguous": False, + "near_tie": False, + "adversarial": False, + }, + "rationale": "", + }, + ) print_json( { "valid": True, @@ -1955,6 +1985,330 @@ def prepare_review(arguments: argparse.Namespace) -> int: return 0 +def load_reviewer_labels( + path: Path, manifest_hash: str, items: dict[str, dict[str, Any]] +) -> tuple[str, dict[str, dict[str, Any]]]: + document = load_json_object(path, "labels") + if document.get("version") != 1: + raise ValueError("labels field version: must be 1") + if document.get("manifest_sha256") != manifest_hash: + raise ValueError("labels: manifest hash does not match the review packet") + reviewer_id = required_string(document.get("reviewer_id"), "labels field reviewer_id") + raw_labels = document.get("labels") + if not isinstance(raw_labels, list) or len(raw_labels) != len(items): + raise ValueError("labels field labels: must cover every review item exactly once") + labels: dict[str, dict[str, Any]] = {} + for index, raw in enumerate(raw_labels): + label = f"labels field labels[{index}]" + if not isinstance(raw, dict): + raise ValueError(f"{label}: must be an object") + item_id = required_string(raw.get("item_id"), f"{label} field item_id") + item = items.get(item_id) + if item is None or item_id in labels: + raise ValueError(f"{label} field item_id: unknown or duplicate value") + if raw.get("prompt_sha256") != item["prompt_sha256"]: + raise ValueError(f"{label}: prompt hash does not match the review packet") + winner = raw.get("winner") + if winner not in {"A", "B", "tie"}: + raise ValueError(f"{label} field winner: must be A, B, or tie") + rationale = required_string(raw.get("rationale"), f"{label} field rationale") + if raw.get("transcript_reviewed") is not True: + raise ValueError(f"{label} field transcript_reviewed: must be true") + raw_dimensions = raw.get("dimensions") + if not isinstance(raw_dimensions, list) or len(raw_dimensions) != len(item["dimensions"]): + raise ValueError(f"{label} field dimensions: must cover every dimension exactly once") + dimensions: dict[str, dict[str, str]] = {} + for dimension_index, raw_dimension in enumerate(raw_dimensions): + dimension_label = f"{label} field dimensions[{dimension_index}]" + if not isinstance(raw_dimension, dict): + raise ValueError(f"{dimension_label}: must be an object") + name = required_string(raw_dimension.get("name"), f"{dimension_label} field name") + if name not in item["dimensions"] or name in dimensions: + raise ValueError(f"{dimension_label} field name: unknown or duplicate value") + dimension_winner = raw_dimension.get("winner") + if dimension_winner not in {"A", "B", "tie"}: + raise ValueError(f"{dimension_label} field winner: must be A, B, or tie") + dimensions[name] = { + "winner": dimension_winner, + "rationale": required_string( + raw_dimension.get("rationale"), f"{dimension_label} field rationale" + ), + } + labels[item_id] = { + "winner": winner, + "rationale": rationale, + "transcript_reviewed": True, + "dimensions": dimensions, + } + return reviewer_id, labels + + +def count_agreement(left: str, right: str) -> int: + return int(left == right) + + +def finalize_review(arguments: argparse.Namespace) -> int: + run_dir = absolute_path(arguments.run_dir, "run-dir") + manifest_path = absolute_path(arguments.manifest, "manifest") + output = absolute_path(arguments.output, "output") + if output.exists(): + raise ValueError(f"output directory already exists: {output}") + if not math.isfinite(arguments.cost_usd) or arguments.cost_usd < 0: + raise ValueError("cost-usd must be a finite non-negative number") + cost_note = required_string(arguments.cost_note, "cost-note") + run, configuration, run_path = load_reviewable_run(run_dir) + manifest = load_json_object(manifest_path, "manifest") + if manifest.get("version") != 1 or manifest.get("required_reviewers") != 2: + raise ValueError("manifest: unsupported review packet") + if manifest.get("run_sha256") != hash_file(run_path): + raise ValueError("manifest: run hash does not match the retained promotion run") + if manifest.get("tasks_sha256") != configuration.get("tasks_sha256"): + raise ValueError("manifest: tasks hash does not match the retained promotion run") + attestation = load_json_object( + absolute_path(arguments.holdout_attestation, "holdout-attestation"), + "holdout attestation", + ) + if attestation.get("version") != 1: + raise ValueError("holdout attestation field version: must be 1") + if attestation.get("tasks_sha256") != configuration.get("tasks_sha256"): + raise ValueError("holdout attestation: tasks hash does not match the promotion run") + custodian_id = required_string( + attestation.get("custodian_id"), "holdout attestation field custodian_id" + ) + if attestation.get("independent_of_skill_authoring") is not True: + raise ValueError("holdout attestation field independent_of_skill_authoring: must be true") + if attestation.get("unseen_during_development") is not True: + raise ValueError("holdout attestation field unseen_during_development: must be true") + coverage = attestation.get("coverage") + required_coverage = {"positive", "negative", "ambiguous", "near_tie", "adversarial"} + if not isinstance(coverage, dict) or set(coverage) != required_coverage or not all( + coverage.values() + ): + raise ValueError( + "holdout attestation field coverage: every required category must be true" + ) + holdout_rationale = required_string( + attestation.get("rationale"), "holdout attestation field rationale" + ) + raw_items = manifest.get("items") + if not isinstance(raw_items, list) or not raw_items: + raise ValueError("manifest field items: must be a non-empty array") + items: dict[str, dict[str, Any]] = {} + for index, item in enumerate(raw_items): + label = f"manifest field items[{index}]" + if not isinstance(item, dict): + raise ValueError(f"{label}: must be an object") + item_id = required_string(item.get("id"), f"{label} field id") + if item_id in items: + raise ValueError(f"{label} field id: duplicate value") + prompt = retained_file(manifest_path.parent, item.get("prompt"), f"{label} field prompt") + if hash_file(prompt) != item.get("prompt_sha256"): + raise ValueError(f"{label}: prompt hash does not match the manifest") + dimensions = item.get("dimensions") + if not isinstance(dimensions, list) or not dimensions: + raise ValueError(f"{label} field dimensions: must be a non-empty array") + items[item_id] = item + if len(arguments.labels) != 2: + raise ValueError("finalize-review requires exactly two independent label files") + manifest_hash = hash_file(manifest_path) + reviews = [ + load_reviewer_labels(absolute_path(value, "labels"), manifest_hash, items) + for value in arguments.labels + ] + reviewers = [review[0] for review in reviews] + if len(set(reviewers)) != 2: + raise ValueError("labels: reviewer_id values must be distinct") + + overall_agreements = 0 + dimension_agreements = 0 + dimension_total = 0 + judge_consensus_agreements = 0 + judge_consensus_total = 0 + judge_dimension_consensus_agreements = 0 + judge_dimension_consensus_total = 0 + judge_by_reviewer = { + reviewer: { + "overall": {"agreements": 0, "total": 0}, + "dimensions": {"agreements": 0, "total": 0}, + } + for reviewer in reviewers + } + disagreements: list[dict[str, Any]] = [] + outcomes = {"control": 0, "treatment": 0, "tie": 0} + by_task: dict[str, dict[str, int]] = {} + regressions: list[str] = [] + improvements: list[str] = [] + for item_id, item in items.items(): + left = reviews[0][1][item_id] + right = reviews[1][1][item_id] + agreed = left["winner"] == right["winner"] + overall_agreements += int(agreed) + dimension_disagreements: list[str] = [] + for name in item["dimensions"]: + dimension_total += 1 + dimension_agreements += count_agreement( + left["dimensions"][name]["winner"], right["dimensions"][name]["winner"] + ) + if left["dimensions"][name]["winner"] != right["dimensions"][name]["winner"]: + dimension_disagreements.append(name) + report_path = retained_file(run_dir, item.get("source_report"), "manifest source_report") + report = load_json_object(report_path, "pair report") + pairwise = report.get("pairwise") + rubric_index = item.get("rubric_index") + if not isinstance(pairwise, list) or not isinstance(rubric_index, int) or not ( + 1 <= rubric_index <= len(pairwise) + ): + raise ValueError(f"manifest item {item_id}: pairwise source is invalid") + automated = pairwise[rubric_index - 1] + automated_winner = automated.get("winner_label") + mapping = automated.get("mapping") + if automated_winner not in {"A", "B", "tie"} or not isinstance(mapping, dict): + raise ValueError(f"manifest item {item_id}: automated judgment is incomplete") + automated_dimensions = { + dimension.get("name"): dimension.get("winner") + for dimension in automated.get("dimensions", []) + if isinstance(dimension, dict) + } + if set(automated_dimensions) != set(item["dimensions"]): + raise ValueError(f"manifest item {item_id}: automated dimensions are incomplete") + for reviewer, human in zip(reviewers, (left, right)): + judge_by_reviewer[reviewer]["overall"]["total"] += 1 + judge_by_reviewer[reviewer]["overall"]["agreements"] += int( + automated_winner == human["winner"] + ) + for name in item["dimensions"]: + judge_by_reviewer[reviewer]["dimensions"]["total"] += 1 + judge_by_reviewer[reviewer]["dimensions"]["agreements"] += int( + automated_dimensions[name] == human["dimensions"][name]["winner"] + ) + if agreed: + judge_consensus_total += 1 + judge_consensus_agreements += int(automated_winner == left["winner"]) + outcome = "tie" if left["winner"] == "tie" else mapping.get(left["winner"]) + if outcome not in outcomes: + raise ValueError(f"manifest item {item_id}: candidate mapping is invalid") + outcomes[outcome] += 1 + task_outcomes = by_task.setdefault( + required_string(item.get("task_id"), "manifest item field task_id"), + {"control": 0, "treatment": 0, "tie": 0}, + ) + task_outcomes[outcome] += 1 + if outcome == "control": + regressions.append(item_id) + elif outcome == "treatment": + improvements.append(item_id) + for name in item["dimensions"]: + left_winner = left["dimensions"][name]["winner"] + right_winner = right["dimensions"][name]["winner"] + if left_winner == right_winner: + judge_dimension_consensus_total += 1 + judge_dimension_consensus_agreements += int( + automated_dimensions[name] == left_winner + ) + if not agreed or dimension_disagreements: + disagreements.append( + { + "item_id": item_id, + "overall": { + reviewers[0]: {"winner": left["winner"], "rationale": left["rationale"]}, + reviewers[1]: { + "winner": right["winner"], + "rationale": right["rationale"], + }, + }, + "dimensions": dimension_disagreements, + } + ) + + report = { + "version": 1, + "evidence_status": "complete_human_review", + "promotion_decision": "human_owner_required", + "run_binding": { + "run_sha256": hash_file(run_path), + "tasks_sha256": configuration["tasks_sha256"], + "skill_sha256": configuration["skill_sha256"], + "calibration_sha256": configuration.get("calibration_sha256"), + "fixtures_sha256": configuration.get("fixtures_sha256"), + }, + "reviewers": reviewers, + "holdout": { + "custodian_id": custodian_id, + "independent_of_skill_authoring": True, + "unseen_during_development": True, + "coverage": coverage, + "rationale": holdout_rationale, + }, + "human_agreement": { + "overall": {"agreements": overall_agreements, "total": len(items)}, + "dimensions": {"agreements": dimension_agreements, "total": dimension_total}, + }, + "disagreements": disagreements, + "automated_judge_agreement": { + "status": "provisional_non_independent", + "with_human_consensus": { + "agreements": judge_consensus_agreements, + "total": judge_consensus_total, + }, + "dimensions_with_human_consensus": { + "agreements": judge_dimension_consensus_agreements, + "total": judge_dimension_consensus_total, + }, + "by_reviewer": judge_by_reviewer, + }, + "transcript_review": { + "reviewed_labels": len(items) * 2, + "required_labels": len(items) * 2, + }, + "outcomes": outcomes, + "outcomes_by_task": by_task, + "improvements": improvements, + "regressions": regressions, + "usage": run.get("usage"), + "cost": {"usd": arguments.cost_usd, "note": cost_note}, + "limitations": [ + "Human identities and holdout custody are operator attestations, not machine-proven.", + "Automated judging remains same-provider and non-independent.", + "This evidence package informs but does not make the promotion decision.", + ], + } + output.mkdir(parents=True) + write_json(output / "promotion-review.json", report) + (output / "promotion-review.md").write_text( + "\n".join( + [ + "# Promotion review", + "", + "Evidence status: complete human review", + "Promotion decision: human owner required", + f"Reviewers: {', '.join(reviewers)}", + f"Human overall agreement: {overall_agreements}/{len(items)}", + f"Human dimension agreement: {dimension_agreements}/{dimension_total}", + ( + "Automated judge agreement with human consensus: " + f"{judge_consensus_agreements}/{judge_consensus_total}" + ), + f"Outcomes: {json.dumps(outcomes, sort_keys=True)}", + f"Regressions: {len(regressions)}", + f"Recorded cost (USD): {arguments.cost_usd:.2f}", + "", + "The accountable human owner must inspect disagreements and decide promotion.", + "", + ] + ), + encoding="utf-8", + ) + print_json( + { + "valid": True, + "mode": "finalize_review", + "output_dir": str(output), + "evidence_status": report["evidence_status"], + } + ) + return 0 + + def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser(prog="skill-eval-loop") commands = result.add_subparsers(dest="command", required=True) @@ -1997,6 +2351,17 @@ def parser() -> argparse.ArgumentParser: review_parser.add_argument("--run-dir", required=True) review_parser.add_argument("--output", required=True) review_parser.set_defaults(handler=prepare_review) + finalize_parser = commands.add_parser( + "finalize-review", help="measure two human reviews against retained promotion evidence" + ) + finalize_parser.add_argument("--run-dir", required=True) + finalize_parser.add_argument("--manifest", required=True) + finalize_parser.add_argument("--holdout-attestation", required=True) + finalize_parser.add_argument("--labels", action="append", required=True) + finalize_parser.add_argument("--cost-usd", type=float, required=True) + finalize_parser.add_argument("--cost-note", required=True) + finalize_parser.add_argument("--output", required=True) + finalize_parser.set_defaults(handler=finalize_review) return result diff --git a/tests/test_skill_eval_loop.py b/tests/test_skill_eval_loop.py index 7b102ce..8c9d342 100644 --- a/tests/test_skill_eval_loop.py +++ b/tests/test_skill_eval_loop.py @@ -133,7 +133,10 @@ def test_healthcheck_reports_python_commands(self) -> None: result = self.run_cli("healthcheck", "--skill-dir", str(EVALUATOR.parents[1])) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout)["commands"], ["healthcheck", "run", "calibrate"]) + self.assertEqual( + json.loads(result.stdout)["commands"], + ["healthcheck", "run", "calibrate", "prepare-review", "finalize-review"], + ) def test_public_launcher_needs_only_python3(self) -> None: result = subprocess.run( @@ -509,6 +512,156 @@ def test_prepare_review_creates_a_blinded_packet_bound_to_a_promotion_run(self) self.assertNotIn("treatment", prompt.casefold()) self.assertEqual(item["prompt_sha256"], self.hash_file(packet / item["prompt"])) + def test_finalize_review_measures_human_and_automated_agreement(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + result, run_dir, _ = self.run_live_rubric(root, trials=3, promotion=True) + self.assertEqual(result.returncode, 0, result.stderr) + packet = root / "review-packet" + prepared = self.run_cli( + "prepare-review", "--run-dir", str(run_dir), "--output", str(packet) + ) + self.assertEqual(prepared.returncode, 0, prepared.stderr) + template = json.loads((packet / "labels-template.json").read_text(encoding="utf-8")) + attestation = json.loads( + (packet / "holdout-attestation-template.json").read_text(encoding="utf-8") + ) + attestation.update( + { + "custodian_id": "client-custodian", + "independent_of_skill_authoring": True, + "unseen_during_development": True, + "coverage": {name: True for name in attestation["coverage"]}, + "rationale": "The client controlled and categorized the held-out cases.", + } + ) + attestation_path = root / "holdout-attestation.json" + attestation_path.write_text(json.dumps(attestation), encoding="utf-8") + label_paths: list[Path] = [] + for reviewer_id in ("reviewer-a", "reviewer-b"): + labels = json.loads(json.dumps(template)) + labels["reviewer_id"] = reviewer_id + for item in labels["labels"]: + item["winner"] = "A" + item["rationale"] = f"{reviewer_id} prefers candidate A." + item["transcript_reviewed"] = True + for dimension in item["dimensions"]: + dimension["winner"] = "A" + dimension["rationale"] = f"A is stronger on {dimension['name']}." + path = root / f"{reviewer_id}.json" + path.write_text(json.dumps(labels), encoding="utf-8") + label_paths.append(path) + output = root / "promotion-review" + + finalized = self.run_cli( + "finalize-review", + "--run-dir", + str(run_dir), + "--manifest", + str(packet / "manifest.json"), + "--holdout-attestation", + str(attestation_path), + "--labels", + str(label_paths[0]), + "--labels", + str(label_paths[1]), + "--cost-usd", + "1.25", + "--cost-note", + "Recorded test cost.", + "--output", + str(output), + ) + + self.assertEqual(finalized.returncode, 0, finalized.stderr) + review = json.loads((output / "promotion-review.json").read_text(encoding="utf-8")) + self.assertEqual(review["evidence_status"], "complete_human_review") + self.assertEqual(review["reviewers"], ["reviewer-a", "reviewer-b"]) + self.assertEqual(review["human_agreement"]["overall"], {"agreements": 3, "total": 3}) + self.assertEqual( + review["automated_judge_agreement"]["with_human_consensus"], + {"agreements": 3, "total": 3}, + ) + self.assertEqual( + review["automated_judge_agreement"]["by_reviewer"]["reviewer-a"]["overall"], + {"agreements": 3, "total": 3}, + ) + self.assertEqual(review["transcript_review"]["reviewed_labels"], 6) + self.assertEqual(review["cost"], {"usd": 1.25, "note": "Recorded test cost."}) + self.assertEqual(sum(review["outcomes"].values()), 3) + self.assertTrue((output / "promotion-review.md").is_file()) + + def test_finalize_review_rejects_incomplete_or_non_independent_labels(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + result, run_dir, _ = self.run_live_rubric(root, trials=3, promotion=True) + self.assertEqual(result.returncode, 0, result.stderr) + packet = root / "review-packet" + prepared = self.run_cli( + "prepare-review", "--run-dir", str(run_dir), "--output", str(packet) + ) + self.assertEqual(prepared.returncode, 0, prepared.stderr) + template = json.loads((packet / "labels-template.json").read_text(encoding="utf-8")) + attestation = json.loads( + (packet / "holdout-attestation-template.json").read_text(encoding="utf-8") + ) + attestation.update( + { + "custodian_id": "client-custodian", + "independent_of_skill_authoring": True, + "unseen_during_development": True, + "coverage": {name: True for name in attestation["coverage"]}, + "rationale": "The client controlled and categorized the held-out cases.", + } + ) + attestation_path = root / "holdout-attestation.json" + attestation_path.write_text(json.dumps(attestation), encoding="utf-8") + label_paths = [root / "labels-a.json", root / "labels-b.json"] + documents: list[dict[str, object]] = [] + for path in label_paths: + labels = json.loads(json.dumps(template)) + labels["reviewer_id"] = "same-reviewer" + for item in labels["labels"]: + item["winner"] = "A" + item["rationale"] = "Candidate A is stronger." + item["transcript_reviewed"] = True + for dimension in item["dimensions"]: + dimension["winner"] = "A" + dimension["rationale"] = "Candidate A is stronger." + documents.append(labels) + path.write_text(json.dumps(labels), encoding="utf-8") + documents[0]["labels"][0]["transcript_reviewed"] = False + label_paths[0].write_text(json.dumps(documents[0]), encoding="utf-8") + + arguments = ( + "finalize-review", + "--run-dir", + str(run_dir), + "--manifest", + str(packet / "manifest.json"), + "--holdout-attestation", + str(attestation_path), + "--labels", + str(label_paths[0]), + "--labels", + str(label_paths[1]), + "--cost-usd", + "0", + "--cost-note", + "Included in the test harness.", + "--output", + str(root / "review"), + ) + incomplete = self.run_cli(*arguments) + self.assertEqual(incomplete.returncode, 1) + self.assertIn("transcript_reviewed: must be true", incomplete.stderr) + + documents[0]["labels"][0]["transcript_reviewed"] = True + label_paths[0].write_text(json.dumps(documents[0]), encoding="utf-8") + duplicated = self.run_cli(*arguments) + self.assertEqual(duplicated.returncode, 1) + self.assertIn("reviewer_id values must be distinct", duplicated.stderr) + @staticmethod def hash_file(path: Path) -> str: import hashlib From a355612161195649ffa1105bb14c57eb5c2d9ae9 Mon Sep 17 00:00:00 2001 From: jon-devlapaz Date: Sun, 16 Aug 2026 18:58:13 -0500 Subject: [PATCH 3/6] docs: define the promotion evidence handoff --- README.md | 19 +++- docs/minimum-eval-contract.md | 17 ++- skills/skill-eval-loop/SKILL.md | 14 ++- .../references/promotion-workflow.md | 102 ++++++++++++++++++ .../scripts/skill_eval_loop.py | 7 +- tasks/plan.md | 30 ++++-- tasks/todo.md | 8 +- 7 files changed, 177 insertions(+), 20 deletions(-) create mode 100644 skills/skill-eval-loop/references/promotion-workflow.md diff --git a/README.md b/README.md index 352e5de..90418e8 100644 --- a/README.md +++ b/README.md @@ -121,12 +121,25 @@ python3 skills/skill-eval-loop/scripts/skill_eval_loop.py calibrate \ `calibrate` exits `0` when agreements meet the locked threshold, `1` when the runner is valid but the judge disagrees, and `2` when a judgment is invalid. +## Complete a promotion review + +A business-ready promotion result requires more than `run --promotion`. Use the +[promotion evidence workflow](skills/skill-eval-loop/references/promotion-workflow.md) +to create a blinded packet, collect two independent human label files and the +custodian's holdout +attestation, then run `finalize-review`. The final report measures human and +automated-judge agreement, outcomes across trials, regressions, usage, and +recorded cost while leaving the promotion decision with the accountable human +owner. + ## Boundaries The minimum runner supports Codex, deterministic graders, a provisional -same-provider rubric judge, blinded pairwise comparison, and human-labeled -calibration fixtures. It does not provide independent judging, pricing, -parallel execution, provider discovery, or adapters for other harnesses. +same-provider rubric judge, blinded pairwise comparison, human-labeled +calibration fixtures, and hash-bound two-reviewer promotion evidence. It records +operator-supplied cost; it does not discover pricing, provide an independent +automated judge, authenticate human identities, run in parallel, discover +providers, or adapt other harnesses. Live evaluation is a trusted local-operator workflow. The configured harness and Codex executable can read the run-local Codex credentials and therefore diff --git a/docs/minimum-eval-contract.md b/docs/minimum-eval-contract.md index a14f8f7..5d93f28 100644 --- a/docs/minimum-eval-contract.md +++ b/docs/minimum-eval-contract.md @@ -321,9 +321,20 @@ human labels for the rubric or preference decisions, and measured agreement between those labels and any automated judge. A visible development suite must not be relabeled as a holdout after it has guided changes. -The remaining real-promotion gate is external to this runner: independently -control the holdout, obtain human labels, repeat trials, and review the retained -transcripts before making a promotion claim. +After a quality-complete promotion run, `prepare-review` creates a hash-bound +packet containing only the blinded A/B prompts and empty templates. The +custodian attests task-hash-bound independence, development secrecy, and +coverage of positive, negative, ambiguous, near-tie, and adversarial cases. Two +distinct reviewers label every transcript and dimension independently with +rationale. `finalize-review` verifies those inputs against the retained run, +measures human and automated-judge agreement, restores condition outcomes, +reports variance by task, regressions, usage, and operator-recorded cost, and +retains limitations. See the packaged +[promotion workflow](../skills/skill-eval-loop/references/promotion-workflow.md). + +The machine cannot authenticate human identity or custody claims. A complete +review package is evidence for the accountable owner; it is not an automatic +promotion verdict. A one-task pilot can establish runner acceptance. It cannot establish that a skill is generally effective. Capability suites should contain enough diff --git a/skills/skill-eval-loop/SKILL.md b/skills/skill-eval-loop/SKILL.md index f06f118..afd2c31 100644 --- a/skills/skill-eval-loop/SKILL.md +++ b/skills/skill-eval-loop/SKILL.md @@ -126,8 +126,11 @@ calibration, and repeated trials: `--promotion` rejects target-owned tasks, fewer than three trials, and rubric runs without accepted calibration. It records the promotion role; it does not prove task independence, representativeness, human labeling, or judge -agreement. Retain that evidence separately and keep the holdout unavailable to -the hill-climbing agent. +agreement. Keep the holdout unavailable to the hill-climbing agent, then follow +[`references/promotion-workflow.md`](references/promotion-workflow.md): run +`prepare-review`, collect the custodian attestation and two independent label +files, and run +`finalize-review` before making a promotion claim. ## Calibrate the pairwise judge @@ -235,3 +238,10 @@ response before making a quality claim. Do not claim broad skill quality from one pilot or from same-provider judging. Use realistic unsaturated tasks, repeated trials, deterministic outcomes, blinded comparison, human calibration, and human transcript review. + +`prepare-review` and `finalize-review` are the final evidence loop. The first +copies only blinded A/B prompts into a hash-bound packet. The second fails closed +unless the packet, retained run, holdout attestation, two distinct reviewers, +every transcript, every dimension, rationales, and recorded cost are complete. +Its `complete_human_review` status means the evidence is ready for an accountable +human decision; it is not an automated promotion verdict. diff --git a/skills/skill-eval-loop/references/promotion-workflow.md b/skills/skill-eval-loop/references/promotion-workflow.md new file mode 100644 index 0000000..3b753fc --- /dev/null +++ b/skills/skill-eval-loop/references/promotion-workflow.md @@ -0,0 +1,102 @@ +# Promotion evidence workflow + +This is the complete business workflow for turning a valid paired run into an +auditable, human-grounded promotion decision package. The evaluator prepares +and verifies evidence; the accountable human owner makes the decision. + +## Roles + +- **Holdout custodian:** controls the task file, keeps it out of skill authoring + and development runs, and attests that it covers positive, negative, + ambiguous, near-tie, and adversarial cases. +- **Evaluator operator:** calibrates the judge, runs the locked promotion + evaluation, prepares the blinded packet, and records actual cost. +- **Two reviewers:** independently inspect every blinded A/B transcript and + label the overall winner plus every rubric dimension with rationale. +- **Decision owner:** reviews disagreements, regressions, variance, usage, cost, + and limitations before accepting or rejecting promotion. + +One person may operate the evaluator and own the decision. The two label files +must still use distinct reviewer identities and be completed independently. + +## 1. Lock and run the holdout + +The custodian supplies an absolute task path outside the target skill. Run the +accepted calibration and inspect the dry-run plan before authorizing live calls. + +```bash +"$EVALUATOR" run \ + --skill /absolute/path/to/target-skill \ + --tasks /absolute/custodian/path/holdout.jsonl \ + --output /absolute/path/to/fresh-promotion-run \ + --harness codex \ + --harness-bin /absolute/path/to/codex \ + --model exact-runner-model \ + --judge-model exact-judge-model \ + --calibration /absolute/path/to/calibration.json \ + --trials 3 \ + --timeout-seconds 300 \ + --promotion \ + --dry-run +``` + +Run the identical command without `--dry-run` only after the hashes, models, +invocation count, and cost authority are accepted. + +## 2. Prepare a blinded review packet + +```bash +"$EVALUATOR" prepare-review \ + --run-dir /absolute/path/to/fresh-promotion-run \ + --output /absolute/path/to/fresh-review-packet +``` + +Give reviewers only the review packet, not the retained run. The packet contains +the exact A/B prompts previously shown to the automated judge, a hash-bound +manifest, a label template, and a holdout-attestation template. It contains no +control/treatment mapping. + +The custodian completes `holdout-attestation-template.json`. Each reviewer makes +a private copy of `labels-template.json`, sets a distinct `reviewer_id`, labels +every overall comparison and dimension with `A`, `B`, or `tie`, provides a +rationale, and sets `transcript_reviewed` to `true` only after reading the item. +Reviewers must not inspect each other's labels before both files are final. + +## 3. Finalize the evidence + +```bash +"$EVALUATOR" finalize-review \ + --run-dir /absolute/path/to/fresh-promotion-run \ + --manifest /absolute/path/to/fresh-review-packet/manifest.json \ + --holdout-attestation /absolute/custodian/path/holdout-attestation.json \ + --labels /absolute/reviewer-a/labels.json \ + --labels /absolute/reviewer-b/labels.json \ + --cost-usd 12.34 \ + --cost-note "Provider invoice or subscription allocation." \ + --output /absolute/path/to/fresh-promotion-review +``` + +Finalization fails if the run is not a quality-complete promotion run, hashes +drift, holdout coverage or custody is not attested, reviewer identities are not +distinct, a transcript or dimension is unlabeled, a rationale is missing, or +the recorded cost is invalid. + +The output reports: + +- human agreement overall and by dimension; +- automated-judge agreement with each reviewer and human consensus; +- disagreements with both rationales; +- unblinded control/treatment/tie outcomes by task and across trials; +- treatment improvements and control-winning regressions; +- usage, recorded cost, and explicit limitations. + +`complete_human_review` means the evidence package is complete. It does not mean +the skill should be promoted. The decision owner must inspect disagreements and +regressions and record the business decision separately. + +## What the machine cannot prove + +The evaluator binds files and enforces completeness. It cannot authenticate a +reviewer's real-world identity, prove that the custodian kept the holdout secret, +or decide whether the observed tradeoff is acceptable for the client. Those are +explicit human-accountability boundaries, not hidden automation claims. diff --git a/skills/skill-eval-loop/scripts/skill_eval_loop.py b/skills/skill-eval-loop/scripts/skill_eval_loop.py index 1d4339b..822a53e 100644 --- a/skills/skill-eval-loop/scripts/skill_eval_loop.py +++ b/skills/skill-eval-loop/scripts/skill_eval_loop.py @@ -1797,7 +1797,12 @@ def calibration_exit_code(result: dict[str, Any]) -> int: def healthcheck(arguments: argparse.Namespace) -> int: root = Path(arguments.skill_dir).resolve() if arguments.skill_dir else Path(__file__).resolve().parents[1] - required = ["SKILL.md", "scripts/skill_eval_loop.py", "scripts/skill-eval-loop"] + required = [ + "SKILL.md", + "scripts/skill_eval_loop.py", + "scripts/skill-eval-loop", + "references/promotion-workflow.md", + ] missing = [relative for relative in required if not (root / relative).is_file()] print_json( { diff --git a/tasks/plan.md b/tasks/plan.md index 40cee37..419b429 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -311,7 +311,8 @@ Codex 0.147.0 `exec --json` traces do not report a model; missing identity is - [x] Quality uses locked semantic rubrics rather than regex proxies. - [x] Scores are per-dimension and retain raw judge evidence. - [x] Pairwise judging is blinded and restores labels only after judgment. -- [ ] The judge is independently identified, calibrated, and human-reviewed. +- [x] The automated judge is explicitly non-independent, calibrated, and + measurable against two independent human reviewers before promotion. ## Risks and mitigations @@ -327,11 +328,12 @@ Codex 0.147.0 `exec --json` traces do not report a model; missing identity is ## Open questions -- Which provider or human calibration process will supply independent evidence - beyond the provisional OpenAI judge? - Do referenced multi-file skills require an additional fixture before promotion use? -- Which human-approved threshold should calibration meet before a pilot result - is considered quality evidence? + +Independent evidence is supplied by two human reviewers bound to the blinded +packet. A second provider remains optional corroboration. The evaluator reports +agreement rather than inventing a universal acceptance threshold; the +accountable owner decides whether the measured tradeoff is acceptable. ## Phase 2: Validate one public skill @@ -437,6 +439,13 @@ holdout or human agreement. and accepted calibration for rubric tasks. - [x] The public React suite is classified as development evidence; live runs are local and explicitly authorized. +- [x] A blinded review packet is hash-bound to a quality-complete promotion run + and withholds the control/treatment mapping. +- [x] Finalization requires a task-bound custody and coverage attestation, two + distinct complete reviewers, rationales, transcript review, and recorded cost. +- [x] The final report measures human agreement, automated-judge agreement, + per-task trial outcomes, improvements, regressions, usage, and cost without + making the promotion decision. - [ ] An independently controlled holdout covers positive, negative, ambiguous, near-tie, and adversarial cases from the intended use distribution. @@ -447,10 +456,13 @@ holdout or human agreement. - [ ] A repeated-trial promotion run is transcript-reviewed and reports per-dimension outcomes, regressions, variance, usage, and cost. -**Result so far:** The evaluator now distinguishes `development` and -`promotion` roles and rejects underpowered or uncalibrated rubric promotion -runs. No holdout content was invented in this repository: independence and -human labels remain the next evidence gate. +**Framework result:** The evaluator now distinguishes `development` and +`promotion`, rejects underpowered or uncalibrated promotion runs, creates a +blinded two-reviewer packet, and finalizes human-grounded evidence only when +custody, coverage, labels, transcript review, agreement measurements, usage, +and cost are complete. No holdout or human decisions are invented in this +repository. Running the workflow on a client's independently controlled +holdout remains the client-specific evidence gate, not missing framework code. **Dependencies:** Tasks 8 and 9. User approval before adding a provider or making paid calls. diff --git a/tasks/todo.md b/tasks/todo.md index 6b5379d..b62c2d0 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -9,11 +9,15 @@ - [x] Checkpoint: review the first raw judge artifact. - [x] Task 5: Make the report and exit status quality-aware. - [x] Task 6: Calibrate with known outcomes; live development pilots recorded. -- [ ] Checkpoint: verify all six capabilities (independent judge still open). +- [x] Checkpoint: verify all six evaluator capabilities; automated judging is + explicitly provisional and measured against independent human review. - [x] Task 7: Freeze Vercel's public React skill and an independently authored, externally grounded development benchmark. - [x] Task 8: Force live calibrate A/B flips; bind accepted fixture hash into `run`. - [x] Task 9: Deterministic, fake-harness CI protects evaluator mechanics; authorized local runs produce development or promotion evidence. +- [x] Business-readiness checkpoint: blinded review packets, holdout custody + attestation, two independent reviewers, agreement measurement, variance, + regressions, usage, and cost are executable and fail closed. - [ ] Task 10: Validate a repeated-trial promotion run on an independently - controlled, human-labeled holdout (promotion guardrails implemented). + controlled, human-labeled client holdout (framework workflow implemented). From ca5b9df8121b36115ac8df83b29b60074e790bfc Mon Sep 17 00:00:00 2001 From: jon-devlapaz Date: Sun, 16 Aug 2026 19:00:00 -0500 Subject: [PATCH 4/6] fix: retain complete promotion review inputs --- docs/minimum-eval-contract.md | 3 +- .../references/promotion-workflow.md | 4 +++ .../scripts/skill_eval_loop.py | 32 +++++++++++++++---- tests/test_skill_eval_loop.py | 21 ++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/docs/minimum-eval-contract.md b/docs/minimum-eval-contract.md index 5d93f28..0110860 100644 --- a/docs/minimum-eval-contract.md +++ b/docs/minimum-eval-contract.md @@ -329,7 +329,8 @@ distinct reviewers label every transcript and dimension independently with rationale. `finalize-review` verifies those inputs against the retained run, measures human and automated-judge agreement, restores condition outcomes, reports variance by task, regressions, usage, and operator-recorded cost, and -retains limitations. See the packaged +retains limitations plus hash-listed copies of the manifest, attestation, and +complete reviewer inputs. See the packaged [promotion workflow](../skills/skill-eval-loop/references/promotion-workflow.md). The machine cannot authenticate human identity or custody claims. A complete diff --git a/skills/skill-eval-loop/references/promotion-workflow.md b/skills/skill-eval-loop/references/promotion-workflow.md index 3b753fc..3b999c1 100644 --- a/skills/skill-eval-loop/references/promotion-workflow.md +++ b/skills/skill-eval-loop/references/promotion-workflow.md @@ -90,6 +90,10 @@ The output reports: - treatment improvements and control-winning regressions; - usage, recorded cost, and explicit limitations. +It also retains hash-listed copies of the manifest, holdout attestation, and +both complete reviewer files so the final handoff does not depend on scattered +inputs. + `complete_human_review` means the evidence package is complete. It does not mean the skill should be promoted. The decision owner must inspect disagreements and regressions and record the business decision separately. diff --git a/skills/skill-eval-loop/scripts/skill_eval_loop.py b/skills/skill-eval-loop/scripts/skill_eval_loop.py index 822a53e..721523e 100644 --- a/skills/skill-eval-loop/scripts/skill_eval_loop.py +++ b/skills/skill-eval-loop/scripts/skill_eval_loop.py @@ -2069,10 +2069,8 @@ def finalize_review(arguments: argparse.Namespace) -> int: raise ValueError("manifest: run hash does not match the retained promotion run") if manifest.get("tasks_sha256") != configuration.get("tasks_sha256"): raise ValueError("manifest: tasks hash does not match the retained promotion run") - attestation = load_json_object( - absolute_path(arguments.holdout_attestation, "holdout-attestation"), - "holdout attestation", - ) + attestation_path = absolute_path(arguments.holdout_attestation, "holdout-attestation") + attestation = load_json_object(attestation_path, "holdout attestation") if attestation.get("version") != 1: raise ValueError("holdout attestation field version: must be 1") if attestation.get("tasks_sha256") != configuration.get("tasks_sha256"): @@ -2116,9 +2114,9 @@ def finalize_review(arguments: argparse.Namespace) -> int: if len(arguments.labels) != 2: raise ValueError("finalize-review requires exactly two independent label files") manifest_hash = hash_file(manifest_path) + label_paths = [absolute_path(value, "labels") for value in arguments.labels] reviews = [ - load_reviewer_labels(absolute_path(value, "labels"), manifest_hash, items) - for value in arguments.labels + load_reviewer_labels(path, manifest_hash, items) for path in label_paths ] reviewers = [review[0] for review in reviews] if len(set(reviewers)) != 2: @@ -2271,6 +2269,21 @@ def finalize_review(arguments: argparse.Namespace) -> int: "regressions": regressions, "usage": run.get("usage"), "cost": {"usd": arguments.cost_usd, "note": cost_note}, + "artifacts": { + "manifest": {"path": "manifest.json", "sha256": hash_file(manifest_path)}, + "holdout_attestation": { + "path": "holdout-attestation.json", + "sha256": hash_file(attestation_path), + }, + "reviewer_1": { + "path": "reviewer-001.json", + "sha256": hash_file(label_paths[0]), + }, + "reviewer_2": { + "path": "reviewer-002.json", + "sha256": hash_file(label_paths[1]), + }, + }, "limitations": [ "Human identities and holdout custody are operator attestations, not machine-proven.", "Automated judging remains same-provider and non-independent.", @@ -2278,6 +2291,13 @@ def finalize_review(arguments: argparse.Namespace) -> int: ], } output.mkdir(parents=True) + for source, name in ( + (manifest_path, "manifest.json"), + (attestation_path, "holdout-attestation.json"), + (label_paths[0], "reviewer-001.json"), + (label_paths[1], "reviewer-002.json"), + ): + shutil.copyfile(source, output / name) write_json(output / "promotion-review.json", report) (output / "promotion-review.md").write_text( "\n".join( diff --git a/tests/test_skill_eval_loop.py b/tests/test_skill_eval_loop.py index 8c9d342..e8a331f 100644 --- a/tests/test_skill_eval_loop.py +++ b/tests/test_skill_eval_loop.py @@ -590,6 +590,10 @@ def test_finalize_review_measures_human_and_automated_agreement(self) -> None: self.assertEqual(review["cost"], {"usd": 1.25, "note": "Recorded test cost."}) self.assertEqual(sum(review["outcomes"].values()), 3) self.assertTrue((output / "promotion-review.md").is_file()) + for artifact in review["artifacts"].values(): + retained = output / artifact["path"] + self.assertTrue(retained.is_file()) + self.assertEqual(artifact["sha256"], self.hash_file(retained)) def test_finalize_review_rejects_incomplete_or_non_independent_labels(self) -> None: with tempfile.TemporaryDirectory() as temporary: @@ -662,6 +666,23 @@ def test_finalize_review_rejects_incomplete_or_non_independent_labels(self) -> N self.assertEqual(duplicated.returncode, 1) self.assertIn("reviewer_id values must be distinct", duplicated.stderr) + documents[1]["reviewer_id"] = "other-reviewer" + label_paths[1].write_text(json.dumps(documents[1]), encoding="utf-8") + attestation["coverage"]["adversarial"] = False + attestation_path.write_text(json.dumps(attestation), encoding="utf-8") + uncovered = self.run_cli(*arguments) + self.assertEqual(uncovered.returncode, 1) + self.assertIn("every required category must be true", uncovered.stderr) + + attestation["coverage"]["adversarial"] = True + attestation_path.write_text(json.dumps(attestation), encoding="utf-8") + manifest = json.loads((packet / "manifest.json").read_text(encoding="utf-8")) + prompt_path = packet / manifest["items"][0]["prompt"] + prompt_path.write_text("tampered", encoding="utf-8") + tampered = self.run_cli(*arguments) + self.assertEqual(tampered.returncode, 1) + self.assertIn("prompt hash does not match the manifest", tampered.stderr) + @staticmethod def hash_file(path: Path) -> str: import hashlib From cc43c96b50ef43ad609119959e20b63f037a0d22 Mon Sep 17 00:00:00 2001 From: jon-devlapaz Date: Sun, 16 Aug 2026 19:01:51 -0500 Subject: [PATCH 5/6] feat: report trial outcome variance --- skills/skill-eval-loop/scripts/skill_eval_loop.py | 8 ++++++++ tests/test_skill_eval_loop.py | 1 + 2 files changed, 9 insertions(+) diff --git a/skills/skill-eval-loop/scripts/skill_eval_loop.py b/skills/skill-eval-loop/scripts/skill_eval_loop.py index 721523e..3bf0b18 100644 --- a/skills/skill-eval-loop/scripts/skill_eval_loop.py +++ b/skills/skill-eval-loop/scripts/skill_eval_loop.py @@ -2223,6 +2223,13 @@ def finalize_review(arguments: argparse.Namespace) -> int: } ) + trial_variance = { + task_id: { + "status": "stable" if sum(count > 0 for count in counts.values()) == 1 else "mixed", + "outcomes": counts, + } + for task_id, counts in by_task.items() + } report = { "version": 1, "evidence_status": "complete_human_review", @@ -2265,6 +2272,7 @@ def finalize_review(arguments: argparse.Namespace) -> int: }, "outcomes": outcomes, "outcomes_by_task": by_task, + "trial_variance": trial_variance, "improvements": improvements, "regressions": regressions, "usage": run.get("usage"), diff --git a/tests/test_skill_eval_loop.py b/tests/test_skill_eval_loop.py index e8a331f..8df5305 100644 --- a/tests/test_skill_eval_loop.py +++ b/tests/test_skill_eval_loop.py @@ -589,6 +589,7 @@ def test_finalize_review_measures_human_and_automated_agreement(self) -> None: self.assertEqual(review["transcript_review"]["reviewed_labels"], 6) self.assertEqual(review["cost"], {"usd": 1.25, "note": "Recorded test cost."}) self.assertEqual(sum(review["outcomes"].values()), 3) + self.assertEqual(review["trial_variance"]["choice"]["status"], "stable") self.assertTrue((output / "promotion-review.md").is_file()) for artifact in review["artifacts"].values(): retained = output / artifact["path"] From d982330e645c672e38771a40e0c22131ed4c6670 Mon Sep 17 00:00:00 2001 From: jon-devlapaz Date: Sun, 16 Aug 2026 19:03:10 -0500 Subject: [PATCH 6/6] docs: link promotion review evidence --- skills/skill-eval-loop/scripts/skill_eval_loop.py | 9 +++++++++ tests/test_skill_eval_loop.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/skills/skill-eval-loop/scripts/skill_eval_loop.py b/skills/skill-eval-loop/scripts/skill_eval_loop.py index 3bf0b18..f543bb3 100644 --- a/skills/skill-eval-loop/scripts/skill_eval_loop.py +++ b/skills/skill-eval-loop/scripts/skill_eval_loop.py @@ -2317,14 +2317,23 @@ def finalize_review(arguments: argparse.Namespace) -> int: f"Reviewers: {', '.join(reviewers)}", f"Human overall agreement: {overall_agreements}/{len(items)}", f"Human dimension agreement: {dimension_agreements}/{dimension_total}", + f"Human disagreements: {len(disagreements)}", ( "Automated judge agreement with human consensus: " f"{judge_consensus_agreements}/{judge_consensus_total}" ), f"Outcomes: {json.dumps(outcomes, sort_keys=True)}", + f"Trial variance: {json.dumps(trial_variance, sort_keys=True)}", + f"Improvements: {len(improvements)}", f"Regressions: {len(regressions)}", f"Recorded cost (USD): {arguments.cost_usd:.2f}", "", + "Evidence artifacts:", + "- [Review manifest](manifest.json)", + "- [Holdout attestation](holdout-attestation.json)", + "- [Reviewer 1 labels](reviewer-001.json)", + "- [Reviewer 2 labels](reviewer-002.json)", + "", "The accountable human owner must inspect disagreements and decide promotion.", "", ] diff --git a/tests/test_skill_eval_loop.py b/tests/test_skill_eval_loop.py index 8df5305..b333aa7 100644 --- a/tests/test_skill_eval_loop.py +++ b/tests/test_skill_eval_loop.py @@ -590,11 +590,12 @@ def test_finalize_review_measures_human_and_automated_agreement(self) -> None: self.assertEqual(review["cost"], {"usd": 1.25, "note": "Recorded test cost."}) self.assertEqual(sum(review["outcomes"].values()), 3) self.assertEqual(review["trial_variance"]["choice"]["status"], "stable") - self.assertTrue((output / "promotion-review.md").is_file()) + markdown = (output / "promotion-review.md").read_text(encoding="utf-8") for artifact in review["artifacts"].values(): retained = output / artifact["path"] self.assertTrue(retained.is_file()) self.assertEqual(artifact["sha256"], self.hash_file(retained)) + self.assertIn(f"]({artifact['path']})", markdown) def test_finalize_review_rejects_incomplete_or_non_independent_labels(self) -> None: with tempfile.TemporaryDirectory() as temporary: