diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 2d1b7bb71..1f57799ca 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -6461,7 +6461,7 @@ jobs: fi if same_head_opencode_approval_exists; then - printf '::notice::MODEL_OUTPUT_UNAVAILABLE: same-head OpenCode approval already exists for head %s, and current-head coverage, peer checks, code-scanning alerts, and review threads are clean; succeeding the required check without publishing a duplicate approval review.\n' "$HEAD_SHA" + printf '::notice::MODEL_OUTPUT_UNAVAILABLE: same-head real-model OpenCode approval with passed adversarial evidence already exists for head %s, and current-head coverage, peer checks, code-scanning alerts, and review threads are clean; succeeding the required check without publishing a duplicate approval review.\n' "$HEAD_SHA" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then { printf '## OpenCode required check satisfied by existing same-head approval\n\n' @@ -6470,7 +6470,7 @@ jobs: printf -- '- Workflow run: %s\n' "$RUN_ID" printf -- '- Workflow attempt: %s\n' "$RUN_ATTEMPT" printf -- '- Model-pool outcome: `%s`\n' "${OPENCODE_MODEL_POOL_OUTCOME:-unknown}" - printf -- '- Reason: a prior OpenCode APPROVED review already targets this exact head, and the fallback rechecked coverage, peer checks, code-scanning alerts, and unresolved review threads before accepting it.\n' + printf -- '- Reason: a prior real-model OpenCode APPROVED review with passed structured adversarial probes already targets this exact head, and the fallback rechecked coverage, peer checks, code-scanning alerts, and unresolved review threads before accepting it.\n' printf -- '- Review state: unchanged; no duplicate APPROVE review was posted from model-output-unavailable evidence.\n\n' } >>"$GITHUB_STEP_SUMMARY" fi @@ -6520,7 +6520,7 @@ jobs: } same_head_opencode_approval_exists() { - local review_lookup_token reviews_json approval_count lookup_error_file + local review_lookup_token reviews_json lookup_error_file review_lookup_token="${CHECK_LOOKUP_GH_TOKEN:-${GH_TOKEN:-}}" if [ -z "$review_lookup_token" ]; then printf '::notice::Existing same-head OpenCode approval lookup skipped because no review read token was configured.\n' >&2 @@ -6536,20 +6536,8 @@ jobs: fi rm -f "$lookup_error_file" - approval_count="$( - printf '%s\n' "$reviews_json" | - jq --arg head "$HEAD_SHA" ' - [ - .[][] - | select(.state == "APPROVED") - | select(.commit_id == $head) - | select((.user.login // "") as $login | ["opencode-agent", "opencode-agent[bot]", "github-actions[bot]"] | index($login)) - ] - | length - ' - )" - - [ "${approval_count:-0}" -gt 0 ] + printf '%s\n' "$reviews_json" | + python3 scripts/ci/opencode_existing_approval_gate.py --head "$HEAD_SHA" } request_changes_for_merge_conflict_if_present() { diff --git a/PR_GOVERNANCE_AUDIT.md b/PR_GOVERNANCE_AUDIT.md index 878f6ab92..d54b5237a 100644 --- a/PR_GOVERNANCE_AUDIT.md +++ b/PR_GOVERNANCE_AUDIT.md @@ -324,6 +324,20 @@ PR #381: wait: OpenCode review is already in progress ## Remaining Proof Gaps +- 2026-07-13 KST `.github` PR #510 merged at `c7a568bde942d25d2a735b1bbfbb52b057b53b2f` + while GitHub still reported `reviewDecision=REVIEW_REQUIRED` and the complete + REST review list was empty. Although an auto-squash request had been enabled, + the resulting commit is a separate two-parent `MERGE` attributed to the user, + created while Required OpenCode run `29225918664` attempt 7 was still in + progress. The repository ruleset also required zero approvals even though the + legacy branch protection required one; ruleset `17921150` now independently + requires one approval, last-push approval, stale-review dismissal, and thread + resolution with no bypass actors. Existing-approval reuse must not + treat an actor, state, and commit match as sufficient evidence: the review body + must also contain the exact real-model marker, current head/run/attempt, an + `APPROVE` result, and a passed adversarial-validation object whose material + probes were falsified. Deterministic, fallback, and model-unavailable markers + remain explicitly ineligible and every rejection reason is emitted to the log. - 2026-07-13 KST `.github` workflow-dispatch run `29227653777` produced a current-head real-model approval for PR #506 after 409 tests, 100% executable coverage, 100% docstring coverage, and three falsified adversarial probes, but diff --git a/scripts/ci/opencode_existing_approval_gate.py b/scripts/ci/opencode_existing_approval_gate.py new file mode 100644 index 000000000..9022fc4e4 --- /dev/null +++ b/scripts/ci/opencode_existing_approval_gate.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Validate that a reusable same-head approval came from a real model review.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from typing import Any, TextIO + + +APPROVAL_AUTHORS = frozenset( + {"opencode-agent", "opencode-agent[bot]", "github-actions[bot]"} +) +FALLBACK_MARKERS = ( + "deterministic current-head evidence", + "deterministic fallback approval", + "model-unavailable evidence fallback", + "did not emit a usable current-head control block", + "scope: `unsupported`", + "model-pool outcome: `unknown`", +) +PRIMARY_APPROVAL_MARKER = ( + "OpenCode reviewed the current-head bounded evidence and found no blocking issues." +) +ADVERSARIAL_BLOCK_RE = re.compile( + r"## Adversarial validation\s*```json\s*(?P.*?)\s*```", + re.IGNORECASE | re.DOTALL, +) +SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") +WORKFLOW_RUN_RE = re.compile(r"(?m)^- Workflow run: [1-9][0-9]*\s*$") +WORKFLOW_ATTEMPT_RE = re.compile(r"(?m)^- Workflow attempt: [1-9][0-9]*\s*$") +REQUIRED_PROBE_FIELDS = ( + "path", + "hypothesis", + "attack_or_counterexample", + "evidence", + "outcome", +) + + +def flatten_reviews(document: object) -> list[dict[str, Any]]: + """Flatten REST pagination output while rejecting malformed review entries.""" + if not isinstance(document, list): + raise ValueError("review payload must be a JSON array") + + reviews: list[dict[str, Any]] = [] + for page in document: + entries = page if isinstance(page, list) else [page] + for review in entries: + if not isinstance(review, dict): + raise ValueError("every review entry must be a JSON object") + reviews.append(review) + return reviews + + +def extract_adversarial_evidence(body: str) -> dict[str, Any] | None: + """Return the last parseable adversarial-validation JSON block.""" + evidence: dict[str, Any] | None = None + for match in ADVERSARIAL_BLOCK_RE.finditer(body): + try: + candidate = json.loads(match.group("payload")) + except json.JSONDecodeError: + continue + if isinstance(candidate, dict): + evidence = candidate + return evidence + + +def adversarial_rejection_reason(body: str) -> str | None: + """Explain why structured adversarial evidence is not reusable.""" + evidence = extract_adversarial_evidence(body) + if evidence is None: + return "missing parseable adversarial-validation JSON" + if str(evidence.get("status") or "").lower() != "passed": + return "adversarial-validation status is not passed" + + probes = evidence.get("probes") + if not isinstance(probes, list) or not probes: + return "adversarial-validation probes are empty" + for probe in probes: + if not isinstance(probe, dict): + return "adversarial-validation probe is not an object" + line = probe.get("line") + if isinstance(line, bool) or not isinstance(line, int) or line < 1: + return "adversarial-validation probe line is not a positive integer" + for field in REQUIRED_PROBE_FIELDS: + if not isinstance(probe.get(field), str) or not probe[field].strip(): + return f"adversarial-validation probe is missing {field}" + if probe["outcome"].strip().lower() != "falsified": + return "approval probe outcome is not falsified" + + residual_risk = evidence.get("residual_risk") + if not isinstance(residual_risk, str) or not residual_risk.strip(): + return "adversarial-validation residual_risk is missing" + return None + + +def review_rejection_reason(review: dict[str, Any], head_sha: str) -> str | None: + """Explain why a review cannot prove a real current-head model approval.""" + if str(review.get("state") or "").upper() != "APPROVED": + return "review state is not APPROVED" + if str(review.get("commit_id") or "").lower() != head_sha.lower(): + return "review commit does not match current head" + + login = str((review.get("user") or {}).get("login") or "") + if login not in APPROVAL_AUTHORS: + return "review author is not an OpenCode publication actor" + + body = str(review.get("body") or "") + body_lower = body.lower() + if any(marker in body_lower for marker in FALLBACK_MARKERS): + return "review body is deterministic or model-unavailable fallback evidence" + if PRIMARY_APPROVAL_MARKER not in body: + return "review body lacks the real-model approval marker" + if "- Result: APPROVE" not in body: + return "review body lacks an APPROVE result" + if f"- Head SHA: `{head_sha}`" not in body: + return "review body lacks the exact current-head SHA" + if not WORKFLOW_RUN_RE.search(body): + return "review body lacks a workflow run id" + if not WORKFLOW_ATTEMPT_RE.search(body): + return "review body lacks a workflow attempt" + return adversarial_rejection_reason(body) + + +def has_reusable_real_model_approval( + reviews: list[dict[str, Any]], head_sha: str, *, log: TextIO +) -> bool: + """Return whether reviews contain a real-model approval for the exact head.""" + candidate_count = 0 + for review in reversed(reviews): + state = str(review.get("state") or "").upper() + commit_id = str(review.get("commit_id") or "") + login = str((review.get("user") or {}).get("login") or "") + if state != "APPROVED" or commit_id.lower() != head_sha.lower(): + continue + if login not in APPROVAL_AUTHORS: + continue + candidate_count += 1 + reason = review_rejection_reason(review, head_sha) + review_id = review.get("id", "unknown") + if reason is None: + print( + "existing-approval gate accepted real-model review " + f"id={review_id} author={login} head={head_sha}", + file=log, + ) + return True + print( + "existing-approval gate rejected same-head review " + f"id={review_id} author={login}: {reason}", + file=log, + ) + + print( + "existing-approval gate found no reusable real-model approval " + f"for head={head_sha}; same-head candidates={candidate_count}", + file=log, + ) + return False + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse existing-approval gate command-line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument("--head", required=True) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + """Read paginated reviews from stdin and evaluate reusable approval evidence.""" + args = parse_args(argv) + if not SHA_RE.fullmatch(args.head): + print("existing-approval gate requires a 40-character head SHA", file=sys.stderr) + return 2 + try: + reviews = flatten_reviews(json.load(sys.stdin)) + except (json.JSONDecodeError, ValueError) as exc: + print(f"existing-approval gate could not parse reviews: {exc}", file=sys.stderr) + return 2 + return 0 if has_reusable_real_model_approval(reviews, args.head, log=sys.stderr) else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 60820adad..0aa38bf5c 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -680,6 +680,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "same_head_opencode_approval_exists" "model-unavailable path reuses an existing same-head OpenCode approval before publishing fallback approval" assert_file_contains "$workflow_file" "EXISTING_CURRENT_HEAD_APPROVAL" "existing same-head approval fallback logs an explicit required-check result" assert_file_contains "$workflow_file" "no duplicate APPROVE review was posted" "existing same-head approval fallback does not publish a duplicate approval review" + assert_file_contains "$workflow_file" "opencode_existing_approval_gate.py" "existing approval reuse requires machine-validated real-model adversarial evidence" assert_file_contains "$workflow_file" "no adversarial_validation block was fabricated" "deterministic model-unavailable approval must not fabricate model adversarial evidence" assert_file_contains "$workflow_file" 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' "deterministic model-unavailable approval is explicit and source-evidence gated" assert_file_contains "$workflow_file" "approval still pending" "pending peer checks cannot satisfy the required OpenCode gate without a review" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 7ca951c8a..14e3e61f9 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -1265,6 +1265,8 @@ def test_opencode_model_pool_failure_uses_gated_clean_evidence_fallback(): assert "MODEL_UNAVAILABLE_CLEAN_EVIDENCE" in workflow assert "no adversarial_validation block was fabricated" in workflow assert "no duplicate APPROVE review was posted" in workflow + assert 'opencode_existing_approval_gate.py --head "$HEAD_SHA"' in workflow + assert "same-head real-model OpenCode approval with passed adversarial evidence" in workflow assert 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' in workflow model_unavailable_block = re.search( r"if \[ \"\$opencode_review_outcome\" != \"success\" \]; then" diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py new file mode 100644 index 000000000..77bcf83cb --- /dev/null +++ b/tests/test_opencode_existing_approval_gate.py @@ -0,0 +1,224 @@ +import io +import json +import sys + +import pytest + +from scripts.ci import opencode_existing_approval_gate as gate + + +HEAD = "a" * 40 + + +def valid_body(head: str = HEAD) -> str: + """Build a real-model review body with structured adversarial evidence.""" + evidence = { + "status": "passed", + "probes": [ + { + "path": ".github/workflows/opencode-review.yml", + "line": 1, + "hypothesis": "A fallback approval could be reused.", + "attack_or_counterexample": "Supply a deterministic approval body.", + "evidence": "The gate rejected the fallback marker.", + "outcome": "falsified", + } + ], + "residual_risk": "Hosted token permissions remain externally enforced.", + } + return "\n".join( + ( + "## Pull request overview", + "", + gate.PRIMARY_APPROVAL_MARKER, + "", + "## Adversarial validation", + "", + "```json", + json.dumps(evidence), + "```", + "", + "- Result: APPROVE", + f"- Head SHA: `{head}`", + "- Workflow run: 123", + "- Workflow attempt: 2", + ) + ) + + +def review(**overrides): + """Build a minimal REST review object for approval-gate tests.""" + value = { + "id": 7, + "state": "APPROVED", + "commit_id": HEAD, + "user": {"login": "opencode-agent[bot]"}, + "body": valid_body(), + } + value.update(overrides) + return value + + +@pytest.mark.parametrize("payload", [[review()], [[review()]]]) +def test_flatten_reviews_and_accept_real_model_approval(payload): + reviews = gate.flatten_reviews(payload) + log = io.StringIO() + assert gate.has_reusable_real_model_approval(reviews, HEAD, log=log) + assert "accepted real-model review" in log.getvalue() + + +@pytest.mark.parametrize("payload", [{}, ["bad"], [["bad"]]]) +def test_flatten_reviews_rejects_malformed_payload(payload): + with pytest.raises(ValueError): + gate.flatten_reviews(payload) + + +def test_extract_adversarial_evidence_uses_last_parseable_block(): + body = "## Adversarial validation\n```json\nnot-json\n```\n" + valid_body() + assert gate.extract_adversarial_evidence(body)["status"] == "passed" + assert gate.extract_adversarial_evidence("none") is None + + +@pytest.mark.parametrize( + ("mutate", "reason"), + [ + (lambda value: value.update(state="COMMENTED"), "state"), + (lambda value: value.update(commit_id="b" * 40), "commit"), + (lambda value: value.update(user={"login": "unknown"}), "author"), + ( + lambda value: value.update(body=value["body"] + "\ndeterministic fallback approval"), + "fallback", + ), + ( + lambda value: value.update(body=value["body"].replace(gate.PRIMARY_APPROVAL_MARKER, "missing")), + "real-model approval marker", + ), + (lambda value: value.update(body=value["body"].replace("- Result: APPROVE", "")), "APPROVE result"), + (lambda value: value.update(body=value["body"].replace(f"- Head SHA: `{HEAD}`", "")), "current-head"), + (lambda value: value.update(body=value["body"].replace("- Workflow run: 123", "")), "workflow run"), + (lambda value: value.update(body=value["body"].replace("- Workflow attempt: 2", "")), "workflow attempt"), + ( + lambda value: value.update(body=value["body"].replace("```json", "```text")), + "parseable adversarial", + ), + ], +) +def test_review_rejection_reason_rejects_non_model_evidence(mutate, reason): + value = review() + mutate(value) + assert reason in gate.review_rejection_reason(value, HEAD) + + +@pytest.mark.parametrize( + ("evidence", "reason"), + [ + ({"status": "failed", "probes": [{}], "residual_risk": "risk"}, "status"), + ({"status": "passed", "probes": [], "residual_risk": "risk"}, "probes are empty"), + ({"status": "passed", "probes": ["bad"], "residual_risk": "risk"}, "not an object"), + ( + { + "status": "passed", + "probes": [ + { + "path": "file", + "line": 0, + "hypothesis": "hypothesis", + "attack_or_counterexample": "attack", + "evidence": "evidence", + "outcome": "falsified", + } + ], + "residual_risk": "risk", + }, + "positive integer", + ), + ( + { + "status": "passed", + "probes": [ + { + "path": "file", + "line": 1, + "hypothesis": "hypothesis", + "attack_or_counterexample": "attack", + "evidence": "evidence", + } + ], + "residual_risk": "risk", + }, + "missing outcome", + ), + ( + { + "status": "passed", + "probes": [ + { + "path": "file", + "line": 1, + "hypothesis": "hypothesis", + "attack_or_counterexample": "attack", + "evidence": "evidence", + "outcome": "confirmed", + } + ], + "residual_risk": "risk", + }, + "not falsified", + ), + ( + { + "status": "passed", + "probes": [ + { + "path": "file", + "line": 1, + "hypothesis": "hypothesis", + "attack_or_counterexample": "attack", + "evidence": "evidence", + "outcome": "falsified", + } + ], + "residual_risk": "", + }, + "residual_risk", + ), + ], +) +def test_adversarial_rejection_reason_explains_invalid_evidence(evidence, reason): + body = f"## Adversarial validation\n```json\n{json.dumps(evidence)}\n```" + assert reason in gate.adversarial_rejection_reason(body) + + +def test_has_reusable_real_model_approval_logs_rejected_candidates(): + fallback = review(body=valid_body() + "\nmodel-unavailable evidence fallback") + log = io.StringIO() + assert not gate.has_reusable_real_model_approval( + [ + review(state="COMMENTED"), + review(commit_id="b" * 40), + review(user={"login": "unknown"}), + fallback, + ], + HEAD, + log=log, + ) + assert "rejected same-head review" in log.getvalue() + assert "same-head candidates=1" in log.getvalue() + + +def test_parse_args_and_main(monkeypatch, capsys): + assert gate.parse_args(["--head", HEAD]).head == HEAD + + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps([[review()]]))) + assert gate.main(["--head", HEAD]) == 0 + + monkeypatch.setattr(sys, "stdin", io.StringIO("not-json")) + assert gate.main(["--head", HEAD]) == 2 + assert "could not parse reviews" in capsys.readouterr().err + + monkeypatch.setattr(sys, "stdin", io.StringIO("[]")) + assert gate.main(["--head", "short"]) == 2 + assert "40-character" in capsys.readouterr().err + + monkeypatch.setattr(sys, "stdin", io.StringIO("[]")) + assert gate.main(["--head", HEAD]) == 1