Skip to content

fix(ci): bound anonymous review provider budget #722

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
seonghobae wants to merge 5 commits into from
Closed

fix(ci): bound anonymous review provider budget #722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
601fb64
fix(ci): bound anonymous review provider budget
seonghobae Aug 3, 2026
14baf54
fix(ci): align scheduled CodeQL action refs
seonghobae Aug 3, 2026
851e686
fix(ci): feed canonical validator feedback to review repair
seonghobae Aug 3, 2026
e576f7f
fix(ci): repair complete approval posture
seonghobae Aug 3, 2026
023db07
fix(ci): publish same-repo status with workflow token
seonghobae Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .github/workflows/opencode-review-dispatch.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4342,6 +4342,10 @@ jobs:
OPENCODE_NVIDIA_NIM_RUN_TIMEOUT_SECONDS: "180"
OPENCODE_NVIDIA_NIM_TOTAL_BUDGET_SECONDS: "900"
OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "3600"
# Anonymous free candidates share fifteen minutes total. A quota-starved
# catalog therefore cannot consume the entire 195-minute retry budget
# before keyed Terra/OpenAI/OpenRouter/GitHub Models fallbacks run.
OPENCODE_FREE_TOTAL_BUDGET_SECONDS: "900"
# This installation currently reports a 4k request-body limit for
# GitHub Models GPT-5 endpoints even though the public catalog is
# larger. Keep the exact runtime failure visible without spending a
Expand Down Expand Up @@ -7826,14 +7830,14 @@ jobs:
&& needs.validate-pr-metadata.outputs.target_repository != ''
&& needs.validate-pr-metadata.outputs.head_sha != ''
env:
GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || steps.opencode_app_token.outputs.token || github.token }}
GH_TOKEN: ${{ needs.validate-pr-metadata.outputs.target_repository == github.repository && github.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || steps.opencode_app_token.outputs.token || github.token }}
GH_REPOSITORY: ${{ needs.validate-pr-metadata.outputs.target_repository }}
PR_NUMBER: ${{ needs.validate-pr-metadata.outputs.pr_number }}
PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
OPENCODE_MODEL_POOL_OUTCOME: ${{ steps.opencode_review_model_pool.outputs.review_status }}
COVERAGE_EVIDENCE_RESULT: ${{ needs.coverage-evidence.result }}
OPENCODE_STATUS_TOKEN_SOURCE: ${{ secrets.PR_REVIEW_MERGE_TOKEN != '' && 'PR_REVIEW_MERGE_TOKEN' || secrets.OPENCODE_APPROVE_TOKEN != '' && 'OPENCODE_APPROVE_TOKEN' || steps.opencode_app_token.outputs.available == 'true' && 'opencode-app' || 'github-token' }}
OPENCODE_STATUS_TOKEN_SOURCE: ${{ needs.validate-pr-metadata.outputs.target_repository == github.repository && 'github-token' || secrets.PR_REVIEW_MERGE_TOKEN != '' && 'PR_REVIEW_MERGE_TOKEN' || secrets.OPENCODE_APPROVE_TOKEN != '' && 'OPENCODE_APPROVE_TOKEN' || steps.opencode_app_token.outputs.available == 'true' && 'opencode-app' || 'github-token' }}
OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt
OPENCODE_ARTIFACT_MANIFEST_SHA256: ${{ steps.seal_artifacts.outputs.manifest_sha256 }}
OPENCODE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-pr-head
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/scheduled-security-scan.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,13 @@ jobs:
with:
persist-credentials: false
- name: Initialize CodeQL
uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
- name: Perform CodeQL Analysis
continue-on-error: true
uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
with:
category: "/language:${{ matrix.language }}-scheduled"

Expand Down
122 changes: 110 additions & 12 deletions scripts/ci/run_opencode_review_model_pool.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,38 @@ set -euo pipefail

: "${GITHUB_OUTPUT:=/dev/null}"

# Only canonical, repository-owned validator categories may be replayed into a
# repair prompt. Provider output is untrusted and must never be reflected back
# verbatim as instructions.
LAST_CONTROL_REJECTION_KIND=""

classify_control_rejection() {
local diagnostics_file="$1"

LAST_CONTROL_REJECTION_KIND="unknown-control-contract"
if grep -Fq "no top-level current-run control JSON object was found" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="missing-current-run-control-envelope"
elif grep -Fq "expected exactly one top-level current-run control JSON object" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="multiple-current-run-control-objects"
elif grep -Fq "must cite the exact probe path and positive line" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="probe-path-line-mismatch"
elif grep -Fq "must state the observed proof result" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="probe-observed-result-missing"
elif grep -Fq "must cite an executed command, test/assertion, log/check/SARIF receipt, source trace, diff, or CodeGraph path" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="probe-proof-anchor-missing"
elif grep -Fq "source-line-sha256 receipt" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="probe-source-line-receipt-invalid"
elif grep -Fq "approval does not prove 100% coverage or an explicit no-source exception" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="approval-coverage-proof-missing"
elif grep -Fq "approval does not cite changed-file evidence" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="approval-changed-file-evidence-missing"
elif grep -Fq "approval does not include the required verification posture" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="approval-verification-posture-missing"
elif grep -Fq "adversarial_validation" "$diagnostics_file"; then
LAST_CONTROL_REJECTION_KIND="adversarial-validation-contract"
fi
}

record_review_status() {
printf 'review_status=%s\n' "$1" >>"$GITHUB_OUTPUT"
}
Expand Down Expand Up @@ -35,19 +67,28 @@ normalize_opencode_output() {
# copy, then normalize — so the pool only records success for output the
# publish step will accept, and leave output_file pristine for the publish
# step to normalize itself.
local probe rc
local probe rc diagnostics_file
probe="$(mktemp)"
diagnostics_file="$(mktemp)"
perl -pe 's/\x1b\[[0-9;?]*[A-Za-z]//g' "$output_file" >"$probe" 2>/dev/null || cp "$output_file" "$probe"

if python3 "$GITHUB_WORKSPACE/scripts/ci/opencode_review_normalize_output.py" \
"$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" "$probe"; then
"$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" "$probe" 2>"$diagnostics_file"; then
bash "$GITHUB_WORKSPACE/scripts/ci/opencode_review_approve_gate.sh" \
"$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" "$probe" >/dev/null
"$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" "$probe" >/dev/null 2>>"$diagnostics_file"
rc=$?
else
rc=1
fi
rm -f "$probe"
if [ -s "$diagnostics_file" ]; then
cat "$diagnostics_file" >&2
fi
if [ "$rc" -ne 0 ]; then
classify_control_rejection "$diagnostics_file"
else
LAST_CONTROL_REJECTION_KIND=""
fi
rm -f "$probe" "$diagnostics_file"
return "$rc"
}

Expand Down Expand Up @@ -225,14 +266,22 @@ PY
write_schema_repair_prompt() {
local model_candidate="$1"
local prompt_file="$2"
local rejection_kind="${LAST_CONTROL_REJECTION_KIND:-unknown-control-contract}"

write_prompt "$model_candidate" "$prompt_file"
{
printf '\nA previous response from this same provider reached the trusted validator but failed the control schema. Perform the review again from the same trusted evidence and return one corrected review body only.\n'
printf 'The trusted validator classified the previous rejection as `%s`. This fixed category contains no provider-controlled text.\n' "$rejection_kind"
printf 'This is a schema repair opportunity, not permission to weaken, omit, or fabricate evidence. Check every item before returning:\n'
printf -- '- Emit exactly one sentinel and exactly one current-run JSON control object; do not quote any example object or earlier response.\n'
printf -- '- The first output line must be exactly `<!-- opencode-review-gate head_sha=%s run_id=%s run_attempt=%s -->`; emit no prose or Markdown before it.\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT"
printf -- '- After that sentinel, emit exactly one `<!-- opencode-review-control-v1` block containing exactly one JSON object and close it with `-->`; do not quote any example object or earlier response.\n'
printf -- '- The JSON identity values must be head_sha=%s, run_id=%s, and run_attempt=%s.\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT"
printf -- '- Choose exactly APPROVE or REQUEST_CHANGES, with a non-empty reason, summary, and residual_risk.\n'
printf -- '- Include "adversarial_validation" as an object with at least the required probe count. Copy each path, line, and source-line-sha256 receipt exactly from trusted bounded evidence.\n'
printf -- '- For every probe, repeat its exact `path:line` inside evidence; name a concrete test, assertion, check, log, source trace, diff, or CodeGraph path; state the observed passed, failed, rejected, returned, showed, or exit-code result; include exactly one matching source-line-sha256 receipt.\n'
printf -- '- In an APPROVE reason or summary, name at least one exact current-head changed-file path copied from trusted bounded evidence.\n'
printf -- '- In an APPROVE reason or summary, include `CodeGraph` and every required verification-posture label exactly: `Approval sufficiency:`, `Verification posture:`, `Linter/static:`, `TDD/regression:`, `Coverage:`, `Docstring coverage:`, `DAG:`, `PoC/execution:`, `DDD/domain:`, `CDD/context:`, `Similar issues:`, `Claim/concept check:`, `Standards search:`, `Compatibility/convention:`, `Breaking-change/backcompat:`, `Performance:`, `Developer experience:`, `User experience:`, `Visual/DOM:`, `Accessibility/i18n:`, `Supply-chain/license:`, `Packaging:`, and `Security/privacy:`. State the evidence-backed outcome for each; do not fabricate unavailable evidence.\n'
printf -- '- In an APPROVE summary, include both `Coverage:` and `Docstring coverage:` and cite the exact trusted Coverage execution evidence that supported repository test suites passed and configured docstring gates passed or were advisory, or the exact explicit no-source exception.\n'
printf -- '- APPROVE requires status=passed, every probe outcome=falsified, and findings=[].\n'
printf -- '- REQUEST_CHANGES requires status=failed, at least one outcome=confirmed, and a non-empty source-backed finding at the same path and line.\n'
printf 'Return only the corrected review body now.\n'
Expand Down Expand Up @@ -368,6 +417,13 @@ is_nvidia_nim_candidate() {
esac
}

is_opencode_free_candidate() {
case "$1" in
opencode-free/*) return 0 ;;
*) return 1 ;;
esac
}

is_schema_repair_candidate() {
case "$1" in
nvidia-nim/* | opencode-free/*) return 0 ;;
Expand Down Expand Up @@ -548,7 +604,8 @@ main() {
local changed_file_count small_file_threshold medium_file_threshold
local invalid_control_cap max_total_attempts total_attempts alive_candidates
local nim_budget_seconds nim_elapsed_seconds nim_remaining_seconds
local nim_attempt_started nim_attempt_elapsed non_nim_candidate_count
local free_budget_seconds free_elapsed_seconds free_remaining_seconds
local attempt_started attempt_elapsed non_nim_candidate_count keyed_candidate_count
local -A dead_candidate_reasons invalid_control_counts
local -a model_candidates

Expand Down Expand Up @@ -615,11 +672,18 @@ main() {
fi
nim_budget_seconds="$(env_integer_or_default OPENCODE_NVIDIA_NIM_TOTAL_BUDGET_SECONDS 900)"
nim_elapsed_seconds=0
free_budget_seconds="$(env_integer_or_default OPENCODE_FREE_TOTAL_BUDGET_SECONDS 900)"
free_elapsed_seconds=0
non_nim_candidate_count=0
keyed_candidate_count=0
for model_candidate in "${model_candidates[@]}"; do
if ! is_nvidia_nim_candidate "$model_candidate"; then
non_nim_candidate_count=$((non_nim_candidate_count + 1))
fi
if ! is_nvidia_nim_candidate "$model_candidate" &&
! is_opencode_free_candidate "$model_candidate"; then
keyed_candidate_count=$((keyed_candidate_count + 1))
fi
done
if [ "$non_nim_candidate_count" -gt 0 ] &&
[ "$budget_seconds" -gt 0 ] &&
Expand All @@ -628,8 +692,15 @@ main() {
printf 'OpenCode NVIDIA NIM combined runtime budget was capped at %ss so %s non-NIM fallback candidate(s) retain retry budget.\n' \
"$nim_budget_seconds" "$non_nim_candidate_count"
fi
printf 'Configured OpenCode model pool: candidates=%s attempts=%s per-model-timeout=%ss retry-budget=%ss max-cycles=%s NVIDIA-NIM-combined-budget=%ss.\n' \
"${#model_candidates[@]}" "$attempts" "$original_run_timeout" "$budget_seconds" "$max_cycles" "$nim_budget_seconds"
if [ "$keyed_candidate_count" -gt 0 ] &&
[ "$budget_seconds" -gt 0 ] &&
[ "$free_budget_seconds" -ge "$budget_seconds" ]; then
free_budget_seconds=$((budget_seconds / 2))
printf 'OpenCode anonymous-free combined runtime budget was capped at %ss so %s keyed fallback candidate(s) retain retry budget.\n' \
"$free_budget_seconds" "$keyed_candidate_count"
fi
printf 'Configured OpenCode model pool: candidates=%s attempts=%s per-model-timeout=%ss retry-budget=%ss max-cycles=%s NVIDIA-NIM-combined-budget=%ss anonymous-free-combined-budget=%ss.\n' \
"${#model_candidates[@]}" "$attempts" "$original_run_timeout" "$budget_seconds" "$max_cycles" "$nim_budget_seconds" "$free_budget_seconds"

cycle=1
while :; do
Expand All @@ -649,6 +720,12 @@ main() {
"$model_candidate" "$nim_budget_seconds"
continue
fi
if is_opencode_free_candidate "$model_candidate" &&
[ "$free_elapsed_seconds" -ge "$free_budget_seconds" ]; then
printf 'Skipping OpenCode %s because the anonymous-free combined runtime budget of %ss is exhausted; preserving the remaining retry budget for keyed fallback candidates.\n' \
"$model_candidate" "$free_budget_seconds"
continue
fi
assert_reasoning_effort_for_candidate "$model_candidate"
safe_model="${model_candidate//[\/:]/-}"
prompt_file="${RUNNER_TEMP}/opencode-review-${safe_model}-prompt.md"
Expand All @@ -673,6 +750,12 @@ main() {
"$model_candidate" "$nim_budget_seconds"
break
fi
if is_opencode_free_candidate "$model_candidate" &&
[ "$free_elapsed_seconds" -ge "$free_budget_seconds" ]; then
printf 'Stopping OpenCode %s retries because the anonymous-free combined runtime budget of %ss is exhausted.\n' \
"$model_candidate" "$free_budget_seconds"
break
fi
if [ "$deadline" -gt 0 ] && [ "$now" -ge "$deadline" ]; then
printf 'OpenCode model pool retry deadline elapsed before %s attempt %s/%s.\n' "$model_candidate" "$attempt" "$effective_attempts"
if finish_pool_without_model; then
Expand Down Expand Up @@ -704,6 +787,14 @@ main() {
OPENCODE_RUN_TIMEOUT_SECONDS="$nim_remaining_seconds"
fi
fi
if is_opencode_free_candidate "$model_candidate"; then
free_remaining_seconds=$((free_budget_seconds - free_elapsed_seconds))
if [ "$OPENCODE_RUN_TIMEOUT_SECONDS" -gt "$free_remaining_seconds" ]; then
printf 'OpenCode %s combined anonymous-free budget cap selected %ss instead of %ss so keyed fallback candidates retain retry budget.\n' \
"$model_candidate" "$free_remaining_seconds" "$OPENCODE_RUN_TIMEOUT_SECONDS"
OPENCODE_RUN_TIMEOUT_SECONDS="$free_remaining_seconds"
fi
fi
uncapped_run_timeout="$OPENCODE_RUN_TIMEOUT_SECONDS"
OPENCODE_RUN_TIMEOUT_SECONDS="$(cap_model_run_timeout "$model_candidate" "$OPENCODE_RUN_TIMEOUT_SECONDS")"
if [ "$OPENCODE_RUN_TIMEOUT_SECONDS" -lt "$uncapped_run_timeout" ]; then
Expand All @@ -717,7 +808,7 @@ main() {
agent="$OPENCODE_FIRST_ATTEMPT_AGENT"
fi
run_status=0
nim_attempt_started="$SECONDS"
attempt_started="$SECONDS"
if run_one_model_attempt "$model_candidate" "$attempt" "$effective_attempts" "$agent" "$prompt_file" "$candidate_output_file" "$opencode_json_file" "$opencode_export_file"; then
cp "$candidate_output_file" "$OPENCODE_OUTPUT_FILE"
record_review_model "$model_candidate"
Expand All @@ -726,12 +817,17 @@ main() {
else
run_status=$?
fi
attempt_elapsed=$((SECONDS - attempt_started))
if is_nvidia_nim_candidate "$model_candidate"; then
nim_attempt_elapsed=$((SECONDS - nim_attempt_started))
nim_elapsed_seconds=$((nim_elapsed_seconds + nim_attempt_elapsed))
nim_elapsed_seconds=$((nim_elapsed_seconds + attempt_elapsed))
printf 'OpenCode NVIDIA NIM combined runtime used %ss/%ss after %s attempt %s/%s.\n' \
"$nim_elapsed_seconds" "$nim_budget_seconds" "$model_candidate" "$attempt" "$effective_attempts"
fi
if is_opencode_free_candidate "$model_candidate"; then
free_elapsed_seconds=$((free_elapsed_seconds + attempt_elapsed))
printf 'OpenCode anonymous-free combined runtime used %ss/%ss after %s attempt %s/%s.\n' \
"$free_elapsed_seconds" "$free_budget_seconds" "$model_candidate" "$attempt" "$effective_attempts"
fi
if [ "$run_status" -ne 3 ] && is_credit_exhausted_failure "$opencode_json_file" "${opencode_json_file}.stderr"; then
dead_candidate_reasons[$model_candidate]="provider credits exhausted (HTTP 402 / payment required)"
printf 'OpenCode %s provider credits are exhausted; marking this candidate failed for the rest of the run so retries cannot accrue further spend.\n' "$model_candidate"
Expand Down Expand Up @@ -805,4 +901,6 @@ main() {
done
}

main "$@"
if [ "${BASH_SOURCE[0]}" = "$0" ]; then
main "$@"
fi
Loading
Loading