diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 191c90451..8f68a8393 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -1438,10 +1438,10 @@ jobs: # continue-on-error, so it survives the job failing. model_pool_outcome: ${{ steps.opencode_review_model_pool.outputs.review_status }} # Coverage and current-head evidence are prepared before the model pool. - # The model pool has a 25-minute retry budget and its publish gate has an - # 8-minute cap, so 45 minutes releases genuinely hung runners without + # The model pool has a 60-minute retry budget and its publish gate has an + # 8-minute cap, so 90 minutes releases genuinely hung runners without # truncating the bounded multi-provider review path. - timeout-minutes: 45 + timeout-minutes: 90 permissions: actions: read checks: read @@ -3178,7 +3178,7 @@ jobs: - name: Run OpenCode PR Review model pool id: opencode_review_model_pool if: needs.coverage-evidence.result == 'success' - timeout-minutes: 12 + timeout-minutes: 65 continue-on-error: true env: STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} @@ -3210,10 +3210,10 @@ jobs: # Keep stale providers from pinning required review jobs for hours. # Adversarial validation needs enough room to read the evidence, but # dynamic cadence and the outer watchdog still bound each current-head run. - OPENCODE_RUN_TIMEOUT_SECONDS: "180" + OPENCODE_RUN_TIMEOUT_SECONDS: "600" OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540" - OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "540" + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600" + OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "3600" # Visit the high-sensitivity candidate catalog once. Per-provider # failures and timeouts are logged before the publish gate evaluates # the current-head result and fails closed when no verdict exists. @@ -3221,14 +3221,14 @@ jobs: OPENCODE_DYNAMIC_REVIEW_CADENCE: "true" OPENCODE_SMALL_CHANGE_FILE_THRESHOLD: "3" OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD: "20" - OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "90" - OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "180" - OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "120" - OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "360" - OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "180" - OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "540" - OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "120" - OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "360" + OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "600" + OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "600" + OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "600" + OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "600" + OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600" # This installation currently reports a 4k request-body limit for # GitHub Models GPT-5 endpoints even though the public catalog is # larger. Keep the exact runtime failure visible without spending a @@ -3237,8 +3237,8 @@ jobs: OPENCODE_DYNAMIC_MAX_CYCLES: "1" CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE: ${{ steps.central_review_process_fallback_scope.outputs.eligible || 'false' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "120" - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "180" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "600" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600" OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" @@ -3259,13 +3259,13 @@ jobs: run: | set -euo pipefail set +e - timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-540}s" \ + timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-3600}s" \ bash "$GITHUB_WORKSPACE/scripts/ci/run_opencode_review_model_pool.sh" pool_status=$? set -e if [ "$pool_status" -eq 124 ] || [ "$pool_status" -eq 137 ] || [ "$pool_status" -eq 143 ]; then printf 'OpenCode model pool exceeded the outer %ss step budget; marking the pool exhausted so current-head evidence fallback can publish a bounded reason instead of blocking the org queue.\n' \ - "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-540}" + "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-3600}" { printf 'review_model=\n' printf 'review_status=exhausted\n' @@ -6481,42 +6481,8 @@ jobs: return 0 fi - clean_evidence_fallback_body="$(printf '%s\n' \ - "## Pull request overview" \ - "" \ - "OpenCode model providers were unavailable for this same-head run, but deterministic current-head evidence is clean: coverage evidence passed, peer GitHub Checks are complete, medium-or-higher code-scanning alerts are clear, mergeability is clean, and reviewer threads are resolved or outdated." \ - "" \ - "## Findings" \ - "" \ - "No blocking findings in deterministic current-head evidence." \ - "" \ - "## Evidence" \ - "" \ - "- Result: APPROVE" \ - "- Reason: MODEL_UNAVAILABLE_CLEAN_EVIDENCE" \ - "- Model-pool outcome: \`${OPENCODE_MODEL_POOL_OUTCOME:-unknown}\`" \ - "- Head SHA: \`${HEAD_SHA}\`" \ - "- Workflow run: ${RUN_ID}" \ - "- Workflow attempt: ${RUN_ATTEMPT}" \ - "- Review provenance: deterministic fallback after model-output unavailability; no adversarial_validation block was fabricated." \ - "" \ - "This fallback does not suppress failed checks, medium-or-higher code-scanning alerts, merge conflicts, unresolved reviewer threads, or failed coverage evidence; any of those conditions still publish REQUEST_CHANGES or leave the approval state unchanged." - )" - printf '::notice::MODEL_OUTPUT_UNAVAILABLE: current-head deterministic evidence is clean for head %s; publishing MODEL_UNAVAILABLE_CLEAN_EVIDENCE approval so provider outage is not treated as a source blocker.\n' "$HEAD_SHA" - create_pull_review "APPROVE" "$clean_evidence_fallback_body" - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - printf '## OpenCode required check satisfied by clean deterministic evidence\n\n' - printf -- '- Result: `MODEL_UNAVAILABLE_CLEAN_EVIDENCE`\n' - printf -- '- Head SHA: `%s`\n' "$HEAD_SHA" - printf -- '- Workflow run: %s\n' "$RUN_ID" - printf -- '- Workflow attempt: %s\n' "$RUN_ATTEMPT" - printf -- '- Model-pool outcome: `%s`\n' "${OPENCODE_MODEL_POOL_OUTCOME:-unknown}" - printf -- '- Reason: model providers were unavailable, while coverage, peer checks, code-scanning alerts, mergeability, and review threads were clean for this exact head.\n' - printf -- '- Review state: APPROVE posted from deterministic current-head evidence; no adversarial_validation block was fabricated.\n\n' - } >>"$GITHUB_STEP_SUMMARY" - fi - return 0 + printf '::notice::MODEL_OUTPUT_UNAVAILABLE: deterministic evidence fallback will not approve %s#%s because model-unavailable approvals are limited to existing same-head real-model approvals or allowlisted central review-process self-repair.\n' "${GH_REPOSITORY:-unknown}" "${PR_NUMBER:-unknown}" + return 1 } same_head_opencode_approval_exists() { diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index ad7977ce3..8ea8aeb6a 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -651,8 +651,8 @@ jobs: # Capture the gate exit code plus its console output. The gate returns # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" - # rate limits, 413 tokens_limit_reached token-cap, connection/warm-up - # failures) that could not complete a scan. A backend outage is CI + # rate limits, OpenAI quota starvation, 413 tokens_limit_reached + # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI # infrastructure noise, not a security finding, so it must not fail # the required check and block merges. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" @@ -674,7 +674,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index a6a0f1c6c..291ef5766 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -626,8 +626,8 @@ main() { budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500}" max_cycles="${OPENCODE_POOL_MAX_CYCLES:-0}" if [ "${CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE:-false}" = "true" ]; then - original_run_timeout="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS:-300}" - budget_seconds="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS:-420}" + original_run_timeout="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS:-600}" + budget_seconds="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS:-3600}" max_cycles="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES:-1}" printf 'Central review-process evidence fallback eligible for scope "%s"; limiting OpenCode model pool to %ss per attempt, %ss total budget, and %s cycle(s) so provider delay is logged before the publish fallback evaluates current-head peer evidence.\n' \ "${CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL:-unsupported}" "$original_run_timeout" "$budget_seconds" "$max_cycles" diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 4f174935a..5884785aa 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2735,6 +2735,11 @@ is_rate_limit_error() { return 0 fi + if grep -Eiq '(exceeded your current quota|insufficient_quota|billing details)' "$STRIX_LOG" && + grep -Eiq "$LLM_PROVIDER_ONLY_REGEX" "$STRIX_LOG"; then + return 0 + fi + # Bare HTTP 429 — require a provider marker so we don't misclassify # target-application rate-limit responses as LLM provider errors. if grep -Eq '(^|[^0-9])429([^0-9]|$)' "$STRIX_LOG" && diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7e846fcae..0043e53a7 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -575,13 +575,15 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "not a generic model-exhaustion message" "opencode review tells models to return concrete missing-evidence findings instead of progress-only output" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" - assert_file_contains "$workflow_file" 'timeout-minutes: 45' "opencode review target releases stalled review runners within the bounded queue budget" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" + assert_file_contains "$workflow_file" 'timeout-minutes: 90' "opencode review target releases stalled review runners within the bounded queue budget" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" - assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode model pool gives multiple candidates a bounded review window while capping stalled model attempts" + assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model pool gives multiple candidates a bounded review window while capping stalled model attempts" assert_file_contains "$workflow_file" 'timeout-minutes: 8' "opencode approval publication is bounded so a stalled review releases queue capacity promptly" assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' "opencode primary review advances after a bounded stalled provider attempt" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review advances after a bounded stalled provider attempt" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "1"' "opencode model pool visits the candidate catalog once without reoccupying the queue" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" @@ -676,13 +678,12 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" "approve_current_head_after_model_unavailable" "opencode general PRs cannot approve without model-backed adversarial evidence" assert_file_contains "$workflow_file" "publish_blockers_after_model_unavailable" "opencode still publishes source-backed blockers after model-output failures" assert_file_contains "$workflow_file" "Current-head model-unavailable evidence fallback candidate" "opencode model-unavailable fallback logs repository, head, and scope evidence" - assert_file_contains "$workflow_file" "MODEL_UNAVAILABLE_CLEAN_EVIDENCE" "model-unavailable path can satisfy the required check only after clean deterministic current-head evidence" + assert_file_contains "$workflow_file" "model-unavailable approvals are limited to existing same-head real-model approvals or allowlisted central review-process self-repair" "model-unavailable path refuses generic deterministic approvals" assert_file_contains "$workflow_file" "same_head_opencode_approval_exists" "model-unavailable path reuses an existing same-head OpenCode approval before publishing fallback approval" assert_file_contains "$workflow_file" "EXISTING_CURRENT_HEAD_APPROVAL" "existing same-head approval fallback logs an explicit required-check result" assert_file_contains "$workflow_file" "no duplicate APPROVE review was posted" "existing same-head approval fallback does not publish a duplicate approval review" assert_file_contains "$workflow_file" "opencode_existing_approval_gate.py" "existing approval reuse requires machine-validated real-model adversarial evidence" - assert_file_contains "$workflow_file" "no adversarial_validation block was fabricated" "deterministic model-unavailable approval must not fabricate model adversarial evidence" - assert_file_contains "$workflow_file" 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' "deterministic model-unavailable approval is explicit and source-evidence gated" + assert_file_not_contains "$workflow_file" 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' "model-unavailable path must not publish generic deterministic approval reviews" assert_file_contains "$workflow_file" "approval still pending" "pending peer checks cannot satisfy the required OpenCode gate without a review" assert_file_contains "$workflow_file" "Cross-repository workflow_dispatch approval hold" "cross-repository pending approvals avoid poisoning the central source-branch check" assert_file_contains "$workflow_file" "CENTRAL_FAST_APPROVAL_ADVERSARIAL_INVALID" "central fast approval revalidates structured adversarial evidence" @@ -726,7 +727,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' "opencode catalog fallback advances after a bounded stalled provider attempt" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback advances after a bounded stalled provider attempt" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review tries DeepSeek V3 before OpenAI fallbacks" @@ -1018,7 +1019,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "collect_failed_check_evidence.sh" "opencode review workflow collects failed check logs and annotations" assert_file_contains "$workflow_file" 'HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.inputs.pr_head_sha }}' "opencode evidence step passes HEAD_SHA to failed-check evidence collection" assert_file_contains "$workflow_file" "FAILED_CHECK_EVIDENCE_ATTEMPTS" "opencode review workflow bounds waiting for peer check failures before model review" - assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode model stage has a bounded multi-provider timeout" + assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model stage has a bounded multi-provider timeout" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation has a bounded peer-check wait timeout" assert_file_contains "$workflow_file" 'FAILED_CHECK_EVIDENCE_ATTEMPTS: "6"' "opencode review workflow keeps pre-model peer-check waiting bounded for required workflow DX" assert_file_contains "$workflow_file" 'FAILED_CHECK_EVIDENCE_SLEEP_SECONDS: "5"' "opencode review workflow retries peer-check evidence without stalling the model stage for Strix-scale durations" @@ -3198,6 +3199,22 @@ case "${FAKE_STRIX_SCENARIO:?}" in ;; esac ;; + openai-primary-quota-fallback-success) + case "${STRIX_LLM:-}" in + openai/quota-primary) + echo "openai.agents: Error streaming response: You exceeded your current quota, please check your plan and billing details." + exit 1 + ;; + openai/fallback-one) + echo "scan ok after quota fallback" + exit 0 + ;; + *) + echo "Error: quota fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 24 + ;; + esac + ;; vertex-primary-429-fallback-success) case "${STRIX_LLM:-}" in vertex_ai/http429-primary) @@ -8398,6 +8415,16 @@ run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback- "vertex_ai/resource-exhausted-primary|vertex_ai/fallback-one" \ "|" +run_gate_case_allow_provider_signal "openai-primary-quota-fallback-success" \ + "openai/quota-primary" \ + "openai/fallback-one openai/fallback-two" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'openai/fallback-one' in [0-9]+s\\." \ + "2" \ + "openai/quota-primary|openai/fallback-one" \ + "|" \ + "openai" + run_gate_case_allow_provider_signal "vertex-primary-429-fallback-success" \ "vertex_ai/http429-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 14e3e61f9..ecdba16cd 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -619,7 +619,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "publish_blockers_after_model_unavailable" in workflow assert 'OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION: "true"' in workflow assert "CENTRAL_FAST_APPROVAL_ADVERSARIAL_INVALID" in workflow - assert "MODEL_UNAVAILABLE_CLEAN_EVIDENCE" in workflow + assert "model-unavailable approvals are limited to existing same-head real-model approvals" in workflow assert '"adversarial_validation"' in model_pool_runner assert "ContextualWisdomLab/.github:ci-review-prompt.md | \\" in workflow assert "ContextualWisdomLab/.github:code-reviewer-prompt.md | \\" in workflow @@ -660,21 +660,23 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "--only-binary=:all: -r requirements-opencode-review-ci-hashes.txt" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL" in workflow - assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "180"' in workflow + assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600"' in workflow assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1"' in workflow assert "Central review-process evidence fallback eligible" in model_pool_runner assert "hash-pinned uv runtime is not installed in the model-pool job" in model_pool_runner assert "provider delay is logged before the publish fallback evaluates current-head peer evidence" in model_pool_runner assert "model pool was intentionally skipped" not in workflow - assert "current-head deterministic evidence is clean" in workflow + assert "current-head deterministic central review-process evidence is clean" in workflow assert 'collect_github_checks_with_retry collect_pending_github_checks "$pending_checks_file"' in workflow current_head_fallback = workflow.split("publish_blockers_after_model_unavailable()", 1)[1].split( "request_changes_for_merge_conflict_if_present()", 1 )[0] assert "wait_for_peer_github_checks" not in current_head_fallback - assert 'if [ "${CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE:-false}" != "true" ]' not in current_head_fallback - assert 'if [ "${GH_REPOSITORY:-}" != "ContextualWisdomLab/.github" ]' not in current_head_fallback + assert "if approve_central_review_process_after_model_unavailable; then" in current_head_fallback + assert "allowlisted central review-process self-repair" in current_head_fallback + assert "clean_evidence_fallback_body" not in current_head_fallback + assert 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' not in workflow assert "collect_open_code_scanning_alerts" in workflow assert ( "CODE_SCANNING_GH_TOKEN: ${{ github.token || secrets.PR_REVIEW_MERGE_TOKEN || " @@ -703,17 +705,17 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert '"## Check outcome"' not in workflow assert "publish REQUEST_CHANGES when coverage-evidence blocker states" in workflow assert re.search(r"Prepare bounded OpenCode review evidence[\s\S]{0,120}timeout-minutes: 12", workflow) - assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 45", workflow) + assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 90", workflow) assert 'timeout-minutes: 12' in workflow - assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 12", workflow) - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "180"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "540"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "540"' in workflow - assert 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-540}s"' in workflow + assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 65", workflow) + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "3600"' in workflow + assert 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-3600}s"' in workflow assert "OpenCode model pool exceeded the outer" in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,280}continue-on-error: true", workflow) @@ -734,21 +736,21 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'github-models/deepseek/deepseek-r1"' ) in workflow assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' in workflow assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "540"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "3600"' in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_DYNAMIC_REVIEW_CADENCE: "true"' in workflow assert 'OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt' in workflow - assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "90"' in workflow - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "180"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow - assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "180"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "540"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow + assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow @@ -1241,8 +1243,8 @@ def test_opencode_jq_filters_do_not_embed_literal_expression_openers(): assert 'contains("$" + "{{")' in workflow -def test_opencode_model_pool_failure_uses_gated_clean_evidence_fallback(): - """A model-pool failure may approve only after all deterministic gates are clean.""" +def test_opencode_model_pool_failure_uses_only_real_or_central_fallback(): + """A model-pool failure may not publish a generic deterministic APPROVE review.""" workflow = Path(".github/workflows/opencode-review.yml").read_text( encoding="utf-8" ) @@ -1262,12 +1264,12 @@ def test_opencode_model_pool_failure_uses_gated_clean_evidence_fallback(): assert 'stop_approval_without_review "MODEL_OUTPUT_UNAVAILABLE" "$body"' in workflow assert "same_head_opencode_approval_exists" in workflow assert "EXISTING_CURRENT_HEAD_APPROVAL" in workflow - assert "MODEL_UNAVAILABLE_CLEAN_EVIDENCE" in workflow - assert "no adversarial_validation block was fabricated" in workflow + assert "allowlisted central review-process self-repair" in workflow + assert "model-unavailable approvals are limited to existing same-head real-model approvals" in workflow assert "no duplicate APPROVE review was posted" in workflow assert 'opencode_existing_approval_gate.py --head "$HEAD_SHA"' in workflow assert "same-head real-model OpenCode approval with passed adversarial evidence" in workflow - assert 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' in workflow + assert 'create_pull_review "APPROVE" "$clean_evidence_fallback_body"' not in workflow model_unavailable_block = re.search( r"if \[ \"\$opencode_review_outcome\" != \"success\" \]; then" r"(?P[\s\S]{0,900})stop_without_review_after_model_unavailable", diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 08d34e4d7..13e5f7403 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -612,6 +612,8 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: workflow = workflow_text("strix.yml") assert "RateLimitError|Too many requests" in workflow + assert "exceeded your current quota" in workflow + assert "billing details" in workflow assert "LLM warm-up failed" in workflow assert "zero_vulnerabilities_signal" not in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow