From fc6c717422ffa43e0ae508e7a0a4a7148b82a1d8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 14 Jul 2026 00:10:40 +0900 Subject: [PATCH 1/2] fix(opencode): wait for long package build checks --- .github/workflows/opencode-review.yml | 46 +++++++++++++++++----- scripts/ci/test_strix_quick_gate.sh | 8 ++-- tests/test_opencode_agent_contract.py | 55 +++++++++++++++++---------- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 5a3c11ea9..9dea6b25b 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -1435,10 +1435,10 @@ jobs: # continue-on-error, so it survives the job failing. model_pool_outcome: ${{ steps.opencode_review_model_pool.outputs.review_status }} # Coverage and current-head evidence are prepared before the model pool. - # The model pool has a 60-minute retry budget and its publish gate has an - # 8-minute cap, so 90 minutes releases genuinely hung runners without - # truncating the bounded multi-provider review path. - timeout-minutes: 90 + # The model pool has a 60-minute retry budget and its publish gate can wait + # for long same-head package/GPU checks, so 150 minutes releases genuinely + # hung runners without truncating the bounded multi-provider review path. + timeout-minutes: 150 permissions: actions: read checks: read @@ -3691,9 +3691,9 @@ jobs: && steps.opencode_review_model_pool.outputs.review_status == 'success' && steps.central_review_process_fallback_scope.outputs.eligible == 'true' continue-on-error: true - # Keep the normal peer-check hold short, but leave bounded room for the - # dynamic image-validation extension and review publication overhead. - timeout-minutes: 14 + # Keep the normal peer-check hold short, but leave bounded room for + # dynamic image/package-build extensions and review publication overhead. + timeout-minutes: 34 env: GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }} CHECK_LOOKUP_GH_TOKEN: ${{ github.token }} @@ -3713,6 +3713,7 @@ jobs: CENTRAL_REVIEW_PROCESS_FALLBACK_CHANGED_COUNT: ${{ steps.central_review_process_fallback_scope.outputs.changed_count || '0' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} APPROVAL_CHECK_WAIT_ATTEMPTS: "36" + APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180" APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60" APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10" REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS: "15" @@ -3802,8 +3803,14 @@ jobs: pending_checks_file="$(mktemp)" failed_checks_file="$(mktemp)" attempts="${APPROVAL_CHECK_WAIT_ATTEMPTS:-36}" + slow_build_attempts="${APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS:-180}" slow_image_attempts="${APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS:-60}" attempt=1 + pending_checks_need_slow_build_wait() { + local pending_file="$1" + grep -Eiq -- '^- ([^/]+/)?gpu-build([[:space:](]|:)' "$pending_file" || + grep -Eiq -- '^- ([^/]+/)?build \([^)]*(src-tauri/target/release/bundle|bundle/|\.msi|\.dmg|\.deb|\.appimage|AppImage)' "$pending_file" + } while [ "$attempt" -le "$attempts" ]; do curl_api_read "${api_url}/repos/${GH_REPOSITORY}/commits/${HEAD_SHA}/check-runs?per_page=100" >"$check_runs_file" jq -r "${self_check_filter} @@ -3819,6 +3826,11 @@ jobs: printf '::notice::Extending central fast approval peer-check wait from %s to %s attempts because current-head image validation is still running.\n' "$attempts" "$slow_image_attempts" attempts="$slow_image_attempts" fi + if [ "$attempts" -lt "$slow_build_attempts" ] && + pending_checks_need_slow_build_wait "$pending_checks_file"; then + printf '::notice::Extending central fast approval peer-check wait from %s to %s attempts because current-head package/GPU build checks are still running.\n' "$attempts" "$slow_build_attempts" + attempts="$slow_build_attempts" + fi if [ "$attempt" -lt "$attempts" ]; then printf 'Central fast approval waiting for peer checks (%s/%s):\n' "$attempt" "$attempts" cat "$pending_checks_file" @@ -3983,9 +3995,10 @@ jobs: # keeps GitHub review publication bounded. Failed-check evidence is # collected from logs/SARIF before this point; central review-process # self-repair must not run a second model pass from the publish step. - # The approval gate normally waits about six minutes, with a bounded - # extension for image validation plus API and publication overhead. - timeout-minutes: 16 + # The approval gate normally waits about six minutes, with bounded + # extensions for image validation or package/GPU builds plus API and + # publication overhead. + timeout-minutes: 36 env: GH_TOKEN: ${{ steps.opencode_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} CHECK_LOOKUP_GH_TOKEN: ${{ github.token }} @@ -4029,6 +4042,7 @@ jobs: PR_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.inputs.pr_base_sha }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.inputs.pr_head_sha }} APPROVAL_CHECK_WAIT_ATTEMPTS: "36" + APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180" APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60" APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10" CHECK_LOOKUP_RETRY_ATTEMPTS: "1" @@ -6301,9 +6315,16 @@ jobs: return 1 } + pending_checks_need_slow_build_wait() { + local pending_file="$1" + grep -Eiq -- '^- ([^/]+/)?gpu-build([[:space:](]|:)' "$pending_file" || + grep -Eiq -- '^- ([^/]+/)?build \([^)]*(src-tauri/target/release/bundle|bundle/|\.msi|\.dmg|\.deb|\.appimage|AppImage)' "$pending_file" + } + wait_for_peer_github_checks() { local output_file="$1" local attempts="${APPROVAL_CHECK_WAIT_ATTEMPTS:-36}" + local slow_build_attempts="${APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS:-180}" local slow_image_attempts="${APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS:-60}" local sleep_seconds="${APPROVAL_CHECK_WAIT_SLEEP_SECONDS:-10}" local attempt=1 @@ -6320,6 +6341,11 @@ jobs: printf '::notice::Extending OpenCode peer-check wait from %s to %s attempts because current-head image validation is still running.\n' "$attempts" "$slow_image_attempts" attempts="$slow_image_attempts" fi + if [ "$attempts" -lt "$slow_build_attempts" ] && + pending_checks_need_slow_build_wait "$output_file"; then + printf '::notice::Extending OpenCode peer-check wait from %s to %s attempts because current-head package/GPU build checks are still running.\n' "$attempts" "$slow_build_attempts" + attempts="$slow_build_attempts" + fi if [ "$attempt" -lt "$attempts" ]; then printf 'Waiting for peer GitHub Checks before OpenCode approval (%s/%s):\n' "$attempt" "$attempts" cat "$output_file" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index a4fdafbc1..a7b8092ee 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -591,10 +591,10 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" - assert_file_contains "$workflow_file" 'timeout-minutes: 90' "opencode review target releases stalled review runners within the bounded queue budget" + assert_file_contains "$workflow_file" 'timeout-minutes: 150' "opencode review target releases stalled review runners within the bounded queue budget" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model pool gives multiple candidates a bounded review window while capping stalled model attempts" - assert_file_contains "$workflow_file" 'timeout-minutes: 14' "opencode fast approval publication is bounded around the dynamic image-check wait" + assert_file_contains "$workflow_file" 'timeout-minutes: 34' "opencode fast approval publication is bounded around the dynamic image and package/GPU check wait" assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review advances after a bounded stalled provider attempt" assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" @@ -670,15 +670,17 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'load_selected_review_output()' "opencode approval step has a direct selected-output fallback when the overview comment is stale or invalid" assert_file_contains "$workflow_file" "gate result from Review Overview comment" "opencode approval step distinguishes overview-comment gate results" assert_file_contains "$workflow_file" "gate result from selected OpenCode output" "opencode approval step can recover from an invalid overview by validating the selected successful output" - assert_file_contains "$workflow_file" 'timeout-minutes: 16' "opencode approval step has a bounded wall-clock timeout that covers dynamically extended image checks" + assert_file_contains "$workflow_file" 'timeout-minutes: 36' "opencode approval step has a bounded wall-clock timeout that covers dynamically extended image and package/GPU checks" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "120"' "opencode publish-stage diagnosis is a short best-effort augmentation" assert_file_not_contains "$workflow_file" "rekick_model_pool_on_exhaustion" "opencode publication must not rerun the exhausted model catalog after the model-pool step" assert_file_contains "$workflow_file" "publish stage performs no duplicate model-catalog pass" "opencode publication logs that exhausted model retries are delegated to the scheduler" assert_file_contains "$workflow_file" 'timeout --kill-after=15s "${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}s"' "opencode failed-check diagnosis bounds export so the publication gate cannot hang silently" assert_file_contains "$workflow_file" 'APPROVAL_CHECK_WAIT_ATTEMPTS: "36"' "opencode approval gives slow peer checks a bounded six-minute hold window before scheduler retry" + assert_file_contains "$workflow_file" 'APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180"' "opencode approval dynamically extends its bounded hold for current-head package and GPU builds" assert_file_contains "$workflow_file" 'APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60"' "opencode approval dynamically extends its bounded hold only for current-head image validation" assert_file_contains "$workflow_file" 'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10"' "opencode approval poll cadence keeps peer-check API volume bounded" assert_file_contains "$workflow_file" "current-head image validation is still running" "opencode approval logs why the peer-check wait budget was dynamically extended" + assert_file_contains "$workflow_file" "current-head package/GPU build checks are still running" "opencode approval logs why package/GPU peer-check waits were dynamically extended" assert_file_not_contains "$workflow_file" 'REVIEW_PUBLISH_STEP_TIMEOUT_SECONDS' "opencode review publication relies on the Actions step timeout instead of a background watchdog" assert_file_not_contains "$workflow_file" "PUBLISH_STEP_TIMEOUT" "opencode review publication does not leave orphaned watchdog processes" assert_file_not_contains "$workflow_file" "OPENCODE_PUBLISH_TIMEOUT_WRAPPED" "opencode review publication does not re-exec the runner shell script" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 555d7d824..db11a472a 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -859,7 +859,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): r"Prepare bounded OpenCode review evidence[\s\S]{0,120}timeout-minutes: 12", workflow, ) - assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 90", workflow) + assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 150", workflow) assert "timeout-minutes: 12" in workflow assert re.search( r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 65", workflow @@ -879,16 +879,18 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): workflow, ) assert re.search( - r"Publish central OpenCode fast approval[\s\S]{0,900}timeout-minutes: 14", + r"Publish central OpenCode fast approval[\s\S]{0,900}timeout-minutes: 34", workflow, ) assert re.search( - r"Publish OpenCode review outcome[\s\S]{0,900}timeout-minutes: 16", workflow + r"Publish OpenCode review outcome[\s\S]{0,900}timeout-minutes: 36", workflow ) assert workflow.count('APPROVAL_CHECK_WAIT_ATTEMPTS: "36"') == 2 + assert workflow.count('APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180"') == 2 assert workflow.count('APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60"') == 2 assert 'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10"' in workflow assert workflow.count("current-head image validation is still running") == 2 + assert workflow.count("current-head package/GPU build checks are still running") == 2 assert 'CHECK_LOOKUP_GH_API_TIMEOUT_SECONDS: "15"' in workflow assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "120"' in workflow assert ( @@ -1640,6 +1642,12 @@ def test_peer_check_wait_budget_fits_publication_step_timeouts(): r'APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "(\d+)"', workflow ) ] + slow_build_attempts = [ + int(value) + for value in re.findall( + r'APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "(\d+)"', workflow + ) + ] sleeps = [ int(value) for value in re.findall(r'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "(\d+)"', workflow) @@ -1654,11 +1662,12 @@ def test_peer_check_wait_budget_fits_publication_step_timeouts(): ) assert normal_attempts == [36, 36] + assert slow_build_attempts == [180, 180] assert slow_image_attempts == [60, 60] assert sleeps == [10, 10] assert fast_timeout is not None assert publish_timeout is not None - wait_seconds = (slow_image_attempts[0] - 1) * sleeps[0] + wait_seconds = (max(slow_build_attempts[0], slow_image_attempts[0]) - 1) * sleeps[0] assert int(fast_timeout.group(1)) * 60 - wait_seconds >= 120 assert int(publish_timeout.group(1)) * 60 - wait_seconds >= 240 @@ -1684,23 +1693,29 @@ def test_slow_peer_wait_matches_only_image_validation_checks(): ("- docs image validation: in_progress\n", False, False), ) for candidate, fast_expected, general_expected in probes: - fast_match = ( - subprocess.run( - ["grep", "-Eiq", "--", fast_pattern], - input=candidate, - text=True, - check=False, - ).returncode - == 0 - ) + fast_match = re.search(fast_pattern, candidate, re.IGNORECASE) is not None general_match = ( - subprocess.run( - ["grep", "-Eiq", "--", general_pattern], - input=candidate, - text=True, - check=False, - ).returncode - == 0 + re.search(general_pattern, candidate, re.IGNORECASE) is not None ) assert fast_match is fast_expected, candidate assert general_match is general_expected, candidate + + gpu_pattern = r"^- ([^/]+/)?gpu-build([\s(]|:)" + package_build_pattern = ( + r"^- ([^/]+/)?build \([^)]*" + r"(src-tauri/target/release/bundle|bundle/|\.msi|\.dmg|\.deb|\.appimage|AppImage)" + ) + slow_build_probes = ( + ("- Release/gpu-build (ubuntu-22.04): IN_PROGRESS\n", True), + ("- gpu-build (windows-2022) check run: in_progress\n", True), + ("- Release/build (windows-latest, src-tauri/target/release/bundle/msi/*.msi): IN_PROGRESS\n", True), + ("- build (macos-latest, src-tauri/target/release/bundle/dmg/*.dmg): IN_PROGRESS\n", True), + ("- build (ubuntu-latest, unit tests): IN_PROGRESS\n", False), + ("- docs-build: IN_PROGRESS\n", False), + ) + for candidate, slow_build_expected in slow_build_probes: + slow_build_match = ( + re.search(gpu_pattern, candidate, re.IGNORECASE) is not None + or re.search(package_build_pattern, candidate, re.IGNORECASE) is not None + ) + assert slow_build_match is slow_build_expected, candidate From 3eb9f04948981965400dcb107815ccd978550dfe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 14 Jul 2026 00:23:37 +0900 Subject: [PATCH 2/2] fix(opencode): ignore superseded cancelled check rollups --- .github/workflows/opencode-review.yml | 63 +++++++++++++++++++++++++-- tests/test_opencode_agent_contract.py | 16 +++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 9dea6b25b..164c8f902 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -5757,6 +5757,55 @@ jobs: rm -f "$runs_json" } + collect_current_head_successful_check_run_names() { + local output_file="$1" + + timeout "$(check_lookup_api_timeout_seconds)s" \ + gh api -X GET "repos/${GH_REPOSITORY}/commits/${HEAD_SHA}/check-runs" \ + -f per_page=100 \ + --paginate \ + --slurp | + jq -r ' + [.[].check_runs[]?] + | sort_by((.started_at // .completed_at // .created_at // ""), (.id // 0)) + | group_by(.name // "") + | map(last) + | .[]? + | select((.status // "") == "completed") + | select((.conclusion // "" | ascii_downcase) == "success") + | .name // empty + ' >"$output_file" + } + + filter_superseded_cancelled_rollup_checks() { + local input_file="$1" + local successful_names_file="$2" + local output_file="$3" + + awk ' + BEGIN { + while ((getline name < successful_names_file) > 0) { + successful[name] = 1 + } + } + { + line = $0 + if (line ~ /^- .*: CANCELLED/) { + label = line + sub(/^- /, "", label) + sub(/: CANCELLED.*/, "", label) + name = label + sub(/^.*\//, "", name) + if (successful[name] || successful[label]) { + printf "Ignoring superseded cancelled check rollup: %s\n", line > "/dev/stderr" + next + } + } + print + } + ' successful_names_file="$successful_names_file" "$input_file" >"$output_file" + } + collect_current_head_commit_check_runs() { local output_file="$1" local mode="$2" @@ -5989,10 +6038,12 @@ jobs: local strix_runs_file local commit_check_runs_file local filtered_rollup_file + local successful_check_names_file rollup_file="$(mktemp)" strix_runs_file="$(mktemp)" commit_check_runs_file="$(mktemp)" filtered_rollup_file="$(mktemp)" + successful_check_names_file="$(mktemp)" if ! pr_node_id="$(timeout "$(check_lookup_api_timeout_seconds)s" gh api graphql \ -f owner="$owner" \ -f name="$name" \ @@ -6106,13 +6157,19 @@ jobs: fi filter_superseded_strix_failures "$rollup_file" "$filtered_rollup_file" mv "$filtered_rollup_file" "$rollup_file" + if ! collect_current_head_successful_check_run_names "$successful_check_names_file"; then + rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" "$successful_check_names_file" + return 1 + fi + filter_superseded_cancelled_rollup_checks "$rollup_file" "$successful_check_names_file" "$filtered_rollup_file" + mv "$filtered_rollup_file" "$rollup_file" if ! collect_current_head_strix_workflow_runs "$strix_runs_file" failed; then - rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" + rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" "$successful_check_names_file" return 1 fi if ! collect_current_head_commit_check_runs "$commit_check_runs_file" failed; then - rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" + rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" "$successful_check_names_file" return 1 fi if grep -Fq -- "Strix Security Scan/strix:" "$rollup_file"; then @@ -6120,7 +6177,7 @@ jobs: else cat "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" | sort -u >"$output_file" fi - rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" + rm -f "$rollup_file" "$strix_runs_file" "$commit_check_runs_file" "$filtered_rollup_file" "$successful_check_names_file" } diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index db11a472a..c1e3e7981 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -216,6 +216,22 @@ def test_opencode_bounded_evidence_context_is_resolved_from_event_payload(): assert "GITHUB_ENV" not in step +def test_opencode_ignores_superseded_cancelled_rollup_checks(): + """Do not fail approval on stale cancelled queue entries after same-head success.""" + workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") + + assert "collect_current_head_successful_check_run_names()" in workflow + assert "filter_superseded_cancelled_rollup_checks()" in workflow + assert "Ignoring superseded cancelled check rollup" in workflow + assert 'if (line ~ /^- .*: CANCELLED/)' in workflow + assert 'sub(/^.*\\//, "", name)' in workflow + assert "successful[name] || successful[label]" in workflow + assert ( + 'filter_superseded_cancelled_rollup_checks "$rollup_file" ' + '"$successful_check_names_file" "$filtered_rollup_file"' + ) in workflow + + def test_opencode_target_coverage_materializes_merge_tree_without_checkout_action(): """Avoid pull_request_target action checkouts of untrusted PR refs.""" workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8")