From 1eab7d587018fdb3027847ec8eb84bb6f2e753d0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 13 Jul 2026 21:46:38 +0900 Subject: [PATCH] Give direct-OpenAI Strix scans GitHub Models fallbacks After #524 the first post-migration Strix runs passed only through the backend-unavailable neutral skip: the OpenAI project quota re-tripped (429 insufficient_quota), so every org Strix check went green with no security evidence at all. Direct-OpenAI mode had no fallback models - STRIX_FALLBACK_MODELS was github_models-mode only, and the gate could not route a github_models/* fallback anyway because the single global LLM_API_BASE_FILE/LLM_API_KEY pair cannot serve two providers in one run (a GitHub Models base paired with a non-compatible primary is rejected by design). Mirror the OpenCode model pool: when the primary provider dies, fall through to GitHub Models candidates with per-model credentials. - strix.yml: openai_direct runs get STRIX_FALLBACK_MODELS 'github_models/openai/o3 github_models/openai/gpt-5-chat' and a new "Prepare GitHub Models fallback credentials" step that writes the GitHub Models token and inference endpoint to STRIX_GITHUB_MODELS_KEY_FILE / STRIX_GITHUB_MODELS_API_BASE_FILE (github_models primary mode is unchanged) - strix_quick_gate.sh: resolved_llm_api_base_for_model consults the new base file for github_models/* models when LLM_API_BASE_FILE is absent, and run_strix_once authenticates github_models/* models with the GitHub Models token instead of the direct-OpenAI key; both new inputs go through the same trusted-file validation as the existing key/base files, and the fail-closed error for github_models models without any endpoint stays intact - test_strix_quick_gate.sh: new gate case openai-direct-quota-github-models-fallback-success proves a 429 insufficient_quota primary falls through to openai/o3 with the API base and API key both switched per model (the fake strix asserts the key swap); workflow/gate contract assertions added - strix_required_workflow_smoke.sh: pins the fallback-credentials step and gate support on the CI-required path The rate-limit classification (upstream added insufficient_quota recognition to is_rate_limit_error) makes the quota failure transient, so the existing retry/fallback loop advances to the GitHub Models candidates without loop changes. Co-Authored-By: Claude Fable 5 --- .github/workflows/strix.yml | 27 +++++- PR_GOVERNANCE_AUDIT.md | 13 ++- scripts/ci/strix_quick_gate.sh | 41 ++++++++- scripts/ci/strix_required_workflow_smoke.sh | 2 + scripts/ci/test_strix_quick_gate.sh | 97 +++++++++++++++++++++ 5 files changed, 175 insertions(+), 5 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8ea8aeb6a..8c93519f2 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -506,6 +506,29 @@ jobs: printf '%s' 'https://models.github.ai/inference' > "$llm_api_base_file" echo "LLM_API_BASE_FILE=$llm_api_base_file" >> "$GITHUB_ENV" + - name: Prepare GitHub Models fallback credentials + if: steps.gate.outputs.provider_mode == 'openai_direct' + env: + GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} + run: | + # Direct-OpenAI scans keep GitHub Models candidates as fallbacks, so + # a provider quota outage degrades to a slower model instead of a + # neutral skip with no security evidence. github_models/* fallback + # models read this token and endpoint; the primary keeps its own key. + umask 077 + sanitized="$(printf '%s' "$GITHUB_MODELS_FALLBACK_TOKEN" | tr -d '\r\n')" + trimmed="$(printf '%s' "$sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [ -z "$trimmed" ]; then + echo '::notice::No GitHub Models token available; direct-OpenAI Strix scans run without GitHub Models fallbacks.' + exit 0 + fi + github_models_key_file="$RUNNER_TEMP/github_models_fallback_key.txt" + printf '%s' "$sanitized" > "$github_models_key_file" + echo "STRIX_GITHUB_MODELS_KEY_FILE=$github_models_key_file" >> "$GITHUB_ENV" + github_models_api_base_file="$RUNNER_TEMP/github_models_api_base.txt" + printf '%s' 'https://models.github.ai/inference' > "$github_models_api_base_file" + echo "STRIX_GITHUB_MODELS_API_BASE_FILE=$github_models_api_base_file" >> "$GITHUB_ENV" + - name: Prepare Vertex AI credentials if: steps.gate.outputs.provider_mode == 'vertex_ai' env: @@ -626,7 +649,9 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }} + STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} + STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" NPM_CONFIG_IGNORE_SCRIPTS: "true" diff --git a/PR_GOVERNANCE_AUDIT.md b/PR_GOVERNANCE_AUDIT.md index 141cb3208..f7897c29f 100644 --- a/PR_GOVERNANCE_AUDIT.md +++ b/PR_GOVERNANCE_AUDIT.md @@ -135,10 +135,19 @@ the organization GitHub Models budget cap made LLM-backed security evidence unreliable) to direct OpenAI `gpt-5.6-luna` through the existing `openai_direct` gate path, reusing the organization `OPENAI_API_KEY` secret when the dedicated `STRIX_OPENAI_API_KEY` secret is absent. GitHub Models -`o3`/`gpt-5-chat` fallbacks remain wired for github_models mode only; direct -mode keeps failing closed on provider signals instead of downgrading models. +`o3`/`gpt-5-chat` fallbacks remain wired for github_models mode only. Recording the migration in this audit also routes its PR through the standard small-change review cadence instead of the central-scope fast path. +Follow-up (same day): the first post-migration runs passed only through the +backend-unavailable neutral skip because the OpenAI project quota re-tripped, +which meant green checks with no security evidence. Direct-OpenAI scans now +mirror the OpenCode model pool's fall-through behavior: the same +`github_models/openai/o3 github_models/openai/gpt-5-chat` fallback list used +by github_models mode is wired for openai_direct mode, with the GitHub Models +token and inference endpoint supplied per fallback model +(`STRIX_GITHUB_MODELS_KEY_FILE`/`STRIX_GITHUB_MODELS_API_BASE_FILE`), so a +provider quota outage degrades to a slower scan instead of skipping evidence. +The pinned ban on GPT-4.1-or-weaker Strix evidence stays in force. ## Live Repository Inventory diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 5884785aa..5b2bdebc4 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -36,6 +36,7 @@ DEFAULT_PROVIDER_RAW="${STRIX_LLM_DEFAULT_PROVIDER:-}" # shellcheck disable=SC2034 # consumed indirectly by sourced model helper functions DEFAULT_PROVIDER="" LLM_API_BASE_FILE="${LLM_API_BASE_FILE:-}" +STRIX_GITHUB_MODELS_API_BASE_FILE="${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" STRIX_INPUT_FILE_ROOT="${STRIX_INPUT_FILE_ROOT:-${RUNNER_TEMP:-}}" STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}" STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}" @@ -269,6 +270,27 @@ if [ -z "$LLM_API_KEY" ] && ! is_vertex_model "$NORMALIZED_STRIX_LLM"; then exit 2 fi +# Optional cross-provider fallback credentials: when the primary model runs +# against the direct OpenAI API, github_models/* fallback models still need +# the GitHub Models token and inference endpoint. Both files are optional; +# without them github_models models keep requiring LLM_API_BASE_FILE. +STRIX_GITHUB_MODELS_KEY_FILE="${STRIX_GITHUB_MODELS_KEY_FILE:-}" +if [ -n "$STRIX_GITHUB_MODELS_KEY_FILE" ] && { [ ! -f "$STRIX_GITHUB_MODELS_KEY_FILE" ] || [ -L "$STRIX_GITHUB_MODELS_KEY_FILE" ]; }; then + echo "ERROR: STRIX_GITHUB_MODELS_KEY_FILE must reference a regular file containing the API key." >&2 + exit 2 +fi +if [ -n "$STRIX_GITHUB_MODELS_KEY_FILE" ] && ! STRIX_GITHUB_MODELS_KEY_FILE="$(resolve_trusted_input_file "STRIX_GITHUB_MODELS_KEY_FILE" "$STRIX_GITHUB_MODELS_KEY_FILE")"; then + exit 2 +fi +STRIX_GITHUB_MODELS_KEY="" +if [ -n "$STRIX_GITHUB_MODELS_KEY_FILE" ]; then + STRIX_GITHUB_MODELS_KEY="$(trim_whitespace "$(cat -- "$STRIX_GITHUB_MODELS_KEY_FILE")")" + if [ -z "$STRIX_GITHUB_MODELS_KEY" ]; then + echo "ERROR: STRIX_GITHUB_MODELS_KEY_FILE must contain a non-empty API key." >&2 + exit 2 + fi +fi + require_non_negative_integer() { local value="$1" local label="$2" @@ -2234,7 +2256,17 @@ resolved_llm_api_base_for_model() { return 0 fi - if [ -z "$LLM_API_BASE_FILE" ]; then + local api_base_file="$LLM_API_BASE_FILE" + local api_base_file_name="LLM_API_BASE_FILE" + if [ -z "$api_base_file" ] && is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then + # Cross-provider fallback: a direct-OpenAI primary run keeps its own + # key and no API base, while github_models/* fallback models route + # through the GitHub Models inference endpoint supplied here. + api_base_file="$STRIX_GITHUB_MODELS_API_BASE_FILE" + api_base_file_name="STRIX_GITHUB_MODELS_API_BASE_FILE" + fi + + if [ -z "$api_base_file" ]; then if is_github_models_model "$model"; then echo "ERROR: GitHub Models Strix scans require LLM_API_BASE_FILE to select the GitHub Models inference endpoint." >&2 return 2 @@ -2242,7 +2274,7 @@ resolved_llm_api_base_for_model() { return 0 fi local resolved_llm_api_base_file - if ! resolved_llm_api_base_file="$(resolve_trusted_input_file "LLM_API_BASE_FILE" "$LLM_API_BASE_FILE")"; then + if ! resolved_llm_api_base_file="$(resolve_trusted_input_file "$api_base_file_name" "$api_base_file")"; then return 2 fi @@ -2339,6 +2371,11 @@ run_strix_once() { local child_llm_api_key="" if ! is_vertex_model "$(normalize_model "$model")"; then child_llm_api_key="$LLM_API_KEY" + if is_github_models_model "$(normalize_model "$model")" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then + # Cross-provider fallback: github_models/* models authenticate + # with the GitHub Models token, not the direct-OpenAI key. + child_llm_api_key="$STRIX_GITHUB_MODELS_KEY" + fi fi set -o pipefail set +e diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 28b839d05..57df964a1 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -147,6 +147,8 @@ assert_file_contains "$workflow_file" 'context="strix"' "Strix workflow publishe assert_file_contains "$workflow_file" "Existing current-run Strix success status is already present" "Strix manual follow-up status publisher accepts already-published same-run evidence" assert_file_not_contains "$workflow_file" 'repository: ${{ github.repository }}' "Strix workflow must not checkout target repository with actions/checkout in privileged context" assert_file_not_contains "$workflow_file" 'bash "$TRUSTED_STRIX_GATE_TEST"' "Strix required path must not execute the full long-form gate harness" +assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "Strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" +assert_file_contains "$gate_script" "STRIX_GITHUB_MODELS_KEY_FILE" "Strix gate supports GitHub Models fallback credentials for cross-provider fallback" assert_file_contains "$gate_script" "STRIX_REPO_ROOT" "Strix gate consumes explicit target root" assert_file_contains "$gate_script" "STRIX_REPO_ROOT must reference a regular directory" "Strix gate rejects invalid or symlink target roots" assert_file_contains "$gate_script" "TARGET_PATH_IS_INTERNAL_PR_SCOPE" "Strix gate separates generated PR scopes from user paths" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 373f4f9d6..91e3e8058 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -268,6 +268,10 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "LLM_API_BASE_FILE" "strix workflow passes the GitHub Models API base through a trusted input file" assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "github_models/openai/o3 github_models/openai/gpt-5-chat" "strix workflow keeps GitHub Models fallback on tool-capable OpenAI models without GPT-4.1 downgrade" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives direct-OpenAI scans GitHub Models fallbacks so provider quota outages degrade instead of skipping" + assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" + assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" + assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" assert_file_contains "$workflow_file" '${strix_model#github_models/}' "strix workflow strips manual github_models routing prefix for OpenAI GPT model names before passing model names to LiteLLM" assert_file_contains "$workflow_file" "openai_direct/%s" "strix workflow keeps manual direct OpenAI scans distinct from GitHub Models openai/gpt-* routing" @@ -3038,6 +3042,31 @@ case "${FAKE_STRIX_SCENARIO:?}" in ;; esac ;; + openai-direct-quota-github-models-fallback-success) + case "${STRIX_LLM:-}" in + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected direct-OpenAI key for primary (${LLM_API_KEY:-})" >&2 + exit 15 + fi + echo "Error getting response: Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details.', 'type': 'insufficient_quota', 'code': 'insufficient_quota'}}" + echo "openai.RateLimitError: Error code: 429" + exit 1 + ;; + openai/o3) + if [ "${LLM_API_KEY:-}" != "github-models-fallback-token" ]; then + echo "unexpected GitHub Models key for fallback (${LLM_API_KEY:-})" >&2 + exit 16 + fi + echo "scan ok with GitHub Models fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -5107,6 +5136,12 @@ PY FAKE_STRIX_OUTSIDE_REPORT_DIR="$repo_root_dir/outside-strix-report" ) fi + if [ "$scenario" = "openai-direct-quota-github-models-fallback-success" ]; then + printf '%s' 'https://models.github.ai/inference' >"$tmp_dir/github_models_api_base.txt" + printf '%s' 'github-models-fallback-token' >"$tmp_dir/github_models_key.txt" + env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") + env_cmd+=(STRIX_GITHUB_MODELS_KEY_FILE="$tmp_dir/github_models_key.txt") + fi if [ "$min_fail_severity" = "__UNSET__" ]; then local next_env_cmd=() local env_pair @@ -5374,6 +5409,36 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" ;; + openai-direct-quota-github-models-fallback-success) + run_gate_case "openai-direct-quota-github-models-fallback-success" \ + "openai_direct/gpt-5.6-luna" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-luna|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; gemini-timeout-fallback-success) run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ "gemini/timeout-fallback-primary" \ @@ -11051,6 +11116,38 @@ run_gate_case "github-models-token-limit-fallback-success" \ "" \ "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" +# Direct-OpenAI primary hits a quota/rate-limit error and falls back to a +# GitHub Models candidate, switching both the API base and the API key per +# model (the fake strix asserts the key swap and exits nonzero on a leak). +run_gate_case "openai-direct-quota-github-models-fallback-success" \ + "openai_direct/gpt-5.6-luna" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-luna|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \