From 3be60058904fcbdcd62aaa9e8430e2973f094284 Mon Sep 17 00:00:00 2001 From: Kenan Salim Date: Tue, 16 Jun 2026 09:57:05 +0300 Subject: [PATCH 1/4] ci: advisory AI failure-investigation workflow (optional, key-gated) Split out of #1317 so the ANTHROPIC_API_KEY funding/ownership decision doesn't block the real quality gates. This workflow is advisory only (never a merge gate): on a same-repo gate failure it reads the logs + co-located specs and posts a root-cause comment. Needs an Anthropic API key as the ANTHROPIC_API_KEY Actions secret; until that's set (and funded), this PR stays parked. Signed-off-by: Kenan Salim --- .github/workflows/ai-investigate.yml | 102 +++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/ai-investigate.yml diff --git a/.github/workflows/ai-investigate.yml b/.github/workflows/ai-investigate.yml new file mode 100644 index 000000000..83f0ac810 --- /dev/null +++ b/.github/workflows/ai-investigate.yml @@ -0,0 +1,102 @@ +# AI failure investigation — when a gate fails, the AI investigates with FULL +# context, because everything it needs is co-located in the repo: +# tests + PRD + DESIGN + ADRs + schemas + code, per component +# (docs//specs/* and src/**/specs/*, registered in artifacts.toml). +# +# Trigger: any mandatory gate workflow finishing red on a PR branch. +# Output: an investigation comment on the PR — root-cause hypothesis, the +# spec section the failure violates (PRD/DESIGN traceability), suspect files, +# and a proposed fix or test. Advisory, never a gate: humans decide, AI digs. +# +# Requires: ANTHROPIC_API_KEY repo secret. + +name: AI Investigate Failure + +on: + workflow_run: + workflows: + - "Backend Lint & Test" + - "E2E — Bronze to API" + - "Security Gates" + - "Helm Validate" + - "Docs Gates" + - "Nightly Install Test" + - "Release Train" + types: [completed] + +permissions: + contents: read + actions: read + pull-requests: write + issues: write + +jobs: + investigate: + # Only for failures on SAME-REPO branches. A malicious fork cannot trigger + # this (which would run on the default branch with secret access + Bash and + # a head_sha checkout) — closes the workflow_run secret-exfiltration vector. + if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.head_repository.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + # nosemgrep: yaml.github-actions.security.workflow-run-target-code-checkout.workflow-run-target-code-checkout -- mitigated: job gated on head_repository == this repo (same-repo only), persist-credentials:false, no secrets used + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.workflow_run.head_sha }} + persist-credentials: false + + - name: Collect failing job logs + # Untrusted workflow_run.* values are passed via env (shell quotes them), + # never interpolated into the script body — avoids template injection. + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + WF_NAME: ${{ github.event.workflow_run.name }} + WF_BRANCH: ${{ github.event.workflow_run.head_branch }} + WF_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + mkdir -p /tmp/failure + gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --log-failed > /tmp/failure/failed-jobs.log 2>/dev/null || \ + gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" --log \ + | tail -800 > /tmp/failure/failed-jobs.log + { + echo "workflow: $WF_NAME" + echo "branch: $WF_BRANCH" + echo "sha: $WF_SHA" + } > /tmp/failure/meta.txt + + - name: AI investigation (full co-located context) + # Pinned to the supported moving major @v1 per Anthropic's published + # guidance (the v1 tag is the maintained release channel; a frozen SHA + # would miss security fixes). This step runs only on workflow_run after + # a gate already failed, is advisory, and creates no release artifact. + uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + prompt: | + A mandatory CI gate failed. Investigate the root cause using the + full co-located context of this repository. + + Failure metadata: /tmp/failure/meta.txt (workflow, branch, sha) + Failing job logs: /tmp/failure/failed-jobs.log + + Method: + 1. Read the logs; identify the first real failure (not cascades). + 2. Locate the failing code/test/model. Read its CO-LOCATED specs: + the component's specs/PRD.md, specs/DESIGN.md, specs/ADR/*, + specs/schemas/* (see docs/DOCS_MAP.md and + cypilot/config/artifacts.toml for the code↔spec mapping). + 3. Determine: is the code violating the spec, is the test wrong, + or is the spec stale? Cite the exact PRD/DESIGN section. + 4. Identify suspect files/commits (git log on the touched paths). + + Then post ONE comment on the PR for the branch named in + /tmp/failure/meta.txt (use `gh pr comment`) containing: ① root-cause + hypothesis with confidence, ② the spec section it traces to, + ③ suspect files with line refs, ④ a concrete proposed fix or + failing-test reproduction, ⑤ what to check if the hypothesis is + wrong. Be brief and factual. If no open PR exists for the branch, + open an issue titled "CI failure investigation" instead. + claude_args: "--allowedTools Bash,Read,Grep,Glob --max-turns 30" From 356b83199a387dd6d0e535448f251fa13f5361f6 Mon Sep 17 00:00:00 2001 From: Kenan Salim Date: Tue, 16 Jun 2026 14:22:40 +0300 Subject: [PATCH 2/4] ci: skip AI investigation gracefully when ANTHROPIC_API_KEY is absent Review feedback on #1351 (cyberantonz): if the secret is not present the action would error. Gate the work on the key: - job-level env HAS_ANTHROPIC_KEY = (secrets.ANTHROPIC_API_KEY != '') - "Collect failing job logs" and "AI investigation" steps run only when the key is set; otherwise a note step logs the skip and the job passes. The workflow is advisory and never a gate, so a missing key must not fail CI. Signed-off-by: Kenan Salim --- .github/workflows/ai-investigate.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ai-investigate.yml b/.github/workflows/ai-investigate.yml index 83f0ac810..616309867 100644 --- a/.github/workflows/ai-investigate.yml +++ b/.github/workflows/ai-investigate.yml @@ -8,7 +8,8 @@ # spec section the failure violates (PRD/DESIGN traceability), suspect files, # and a proposed fix or test. Advisory, never a gate: humans decide, AI digs. # -# Requires: ANTHROPIC_API_KEY repo secret. +# Optional: ANTHROPIC_API_KEY repo secret. When it is not set, the job skips +# its work and passes cleanly — this workflow is advisory and never a gate. name: AI Investigate Failure @@ -38,6 +39,12 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.head_repository.full_name == github.repository }} runs-on: ubuntu-latest timeout-minutes: 15 + # Job-level env can read `secrets`; step-level `if` then gates on it. The + # ANTHROPIC_API_KEY secret is optional, so when it is absent the heavy steps + # are skipped and the workflow still passes (advisory, never a gate) instead + # of erroring out inside the action with no key. + env: + HAS_ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_API_KEY != '' }} steps: # nosemgrep: yaml.github-actions.security.workflow-run-target-code-checkout.workflow-run-target-code-checkout -- mitigated: job gated on head_repository == this repo (same-repo only), persist-credentials:false, no secrets used - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -45,7 +52,12 @@ jobs: ref: ${{ github.event.workflow_run.head_sha }} persist-credentials: false + - name: Note when skipped (no API key) + if: env.HAS_ANTHROPIC_KEY != 'true' + run: echo "ANTHROPIC_API_KEY not configured — advisory AI investigation skipped; workflow passes." + - name: Collect failing job logs + if: env.HAS_ANTHROPIC_KEY == 'true' # Untrusted workflow_run.* values are passed via env (shell quotes them), # never interpolated into the script body — avoids template injection. env: @@ -68,6 +80,7 @@ jobs: } > /tmp/failure/meta.txt - name: AI investigation (full co-located context) + if: env.HAS_ANTHROPIC_KEY == 'true' # Pinned to the supported moving major @v1 per Anthropic's published # guidance (the v1 tag is the maintained release channel; a frozen SHA # would miss security fixes). This step runs only on workflow_run after From d54a17b52dcf71278470ebed285d6fa68a725d94 Mon Sep 17 00:00:00 2001 From: SharedQA <122366558+SharedQA@users.noreply.github.com> Date: Tue, 23 Jun 2026 12:19:56 +0300 Subject: [PATCH 3/4] ci(ai-investigate): scrub API key from subprocesses, SHA-pin the action, best-effort logs Address review on #1351: - Set CLAUDE_CODE_SUBPROCESS_ENV_SCRUB=1 so claude-code-action's Bash tool can't leak ANTHROPIC_API_KEY to subprocesses (defense-in-depth on the same-repo path; forks are already gated). - SHA-pin anthropics/claude-code-action to the v1 commit per the repo's hash-pin policy (matches actions/checkout), '# v1' for Dependabot bumps. - Make failing-log collection best-effort so a non-zero gh call can't abort this advisory job under 'set -euo pipefail' (which would skip the AI step). Signed-off-by: SharedQA <122366558+SharedQA@users.noreply.github.com> --- .github/workflows/ai-investigate.yml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ai-investigate.yml b/.github/workflows/ai-investigate.yml index 616309867..260418b15 100644 --- a/.github/workflows/ai-investigate.yml +++ b/.github/workflows/ai-investigate.yml @@ -45,6 +45,11 @@ jobs: # of erroring out inside the action with no key. env: HAS_ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_API_KEY != '' }} + # Defense-in-depth: best-effort scrub of ANTHROPIC_API_KEY from the + # subprocesses the action's Bash tool spawns, so a same-repo branch + # can't exfiltrate it via a planted command. The same-repo gate above + # already blocks forks; this hardens the same-repo path. + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" steps: # nosemgrep: yaml.github-actions.security.workflow-run-target-code-checkout.workflow-run-target-code-checkout -- mitigated: job gated on head_repository == this repo (same-repo only), persist-credentials:false, no secrets used - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -69,10 +74,15 @@ jobs: run: | set -euo pipefail mkdir -p /tmp/failure - gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --log-failed > /tmp/failure/failed-jobs.log 2>/dev/null || \ - gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" --log \ - | tail -800 > /tmp/failure/failed-jobs.log + # Best-effort: this is an advisory job, so log retrieval must never + # abort it (under pipefail a failing fallback pipeline would, which + # then skips the AI step via implicit success()). + if ! gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" --log-failed \ + > /tmp/failure/failed-jobs.log 2>/dev/null; then + gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" --log 2>/dev/null \ + | tail -800 > /tmp/failure/failed-jobs.log || true + fi + [ -s /tmp/failure/failed-jobs.log ] || echo "(no logs retrieved)" > /tmp/failure/failed-jobs.log { echo "workflow: $WF_NAME" echo "branch: $WF_BRANCH" @@ -81,11 +91,11 @@ jobs: - name: AI investigation (full co-located context) if: env.HAS_ANTHROPIC_KEY == 'true' - # Pinned to the supported moving major @v1 per Anthropic's published - # guidance (the v1 tag is the maintained release channel; a frozen SHA - # would miss security fixes). This step runs only on workflow_run after - # a gate already failed, is advisory, and creates no release artifact. - uses: anthropics/claude-code-action@v1 + # SHA-pinned to v1 per the repo's hash-pin policy (CI supply-chain + # hardening — same as actions/checkout above); the trailing `# v1` + # documents the channel so Dependabot/Renovate can bump the pin. Runs + # only on workflow_run after a gate failed; advisory, no release artifact. + uses: anthropics/claude-code-action@30544b674398ee15c84819bd87caf8a87e8c7b55 # v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} prompt: | From 17f7d8669515d8d35be53d014d4d5bf7c428fea2 Mon Sep 17 00:00:00 2001 From: SharedQA <122366558+SharedQA@users.noreply.github.com> Date: Tue, 23 Jun 2026 12:28:44 +0300 Subject: [PATCH 4/4] ci(ai-investigate): drop blanket Bash; scope to the exact git/gh commands Public-repo hardening. The agent runs with an API key on CI failures, so a prompt-injection planted in the failing logs/code could abuse a general shell to exfiltrate ANTHROPIC_API_KEY. Replace '--allowedTools Bash' with read tools (Read/Grep/Glob) plus only the commands the prompt actually needs: Bash(git log:*), Bash(gh pr list|view|comment:*), Bash(gh issue create|list:*) No arbitrary shell. Layered with CLAUDE_CODE_SUBPROCESS_ENV_SCRUB + same-repo gate. Signed-off-by: SharedQA <122366558+SharedQA@users.noreply.github.com> --- .github/workflows/ai-investigate.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ai-investigate.yml b/.github/workflows/ai-investigate.yml index 260418b15..b2365f7c0 100644 --- a/.github/workflows/ai-investigate.yml +++ b/.github/workflows/ai-investigate.yml @@ -122,4 +122,9 @@ jobs: failing-test reproduction, ⑤ what to check if the hypothesis is wrong. Be brief and factual. If no open PR exists for the branch, open an issue titled "CI failure investigation" instead. - claude_args: "--allowedTools Bash,Read,Grep,Glob --max-turns 30" + # Public-repo hardening: NO blanket Bash. The agent gets read/analysis + # tools plus only the exact git/gh commands it needs to inspect history + # and post its findings — so a prompt-injection in the failing logs + # can't run an arbitrary command (e.g. curl out ANTHROPIC_API_KEY). + # Layered with CLAUDE_CODE_SUBPROCESS_ENV_SCRUB + the same-repo gate. + claude_args: "--allowedTools Read,Grep,Glob,Bash(git log:*),Bash(gh pr list:*),Bash(gh pr view:*),Bash(gh pr comment:*),Bash(gh issue create:*),Bash(gh issue list:*) --max-turns 30"