From ce5afea86a94d3257389b401847a94e5176c5370 Mon Sep 17 00:00:00 2001 From: Augustine Uzokwe Date: Mon, 1 Jun 2026 15:19:43 +0200 Subject: [PATCH] test: switch integration smoke from Claude to Gemini (#345) The integration smoke still called Anthropic Claude Haiku - a different provider and billing surface than the rest of the pipeline, left over from the ADR-0006 Gemini cutover. Switch it to Gemini so a manual smoke run mirrors the production stack and the eval gate. - _make_llm builds ChatGoogleGenerativeAI (was ChatAnthropic), same timeout/max_retries kwargs the judge already uses. - DEFAULT_INTEGRATION_MODEL tracks agents.config.DEFAULT_MODEL so a model bump flows through automatically. - nightly-integration.yml uses GOOGLE_API_KEY (GOOGLE_API_KEY_CI || GOOGLE_API_KEY, same as the eval gate), Gemini model input default, de-Anthropic-ified descriptions. - Docstring/comments updated; fence-stripping rationale now points at the real Gemini quirk (agents/_llm_utils.strip_json_fence). Stays manual (workflow_dispatch only); no nightly cron re-enabled (RTIA is not live - no recurring spend). Closes #345 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/nightly-integration.yml | 26 ++++++++-------- scripts/run_integration_smoke.py | 37 +++++++++++------------ 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/.github/workflows/nightly-integration.yml b/.github/workflows/nightly-integration.yml index 7f4e4d8..1f28475 100644 --- a/.github/workflows/nightly-integration.yml +++ b/.github/workflows/nightly-integration.yml @@ -1,30 +1,31 @@ name: Integration smoke (manual) # Runs the live agent pipeline against every sample requirement using the -# cheapest dated Anthropic model and a hard token budget. Catches behaviour -# regressions that mocked unit tests cannot see (prompt drift, schema -# mismatches, missing assumptions, accidental cost blowups). +# same Gemini Flash model as the pipeline and eval gate, plus a hard token +# budget. Catches behaviour regressions that mocked unit tests cannot see +# (prompt drift, schema mismatches, missing assumptions, accidental cost +# blowups). # # **Manual trigger only.** The nightly cron was intentionally removed - RTIA # is local-dev today, so the value of catching overnight drift does not yet # justify the recurring (small) API spend. Re-enable the cron when the # Phase 14 API endpoint lands and the pipeline starts serving real requests. -# Trigger on demand via: gh workflow run integration-smoke.yml +# Trigger on demand via: gh workflow run nightly-integration.yml on: workflow_dispatch: inputs: model: - description: "Anthropic model ID (override the script default if needed)" + description: "Gemini model ID (override the script default if needed)" required: false - default: "claude-haiku-4-5-20251001" + default: "gemini-3.5-flash" permissions: contents: read jobs: smoke: - name: Live agent pipeline (all samples, low-cost model) + name: Live agent pipeline (all samples, Gemini Flash) runs-on: ubuntu-latest # Budget should comfortably cover the script's own ~2-3 minute runtime # while still failing fast on a wedged call. @@ -43,12 +44,11 @@ jobs: - name: Run integration smoke env: - # Distinct secret from any human-developer key so usage shows up - # in billing under its own label and can be capped/rotated - # independently. Falls back to the standard key only when present - # to make ad-hoc workflow_dispatch runs from a fork easier. - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_INTEGRATION || secrets.ANTHROPIC_API_KEY }} + # Same Gemini key the eval gate uses: GOOGLE_API_KEY_CI when set + # (so CI/automation spend is billed under its own label and can be + # capped/rotated independently), falling back to GOOGLE_API_KEY. + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY_CI || secrets.GOOGLE_API_KEY }} run: | set -euo pipefail uv run python scripts/run_integration_smoke.py \ - --model "${{ github.event.inputs.model || 'claude-haiku-4-5-20251001' }}" + --model "${{ github.event.inputs.model || 'gemini-3.5-flash' }}" diff --git a/scripts/run_integration_smoke.py b/scripts/run_integration_smoke.py index 67868ae..73c6969 100644 --- a/scripts/run_integration_smoke.py +++ b/scripts/run_integration_smoke.py @@ -17,7 +17,7 @@ Usage: uv run python scripts/run_integration_smoke.py uv run python scripts/run_integration_smoke.py \ - --model claude-haiku-4-5-20251001 \ + --model gemini-3.5-flash \ --budget-input-tokens 50000 \ --budget-output-tokens 5000 @@ -47,10 +47,10 @@ sys.path.insert(0, str(_REPO_ROOT)) from dotenv import load_dotenv -from langchain_anthropic import ChatAnthropic from langchain_core.messages import HumanMessage, SystemMessage +from langchain_google_genai import ChatGoogleGenerativeAI -from agents.config import DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT_SECONDS +from agents.config import DEFAULT_MAX_RETRIES, DEFAULT_MODEL, DEFAULT_TIMEOUT_SECONDS from agents.final_artifact import FinalUserStory from agents.requirements_analyst import _PROMPT_HASH as ANALYST_PROMPT_HASH from agents.requirements_analyst import AnalystOutput @@ -64,15 +64,14 @@ USER_PROMPT_TEMPLATE as ANALYST_USER_PROMPT_TEMPLATE, ) -# Default model for nightly runs: cheapest dated ID currently published. -# (Anthropic dated IDs only exist for Haiku 4.5 and below today; see ADR-0001.) -DEFAULT_INTEGRATION_MODEL = "claude-haiku-4-5-20251001" +# Default model: the same Gemini Flash the pipeline and eval gate use, so a +# manual smoke run mirrors the production stack (ADR-0006 / ADR-0007). Tracks +# agents.config.DEFAULT_MODEL so a model bump flows here automatically. +DEFAULT_INTEGRATION_MODEL = DEFAULT_MODEL -# Token budget for a single nightly run across all samples. Calibrated to be -# roughly 2x the observed Phase 6 baseline (input≈6.9k, output≈0.9k on Opus -# for Analyst alone; adding Story Writer ≈ doubles it). The budget is a -# regression tripwire, not a SLO - bump it deliberately when a real prompt -# change moves the floor. +# Token budget for a single run across all samples. A generous tripwire +# (~2x a typical baseline) to catch a cost blowup, not an SLO - bump it +# deliberately when a real prompt change moves the floor. DEFAULT_BUDGET_INPUT_TOKENS = 50_000 DEFAULT_BUDGET_OUTPUT_TOKENS = 5_000 @@ -107,8 +106,8 @@ def passed(self) -> bool: return not self.failures and self.final_story is not None -def _make_llm(model: str) -> ChatAnthropic: - return ChatAnthropic( +def _make_llm(model: str) -> ChatGoogleGenerativeAI: + return ChatGoogleGenerativeAI( model=model, timeout=DEFAULT_TIMEOUT_SECONDS, max_retries=DEFAULT_MAX_RETRIES, @@ -127,13 +126,13 @@ def _collect_usage(response) -> UsageTelemetry: def _strip_json_fences(raw: str) -> str: - """Strip ```json ... ``` fences that smaller models sometimes wrap output in. + """Strip ```json ... ``` fences the model sometimes wraps output in. - The agent prompts explicitly forbid markdown fences, and Opus respects - that. Haiku/Sonnet are less reliable on this exact instruction, so the - smoke script (which deliberately runs against a cheaper model) strips - them defensively rather than treating a wrapped-but-valid JSON response - as a failure. The pipeline agent path keeps strict parsing. + The agent prompts explicitly forbid markdown fences, but Gemini + occasionally adds them anyway (a known quirk - see + agents/_llm_utils.strip_json_fence, which the real pipeline applies + for the same reason). This helper mirrors that defence so a + wrapped-but-valid JSON response is parsed rather than failed. """ text = raw.strip() if text.startswith("```"):