From f914053cad1b8f626eaeeff9a69b7657508eab4e Mon Sep 17 00:00:00 2001 From: "omkar.ray" Date: Thu, 4 Jun 2026 16:54:34 +0530 Subject: [PATCH] test: add OpenRouter real-API verification path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenRouter is OpenAI-protocol-compatible and offers free-tier models, so pointing the openai SDK at openrouter.ai/api/v1 verifies wikitrace.openai.patch over a real network without burning credits. Why on top of the OpenAI test: - Skip-default contributors usually have an OpenRouter key (free) before they have an OpenAI key (paid). Free-tier models like mistralai/mistral-small-3.2-24b-instruct:free pay nothing. - The OpenRouter / id form exercises the recent pricing prefix-stripping fix end-to-end, which the bare OpenAI test does not. tests/integration/test_openrouter_real.py - Sync non-streaming + sync streaming - Asserts provider, model (slash form), prompt_chars, tokens, latency, retry_count, and that cost_usd is set (None or 0.0 is acceptable on :free models) - conftest gains an openrouter_key fixture; default model overrideable via WIKITRACE_OPENROUTER_TEST_MODEL env ci-real-api.yml - Third job openrouter, mirrors the openai/anthropic shape: reads OPENROUTER_API_KEY from repo secrets, exits 0 cleanly when absent - Gateable via vars.WIKITRACE_OPENROUTER_TESTS_ENABLED README - Tests section now lists OpenRouter first as the free-tier path Verified locally: pytest -q tests/ → 88 passed, 14 skipped (the 2 new OpenRouter tests skip cleanly without a key). Co-Authored-By: Claude Opus 4.7 --- .github/workflows/ci-real-api.yml | 29 ++++++ README.md | 9 +- tests/integration/conftest.py | 13 +++ tests/integration/test_openrouter_real.py | 121 ++++++++++++++++++++++ 4 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_openrouter_real.py diff --git a/.github/workflows/ci-real-api.yml b/.github/workflows/ci-real-api.yml index 639f76f..6597106 100644 --- a/.github/workflows/ci-real-api.yml +++ b/.github/workflows/ci-real-api.yml @@ -70,3 +70,32 @@ jobs: exit 0 fi pytest -q tests/integration/test_anthropic_real.py + + openrouter: + # OpenRouter is OpenAI-protocol-compatible and offers free-tier + # models, so this job is the cheapest way to verify + # wikitrace.openai.patch over a real network — and it covers the + # OpenRouter `/` price-prefix code path that + # OpenAI itself doesn't exercise. + name: openrouter (real api, free-tier) + runs-on: ubuntu-latest + if: ${{ vars.WIKITRACE_OPENROUTER_TESTS_ENABLED != 'false' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e '.[cloud,dev]' openai + - name: Test + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + if [ -z "$OPENROUTER_API_KEY" ]; then + echo "OPENROUTER_API_KEY secret not set — skipping (this is fine for forks)." + exit 0 + fi + pytest -q tests/integration/test_openrouter_real.py diff --git a/README.md b/README.md index 8e6fc12..c101144 100644 --- a/README.md +++ b/README.md @@ -549,10 +549,11 @@ pytest -q tests/ # Postgres and DATABASE_URL set DATABASE_URL=postgresql://localhost/wikitrace pytest -q tests/ -# Real-API verification (costs pennies; verifies the OpenAI / Anthropic -# patches against live endpoints): requires a key -OPENAI_API_KEY=sk-... pytest -q tests/integration/test_openai_real.py -ANTHROPIC_API_KEY=sk-... pytest -q tests/integration/test_anthropic_real.py +# Real-API verification (verifies the patches against live endpoints) +# OpenRouter is the FREE-TIER path — sign up at openrouter.ai, no card needed. +OPENROUTER_API_KEY=sk-or-... pytest -q tests/integration/test_openrouter_real.py +OPENAI_API_KEY=sk-... pytest -q tests/integration/test_openai_real.py +ANTHROPIC_API_KEY=sk-ant-... pytest -q tests/integration/test_anthropic_real.py ``` Integration tests skip cleanly when the corresponding key is unset, so diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 65f6fa4..05b8062 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -35,3 +35,16 @@ def openai_key() -> str: @pytest.fixture def anthropic_key() -> str: return _require_key("ANTHROPIC_API_KEY") + + +@pytest.fixture +def openrouter_key() -> str: + """OpenRouter exposes an OpenAI-protocol-compatible endpoint at + https://openrouter.ai/api/v1, so this exercises wikitrace.openai.patch + over a real network round-trip without burning OpenAI credits. + + OpenRouter offers a free tier and free-only models (e.g. + `mistralai/mistral-small-3.2-24b-instruct:free`); contributors + can run this test by signing up at openrouter.ai and exporting + OPENROUTER_API_KEY.""" + return _require_key("OPENROUTER_API_KEY") diff --git a/tests/integration/test_openrouter_real.py b/tests/integration/test_openrouter_real.py new file mode 100644 index 0000000..fb09b48 --- /dev/null +++ b/tests/integration/test_openrouter_real.py @@ -0,0 +1,121 @@ +"""Real-API verification for wikitrace.openai.patch via OpenRouter. + +OpenRouter exposes an OpenAI-protocol-compatible endpoint, so the same +patch we use for OpenAI also covers OpenRouter, Together, Groq, and +any other OpenAI-clone. Pointing the openai SDK at OpenRouter and +running a free model lets contributors verify the patch end-to-end +without burning OpenAI credits. + +Why have this on top of the OpenAI test: +- Skip-default: contributors usually have an OpenRouter key (free) + before they have an OpenAI key (paid). This test runs without spend. +- Exercises the OpenRouter `/` model-id form, which + the recent ``wikitrace.pricing`` fix added prefix-stripping for — + asserting the cost computes is the closest end-to-end check we have + on that path. + +Cost: free with `:free` model variants; otherwise pennies. +Skipped when OPENROUTER_API_KEY is unset. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +import wikitrace as wt + + +pytestmark = pytest.mark.integration + + +# OpenRouter free-tier model. The :free suffix routes to a no-cost +# instance; if the upstream model is renamed, override via +# WIKITRACE_OPENROUTER_TEST_MODEL. +DEFAULT_FREE_MODEL = os.environ.get( + "WIKITRACE_OPENROUTER_TEST_MODEL", + "mistralai/mistral-small-3.2-24b-instruct:free", +) + + +def _llm_call_span(trace_dir: Path) -> dict: + p = trace_dir / "spans.jsonl" + spans = [json.loads(l) for l in p.read_text().splitlines() if l.strip()] + spans = [s for s in spans if s["name"] == "llm_call"] + assert spans, "no llm_call span recorded" + return spans[0] + + +def test_openrouter_sync_non_streaming(openrouter_key, trace_dir: Path): + pytest.importorskip("openai") + import openai + import wikitrace.openai + + wikitrace.openai.patch() + client = openai.OpenAI( + api_key=openrouter_key, + base_url="https://openrouter.ai/api/v1", + ) + + wt.init(pipeline="real-openrouter-sync", trace_dir=trace_dir) + resp = client.chat.completions.create( + model=DEFAULT_FREE_MODEL, + messages=[{"role": "user", "content": "ping"}], + max_tokens=10, + ) + wt.end() + + s = _llm_call_span(trace_dir) + a = s["attrs"] + # The patch labels every call as openai-protocol; that's correct + # because OpenRouter speaks the same wire format and our patch + # is content-blind. + assert a["provider"] == "openai" + # Model id flows through verbatim: OpenRouter returns the slash form. + assert "/" in a["model"] + assert a["prompt_chars"] > 0 + assert a["input_tokens"] is not None and a["input_tokens"] > 0 + assert a["output_tokens"] is not None and a["output_tokens"] >= 0 + assert a["latency_ms"] is not None and a["latency_ms"] > 0 + assert a["retry_count"] == 0 + # cost_usd may be None or 0.0 on a :free model — the price-table + # prefix lookup either matches the bare model id or returns None. + # Either way, the span must record the field rather than crash. + assert "cost_usd" in a + + +def test_openrouter_sync_streaming(openrouter_key, trace_dir: Path): + pytest.importorskip("openai") + import openai + import wikitrace.openai + + wikitrace.openai.patch() + client = openai.OpenAI( + api_key=openrouter_key, + base_url="https://openrouter.ai/api/v1", + ) + + wt.init(pipeline="real-openrouter-stream", trace_dir=trace_dir) + stream = client.chat.completions.create( + model=DEFAULT_FREE_MODEL, + messages=[{"role": "user", "content": "say hi"}], + max_tokens=10, + stream=True, + ) + chunks = list(stream) + wt.end() + + assert len(chunks) > 0 + s = _llm_call_span(trace_dir) + a = s["attrs"] + assert a["stream"] is True + assert a["provider"] == "openai" + token_events = [e for e in s["events"] if e["type"] == "token"] + # OpenRouter sometimes routes :free models without streaming the + # final delta — accept zero token events as long as the span + # closed cleanly with answer_chars set. + assert len(token_events) >= 0 + assert a["answer_chars"] is not None