From 202b35898249dccca81316d1931ce0e86b0fa658 Mon Sep 17 00:00:00 2001 From: Valdis Pornieks Date: Tue, 14 Jul 2026 14:25:03 +0300 Subject: [PATCH 1/2] chore: rename Langfuse env vars to QUALOPS_LANGFUSE_ prefix Use QUALOPS_LANGFUSE_SECRET_KEY/QUALOPS_LANGFUSE_PUBLIC_KEY/QUALOPS_LANGFUSE_BASE_URL instead of the plain LANGFUSE_* names, passed explicitly into the LangfuseSpanProcessor/LangfuseClient constructors rather than relying on the SDK's own env auto-read. This lets an organization set these vars once for every repo running qualops without colliding with a repo's own unprefixed LANGFUSE_* vars used for unrelated Langfuse projects. --- .env.example | 6 +-- .github/workflows/ci.yml | 6 +-- CHANGELOG.md | 1 + evals/README.md | 4 +- evals/src/run-eval.ts | 10 +++-- evals/src/upload-datasets.ts | 10 +++-- src/observability/tracing.ts | 9 +++-- .../tracing-langfuse-processor.spec.ts | 39 +++++++++++++++++++ tests/unit/observability/tracing.spec.ts | 24 ++++++------ 9 files changed, 78 insertions(+), 31 deletions(-) create mode 100644 tests/unit/observability/tracing-langfuse-processor.spec.ts diff --git a/.env.example b/.env.example index a1f23eb6..444d3674 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,6 @@ # AWS_SECRET_ACCESS_KEY=your-secret-key # Langfuse (eval observability) -LANGFUSE_SECRET_KEY="sk-lf-your-personal-secret-key" -LANGFUSE_PUBLIC_KEY="pk-lf-your-personal-public-key" -LANGFUSE_BASE_URL="https://cloud.langfuse.com" +QUALOPS_LANGFUSE_SECRET_KEY="sk-lf-your-personal-secret-key" +QUALOPS_LANGFUSE_PUBLIC_KEY="pk-lf-your-personal-public-key" +QUALOPS_LANGFUSE_BASE_URL="https://cloud.langfuse.com" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4bdb474..e1fdf148 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,9 +194,9 @@ jobs: BASE_REF: ${{ github.base_ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} CLAUDE_CODE_EXECUTABLE: ${{ github.workspace }}/node_modules/@anthropic-ai/claude-agent-sdk-linux-x64/claude - LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} - LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} - LANGFUSE_BASE_URL: ${{ vars.LANGFUSE_BASE_URL }} + QUALOPS_LANGFUSE_PUBLIC_KEY: ${{ secrets.QUALOPS_LANGFUSE_PUBLIC_KEY }} + QUALOPS_LANGFUSE_SECRET_KEY: ${{ secrets.QUALOPS_LANGFUSE_SECRET_KEY }} + QUALOPS_LANGFUSE_BASE_URL: ${{ vars.QUALOPS_LANGFUSE_BASE_URL }} run: | node dist/cli.js all \ --base "origin/$BASE_REF" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index d6f9a7d3..64926438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A/B testing for QualOps config changes: run N config files against the same dataset and compare quality (precision/recall/F1) and cost/latency — for testing per-stage model choices, budgets, prompts, or pipeline modes. New `--config=` eval flag runs against an arbitrary config file (validated: inside repo, `.json`, exists; precedence over `--preset`), so real ship configs are A/B-tested with no copy to drift. `evals/src/run-ab.ts` (TypeScript) takes configs via repeatable `--config=` (same flag name as the qualops CLI), runs the full dataset per arm by default (scope with `--dataset`/`--limit`/`--repeats`, matching a normal eval run), and calls `evals/src/compare-experiments.ts`, which reads run-logs from `evals/logs/` (by path or experiment-label prefix via repeatable `--eval-log=`) and renders a metric × arm table — one column per arm, N arms supported — with a held-or-up vs. regressed verdict for the 2-arm case. Run-log file/entry types are shared from `run-log.ts` (`RunLogFile`, `ItemCompleteEntry`). npm aliases: `eval:ab`, `eval:ab:compare`. Documented under "A/B testing configurations" in `evals/README.md`. ### Changed +- Langfuse env vars renamed to `QUALOPS_LANGFUSE_SECRET_KEY`/`QUALOPS_LANGFUSE_PUBLIC_KEY`/`QUALOPS_LANGFUSE_BASE_URL` so orgs can set them once across repos. - `BASH_TOOL_DESCRIPTION` constant removed; callers use `buildBashToolDescription(root)` directly so the description always reflects the actual workspace root rather than a hardcoded `/workspace/pr`. - Updated all dependencies to their latest stable versions (OpenTelemetry, `openai`, `@openai/agents`, `@types/node`, ESLint + `typescript-eslint`, `jest`/`ts-jest`, `prettier`, `zod`; website: Astro 6→7 + Starlight). Dropped the redundant `@types/diff` stub (diff@9 ships its own types) and pinned transitive `uuid` via `overrides`. Build, lint, type-check, unit tests, and evals all pass. diff --git a/evals/README.md b/evals/README.md index 37113a36..cc6d9d0b 100644 --- a/evals/README.md +++ b/evals/README.md @@ -6,8 +6,8 @@ Evaluates QualOps review quality against annotated code samples using [Langfuse] 1. `.env` in the qualops root with: - `ANTHROPIC_API_KEY` (required) - - `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY` (required) - - `LANGFUSE_BASE_URL` (optional, defaults to `https://cloud.langfuse.com`) + - `QUALOPS_LANGFUSE_SECRET_KEY`, `QUALOPS_LANGFUSE_PUBLIC_KEY` (required) + - `QUALOPS_LANGFUSE_BASE_URL` (optional, defaults to `https://cloud.langfuse.com`) - `OPENAI_API_KEY` (optional, for dual-judge scoring) 2. Upload datasets to Langfuse: diff --git a/evals/src/run-eval.ts b/evals/src/run-eval.ts index 41e2d633..7b7bd5e9 100644 --- a/evals/src/run-eval.ts +++ b/evals/src/run-eval.ts @@ -544,11 +544,13 @@ async function runDataset(langfuse: LangfuseClient, datasetName: string, tracer: } function createLangfuseClient(): { langfuse: LangfuseClient; host: string } { - const secretKey = process.env.LANGFUSE_SECRET_KEY; - const publicKey = process.env.LANGFUSE_PUBLIC_KEY; - const host = process.env.LANGFUSE_BASE_URL || 'https://cloud.langfuse.com'; + const secretKey = process.env.QUALOPS_LANGFUSE_SECRET_KEY; + const publicKey = process.env.QUALOPS_LANGFUSE_PUBLIC_KEY; + const host = process.env.QUALOPS_LANGFUSE_BASE_URL || 'https://cloud.langfuse.com'; if (!secretKey || !publicKey) { - console.error('Error: LANGFUSE_SECRET_KEY and LANGFUSE_PUBLIC_KEY must be set in .env'); + console.error( + 'Error: QUALOPS_LANGFUSE_SECRET_KEY and QUALOPS_LANGFUSE_PUBLIC_KEY must be set in .env' + ); process.exit(1); } const langfuse = new LangfuseClient({ secretKey, publicKey, baseUrl: host }); diff --git a/evals/src/upload-datasets.ts b/evals/src/upload-datasets.ts index 94187523..3f47f5db 100644 --- a/evals/src/upload-datasets.ts +++ b/evals/src/upload-datasets.ts @@ -265,12 +265,14 @@ function resolveCrbRepos(repoArg: string): string[] { } async function main(): Promise { - const langfuseSecretKey = process.env.LANGFUSE_SECRET_KEY; - const langfusePublicKey = process.env.LANGFUSE_PUBLIC_KEY; - const langfuseHost = process.env.LANGFUSE_BASE_URL || 'https://cloud.langfuse.com'; + const langfuseSecretKey = process.env.QUALOPS_LANGFUSE_SECRET_KEY; + const langfusePublicKey = process.env.QUALOPS_LANGFUSE_PUBLIC_KEY; + const langfuseHost = process.env.QUALOPS_LANGFUSE_BASE_URL || 'https://cloud.langfuse.com'; if (!langfuseSecretKey || !langfusePublicKey) { - console.error('Error: LANGFUSE_SECRET_KEY and LANGFUSE_PUBLIC_KEY must be set in .env'); + console.error( + 'Error: QUALOPS_LANGFUSE_SECRET_KEY and QUALOPS_LANGFUSE_PUBLIC_KEY must be set in .env' + ); process.exit(1); } diff --git a/src/observability/tracing.ts b/src/observability/tracing.ts index 0261bb4c..dfa9ef1e 100644 --- a/src/observability/tracing.ts +++ b/src/observability/tracing.ts @@ -14,7 +14,7 @@ let _state: TracingState | null = null; * Initialize OpenTelemetry tracing. * * Auto-detection: - * 1. LANGFUSE_SECRET_KEY + LANGFUSE_PUBLIC_KEY → LangfuseSpanProcessor + * 1. QUALOPS_LANGFUSE_SECRET_KEY + QUALOPS_LANGFUSE_PUBLIC_KEY → LangfuseSpanProcessor * 2. OTEL_EXPORTER_OTLP_ENDPOINT → raw OTLPTraceExporter * 3. Neither → no-op (OTel default) */ @@ -29,7 +29,10 @@ export async function setupTracing(): Promise { return; } - const hasLangfuse = !!(process.env.LANGFUSE_SECRET_KEY && process.env.LANGFUSE_PUBLIC_KEY); + const secretKey = process.env.QUALOPS_LANGFUSE_SECRET_KEY; + const publicKey = process.env.QUALOPS_LANGFUSE_PUBLIC_KEY; + const baseUrl = process.env.QUALOPS_LANGFUSE_BASE_URL; + const hasLangfuse = !!(secretKey && publicKey); const hasOtlp = !!process.env.OTEL_EXPORTER_OTLP_ENDPOINT; if (!hasLangfuse && !hasOtlp) return; @@ -38,7 +41,7 @@ export async function setupTracing(): Promise { if (hasLangfuse) { const { LangfuseSpanProcessor } = await import('@langfuse/otel'); - const processor = new LangfuseSpanProcessor(); + const processor = new LangfuseSpanProcessor({ secretKey, publicKey, baseUrl }); const sdk = new NodeSDK({ spanProcessors: [processor] }); sdk.start(); _state = { sdk, processor }; diff --git a/tests/unit/observability/tracing-langfuse-processor.spec.ts b/tests/unit/observability/tracing-langfuse-processor.spec.ts new file mode 100644 index 00000000..bc9ffe91 --- /dev/null +++ b/tests/unit/observability/tracing-langfuse-processor.spec.ts @@ -0,0 +1,39 @@ +import { shutdownTracing } from '@/observability'; + +const constructorSpy = jest.fn(); + +jest.mock('@langfuse/otel', () => ({ + LangfuseSpanProcessor: class { + constructor(params: unknown) { + constructorSpy(params); + } + async forceFlush() {} + async shutdown() {} + onStart() {} + onEnd() {} + }, +})); + +describe('setupTracing (LangfuseSpanProcessor construction)', () => { + afterEach(async () => { + await shutdownTracing(); + delete process.env.QUALOPS_LANGFUSE_SECRET_KEY; + delete process.env.QUALOPS_LANGFUSE_PUBLIC_KEY; + delete process.env.QUALOPS_LANGFUSE_BASE_URL; + }); + + it('constructs LangfuseSpanProcessor with explicit options read from QUALOPS_LANGFUSE_* env vars', async () => { + process.env.QUALOPS_LANGFUSE_SECRET_KEY = 'sk-test'; + process.env.QUALOPS_LANGFUSE_PUBLIC_KEY = 'pk-test'; + process.env.QUALOPS_LANGFUSE_BASE_URL = 'http://localhost:3000'; + + const { setupTracing } = await import('@/observability'); + await setupTracing(); + + expect(constructorSpy).toHaveBeenCalledWith({ + secretKey: 'sk-test', + publicKey: 'pk-test', + baseUrl: 'http://localhost:3000', + }); + }); +}); diff --git a/tests/unit/observability/tracing.spec.ts b/tests/unit/observability/tracing.spec.ts index 3bafccea..af365efa 100644 --- a/tests/unit/observability/tracing.spec.ts +++ b/tests/unit/observability/tracing.spec.ts @@ -5,9 +5,9 @@ import { setupTracing, getTracer, shutdownTracing, forceFlushTracing } from '@/o describe('tracing', () => { afterEach(async () => { await shutdownTracing(); - delete process.env.LANGFUSE_SECRET_KEY; - delete process.env.LANGFUSE_PUBLIC_KEY; - delete process.env.LANGFUSE_BASE_URL; + delete process.env.QUALOPS_LANGFUSE_SECRET_KEY; + delete process.env.QUALOPS_LANGFUSE_PUBLIC_KEY; + delete process.env.QUALOPS_LANGFUSE_BASE_URL; delete process.env.OTEL_EXPORTER_OTLP_ENDPOINT; }); @@ -21,9 +21,9 @@ describe('tracing', () => { }); it('registers a real TracerProvider when Langfuse keys are set', async () => { - process.env.LANGFUSE_SECRET_KEY = 'sk-test'; - process.env.LANGFUSE_PUBLIC_KEY = 'pk-test'; - process.env.LANGFUSE_BASE_URL = 'http://localhost:3000'; + process.env.QUALOPS_LANGFUSE_SECRET_KEY = 'sk-test'; + process.env.QUALOPS_LANGFUSE_PUBLIC_KEY = 'pk-test'; + process.env.QUALOPS_LANGFUSE_BASE_URL = 'http://localhost:3000'; await setupTracing(); @@ -36,9 +36,9 @@ describe('tracing', () => { }); it('is idempotent — calling twice does not register a second provider', async () => { - process.env.LANGFUSE_SECRET_KEY = 'sk-test'; - process.env.LANGFUSE_PUBLIC_KEY = 'pk-test'; - process.env.LANGFUSE_BASE_URL = 'http://localhost:3000'; + process.env.QUALOPS_LANGFUSE_SECRET_KEY = 'sk-test'; + process.env.QUALOPS_LANGFUSE_PUBLIC_KEY = 'pk-test'; + process.env.QUALOPS_LANGFUSE_BASE_URL = 'http://localhost:3000'; await setupTracing(); const providerAfterFirst = (trace.getTracerProvider() as any).getDelegate(); @@ -53,9 +53,9 @@ describe('tracing', () => { describe('shutdownTracing', () => { it('clears state so a subsequent call is a no-op', async () => { - process.env.LANGFUSE_SECRET_KEY = 'sk-test'; - process.env.LANGFUSE_PUBLIC_KEY = 'pk-test'; - process.env.LANGFUSE_BASE_URL = 'http://localhost:3000'; + process.env.QUALOPS_LANGFUSE_SECRET_KEY = 'sk-test'; + process.env.QUALOPS_LANGFUSE_PUBLIC_KEY = 'pk-test'; + process.env.QUALOPS_LANGFUSE_BASE_URL = 'http://localhost:3000'; await setupTracing(); await shutdownTracing(); From c6662f361222fd6fd18bed868fc97c6546d47287 Mon Sep 17 00:00:00 2001 From: Valdis Pornieks Date: Tue, 14 Jul 2026 15:04:47 +0300 Subject: [PATCH 2/2] fix: fall back to Langfuse SDK default when QUALOPS_LANGFUSE_BASE_URL is empty GitHub Actions sets an unreferenced vars.* to an empty string rather than omitting it, so an org without QUALOPS_LANGFUSE_BASE_URL configured was passing baseUrl: '' into LangfuseSpanProcessor. The SDK treats an explicit empty string as provided rather than falling back to its own default, producing a relative export URL and a hard failure at startup. Treat an empty QUALOPS_LANGFUSE_BASE_URL the same as unset. --- src/observability/tracing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/observability/tracing.ts b/src/observability/tracing.ts index dfa9ef1e..ea4efa4b 100644 --- a/src/observability/tracing.ts +++ b/src/observability/tracing.ts @@ -31,7 +31,7 @@ export async function setupTracing(): Promise { const secretKey = process.env.QUALOPS_LANGFUSE_SECRET_KEY; const publicKey = process.env.QUALOPS_LANGFUSE_PUBLIC_KEY; - const baseUrl = process.env.QUALOPS_LANGFUSE_BASE_URL; + const baseUrl = process.env.QUALOPS_LANGFUSE_BASE_URL || undefined; const hasLangfuse = !!(secretKey && publicKey); const hasOtlp = !!process.env.OTEL_EXPORTER_OTLP_ENDPOINT;