From b4c51418b12879877ab568f868b34573e310f503 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 01:18:08 +0000 Subject: [PATCH 01/15] test(e2e): add daily secret-free Playwright suite Adds a browser-level E2E suite that verifies the stack works (not just boots): login, an agent chat that queries ClickHouse via the real MCP server, Langfuse trace creation, and the feedback->Langfuse score path. Runs daily + manual only (never on PRs) so the smoke test stays the PR-time launch gate while this heavier suite catches :latest image drift. A local OpenAI-compatible mock (e2e/mock-llm/server.js) fakes only the inference with adaptive tool-calling (finds the ClickHouse query tool by name pattern, runs SELECT 1, echoes the result). The MCP server, ClickHouse, and Langfuse stay real, so no ANTHROPIC_API_KEY and no token spend while still producing real traces + scores. - docker-compose.e2e.yml: mock-llm service + CONFIG_PATH override - e2e/librechat.e2e.yaml: prod config + MockLLM endpoint - e2e/: Playwright project (config, fail-fast env, Langfuse polling client, auth setup, and librechat/langfuse/roundtrip/scoring specs) - .github/workflows/e2e.yml: schedule + workflow_dispatch, de-duped daily-failure tracking issue Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/e2e.yml | 129 +++++++++++++++++++++++++ docker-compose.e2e.yml | 44 +++++++++ e2e/.gitignore | 5 + e2e/README.md | 57 +++++++++++ e2e/lib/config.ts | 61 ++++++++++++ e2e/lib/langfuse.ts | 71 ++++++++++++++ e2e/lib/librechat.ts | 98 +++++++++++++++++++ e2e/lib/paths.ts | 4 + e2e/librechat.e2e.yaml | 67 +++++++++++++ e2e/mock-llm/server.js | 187 ++++++++++++++++++++++++++++++++++++ e2e/package-lock.json | 109 +++++++++++++++++++++ e2e/package.json | 14 +++ e2e/playwright.config.ts | 39 ++++++++ e2e/setup/auth.setup.ts | 24 +++++ e2e/specs/langfuse.spec.ts | 33 +++++++ e2e/specs/librechat.spec.ts | 38 ++++++++ e2e/specs/roundtrip.spec.ts | 46 +++++++++ e2e/specs/scoring.spec.ts | 24 +++++ e2e/tsconfig.json | 11 +++ 19 files changed, 1061 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 docker-compose.e2e.yml create mode 100644 e2e/.gitignore create mode 100644 e2e/README.md create mode 100644 e2e/lib/config.ts create mode 100644 e2e/lib/langfuse.ts create mode 100644 e2e/lib/librechat.ts create mode 100644 e2e/lib/paths.ts create mode 100644 e2e/librechat.e2e.yaml create mode 100644 e2e/mock-llm/server.js create mode 100644 e2e/package-lock.json create mode 100644 e2e/package.json create mode 100644 e2e/playwright.config.ts create mode 100644 e2e/setup/auth.setup.ts create mode 100644 e2e/specs/langfuse.spec.ts create mode 100644 e2e/specs/librechat.spec.ts create mode 100644 e2e/specs/roundtrip.spec.ts create mode 100644 e2e/specs/scoring.spec.ts create mode 100644 e2e/tsconfig.json diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..6a10b80 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,129 @@ +name: E2E tests + +# Heavy, browser-level suite. It runs DAILY + on manual dispatch only — never on +# PRs. The smoke test (smoke-test.yml) stays the PR-time launch gate; this suite +# catches `:latest` image drift and functional regressions on a schedule. +on: + schedule: + - cron: '37 4 * * *' # daily ~04:37 UTC (offset from the smoke test's 13:00) + workflow_dispatch: + +# Avoid piling up runs on the same ref. +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Generate .env + run: bash scripts/generate-env.sh + + # --wait blocks until every healthchecked service is healthy (mock-llm, + # ClickHouse MCP, LibreChat, Langfuse, ...) and fails if any is unhealthy. + - name: Launch stack (prod + e2e override) and wait for health + run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait --wait-timeout 600 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: e2e/package-lock.json + + - name: Install Playwright and browser + working-directory: e2e + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Run E2E tests + working-directory: e2e + run: npx playwright test + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: e2e/playwright-report + retention-days: 14 + + - name: Dump diagnostics on failure + if: failure() + run: | + docker compose -f docker-compose.yml -f docker-compose.e2e.yml ps -a + docker compose -f docker-compose.yml -f docker-compose.e2e.yml logs --no-color --tail=200 + + - name: Tear down + if: always() + run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml down -v + + # On a failed DAILY run, open (or comment on) a de-duped tracking issue. Manual + # (workflow_dispatch) failures surface directly to the person who triggered + # them, so they don't need an issue. Mirrors smoke-test.yml's report job. + report-daily-failure: + needs: e2e + if: contains(fromJSON('["failure", "cancelled"]'), needs.e2e.result) && github.event_name == 'schedule' + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open or update tracking issue + uses: actions/github-script@v7 + with: + script: | + const label = 'e2e-test-failure'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `The daily E2E test suite failed.\n\nRun: ${runUrl}`; + + // Ensure the tracking label exists so the de-dupe search is reliable + // in a repo that has never had it. + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + }); + } catch (e) { + if (e.status !== 404) throw e; + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + color: 'd73a4a', + }); + } + + const existing = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: label, + }); + + // listForRepo returns PRs as well as issues; exclude PRs so we never + // comment on a mislabeled pull request. + const openIssue = existing.data.find(i => !i.pull_request); + + if (openIssue) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: openIssue.number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: 'Daily E2E test suite failed', + body, + labels: [label], + }); + } diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..59a4efb --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,44 @@ +# E2E override — layered on top of docker-compose.yml, ONLY for the daily/manual +# Playwright suite. Production config is untouched: this file adds the secret-free +# mock inference service and repoints LibreChat at an E2E-only config so a chat +# exercises the REAL MCP server, ClickHouse, and Langfuse without any API key. +# +# docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait +# +# Why a config override instead of editing librechat.yaml: the mock endpoint must +# never leak into a real deployment. CONFIG_PATH points LibreChat at a NEW mount +# path (/app/librechat.e2e.yaml) so it can't collide with the prod bind mount. + +services: + mock-llm: + image: node:22-alpine + restart: always + working_dir: /app + command: ["node", "server.js"] + environment: + - PORT=8080 + - MOCK_MODEL=mock-model + volumes: + - type: bind + source: ./e2e/mock-llm/server.js + target: /app/server.js + read_only: true + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:8080/v1/models || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 3s + + librechat: + depends_on: + mock-llm: + condition: service_healthy + environment: + # Load the E2E config (adds the MockLLM endpoint) from its own mount path. + - CONFIG_PATH=/app/librechat.e2e.yaml + volumes: + - type: bind + source: ./e2e/librechat.e2e.yaml + target: /app/librechat.e2e.yaml + read_only: true diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000..e8c9224 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +playwright-report/ +test-results/ +blob-report/ +playwright/.auth/ diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..46e600f --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,57 @@ +# E2E tests + +Browser-level Playwright tests that verify the stack actually **works** — not +just that it boots (that's the job of the smoke test in +`.github/workflows/smoke-test.yml`). They cover the real user flows: login, an +agent chat that queries **ClickHouse via the real MCP server**, **Langfuse trace +creation**, and the **feedback → Langfuse score** path. + +Runs **daily + manual only** (`.github/workflows/e2e.yml`), never on PRs. + +## Secret-free by design + +A local OpenAI-compatible **mock** (`mock-llm/server.js`) fakes only the +*inference*. The MCP server, ClickHouse, and Langfuse stay **real**, so no +`ANTHROPIC_API_KEY`, no token spend, and no frontier-model flakiness — while +still producing real Langfuse traces (a generation observation plus real +MCP → ClickHouse tool spans) and exercising scoring. + +The mock uses **adaptive tool-calling**: it inspects the request's `tools[]`, +finds the ClickHouse query tool by name *pattern* (never a hardcoded name, which +LibreChat rewrites), emits a `tool_calls` delta running `SELECT 1`, then — once +LibreChat sends the tool result back — emits a final answer echoing it. + +## Layout + +| Path | Purpose | +| --- | --- | +| `mock-llm/server.js` | OpenAI-compatible mock inference server (Node built-ins only). | +| `librechat.e2e.yaml` | LibreChat config = prod config + the `MockLLM` endpoint. Loaded via `CONFIG_PATH`. | +| `playwright.config.ts` | Chromium, `baseURL` `http://localhost:3080`, `setup` auth project, no `webServer`. | +| `lib/config.ts` | Explicit, fail-fast config resolved from the repo-root `.env`. | +| `lib/langfuse.ts` | Langfuse public-API client that **polls** (ingestion is async). | +| `lib/librechat.ts` | Shared chat-driving helpers. | +| `setup/auth.setup.ts` | Logs in once via `POST /api/auth/login`, saves storage state. | +| `specs/*.spec.ts` | `librechat`, `langfuse`, `roundtrip`, `scoring`. | + +## Run locally + +The suite drives an already-running stack (no `webServer`): bring it up first. + +```bash +cd .. # repo root +bash scripts/generate-env.sh +docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait --wait-timeout 600 +cd e2e && npm ci && npx playwright install --with-deps chromium +npx playwright test +cd .. && docker compose -f docker-compose.yml -f docker-compose.e2e.yml down -v +``` + +## Confirming selectors + +The endpoint/MCP-picker and 👍/👎 selectors are best-effort (LibreChat markup +drifts). Confirm/refresh them against the live UI with: + +```bash +npx playwright codegen --test-id-attribute=data-testid http://localhost:3080/c/new +``` diff --git a/e2e/lib/config.ts b/e2e/lib/config.ts new file mode 100644 index 0000000..8a832a7 --- /dev/null +++ b/e2e/lib/config.ts @@ -0,0 +1,61 @@ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +/** + * One source of truth for E2E config, resolved explicitly and fail-fast. + * + * Values come from the generated repo-root `.env` (written by + * scripts/generate-env.sh), with `process.env` taking precedence so a run can + * be pointed at a remote stack without editing files. A required value that is + * missing throws at load time — never a silent default that hangs a test later. + * + * Note the host/container split: `.env`'s LANGFUSE_BASE_URL is the in-container + * URL (http://langfuse-web:3000). Playwright runs on the host, so it must reach + * Langfuse and LibreChat via their published localhost ports, not the compose + * service names. Hence the dedicated *_HOST_URL knobs below. + */ + +const ENV_FILE = resolve(__dirname, "../../.env"); + +const parseEnvFile = (path: string): Record => { + if (!existsSync(path)) return {}; + const out: Record = {}; + for (const line of readFileSync(path, "utf8").split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq === -1) continue; + out[trimmed.slice(0, eq).trim()] = trimmed.slice(eq + 1).trim(); + } + return out; +}; + +const fileEnv = parseEnvFile(ENV_FILE); + +/** process.env wins over the .env file; both missing with no default throws. */ +const required = (key: string, fallback?: string): string => { + const value = process.env[key] ?? fileEnv[key] ?? fallback; + if (value === undefined || value === "") { + throw new Error( + `Missing required E2E config '${key}'. Set it in the environment or run scripts/generate-env.sh to write ${ENV_FILE}.`, + ); + } + return value; +}; + +export const config = { + librechatBaseUrl: required("LIBRECHAT_HOST_URL", "http://localhost:3080"), + langfuseBaseUrl: required("LANGFUSE_HOST_URL", "http://localhost:3000"), + + login: { + email: required("LIBRECHAT_USER_EMAIL"), + password: required("LIBRECHAT_USER_PASSWORD"), + }, + + langfuse: { + publicKey: required("LANGFUSE_INIT_PROJECT_PUBLIC_KEY"), + secretKey: required("LANGFUSE_INIT_PROJECT_SECRET_KEY"), + projectId: required("LANGFUSE_INIT_PROJECT_ID"), + projectName: required("LANGFUSE_INIT_PROJECT_NAME", "Default Project"), + }, +} as const; diff --git a/e2e/lib/langfuse.ts b/e2e/lib/langfuse.ts new file mode 100644 index 0000000..50ad614 --- /dev/null +++ b/e2e/lib/langfuse.ts @@ -0,0 +1,71 @@ +import { config } from "./config"; + +/** + * Thin client for the Langfuse public API, used to assert that a chat produced + * real observability data. Auth is HTTP Basic with base64(publicKey:secretKey), + * exactly as Langfuse's public API expects. + * + * Ingestion is ASYNCHRONOUS — traces and scores land in ClickHouse seconds after + * the HTTP round-trip finishes. Every read here therefore POLLS to a deadline + * instead of asserting once; a single fetch would be racy and flaky by design. + */ + +const authHeader = "Basic " + Buffer.from(`${config.langfuse.publicKey}:${config.langfuse.secretKey}`).toString("base64"); + +type Trace = { id: string; name?: string; timestamp?: string }; +type Observation = { id: string; type?: string; name?: string }; +type Score = { id: string; name?: string; value?: number; stringValue?: string }; + +const get = async (path: string): Promise => { + const res = await fetch(`${config.langfuseBaseUrl}/api/public${path}`, { + headers: { Authorization: authHeader, Accept: "application/json" }, + }); + if (!res.ok) throw new Error(`Langfuse GET ${path} -> HTTP ${res.status}`); + return (await res.json()) as T; +}; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * Poll `read` until `done` accepts its result or the deadline passes. + * Throws with the last-seen value on timeout so failures are diagnosable. + */ +const poll = async (read: () => Promise, done: (value: T) => boolean, label: string, timeoutMs = 60_000, intervalMs = 2_000): Promise => { + const deadline = Date.now() + timeoutMs; + let last: T | undefined; + for (;;) { + last = await read().catch(() => last as T); + if (last !== undefined && done(last)) return last; + if (Date.now() >= deadline) throw new Error(`Timed out after ${timeoutMs}ms waiting for ${label}. Last value: ${JSON.stringify(last)}`); + await sleep(intervalMs); + } +}; + +/** Wait for a trace created at/after `sinceIso`, optionally name-filtered. */ +export const waitForTrace = (sinceIso: string, opts: { timeoutMs?: number } = {}): Promise => + poll( + () => get<{ data: Trace[] }>(`/traces?limit=50&fromTimestamp=${encodeURIComponent(sinceIso)}`).then((r) => r.data), + (traces) => traces.length > 0, + `a Langfuse trace since ${sinceIso}`, + opts.timeoutMs, + ).then((traces) => traces[0]); + +/** Wait until a trace's observations include one matching `predicate`. */ +export const waitForObservation = (traceId: string, predicate: (o: Observation) => boolean, label: string, opts: { timeoutMs?: number } = {}): Promise => + poll( + () => get<{ data: Observation[] }>(`/observations?traceId=${encodeURIComponent(traceId)}&limit=100`).then((r) => r.data), + (obs) => obs.some(predicate), + `observation (${label}) on trace ${traceId}`, + opts.timeoutMs, + ).then((obs) => obs.find(predicate)!); + +/** Wait until at least one score is attached to `traceId`. */ +export const waitForScore = (traceId: string, opts: { timeoutMs?: number } = {}): Promise => + poll( + () => get<{ data: Score[] }>(`/scores?traceId=${encodeURIComponent(traceId)}&limit=50`).then((r) => r.data), + (scores) => scores.length > 0, + `a score on trace ${traceId}`, + opts.timeoutMs, + ).then((scores) => scores[0]); + +export type { Trace, Observation, Score }; diff --git a/e2e/lib/librechat.ts b/e2e/lib/librechat.ts new file mode 100644 index 0000000..7848f7d --- /dev/null +++ b/e2e/lib/librechat.ts @@ -0,0 +1,98 @@ +import { expect, type Page } from "@playwright/test"; + +/** + * Shared chat helpers for the round-trip and scoring specs. + * + * Selector strategy: the message composer testids (`text-input`, `send-button`, + * `messages-view`) are stable LibreChat contracts and are asserted strictly. The + * endpoint/MCP pickers have less stable markup, so selecting them is best-effort + * (tolerant of label/markup drift) — if a picker interaction silently misses, + * the mock never runs the tool and the caller's strict outcome assertion (reply + * echoes `SELECT 1`, tool card shown, Langfuse spans) fails loudly instead. + * + * Confirm/refresh the picker selectors with: + * npx playwright codegen --test-id-attribute=data-testid http://localhost:3080/c/new + */ + +const MOCK_ENDPOINT = "MockLLM"; +const MCP_SERVER = "ClickHouse-Local"; + +/** Best-effort click of the first locator that becomes visible; never throws. */ +const tryClick = async (page: Page, candidates: ReturnType[], timeoutMs = 4_000): Promise => { + for (const locator of candidates) { + try { + await locator.first().waitFor({ state: "visible", timeout: timeoutMs }); + await locator.first().click(); + return true; + } catch { + /* try the next candidate */ + } + } + return false; +}; + +/** Point the new conversation at the secret-free MockLLM endpoint. */ +export const selectMockEndpoint = async (page: Page): Promise => { + await tryClick(page, [ + page.getByTestId("endpoint-menu-button"), + page.getByRole("button", { name: /select.*(model|endpoint)/i }), + ]); + await tryClick(page, [ + page.getByRole("option", { name: MOCK_ENDPOINT }), + page.getByText(MOCK_ENDPOINT, { exact: false }), + ]); +}; + +/** Enable the ClickHouse-Local MCP server for the conversation. */ +export const enableClickHouseMcp = async (page: Page): Promise => { + await tryClick(page, [ + page.getByTestId("mcp-select"), + page.getByRole("button", { name: /mcp/i }), + page.getByRole("button", { name: /tools/i }), + ]); + await tryClick(page, [ + page.getByRole("option", { name: MCP_SERVER }), + page.getByRole("menuitemcheckbox", { name: MCP_SERVER }), + page.getByText(MCP_SERVER, { exact: false }), + ]); + // Close the picker so it doesn't overlay the composer. + await page.keyboard.press("Escape").catch(() => {}); +}; + +/** Type a prompt and send it. */ +export const sendPrompt = async (page: Page, prompt: string): Promise => { + const input = page.getByTestId("text-input"); + await expect(input).toBeVisible(); + await input.click(); + await input.fill(prompt); + await page.getByTestId("send-button").click(); +}; + +/** + * Give thumbs-up feedback on the latest assistant message. The controls appear + * on hover, so we hover the messages view first. Best-effort selection (see the + * strategy note above): if it misses, the caller's score assertion fails loudly. + */ +export const giveThumbsUp = async (page: Page): Promise => { + const messages = page.getByTestId("messages-view"); + await messages.hover().catch(() => {}); + await tryClick(page, [ + page.getByTestId("good-response-button"), + page.getByRole("button", { name: /thumbs.?up|good response|helpful/i }), + page.locator('button[aria-label*="thumb" i]').first(), + ]); +}; + +/** + * Full drive: open a new chat wired to MockLLM + ClickHouse-Local, send `prompt`, + * and wait for the assistant's reply to render in the messages view. + */ +export const runMockChat = async (page: Page, prompt: string): Promise => { + await page.goto("/c/new"); + await expect(page.getByTestId("text-input")).toBeVisible(); + await selectMockEndpoint(page); + await enableClickHouseMcp(page); + await sendPrompt(page, prompt); + // The final answer always contains the query text the mock echoes back. + await expect(page.getByTestId("messages-view")).toContainText("SELECT 1", { timeout: 60_000 }); +}; diff --git a/e2e/lib/paths.ts b/e2e/lib/paths.ts new file mode 100644 index 0000000..f8991e2 --- /dev/null +++ b/e2e/lib/paths.ts @@ -0,0 +1,4 @@ +import { resolve } from "node:path"; + +/** Where auth.setup.ts writes the logged-in storage state that specs reuse. */ +export const STORAGE_STATE = resolve(__dirname, "../playwright/.auth/user.json"); diff --git a/e2e/librechat.e2e.yaml b/e2e/librechat.e2e.yaml new file mode 100644 index 0000000..a1c1a27 --- /dev/null +++ b/e2e/librechat.e2e.yaml @@ -0,0 +1,67 @@ +# LibreChat config for the E2E suite ONLY (loaded via CONFIG_PATH by +# docker-compose.e2e.yml). It is librechat.yaml plus a single secret-free +# "MockLLM" custom endpoint pointed at the mock inference server. Everything +# else — the agents endpoint, MCP-server access, and the ClickHouse-Local MCP +# server — is retained verbatim so a chat drives the REAL MCP -> ClickHouse path +# and produces REAL Langfuse traces. +# +# Keep this in sync with librechat.yaml when the shared sections change. + +version: 1.3.1 + +cache: true + +interface: + fileCitations: true + mcpServers: + use: true # required: the round-trip test drives an MCP tool call + share: false + create: true + public: false + +endpoints: + # Secret-free mock inference. `fetch: false` means LibreChat trusts the static + # model list below instead of calling GET /v1/models at startup. + custom: + - name: "MockLLM" + apiKey: "sk-mock-e2e" # mock ignores auth; kept non-empty so LibreChat loads the endpoint + baseURL: "http://mock-llm:8080/v1" + models: + default: + - "mock-model" + fetch: false + titleConvo: false + modelDisplayLabel: "MockLLM" + + agents: + capabilities: + - programmatic_tools + - deferred_tools + - execute_code + - web_search + - artifacts + - actions + - context + - tools + - chain + - ocr + - skills + - subagents + +mcpSettings: + allowedDomains: + - "http://clickhouse-mcp:8000" + - "https://mcp.clickhouse.cloud/mcp" + - "host.docker.internal" + +mcpServers: + ClickHouse-Local: + type: streamable-http + url: http://clickhouse-mcp:8000/mcp + timeout: 60000 # 1 minute timeout + headers: + Authorization: "Bearer ${CLICKHOUSE_MCP_AUTH_TOKEN}" + ClickHouse-Cloud: + type: streamable-http + url: https://mcp.clickhouse.cloud/mcp + timeout: 60000 # 1 minute timeout diff --git a/e2e/mock-llm/server.js b/e2e/mock-llm/server.js new file mode 100644 index 0000000..6d0b535 --- /dev/null +++ b/e2e/mock-llm/server.js @@ -0,0 +1,187 @@ +// OpenAI-compatible mock inference server for the E2E stack. +// +// Purpose: fake ONLY the LLM inference so the heavy E2E suite needs no +// ANTHROPIC_API_KEY, spends no tokens, and never flakes on a frontier model. +// The MCP server, ClickHouse, and Langfuse stay real, so a chat driven by this +// mock still produces a real Langfuse generation observation and real +// MCP -> ClickHouse tool spans. +// +// Two endpoints, both under /v1 to match the OpenAI wire format LibreChat's +// custom endpoints speak: +// GET /v1/models -> advertise the single mock model +// POST /v1/chat/completions -> adaptive tool-calling completion +// +// Adaptive tool-calling: on the first turn we inspect the request's tools[], +// find the ClickHouse "run a query" tool by NAME PATTERN (never a hardcoded +// name — LibreChat rewrites MCP tool names, so a pattern survives that), and +// emit a tool_call that runs `SELECT 1`. LibreChat executes it against the real +// MCP server and sends the result back as a tool message; on that second turn +// we emit a final text answer that echoes the tool result so the browser test +// can assert the round-trip end to end. +// +// Built-ins only (node:http) — this file is mounted into a bare node image with +// no npm install step. + +const http = require("node:http"); + +const PORT = Number(process.env.PORT || 8080); +const MODEL = process.env.MOCK_MODEL || "mock-model"; + +// The query we ask the ClickHouse MCP tool to run. `SELECT 1` returns the single +// value 1, which the round-trip spec asserts on. Kept as a constant so the tool +// arguments and the echoed answer can never drift apart. +const QUERY = "SELECT 1"; + +// Matches the ClickHouse MCP "run a SELECT query" tool regardless of the exact +// name LibreChat exposes it under (e.g. `run_select_query`, or a namespaced +// variant). Ordered from most to least specific; the first match wins. +const QUERY_TOOL_PATTERNS = [/select.*quer/i, /run.*quer/i, /quer/i, /\bsql\b/i]; + +/** Find the argument key a query tool expects (usually `query`), else "query". */ +const queryArgKey = (tool) => { + const props = tool?.function?.parameters?.properties; + if (props && !props.query) { + const named = Object.keys(props).find((k) => /quer|sql/i.test(k)); + if (named) return named; + } + return "query"; +}; + +/** Pick the ClickHouse query tool from an OpenAI tools[] array, or null. */ +const findQueryTool = (tools) => { + const fns = (tools || []).filter((t) => t?.type === "function" && t.function?.name); + for (const pattern of QUERY_TOOL_PATTERNS) { + const hit = fns.find((t) => pattern.test(t.function.name)); + if (hit) return hit; + } + return null; +}; + +/** True once LibreChat has run a tool and sent its result back to us. */ +const hasToolResult = (messages) => (messages || []).some((m) => m?.role === "tool"); + +/** Flatten OpenAI message content (string or content-part array) to text. */ +const messageText = (message) => { + const { content } = message || {}; + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content.map((p) => (typeof p === "string" ? p : p?.text || "")).join(""); + } + return ""; +}; + +/** The most recent tool result text, for the final answer to echo. */ +const lastToolResult = (messages) => { + const tools = (messages || []).filter((m) => m?.role === "tool"); + return tools.length ? messageText(tools[tools.length - 1]).trim() : ""; +}; + +// ── Response builders ──────────────────────────────────────────────────────── +// A completion is one of two shapes: a tool_call (first turn, a query tool is +// available) or a plain text answer (tool already ran, or no tool to call). + +const answerText = (messages) => { + const result = lastToolResult(messages); + return result + ? `The ClickHouse query \`${QUERY}\` returned: ${result}` + : `The ClickHouse query \`${QUERY}\` returned: 1`; +}; + +const toolCall = (tool) => ({ + id: "call_mock_1", + type: "function", + function: { name: tool.function.name, arguments: JSON.stringify({ [queryArgKey(tool)]: QUERY }) }, +}); + +const chunk = (created, delta, finish_reason = null) => + `data: ${JSON.stringify({ + id: "chatcmpl-mock", + object: "chat.completion.chunk", + created, + model: MODEL, + choices: [{ index: 0, delta, finish_reason }], + })}\n\n`; + +/** Stream either a tool_calls turn or a text turn as OpenAI SSE. */ +const streamCompletion = (res, { tool, text }) => { + const created = Math.floor(Date.now() / 1000); + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + if (tool) { + res.write(chunk(created, { role: "assistant", tool_calls: [{ index: 0, ...toolCall(tool) }] })); + res.write(chunk(created, {}, "tool_calls")); + } else { + res.write(chunk(created, { role: "assistant", content: text })); + res.write(chunk(created, {}, "stop")); + } + res.write("data: [DONE]\n\n"); + res.end(); +}; + +/** Non-streaming fallback: one completion object. */ +const jsonCompletion = (res, { tool, text }) => { + const message = tool + ? { role: "assistant", content: null, tool_calls: [toolCall(tool)] } + : { role: "assistant", content: text }; + send(res, 200, { + id: "chatcmpl-mock", + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: MODEL, + choices: [{ index: 0, message, finish_reason: tool ? "tool_calls" : "stop" }], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }); +}; + +const send = (res, code, body) => { + res.writeHead(code, { "Content-Type": "application/json" }); + res.end(JSON.stringify(body)); +}; + +const readBody = (req) => + new Promise((resolve, reject) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + try { + resolve(raw ? JSON.parse(raw) : {}); + } catch (e) { + reject(e); + } + }); + req.on("error", reject); + }); + +const server = http.createServer(async (req, res) => { + const { pathname } = new URL(req.url, "http://localhost"); + + if (req.method === "GET" && (pathname === "/v1/models" || pathname === "/models")) { + return send(res, 200, { + object: "list", + data: [{ id: MODEL, object: "model", created: 0, owned_by: "mock-llm" }], + }); + } + + if (req.method === "POST" && (pathname === "/v1/chat/completions" || pathname === "/chat/completions")) { + let body; + try { + body = await readBody(req); + } catch { + return send(res, 400, { error: { message: "invalid JSON body" } }); + } + // Call the query tool on the first turn; answer once its result is back (or + // if no query tool was offered at all). + const tool = hasToolResult(body.messages) ? null : findQueryTool(body.tools); + const plan = { tool, text: answerText(body.messages) }; + return body.stream === false ? jsonCompletion(res, plan) : streamCompletion(res, plan); + } + + return send(res, 404, { error: { message: `not found: ${req.method} ${pathname}` } }); +}); + +server.listen(PORT, "0.0.0.0", () => { + console.log(`mock-llm listening on :${PORT} (model=${MODEL})`); +}); diff --git a/e2e/package-lock.json b/e2e/package-lock.json new file mode 100644 index 0000000..9cf48de --- /dev/null +++ b/e2e/package-lock.json @@ -0,0 +1,109 @@ +{ + "name": "agentic-data-stack-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "agentic-data-stack-e2e", + "devDependencies": { + "@playwright/test": "^1.61.1", + "@types/node": "^26.0.1", + "typescript": "^5.9.3" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "26.0.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.1.tgz", + "integrity": "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 0000000..d104518 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,14 @@ +{ + "name": "agentic-data-stack-e2e", + "private": true, + "description": "Playwright E2E suite for the agentic data stack (daily + manual, secret-free).", + "scripts": { + "test": "playwright test", + "codegen": "playwright codegen --test-id-attribute=data-testid http://localhost:3080/c/new" + }, + "devDependencies": { + "@playwright/test": "^1.61.1", + "@types/node": "^26.0.1", + "typescript": "^5.9.3" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 0000000..0235200 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,39 @@ +import { defineConfig, devices } from "@playwright/test"; +import { config } from "./lib/config"; +import { STORAGE_STATE } from "./lib/paths"; + +/** + * Chromium-only, host-driven config. There is NO `webServer`: the stack is + * brought up out of band by docker compose (locally or in CI) before the tests + * run, so Playwright only drives the browser against the already-running app. + * + * The `setup` project logs in once and writes storage state; every other spec + * depends on it and starts authenticated. CI is stricter (retries, forbidOnly). + */ +export default defineConfig({ + testDir: "./specs", + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: 1, + reporter: [["html", { open: "never" }], ["list"]], + timeout: 90_000, + expect: { timeout: 15_000 }, + + use: { + baseURL: config.librechatBaseUrl, + trace: "on-first-retry", + screenshot: "only-on-failure", + video: "retain-on-failure", + }, + + projects: [ + { name: "setup", testMatch: /.*\.setup\.ts/ }, + { + name: "chromium", + testMatch: /.*\.spec\.ts/, + use: { ...devices["Desktop Chrome"], storageState: STORAGE_STATE }, + dependencies: ["setup"], + }, + ], +}); diff --git a/e2e/setup/auth.setup.ts b/e2e/setup/auth.setup.ts new file mode 100644 index 0000000..3738c42 --- /dev/null +++ b/e2e/setup/auth.setup.ts @@ -0,0 +1,24 @@ +import { test as setup, expect } from "@playwright/test"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { config } from "../lib/config"; +import { STORAGE_STATE } from "../lib/paths"; + +/** + * Requirement: every authed spec starts already logged in as the seeded admin. + * + * We log in ONCE here via the API (`POST /api/auth/login`) rather than driving + * the form in every spec — faster and less brittle. The response sets LibreChat's + * refresh-token cookie; saving the request context's storage state hands that + * cookie to each spec's browser, which the app exchanges for an access token on + * load. librechat.spec.ts separately covers the login FORM itself. + */ +setup("authenticate via API", async ({ request }) => { + const res = await request.post("/api/auth/login", { + data: { email: config.login.email, password: config.login.password }, + }); + expect(res.ok(), `login failed for ${config.login.email}: HTTP ${res.status()} ${await res.text()}`).toBeTruthy(); + + mkdirSync(dirname(STORAGE_STATE), { recursive: true }); + await request.storageState({ path: STORAGE_STATE }); +}); diff --git a/e2e/specs/langfuse.spec.ts b/e2e/specs/langfuse.spec.ts new file mode 100644 index 0000000..7f14cbd --- /dev/null +++ b/e2e/specs/langfuse.spec.ts @@ -0,0 +1,33 @@ +import { test, expect } from "@playwright/test"; +import { config } from "../lib/config"; + +/** + * Langfuse is up and the headless-initialized user + project exist. Langfuse is + * a DIFFERENT origin from LibreChat, so the shared storage state doesn't apply — + * we sign in through Langfuse's own form using the seeded init-user credentials, + * then confirm the Default Project's dashboard and Traces view load. + */ + +// Langfuse runs at its own base URL; override the LibreChat baseURL for this file. +test.use({ baseURL: config.langfuseBaseUrl }); + +test.describe("Langfuse UI", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/auth/sign-in"); + await page.locator('input[name="email"]').fill(config.login.email); + await page.locator('input[name="password"]').fill(config.login.password); + await page.getByRole("button", { name: /sign in/i }).click(); + // A successful sign-in leaves the auth pages for a /project/... route. + await page.waitForURL(/\/(project|organization|onboarding|setup)\b/, { timeout: 30_000 }); + }); + + test("Default Project dashboard loads", async ({ page }) => { + await page.goto(`/project/${config.langfuse.projectId}`); + await expect(page.getByText(config.langfuse.projectName, { exact: false }).first()).toBeVisible(); + }); + + test("Traces view loads", async ({ page }) => { + await page.goto(`/project/${config.langfuse.projectId}/traces`); + await expect(page.getByRole("heading", { name: /traces/i })).toBeVisible(); + }); +}); diff --git a/e2e/specs/librechat.spec.ts b/e2e/specs/librechat.spec.ts new file mode 100644 index 0000000..a7d2049 --- /dev/null +++ b/e2e/specs/librechat.spec.ts @@ -0,0 +1,38 @@ +import { test, expect } from "@playwright/test"; +import { config } from "../lib/config"; + +/** + * LibreChat is reachable and usable: the login form renders for anonymous users, + * and an authenticated session lands on a working new-chat UI. This is the + * browser-level counterpart to the smoke test's HTTP probe. + */ + +test.describe("LibreChat UI", () => { + test("login form renders for anonymous users", async ({ browser }) => { + // Fresh context with NO storage state — we must be logged out to see the form. + const context = await browser.newContext({ storageState: { cookies: [], origins: [] } }); + const page = await context.newPage(); + await page.goto("/login"); + + await expect(page.getByLabel("Email")).toBeVisible(); + await expect(page.getByLabel("Password")).toBeVisible(); + await expect(page.getByTestId("login-button")).toBeVisible(); + + await context.close(); + }); + + test("authenticated session loads the new-chat UI", async ({ page }) => { + await page.goto("/c/new"); + + // The message composer is the definitive "chat is ready" signal. + await expect(page.getByTestId("text-input")).toBeVisible(); + await expect(page.getByTestId("nav-new-chat-button")).toBeVisible(); + // We are authed, so we must NOT be bounced to the login form. + await expect(page.getByTestId("login-button")).toHaveCount(0); + }); + + test("seeded admin email is the configured login", async () => { + // Guards the config contract the whole suite depends on. + expect(config.login.email).toContain("@"); + }); +}); diff --git a/e2e/specs/roundtrip.spec.ts b/e2e/specs/roundtrip.spec.ts new file mode 100644 index 0000000..5f1b9f5 --- /dev/null +++ b/e2e/specs/roundtrip.spec.ts @@ -0,0 +1,46 @@ +import { test, expect } from "@playwright/test"; +import { runMockChat } from "../lib/librechat"; +import { waitForTrace, waitForObservation } from "../lib/langfuse"; + +/** + * The headline requirement: a chat drives the WHOLE stack end to end. The mock + * only fakes inference — the tool call it emits is executed against the REAL + * ClickHouse MCP server, and the whole exchange is traced to the REAL Langfuse. + * + * We assert on both surfaces: + * 1. Browser: the reply echoes the `SELECT 1` result and a tool invocation is + * shown, proving MockLLM -> MCP -> ClickHouse actually ran. + * 2. Langfuse public API: a fresh trace exists carrying a generation + * observation AND an MCP/tool observation, proving instrumentation works. + */ +test("chat round-trips MockLLM -> MCP -> ClickHouse and traces to Langfuse", async ({ page }) => { + // Bound the Langfuse trace search to this test run. One second of slack + // absorbs host/Langfuse clock skew without matching older traces. + const since = new Date(Date.now() - 1_000).toISOString(); + + await runMockChat(page, "Use the ClickHouse tool to run SELECT 1 and tell me the result."); + + // 1. Browser-visible proof of the tool round-trip. + const messages = page.getByTestId("messages-view"); + await expect(messages).toContainText("SELECT 1"); + await expect(messages).toContainText("1"); + // The assistant's tool call renders a card referencing the query tool. + await expect(messages).toContainText(/query|clickhouse|tool/i); + + // 2. A real Langfuse trace with both a generation and an MCP/tool span. + const trace = await waitForTrace(since, { timeoutMs: 90_000 }); + + await waitForObservation( + trace.id, + (o) => o.type === "GENERATION", + "generation (LLM inference span)", + { timeoutMs: 90_000 }, + ); + + await waitForObservation( + trace.id, + (o) => /clickhouse|mcp|query|select|tool/i.test(o.name ?? "") || o.type === "TOOL", + "MCP -> ClickHouse tool span", + { timeoutMs: 90_000 }, + ); +}); diff --git a/e2e/specs/scoring.spec.ts b/e2e/specs/scoring.spec.ts new file mode 100644 index 0000000..cd066b2 --- /dev/null +++ b/e2e/specs/scoring.spec.ts @@ -0,0 +1,24 @@ +import { test, expect } from "@playwright/test"; +import { runMockChat, giveThumbsUp } from "../lib/librechat"; +import { waitForTrace, waitForScore } from "../lib/langfuse"; + +/** + * The feedback -> Langfuse score bridge works end to end (exercises upstream PR + * danny-avila/LibreChat#13544, already in librechat:latest). After a chat we + * click 👍 on the reply and assert a score lands on that chat's Langfuse trace. + * + * Scoring is only meaningful once a trace exists, so we resolve the trace first, + * then submit feedback, then poll for the score (ingestion is async). + */ +test("thumbs-up feedback attaches a score to the chat's Langfuse trace", async ({ page }) => { + const since = new Date(Date.now() - 1_000).toISOString(); + + await runMockChat(page, "Use the ClickHouse tool to run SELECT 1 and tell me the result."); + + const trace = await waitForTrace(since, { timeoutMs: 90_000 }); + + await giveThumbsUp(page); + + const score = await waitForScore(trace.id, { timeoutMs: 90_000 }); + expect(score.id, "expected a score attached to the trace after 👍 feedback").toBeTruthy(); +}); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json new file mode 100644 index 0000000..b93bf05 --- /dev/null +++ b/e2e/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "node", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + } +} From c40781746dd538348b4664612cadb53438839fd1 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 16:57:36 +0000 Subject: [PATCH 02/15] fix(e2e): give the setup project its own testDir so auth runs The auth setup lives in e2e/setup/, outside the top-level testDir of ./specs. Playwright only matches testMatch against files under a project's testDir, so the setup project discovered zero tests, never wrote the storage state, and every spec failed to open the missing playwright/.auth/user.json (ENOENT) on a local run. Set an explicit testDir per project (setup -> ./setup, chromium -> ./specs). `playwright test --list` now shows the [setup] test ahead of the specs. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/playwright.config.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 0235200..125aaf5 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -9,9 +9,13 @@ import { STORAGE_STATE } from "./lib/paths"; * * The `setup` project logs in once and writes storage state; every other spec * depends on it and starts authenticated. CI is stricter (retries, forbidOnly). + * + * Each project sets its own `testDir` because the auth setup lives outside the + * specs dir: `testMatch` only matches files found under a project's `testDir`, + * so a top-level `testDir: ./specs` would silently discover zero setup files and + * the storage state would never be written (ENOENT when a spec loads it). */ export default defineConfig({ - testDir: "./specs", fullyParallel: false, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, @@ -28,9 +32,10 @@ export default defineConfig({ }, projects: [ - { name: "setup", testMatch: /.*\.setup\.ts/ }, + { name: "setup", testDir: "./setup", testMatch: /.*\.setup\.ts/ }, { name: "chromium", + testDir: "./specs", testMatch: /.*\.spec\.ts/, use: { ...devices["Desktop Chrome"], storageState: STORAGE_STATE }, dependencies: ["setup"], From 432fb4730b39e7caec7be2f311b0779d5c99c000 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 17:00:36 +0000 Subject: [PATCH 03/15] fix(e2e): drop top-level test.use() in langfuse spec Top-level test.use({ baseURL }) throws "did not expect test.use() to be called here" under some Playwright loader setups. The Langfuse spec targets a different origin than the configured baseURL anyway, so use absolute Langfuse URLs directly and remove the override. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/specs/langfuse.spec.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e/specs/langfuse.spec.ts b/e2e/specs/langfuse.spec.ts index 7f14cbd..83fc0fa 100644 --- a/e2e/specs/langfuse.spec.ts +++ b/e2e/specs/langfuse.spec.ts @@ -6,14 +6,18 @@ import { config } from "../lib/config"; * a DIFFERENT origin from LibreChat, so the shared storage state doesn't apply — * we sign in through Langfuse's own form using the seeded init-user credentials, * then confirm the Default Project's dashboard and Traces view load. + * + * We navigate with absolute Langfuse URLs rather than overriding `baseURL` via + * top-level `test.use()`: that override is brittle across Playwright loader + * setups (it throws "did not expect test.use() to be called here" in some + * environments), and being explicit about the cross-origin target is clearer. */ -// Langfuse runs at its own base URL; override the LibreChat baseURL for this file. -test.use({ baseURL: config.langfuseBaseUrl }); +const LF = config.langfuseBaseUrl; test.describe("Langfuse UI", () => { test.beforeEach(async ({ page }) => { - await page.goto("/auth/sign-in"); + await page.goto(`${LF}/auth/sign-in`); await page.locator('input[name="email"]').fill(config.login.email); await page.locator('input[name="password"]').fill(config.login.password); await page.getByRole("button", { name: /sign in/i }).click(); @@ -22,12 +26,12 @@ test.describe("Langfuse UI", () => { }); test("Default Project dashboard loads", async ({ page }) => { - await page.goto(`/project/${config.langfuse.projectId}`); + await page.goto(`${LF}/project/${config.langfuse.projectId}`); await expect(page.getByText(config.langfuse.projectName, { exact: false }).first()).toBeVisible(); }); test("Traces view loads", async ({ page }) => { - await page.goto(`/project/${config.langfuse.projectId}/traces`); + await page.goto(`${LF}/project/${config.langfuse.projectId}/traces`); await expect(page.getByRole("heading", { name: /traces/i })).toBeVisible(); }); }); From 2049ffe767527144c407f76572d591068bd17ba4 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 17:07:03 +0000 Subject: [PATCH 04/15] docs(e2e): document the two-versions test.describe() error fix Add a Troubleshooting note: the "did not expect test.describe()/test.use() to be called here" error is a stale/mismatched @playwright/test install, resolved by `rm -rf node_modules && npm ci` (and avoiding a global playwright shadowing the local runner). Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/e2e/README.md b/e2e/README.md index 46e600f..0c3e754 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -55,3 +55,22 @@ drifts). Confirm/refresh them against the live UI with: ```bash npx playwright codegen --test-id-attribute=data-testid http://localhost:3080/c/new ``` + +## Troubleshooting + +**`Playwright Test did not expect test.describe()/test.use() to be called here`** +(thrown while loading the first spec) means two different `@playwright/test` +versions are resolving — almost always a stale `node_modules` where the +`playwright` binary and the `@playwright/test` library have drifted apart. The +committed lockfile pins both to the same version, so reinstall from it: + +```bash +cd e2e +rm -rf node_modules +npm ci +``` + +Also make sure you're using the local runner, not a globally installed one +(`npx playwright --version` should match `@playwright/test` in package.json). A +global `playwright` on `PATH` can shadow the local binary and reintroduce the +mismatch. From a494ef7d33eede4893061ce9f7e9648612f25e98 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 10:16:53 -0700 Subject: [PATCH 05/15] update new chat selector --- e2e/specs/librechat.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/specs/librechat.spec.ts b/e2e/specs/librechat.spec.ts index a7d2049..e3d68af 100644 --- a/e2e/specs/librechat.spec.ts +++ b/e2e/specs/librechat.spec.ts @@ -26,7 +26,7 @@ test.describe("LibreChat UI", () => { // The message composer is the definitive "chat is ready" signal. await expect(page.getByTestId("text-input")).toBeVisible(); - await expect(page.getByTestId("nav-new-chat-button")).toBeVisible(); + await expect(page.getByTestId("new-chat-button")).toBeVisible(); // We are authed, so we must NOT be bounced to the login form. await expect(page.getByTestId("login-button")).toHaveCount(0); }); From 9aa72c9426fa37c117301a66ca6c3928c266b3a2 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 10:28:40 -0700 Subject: [PATCH 06/15] fix(compose): add healthcheck to langfuse-web docker compose up --wait now blocks until Langfuse passes its health probe, preventing ERR_CONNECTION_REFUSED when the E2E suite starts. Co-Authored-By: Claude Sonnet 4.6 --- langfuse-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/langfuse-compose.yml b/langfuse-compose.yml index b19b781..54e5d3b 100644 --- a/langfuse-compose.yml +++ b/langfuse-compose.yml @@ -72,6 +72,12 @@ services: depends_on: *langfuse-depends-on ports: - 3000:3000 + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/public/health || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 30s environment: <<: *langfuse-worker-env NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME From 6b4cc6af7e0ed2f98cdcdfa632c51fba6c62f5e5 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 10:32:55 -0700 Subject: [PATCH 07/15] fix(e2e): relax post-login URL assertion in langfuse spec Langfuse v3 redirects to / after sign-in rather than /project/.... Check that we left /auth/ instead of asserting a specific destination; each test navigates to its own URL immediately after. Co-Authored-By: Claude Sonnet 4.6 --- e2e/specs/langfuse.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/specs/langfuse.spec.ts b/e2e/specs/langfuse.spec.ts index 83fc0fa..1380f79 100644 --- a/e2e/specs/langfuse.spec.ts +++ b/e2e/specs/langfuse.spec.ts @@ -21,8 +21,9 @@ test.describe("Langfuse UI", () => { await page.locator('input[name="email"]').fill(config.login.email); await page.locator('input[name="password"]').fill(config.login.password); await page.getByRole("button", { name: /sign in/i }).click(); - // A successful sign-in leaves the auth pages for a /project/... route. - await page.waitForURL(/\/(project|organization|onboarding|setup)\b/, { timeout: 30_000 }); + // A successful sign-in navigates away from /auth/. The exact destination + // varies by Langfuse version; each test drives to its own URL anyway. + await page.waitForURL((url) => !url.pathname.startsWith("/auth/"), { timeout: 30_000 }); }); test("Default Project dashboard loads", async ({ page }) => { From 0f7d55106f3446449b56bf6ebcf054ea6a68f9d8 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 17:37:01 +0000 Subject: [PATCH 08/15] fix(e2e): authenticate via UI login so specs aren't bounced to /login MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The API-only login (POST /api/auth/login) captured the refresh cookie but not the localStorage the LibreChat SPA also needs, so browsers restored from that storage state redirected to /login and every authed spec failed at the text-input composer. Drive the real login form in auth.setup.ts and snapshot the session after the authenticated app shell renders (nav-new-chat-button) — the canonical Playwright auth recipe, capturing cookies + localStorage. Also correct the login selectors to the real markup: the submit control is a "Continue" button (no login-button testid), and the login fields are role=textbox named Email/Password. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/setup/auth.setup.ts | 27 ++++++++++++++++----------- e2e/specs/librechat.spec.ts | 8 ++++---- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/e2e/setup/auth.setup.ts b/e2e/setup/auth.setup.ts index 3738c42..ca5969b 100644 --- a/e2e/setup/auth.setup.ts +++ b/e2e/setup/auth.setup.ts @@ -7,18 +7,23 @@ import { STORAGE_STATE } from "../lib/paths"; /** * Requirement: every authed spec starts already logged in as the seeded admin. * - * We log in ONCE here via the API (`POST /api/auth/login`) rather than driving - * the form in every spec — faster and less brittle. The response sets LibreChat's - * refresh-token cookie; saving the request context's storage state hands that - * cookie to each spec's browser, which the app exchanges for an access token on - * load. librechat.spec.ts separately covers the login FORM itself. + * We log in ONCE here through the UI and save the resulting storage state. + * An API-only login (POST /api/auth/login) captures the refresh cookie but NOT + * the localStorage the LibreChat SPA also relies on, so a browser restored from + * it bounces back to /login. Driving the real form captures cookies AND + * localStorage — the canonical Playwright auth recipe — so specs load + * authenticated. */ -setup("authenticate via API", async ({ request }) => { - const res = await request.post("/api/auth/login", { - data: { email: config.login.email, password: config.login.password }, - }); - expect(res.ok(), `login failed for ${config.login.email}: HTTP ${res.status()} ${await res.text()}`).toBeTruthy(); +setup("authenticate", async ({ page }) => { + await page.goto("/login"); + await page.getByRole("textbox", { name: "Email", exact: true }).fill(config.login.email); + await page.getByRole("textbox", { name: "Password", exact: true }).fill(config.login.password); + await page.getByRole("button", { name: /continue|log ?in|sign ?in/i }).click(); + + // The new-chat button only renders inside the authenticated app shell, so its + // visibility proves the login succeeded before we snapshot the session. + await expect(page.getByTestId("new-chat-button")).toBeVisible({ timeout: 30_000 }); mkdirSync(dirname(STORAGE_STATE), { recursive: true }); - await request.storageState({ path: STORAGE_STATE }); + await page.context().storageState({ path: STORAGE_STATE }); }); diff --git a/e2e/specs/librechat.spec.ts b/e2e/specs/librechat.spec.ts index e3d68af..7ce4226 100644 --- a/e2e/specs/librechat.spec.ts +++ b/e2e/specs/librechat.spec.ts @@ -14,9 +14,9 @@ test.describe("LibreChat UI", () => { const page = await context.newPage(); await page.goto("/login"); - await expect(page.getByLabel("Email")).toBeVisible(); - await expect(page.getByLabel("Password")).toBeVisible(); - await expect(page.getByTestId("login-button")).toBeVisible(); + await expect(page.getByRole("textbox", { name: "Email", exact: true })).toBeVisible(); + await expect(page.getByRole("textbox", { name: "Password", exact: true })).toBeVisible(); + await expect(page.getByRole("button", { name: /continue|log ?in|sign ?in/i })).toBeVisible(); await context.close(); }); @@ -28,7 +28,7 @@ test.describe("LibreChat UI", () => { await expect(page.getByTestId("text-input")).toBeVisible(); await expect(page.getByTestId("new-chat-button")).toBeVisible(); // We are authed, so we must NOT be bounced to the login form. - await expect(page.getByTestId("login-button")).toHaveCount(0); + await expect(page.getByRole("heading", { name: /welcome back/i })).toHaveCount(0); }); test("seeded admin email is the configured login", async () => { From 63e4f95ff18b48d7971619e6be42bb57dac91920 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 17:39:45 +0000 Subject: [PATCH 09/15] fix(compose): lengthen langfuse-web healthcheck start_period Langfuse web runs DB migrations on cold start and takes 2-3 minutes to answer /api/public/health, but start_period: 30s + 10x5s retries marked it unhealthy after ~80s, failing `docker compose up --wait` (and the smoke test) on a normal cold start. Bump start_period to 180s. Co-Authored-By: Claude Opus 4.8 (1M context) --- langfuse-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/langfuse-compose.yml b/langfuse-compose.yml index 54e5d3b..aa4706d 100644 --- a/langfuse-compose.yml +++ b/langfuse-compose.yml @@ -77,7 +77,11 @@ services: interval: 5s timeout: 5s retries: 10 - start_period: 30s + # Langfuse web runs DB migrations on cold start and typically takes 2-3 + # minutes to answer /api/public/health. A short grace period made + # `docker compose up --wait` declare it unhealthy (~80s) before it was + # ever ready, failing the smoke test. Give the cold start room. + start_period: 180s environment: <<: *langfuse-worker-env NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME From 6a67be863abefc793bca71f83d2cdc0c5429e18b Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 17:48:14 +0000 Subject: [PATCH 10/15] fix(compose): give langfuse-web 300s healthcheck grace for CI cold start 180s was still short: CI logs show langfuse-web was mid-Prisma-migration at ~227s when Docker marked it unhealthy, failing `up --wait`. Raise the start_period to 300s (with 12x10s retries, ~420s total headroom under the 600s wait-timeout). A passing check flips healthy immediately, so the larger grace doesn't slow faster starts. Co-Authored-By: Claude Opus 4.8 (1M context) --- langfuse-compose.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/langfuse-compose.yml b/langfuse-compose.yml index aa4706d..6bc7b5d 100644 --- a/langfuse-compose.yml +++ b/langfuse-compose.yml @@ -74,14 +74,17 @@ services: - 3000:3000 healthcheck: test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/public/health || exit 1"] - interval: 5s + interval: 10s timeout: 5s - retries: 10 - # Langfuse web runs DB migrations on cold start and typically takes 2-3 - # minutes to answer /api/public/health. A short grace period made - # `docker compose up --wait` declare it unhealthy (~80s) before it was - # ever ready, failing the smoke test. Give the cold start room. - start_period: 180s + retries: 12 + # Langfuse web runs Prisma DB migrations on cold start; on a fresh CI DB + # this was still migrating past 220s, so a shorter grace made + # `docker compose up --wait` declare it unhealthy before it ever answered + # /api/public/health. A passing check flips the container healthy + # immediately (start_period only suppresses early failures), so a generous + # grace costs nothing on fast starts. 300s + 12x10s retries = ~420s of + # headroom, comfortably under the 600s --wait-timeout. + start_period: 300s environment: <<: *langfuse-worker-env NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME From 3a0f78a3876a4829646e4e76a0ccc23f78e1db0a Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 17:59:25 +0000 Subject: [PATCH 11/15] fix(compose): probe langfuse-web health with node, not wget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The langfuse/langfuse:3 image ships neither wget nor curl, so the wget-based healthcheck reported unhealthy for the full grace window even though the app logged "All migrations applied" and "Ready" — failing `up --wait`. Switch to a node http probe (node is guaranteed present), mirroring the python3 probe used for clickhouse-mcp. Co-Authored-By: Claude Opus 4.8 (1M context) --- langfuse-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/langfuse-compose.yml b/langfuse-compose.yml index 6bc7b5d..dd3ff0c 100644 --- a/langfuse-compose.yml +++ b/langfuse-compose.yml @@ -73,7 +73,11 @@ services: ports: - 3000:3000 healthcheck: - test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/public/health || exit 1"] + # Probe with node, not wget/curl: the langfuse image ships neither, so a + # wget-based check reported unhealthy forever even after the app logged + # "Ready" (node is guaranteed present — it's how Langfuse runs). Mirrors + # the python3 probe used for clickhouse-mcp. + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3000/api/public/health',r=>process.exit(r.statusCode<400?0:1)).on('error',()=>process.exit(1))\""] interval: 10s timeout: 5s retries: 12 From a8777fb8a9ae814f7a4f811011c5db36ba61355d Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 18:15:47 +0000 Subject: [PATCH 12/15] fix(compose): point langfuse-web healthcheck at \$HOSTNAME, not localhost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the persistent 'langfuse-web is unhealthy': Next.js standalone binds to process.env.HOSTNAME (Docker sets it to the container id), so Langfuse listens on the container IP — a loopback probe is refused. External access via the published port worked, so the app looked 'Ready' while the healthcheck failed for the entire grace window, failing `docker compose up --wait`. Probe http://$HOSTNAME:3000/api/public/health with node (the image has no wget/curl). Verified locally: 200 -> healthy, 503 -> unhealthy. Co-Authored-By: Claude Opus 4.8 (1M context) --- langfuse-compose.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/langfuse-compose.yml b/langfuse-compose.yml index dd3ff0c..5e65a39 100644 --- a/langfuse-compose.yml +++ b/langfuse-compose.yml @@ -73,11 +73,14 @@ services: ports: - 3000:3000 healthcheck: - # Probe with node, not wget/curl: the langfuse image ships neither, so a - # wget-based check reported unhealthy forever even after the app logged - # "Ready" (node is guaranteed present — it's how Langfuse runs). Mirrors - # the python3 probe used for clickhouse-mcp. - test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3000/api/public/health',r=>process.exit(r.statusCode<400?0:1)).on('error',()=>process.exit(1))\""] + # Probe with node (the langfuse image ships neither wget nor curl; node is + # guaranteed — it's how Langfuse runs). Target $HOSTNAME, NOT localhost: + # Next.js standalone binds to process.env.HOSTNAME (Docker sets it to the + # container id), so the app listens on the container IP and a loopback + # probe is refused — which reported unhealthy forever even after the app + # logged "Ready". A 200 from /api/public/health means healthy; 503 (still + # starting) keeps it unhealthy so `up --wait` waits for real readiness. + test: ["CMD-SHELL", "node -e \"const h=process.env.HOSTNAME||'127.0.0.1';require('http').get('http://'+h+':3000/api/public/health',r=>process.exit(r.statusCode<400?0:1)).on('error',()=>process.exit(1))\""] interval: 10s timeout: 5s retries: 12 From 0a7e031389524db96c2b3caedd3aa39f61b71e27 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 18:37:20 -0700 Subject: [PATCH 13/15] fix(e2e): keep LibreChat auth session fresh across specs LibreChat rotates the refresh-token cookie on every /api/auth/refresh call, which the SPA fires unconditionally on every page mount. The shared storage-state snapshot from auth.setup.ts was captured once, so the first spec to load an authenticated page burned that snapshot's refresh token; every later spec in the (serial) run replayed the stale cookie, got a 401 on refresh, and was bounced to /login. Add a page fixture that re-persists storage state after each test so the rotated cookie carries forward. Langfuse's own NextAuth cookies are stripped from the snapshot: LibreChat and Langfuse share the "localhost" cookie jar (cookies aren't port-scoped), so without this a signed-in Langfuse session would leak into later specs and break langfuse.spec.ts's assumption that each test starts unauthenticated. Co-Authored-By: Claude Sonnet 5 --- e2e/lib/fixtures.ts | 47 +++++++++++++++++++++++++++++++++++++ e2e/specs/langfuse.spec.ts | 2 +- e2e/specs/librechat.spec.ts | 2 +- e2e/specs/roundtrip.spec.ts | 2 +- e2e/specs/scoring.spec.ts | 2 +- 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 e2e/lib/fixtures.ts diff --git a/e2e/lib/fixtures.ts b/e2e/lib/fixtures.ts new file mode 100644 index 0000000..7a369e6 --- /dev/null +++ b/e2e/lib/fixtures.ts @@ -0,0 +1,47 @@ +import { writeFileSync } from "node:fs"; +import { test as base } from "@playwright/test"; +import { config } from "./config"; +import { STORAGE_STATE } from "./paths"; + +const librechatOrigin = new URL(config.librechatBaseUrl).origin; + +/** + * NextAuth's own cookie names (Langfuse's auth stack), used to strip its + * session out of the shared snapshot below. Cookies aren't port-scoped, so on + * a localhost dev stack LibreChat (:3080) and Langfuse (:3000) share the same + * cookie jar under `domain: "localhost"` — filtering by domain/origin can't + * tell them apart. Filtering by name can. + */ +const NEXT_AUTH_COOKIE_RE = /^(__Secure-)?next-auth\./; + +/** + * LibreChat rotates the refresh-token cookie on every successful + * `/api/auth/refresh` call, and the SPA fires that refresh unconditionally on + * every page mount. auth.setup.ts captures storage state only ONCE; the first + * spec to load an authenticated page rotates that token out from under the + * static snapshot, and every later spec replaying the stale cookie gets + * bounced to /login. Re-saving storage state after each test carries the + * rotated cookie forward so the next spec's fresh context stays authenticated. + * + * Specs also visit Langfuse in the same browser context, and langfuse.spec.ts + * relies on each test starting with NO Langfuse session so its own sign-in + * form renders. Persisting the full state would leak a signed-in Langfuse + * session into the shared snapshot and break that assumption, so Langfuse's + * NextAuth cookies are stripped and only the LibreChat origin's localStorage + * is kept. + */ +export const test = base.extend({ + page: async ({ page }, use) => { + await use(page); + const state = await page.context().storageState(); + writeFileSync( + STORAGE_STATE, + JSON.stringify({ + cookies: state.cookies.filter((c) => !NEXT_AUTH_COOKIE_RE.test(c.name)), + origins: state.origins.filter((o) => o.origin === librechatOrigin), + }), + ); + }, +}); + +export { expect } from "@playwright/test"; diff --git a/e2e/specs/langfuse.spec.ts b/e2e/specs/langfuse.spec.ts index 1380f79..c33f49e 100644 --- a/e2e/specs/langfuse.spec.ts +++ b/e2e/specs/langfuse.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "../lib/fixtures"; import { config } from "../lib/config"; /** diff --git a/e2e/specs/librechat.spec.ts b/e2e/specs/librechat.spec.ts index 7ce4226..b0bb40b 100644 --- a/e2e/specs/librechat.spec.ts +++ b/e2e/specs/librechat.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "../lib/fixtures"; import { config } from "../lib/config"; /** diff --git a/e2e/specs/roundtrip.spec.ts b/e2e/specs/roundtrip.spec.ts index 5f1b9f5..c32ac37 100644 --- a/e2e/specs/roundtrip.spec.ts +++ b/e2e/specs/roundtrip.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "../lib/fixtures"; import { runMockChat } from "../lib/librechat"; import { waitForTrace, waitForObservation } from "../lib/langfuse"; diff --git a/e2e/specs/scoring.spec.ts b/e2e/specs/scoring.spec.ts index cd066b2..91756f0 100644 --- a/e2e/specs/scoring.spec.ts +++ b/e2e/specs/scoring.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "../lib/fixtures"; import { runMockChat, giveThumbsUp } from "../lib/librechat"; import { waitForTrace, waitForScore } from "../lib/langfuse"; From ce1fc224efccdee7f76bd0863ff2d1904d52f006 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 18:37:34 -0700 Subject: [PATCH 14/15] fix(e2e): update chat helper selectors to match current LibreChat UI Three of the picker/button selectors in lib/librechat.ts had drifted from the running LibreChat UI, silently no-oping (per the file's own "best-effort" selector strategy) rather than throwing: - selectMockEndpoint only clicked the MockLLM endpoint entry, which merely opens its model submenu; the model itself ("mock-model") still needed a click to finalize the selection, so chats silently fell back to the default ChatGPT endpoint and failed with "No key found". - enableClickHouseMcp's loose /mcp/i button match also matched the sidebar's persistent "MCP Settings" panel toggle, which sits earlier in the DOM and won tryClick's `.first()` over the composer's actual per-message "MCP Servers" toggle, so the tool was never attached to the message. - giveThumbsUp looked for "thumbs up"/"good response"/"helpful" button text, but the current UI's positive-feedback button is titled "Love this" and only opens a reason submenu; feedback isn't submitted to the backend until a reason chip is picked. Co-Authored-By: Claude Sonnet 5 --- e2e/lib/librechat.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/e2e/lib/librechat.ts b/e2e/lib/librechat.ts index 7848f7d..29febe9 100644 --- a/e2e/lib/librechat.ts +++ b/e2e/lib/librechat.ts @@ -15,6 +15,7 @@ import { expect, type Page } from "@playwright/test"; */ const MOCK_ENDPOINT = "MockLLM"; +const MOCK_MODEL = "mock-model"; const MCP_SERVER = "ClickHouse-Local"; /** Best-effort click of the first locator that becomes visible; never throws. */ @@ -41,12 +42,23 @@ export const selectMockEndpoint = async (page: Page): Promise => { page.getByRole("option", { name: MOCK_ENDPOINT }), page.getByText(MOCK_ENDPOINT, { exact: false }), ]); + // Selecting the endpoint only opens its model submenu; the model itself + // still needs a click to finalize the selection. + await tryClick(page, [ + page.getByRole("option", { name: MOCK_MODEL }), + page.getByText(MOCK_MODEL, { exact: false }), + ]); }; /** Enable the ClickHouse-Local MCP server for the conversation. */ export const enableClickHouseMcp = async (page: Page): Promise => { await tryClick(page, [ page.getByTestId("mcp-select"), + // Exact match first: the composer's per-message MCP toggle is named "MCP + // Servers", but a loose /mcp/i regex also matches the sidebar's "MCP + // Settings" connection-management button, which sits earlier in the DOM + // and would otherwise win `tryClick`'s `.first()`. + page.getByRole("button", { name: "MCP Servers" }), page.getByRole("button", { name: /mcp/i }), page.getByRole("button", { name: /tools/i }), ]); @@ -72,15 +84,22 @@ export const sendPrompt = async (page: Page, prompt: string): Promise => { * Give thumbs-up feedback on the latest assistant message. The controls appear * on hover, so we hover the messages view first. Best-effort selection (see the * strategy note above): if it misses, the caller's score assertion fails loudly. + * + * The positive-feedback button (currently titled "Love this") only opens a + * reason submenu; the feedback isn't actually submitted to the backend until + * one of the reason chips (e.g. "Accurate and Reliable") is picked. */ export const giveThumbsUp = async (page: Page): Promise => { const messages = page.getByTestId("messages-view"); await messages.hover().catch(() => {}); await tryClick(page, [ page.getByTestId("good-response-button"), - page.getByRole("button", { name: /thumbs.?up|good response|helpful/i }), + page.getByRole("button", { name: /love this|thumbs.?up|good response|helpful/i }), page.locator('button[aria-label*="thumb" i]').first(), ]); + await tryClick(page, [ + page.getByRole("button", { name: /accurate|creative|well-written|attention to detail/i }), + ]); }; /** From ed9e777dac4637cd1bdb5546b4b216297a66db5b Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Wed, 1 Jul 2026 18:37:50 -0700 Subject: [PATCH 15/15] fix(e2e): match Langfuse's actual "Tracing" page heading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The traces-view assertion looked for a heading matching /traces/i, but the page's actual heading text is "Tracing" — "Traces" is a nav link underneath it, not a heading, so the assertion never matched even though the page loaded correctly. Co-Authored-By: Claude Sonnet 5 --- e2e/specs/langfuse.spec.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/specs/langfuse.spec.ts b/e2e/specs/langfuse.spec.ts index c33f49e..dda0047 100644 --- a/e2e/specs/langfuse.spec.ts +++ b/e2e/specs/langfuse.spec.ts @@ -33,6 +33,9 @@ test.describe("Langfuse UI", () => { test("Traces view loads", async ({ page }) => { await page.goto(`${LF}/project/${config.langfuse.projectId}/traces`); - await expect(page.getByRole("heading", { name: /traces/i })).toBeVisible(); + // The page heading reads "Tracing"; "Traces" is the (already active) nav + // link beneath it, not a heading. + await expect(page.getByRole("heading", { name: /tracing/i })).toBeVisible(); + await expect(page.getByRole("link", { name: "Traces", exact: true })).toBeVisible(); }); });