test(e2e): add daily secret-free Playwright suite [murmur:agentic-data-stack/agentic-data-stack-issue-12]

.github/workflows/e2e.yml

@@ -0,0 +1,129 @@
+name: E2E tests
+
+# Heavy, browser-level suite. It runs DAILY + on manual dispatch only — never on
+# PRs. The smoke test (smoke-test.yml) stays the PR-time launch gate; this suite
+# catches `:latest` image drift and functional regressions on a schedule.
+on:
+  schedule:
+    - cron: '37 4 * * *'   # daily ~04:37 UTC (offset from the smoke test's 13:00)
+  workflow_dispatch:
+
+# Avoid piling up runs on the same ref.
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Generate .env
+        run: bash scripts/generate-env.sh
+
+      # --wait blocks until every healthchecked service is healthy (mock-llm,
+      # ClickHouse MCP, LibreChat, Langfuse, ...) and fails if any is unhealthy.
+      - name: Launch stack (prod + e2e override) and wait for health
+        run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait --wait-timeout 600
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: e2e/package-lock.json
+
+      - name: Install Playwright and browser
+        working-directory: e2e
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Run E2E tests
+        working-directory: e2e
+        run: npx playwright test
+
+      - name: Upload Playwright report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: e2e/playwright-report
+          retention-days: 14
+
+      - name: Dump diagnostics on failure
+        if: failure()
+        run: |
+          docker compose -f docker-compose.yml -f docker-compose.e2e.yml ps -a
+          docker compose -f docker-compose.yml -f docker-compose.e2e.yml logs --no-color --tail=200
+
+      - name: Tear down
+        if: always()
+        run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml down -v
+
+  # On a failed DAILY run, open (or comment on) a de-duped tracking issue. Manual
+  # (workflow_dispatch) failures surface directly to the person who triggered
+  # them, so they don't need an issue. Mirrors smoke-test.yml's report job.
+  report-daily-failure:
+    needs: e2e
+    if: contains(fromJSON('["failure", "cancelled"]'), needs.e2e.result) && github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: Open or update tracking issue
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const label = 'e2e-test-failure';
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const body = `The daily E2E test suite failed.\n\nRun: ${runUrl}`;
+
+            // Ensure the tracking label exists so the de-dupe search is reliable
+            // in a repo that has never had it.
+            try {
+              await github.rest.issues.getLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: label,
+              });
+            } catch (e) {
+              if (e.status !== 404) throw e;
+              await github.rest.issues.createLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: label,
+                color: 'd73a4a',
+              });
+            }
+
+            const existing = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: label,
+            });
+
+            // listForRepo returns PRs as well as issues; exclude PRs so we never
+            // comment on a mislabeled pull request.
+            const openIssue = existing.data.find(i => !i.pull_request);
+
+            if (openIssue) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: openIssue.number,
+                body,
+              });
+            } else {
+              await github.rest.issues.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                title: 'Daily E2E test suite failed',
+                body,
+                labels: [label],
+              });
+            }

docker-compose.e2e.yml

@@ -0,0 +1,44 @@
+# E2E override — layered on top of docker-compose.yml, ONLY for the daily/manual
+# Playwright suite. Production config is untouched: this file adds the secret-free
+# mock inference service and repoints LibreChat at an E2E-only config so a chat
+# exercises the REAL MCP server, ClickHouse, and Langfuse without any API key.
+#
+#   docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait
+#
+# Why a config override instead of editing librechat.yaml: the mock endpoint must
+# never leak into a real deployment. CONFIG_PATH points LibreChat at a NEW mount
+# path (/app/librechat.e2e.yaml) so it can't collide with the prod bind mount.
+
+services:
+  mock-llm:
+    image: node:22-alpine
+    restart: always
+    working_dir: /app
+    command: ["node", "server.js"]
+    environment:
+      - PORT=8080
+      - MOCK_MODEL=mock-model
+    volumes:
+      - type: bind
+        source: ./e2e/mock-llm/server.js
+        target: /app/server.js
+        read_only: true
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:8080/v1/models || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+
+  librechat:
+    depends_on:
+      mock-llm:
+        condition: service_healthy
+    environment:
+      # Load the E2E config (adds the MockLLM endpoint) from its own mount path.
+      - CONFIG_PATH=/app/librechat.e2e.yaml
+    volumes:
+      - type: bind
+        source: ./e2e/librechat.e2e.yaml
+        target: /app/librechat.e2e.yaml
+        read_only: true

e2e/.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+playwright-report/
+test-results/
+blob-report/
+playwright/.auth/

e2e/README.md

@@ -0,0 +1,76 @@
+# E2E tests
+
+Browser-level Playwright tests that verify the stack actually **works** — not
+just that it boots (that's the job of the smoke test in
+`.github/workflows/smoke-test.yml`). They cover the real user flows: login, an
+agent chat that queries **ClickHouse via the real MCP server**, **Langfuse trace
+creation**, and the **feedback → Langfuse score** path.
+
+Runs **daily + manual only** (`.github/workflows/e2e.yml`), never on PRs.
+
+## Secret-free by design
+
+A local OpenAI-compatible **mock** (`mock-llm/server.js`) fakes only the
+*inference*. The MCP server, ClickHouse, and Langfuse stay **real**, so no
+`ANTHROPIC_API_KEY`, no token spend, and no frontier-model flakiness — while
+still producing real Langfuse traces (a generation observation plus real
+MCP → ClickHouse tool spans) and exercising scoring.
+
+The mock uses **adaptive tool-calling**: it inspects the request's `tools[]`,
+finds the ClickHouse query tool by name *pattern* (never a hardcoded name, which
+LibreChat rewrites), emits a `tool_calls` delta running `SELECT 1`, then — once
+LibreChat sends the tool result back — emits a final answer echoing it.
+
+## Layout
+
+| Path | Purpose |
+| --- | --- |
+| `mock-llm/server.js` | OpenAI-compatible mock inference server (Node built-ins only). |
+| `librechat.e2e.yaml` | LibreChat config = prod config + the `MockLLM` endpoint. Loaded via `CONFIG_PATH`. |
+| `playwright.config.ts` | Chromium, `baseURL` `http://localhost:3080`, `setup` auth project, no `webServer`. |
+| `lib/config.ts` | Explicit, fail-fast config resolved from the repo-root `.env`. |
+| `lib/langfuse.ts` | Langfuse public-API client that **polls** (ingestion is async). |
+| `lib/librechat.ts` | Shared chat-driving helpers. |
+| `setup/auth.setup.ts` | Logs in once via `POST /api/auth/login`, saves storage state. |
+| `specs/*.spec.ts` | `librechat`, `langfuse`, `roundtrip`, `scoring`. |
+
+## Run locally
+
+The suite drives an already-running stack (no `webServer`): bring it up first.
+
+```bash
+cd ..                      # repo root
+bash scripts/generate-env.sh
+docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --wait --wait-timeout 600
+cd e2e && npm ci && npx playwright install --with-deps chromium
+npx playwright test
+cd .. && docker compose -f docker-compose.yml -f docker-compose.e2e.yml down -v
+```
+
+## Confirming selectors
+
+The endpoint/MCP-picker and 👍/👎 selectors are best-effort (LibreChat markup
+drifts). Confirm/refresh them against the live UI with:
+
+```bash
+npx playwright codegen --test-id-attribute=data-testid http://localhost:3080/c/new
+```
+
+## Troubleshooting
+
+**`Playwright Test did not expect test.describe()/test.use() to be called here`**
+(thrown while loading the first spec) means two different `@playwright/test`
+versions are resolving — almost always a stale `node_modules` where the
+`playwright` binary and the `@playwright/test` library have drifted apart. The
+committed lockfile pins both to the same version, so reinstall from it:
+
+```bash
+cd e2e
+rm -rf node_modules
+npm ci
+```
+
+Also make sure you're using the local runner, not a globally installed one
+(`npx playwright --version` should match `@playwright/test` in package.json). A
+global `playwright` on `PATH` can shadow the local binary and reintroduce the
+mismatch.

e2e/lib/config.ts

@@ -0,0 +1,61 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+/**
+ * One source of truth for E2E config, resolved explicitly and fail-fast.
+ *
+ * Values come from the generated repo-root `.env` (written by
+ * scripts/generate-env.sh), with `process.env` taking precedence so a run can
+ * be pointed at a remote stack without editing files. A required value that is
+ * missing throws at load time — never a silent default that hangs a test later.
+ *
+ * Note the host/container split: `.env`'s LANGFUSE_BASE_URL is the in-container
+ * URL (http://langfuse-web:3000). Playwright runs on the host, so it must reach
+ * Langfuse and LibreChat via their published localhost ports, not the compose
+ * service names. Hence the dedicated *_HOST_URL knobs below.
+ */
+
+const ENV_FILE = resolve(__dirname, "../../.env");
+
+const parseEnvFile = (path: string): Record<string, string> => {
+  if (!existsSync(path)) return {};
+  const out: Record<string, string> = {};
+  for (const line of readFileSync(path, "utf8").split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eq = trimmed.indexOf("=");
+    if (eq === -1) continue;
+    out[trimmed.slice(0, eq).trim()] = trimmed.slice(eq + 1).trim();
+  }
+  return out;
+};
+
+const fileEnv = parseEnvFile(ENV_FILE);
+
+/** process.env wins over the .env file; both missing with no default throws. */
+const required = (key: string, fallback?: string): string => {
+  const value = process.env[key] ?? fileEnv[key] ?? fallback;
+  if (value === undefined || value === "") {
+    throw new Error(
+      `Missing required E2E config '${key}'. Set it in the environment or run scripts/generate-env.sh to write ${ENV_FILE}.`,
+    );
+  }
+  return value;
+};
+
+export const config = {
+  librechatBaseUrl: required("LIBRECHAT_HOST_URL", "http://localhost:3080"),
+  langfuseBaseUrl: required("LANGFUSE_HOST_URL", "http://localhost:3000"),
+
+  login: {
+    email: required("LIBRECHAT_USER_EMAIL"),
+    password: required("LIBRECHAT_USER_PASSWORD"),
+  },
+
+  langfuse: {
+    publicKey: required("LANGFUSE_INIT_PROJECT_PUBLIC_KEY"),
+    secretKey: required("LANGFUSE_INIT_PROJECT_SECRET_KEY"),
+    projectId: required("LANGFUSE_INIT_PROJECT_ID"),
+    projectName: required("LANGFUSE_INIT_PROJECT_NAME", "Default Project"),
+  },
+} as const;

e2e/lib/fixtures.ts

@@ -0,0 +1,47 @@
+import { writeFileSync } from "node:fs";
+import { test as base } from "@playwright/test";
+import { config } from "./config";
+import { STORAGE_STATE } from "./paths";
+
+const librechatOrigin = new URL(config.librechatBaseUrl).origin;
+
+/**
+ * NextAuth's own cookie names (Langfuse's auth stack), used to strip its
+ * session out of the shared snapshot below. Cookies aren't port-scoped, so on
+ * a localhost dev stack LibreChat (:3080) and Langfuse (:3000) share the same
+ * cookie jar under `domain: "localhost"` — filtering by domain/origin can't
+ * tell them apart. Filtering by name can.
+ */
+const NEXT_AUTH_COOKIE_RE = /^(__Secure-)?next-auth\./;
+
+/**
+ * LibreChat rotates the refresh-token cookie on every successful
+ * `/api/auth/refresh` call, and the SPA fires that refresh unconditionally on
+ * every page mount. auth.setup.ts captures storage state only ONCE; the first
+ * spec to load an authenticated page rotates that token out from under the
+ * static snapshot, and every later spec replaying the stale cookie gets
+ * bounced to /login. Re-saving storage state after each test carries the
+ * rotated cookie forward so the next spec's fresh context stays authenticated.
+ *
+ * Specs also visit Langfuse in the same browser context, and langfuse.spec.ts
+ * relies on each test starting with NO Langfuse session so its own sign-in
+ * form renders. Persisting the full state would leak a signed-in Langfuse
+ * session into the shared snapshot and break that assumption, so Langfuse's
+ * NextAuth cookies are stripped and only the LibreChat origin's localStorage
+ * is kept.
+ */
+export const test = base.extend({
+  page: async ({ page }, use) => {
+    await use(page);
+    const state = await page.context().storageState();
+    writeFileSync(
+      STORAGE_STATE,
+      JSON.stringify({
+        cookies: state.cookies.filter((c) => !NEXT_AUTH_COOKIE_RE.test(c.name)),
+        origins: state.origins.filter((o) => o.origin === librechatOrigin),
+      }),
+    );
+  },
+});
+
+export { expect } from "@playwright/test";