From 75e2ed5e3fd45cbc730bd9c9fa9e1a6d97bd00af Mon Sep 17 00:00:00 2001 From: John Durrant Date: Mon, 8 Jun 2026 14:01:55 +0100 Subject: [PATCH] feat(questionnaires): F8.1 admin analytics dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the admin read-side view of completed-session data, scoped to a questionnaire version: per-question answer distributions, an invitation completion funnel, and per-version cost actuals — with tag-aware filtering. Read-only and aggregate-only (free-text answer values are never surfaced), so the surface is PII-safe ahead of F8.3 anonymous-mode hardening. - Core aggregators (pure, Prisma at the seam) in lib/app/questionnaire/analytics: distributions, funnel, cost (+ shared query schema / scope resolver). - Three admin-only, flag-gated, version-scoped GET routes under .../versions/[vid]/analytics/{distributions,funnel,cost}. - Version-scoped Analytics tab + client UI (filter, distribution, funnel, cost panels); recharts for the cost trend. - Cost attribution: standardise the four session-bound capabilities on the canonical AiCostLog metadata key `appQuestionnaireSessionId` (runtime spend), with design-time spend read via `metadata.versionId`. No migration. Tests: 45 unit (aggregator math, per-type distributions, funnel staging, cost runtime/design split + edge branches) + 21 route integration tests (auth, version-scope, validation, response shape). Docs + F8.1 tracker added. Co-Authored-By: Claude Opus 4.8 (1M context) --- .context/admin/questionnaire-analytics.md | 80 ++++ .context/app/planning/features/f8.1.md | 85 +++++ .../questionnaires/[id]/analytics/page.tsx | 207 ++++++++++ app/admin/questionnaires/[id]/page.tsx | 7 + .../versions/[vid]/analytics/cost/route.ts | 48 +++ .../[vid]/analytics/distributions/route.ts | 50 +++ .../versions/[vid]/analytics/funnel/route.ts | 49 +++ .../analytics/analytics-filters.tsx | 139 +++++++ .../analytics/analytics-view.tsx | 60 +++ .../analytics/completion-funnel-panel.tsx | 95 +++++ .../questionnaires/analytics/cost-panel.tsx | 159 ++++++++ .../analytics/question-distribution-panel.tsx | 194 ++++++++++ lib/api/endpoints.ts | 9 + lib/app/questionnaire/analytics/cost.ts | 212 +++++++++++ .../questionnaire/analytics/distributions.ts | 329 ++++++++++++++++ lib/app/questionnaire/analytics/funnel.ts | 117 ++++++ lib/app/questionnaire/analytics/index.ts | 41 ++ .../questionnaire/analytics/query-schema.ts | 61 +++ lib/app/questionnaire/analytics/views.ts | 180 +++++++++ .../capabilities/compose-completion-offer.ts | 5 +- .../capabilities/detect-contradictions.ts | 2 +- .../capabilities/extract-answer-slots.ts | 2 +- .../capabilities/refine-answer.ts | 2 +- .../questionnaires/analytics-routes.test.ts | 166 ++++++++ .../completion-capability.test.ts | 9 +- .../refinement-capability.test.ts | 4 +- .../app/questionnaire/analytics/cost.test.ts | 217 +++++++++++ .../analytics/distributions.test.ts | 358 ++++++++++++++++++ .../questionnaire/analytics/funnel.test.ts | 108 ++++++ 29 files changed, 2987 insertions(+), 8 deletions(-) create mode 100644 .context/admin/questionnaire-analytics.md create mode 100644 .context/app/planning/features/f8.1.md create mode 100644 app/admin/questionnaires/[id]/analytics/page.tsx create mode 100644 app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/cost/route.ts create mode 100644 app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/distributions/route.ts create mode 100644 app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/funnel/route.ts create mode 100644 components/admin/questionnaires/analytics/analytics-filters.tsx create mode 100644 components/admin/questionnaires/analytics/analytics-view.tsx create mode 100644 components/admin/questionnaires/analytics/completion-funnel-panel.tsx create mode 100644 components/admin/questionnaires/analytics/cost-panel.tsx create mode 100644 components/admin/questionnaires/analytics/question-distribution-panel.tsx create mode 100644 lib/app/questionnaire/analytics/cost.ts create mode 100644 lib/app/questionnaire/analytics/distributions.ts create mode 100644 lib/app/questionnaire/analytics/funnel.ts create mode 100644 lib/app/questionnaire/analytics/index.ts create mode 100644 lib/app/questionnaire/analytics/query-schema.ts create mode 100644 lib/app/questionnaire/analytics/views.ts create mode 100644 tests/integration/api/v1/app/questionnaires/analytics-routes.test.ts create mode 100644 tests/unit/lib/app/questionnaire/analytics/cost.test.ts create mode 100644 tests/unit/lib/app/questionnaire/analytics/distributions.test.ts create mode 100644 tests/unit/lib/app/questionnaire/analytics/funnel.test.ts diff --git a/.context/admin/questionnaire-analytics.md b/.context/admin/questionnaire-analytics.md new file mode 100644 index 000000000..1aff0a70f --- /dev/null +++ b/.context/admin/questionnaire-analytics.md @@ -0,0 +1,80 @@ +# Questionnaire analytics (F8.1) + +Version-scoped admin page at `/admin/questionnaires/[id]/analytics?v=[versionId]` — the read-side +view of a version's completed-session data. Three surfaces in tabs: per-question **distributions**, +the completion **funnel**, and **cost** actuals. Reached from the `Analytics` button on the +questionnaire detail page. Read-only, admin-only, master-flag-gated (`APP_QUESTIONNAIRES_ENABLED`). + +> **Source of truth:** aggregators in `lib/app/questionnaire/analytics/`, routes under +> `app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/`, UI in +> `app/admin/questionnaires/[id]/analytics/` + `components/admin/questionnaires/analytics/`. +> Update this doc when those change. + +## Scope & filter + +One shared filter (`analytics-filters.tsx`) drives all three views through the URL: + +- **Date window** — `from`/`to` (`YYYY-MM-DD`), default last 30 days (reuses the platform + `resolveAnalyticsDateRange` so questionnaire and orchestration share one "30 days"). +- **Tag filter** — `tagIds` (comma-separated). Restricts the **distributions** view to questions + carrying any selected tag; the funnel and cost views ignore it. +- **Version** — `?v=` (the page owns this, SSR links), like the evaluations/invitations sub-pages. + +All aggregations count **non-preview** sessions only (`isPreview = false`). + +## The three surfaces + +**Distributions** (`distributions.ts`) — per question, a type-appropriate breakdown over the +answers captured in scope: + +| Type | Detail | +| ------------------- | -------------------------------------------------------------- | +| single/multi choice | count per option + an "other/unlisted" bucket | +| likert | count per scale point + mean | +| numeric | min/max/mean/median + histogram | +| boolean | true/false counts (custom labels) | +| date | counts bucketed by month | +| **free_text** | **no values** — response rate, avg confidence, provenance only | + +Every question also reports answered/unanswered counts, response rate (denominator = sessions in +scope), avg confidence, and the provenance mix (`direct`/`inferred`/`synthesised`/`refined`). +**Free-text answer values are never serialised** — F8.1 stays PII-safe ahead of F8.3. + +**Funnel** (`funnel.ts`) — invited → opened → started → completed, with per-stage drop-off, +retention (vs invited), and step conversion. Invited/opened come from invitation timestamps +(`sentAt`/`openedAt`, excluding revoked); started/completed are derived from real sessions matched +to invited respondents by `userId`. **Anonymous (un-invited) sessions** are reported separately +(they enter at "started") so the invite funnel isn't overstated. + +**Cost** (`cost.ts`) — total spend split into **respondent runtime** vs **design-time**, a +per-capability breakdown, a daily trend, and the top sessions by spend. Reads the platform +`AiCostLog` ledger via raw SQL over its `metadata` JSON. + +## Cost attribution contract + +Questionnaire LLM spend is attributed in `AiCostLog.metadata`: + +- **Runtime** (live respondent turns) → `metadata.appQuestionnaireSessionId` = the session cuid. + Stamped by the session-bound capabilities (`extract_answer_slots`, `detect_contradictions`, + `refine_answer`, `compose_completion_offer`) and the adaptive selector. F8.1 standardised these + on the `appQuestionnaireSessionId` key (previously a bare `sessionId` in the four capabilities). +- **Design-time** (structure evaluation) → `metadata.versionId`, stamped by `evaluate_structure`. + +A version's spend = the ledger rows for its non-preview sessions (runtime) ∪ the rows tagged with +its version id (design-time). The two key sets are disjoint, so there's no double-counting. +`AppQuestionnaireTurn.costUsd` is unrelated to this view — it's the F6.3 budget-enforcement basis +and is left untouched. One-time ingest cost (`extract_questionnaire_structure`, logged before a +version exists) is not version-attributable and is excluded. + +## API + +All three are `GET`, admin-only, version-scoped (404 on a cross-version id), and accept the shared +query (`from`, `to`, `tagIds`). Rate limiting is the automatic section cap (read-only, no sub-cap). + +| Endpoint | Returns | +| ------------------------------------------ | ----------------------------- | +| `…/versions/[vid]/analytics/distributions` | `QuestionDistributionsResult` | +| `…/versions/[vid]/analytics/funnel` | `CompletionFunnelResult` | +| `…/versions/[vid]/analytics/cost` | `QuestionnaireCostResult` | + +Endpoint builders: `API.APP.QUESTIONNAIRES.versionAnalytics{Distributions,Funnel,Cost}(id, vid)`. diff --git a/.context/app/planning/features/f8.1.md b/.context/app/planning/features/f8.1.md new file mode 100644 index 000000000..756019a57 --- /dev/null +++ b/.context/app/planning/features/f8.1.md @@ -0,0 +1,85 @@ +--- +feature: F8.1 +title: Admin analytics dashboards +phase: P8 — Admin analytics, exports, anonymous mode +status: in flight +owner: TBD +deps: F4.2 (answer slots), F4.6 (session events), F3.2 (invitations), F6.1 (live turns) +opened: 2026-06-08 +plan: .context/app/planning/development-plan.md#f81--admin-analytics-dashboards +docs: .context/admin/questionnaire-analytics.md +--- + +# F8.1 — Admin analytics dashboards + +> Committable tracker for **F8.1**, the opening feature of P8. The admin's read-side view of +> completed-session data, scoped to a version: per-question answer distributions, an +> invitation completion funnel, and per-version cost actuals. Tag-aware filtering. Read-only, +> aggregate-only — no new models, no migration. Gated by `APP_QUESTIONNAIRES_ENABLED`. + +## Intent + +P1–P7 produced the data: answer slots (F4.2), the session state machine + events (F4.6), +invitations (F3.2), live respondent turns + cost (F6.1/F6.3). F8.1 makes that data +**readable** for the admin without mutating anything: three aggregations over a version's +non-preview sessions, surfaced as a version-scoped `Analytics` tab. + +It is deliberately **aggregate-only**: free-text answer _values_ are never surfaced (only +response rate, confidence, provenance), so F8.1 is PII-safe by construction. Full +anonymous-mode hardening across every surface is F8.3. + +## Decisions (confirmed with the user) + +- **Cost source = `AiCostLog`, made properly attributable** (not an `AppQuestionnaireTurn.costUsd` + shortcut). The four session-bound capabilities already stamped a session link in `metadata` + under a bare `sessionId`; the adaptive selector (via `costLogMetadata`) used the canonical + `appQuestionnaireSessionId`. F8.1 **standardises the four capabilities on + `appQuestionnaireSessionId`** so runtime cost is attributable by one key, and reads + design-time cost via `metadata.versionId` (already stamped by `evaluate-structure`). The two + tag sets are disjoint, so runtime + design-time never double-count. `AppQuestionnaireTurn.costUsd` + is left untouched — it remains the F6.3 budget-enforcement basis. No existing data, so no + backfill concern. +- **Placement = version-scoped tab** under `app/admin/questionnaires/[id]/analytics`, alongside + the existing `invitations` and `evaluations` sub-pages (`?v=` selects the version). +- **Funnel from session reality, not invitation status.** Invited/opened come from + `AppQuestionnaireInvitation` timestamps; started/completed are derived from real sessions + matched to invited respondents by `userId`. Anonymous (un-invited) sessions can't enter the + invite funnel, so they're reported separately (entering at "started"). +- **No migration.** `AiCostLog.metadata` is already `Json`; everything else aggregates over + existing models. + +## Build shape (branch `feat/F8.1-admin-analytics-dashboards`) + +- **Cost attribution** — metadata key standardised to `appQuestionnaireSessionId` in + `lib/app/questionnaire/capabilities/{extract-answer-slots,detect-contradictions,refine-answer,compose-completion-offer}.ts`. +- **Core aggregators** (pure, Prisma at the seam) — `lib/app/questionnaire/analytics/`: + `views.ts` (client-safe types), `query-schema.ts` (`questionnaireAnalyticsQuerySchema` + + `resolveAnalyticsScope`, reusing the platform date-range helpers), `distributions.ts`, + `funnel.ts`, `cost.ts` (raw SQL over `ai_cost_log` JSON metadata), `index.ts`. +- **API routes** (admin-only, master-flag-gated, version-scoped, read-only) — + `app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/{distributions,funnel,cost}/route.ts`; + endpoint constants in `lib/api/endpoints.ts`. +- **UI** — `app/admin/questionnaires/[id]/analytics/page.tsx` (server, parallel fetch), + `Analytics` link on the detail page, and the client island + `components/admin/questionnaires/analytics/{analytics-view,analytics-filters,question-distribution-panel,completion-funnel-panel,cost-panel}.tsx` + (tabs; recharts for the cost trend; CSS bars elsewhere; reuses `TagChip` + `FieldHelp`). + +## Tests + +- Unit (`tests/unit/lib/app/questionnaire/analytics/`): distribution math per type incl. + free-text value suppression and tag-filter wiring; funnel stage counting, drop-off math, + anonymous separation, preview exclusion; cost runtime/design split, per-capability merge, + top-session ranking, trend mapping, no-session short-circuit. 16 tests. + +## Known limitations / deferred + +- One-time ingest cost (`extract-questionnaire-structure`, stamped pre-version) is not + version-attributable and is excluded from the per-version cost view. +- JSON-path cost filters aren't indexed; fine at admin scale, revisit with a GIN index if needed. +- CSV/JSON result exports → F8.2. Cross-surface anonymous-mode PII audit → F8.3. + +## Docs and CHANGELOG + +- Admin doc: `.context/admin/questionnaire-analytics.md`. +- **No CHANGELOG entry** — the CHANGELOG tracks only the Sunrise platform surface; ConQuest + app routes/models are out of scope. diff --git a/app/admin/questionnaires/[id]/analytics/page.tsx b/app/admin/questionnaires/[id]/analytics/page.tsx new file mode 100644 index 000000000..e177d46a5 --- /dev/null +++ b/app/admin/questionnaires/[id]/analytics/page.tsx @@ -0,0 +1,207 @@ +import type { Metadata } from 'next'; +import Link from 'next/link'; +import { notFound } from 'next/navigation'; + +import { AnalyticsView } from '@/components/admin/questionnaires/analytics/analytics-view'; +import { API } from '@/lib/api/endpoints'; +import { parseApiResponse, serverFetch } from '@/lib/api/server-fetch'; +import { logger } from '@/lib/logging'; +import { isQuestionnairesEnabled } from '@/lib/app/questionnaire/feature-flag'; +import { getAnalyticsDefaultDateInputs } from '@/lib/app/questionnaire/analytics'; +import type { + CompletionFunnelResult, + QuestionDistributionsResult, + QuestionnaireCostResult, +} from '@/lib/app/questionnaire/analytics'; +import type { QuestionnaireDetail, TagView, VersionGraphView } from '@/lib/app/questionnaire/views'; + +export const metadata: Metadata = { + title: 'Analytics', + description: 'Per-question distributions, completion funnel, and cost actuals for a version.', +}; + +interface PageProps { + params: Promise<{ id: string }>; + searchParams: Promise<{ v?: string; from?: string; to?: string; tagIds?: string }>; +} + +/** Build the shared analytics query string (date window + tag filter). */ +function buildQuery(sp: { from?: string; to?: string; tagIds?: string }): string { + const qs = new URLSearchParams(); + if (sp.from) qs.set('from', sp.from); + if (sp.to) qs.set('to', sp.to); + if (sp.tagIds) qs.set('tagIds', sp.tagIds); + return qs.toString() ? `?${qs.toString()}` : ''; +} + +async function getDetail(id: string): Promise { + try { + const res = await serverFetch(API.APP.QUESTIONNAIRES.byId(id)); + if (!res.ok) return null; + const body = await parseApiResponse(res); + return body.success ? body.data : null; + } catch (err) { + logger.error('analytics page: detail fetch failed', err); + return null; + } +} + +async function getTagVocabulary(id: string, versionId: string): Promise { + try { + const res = await serverFetch(API.APP.QUESTIONNAIRES.versionGraph(id, versionId)); + if (!res.ok) return []; + const body = await parseApiResponse(res); + return body.success ? body.data.tags : []; + } catch (err) { + logger.error('analytics page: tag vocabulary fetch failed', err); + return []; + } +} + +async function getDistributions( + id: string, + versionId: string, + query: string +): Promise { + try { + const res = await serverFetch( + `${API.APP.QUESTIONNAIRES.versionAnalyticsDistributions(id, versionId)}${query}` + ); + if (!res.ok) return null; + const body = await parseApiResponse(res); + return body.success ? body.data : null; + } catch (err) { + logger.error('analytics page: distributions fetch failed', err); + return null; + } +} + +async function getFunnel( + id: string, + versionId: string, + query: string +): Promise { + try { + const res = await serverFetch( + `${API.APP.QUESTIONNAIRES.versionAnalyticsFunnel(id, versionId)}${query}` + ); + if (!res.ok) return null; + const body = await parseApiResponse(res); + return body.success ? body.data : null; + } catch (err) { + logger.error('analytics page: funnel fetch failed', err); + return null; + } +} + +async function getCost( + id: string, + versionId: string, + query: string +): Promise { + try { + const res = await serverFetch( + `${API.APP.QUESTIONNAIRES.versionAnalyticsCost(id, versionId)}${query}` + ); + if (!res.ok) return null; + const body = await parseApiResponse(res); + return body.success ? body.data : null; + } catch (err) { + logger.error('analytics page: cost fetch failed', err); + return null; + } +} + +export default async function QuestionnaireAnalyticsPage({ params, searchParams }: PageProps) { + if (!(await isQuestionnairesEnabled())) notFound(); + + const { id } = await params; + const sp = await searchParams; + + const detail = await getDetail(id); + if (!detail) notFound(); + + // Version selection mirrors the detail/evaluations pages: `?v=` or the newest. + const selected = detail.versions.find((ver) => ver.id === sp.v) ?? detail.versions[0] ?? null; + + const { from: defaultFrom, to: defaultTo } = getAnalyticsDefaultDateInputs(); + const filters = { + from: sp.from || defaultFrom, + to: sp.to || defaultTo, + tagIds: (sp.tagIds ?? '').split(',').filter((t) => t.length > 0), + }; + const query = buildQuery(sp); + + const [tagVocabulary, distributions, funnel, cost] = selected + ? await Promise.all([ + getTagVocabulary(id, selected.id), + getDistributions(id, selected.id, query), + getFunnel(id, selected.id, query), + getCost(id, selected.id, query), + ]) + : [[] as TagView[], null, null, null]; + + return ( +
+ + +
+

Analytics

+

+ The read-side view of completed-session data for a version: per-question distributions, + the invitation completion funnel, and cost actuals. Aggregate-only — individual free-text + answers are never shown. +

+
+ + {detail.versions.length === 0 || !selected ? ( +

This questionnaire has no versions.

+ ) : ( + <> + {/* Version selector — SSR links that set ?v= on this sub-route. */} +
+ {detail.versions.map((ver) => { + const active = ver.id === selected.id; + return ( + + v{ver.versionNumber} + + {' '} + · {ver.status} + + + ); + })} +
+ + + + )} +
+ ); +} diff --git a/app/admin/questionnaires/[id]/page.tsx b/app/admin/questionnaires/[id]/page.tsx index 94f5a6a79..17131f039 100644 --- a/app/admin/questionnaires/[id]/page.tsx +++ b/app/admin/questionnaires/[id]/page.tsx @@ -176,6 +176,13 @@ export default async function QuestionnaireDetailPage({ params, searchParams }: + {/* Analytics (F8.1) — read-side completed-session view, scoped to the + selected version. Always available (read-only, no paid work). */} + {/* Design-time evaluation (F5.2) — only when the sub-flag is on (the run route 404s otherwise), scoped to the selected version. */} {designEvalEnabled && ( diff --git a/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/cost/route.ts b/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/cost/route.ts new file mode 100644 index 000000000..2ddb5a394 --- /dev/null +++ b/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/cost/route.ts @@ -0,0 +1,48 @@ +/** + * Per-version cost actuals (F8.1). + * + * GET /api/v1/app/questionnaires/:id/versions/:vid/analytics/cost + * Admin-only. Aggregates the version's `AiCostLog` spend over the window: total, + * runtime vs design-time split, per-capability breakdown, daily trend, and top + * sessions by spend. Query params: `from`/`to` (YYYY-MM-DD, default last 30 days). + * Read-only — master-flag-gated and version-scoped; no sub-flag. + */ + +import { errorResponse, successResponse } from '@/lib/api/responses'; +import { getRouteLogger } from '@/lib/api/context'; +import { withAdminAuth } from '@/lib/auth/guards'; +import { validateQueryParams } from '@/lib/api/validation'; + +import { withQuestionnairesEnabled } from '@/lib/app/questionnaire/feature-flag'; +import { + questionnaireAnalyticsQuerySchema, + resolveAnalyticsScope, + getQuestionnaireCostBreakdown, +} from '@/lib/app/questionnaire/analytics'; +import { loadScopedVersion } from '@/app/api/v1/app/questionnaires/_lib/authoring-routes'; + +const handleGet = withAdminAuth<{ id: string; vid: string }>( + async (request, _session, { params }) => { + const log = await getRouteLogger(request); + const { id, vid } = await params; + + const scoped = await loadScopedVersion(id, vid); + if (!scoped) { + return errorResponse('Questionnaire version not found', { code: 'NOT_FOUND', status: 404 }); + } + + const { searchParams } = new URL(request.url); + const query = validateQueryParams(searchParams, questionnaireAnalyticsQuerySchema); + const scope = resolveAnalyticsScope(vid, query); + + const result = await getQuestionnaireCostBreakdown(scope); + log.info('Questionnaire analytics cost computed', { + versionId: vid, + totalCostUsd: result.totalCostUsd, + }); + + return successResponse(result); + } +); + +export const GET = withQuestionnairesEnabled(handleGet); diff --git a/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/distributions/route.ts b/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/distributions/route.ts new file mode 100644 index 000000000..a87267d7b --- /dev/null +++ b/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/distributions/route.ts @@ -0,0 +1,50 @@ +/** + * Per-question answer distributions (F8.1). + * + * GET /api/v1/app/questionnaires/:id/versions/:vid/analytics/distributions + * Admin-only. Aggregates the answers captured across a version's non-preview + * sessions into a per-question, type-appropriate distribution. Query params: + * `from`/`to` (YYYY-MM-DD, default last 30 days), `tagIds` (comma-separated, + * restricts to tagged questions). Read-only — master-flag-gated and + * version-scoped; no sub-flag (no paid work). + */ + +import { errorResponse, successResponse } from '@/lib/api/responses'; +import { getRouteLogger } from '@/lib/api/context'; +import { withAdminAuth } from '@/lib/auth/guards'; +import { validateQueryParams } from '@/lib/api/validation'; + +import { withQuestionnairesEnabled } from '@/lib/app/questionnaire/feature-flag'; +import { + questionnaireAnalyticsQuerySchema, + resolveAnalyticsScope, + getQuestionDistributions, +} from '@/lib/app/questionnaire/analytics'; +import { loadScopedVersion } from '@/app/api/v1/app/questionnaires/_lib/authoring-routes'; + +const handleGet = withAdminAuth<{ id: string; vid: string }>( + async (request, _session, { params }) => { + const log = await getRouteLogger(request); + const { id, vid } = await params; + + const scoped = await loadScopedVersion(id, vid); + if (!scoped) { + return errorResponse('Questionnaire version not found', { code: 'NOT_FOUND', status: 404 }); + } + + const { searchParams } = new URL(request.url); + const query = validateQueryParams(searchParams, questionnaireAnalyticsQuerySchema); + const scope = resolveAnalyticsScope(vid, query); + + const result = await getQuestionDistributions(scope); + log.info('Questionnaire analytics distributions computed', { + versionId: vid, + totalSessions: result.totalSessions, + questionCount: result.questions.length, + }); + + return successResponse(result); + } +); + +export const GET = withQuestionnairesEnabled(handleGet); diff --git a/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/funnel/route.ts b/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/funnel/route.ts new file mode 100644 index 000000000..542ae118e --- /dev/null +++ b/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/funnel/route.ts @@ -0,0 +1,49 @@ +/** + * Completion funnel (F8.1). + * + * GET /api/v1/app/questionnaires/:id/versions/:vid/analytics/funnel + * Admin-only. Computes the invited → opened → started → completed funnel for a + * version, with per-stage drop-off and a separate count of anonymous (un-invited) + * sessions. Query params: `from`/`to` (YYYY-MM-DD, default last 30 days). + * Read-only — master-flag-gated and version-scoped; no sub-flag. + */ + +import { errorResponse, successResponse } from '@/lib/api/responses'; +import { getRouteLogger } from '@/lib/api/context'; +import { withAdminAuth } from '@/lib/auth/guards'; +import { validateQueryParams } from '@/lib/api/validation'; + +import { withQuestionnairesEnabled } from '@/lib/app/questionnaire/feature-flag'; +import { + questionnaireAnalyticsQuerySchema, + resolveAnalyticsScope, + getCompletionFunnel, +} from '@/lib/app/questionnaire/analytics'; +import { loadScopedVersion } from '@/app/api/v1/app/questionnaires/_lib/authoring-routes'; + +const handleGet = withAdminAuth<{ id: string; vid: string }>( + async (request, _session, { params }) => { + const log = await getRouteLogger(request); + const { id, vid } = await params; + + const scoped = await loadScopedVersion(id, vid); + if (!scoped) { + return errorResponse('Questionnaire version not found', { code: 'NOT_FOUND', status: 404 }); + } + + const { searchParams } = new URL(request.url); + const query = validateQueryParams(searchParams, questionnaireAnalyticsQuerySchema); + const scope = resolveAnalyticsScope(vid, query); + + const result = await getCompletionFunnel(scope); + log.info('Questionnaire analytics funnel computed', { + versionId: vid, + invited: result.stages[0]?.count ?? 0, + completed: result.stages[result.stages.length - 1]?.count ?? 0, + }); + + return successResponse(result); + } +); + +export const GET = withQuestionnairesEnabled(handleGet); diff --git a/components/admin/questionnaires/analytics/analytics-filters.tsx b/components/admin/questionnaires/analytics/analytics-filters.tsx new file mode 100644 index 000000000..450cf3f0c --- /dev/null +++ b/components/admin/questionnaires/analytics/analytics-filters.tsx @@ -0,0 +1,139 @@ +'use client'; + +/** + * Shared scope/filter control for the F8.1 analytics views. + * + * Drives the date window and tag filter through the URL (so the SSR page re-fetches + * and the filter state is shareable/back-button-safe), preserving the `?v=` version + * selection the page owns. The tag filter only affects the distributions view; the + * funnel and cost views ignore `tagIds`. + */ + +import { useRouter, usePathname, useSearchParams } from 'next/navigation'; + +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { FieldHelp } from '@/components/ui/field-help'; +import { TagChip } from '@/components/admin/questionnaires/tag-chip'; +import { cn } from '@/lib/utils'; +import type { TagView } from '@/lib/app/questionnaire/views'; + +export interface AnalyticsFiltersProps { + tagVocabulary: TagView[]; + filters: { from: string; to: string; tagIds: string[] }; +} + +export function AnalyticsFilters({ tagVocabulary, filters }: AnalyticsFiltersProps) { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + + const push = (mutate: (params: URLSearchParams) => void) => { + const params = new URLSearchParams(searchParams.toString()); + mutate(params); + router.push(`${pathname}?${params.toString()}`, { scroll: false }); + }; + + const setDate = (key: 'from' | 'to', value: string) => + push((params) => { + if (value) params.set(key, value); + else params.delete(key); + }); + + const toggleTag = (tagId: string) => + push((params) => { + const next = new Set(filters.tagIds); + if (next.has(tagId)) next.delete(tagId); + else next.add(tagId); + if (next.size > 0) params.set('tagIds', [...next].join(',')); + else params.delete('tagIds'); + }); + + const clearTags = () => + push((params) => { + params.delete('tagIds'); + }); + + const selected = new Set(filters.tagIds); + + return ( +
+
+
+ + setDate('from', e.target.value)} + className="w-40" + /> +
+
+ + setDate('to', e.target.value)} + className="w-40" + /> +
+
+ + {tagVocabulary.length > 0 && ( +
+
+ + Filter questions by tag + + + Restricts the per-question distributions to questions carrying any selected tag. Does + not affect the funnel or cost views. + + {selected.size > 0 && ( + + )} +
+
+ {tagVocabulary.map((tag) => { + const isOn = selected.has(tag.id); + return ( + + ); + })} +
+
+ )} +
+ ); +} diff --git a/components/admin/questionnaires/analytics/analytics-view.tsx b/components/admin/questionnaires/analytics/analytics-view.tsx new file mode 100644 index 000000000..29c87cd26 --- /dev/null +++ b/components/admin/questionnaires/analytics/analytics-view.tsx @@ -0,0 +1,60 @@ +'use client'; + +/** + * F8.1 analytics client island. + * + * Renders the shared filter, then the three analytics surfaces in tabs. The version + * selection and SSR data fetch live in the page; this component is presentational + + * URL-driven filtering only (no per-row fetches). + */ + +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { AnalyticsFilters } from '@/components/admin/questionnaires/analytics/analytics-filters'; +import { QuestionDistributionPanel } from '@/components/admin/questionnaires/analytics/question-distribution-panel'; +import { CompletionFunnelPanel } from '@/components/admin/questionnaires/analytics/completion-funnel-panel'; +import { CostPanel } from '@/components/admin/questionnaires/analytics/cost-panel'; +import type { TagView } from '@/lib/app/questionnaire/views'; +import type { + CompletionFunnelResult, + QuestionDistributionsResult, + QuestionnaireCostResult, +} from '@/lib/app/questionnaire/analytics'; + +export interface AnalyticsViewProps { + tagVocabulary: TagView[]; + distributions: QuestionDistributionsResult | null; + funnel: CompletionFunnelResult | null; + cost: QuestionnaireCostResult | null; + filters: { from: string; to: string; tagIds: string[] }; +} + +export function AnalyticsView({ + tagVocabulary, + distributions, + funnel, + cost, + filters, +}: AnalyticsViewProps) { + return ( +
+ + + + + Distributions + Completion funnel + Cost + + + + + + + + + + + +
+ ); +} diff --git a/components/admin/questionnaires/analytics/completion-funnel-panel.tsx b/components/admin/questionnaires/analytics/completion-funnel-panel.tsx new file mode 100644 index 000000000..250ef8a75 --- /dev/null +++ b/components/admin/questionnaires/analytics/completion-funnel-panel.tsx @@ -0,0 +1,95 @@ +'use client'; + +/** + * Completion funnel panel (F8.1): invited → opened → started → completed. + * + * Each stage is a horizontal bar widthed by its retention from the first stage, + * annotated with the count, step conversion, and absolute drop-off. Anonymous + * (un-invited) sessions are shown separately since they don't pass through the + * invite stages. + */ + +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import type { CompletionFunnelResult, FunnelStage } from '@/lib/app/questionnaire/analytics'; + +function pct(n: number): string { + return `${(n * 100).toFixed(0)}%`; +} + +function StageBar({ stage, isFirst }: { stage: FunnelStage; isFirst: boolean }) { + const width = Math.max(2, stage.retention * 100); + return ( +
+
+ {stage.label} + + {stage.count} · {pct(stage.retention)} of invited + +
+
+
+ {stage.count > 0 ? stage.count : ''} +
+
+ {!isFirst && ( +

+ {pct(stage.conversionFromPrev)} from previous · {stage.dropoff} dropped +

+ )} +
+ ); +} + +export function CompletionFunnelPanel({ data }: { data: CompletionFunnelResult | null }) { + if (!data) { + return

Funnel data could not be loaded.

; + } + + const invited = data.stages[0]?.count ?? 0; + + return ( +
+ + + Invitation funnel + + + {invited === 0 ? ( +

+ No invitations sent in this window. +

+ ) : ( + data.stages.map((stage, i) => ( + + )) + )} + + + + + + Anonymous sessions + + +

+ Public-link respondents with no invitation. They enter at “started”, so they sit outside + the invite funnel. +

+
+
+
{data.anonymous.started}
+
started
+
+
+
{data.anonymous.completed}
+
completed
+
+
+ + +
+ ); +} diff --git a/components/admin/questionnaires/analytics/cost-panel.tsx b/components/admin/questionnaires/analytics/cost-panel.tsx new file mode 100644 index 000000000..c01b6cfbe --- /dev/null +++ b/components/admin/questionnaires/analytics/cost-panel.tsx @@ -0,0 +1,159 @@ +'use client'; + +/** + * Cost panel (F8.1): per-version spend from `AiCostLog`. + * + * Summary cards (total / runtime / design-time), a per-capability breakdown, a daily + * spend trend (recharts line), and the top respondent sessions by spend. + */ + +import { + CartesianGrid, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { Badge } from '@/components/ui/badge'; +import { formatUsd } from '@/lib/utils/format-currency'; +import type { QuestionnaireCostResult } from '@/lib/app/questionnaire/analytics'; + +function StatCard({ label, value }: { label: string; value: string }) { + return ( + + +
{value}
+
{label}
+ + + ); +} + +export function CostPanel({ data }: { data: QuestionnaireCostResult | null }) { + if (!data) { + return

Cost data could not be loaded.

; + } + + const maxCapability = Math.max(1, ...data.byCapability.map((c) => c.costUsd)); + + return ( +
+
+ + + +
+ +
+ + + Spend by capability + + + {data.byCapability.length === 0 ? ( +

No spend in this window.

+ ) : ( +
+ {data.byCapability.map((c) => ( +
+ + {c.label} + +
+
+
+ + {formatUsd(c.costUsd)} + +
+ ))} +
+ )} + + + + + + Daily spend + + + {data.trend.length === 0 ? ( +

No spend to chart.

+ ) : ( +
+ + + + + formatUsd(v, { compact: true })} + /> + [formatUsd(Number(value)), 'Spend']} + /> + + + +
+ )} + + +
+ + + + Top sessions by spend + + + {data.topSessions.length === 0 ? ( +

No respondent session spend yet.

+ ) : ( + + + + Session + Status + Started + Spend + + + + {data.topSessions.map((s) => ( + + {s.sessionId.slice(0, 12)}… + + {s.status} + + + {new Date(s.createdAt).toLocaleDateString()} + + + {formatUsd(s.costUsd)} + + + ))} + +
+ )} + + +
+ ); +} diff --git a/components/admin/questionnaires/analytics/question-distribution-panel.tsx b/components/admin/questionnaires/analytics/question-distribution-panel.tsx new file mode 100644 index 000000000..381fa3a25 --- /dev/null +++ b/components/admin/questionnaires/analytics/question-distribution-panel.tsx @@ -0,0 +1,194 @@ +'use client'; + +/** + * Per-question distribution panel (F8.1). + * + * One card per question, with a type-appropriate breakdown rendered as lightweight + * CSS bars (no chart lib instance per question — there can be many). Free-text + * questions show only response rate / confidence / provenance; their answer values + * are never rendered (PII-safe by design). + */ + +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { TagChip } from '@/components/admin/questionnaires/tag-chip'; +import { QUESTION_TYPE_LABELS } from '@/lib/app/questionnaire/types'; +import type { + DistributionDetail, + QuestionDistribution, + QuestionDistributionsResult, +} from '@/lib/app/questionnaire/analytics'; + +function pct(n: number): string { + return `${(n * 100).toFixed(0)}%`; +} + +function confidenceLabel(n: number | null): string { + return n == null ? '—' : `${(n * 100).toFixed(0)}%`; +} + +/** A labelled horizontal bar: `label …… count` with a fill proportional to `max`. */ +function BarRow({ label, count, max }: { label: string; count: number; max: number }) { + const width = max > 0 ? Math.max(2, (count / max) * 100) : 0; + return ( +
+ + {label} + +
+
+
+ {count} +
+ ); +} + +function DetailBody({ detail }: { detail: DistributionDetail }) { + switch (detail.kind) { + case 'free_text': + return ( +

+ Free-text responses — values are not shown. See response rate and confidence above. +

+ ); + + case 'choice': { + const max = Math.max(1, ...detail.buckets.map((b) => b.count)); + return ( +
+ {detail.buckets.map((b) => ( + + ))} + {detail.otherCount > 0 && ( + + )} +
+ ); + } + + case 'likert': { + const max = Math.max(1, ...detail.buckets.map((b) => b.count)); + return ( +
+ {detail.buckets.map((b) => ( + + ))} + {detail.mean != null && ( +

Mean: {detail.mean.toFixed(2)}

+ )} +
+ ); + } + + case 'numeric': { + if (!detail.summary) { + return

No numeric answers yet.

; + } + const { summary, histogram } = detail; + const max = Math.max(1, ...histogram.map((b) => b.count)); + return ( +
+
+ min {summary.min} + max {summary.max} + mean {summary.mean.toFixed(1)} + median {summary.median} +
+
+ {histogram.map((b) => ( + + ))} +
+
+ ); + } + + case 'boolean': { + const max = Math.max(1, detail.trueCount, detail.falseCount); + return ( +
+ + +
+ ); + } + + case 'date': { + if (detail.buckets.length === 0) { + return

No date answers yet.

; + } + const max = Math.max(1, ...detail.buckets.map((b) => b.count)); + return ( +
+ {detail.buckets.map((b) => ( + + ))} +
+ ); + } + } +} + +function QuestionCard({ q }: { q: QuestionDistribution }) { + return ( + + +
+ {q.prompt} + + {QUESTION_TYPE_LABELS[q.type]} + +
+
+ {q.sectionTitle} + + {q.answeredCount} answered · {pct(q.responseRate)} response rate + + avg confidence {confidenceLabel(q.avgConfidence)} + {q.required && required} +
+ {q.tags.length > 0 && ( +
+ {q.tags.map((t) => ( + + ))} +
+ )} + + + +
+ direct {q.provenance.direct} + inferred {q.provenance.inferred} + synthesised {q.provenance.synthesised} + refined {q.provenance.refined} +
+ + + ); +} + +export function QuestionDistributionPanel({ data }: { data: QuestionDistributionsResult | null }) { + if (!data) { + return

Distribution data could not be loaded.

; + } + return ( +
+

+ {data.totalSessions} session{data.totalSessions === 1 ? '' : 's'} in range ·{' '} + {data.completedSessions} completed +

+ {data.questions.length === 0 ? ( +

+ No questions match the current filter. +

+ ) : ( +
+ {data.questions.map((q) => ( + + ))} +
+ )} +
+ ); +} diff --git a/lib/api/endpoints.ts b/lib/api/endpoints.ts index fdedfdadd..90bbb7cbe 100644 --- a/lib/api/endpoints.ts +++ b/lib/api/endpoints.ts @@ -430,6 +430,15 @@ export const API = { findingId: string ): string => `/api/v1/app/questionnaires/${id}/versions/${versionId}/evaluations/${runId}/findings/${findingId}/apply`, + /** Per-question answer distributions for a version (GET — F8.1). */ + versionAnalyticsDistributions: (id: string, versionId: string): string => + `/api/v1/app/questionnaires/${id}/versions/${versionId}/analytics/distributions`, + /** Completion funnel (invited → opened → started → completed) for a version (GET — F8.1). */ + versionAnalyticsFunnel: (id: string, versionId: string): string => + `/api/v1/app/questionnaires/${id}/versions/${versionId}/analytics/funnel`, + /** Per-version cost actuals from `AiCostLog` (GET — F8.1). */ + versionAnalyticsCost: (id: string, versionId: string): string => + `/api/v1/app/questionnaires/${id}/versions/${versionId}/analytics/cost`, /** Invitations for a questionnaire (GET list, POST send single/bulk — F3.2). */ invitations: (id: string): string => `/api/v1/app/questionnaires/${id}/invitations`, /** Single invitation (PATCH revoke — F3.2). */ diff --git a/lib/app/questionnaire/analytics/cost.ts b/lib/app/questionnaire/analytics/cost.ts new file mode 100644 index 000000000..f0a95514d --- /dev/null +++ b/lib/app/questionnaire/analytics/cost.ts @@ -0,0 +1,212 @@ +/** + * Per-version cost actuals (F8.1) — read from the platform `AiCostLog` ledger. + * + * Questionnaire LLM spend lands in `ai_cost_log` tagged via `metadata`: + * - live respondent turns → `metadata.appQuestionnaireSessionId` (the session cuid) + * - design-time work (evaluate)→ `metadata.versionId` + * so a version's spend = the rows for its non-preview sessions (runtime) plus the + * rows tagged with its version id (design-time). The two tag sets are disjoint, so + * summing them never double-counts. + * + * `AppQuestionnaireTurn.costUsd` stays the F6.3 budget-enforcement basis and is left + * untouched; this surface reads the richer ledger so it can break spend down by the + * capability that incurred it. + * + * Postgres-only raw SQL (the repo is already pinned to Postgres via pgvector) — the + * metadata filters are JSON path extractions Prisma's typed `where` can't express + * for a set of session ids. Mirrors `lib/orchestration/llm/cost-reports.ts`. + */ + +import { prisma } from '@/lib/db/client'; +import { narrowToEnum, SESSION_STATUSES, type SessionStatus } from '@/lib/app/questionnaire/types'; +import type { AnalyticsScope } from '@/lib/app/questionnaire/analytics/query-schema'; +import type { + CostCapabilityBucket, + CostDayPoint, + QuestionnaireCostResult, + SessionCostRow, +} from '@/lib/app/questionnaire/analytics/views'; + +/** Cap on the top-spending sessions table. */ +const TOP_SESSIONS_LIMIT = 10; + +/** Human labels for the capability slugs that stamp cost rows; falls back to the raw key. */ +const CAPABILITY_LABELS: Record = { + extract_answer_slots: 'Answer extraction', + detect_contradictions: 'Contradiction detection', + refine_answer: 'Answer refinement', + compose_completion_offer: 'Completion offer', + evaluate_structure: 'Design evaluation', + extract_questionnaire_structure: 'Structure extraction', + chat: 'Question selection', +}; + +function labelFor(key: string): string { + return CAPABILITY_LABELS[key] ?? key; +} + +/** Coerce a raw-SQL aggregate (bigint / numeric / null) to a finite JS number. */ +function num(value: unknown): number { + if (value === null || value === undefined) return 0; + const n = Number(value); + return Number.isFinite(n) ? n : 0; +} + +interface CapabilityRow { + capability: string | null; + cost: number | string | null; + calls: bigint | number | null; +} +interface SessionRow { + session_id: string | null; + cost: number | string | null; +} +interface TrendRow { + day: Date | string; + cost: number | string | null; +} + +/** + * Aggregate a version's `AiCostLog` spend over the window: total, runtime vs + * design-time split, per-capability breakdown, daily trend, and top sessions. + */ +export async function getQuestionnaireCostBreakdown( + scope: AnalyticsScope +): Promise { + const range = { from: scope.from.toISOString(), to: scope.to.toISOString() }; + + // Non-preview sessions for the version (all time — cost rows are date-filtered by + // their own `createdAt`). Carries status + createdAt for the top-sessions table. + const sessions = await prisma.appQuestionnaireSession.findMany({ + where: { versionId: scope.versionId, isPreview: false }, + select: { id: true, status: true, createdAt: true }, + }); + const sessionMeta = new Map(sessions.map((s) => [s.id, s])); + const sessionIds = sessions.map((s) => s.id); + const hasSessions = sessionIds.length > 0; + + // Per-capability merge (runtime ∪ design-time). + const byCapability = new Map(); + const addCapability = (key: string, costUsd: number, calls: number) => { + const cur = byCapability.get(key) ?? { costUsd: 0, callCount: 0 }; + cur.costUsd += costUsd; + cur.callCount += calls; + byCapability.set(key, cur); + }; + + // 1. Runtime cost: rows tagged with one of this version's session ids. + let runtimeCostUsd = 0; + const sessionCost = new Map(); + if (hasSessions) { + const capRows = await prisma.$queryRawUnsafe( + ` + SELECT COALESCE("metadata"->>'capability', operation) AS capability, + SUM("totalCostUsd") AS cost, + COUNT(*) AS calls + FROM "ai_cost_log" + WHERE "createdAt" >= $1 AND "createdAt" < $2 + AND "metadata"->>'appQuestionnaireSessionId' = ANY($3::text[]) + GROUP BY capability + `, + scope.from, + scope.to, + sessionIds + ); + for (const r of capRows) { + const cost = num(r.cost); + runtimeCostUsd += cost; + addCapability(r.capability ?? 'chat', cost, num(r.calls)); + } + + const sessionRows = await prisma.$queryRawUnsafe( + ` + SELECT "metadata"->>'appQuestionnaireSessionId' AS session_id, + SUM("totalCostUsd") AS cost + FROM "ai_cost_log" + WHERE "createdAt" >= $1 AND "createdAt" < $2 + AND "metadata"->>'appQuestionnaireSessionId' = ANY($3::text[]) + GROUP BY session_id + `, + scope.from, + scope.to, + sessionIds + ); + for (const r of sessionRows) { + if (r.session_id) sessionCost.set(r.session_id, num(r.cost)); + } + } + + // 2. Design-time cost: rows tagged with this version id. + let designTimeCostUsd = 0; + const designRows = await prisma.$queryRawUnsafe( + ` + SELECT COALESCE("metadata"->>'capability', operation) AS capability, + SUM("totalCostUsd") AS cost, + COUNT(*) AS calls + FROM "ai_cost_log" + WHERE "createdAt" >= $1 AND "createdAt" < $2 + AND "metadata"->>'versionId' = $3 + GROUP BY capability + `, + scope.from, + scope.to, + scope.versionId + ); + for (const r of designRows) { + const cost = num(r.cost); + designTimeCostUsd += cost; + addCapability(r.capability ?? 'chat', cost, num(r.calls)); + } + + // 3. Daily trend over both attribution paths. + const trendParams: unknown[] = [scope.from, scope.to, scope.versionId]; + let attribution = `"metadata"->>'versionId' = $3`; + if (hasSessions) { + trendParams.push(sessionIds); + attribution = `("metadata"->>'versionId' = $3 OR "metadata"->>'appQuestionnaireSessionId' = ANY($4::text[]))`; + } + const trendRows = await prisma.$queryRawUnsafe( + ` + SELECT date_trunc('day', "createdAt") AS day, SUM("totalCostUsd") AS cost + FROM "ai_cost_log" + WHERE "createdAt" >= $1 AND "createdAt" < $2 AND ${attribution} + GROUP BY day + ORDER BY day ASC + `, + ...trendParams + ); + const trend: CostDayPoint[] = trendRows.map((r) => ({ + date: (r.day instanceof Date ? r.day : new Date(r.day)).toISOString().slice(0, 10), + costUsd: num(r.cost), + })); + + const capabilityBuckets: CostCapabilityBucket[] = [...byCapability.entries()] + .map(([key, v]) => ({ key, label: labelFor(key), costUsd: v.costUsd, callCount: v.callCount })) + .sort((a, b) => b.costUsd - a.costUsd); + + const topSessions: SessionCostRow[] = [...sessionCost.entries()] + .map(([sessionId, costUsd]) => { + const meta = sessionMeta.get(sessionId); + return { + sessionId, + status: meta + ? narrowToEnum(meta.status, SESSION_STATUSES, 'active') + : 'active', + costUsd, + createdAt: (meta?.createdAt ?? scope.from).toISOString(), + }; + }) + .sort((a, b) => b.costUsd - a.costUsd) + .slice(0, TOP_SESSIONS_LIMIT); + + return { + versionId: scope.versionId, + range, + totalCostUsd: runtimeCostUsd + designTimeCostUsd, + runtimeCostUsd, + designTimeCostUsd, + byCapability: capabilityBuckets, + trend, + topSessions, + }; +} diff --git a/lib/app/questionnaire/analytics/distributions.ts b/lib/app/questionnaire/analytics/distributions.ts new file mode 100644 index 000000000..66880f2ba --- /dev/null +++ b/lib/app/questionnaire/analytics/distributions.ts @@ -0,0 +1,329 @@ +/** + * Per-question answer distributions (F8.1). + * + * For each question in a version (optionally filtered to a tag set), aggregate the + * answers captured across the **non-preview** sessions in the date window into a + * type-appropriate distribution: choice/likert option counts, a numeric summary + + * histogram, boolean true/false, date buckets. `free_text` deliberately carries no + * value detail — only response rate, confidence, and provenance — so respondent + * prose never crosses this boundary (PII-safe ahead of F8.3). + * + * Three queries, no N+1: the version's slots (+ section + tags), the sessions in + * scope (the response-rate denominator), and the answer rows for those sessions. + * Everything else is in-memory grouping. + */ + +import { prisma } from '@/lib/db/client'; +import { + ANSWER_PROVENANCES, + QUESTION_TYPE_LABELS, + narrowToEnum, + type AnswerProvenance, + type QuestionType, + QUESTION_TYPES, +} from '@/lib/app/questionnaire/types'; +import { typeConfigSchemaFor } from '@/lib/app/questionnaire/authoring/type-config-schema'; +import type { TagColor } from '@/lib/app/questionnaire/types'; +import type { TagView } from '@/lib/app/questionnaire/views'; +import type { AnalyticsScope } from '@/lib/app/questionnaire/analytics/query-schema'; +import type { + DistributionDetail, + HistogramBin, + ProvenanceBreakdown, + QuestionDistribution, + QuestionDistributionsResult, + ValueBucket, +} from '@/lib/app/questionnaire/analytics/views'; + +/** Max histogram bins for a numeric question (Sturges-ish, capped for a tidy chart). */ +const MAX_NUMERIC_BINS = 8; + +/** A zeroed provenance breakdown — one counter per label in the vocabulary. */ +function emptyProvenance(): ProvenanceBreakdown { + const acc: ProvenanceBreakdown = { direct: 0, inferred: 0, synthesised: 0, refined: 0 }; + return acc; +} + +/** Read a choice question's `[{ value, label }]` options from its stored config. */ +function readChoices( + type: QuestionType, + typeConfig: unknown +): Array<{ value: string; label: string }> { + const parsed = typeConfigSchemaFor(type).safeParse(typeConfig); + if (!parsed.success) return []; + const cfg = parsed.data as { choices?: Array<{ value: string; label: string }> }; + return Array.isArray(cfg.choices) ? cfg.choices : []; +} + +function readLikert(typeConfig: unknown): { + min: number; + max: number; + minLabel?: string; + maxLabel?: string; +} | null { + const parsed = typeConfigSchemaFor('likert').safeParse(typeConfig); + if (!parsed.success) return null; + return parsed.data as { min: number; max: number; minLabel?: string; maxLabel?: string }; +} + +function readBooleanLabels(typeConfig: unknown): { trueLabel: string; falseLabel: string } { + const parsed = typeConfigSchemaFor('boolean').safeParse(typeConfig ?? {}); + const cfg = parsed.success ? (parsed.data as { trueLabel?: string; falseLabel?: string }) : {}; + return { trueLabel: cfg.trueLabel ?? 'True', falseLabel: cfg.falseLabel ?? 'False' }; +} + +/** Coerce a stored answer value to a finite number, or null. Mirrors extraction's strictness. */ +function asFiniteNumber(value: unknown): number | null { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + return null; +} + +function median(sorted: number[]): number { + const n = sorted.length; + if (n === 0) return 0; + const mid = Math.floor(n / 2); + return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +/** Build a numeric histogram with up to {@link MAX_NUMERIC_BINS} equal-width bins. */ +function buildHistogram(values: number[]): HistogramBin[] { + if (values.length === 0) return []; + const min = Math.min(...values); + const max = Math.max(...values); + if (min === max) { + return [{ label: `${min}`, min, max, count: values.length }]; + } + const binCount = Math.min(MAX_NUMERIC_BINS, Math.max(1, Math.ceil(Math.sqrt(values.length)))); + const width = (max - min) / binCount; + const bins: HistogramBin[] = Array.from({ length: binCount }, (_, i) => { + const lo = min + i * width; + const hi = i === binCount - 1 ? max : min + (i + 1) * width; + return { + label: `${formatBound(lo)}–${formatBound(hi)}`, + min: lo, + max: hi, + count: 0, + }; + }); + for (const v of values) { + // Top bin is inclusive of `max`; others are half-open [lo, hi). + const idx = v === max ? binCount - 1 : Math.min(binCount - 1, Math.floor((v - min) / width)); + bins[idx].count += 1; + } + return bins; +} + +function formatBound(n: number): string { + return Number.isInteger(n) ? `${n}` : n.toFixed(1); +} + +/** Compute the type-appropriate {@link DistributionDetail} from a question's answers. */ +function buildDetail( + type: QuestionType, + typeConfig: unknown, + values: unknown[] +): DistributionDetail { + switch (type) { + case 'free_text': + return { kind: 'free_text' }; + + case 'single_choice': + case 'multi_choice': { + const choices = readChoices(type, typeConfig); + const labelByValue = new Map(choices.map((c) => [c.value, c.label])); + const counts = new Map(); + let otherCount = 0; + for (const raw of values) { + const picks = type === 'multi_choice' ? (Array.isArray(raw) ? raw : []) : [raw]; + for (const pick of picks) { + if (typeof pick !== 'string') continue; + if (labelByValue.has(pick)) { + counts.set(pick, (counts.get(pick) ?? 0) + 1); + } else { + otherCount += 1; + } + } + } + const buckets: ValueBucket[] = choices.map((c) => ({ + value: c.value, + label: c.label, + count: counts.get(c.value) ?? 0, + })); + return { kind: 'choice', buckets, otherCount }; + } + + case 'likert': { + const bounds = readLikert(typeConfig); + const min = bounds?.min ?? 1; + const max = bounds?.max ?? 5; + const counts = new Map(); + const nums: number[] = []; + for (const raw of values) { + const n = asFiniteNumber(raw); + if (n === null || !Number.isInteger(n) || n < min || n > max) continue; + counts.set(n, (counts.get(n) ?? 0) + 1); + nums.push(n); + } + const buckets: ValueBucket[] = []; + for (let v = min; v <= max; v += 1) { + const label = + v === min && bounds?.minLabel + ? `${v} (${bounds.minLabel})` + : v === max && bounds?.maxLabel + ? `${v} (${bounds.maxLabel})` + : `${v}`; + buckets.push({ value: `${v}`, label, count: counts.get(v) ?? 0 }); + } + const mean = nums.length > 0 ? nums.reduce((a, b) => a + b, 0) / nums.length : null; + return { kind: 'likert', min, max, buckets, mean }; + } + + case 'numeric': { + const nums = values.map(asFiniteNumber).filter((n): n is number => n !== null); + if (nums.length === 0) { + return { kind: 'numeric', summary: null, histogram: [] }; + } + const sorted = [...nums].sort((a, b) => a - b); + const summary = { + count: nums.length, + min: sorted[0], + max: sorted[sorted.length - 1], + mean: nums.reduce((a, b) => a + b, 0) / nums.length, + median: median(sorted), + }; + return { kind: 'numeric', summary, histogram: buildHistogram(nums) }; + } + + case 'boolean': { + const { trueLabel, falseLabel } = readBooleanLabels(typeConfig); + let trueCount = 0; + let falseCount = 0; + for (const raw of values) { + if (raw === true) trueCount += 1; + else if (raw === false) falseCount += 1; + } + return { kind: 'boolean', trueLabel, falseLabel, trueCount, falseCount }; + } + + case 'date': { + // Bucket by calendar month — coarse enough to read, fine enough to spot spread. + const counts = new Map(); + for (const raw of values) { + if (typeof raw !== 'string') continue; + const month = raw.slice(0, 7); // YYYY-MM from an ISO date/datetime + if (!/^\d{4}-\d{2}$/.test(month)) continue; + counts.set(month, (counts.get(month) ?? 0) + 1); + } + const buckets = [...counts.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([label, count]) => ({ label, count })); + return { kind: 'date', buckets }; + } + } +} + +/** + * Aggregate per-question answer distributions for a version over the non-preview + * sessions in scope. Tag-filtered when `scope.tagIds` is non-empty. + */ +export async function getQuestionDistributions( + scope: AnalyticsScope +): Promise { + const range = { from: scope.from.toISOString(), to: scope.to.toISOString() }; + + // 1. The version's questions (optionally restricted to the tag filter), with + // section + tag projections, in display order. + const slots = await prisma.appQuestionSlot.findMany({ + where: { + versionId: scope.versionId, + ...(scope.tagIds.length > 0 ? { tags: { some: { tagId: { in: scope.tagIds } } } } : {}), + }, + select: { + id: true, + key: true, + prompt: true, + type: true, + typeConfig: true, + required: true, + ordinal: true, + section: { select: { title: true, ordinal: true } }, + tags: { select: { tag: { select: { id: true, label: true, color: true } } } }, + }, + orderBy: [{ section: { ordinal: 'asc' } }, { ordinal: 'asc' }], + }); + + // 2. The non-preview sessions in the window — the response-rate denominator. + const sessions = await prisma.appQuestionnaireSession.findMany({ + where: { + versionId: scope.versionId, + isPreview: false, + createdAt: { gte: scope.from, lt: scope.to }, + }, + select: { id: true, status: true }, + }); + const sessionIds = sessions.map((s) => s.id); + const totalSessions = sessions.length; + const completedSessions = sessions.filter((s) => s.status === 'completed').length; + + // 3. The answers for those sessions + those questions, in one query. + const questionIds = slots.map((s) => s.id); + const answers = + sessionIds.length > 0 && questionIds.length > 0 + ? await prisma.appAnswerSlot.findMany({ + where: { sessionId: { in: sessionIds }, questionSlotId: { in: questionIds } }, + select: { questionSlotId: true, value: true, confidence: true, provenanceLabel: true }, + }) + : []; + + // Group answers by question. + const byQuestion = new Map< + string, + { values: unknown[]; confidences: number[]; provenance: ProvenanceBreakdown } + >(); + for (const slot of slots) { + byQuestion.set(slot.id, { values: [], confidences: [], provenance: emptyProvenance() }); + } + for (const a of answers) { + const bucket = byQuestion.get(a.questionSlotId); + if (!bucket) continue; + bucket.values.push(a.value); + if (typeof a.confidence === 'number') bucket.confidences.push(a.confidence); + const label = narrowToEnum(a.provenanceLabel, ANSWER_PROVENANCES, 'direct'); + bucket.provenance[label] += 1; + } + + const questions: QuestionDistribution[] = slots.map((slot) => { + const type = narrowToEnum(slot.type, QUESTION_TYPES, 'free_text'); + const bucket = byQuestion.get(slot.id)!; + const answeredCount = bucket.values.length; + const avgConfidence = + bucket.confidences.length > 0 + ? bucket.confidences.reduce((a, b) => a + b, 0) / bucket.confidences.length + : null; + const tags: TagView[] = slot.tags.map((t) => ({ + id: t.tag.id, + label: t.tag.label, + color: (t.tag.color as TagColor | null) ?? null, + })); + return { + questionId: slot.id, + key: slot.key, + prompt: slot.prompt, + type, + sectionTitle: slot.section.title, + required: slot.required, + tags, + answeredCount, + unansweredCount: Math.max(0, totalSessions - answeredCount), + responseRate: totalSessions > 0 ? answeredCount / totalSessions : 0, + avgConfidence, + provenance: bucket.provenance, + detail: buildDetail(type, slot.typeConfig, bucket.values), + }; + }); + + return { versionId: scope.versionId, range, totalSessions, completedSessions, questions }; +} + +/** Re-exported for the read view / tests that label types. */ +export { QUESTION_TYPE_LABELS }; diff --git a/lib/app/questionnaire/analytics/funnel.ts b/lib/app/questionnaire/analytics/funnel.ts new file mode 100644 index 000000000..bc9a0824a --- /dev/null +++ b/lib/app/questionnaire/analytics/funnel.ts @@ -0,0 +1,117 @@ +/** + * Completion funnel (F8.1): invited → opened → started → completed, with drop-off. + * + * The invite stages come from `AppQuestionnaireInvitation` timestamps (set reliably + * by F3.2: `sentAt`, `openedAt`). The downstream stages are derived from real + * session data rather than the invitation's `status` string, so the funnel reflects + * what respondents actually did: an invited respondent is "started" once they have a + * non-preview session for the version, and "completed" once one of those sessions + * reaches `completed`. Respondents are matched to invitations by `userId` + * (set on registration) — the only link between the two (no FK, UG-1). + * + * Anonymous / public-link sessions have no invitation, so they can't enter the + * invite funnel; they're reported separately (entering at "started") to avoid + * overstating invited-stage drop-off. + */ + +import { prisma } from '@/lib/db/client'; +import type { AnalyticsScope } from '@/lib/app/questionnaire/analytics/query-schema'; +import type { + CompletionFunnelResult, + FunnelStage, + FunnelStageKey, +} from '@/lib/app/questionnaire/analytics/views'; + +const STAGE_LABELS: Record = { + invited: 'Invited', + opened: 'Opened', + started: 'Started', + completed: 'Completed', +}; + +/** Build the ordered stage list with drop-off, retention, and step conversion. */ +function buildStages(counts: Record): FunnelStage[] { + const order: FunnelStageKey[] = ['invited', 'opened', 'started', 'completed']; + const base = counts.invited; + let prev = counts.invited; + return order.map((key, i) => { + const count = counts[key]; + const stage: FunnelStage = { + key, + label: STAGE_LABELS[key], + count, + dropoff: i === 0 ? 0 : Math.max(0, prev - count), + retention: base > 0 ? count / base : 0, + conversionFromPrev: i === 0 ? 1 : prev > 0 ? count / prev : 0, + }; + prev = count; + return stage; + }); +} + +/** + * Compute the completion funnel for a version over the date window. Invitations are + * scoped by `createdAt`; sessions by `createdAt` and `isPreview = false`. + */ +export async function getCompletionFunnel(scope: AnalyticsScope): Promise { + const range = { from: scope.from.toISOString(), to: scope.to.toISOString() }; + + // Invitations in scope, excluding revoked. `userId` links to the respondent. + const invitations = await prisma.appQuestionnaireInvitation.findMany({ + where: { + versionId: scope.versionId, + createdAt: { gte: scope.from, lt: scope.to }, + revokedAt: null, + }, + select: { sentAt: true, openedAt: true, userId: true }, + }); + + // Non-preview sessions in scope, grouped by respondent. + const sessions = await prisma.appQuestionnaireSession.findMany({ + where: { + versionId: scope.versionId, + isPreview: false, + createdAt: { gte: scope.from, lt: scope.to }, + }, + select: { respondentUserId: true, status: true }, + }); + + // Which respondents have any session / a completed session. + const startedUsers = new Set(); + const completedUsers = new Set(); + for (const s of sessions) { + if (!s.respondentUserId) continue; + startedUsers.add(s.respondentUserId); + if (s.status === 'completed') completedUsers.add(s.respondentUserId); + } + + const invitedUserIds = new Set( + invitations.map((i) => i.userId).filter((id): id is string => id !== null) + ); + + const invited = invitations.filter((i) => i.sentAt !== null).length; + const opened = invitations.filter((i) => i.openedAt !== null).length; + let started = 0; + let completed = 0; + for (const userId of invitedUserIds) { + if (startedUsers.has(userId)) started += 1; + if (completedUsers.has(userId)) completed += 1; + } + + // Anonymous sessions: those whose respondent was never invited (incl. no userId). + let anonStarted = 0; + let anonCompleted = 0; + for (const s of sessions) { + const invitedRespondent = s.respondentUserId !== null && invitedUserIds.has(s.respondentUserId); + if (invitedRespondent) continue; + anonStarted += 1; + if (s.status === 'completed') anonCompleted += 1; + } + + return { + versionId: scope.versionId, + range, + stages: buildStages({ invited, opened, started, completed }), + anonymous: { started: anonStarted, completed: anonCompleted }, + }; +} diff --git a/lib/app/questionnaire/analytics/index.ts b/lib/app/questionnaire/analytics/index.ts new file mode 100644 index 000000000..77d401353 --- /dev/null +++ b/lib/app/questionnaire/analytics/index.ts @@ -0,0 +1,41 @@ +/** + * F8.1 admin analytics — public surface. + * + * Three read-only aggregators over a version's completed-session data (distributions, + * funnel, cost) plus the shared query contract / scope resolver. Server-only at the + * seam (the aggregators touch Prisma); the `views` types are client-safe and imported + * by the admin UI. + */ + +export { + questionnaireAnalyticsQuerySchema, + resolveAnalyticsScope, + getAnalyticsDefaultDateInputs, + type QuestionnaireAnalyticsQuery, + type AnalyticsScope, +} from '@/lib/app/questionnaire/analytics/query-schema'; + +export { + getQuestionDistributions, + QUESTION_TYPE_LABELS, +} from '@/lib/app/questionnaire/analytics/distributions'; +export { getCompletionFunnel } from '@/lib/app/questionnaire/analytics/funnel'; +export { getQuestionnaireCostBreakdown } from '@/lib/app/questionnaire/analytics/cost'; + +export type { + AnalyticsRange, + ProvenanceBreakdown, + ValueBucket, + NumericSummary, + HistogramBin, + DistributionDetail, + QuestionDistribution, + QuestionDistributionsResult, + FunnelStageKey, + FunnelStage, + CompletionFunnelResult, + CostCapabilityBucket, + CostDayPoint, + SessionCostRow, + QuestionnaireCostResult, +} from '@/lib/app/questionnaire/analytics/views'; diff --git a/lib/app/questionnaire/analytics/query-schema.ts b/lib/app/questionnaire/analytics/query-schema.ts new file mode 100644 index 000000000..7a2b7fa80 --- /dev/null +++ b/lib/app/questionnaire/analytics/query-schema.ts @@ -0,0 +1,61 @@ +/** + * Shared query contract + scope resolution for the F8.1 analytics endpoints. + * + * The three GET routes (distributions / funnel / cost) accept the same filter: + * a date window and an optional tag filter. The window defaults reuse the + * platform's analytics helpers (`lib/orchestration/analytics`) so the + * questionnaire surface and the orchestration surface share one "last 30 days" + * definition. + * + * `tagIds` arrives as a single comma-separated query param (the route layer + * flattens repeated params to last-value-wins; see `validateQueryParams`), so we + * split it here into the array the aggregators consume. + */ + +import { z } from 'zod'; + +import { + resolveAnalyticsDateRange, + getAnalyticsDefaultDateInputs, +} from '@/lib/orchestration/analytics/date-range'; + +/** Validated raw query for any of the three analytics endpoints. */ +export const questionnaireAnalyticsQuerySchema = z.object({ + /** Inclusive lower bound, `YYYY-MM-DD`. Defaults to 30 days before `to`. */ + from: z.string().date().optional(), + /** Upper bound, `YYYY-MM-DD`. Defaults to now. */ + to: z.string().date().optional(), + /** Comma-separated `AppQuestionTag` ids; restricts the distributions view. */ + tagIds: z.string().optional(), +}); + +export type QuestionnaireAnalyticsQuery = z.infer; + +/** + * The resolved scope the aggregators take: a concrete date window (Date objects), + * the target version, and the parsed tag filter. `from`/`to` follow + * {@link resolveAnalyticsDateRange} (inclusive `from`, exclusive `to`). + */ +export interface AnalyticsScope { + versionId: string; + from: Date; + to: Date; + /** Empty array = no tag filter (all questions). */ + tagIds: string[]; +} + +/** Build a concrete {@link AnalyticsScope} from a validated query + version id. */ +export function resolveAnalyticsScope( + versionId: string, + query: QuestionnaireAnalyticsQuery +): AnalyticsScope { + const { from, to } = resolveAnalyticsDateRange(query); + const tagIds = (query.tagIds ?? '') + .split(',') + .map((id) => id.trim()) + .filter((id) => id.length > 0); + return { versionId, from, to, tagIds }; +} + +/** `YYYY-MM-DD` defaults for the filter's `` controls. */ +export { getAnalyticsDefaultDateInputs }; diff --git a/lib/app/questionnaire/analytics/views.ts b/lib/app/questionnaire/analytics/views.ts new file mode 100644 index 000000000..faa5df400 --- /dev/null +++ b/lib/app/questionnaire/analytics/views.ts @@ -0,0 +1,180 @@ +/** + * Read-side view types for the F8.1 admin analytics surface. + * + * The shapes the three analytics GET endpoints return and the admin UI consumes. + * Pure types, **client-safe** (no Prisma, no Next) — the route serializers and the + * `'use client'` panels import the same contract. Dates cross the HTTP boundary as + * ISO strings, so there are no `Date` objects here. + * + * Three surfaces, one shared scope (`AnalyticsRange` echoes the resolved window): + * - {@link QuestionDistributionsResult} — per-question answer distributions + * - {@link CompletionFunnelResult} — invited → opened → started → completed + * - {@link QuestionnaireCostResult} — per-version cost actuals from `AiCostLog` + * + * Deliberately aggregate-only: free-text answer *values* are never surfaced (only + * counts/confidence), so F8.1 is PII-safe by construction ahead of F8.3's + * anonymous-mode hardening. + */ + +import type { QuestionType, AnswerProvenance, SessionStatus } from '@/lib/app/questionnaire/types'; +import type { TagView } from '@/lib/app/questionnaire/views'; + +/** The resolved analytics window, echoed back so the UI can label what it shows. */ +export interface AnalyticsRange { + /** Inclusive lower bound, ISO-8601. */ + from: string; + /** Exclusive upper bound, ISO-8601. */ + to: string; +} + +/* ── Per-question distributions ─────────────────────────────────────────── */ + +/** Answer counts split by how each value was arrived at ({@link AnswerProvenance}). */ +export type ProvenanceBreakdown = Record; + +/** One bucket of a choice/likert distribution: the option and how often it was chosen. */ +export interface ValueBucket { + /** Stored value (choice slug, or stringified likert integer). */ + value: string; + /** Human label (choice label, likert bound label, or the value itself). */ + label: string; + count: number; +} + +/** Summary statistics for a numeric question's answers. */ +export interface NumericSummary { + count: number; + min: number; + max: number; + mean: number; + median: number; +} + +/** One bin of a numeric histogram (half-open `[min, max)`, top bin inclusive). */ +export interface HistogramBin { + label: string; + min: number; + max: number; + count: number; +} + +/** + * The type-appropriate shape of a question's answer distribution. `free_text` + * carries no value detail by design (PII / not meaningful as a distribution). + */ +export type DistributionDetail = + | { kind: 'choice'; buckets: ValueBucket[]; otherCount: number } + | { kind: 'likert'; min: number; max: number; buckets: ValueBucket[]; mean: number | null } + | { kind: 'numeric'; summary: NumericSummary | null; histogram: HistogramBin[] } + | { + kind: 'boolean'; + trueLabel: string; + falseLabel: string; + trueCount: number; + falseCount: number; + } + | { kind: 'date'; buckets: { label: string; count: number }[] } + | { kind: 'free_text' }; + +/** One question's distribution over the non-preview sessions in scope. */ +export interface QuestionDistribution { + questionId: string; + key: string; + prompt: string; + type: QuestionType; + sectionTitle: string; + required: boolean; + tags: TagView[]; + /** Distinct sessions in scope that answered this question. */ + answeredCount: number; + /** Sessions in scope that did not answer it (`totalSessions - answeredCount`). */ + unansweredCount: number; + /** `answeredCount / totalSessions` (0 when there are no sessions). */ + responseRate: number; + /** Mean of the recorded answer confidences (0–1), or null when none scored. */ + avgConfidence: number | null; + provenance: ProvenanceBreakdown; + detail: DistributionDetail; +} + +export interface QuestionDistributionsResult { + versionId: string; + range: AnalyticsRange; + /** Non-preview sessions in scope — the denominator for every response rate. */ + totalSessions: number; + /** Of those, how many reached `completed`. */ + completedSessions: number; + questions: QuestionDistribution[]; +} + +/* ── Completion funnel ──────────────────────────────────────────────────── */ + +export type FunnelStageKey = 'invited' | 'opened' | 'started' | 'completed'; + +export interface FunnelStage { + key: FunnelStageKey; + label: string; + count: number; + /** Absolute drop from the previous stage (0 for the first stage). */ + dropoff: number; + /** Fraction of the first stage (`invited`) retained here, 0–1. */ + retention: number; + /** Conversion from the immediately previous stage, 0–1 (1 for the first). */ + conversionFromPrev: number; +} + +export interface CompletionFunnelResult { + versionId: string; + range: AnalyticsRange; + /** invited → opened → started → completed, in order. */ + stages: FunnelStage[]; + /** + * Respondent sessions with no invitation (anonymous / public link). They enter + * the journey at "started", so they're reported separately rather than folded + * into the invite funnel (which would misstate invited-stage retention). + */ + anonymous: { + started: number; + completed: number; + }; +} + +/* ── Cost actuals ───────────────────────────────────────────────────────── */ + +/** Spend grouped by the capability (or LLM operation) that incurred it. */ +export interface CostCapabilityBucket { + key: string; + label: string; + costUsd: number; + callCount: number; +} + +/** A single day's total questionnaire spend (zero-spend days omitted). */ +export interface CostDayPoint { + /** `YYYY-MM-DD`. */ + date: string; + costUsd: number; +} + +/** One respondent session's total spend, for the top-spenders table. */ +export interface SessionCostRow { + sessionId: string; + status: SessionStatus; + costUsd: number; + createdAt: string; +} + +export interface QuestionnaireCostResult { + versionId: string; + range: AnalyticsRange; + totalCostUsd: number; + /** Spend on live respondent turns (attributed via `metadata.appQuestionnaireSessionId`). */ + runtimeCostUsd: number; + /** Design-time spend: structure extraction + evaluation (attributed via `metadata.versionId`). */ + designTimeCostUsd: number; + byCapability: CostCapabilityBucket[]; + /** Daily total spend across the window, ascending. */ + trend: CostDayPoint[]; + /** Highest-spend respondent sessions, descending (capped). */ + topSessions: SessionCostRow[]; +} diff --git a/lib/app/questionnaire/capabilities/compose-completion-offer.ts b/lib/app/questionnaire/capabilities/compose-completion-offer.ts index f823921f0..1b0a3ceee 100644 --- a/lib/app/questionnaire/capabilities/compose-completion-offer.ts +++ b/lib/app/questionnaire/capabilities/compose-completion-offer.ts @@ -294,7 +294,10 @@ export class AppComposeCompletionOfferCapability extends BaseCapability< provider: providerSlug, inputTokens: completion.tokenUsage.input, outputTokens: completion.tokenUsage.output, - metadata: { capability: SLUG, ...(args.sessionId ? { sessionId: args.sessionId } : {}) }, + metadata: { + capability: SLUG, + ...(args.sessionId ? { appQuestionnaireSessionId: args.sessionId } : {}), + }, }).catch((err) => { logger.error('compose_completion_offer: logCost rejected', { agentId: context.agentId, diff --git a/lib/app/questionnaire/capabilities/detect-contradictions.ts b/lib/app/questionnaire/capabilities/detect-contradictions.ts index ec437991a..85dc9a7f4 100644 --- a/lib/app/questionnaire/capabilities/detect-contradictions.ts +++ b/lib/app/questionnaire/capabilities/detect-contradictions.ts @@ -347,7 +347,7 @@ export class AppDetectContradictionsCapability extends BaseCapability< provider: providerSlug, inputTokens: completion.tokenUsage.input, outputTokens: completion.tokenUsage.output, - metadata: { capability: SLUG, sessionId: detectionContext.sessionId }, + metadata: { capability: SLUG, appQuestionnaireSessionId: detectionContext.sessionId }, }).catch((err) => { logger.error('detect_contradictions: logCost rejected', { agentId: context.agentId, diff --git a/lib/app/questionnaire/capabilities/extract-answer-slots.ts b/lib/app/questionnaire/capabilities/extract-answer-slots.ts index 5aaa2bee9..ee966c706 100644 --- a/lib/app/questionnaire/capabilities/extract-answer-slots.ts +++ b/lib/app/questionnaire/capabilities/extract-answer-slots.ts @@ -386,7 +386,7 @@ export class AppExtractAnswerSlotsCapability extends BaseCapability< provider: providerSlug, inputTokens: completion.tokenUsage.input, outputTokens: completion.tokenUsage.output, - metadata: { capability: SLUG, sessionId: extractionContext.sessionId }, + metadata: { capability: SLUG, appQuestionnaireSessionId: extractionContext.sessionId }, }).catch((err) => { logger.error('extract_answer_slots: logCost rejected', { agentId: context.agentId, diff --git a/lib/app/questionnaire/capabilities/refine-answer.ts b/lib/app/questionnaire/capabilities/refine-answer.ts index a44651a77..e69379195 100644 --- a/lib/app/questionnaire/capabilities/refine-answer.ts +++ b/lib/app/questionnaire/capabilities/refine-answer.ts @@ -352,7 +352,7 @@ export class AppRefineAnswerCapability extends BaseCapability { logger.error('refine_answer: logCost rejected', { agentId: context.agentId, diff --git a/tests/integration/api/v1/app/questionnaires/analytics-routes.test.ts b/tests/integration/api/v1/app/questionnaires/analytics-routes.test.ts new file mode 100644 index 000000000..ed696f281 --- /dev/null +++ b/tests/integration/api/v1/app/questionnaires/analytics-routes.test.ts @@ -0,0 +1,166 @@ +/** + * Integration test: questionnaire analytics read routes (F8.1). + * + * Pins the route → flag-gate → auth → version-scope → query-validation → aggregator + * wiring for the three GET endpoints (distributions / funnel / cost). The aggregators + * themselves are unit-tested separately (lib/app/questionnaire/analytics/*.test.ts); + * here they're stubbed so the test exercises only the route shell: + * - 404 when the master flag is off (before auth — the app looks absent) + * - 401 unauthenticated / 403 non-admin + * - 404 when the version doesn't resolve under the questionnaire + * - 400 on an invalid query (bad date) + * - 200 + payload on the happy path, with the resolved scope passed to the aggregator + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { NextRequest } from 'next/server'; + +// ─── Mocks (hoisted) ────────────────────────────────────────────────────────── + +vi.mock('@/lib/feature-flags', () => ({ isFeatureEnabled: vi.fn() })); +vi.mock('@/lib/auth/config', () => ({ auth: { api: { getSession: vi.fn() } } })); +vi.mock('next/headers', () => ({ headers: vi.fn(() => Promise.resolve(new Headers())) })); + +const prismaMock = vi.hoisted(() => ({ + appQuestionnaireVersion: { findFirst: vi.fn() }, +})); +vi.mock('@/lib/db/client', () => ({ prisma: prismaMock })); + +// Keep the real query schema + scope resolver; stub only the DB-touching aggregators. +const analyticsMock = vi.hoisted(() => ({ + getQuestionDistributions: vi.fn(), + getCompletionFunnel: vi.fn(), + getQuestionnaireCostBreakdown: vi.fn(), +})); +vi.mock('@/lib/app/questionnaire/analytics', async (importOriginal) => ({ + ...(await importOriginal()), + ...analyticsMock, +})); + +// ─── Imports (after mocks) ──────────────────────────────────────────────────── + +import { GET as getDistributions } from '@/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/distributions/route'; +import { GET as getFunnel } from '@/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/funnel/route'; +import { GET as getCost } from '@/app/api/v1/app/questionnaires/[id]/versions/[vid]/analytics/cost/route'; + +import { isFeatureEnabled } from '@/lib/feature-flags'; +import { auth } from '@/lib/auth/config'; +import { APP_QUESTIONNAIRES_FLAG } from '@/lib/app/questionnaire/constants'; +import { + mockAdminUser, + mockAuthenticatedUser, + mockUnauthenticatedUser, +} from '@/tests/helpers/auth'; + +type Mock = ReturnType; + +const BASE = 'http://localhost:3000/api/v1/app/questionnaires/qn-1/versions/v1/analytics'; +const PARAMS = { id: 'qn-1', vid: 'v1' }; + +function req(path: 'distributions' | 'funnel' | 'cost', search = ''): NextRequest { + return { url: `${BASE}/${path}${search}`, headers: new Headers() } as unknown as NextRequest; +} +function ctx>(params: T): { params: Promise } { + return { params: Promise.resolve(params) }; +} +function setAuth(session: ReturnType | null) { + (auth.api.getSession as unknown as Mock).mockResolvedValue(session); +} + +// The three routes, each paired with its stubbed aggregator + a canned payload. +const ROUTES = [ + { + name: 'distributions', + handler: getDistributions, + agg: analyticsMock.getQuestionDistributions, + payload: { versionId: 'v1', totalSessions: 3, completedSessions: 1, questions: [] }, + }, + { + name: 'funnel', + handler: getFunnel, + agg: analyticsMock.getCompletionFunnel, + payload: { versionId: 'v1', stages: [], anonymous: { started: 0, completed: 0 } }, + }, + { + name: 'cost', + handler: getCost, + agg: analyticsMock.getQuestionnaireCostBreakdown, + payload: { versionId: 'v1', totalCostUsd: 0, byCapability: [], trend: [], topSessions: [] }, + }, +] as const; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(isFeatureEnabled).mockImplementation((flag) => + Promise.resolve(flag === APP_QUESTIONNAIRES_FLAG) + ); + setAuth(mockAdminUser()); + prismaMock.appQuestionnaireVersion.findFirst.mockResolvedValue({ + id: 'v1', + questionnaireId: 'qn-1', + versionNumber: 1, + status: 'launched', + }); + analyticsMock.getQuestionDistributions.mockResolvedValue(ROUTES[0].payload); + analyticsMock.getCompletionFunnel.mockResolvedValue(ROUTES[1].payload); + analyticsMock.getQuestionnaireCostBreakdown.mockResolvedValue(ROUTES[2].payload); +}); + +describe.each(ROUTES)('GET analytics/$name', ({ name, handler, agg, payload }) => { + const path = name; + + it('404s when the master flag is off, before auth', async () => { + vi.mocked(isFeatureEnabled).mockResolvedValue(false); + setAuth(null); + const res = await handler(req(path), ctx(PARAMS)); + expect(res.status).toBe(404); + expect(agg).not.toHaveBeenCalled(); + }); + + it('401s when unauthenticated', async () => { + setAuth(mockUnauthenticatedUser()); + expect((await handler(req(path), ctx(PARAMS))).status).toBe(401); + }); + + it('403s for a non-admin', async () => { + setAuth(mockAuthenticatedUser()); + expect((await handler(req(path), ctx(PARAMS))).status).toBe(403); + }); + + it('404s with the error envelope when the version does not resolve', async () => { + prismaMock.appQuestionnaireVersion.findFirst.mockResolvedValue(null); + const res = await handler(req(path), ctx(PARAMS)); + expect(res.status).toBe(404); + const body = await res.json(); + expect(body.success).toBe(false); + expect(body.error?.code).toBe('NOT_FOUND'); + expect(agg).not.toHaveBeenCalled(); + }); + + it('400s with the error envelope on an invalid date query', async () => { + const res = await handler(req(path, '?from=not-a-date'), ctx(PARAMS)); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.success).toBe(false); + expect(body.error?.code).toBeDefined(); + expect(agg).not.toHaveBeenCalled(); + }); + + it('200s on the happy path and returns the aggregator payload', async () => { + const res = await handler(req(path), ctx(PARAMS)); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.success).toBe(true); + expect(body.data).toEqual(payload); + expect(agg).toHaveBeenCalledTimes(1); + }); + + it('passes the resolved scope (version + parsed tag filter) to the aggregator', async () => { + await handler(req(path, '?tagIds=t1,t2'), ctx(PARAMS)); + const scope = (agg as unknown as Mock).mock.calls[0][0]; + expect(scope.versionId).toBe('v1'); + expect(scope.tagIds).toEqual(['t1', 't2']); + expect(scope.from).toBeInstanceOf(Date); + expect(scope.to).toBeInstanceOf(Date); + }); +}); diff --git a/tests/integration/lib/app/questionnaire/completion-capability.test.ts b/tests/integration/lib/app/questionnaire/completion-capability.test.ts index 242069488..1e08de8f3 100644 --- a/tests/integration/lib/app/questionnaire/completion-capability.test.ts +++ b/tests/integration/lib/app/questionnaire/completion-capability.test.ts @@ -225,7 +225,10 @@ describe('AppComposeCompletionOfferCapability — dispatch', () => { provider: 'test-provider', inputTokens: 321, outputTokens: 123, - metadata: expect.objectContaining({ capability: SLUG, sessionId: 'sess-1' }), + metadata: expect.objectContaining({ + capability: SLUG, + appQuestionnaireSessionId: 'sess-1', + }), }) ); }); @@ -385,7 +388,7 @@ describe('AppComposeCompletionOfferCapability — dispatch', () => { expect(result.error?.message).toBe('provider blew up'); }); - it('logs cost without an agentId, and without a sessionId in metadata, when both are omitted', async () => { + it('logs cost without an agentId, and without a session id in metadata, when both are omitted', async () => { (getProvider as Mock).mockResolvedValue(makeProvider([{ content: VALID_JSON }])); await capabilityDispatcher.dispatch( @@ -405,7 +408,7 @@ describe('AppComposeCompletionOfferCapability — dispatch', () => { metadata: { capability: SLUG }, }); expect(chatArg).not.toHaveProperty('agentId'); - expect(chatArg.metadata).not.toHaveProperty('sessionId'); + expect(chatArg.metadata).not.toHaveProperty('appQuestionnaireSessionId'); }); }); diff --git a/tests/integration/lib/app/questionnaire/refinement-capability.test.ts b/tests/integration/lib/app/questionnaire/refinement-capability.test.ts index 99537c0e9..c31f56ad4 100644 --- a/tests/integration/lib/app/questionnaire/refinement-capability.test.ts +++ b/tests/integration/lib/app/questionnaire/refinement-capability.test.ts @@ -564,7 +564,9 @@ describe('AppRefineAnswerCapability — dispatch', () => { expect((result.data as { decisions: unknown[] }).decisions).toHaveLength(1); // The cost log uses the dispatch-derived sessionId fallback (no sessionId supplied). expect(logCost).toHaveBeenCalledWith( - expect.objectContaining({ metadata: { capability: SLUG, sessionId: 'dispatch-refine' } }) + expect.objectContaining({ + metadata: { capability: SLUG, appQuestionnaireSessionId: 'dispatch-refine' }, + }) ); }); diff --git a/tests/unit/lib/app/questionnaire/analytics/cost.test.ts b/tests/unit/lib/app/questionnaire/analytics/cost.test.ts new file mode 100644 index 000000000..5a6f863f0 --- /dev/null +++ b/tests/unit/lib/app/questionnaire/analytics/cost.test.ts @@ -0,0 +1,217 @@ +/** + * Unit test: per-version cost aggregation (F8.1). + * + * Mocks the session read + the raw-SQL ledger queries and asserts the runtime vs + * design-time split, per-capability merge, top-session ranking, and trend mapping. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const findManySessions = vi.fn(); +const queryRawUnsafe = vi.fn(); + +vi.mock('@/lib/db/client', () => ({ + prisma: { + appQuestionnaireSession: { findMany: (...a: unknown[]) => findManySessions(...a) }, + $queryRawUnsafe: (...a: unknown[]) => queryRawUnsafe(...a), + }, +})); + +import { getQuestionnaireCostBreakdown } from '@/lib/app/questionnaire/analytics/cost'; +import type { AnalyticsScope } from '@/lib/app/questionnaire/analytics/query-schema'; + +const scope: AnalyticsScope = { + versionId: 'v1', + from: new Date('2026-01-01T00:00:00.000Z'), + to: new Date('2026-02-01T00:00:00.000Z'), + tagIds: [], +}; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +/** + * Route each raw query to canned rows by inspecting its SQL. The four queries are + * discriminated by anchored, mutually-exclusive markers (not check-order): the two + * capability rollups both carry `GROUP BY capability` and are split by the presence + * of the session `ANY(...)` clause (runtime) vs the bare `versionId` filter (design). + */ +function wireRawSql() { + queryRawUnsafe.mockImplementation((sql: string) => { + const groupBySession = sql.includes('GROUP BY session_id'); + const groupByCapability = sql.includes('GROUP BY capability'); + const hasSessionAny = sql.includes("'appQuestionnaireSessionId' = ANY"); + const isTrend = sql.includes('date_trunc'); + + if (groupBySession) { + return Promise.resolve([ + { session_id: 's1', cost: 0.5 }, + { session_id: 's2', cost: 1.5 }, + ]); + } + if (isTrend) { + return Promise.resolve([ + { day: new Date('2026-01-05T00:00:00.000Z'), cost: 1.0 }, + { day: new Date('2026-01-06T00:00:00.000Z'), cost: 1.2 }, + ]); + } + if (groupByCapability && hasSessionAny) { + // runtime by-capability + return Promise.resolve([ + { capability: 'extract_answer_slots', cost: 1.5, calls: 3n }, + { capability: 'chat', cost: 0.5, calls: 2n }, + ]); + } + if (groupByCapability) { + // design-time by-capability (versionId filter, no session ANY clause) + return Promise.resolve([{ capability: 'evaluate_structure', cost: 0.8, calls: 7n }]); + } + return Promise.resolve([]); + }); +} + +describe('getQuestionnaireCostBreakdown', () => { + it('splits runtime vs design-time spend and totals them', async () => { + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed', createdAt: new Date('2026-01-05T00:00:00.000Z') }, + { id: 's2', status: 'abandoned', createdAt: new Date('2026-01-06T00:00:00.000Z') }, + ]); + wireRawSql(); + + const result = await getQuestionnaireCostBreakdown(scope); + + expect(result.runtimeCostUsd).toBeCloseTo(2.0, 5); // 1.5 + 0.5 + expect(result.designTimeCostUsd).toBeCloseTo(0.8, 5); + expect(result.totalCostUsd).toBeCloseTo(2.8, 5); + }); + + it('merges runtime + design-time into a per-capability breakdown, sorted desc', async () => { + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed', createdAt: new Date('2026-01-05T00:00:00.000Z') }, + { id: 's2', status: 'abandoned', createdAt: new Date('2026-01-06T00:00:00.000Z') }, + ]); + wireRawSql(); + + const result = await getQuestionnaireCostBreakdown(scope); + const keys = result.byCapability.map((c) => c.key); + expect(keys).toEqual(['extract_answer_slots', 'evaluate_structure', 'chat']); + expect(result.byCapability[0]).toMatchObject({ + key: 'extract_answer_slots', + label: 'Answer extraction', + costUsd: 1.5, + callCount: 3, + }); + }); + + it('ranks top sessions by spend and joins status + createdAt', async () => { + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed', createdAt: new Date('2026-01-05T00:00:00.000Z') }, + { id: 's2', status: 'abandoned', createdAt: new Date('2026-01-06T00:00:00.000Z') }, + ]); + wireRawSql(); + + const result = await getQuestionnaireCostBreakdown(scope); + expect(result.topSessions.map((s) => s.sessionId)).toEqual(['s2', 's1']); // 1.5 before 0.5 + expect(result.topSessions[0]).toMatchObject({ + sessionId: 's2', + status: 'abandoned', + costUsd: 1.5, + }); + }); + + it('maps the daily trend to YYYY-MM-DD points', async () => { + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed', createdAt: new Date('2026-01-05T00:00:00.000Z') }, + ]); + wireRawSql(); + + const result = await getQuestionnaireCostBreakdown(scope); + expect(result.trend).toEqual([ + { date: '2026-01-05', costUsd: 1.0 }, + { date: '2026-01-06', costUsd: 1.2 }, + ]); + }); + + it('coerces string/bigint aggregates and falls back for an unknown session row', async () => { + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed', createdAt: new Date('2026-01-05T00:00:00.000Z') }, + ]); + queryRawUnsafe.mockImplementation((sql: string) => { + if (sql.includes('GROUP BY session_id')) { + // 's9' is not in findMany → must fall back to status 'active' + scope.from. + return Promise.resolve([ + { session_id: 's1', cost: '0.25' }, + { session_id: 's9', cost: '2.00' }, + ]); + } + if (sql.includes('date_trunc')) return Promise.resolve([]); + if (sql.includes("'appQuestionnaireSessionId' = ANY")) { + return Promise.resolve([{ capability: 'extract_answer_slots', cost: '2.25', calls: '5' }]); + } + return Promise.resolve([]); // design-time empty + }); + + const result = await getQuestionnaireCostBreakdown(scope); + expect(result.runtimeCostUsd).toBeCloseTo(2.25, 5); // string coerced + expect(result.byCapability[0]).toMatchObject({ callCount: 5 }); // string calls coerced + // 's9' ranks first and uses the fallback status/createdAt. + expect(result.topSessions[0]).toMatchObject({ + sessionId: 's9', + status: 'active', + costUsd: 2, + createdAt: scope.from.toISOString(), + }); + }); + + it('handles null/undefined aggregates, null capability, and string trend days', async () => { + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed', createdAt: new Date('2026-01-05T00:00:00.000Z') }, + ]); + queryRawUnsafe.mockImplementation((sql: string) => { + if (sql.includes('GROUP BY session_id')) { + // a null session_id row must be ignored. + return Promise.resolve([ + { session_id: null, cost: 1 }, + { session_id: 's1', cost: 0.5 }, + ]); + } + if (sql.includes('date_trunc')) { + // day arriving as a string + a null cost coerces to 0. + return Promise.resolve([{ day: '2026-01-05T00:00:00.000Z', cost: null }]); + } + if (sql.includes("'appQuestionnaireSessionId' = ANY")) { + // null capability → labelled via the 'chat' fallback. + return Promise.resolve([{ capability: null, cost: 0.5, calls: 2n }]); + } + return Promise.resolve([]); // design-time empty + }); + + const result = await getQuestionnaireCostBreakdown(scope); + expect(result.runtimeCostUsd).toBeCloseTo(0.5, 5); + expect(result.byCapability[0]).toMatchObject({ key: 'chat', label: 'Question selection' }); + expect(result.trend).toEqual([{ date: '2026-01-05', costUsd: 0 }]); // string day parsed, null cost → 0 + expect(result.topSessions.map((s) => s.sessionId)).toEqual(['s1']); // null session_id dropped + }); + + it('skips runtime queries when the version has no sessions', async () => { + findManySessions.mockResolvedValue([]); + const executedSql: string[] = []; + queryRawUnsafe.mockImplementation((sql: string) => { + executedSql.push(sql); + if (sql.includes('date_trunc')) return Promise.resolve([]); + return Promise.resolve([{ capability: 'evaluate_structure', cost: 0.4, calls: 7n }]); + }); + + const result = await getQuestionnaireCostBreakdown(scope); + + // No query may reference sessions, and the two session-scoped queries must not run. + expect(executedSql.some((sql) => sql.includes("'appQuestionnaireSessionId' = ANY"))).toBe( + false + ); + expect(executedSql.some((sql) => sql.includes('GROUP BY session_id'))).toBe(false); + expect(result.runtimeCostUsd).toBe(0); + expect(result.designTimeCostUsd).toBeCloseTo(0.4, 5); + expect(result.topSessions).toEqual([]); + }); +}); diff --git a/tests/unit/lib/app/questionnaire/analytics/distributions.test.ts b/tests/unit/lib/app/questionnaire/analytics/distributions.test.ts new file mode 100644 index 000000000..a2f55c53d --- /dev/null +++ b/tests/unit/lib/app/questionnaire/analytics/distributions.test.ts @@ -0,0 +1,358 @@ +/** + * Unit test: per-question distribution aggregation (F8.1). + * + * Mocks the three Prisma reads and asserts the computed distribution math per type, + * the response-rate denominator, provenance tallies, and that the session query + * excludes preview sessions. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const findManySlots = vi.fn(); +const findManySessions = vi.fn(); +const findManyAnswers = vi.fn(); + +vi.mock('@/lib/db/client', () => ({ + prisma: { + appQuestionSlot: { findMany: (...a: unknown[]) => findManySlots(...a) }, + appQuestionnaireSession: { findMany: (...a: unknown[]) => findManySessions(...a) }, + appAnswerSlot: { findMany: (...a: unknown[]) => findManyAnswers(...a) }, + }, +})); + +import { getQuestionDistributions } from '@/lib/app/questionnaire/analytics/distributions'; +import type { AnalyticsScope } from '@/lib/app/questionnaire/analytics/query-schema'; + +const scope: AnalyticsScope = { + versionId: 'v1', + from: new Date('2026-01-01T00:00:00.000Z'), + to: new Date('2026-02-01T00:00:00.000Z'), + tagIds: [], +}; + +function slot(overrides: Record) { + return { + id: 'q?', + key: 'k', + prompt: 'P', + type: 'free_text', + typeConfig: null, + required: false, + ordinal: 0, + section: { title: 'Section', ordinal: 0 }, + tags: [], + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('getQuestionDistributions', () => { + it('excludes preview sessions and reports the session denominator', async () => { + findManySlots.mockResolvedValue([]); + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed' }, + { id: 's2', status: 'active' }, + { id: 's3', status: 'abandoned' }, + ]); + findManyAnswers.mockResolvedValue([]); + + const result = await getQuestionDistributions(scope); + + expect(result.totalSessions).toBe(3); + expect(result.completedSessions).toBe(1); + // The session read must filter out previews and scope to the version + window. + const where = findManySessions.mock.calls[0][0].where; + expect(where.isPreview).toBe(false); + expect(where.versionId).toBe('v1'); + expect(where.createdAt).toEqual({ gte: scope.from, lt: scope.to }); + }); + + it('buckets single_choice answers by option and counts unlisted values as other', async () => { + findManySlots.mockResolvedValue([ + slot({ + id: 'q1', + type: 'single_choice', + typeConfig: { + choices: [ + { value: 'a', label: 'Apple' }, + { value: 'b', label: 'Banana' }, + ], + }, + }), + ]); + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed' }, + { id: 's2', status: 'completed' }, + { id: 's3', status: 'completed' }, + { id: 's4', status: 'active' }, + ]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'q1', value: 'a', confidence: 0.8, provenanceLabel: 'direct' }, + { questionSlotId: 'q1', value: 'a', confidence: 0.6, provenanceLabel: 'inferred' }, + { questionSlotId: 'q1', value: 'b', confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'q1', value: 'zzz', confidence: null, provenanceLabel: 'direct' }, + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + + expect(q.answeredCount).toBe(4); + expect(q.unansweredCount).toBe(0); // 4 sessions, 4 answers + expect(q.responseRate).toBe(1); + // avg of the two scored confidences (0.8, 0.6); nulls ignored. + expect(q.avgConfidence).toBeCloseTo(0.7, 5); + expect(q.provenance).toEqual({ direct: 3, inferred: 1, synthesised: 0, refined: 0 }); + + expect(q.detail).toMatchObject({ kind: 'choice', otherCount: 1 }); + if (q.detail.kind === 'choice') { + expect(q.detail.buckets).toEqual([ + { value: 'a', label: 'Apple', count: 2 }, + { value: 'b', label: 'Banana', count: 1 }, + ]); + } + }); + + it('summarises numeric answers with stats and a histogram', async () => { + findManySlots.mockResolvedValue([slot({ id: 'q2', type: 'numeric', typeConfig: {} })]); + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed' }, + { id: 's2', status: 'completed' }, + ]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'q2', value: 10, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'q2', value: 20, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'q2', value: 30, confidence: null, provenanceLabel: 'direct' }, + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail.kind).toBe('numeric'); + if (q.detail.kind === 'numeric') { + expect(q.detail.summary).toEqual({ count: 3, min: 10, max: 30, mean: 20, median: 20 }); + const total = q.detail.histogram.reduce((acc, b) => acc + b.count, 0); + expect(total).toBe(3); // every value lands in a bin + } + }); + + it('never exposes free-text values, only counts/confidence/provenance', async () => { + findManySlots.mockResolvedValue([slot({ id: 'q3', type: 'free_text', typeConfig: null })]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue([ + { + questionSlotId: 'q3', + value: 'my secret personal answer', + confidence: 0.9, + provenanceLabel: 'direct', + }, + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail).toEqual({ kind: 'free_text' }); + expect(q.answeredCount).toBe(1); + // The serialized detail must not carry the prose value anywhere. + expect(JSON.stringify(q.detail)).not.toContain('secret'); + }); + + it('counts multi_choice picks across the answer arrays', async () => { + findManySlots.mockResolvedValue([ + slot({ + id: 'qm', + type: 'multi_choice', + typeConfig: { + choices: [ + { value: 'a', label: 'Apple' }, + { value: 'b', label: 'Banana' }, + { value: 'c', label: 'Cherry' }, + ], + }, + }), + ]); + findManySessions.mockResolvedValue([ + { id: 's1', status: 'completed' }, + { id: 's2', status: 'completed' }, + ]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'qm', value: ['a', 'b'], confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qm', value: ['a', 'zzz'], confidence: null, provenanceLabel: 'inferred' }, + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.answeredCount).toBe(2); // two answer rows (sessions), not picks + expect(q.detail).toMatchObject({ kind: 'choice', otherCount: 1 }); + if (q.detail.kind === 'choice') { + expect(q.detail.buckets).toEqual([ + { value: 'a', label: 'Apple', count: 2 }, + { value: 'b', label: 'Banana', count: 1 }, + { value: 'c', label: 'Cherry', count: 0 }, + ]); + } + }); + + it('buckets likert answers per scale point with bound labels and a mean', async () => { + findManySlots.mockResolvedValue([ + slot({ + id: 'ql', + type: 'likert', + typeConfig: { min: 1, max: 5, minLabel: 'Low', maxLabel: 'High' }, + }), + ]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'ql', value: 1, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'ql', value: 5, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'ql', value: 3, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'ql', value: 9, confidence: null, provenanceLabel: 'direct' }, // out of range → ignored + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail.kind).toBe('likert'); + if (q.detail.kind === 'likert') { + expect(q.detail.min).toBe(1); + expect(q.detail.max).toBe(5); + expect(q.detail.buckets).toHaveLength(5); + expect(q.detail.buckets[0]).toEqual({ value: '1', label: '1 (Low)', count: 1 }); + expect(q.detail.buckets[4]).toEqual({ value: '5', label: '5 (High)', count: 1 }); + expect(q.detail.mean).toBeCloseTo(3, 5); // (1+5+3)/3, out-of-range dropped + } + }); + + it('counts boolean answers with custom labels', async () => { + findManySlots.mockResolvedValue([ + slot({ id: 'qb', type: 'boolean', typeConfig: { trueLabel: 'Yes', falseLabel: 'No' } }), + ]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'qb', value: true, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qb', value: true, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qb', value: false, confidence: null, provenanceLabel: 'direct' }, + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail).toEqual({ + kind: 'boolean', + trueLabel: 'Yes', + falseLabel: 'No', + trueCount: 2, + falseCount: 1, + }); + }); + + it('buckets date answers by month', async () => { + findManySlots.mockResolvedValue([slot({ id: 'qd', type: 'date', typeConfig: null })]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'qd', value: '2026-01-15', confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qd', value: '2026-01-20', confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qd', value: '2026-02-02', confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qd', value: 'not-a-date', confidence: null, provenanceLabel: 'direct' }, + ]); + + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail.kind).toBe('date'); + if (q.detail.kind === 'date') { + expect(q.detail.buckets).toEqual([ + { label: '2026-01', count: 2 }, + { label: '2026-02', count: 1 }, + ]); + } + }); + + it('degrades gracefully on malformed configs, empty stats, and unknown provenance', async () => { + findManySlots.mockResolvedValue([ + slot({ id: 'qc', type: 'single_choice', typeConfig: { garbage: true } }), // unreadable choices + slot({ id: 'ql', type: 'likert', typeConfig: null }), // no bounds → default 1..5, no labels + slot({ id: 'qb', type: 'boolean', typeConfig: null }), // default True/False labels + slot({ id: 'qn', type: 'numeric', typeConfig: {} }), // numeric with no numeric answers + ]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue([ + // choice with no readable config: everything counts as "other" + { questionSlotId: 'qc', value: 'whatever', confidence: null, provenanceLabel: 'weird' }, + // likert mid value, no labels → plain numeric label + { questionSlotId: 'ql', value: 3, confidence: null, provenanceLabel: 'synthesised' }, + { questionSlotId: 'qb', value: true, confidence: null, provenanceLabel: 'refined' }, + // numeric answer that isn't a number → dropped → summary null + { questionSlotId: 'qn', value: 'NaN-ish', confidence: null, provenanceLabel: 'direct' }, + ]); + + const result = await getQuestionDistributions(scope); + const [qc, ql, qb, qn] = result.questions; + + // Unknown provenance label falls back to 'direct'. + expect(qc.provenance.direct).toBe(1); + expect(qc.detail).toMatchObject({ kind: 'choice', otherCount: 1 }); + if (qc.detail.kind === 'choice') expect(qc.detail.buckets).toEqual([]); + + expect(ql.detail.kind).toBe('likert'); + if (ql.detail.kind === 'likert') { + expect(ql.detail.min).toBe(1); + expect(ql.detail.max).toBe(5); + expect(ql.detail.buckets[2]).toEqual({ value: '3', label: '3', count: 1 }); // no bound label + } + + expect(qb.detail).toMatchObject({ trueLabel: 'True', falseLabel: 'False', trueCount: 1 }); + + expect(qn.detail).toEqual({ kind: 'numeric', summary: null, histogram: [] }); + }); + + it('renders a single-bin histogram when all numeric answers are equal', async () => { + findManySlots.mockResolvedValue([slot({ id: 'qn', type: 'numeric', typeConfig: {} })]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue([ + { questionSlotId: 'qn', value: 7, confidence: null, provenanceLabel: 'direct' }, + { questionSlotId: 'qn', value: 7, confidence: null, provenanceLabel: 'direct' }, + ]); + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail.kind).toBe('numeric'); // guard: a wrong kind must fail, not skip the asserts + if (q.detail.kind === 'numeric') { + expect(q.detail.summary).toMatchObject({ min: 7, max: 7, median: 7 }); + expect(q.detail.histogram).toEqual([{ label: '7', min: 7, max: 7, count: 2 }]); + } + }); + + it('averages the two middle values for an even-count numeric median', async () => { + findManySlots.mockResolvedValue([slot({ id: 'qn', type: 'numeric', typeConfig: {} })]); + findManySessions.mockResolvedValue([{ id: 's1', status: 'completed' }]); + findManyAnswers.mockResolvedValue( + [10, 20, 30, 40].map((value) => ({ + questionSlotId: 'qn', + value, + confidence: null, + provenanceLabel: 'direct', + })) + ); + const result = await getQuestionDistributions(scope); + const q = result.questions[0]; + expect(q.detail.kind).toBe('numeric'); + if (q.detail.kind === 'numeric') { + // even count → median = (20 + 30) / 2 = 25, not a single middle element. + expect(q.detail.summary).toMatchObject({ count: 4, median: 25, mean: 25 }); + } + }); + + it('passes the tag filter into the slot query when tagIds are present', async () => { + findManySlots.mockResolvedValue([]); + findManySessions.mockResolvedValue([]); + await getQuestionDistributions({ ...scope, tagIds: ['t1', 't2'] }); + const where = findManySlots.mock.calls[0][0].where; + expect(where.tags).toEqual({ some: { tagId: { in: ['t1', 't2'] } } }); + }); + + it('skips the answer query entirely when there are no sessions', async () => { + findManySlots.mockResolvedValue([slot({ id: 'q1', type: 'free_text' })]); + findManySessions.mockResolvedValue([]); + const result = await getQuestionDistributions(scope); + expect(findManyAnswers).not.toHaveBeenCalled(); + expect(result.questions[0].responseRate).toBe(0); + expect(result.questions[0].answeredCount).toBe(0); + }); +}); diff --git a/tests/unit/lib/app/questionnaire/analytics/funnel.test.ts b/tests/unit/lib/app/questionnaire/analytics/funnel.test.ts new file mode 100644 index 000000000..a7cf35c9e --- /dev/null +++ b/tests/unit/lib/app/questionnaire/analytics/funnel.test.ts @@ -0,0 +1,108 @@ +/** + * Unit test: completion funnel aggregation (F8.1). + * + * Mocks the invitation + session reads and asserts stage counting (derived from real + * sessions, not invitation status), drop-off math, anonymous-session separation, and + * preview exclusion. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const findManyInvitations = vi.fn(); +const findManySessions = vi.fn(); + +vi.mock('@/lib/db/client', () => ({ + prisma: { + appQuestionnaireInvitation: { findMany: (...a: unknown[]) => findManyInvitations(...a) }, + appQuestionnaireSession: { findMany: (...a: unknown[]) => findManySessions(...a) }, + }, +})); + +import { getCompletionFunnel } from '@/lib/app/questionnaire/analytics/funnel'; +import type { AnalyticsScope } from '@/lib/app/questionnaire/analytics/query-schema'; + +const scope: AnalyticsScope = { + versionId: 'v1', + from: new Date('2026-01-01T00:00:00.000Z'), + to: new Date('2026-02-01T00:00:00.000Z'), + tagIds: [], +}; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('getCompletionFunnel', () => { + beforeEach(() => { + // 4 invited (all sent); 3 opened; 3 registered respondents u1/u2/u3. + findManyInvitations.mockResolvedValue([ + { sentAt: new Date(), openedAt: new Date(), userId: 'u1' }, + { sentAt: new Date(), openedAt: new Date(), userId: 'u2' }, + { sentAt: new Date(), openedAt: new Date(), userId: 'u3' }, + { sentAt: new Date(), openedAt: null, userId: null }, // sent, never opened/registered + ]); + // u1 started (active), u2 + u3 completed; plus two anonymous (un-invited). + findManySessions.mockResolvedValue([ + { respondentUserId: 'u1', status: 'active' }, + { respondentUserId: 'u2', status: 'completed' }, + { respondentUserId: 'u3', status: 'completed' }, + { respondentUserId: 'anon1', status: 'completed' }, + { respondentUserId: null, status: 'active' }, + ]); + }); + + it('counts invited → opened → started → completed from session reality', async () => { + const result = await getCompletionFunnel(scope); + const byKey = Object.fromEntries(result.stages.map((s) => [s.key, s.count])); + expect(byKey).toEqual({ invited: 4, opened: 3, started: 3, completed: 2 }); + }); + + it('computes drop-off, retention, and step conversion', async () => { + const result = await getCompletionFunnel(scope); + const completed = result.stages.find((s) => s.key === 'completed')!; + expect(completed.retention).toBeCloseTo(2 / 4, 5); // of invited + expect(completed.conversionFromPrev).toBeCloseTo(2 / 3, 5); // started → completed + expect(completed.dropoff).toBe(1); // 3 started, 2 completed + + const invited = result.stages.find((s) => s.key === 'invited')!; + expect(invited.dropoff).toBe(0); + expect(invited.conversionFromPrev).toBe(1); + }); + + it('reports anonymous (un-invited) sessions separately', async () => { + const result = await getCompletionFunnel(scope); + expect(result.anonymous).toEqual({ started: 2, completed: 1 }); + }); + + it('excludes preview sessions and revoked invitations in its queries', async () => { + await getCompletionFunnel(scope); + expect(findManySessions.mock.calls[0][0].where.isPreview).toBe(false); + expect(findManyInvitations.mock.calls[0][0].where.revokedAt).toBeNull(); + }); + + it('handles a zero-invite window without dividing by zero', async () => { + findManyInvitations.mockResolvedValue([]); + findManySessions.mockResolvedValue([{ respondentUserId: 'anon', status: 'completed' }]); + const result = await getCompletionFunnel(scope); + expect(result.stages).toHaveLength(4); // lock the stage count so .every() can't be vacuous + expect(result.stages.every((s) => s.retention === 0)).toBe(true); + expect(result.anonymous).toEqual({ started: 1, completed: 1 }); + }); + + it('guards conversionFromPrev against a zero-count intermediate stage', async () => { + // 2 invited, but none opened → the started stage converts from a zero "opened" base. + findManyInvitations.mockResolvedValue([ + { sentAt: new Date(), openedAt: null, userId: 'u1' }, + { sentAt: new Date(), openedAt: null, userId: 'u2' }, + ]); + findManySessions.mockResolvedValue([{ respondentUserId: 'u1', status: 'active' }]); + + const result = await getCompletionFunnel(scope); + const byKey = Object.fromEntries(result.stages.map((s) => [s.key, s])); + expect(byKey.opened.count).toBe(0); + expect(byKey.started.count).toBe(1); + // prev (opened) is 0 → guard returns 0, not NaN/Infinity. + expect(byKey.started.conversionFromPrev).toBe(0); + expect(Number.isFinite(byKey.started.conversionFromPrev)).toBe(true); + }); +});