From 225c23c7717ea3e8b0d2de18092a4899cc0aa491 Mon Sep 17 00:00:00 2001 From: Christopher Date: Wed, 1 Jul 2026 23:36:15 +1000 Subject: [PATCH 1/7] feat(dashboard): thread promptfoo tags map through lightweight loader (av-y4ns) Add the promptfoo-shaped `tags` map (Record) to the Dashboard lightweight result path so the server can group runs by an arbitrary tag key. - Add `tags?: Record` to `ResultManifestRecord` and `LightweightResultRecord`; populate it in `loadLightweightResults` from the parsed JSONL row's `tags` field (dropping non-string values). - Read `metadata.tags` from summary.json in the Dashboard run metadata reader and emit `run_tags` (snake_case wire) on `RunMeta`, sourced from row tags with a summary-metadata fallback for in-progress runs. - Add `run_tags?: Record` to the client `RunMeta` type. Load-bearing prerequisite for the Tags tab. Includes the brainstorm/design doc (docs/plans/dashboard-tags-tab-brainstorm.md), not yet on main. Refs av-qsxw, av-y4ns Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/results/manifest.ts | 23 +++ apps/cli/src/commands/results/serve.ts | 25 +++ apps/dashboard/src/lib/types.ts | 7 + docs/plans/dashboard-tags-tab-brainstorm.md | 160 ++++++++++++++++++++ 4 files changed, 215 insertions(+) create mode 100644 docs/plans/dashboard-tags-tab-brainstorm.md diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts index ceb4c9754..52c6bcad4 100644 --- a/apps/cli/src/commands/results/manifest.ts +++ b/apps/cli/src/commands/results/manifest.ts @@ -28,6 +28,8 @@ export interface ResultManifestRecord { readonly suite?: string; readonly category?: string; readonly experiment?: string; + /** promptfoo-shaped tag map (`Record`), e.g. `{experiment, team, env}`. */ + readonly tags?: Record; readonly target?: string; readonly variant?: string; readonly score: number; @@ -315,6 +317,8 @@ export interface LightweightResultRecord { readonly target?: string; readonly variant?: string; readonly experiment?: string; + /** promptfoo-shaped tag map from the JSONL row's `tags` field. */ + readonly tags?: Record; readonly score: number; readonly scores?: readonly Record[]; readonly executionStatus?: string; @@ -324,6 +328,24 @@ export interface LightweightResultRecord { readonly runtimeSource?: RunRuntimeSourceMetadata; } +/** + * Coerce a raw JSONL `tags` value into a `Record`, dropping + * non-string values. Returns undefined when the map is absent or empty so the + * lightweight record stays sparse for old runs that never wrote a tags map. + */ +function normalizeTagMap(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return undefined; + } + const entries: [string, string][] = []; + for (const [key, raw] of Object.entries(value as Record)) { + if (typeof raw === 'string') { + entries.push([key, raw]); + } + } + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + export function loadLightweightResults(sourceFile: string): LightweightResultRecord[] { const resolvedSourceFile = resolveRunManifestPath(sourceFile); const content = readFileSync(resolvedSourceFile, 'utf8'); @@ -335,6 +357,7 @@ export function loadLightweightResults(sourceFile: string): LightweightResultRec target: record.target, variant: record.variant, experiment: record.experiment, + tags: normalizeTagMap(record.tags), score: record.score, scores: record.scores, executionStatus: record.execution_status, diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 3e618cfaa..62634a7af 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -1573,6 +1573,7 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext let target = m.target; let experiment = m.experiment ?? inferExperimentFromRunId(m.raw_filename); const summaryMetadata = readRunSummaryMetadataForDashboard(m.path); + let runTags: Record | undefined = summaryMetadata.tags; let runtimeSource: RunRuntimeSourceMetadata | undefined = summaryMetadata.runtimeSource; let timestamp = m.timestamp; let testCount = m.testCount; @@ -1587,6 +1588,7 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext const qualitySummary = summarizeQualityResults(records, passThreshold); target = records[0].target; experiment = records[0].experiment ?? experiment; + runTags = records[0].tags ?? runTags; timestamp = hasUsableTimestamp(timestamp) || !records[0].timestamp ? timestamp @@ -1632,6 +1634,7 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext on_remote: m.on_remote, ...(target && { target }), ...(experiment && { experiment }), + ...(runTags && { run_tags: runTags }), ...(runtimeSource && { runtime_source: runtimeSource }), ...tagFields, ...(liveStatus && { status: liveStatus }), @@ -1746,10 +1749,29 @@ function attachExternalTraceFields>( interface RunSummaryMetadataForDashboard { readonly evalFile?: string; readonly experiment?: string; + readonly tags?: Record; readonly plannedTestCount?: number; readonly runtimeSource?: RunRuntimeSourceMetadata; } +/** + * Coerce a summary.json `metadata.tags` value into a `Record`, + * dropping non-string values. Returns undefined for absent/empty maps so old + * runs (no tags map) stay sparse. + */ +function normalizeSummaryTagMap(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return undefined; + } + const entries: [string, string][] = []; + for (const [key, raw] of Object.entries(value as Record)) { + if (typeof raw === 'string') { + entries.push([key, raw]); + } + } + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + function readRunSummaryMetadataForDashboard(manifestPath: string): RunSummaryMetadataForDashboard { try { const summaryPath = path.join(path.dirname(manifestPath), 'summary.json'); @@ -1760,16 +1782,19 @@ function readRunSummaryMetadataForDashboard(manifestPath: string): RunSummaryMet metadata?: { eval_file?: string; experiment?: string; + tags?: Record; planned_test_count?: number; runtime_source?: RunRuntimeSourceMetadata; }; }; const planned = parsed.metadata?.planned_test_count; + const tags = normalizeSummaryTagMap(parsed.metadata?.tags); return { ...(typeof parsed.metadata?.eval_file === 'string' && parsed.metadata.eval_file.trim() && { evalFile: parsed.metadata.eval_file.trim() }), ...(typeof parsed.metadata?.experiment === 'string' && parsed.metadata.experiment.trim() && { experiment: parsed.metadata.experiment.trim() }), + ...(tags && { tags }), ...(typeof planned === 'number' && Number.isFinite(planned) && planned > 0 && { plannedTestCount: planned }), diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index 7cb6fcf13..fa094f83c 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -32,6 +32,13 @@ export interface RunMeta { size_bytes: number; target?: string; experiment?: string; + /** + * promptfoo-shaped tag map (`Record`) resolved for this run, + * e.g. `{experiment: "v2", team: "core", env: "ci"}`. Lets the client group + * and filter runs by an arbitrary tag key. Distinct from the legacy manual + * per-run label chips (now removed). + */ + run_tags?: Record; runtime_source?: RunRuntimeSource; source: 'local' | 'remote'; /** diff --git a/docs/plans/dashboard-tags-tab-brainstorm.md b/docs/plans/dashboard-tags-tab-brainstorm.md new file mode 100644 index 000000000..234d657a4 --- /dev/null +++ b/docs/plans/dashboard-tags-tab-brainstorm.md @@ -0,0 +1,160 @@ +# Dashboard "Tags" Tab — Brainstorm / Design + +Status: brainstorm (decision-oriented, not a plan) +Date: 2026-07-01 +Scope: rename the Dashboard "Experiments" tab to "Tags" and let the user pick which tag key drives grouping/comparison. `experiment` becomes one key in a promptfoo-shaped `tags` map rather than a privileged first-class concept. + +--- + +## Operator decisions (2026-07-01) — supersede §2/§7 where they conflict + +- **Remove the legacy manual `tags.json` chips entirely** rather than rename them to "Labels". There is only one tags concept: the promptfoo `tags` map. This deletes the manual `string[]` sidecar and all its surface: `tags.json` read/write, remote/pending tag sync, the editable-chips UI, the compare `?tags=` filter, and the `RunMeta.tags`/`remote_tags`/`pending_tags` + `RunFinalState.tags` + `CompareRunEntry.tags` wire fields. With the manual concept gone, the §2 naming collision disappears (no "Labels" rename, no `?tags=`→`?labels=`). +- **Phase 1 = Option A** (tag-key selector regrouping the table; `experiment` default key). Faceted filters and arbitrary-key compare grouping stay phase 2. + +--- + +## 1. Problem statement & current behavior + +Today the Dashboard has a first-class **Experiments** tab that groups runs purely on the per-row `experiment` **string**. A recently merged feature makes runs also write a promptfoo-shaped `tags` **map** (`Record`, including an `experiment` key), but the Dashboard ignores that map entirely. The goal is to generalize: group/compare by *any* tag key (`experiment`, `team`, `env`, `model`, arbitrary user keys), with `experiment` as just the default-selected key. + +Current wiring (verified): + +- Tab declaration: `apps/dashboard/src/routes/index.tsx:48-53` (`{ id: 'experiments', label: '🧪 Experiments' }`), rendered at `index.tsx:368` (`{activeTab === 'experiments' && }`). +- Grouping table: `apps/dashboard/src/components/ExperimentsTab.tsx` — reads `data.experiments` (`ExperimentSummary[]`), rows keyed on `exp.name`, links to `/experiments/$experimentName` (lines 47-108). +- Detail view: `apps/dashboard/src/components/ExperimentDetail.tsx:48-51` — finds the summary by `entry.name === experimentName` and filters runs with `(run.experiment ?? 'default') === experimentName`. Grouping is on the run's `experiment` string, not tags. +- Routes: `apps/dashboard/src/routes/experiments/$experimentName.tsx` and `apps/dashboard/src/routes/projects/$projectId_/experiments/$experimentName.tsx`. +- Data loading: `apps/dashboard/src/lib/api.ts:201-204` (`experimentsOptions` → `GET /api/experiments`) and `api.ts:634-638` (`projectExperimentsOptions` → `GET {projectApiBase}/experiments`). Types `ExperimentSummary` / `ExperimentsResponse` at `apps/dashboard/src/lib/types.ts:356-370`. +- Server grouping: `handleExperiments` in `apps/cli/src/commands/results/serve.ts:2370-2432`. It reads only `r.experiment ?? 'default'` (`serve.ts:2390`) from lightweight records; the promptfoo `tags` map is never consulted. The compare handler is the same story — `handleCompare` groups on `r.experiment` at `serve.ts:2523-2527` and (importantly) already uses the *other* "tags" for filtering (see §2). + +**The blocking data-plumbing fact:** the server groups off `LightweightResultRecord`, whose type (`apps/cli/src/commands/results/manifest.ts:310-325`) and loader (`loadLightweightResults`, `manifest.ts:327-346`) expose `experiment` but have **no `tags` field**. So even though each JSONL row carries a `tags` map on disk, the Dashboard's lightweight path drops it before `handleExperiments`/`handleCompare` ever see it. Any Tags-tab work that groups by an arbitrary key must first thread the `tags` map through this loader. + +Artifact source of truth (verified in core): + +- `summary.json` `metadata.tags` (`Record`) is written via `aggregateRunDir` → `buildRunSummaryArtifact` in `packages/core/src/evaluation/run-artifacts.ts` (`tags` option threaded at `run-artifacts.ts:168,180,190`; round-tripped through `readRunSummaryMetadata` at `run-artifacts.ts:245-258`). +- Each `index.jsonl` row carries `experiment` (string) and `tags` (`Record`): the `ResultIndexArtifact` type declares both at `run-artifacts.ts:477,479`, and `writePerTestArtifacts` writes each row with `experiment: options?.experiment` and `...(resolvedTags ? { tags: resolvedTags } : {})` at `run-artifacts.ts:2351-2371` (row push at `2369-2370`). +- Lockstep: the run-level `experiment` namespace is derived from the `tags` map's reserved `experiment` key (`run-artifacts.ts:435-439` — "The reserved key `experiment` feeds the experiment namespace"; `experiment_namespace_source: 'tags'` at `run-artifacts.ts:82,310`). The equality is actively enforced at run time by `syncTagsExperiment` (`apps/cli/src/commands/eval/run-eval.ts:~404-424`) with precedence resolved by `resolveExperimentNamespace` (CLI `--experiment` > `tags.experiment` > eval defaults, `run-eval.ts:~426-461`), so the top-level `experiment` field and `tags.experiment` stay equal. The Dashboard-side resolution should therefore read `record.experiment ?? record.tags?.experiment`. +- Confirmed drop point: neither `ResultManifestRecord` (`manifest.ts:25-76`) nor `LightweightResultRecord` (`manifest.ts:310-325`) declares `tags`, and the core row normalizer (`packages/core/src/evaluation/result-row-schema.ts:~189-221`) has no `tags` alias — the map is present in the raw JSONL but dropped by the CLI parse layer before any handler sees it. This is exactly the plumbing §4 must fix. + +--- + +## 2. The "two tags" collision (must be resolved before any UI work) + +There are **two unrelated concepts both named "tags"** in the Dashboard. This is the single biggest source of confusion for this feature and must be named apart in the UI. + +| | promptfoo tags **map** | manual `tags.json` **chips** | +|---|---|---| +| Shape | `Record` (e.g. `{experiment: "v2", team: "core", env: "ci"}`) | `string[]` (e.g. `["baseline", "flaky"]`) | +| Origin | Authored in eval/project config; resolved at run time; written into `summary.json`/`index.jsonl` | User-assigned in the Dashboard, stored in a per-run `tags.json` sidecar | +| Purpose | Structured facets to group/compare runs by (the subject of this brainstorm) | Free-form labels to filter/annotate runs | +| Server surface | Currently **ignored** by the Dashboard | `RunTagFields` / `readRunTagFields` (`serve.ts:1242-1249,1266-1299`); used as the `?tags=` OR-filter in `handleCompare` (`serve.ts:2503-2509`); surfaced on `RunMeta.tags`/`remote_tags`/`pending_tags` (`types.ts:45-52`) and `RunFinalState.tags` (`types.ts:66-69`) | +| Editability | Read-only (derived from config) | Editable + syncs to remote results repo | + +**Recommended naming reconciliation:** + +- Rename the new grouping concept to **"Tags"** in the tab, and consistently call the map keys **tag keys** and the values **tag values**. This is the promptfoo-native vocabulary and matches the map on disk. +- Rename the existing manual `string[]` sidecar concept in the **UI** to **"Labels"** (chips), even though the on-disk file stays `tags.json` and the wire fields stay `tags`/`remote_tags`/`pending_tags` for backward compatibility. Do the rename at the presentation layer only; do not churn the wire format. + - Alternative if "Labels" is too invasive: keep calling them "tags" but always render them as removable chips on a run and never expose them in the Tags-tab grouping UI, so the two never appear in the same control. The map is "group by", the chips are "filter/annotate". This is weaker — the shared word will still confuse — so **prefer the "Labels" rename**. +- Document the distinction in `CONCEPTS.md` (map = "tags", chips = "labels/run labels") so future work doesn't re-collide them. + +Open question for the operator: is renaming the sidecar chips to "Labels" acceptable, or is "tags" load-bearing in user muscle memory / docs? (See §7.) + +--- + +## 3. Design options for the Tags tab + +### Option A — Tag-key selector dropdown drives a single grouping table (recommended) +A dropdown at the top of the tab lists the tag keys present across all runs (union of every row's `tags` map keys, plus a synthetic `experiment`). Selecting a key regroups the existing table by that key's values. Default selection: `experiment`. + +- Pros: smallest UI delta from today's Experiments table; one mental model ("group by this key"); reuses the entire existing table/detail/pass-rate layout; graceful for old runs (only `experiment` key appears). +- Cons: single-key only (no cross-tabulation); requires the server to enumerate available keys. + +### Option B — Faceted multi-key filter + grouping +User picks a **group-by** key *and* optional **filter** facets (e.g. group by `model`, filter `env=ci`). Table shows the group-by values; facet chips narrow the population. +- Pros: powerful; matches how people actually slice eval results; composes with the existing `?tags=` label filter. +- Cons: much bigger UI + server surface; risk of over-building before we know users want cross-facet slicing (YAGNI). Better as a phase-2 layer on top of A. + +### Option C — Keep "Experiments" as a pinned default view, add a general Tags view alongside +Two entry points: the familiar Experiments table (pinned to `tags.experiment`) plus a general Tags explorer. +- Pros: zero behavior change for existing users; lowest migration risk. +- Cons: contradicts the stated intent ("experiment stops being privileged"); two tabs doing nearly the same thing; more surface to maintain. Not recommended, but the "pin experiment as default key" idea is worth keeping — fold it into Option A. + +### Option D — Matrix / pivot (tag-key × tag-key) +Render a pivot table crossing two keys (rows = `team`, cols = `env`) with pass-rate cells. +- Pros: strong for comparison-heavy users; natural extension of the existing compare grid. +- Cons: heaviest to build and hardest to make legible with sparse data; clearly out of scope for a first cut. Park it. + +**Recommendation: ship Option A now**, structured so Option B's facet filter can layer on later without rework. Keep `experiment` as the default-selected key so the tab looks and behaves like today's Experiments tab on first load. + +--- + +## 4. Data / API changes + +**Thread the `tags` map through the lightweight loader (prerequisite for everything).** +- Add `readonly tags?: Record` to `LightweightResultRecord` (`manifest.ts:310-325`) and populate it in `loadLightweightResults` (`manifest.ts:327-346`) from the parsed row's `tags` map. This is the load-bearing change; without it the server literally cannot group by any key. + +**Generalize `handleExperiments` → `handleTags` (keep both routes during transition).** +- New endpoint `GET /api/tags` returns the list of available tag keys and their per-value summaries; accept `?key=` to select the grouping key (default `experiment`). +- Shape sketch: + ``` + GET /api/tags -> { keys: string[] } // for the dropdown + GET /api/tags?key=team -> { key: "team", groups: TagGroupSummary[] } + ``` + where `TagGroupSummary` is essentially today's `ExperimentSummary` (`types.ts:356-366`) with `name` = the tag value. +- Implementation: reuse the `handleExperiments` aggregation loop (`serve.ts:2386-2429`) but key the map on `record.tags?.[key]` instead of `record.experiment`. For `key === 'experiment'`, fall back to `record.experiment ?? record.tags?.experiment ?? 'default'` so lockstep and old runs both resolve. +- Keep `GET /api/experiments` as a thin alias of `handleTags(key='experiment')` for one release so nothing breaks mid-migration; delete once the frontend cuts over. + +**Available-keys enumeration.** `handleTags` (or a cheap `GET /api/tags`) walks every run's rows, unions `Object.keys(row.tags ?? {})`, always includes `experiment`, and returns a sorted key list for the dropdown. This is O(rows) but the loop already exists. + +**Compare view.** `handleCompare` (`serve.ts:2434+`) groups cells on `[experiment, target]` (`serve.ts:2531`). To let compare group by an arbitrary key, add an optional `?group_key=` that swaps `experiment` for `row.tags?.[group_key]`. Out of scope for phase 1; note it as the natural phase-2 follow-on so the tab's "compare" affordance stays coherent. **Do not** conflate this with the existing `?tags=` param on `handleCompare` — that one filters on the manual **label** `string[]` (`serve.ts:2503-2509`), a different concept (see §2). Consider renaming that query param to `?labels=` when the UI rename lands. + +**Backward compatibility (old runs with no tags map).** +- Rows/summaries written before the feature have no `tags` map but do have `experiment`. Under Option A the only key such runs contribute is `experiment` (via the fallback above); they simply don't appear under other keys. That's correct and needs no migration. +- `record.tags?.[key]` for a missing key groups those runs under a `default`/`(none)` bucket — decide whether to show or hide the "(none)" bucket (recommend: show it, labeled `(no )`). + +--- + +## 5. UI / UX sketch + +- **Tab label:** `🏷️ Tags` replacing `🧪 Experiments` at `index.tsx:50`. +- **Tag-key selector:** a small `` populated from `GET /api/tags`, defaulting to `experiment` so day-1 behavior matches the old Experiments tab. The selected key lives in the URL (`?tab=tags&key=`) so the view is shareable/refresh-safe. - `TagsTab` (was ExperimentsTab) regroups the table by the selected key; the first column header is the key label and the empty state is key-aware. - `TagValueDetail` (was ExperimentDetail) filters runs by `run_tags?.[key] === value` with the reserved-`experiment` fallback (`run.experiment ?? run.run_tags?.experiment`). - Routes `/tags/$key/$value` (+ project-scoped); the old `/experiments/$experimentName` routes now redirect to `/tags/experiment/` for one release. Sidebar + breadcrumbs generalized to the tag key/value, and the tags query cache is invalidated on sync/combine. - Add `TagKeysResponse`/`TagGroupsResponse`/`TagGroupSummary` wire types and `tagKeysOptions`/`tagGroupsOptions` (+ project variants) with `DEFAULT_TAG_KEY`. Legacy manual-tags UI removal (av-ccz9): - Remove the tag chips + inline `TagsEditor`, the compare tag-filter bar and its client-side re-aggregation, the `run.tags` heatmap, and the launch-time tags input in RunEvalModal. Drop the manual-tags wire fields (`RunMeta`/`CompareRunEntry`/`RunFinalState`/`RunTagsResponse`, `RunEvalRequest.tags`) and the `saveRunTagsApi`/`deleteRunTagsApi` clients. - Add `run_tags?: Record` to `RunMeta` for client-side detail filtering by tag key. Regenerates routeTree.gen.ts. Dashboard builds, typechecks, tests (145), and lints clean. Refs av-qsxw, av-hd6c, av-ccz9 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/components/AnalyticsCharts.tsx | 115 +--- .../dashboard/src/components/AnalyticsTab.tsx | 601 ++---------------- apps/dashboard/src/components/Breadcrumbs.tsx | 14 +- .../src/components/ExperimentsTab.tsx | 152 ----- .../dashboard/src/components/RunEvalModal.tsx | 133 +--- apps/dashboard/src/components/RunList.tsx | 20 +- apps/dashboard/src/components/Sidebar.tsx | 102 +-- ...xperimentDetail.tsx => TagValueDetail.tsx} | 68 +- apps/dashboard/src/components/TagsTab.tsx | 199 ++++++ .../src/components/run-eval-threshold.test.ts | 6 +- .../src/components/run-eval-threshold.ts | 3 - apps/dashboard/src/lib/api.ts | 95 ++- apps/dashboard/src/lib/navigation.test.ts | 9 +- apps/dashboard/src/lib/navigation.ts | 17 +- apps/dashboard/src/lib/types.ts | 65 +- apps/dashboard/src/routeTree.gen.ts | 43 ++ .../routes/experiments/$experimentName.tsx | 20 +- apps/dashboard/src/routes/index.tsx | 29 +- .../src/routes/projects/$projectId.tsx | 31 +- .../experiments/$experimentName.tsx | 24 +- .../projects/$projectId_/tags/$key.$value.tsx | 17 + .../dashboard/src/routes/tags/$key.$value.tsx | 16 + 22 files changed, 603 insertions(+), 1176 deletions(-) delete mode 100644 apps/dashboard/src/components/ExperimentsTab.tsx rename apps/dashboard/src/components/{ExperimentDetail.tsx => TagValueDetail.tsx} (53%) create mode 100644 apps/dashboard/src/components/TagsTab.tsx create mode 100644 apps/dashboard/src/routes/projects/$projectId_/tags/$key.$value.tsx create mode 100644 apps/dashboard/src/routes/tags/$key.$value.tsx diff --git a/apps/dashboard/src/components/AnalyticsCharts.tsx b/apps/dashboard/src/components/AnalyticsCharts.tsx index dd34dd306..f72dcdd1c 100644 --- a/apps/dashboard/src/components/AnalyticsCharts.tsx +++ b/apps/dashboard/src/components/AnalyticsCharts.tsx @@ -7,10 +7,9 @@ * fields and renders the following charts: * * 1. Normalized gain bar chart (horizontal bars, g per task × target) - * 2. Domain/tag heatmap (pass rate by tag × target) - * 3. Negative delta table (tasks where non-baseline scored worse) - * 4. Filterable score distribution histogram (experiment/category/time) - * 5. Trend-over-time line chart (mean score per target over time) + * 2. Negative delta table (tasks where non-baseline scored worse) + * 3. Filterable score distribution histogram (experiment/category/time) + * 4. Trend-over-time line chart (mean score per target over time) * * All charts use recharts styled with Tailwind-matching colors to * respect the Dashboard dark theme (gray-950 canvas, cyan accents, @@ -144,20 +143,15 @@ export function AnalyticsCharts({ data, projectId }: AnalyticsChartsProps) { )} - {/* 2. Tag heatmap */} - {data.runs && data.runs.length > 0 && targets.length > 1 && ( - - )} - - {/* 3. Negative delta table */} + {/* 2. Negative delta table */} {baseline && baselineData && ( )} - {/* 4. Score distribution histogram */} + {/* 3. Score distribution histogram */} - {/* 5. Trend over time */} + {/* 4. Trend over time */} {trendData.length > 1 && targets.length > 0 && ( )} @@ -301,98 +295,7 @@ function NormalizedGainChart({ data, baseline }: { data: CompareResponse; baseli ); } -// ── 2. Tag heatmap ───────────────────────────────────────────────────── - -function TagHeatmap({ runs, targets }: { runs: CompareRunEntry[]; targets: string[] }) { - const { tags, grid } = useMemo(() => { - // Collect all tags and compute pass rate per (tag, target) - const tagTargetMap = new Map>(); - for (const run of runs) { - for (const tag of run.tags ?? []) { - let targetMap = tagTargetMap.get(tag); - if (!targetMap) { - targetMap = new Map(); - tagTargetMap.set(tag, targetMap); - } - let entry = targetMap.get(run.target); - if (!entry) { - entry = { passed: 0, total: 0 }; - targetMap.set(run.target, entry); - } - entry.passed += run.passed_count; - entry.total += run.eval_count; - } - } - const allTags = [...tagTargetMap.keys()].sort(); - const gridData = allTags.map((tag) => { - const row: Record = { tag }; - const targetMap = tagTargetMap.get(tag); - if (!targetMap) return row; - for (const target of targets) { - const entry = targetMap.get(target); - row[target] = entry && entry.total > 0 ? entry.passed / entry.total : -1; - } - return row; - }); - return { tags: allTags, grid: gridData }; - }, [runs, targets]); - - if (tags.length === 0) return null; - - return ( - -
- - - - - {targets.map((t) => ( - - ))} - - - - {grid.map((row) => ( - - - {targets.map((target) => { - const val = row[target] as number; - if (val < 0) { - return ( - - ); - } - const pct = Math.round(val * 100); - const colorClass = - val >= 0.8 - ? 'bg-emerald-400/20 text-emerald-400' - : val >= 0.5 - ? 'bg-yellow-400/20 text-yellow-400' - : 'bg-red-400/20 text-red-400'; - return ( - - ); - })} - - ))} - -
Tag - {t} -
{row.tag as string} - — - - - {pct}% - -
-
- - ); -} - -// ── 3. Negative delta table ──────────────────────────────────────────── +// ── 2. Negative delta table ──────────────────────────────────────────── function NegativeDeltaTable({ data, baseline }: { data: CompareResponse; baseline: string }) { const negatives = useMemo(() => { @@ -444,7 +347,7 @@ function NegativeDeltaTable({ data, baseline }: { data: CompareResponse; baselin ); } -// ── 4. Score distribution histogram ──────────────────────────────────── +// ── 3. Score distribution histogram ──────────────────────────────────── const DEFAULT_DISTRIBUTION_FILTERS: ScoreDistributionFilters = { experiment: ALL_DISTRIBUTION_FILTER_VALUE, @@ -651,7 +554,7 @@ function scoreDistributionEmptyMessage( return 'No scores match the selected distribution filters.'; } -// ── 5. Trend over time ───────────────────────────────────────────────── +// ── 4. Trend over time ───────────────────────────────────────────────── interface TrendPoint { date: string; diff --git a/apps/dashboard/src/components/AnalyticsTab.tsx b/apps/dashboard/src/components/AnalyticsTab.tsx index cd95956a4..77d675917 100644 --- a/apps/dashboard/src/components/AnalyticsTab.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.tsx @@ -4,8 +4,7 @@ * Two modes: * 1. Aggregated (default) — `(experiment, target)` matrix, one cell per pair. * 2. Per run — individual runs are first-class; users select - * 2+ runs to render a side-by-side comparison, - * and may attach retroactive tags to any run. + * 2+ runs to render a side-by-side comparison. * * Styling matches the rest of AgentV Dashboard: dark gray surfaces * (`bg-gray-900` / `border-gray-800`), cyan accents for interactive elements, @@ -14,8 +13,6 @@ * * Backend contract: * - `GET /api/compare` → { cells, runs? } - * - `PUT /api/runs/:runId/tags` → replaces sidecar tags.json - * - `DELETE /api/runs/:runId/tags` → records an empty tag state * * To extend with a new mode: add a value to `ViewMode`, a button in the mode * toggle, and a new body component in the content switch. Hooks in any new @@ -23,10 +20,8 @@ * hook order does not change across renders. */ -import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useMemo, useState } from 'react'; -import { deleteRunTagsApi, saveRunTagsApi } from '~/lib/api'; import { aggregateQualityCount, executionErrorCount } from '~/lib/result-summary'; import type { CompareCell, CompareResponse, CompareRunEntry, CompareTestResult } from '~/lib/types'; @@ -40,7 +35,7 @@ interface AnalyticsTabProps { error?: Error | null; /** Project scope. Undefined for the unscoped (root) compare view. */ projectId?: string; - /** Read-only mode disables tag editing. */ + /** Read-only mode. Reserved for surfaces that disable mutating actions. */ readOnly?: boolean; } @@ -57,108 +52,9 @@ export function AnalyticsTab({ readOnly, }: AnalyticsTabProps) { const [mode, setMode] = useState('aggregated'); - const [filterTags, setFilterTags] = useState([]); - // Chip list is derived from the UNFILTERED response so chips stay visible - // even when the active filter would otherwise hide the runs that supplied - // them. Sorted alphabetically for stable UI. - const { allTags, tagCounts } = useMemo(() => { - const counts = new Map(); - for (const run of data?.runs ?? []) { - for (const tag of run.tags ?? []) { - counts.set(tag, (counts.get(tag) ?? 0) + 1); - } - } - return { allTags: [...counts.keys()].sort(), tagCounts: counts }; - }, [data?.runs]); - - // When a filter is active, re-aggregate cells/runs client-side from the - // filtered subset of runs. This avoids a network round-trip on every chip - // click and keeps the backend responsible only for the initial fetch. - // Safe because the server already exposes per-run totals; we sum them per - // (experiment, target) bucket, weighting averages by quality_count so - // execution errors do not depress quality scores. - const filteredData = useMemo(() => { - if (!data) return data; - if (filterTags.length === 0) return data; - const filterSet = new Set(filterTags); - const filteredRuns = (data.runs ?? []).filter((r) => - (r.tags ?? []).some((t) => filterSet.has(t)), - ); - - type CellAccum = { - experiment: string; - target: string; - eval_count: number; - quality_count: number; - passed_count: number; - execution_error_count: number; - score_sum: number; - tests: CompareTestResult[]; - }; - const cellMap = new Map(); - const experimentsSet = new Set(); - const targetsSet = new Set(); - - for (const run of filteredRuns) { - experimentsSet.add(run.experiment); - targetsSet.add(run.target); - const key = `${run.experiment}::${run.target}`; - const entry = cellMap.get(key) ?? { - experiment: run.experiment, - target: run.target, - eval_count: 0, - quality_count: 0, - passed_count: 0, - execution_error_count: 0, - score_sum: 0, - tests: [], - }; - const runQualityCount = aggregateQualityCount(run); - entry.eval_count += run.eval_count; - entry.quality_count += runQualityCount; - entry.passed_count += run.passed_count; - entry.execution_error_count += executionErrorCount(run); - entry.score_sum += run.avg_score * runQualityCount; - for (const t of run.tests) entry.tests.push(t); - cellMap.set(key, entry); - } - - const cells: CompareCell[] = [...cellMap.values()].map((e) => { - // Dedupe tests by test_id, last-wins (same pattern as the server). - const dedup = new Map(); - for (const t of e.tests) dedup.set(t.test_id, t); - return { - experiment: e.experiment, - target: e.target, - eval_count: e.eval_count, - quality_count: e.quality_count, - passed_count: e.passed_count, - execution_error_count: e.execution_error_count, - pass_rate: e.quality_count > 0 ? e.passed_count / e.quality_count : 0, - avg_score: e.quality_count > 0 ? e.score_sum / e.quality_count : 0, - tests: [...dedup.values()].slice(-100), - }; - }); - - return { - ...data, - experiments: [...experimentsSet].sort(), - targets: [...targetsSet].sort(), - cells, - runs: filteredRuns, - }; - }, [data, filterTags]); - - const toggleFilterTag = (tag: string) => { - setFilterTags((prev) => (prev.includes(tag) ? prev.filter((x) => x !== tag) : [...prev, tag])); - }; - const clearFilterTags = () => setFilterTags([]); - - const runsCount = filteredData?.runs?.length ?? 0; + const runsCount = data?.runs?.length ?? 0; const underlyingHasData = data && data.cells.length > 0; - const filterYieldsNoRuns = - filterTags.length > 0 && filteredData && (filteredData.runs?.length ?? 0) === 0; return (
@@ -169,38 +65,11 @@ export function AnalyticsTab({ )} {!isLoading && !isError && !underlyingHasData && } - {!isLoading && !isError && underlyingHasData && ( + {!isLoading && !isError && underlyingHasData && data && ( <> - {allTags.length > 0 && ( - - )} - {filterYieldsNoRuns ? ( - `\`${t}\``).join(' + ')}`} - body="Clear the filter or pick a different tag combination." - action={{ label: 'Clear filter', onClick: clearFilterTags }} - /> - ) : ( - filteredData && ( - <> - {mode === 'aggregated' && ( - - )} - {mode === 'per-run' && ( - - )} - - ) + {mode === 'aggregated' && } + {mode === 'per-run' && ( + )} )} @@ -208,72 +77,6 @@ export function AnalyticsTab({ ); } -// ── Tag filter bar ────────────────────────────────────────────────────── - -function TagFilterBar({ - allTags, - tagCounts, - selected, - onToggle, - onClear, -}: { - allTags: string[]; - tagCounts: Map; - selected: string[]; - onToggle: (tag: string) => void; - onClear: () => void; -}) { - const selectedSet = new Set(selected); - const anySelected = selected.length > 0; - return ( -
-
- - Filter by tag - - {allTags.map((tag) => { - const isActive = selectedSet.has(tag); - const count = tagCounts.get(tag) ?? 0; - return ( - - ); - })} - {anySelected && ( - - )} -
-

- Showing runs with any selected tag. -

-
- ); -} - // ── Header ────────────────────────────────────────────────────────────── function Header({ @@ -525,19 +328,10 @@ function TestBreakdown({ tests }: { tests: CompareTestResult[] }) { // ── Per-run view ──────────────────────────────────────────────────────── -function PerRunView({ - data, - projectId, - readOnly, -}: { - data: CompareResponse; - projectId?: string; - readOnly: boolean; -}) { +function PerRunView({ data }: { data: CompareResponse; projectId?: string; readOnly?: boolean }) { const runs = data.runs ?? []; const [selected, setSelected] = useState>(new Set()); const [showingCompare, setShowingCompare] = useState(false); - const [editingRunId, setEditingRunId] = useState(null); const toggleSelect = (runId: string) => { setSelected((prev) => { @@ -575,7 +369,6 @@ function PerRunView({ Timestamp - Tags Experiment Target Tests @@ -590,11 +383,6 @@ function PerRunView({ run={run} checked={selected.has(run.run_id)} onToggle={() => toggleSelect(run.run_id)} - editing={editingRunId === run.run_id} - onStartEdit={() => setEditingRunId(run.run_id)} - onEndEdit={() => setEditingRunId(null)} - projectId={projectId} - readOnly={readOnly} /> ))} @@ -639,331 +427,48 @@ function PerRunRow({ run, checked, onToggle, - editing, - onStartEdit, - onEndEdit, - projectId, - readOnly, }: { run: CompareRunEntry; checked: boolean; onToggle: () => void; - editing: boolean; - onStartEdit: () => void; - onEndEdit: () => void; - projectId?: string; - readOnly: boolean; }) { const avgPct = Math.round(run.avg_score * 100); const qualityCount = aggregateQualityCount(run); const errors = executionErrorCount(run); - const canEdit = !readOnly; - const tagsBtnRef = useRef(null); - const tags = run.tags ?? []; - const metadataDirty = run.metadata_dirty === true; - const runLabel = tags[0] ?? run.run_id; const subLabel = runSubLabel(run.run_id); - const tagsButtonClass = - tags.length > 0 - ? 'inline-flex flex-wrap items-center gap-1 rounded-md px-1 py-0.5 transition-colors hover:bg-gray-800/60' - : metadataDirty - ? 'rounded-md border border-yellow-900/60 bg-yellow-950/20 px-2 py-0.5 text-xs text-yellow-300 transition-colors hover:border-yellow-700' - : 'rounded-md border border-dashed border-gray-700 px-2 py-0.5 text-xs text-gray-500 transition-colors hover:border-cyan-800 hover:text-cyan-400'; - - // Restore focus to the tags trigger button once the inline editor closes, - // so keyboard users don't lose their place in the table. - const wasEditing = useRef(editing); - useEffect(() => { - if (wasEditing.current && !editing) { - tagsBtnRef.current?.focus(); - } - wasEditing.current = editing; - }, [editing]); return ( - <> - - - - - -
- {formatTimestamp(run.started_at)} -
- {subLabel &&
{subLabel}
} - - - {canEdit ? ( - - ) : tags.length > 0 ? ( - - ) : ( - - {metadataDirty ? 'Pending clear' : '—'} - - )} - - {run.experiment} - {run.target} - -
{qualityCount}
- {errors > 0 &&
{errors} errors
} - - - - - {avgPct}% - - {editing && ( - - - - - - )} - - ); -} - -function TagChips({ tags, dirty }: { tags: string[]; dirty: boolean }) { - return ( - - {tags.map((t) => ( - - {t} - - ))} - {dirty ? ( - - Pending sync - - ) : null} - - ); -} - -/** - * Inline chip-based tag editor. - * - * Local state: a `string[]` staged edit of the run's tags. Chips show the - * current staged tags; an input at the end accepts new tags (commit with - * Enter or comma, delete the last chip with Backspace on an empty input). - * Save persists the whole array; Cancel / Escape discards. - * - * The backend's `writeRunTags` handles deduplication, length limits, and - * control-character rejection, so we only lightly normalize in the UI - * (trim + skip duplicates already in the staged array). - */ -function TagsEditor({ - runId, - currentTags, - tagRevision, - source, - projectId, - onClose, -}: { - runId: string; - currentTags: string[]; - tagRevision?: string; - source: 'local' | 'remote'; - projectId?: string; - onClose: () => void; -}) { - const [tags, setTags] = useState(currentTags); - const [input, setInput] = useState(''); - const [err, setErr] = useState(null); - const qc = useQueryClient(); - const inputRef = useRef(null); - - useEffect(() => { - inputRef.current?.focus(); - }, []); - - const saveMut = useMutation({ - mutationFn: () => saveRunTagsApi(runId, tags, projectId, tagRevision), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ['compare'] }); - qc.invalidateQueries({ queryKey: ['runs'] }); - qc.invalidateQueries({ queryKey: ['remote-status', projectId ?? ''] }); - if (projectId) { - qc.invalidateQueries({ queryKey: ['projects', projectId, 'compare'] }); - qc.invalidateQueries({ queryKey: ['projects', projectId, 'runs'] }); - } - onClose(); - }, - onError: (e: Error) => setErr(e.message), - }); - - const clearMut = useMutation({ - mutationFn: () => deleteRunTagsApi(runId, projectId, tagRevision), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ['compare'] }); - qc.invalidateQueries({ queryKey: ['runs'] }); - qc.invalidateQueries({ queryKey: ['remote-status', projectId ?? ''] }); - if (projectId) { - qc.invalidateQueries({ queryKey: ['projects', projectId, 'compare'] }); - qc.invalidateQueries({ queryKey: ['projects', projectId, 'runs'] }); - } - onClose(); - }, - onError: (e: Error) => setErr(e.message), - }); - - const busy = saveMut.isPending || clearMut.isPending; - const hasChanges = - tags.length !== currentTags.length || tags.some((t, i) => t !== currentTags[i]); - - const commitInput = () => { - const trimmed = input.trim(); - if (trimmed === '') return; - if (tags.includes(trimmed)) { - setInput(''); - return; - } - setTags([...tags, trimmed]); - setInput(''); - setErr(null); - }; - - const removeTag = (tag: string) => { - setTags(tags.filter((t) => t !== tag)); - }; - - return ( -
-
- Tag run - - Multi-valued. Enter or comma adds; Backspace removes the last chip. - -
- {source === 'remote' ? ( -
- Remote tag edits are saved as local metadata. Use Sync Metadata to push them to the - results repo. -
- ) : null} -
- {tags.map((t) => ( - - {t} - - - ))} + + { - setErr(null); - setInput(e.target.value); - }} - maxLength={60} - disabled={busy} - onKeyDown={(e) => { - if (e.key === 'Enter' || e.key === ',') { - e.preventDefault(); - commitInput(); - } else if (e.key === 'Backspace' && input === '' && tags.length > 0) { - e.preventDefault(); - setTags(tags.slice(0, -1)); - } else if (e.key === 'Escape') { - onClose(); - } - }} - onBlur={commitInput} + type="checkbox" + className="h-4 w-4 cursor-pointer rounded border-gray-700 bg-gray-900 text-cyan-500 accent-cyan-500 focus:ring-cyan-500" + checked={checked} + onChange={onToggle} + aria-label={`Select run ${run.run_id}`} /> -
-
- - {currentTags.length > 0 && ( - - )} - -
- {err && ( -
- {err} + + +
+ {formatTimestamp(run.started_at)}
- )} -
+ {subLabel &&
{subLabel}
} + + {run.experiment} + {run.target} + +
{qualityCount}
+ {errors > 0 &&
{errors} errors
} + + + + + {avgPct}% + ); } @@ -1090,24 +595,11 @@ function PerRunCompareView({ } function RunColumnHeader({ run }: { run: CompareRunEntry }) { - const tags = run.tags ?? []; return (
{formatTimestamp(run.started_at)}
- {tags.length > 0 && ( -
- {tags.map((t) => ( - - {t} - - ))} -
- )}
{run.experiment} · {run.target}
@@ -1154,28 +646,11 @@ function EmptyState() { ); } -function Notice({ - headline, - body, - action, -}: { - headline: string; - body: string; - action?: { label: string; onClick: () => void }; -}) { +function Notice({ headline, body }: { headline: string; body: string }) { return (

{headline}

{body}

- {action && ( - - )}
); } diff --git a/apps/dashboard/src/components/Breadcrumbs.tsx b/apps/dashboard/src/components/Breadcrumbs.tsx index f5421a3e6..5d7e84058 100644 --- a/apps/dashboard/src/components/Breadcrumbs.tsx +++ b/apps/dashboard/src/components/Breadcrumbs.tsx @@ -10,11 +10,11 @@ import { Link, useMatches } from '@tanstack/react-router'; import { categoryPath, evalPath, - experimentPath, jobPath, projectHomePath, runPath, suitePath, + tagValuePath, } from '~/lib/navigation'; import { useSidebarContext } from '~/lib/sidebar-context'; @@ -102,10 +102,10 @@ function deriveSegments(matches: ReturnType): BreadcrumbSegme label: params.evalId ?? 'Eval', to: evalPath(params.runId, params.evalId ?? 'Eval', params.projectId), }); - } else if (routeId.includes('/projects/$projectId_/experiments/$experimentName')) { + } else if (routeId.includes('/projects/$projectId_/tags/$key/$value')) { segments.push({ - label: params.experimentName ?? 'Experiment', - to: experimentPath(params.experimentName ?? 'Experiment', params.projectId), + label: params.value ?? 'Tag', + to: tagValuePath(params.key ?? 'experiment', params.value ?? 'Tag', params.projectId), }); } else if (routeId.includes('/runs/$runId/category/$category')) { if (!segments.some((s) => s.label === formatBreadcrumbRunLabel(params.runId))) { @@ -151,10 +151,10 @@ function deriveSegments(matches: ReturnType): BreadcrumbSegme label: params.evalId ?? 'Eval', to: evalPath(params.runId, params.evalId ?? 'Eval'), }); - } else if (routeId.includes('/experiments/$experimentName')) { + } else if (routeId.includes('/tags/$key/$value')) { segments.push({ - label: params.experimentName ?? 'Experiment', - to: experimentPath(params.experimentName ?? 'Experiment'), + label: params.value ?? 'Tag', + to: tagValuePath(params.key ?? 'experiment', params.value ?? 'Tag'), }); } else if (routeId === '/settings') { segments.push({ label: 'Settings', to: '/settings' }); diff --git a/apps/dashboard/src/components/ExperimentsTab.tsx b/apps/dashboard/src/components/ExperimentsTab.tsx deleted file mode 100644 index 75605eaa1..000000000 --- a/apps/dashboard/src/components/ExperimentsTab.tsx +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Experiments table showing experiments grouped across all runs. - * - * Displays experiment name, number of runs, targets, pass rate, and - * last run timestamp. Each row links to the experiment detail page. - * - * The table keeps the desktop column layout on mobile by using the same - * overflow container + fixed minimum width pattern as other Dashboard summary - * tables, so right-side metrics remain reachable instead of being clipped. - */ - -import { useQuery } from '@tanstack/react-query'; -import { Link } from '@tanstack/react-router'; - -import { projectExperimentsOptions, useExperiments } from '~/lib/api'; -import { aggregateQualityCount, executionErrorCount } from '~/lib/result-summary'; -import type { ExperimentSummary } from '~/lib/types'; - -import { PassRatePill } from './PassRatePill'; - -interface ExperimentsTabProps { - projectId?: string; -} - -export function ExperimentsTab({ projectId }: ExperimentsTabProps) { - const { data, isLoading } = projectId - ? useQuery(projectExperimentsOptions(projectId)) - : useExperiments(); - - if (isLoading) { - return ; - } - - const experiments = data?.experiments ?? []; - - if (experiments.length === 0) { - return ( -
-

No experiments found

-

- Experiments will appear here once evaluations are run with experiment labels. -

-
- ); - } - - return ( -
- - - - - - - - - - - - - - {experiments.map((exp: ExperimentSummary) => { - const qualityCount = aggregateQualityCount(exp); - const errors = executionErrorCount(exp); - return ( - - - - - - - - - - ); - })} - -
ExperimentRunsTargetsEvalsExecution ErrorsPass RateLast Run
- {projectId ? ( - - {exp.name} - - ) : ( - - {exp.name} - - )} - {exp.run_count} - {exp.target_count} - - {exp.passed_count} - / - {qualityCount} - - {errors > 0 ? errors : 0} - - - - {formatTimestamp(exp.last_run).date} -
-
- ); -} - -function formatTimestamp(ts: string | undefined | null): { date: string; full: string } { - if (!ts) return { date: 'N/A', full: 'N/A' }; - try { - const d = new Date(ts); - if (Number.isNaN(d.getTime())) return { date: 'N/A', full: 'N/A' }; - const full = d.toLocaleString(); - const diffMs = Date.now() - d.getTime(); - const diffMin = Math.floor(diffMs / 60_000); - const diffHour = Math.floor(diffMs / 3_600_000); - let date: string; - if (diffMin < 1) date = 'just now'; - else if (diffMin < 60) date = `${diffMin} min ago`; - else if (diffHour < 24) date = `${diffHour} hour${diffHour === 1 ? '' : 's'} ago`; - else date = d.toLocaleDateString(); - return { date, full }; - } catch { - return { date: 'N/A', full: 'N/A' }; - } -} - -function LoadingSkeleton() { - return ( -
-
-
-
-
- {['sk-1', 'sk-2', 'sk-3', 'sk-4', 'sk-5'].map((id) => ( -
-
-
-
-
-
-
- ))} -
-
- ); -} diff --git a/apps/dashboard/src/components/RunEvalModal.tsx b/apps/dashboard/src/components/RunEvalModal.tsx index fa5911c83..6d126aeeb 100644 --- a/apps/dashboard/src/components/RunEvalModal.tsx +++ b/apps/dashboard/src/components/RunEvalModal.tsx @@ -63,8 +63,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal const [testIds, setTestIds] = useState(prefill?.testIds ?? []); const [target, setTarget] = useState(prefill?.target ?? ''); const [experiment, setExperiment] = useState(DEFAULT_EXPERIMENT); - const [tagInput, setTagInput] = useState(''); - const [tags, setTags] = useState([]); const [threshold, setThreshold] = useState(''); const [thresholdEdited, setThresholdEdited] = useState(false); const [workers, setWorkers] = useState(''); @@ -107,8 +105,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setTestIds(prefill?.testIds ?? []); setTarget(prefill?.target ?? ''); setExperiment(DEFAULT_EXPERIMENT); - setTagInput(''); - setTags([]); setTestIdInput(''); setThreshold(''); setThresholdEdited(false); @@ -131,28 +127,16 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal // Build request body from form state const buildRequest = useCallback((): RunEvalRequest => { - const effectiveTags = mergeUniqueTags(tags, parseTagsInput(tagInput)); return buildRunEvalRequest({ suiteFilter, testIds, target, experiment, - tags: effectiveTags, thresholdInput: threshold, studioThreshold: studioConfig?.threshold, workers, }); - }, [ - experiment, - studioConfig?.threshold, - suiteFilter, - tagInput, - tags, - target, - testIds, - threshold, - workers, - ]); + }, [experiment, studioConfig?.threshold, suiteFilter, target, testIds, threshold, workers]); // Update CLI preview when form changes useEffect(() => { @@ -179,18 +163,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setTestIds(testIds.filter((t) => t !== id)); } - function addTagsFromInput() { - const nextTags = parseTagsInput(tagInput); - if (nextTags.length > 0) { - setTags(mergeUniqueTags(tags, nextTags)); - } - setTagInput(''); - } - - function removeTag(tag: string) { - setTags(tags.filter((t) => t !== tag)); - } - // Launch async function handleLaunch() { setError(null); @@ -344,77 +316,22 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal {/* Run metadata */}
-
-
- - setExperiment(e.target.value)} - placeholder={DEFAULT_EXPERIMENT} - className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" - /> -
-
- -
- setTagInput(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter' || e.key === ',') { - e.preventDefault(); - addTagsFromInput(); - } - }} - onBlur={addTagsFromInput} - placeholder="baseline, prompt-v2" - className="min-w-0 flex-1 rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" - /> - -
-
+
+ + setExperiment(e.target.value)} + placeholder={DEFAULT_EXPERIMENT} + className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" + />
- {tags.length > 0 && ( -
- {tags.map((tag) => ( - - {tag} - - - ))} -
- )} -

- Saved with the new run only; existing runs stay unchanged. -

{/* Advanced options */} @@ -506,24 +423,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal // ── Sub-components ─────────────────────────────────────────────────────── -function parseTagsInput(value: string): string[] { - return value - .split(',') - .map((tag) => tag.trim()) - .filter((tag) => tag.length > 0); -} - -function mergeUniqueTags(existing: string[], incoming: string[]): string[] { - const seen = new Set(existing); - const merged = [...existing]; - for (const tag of incoming) { - if (seen.has(tag)) continue; - seen.add(tag); - merged.push(tag); - } - return merged; -} - function ModalShell({ children, onClose, diff --git a/apps/dashboard/src/components/RunList.tsx b/apps/dashboard/src/components/RunList.tsx index 1e377133e..b0b10a676 100644 --- a/apps/dashboard/src/components/RunList.tsx +++ b/apps/dashboard/src/components/RunList.tsx @@ -77,7 +77,6 @@ interface RunListItemView { passing: boolean; passedCount: number; failedCount: number; - metadataDirty: boolean; experimentNamespace: string; runtimeSourceLabel?: string; runtimeSourceTitle?: string; @@ -113,7 +112,6 @@ export function buildRunListItemView(run: RunMeta, passThreshold: number): RunLi const passing = qualityCount > 0 ? run.pass_rate >= passThreshold : errors === 0; const passedCount = Math.round(run.pass_rate * qualityCount); const failedCount = qualityCount - passedCount; - const metadataDirty = run.metadata_dirty === true; const experimentNamespace = experimentNamespaceLabel(run); const runtimeSourceLabel = run.runtime_source ? runtimeSourceSummary(run.runtime_source) @@ -133,7 +131,6 @@ export function buildRunListItemView(run: RunMeta, passThreshold: number): RunLi passing, passedCount, failedCount, - metadataDirty, experimentNamespace, runtimeSourceLabel, runtimeSourceTitle: runtimeSourceTooltip, @@ -220,6 +217,7 @@ export function RunList({ await Promise.all([ queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'runs'] }), queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'experiments'] }), + queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'tags'] }), queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'compare'] }), queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'targets'] }), ]); @@ -228,6 +226,7 @@ export function RunList({ await Promise.all([ queryClient.invalidateQueries({ queryKey: ['runs'] }), queryClient.invalidateQueries({ queryKey: ['experiments'] }), + queryClient.invalidateQueries({ queryKey: ['tags'] }), queryClient.invalidateQueries({ queryKey: ['compare'] }), queryClient.invalidateQueries({ queryKey: ['targets'] }), ]); @@ -437,7 +436,6 @@ export function RunList({ qualityCount, passedCount, failedCount, - metadataDirty, experimentNamespace, runtimeSourceLabel, runtimeSourceTitle, @@ -487,7 +485,6 @@ export function RunList({ />
- {metadataDirty ? : null} {ts.date} @@ -556,7 +553,6 @@ export function RunList({ qualityCount, passedCount, failedCount, - metadataDirty, experimentNamespace, runtimeSourceLabel, runtimeSourceTitle, @@ -620,7 +616,6 @@ export function RunList({ title={display.title} className="block min-w-0 truncate font-medium text-cyan-400 hover:text-cyan-300 hover:underline" /> - {metadataDirty ? : null}
@@ -771,17 +766,6 @@ function CloudOutlineIcon() { ); } -function PendingSyncBadge() { - return ( - - Pending sync - - ); -} - function RunSourceBadges({ experimentNamespace, runtimeSourceLabel, diff --git a/apps/dashboard/src/components/Sidebar.tsx b/apps/dashboard/src/components/Sidebar.tsx index 543e442dc..35987e0c5 100644 --- a/apps/dashboard/src/components/Sidebar.tsx +++ b/apps/dashboard/src/components/Sidebar.tsx @@ -6,7 +6,7 @@ * - At run detail: shows nearby runs as local review context * - At eval detail: shows evals in the current run with pass/fail indicators * - At suite/category detail: shows the filtered review context - * - At experiment detail: shows nearby experiments + * - At tag-value detail: shows sibling values for the selected tag key * * Responsive behavior is handled by SidebarShell: * - md+ (≥768px): always-visible fixed left panel @@ -22,10 +22,10 @@ import { DEFAULT_APP_NAME, isPassing, projectCategorySuitesOptions, - projectExperimentsOptions, + projectTagGroupsOptions, + tagGroupsOptions, useCategorySuites, useEvalRuns, - useExperiments, useProjectList, useProjectRunDetail, useProjectRunList, @@ -141,11 +141,11 @@ function useCurrentEvalIdentitySearch() { }; } -type ProjectTabId = 'runs' | 'experiments' | 'analytics' | 'targets'; +type ProjectTabId = 'runs' | 'tags' | 'analytics' | 'targets'; const projectNavItems: { id: ProjectTabId; label: string; description: string }[] = [ { id: 'runs', label: 'Recent Runs', description: 'Run review' }, - { id: 'experiments', label: 'Experiments', description: 'Grouped runs' }, + { id: 'tags', label: 'Tags', description: 'Grouped runs' }, { id: 'analytics', label: 'Analytics', description: 'Compare scores' }, { id: 'targets', label: 'Targets', description: 'Target results' }, ]; @@ -224,8 +224,8 @@ export function Sidebar() { to: '/projects/$projectId/runs/$runId', fuzzy: true, }); - const projectExperimentMatch = matchRoute({ - to: '/projects/$projectId/experiments/$experimentName', + const projectTagValueMatch = matchRoute({ + to: '/projects/$projectId/tags/$key/$value', fuzzy: true, }); const projectCategoryMatch = matchRoute({ @@ -293,15 +293,16 @@ export function Sidebar() { } if ( - projectExperimentMatch && - typeof projectExperimentMatch === 'object' && - 'projectId' in projectExperimentMatch + projectTagValueMatch && + typeof projectTagValueMatch === 'object' && + 'projectId' in projectTagValueMatch ) { - const { projectId, experimentName } = projectExperimentMatch as { + const { projectId, key, value } = projectTagValueMatch as { projectId: string; - experimentName: string; + key: string; + value: string; }; - return ; + return ; } // Project home (runs/experiments/targets) @@ -321,8 +322,8 @@ export function Sidebar() { to: '/runs/$runId/suite/$suite', fuzzy: true, }); - const experimentMatch = matchRoute({ - to: '/experiments/$experimentName', + const tagValueMatch = matchRoute({ + to: '/tags/$key/$value', fuzzy: true, }); @@ -341,13 +342,9 @@ export function Sidebar() { return ; } - if ( - experimentMatch && - typeof experimentMatch === 'object' && - 'experimentName' in experimentMatch - ) { - const { experimentName } = experimentMatch as { experimentName: string }; - return ; + if (tagValueMatch && typeof tagValueMatch === 'object' && 'key' in tagValueMatch) { + const { key, value } = tagValueMatch as { key: string; value: string }; + return ; } if (runMatch && typeof runMatch === 'object' && 'runId' in runMatch) { @@ -801,15 +798,22 @@ function ProjectCategorySidebar({ ); } -function ProjectExperimentSidebar({ +function tagKeyLabel(key: string): string { + if (!key) return 'Tag'; + return key.charAt(0).toUpperCase() + key.slice(1); +} + +function ProjectTagValueSidebar({ projectId, - currentExperiment, + tagKey, + currentValue, }: { projectId: string; - currentExperiment: string; + tagKey: string; + currentValue: string; }) { - const { data } = useQuery(projectExperimentsOptions(projectId)); - const experiments = data?.experiments ?? []; + const { data } = useQuery(projectTagGroupsOptions(projectId, tagKey)); + const groups = data?.groups ?? []; return ( @@ -819,33 +823,33 @@ function ProjectExperimentSidebar({ } + search={{ tab: 'tags', key: tagKey } as Record} className="text-xs text-gray-400 hover:text-cyan-400" > - ← All experiments + ← All {tagKeyLabel(tagKey)} values
+ ); +} + +function formatTimestamp(ts: string | undefined | null): { date: string; full: string } { + if (!ts) return { date: 'N/A', full: 'N/A' }; + try { + const d = new Date(ts); + if (Number.isNaN(d.getTime())) return { date: 'N/A', full: 'N/A' }; + const full = d.toLocaleString(); + const diffMs = Date.now() - d.getTime(); + const diffMin = Math.floor(diffMs / 60_000); + const diffHour = Math.floor(diffMs / 3_600_000); + let date: string; + if (diffMin < 1) date = 'just now'; + else if (diffMin < 60) date = `${diffMin} min ago`; + else if (diffHour < 24) date = `${diffHour} hour${diffHour === 1 ? '' : 's'} ago`; + else date = d.toLocaleDateString(); + return { date, full }; + } catch { + return { date: 'N/A', full: 'N/A' }; + } +} + +function LoadingSkeleton() { + return ( +
+
+
+
+
+ {['sk-1', 'sk-2', 'sk-3', 'sk-4', 'sk-5'].map((id) => ( +
+
+
+
+
+
+
+ ))} +
+
+ ); +} diff --git a/apps/dashboard/src/components/run-eval-threshold.test.ts b/apps/dashboard/src/components/run-eval-threshold.test.ts index 414da34cd..84e52360a 100644 --- a/apps/dashboard/src/components/run-eval-threshold.test.ts +++ b/apps/dashboard/src/components/run-eval-threshold.test.ts @@ -28,7 +28,6 @@ describe('buildRunEvalRequest', () => { testIds: [], target: '', experiment: '', - tags: [], thresholdInput: '', studioThreshold: 0.75, workers: '', @@ -46,7 +45,6 @@ describe('buildRunEvalRequest', () => { testIds: [], target: '', experiment: '', - tags: [], thresholdInput: '0.9', studioThreshold: 0.75, workers: '', @@ -57,14 +55,13 @@ describe('buildRunEvalRequest', () => { }); }); - it('submits launch metadata when experiment and tags are set', () => { + it('submits launch metadata when the experiment is set', () => { expect( buildRunEvalRequest({ suiteFilter: 'evals/**/*.eval.yaml', testIds: [], target: '', experiment: 'smoke', - tags: ['baseline', 'prompt-v2'], thresholdInput: '', studioThreshold: 0.75, workers: '', @@ -72,7 +69,6 @@ describe('buildRunEvalRequest', () => { ).toEqual({ suite_filter: 'evals/**/*.eval.yaml', experiment: 'smoke', - tags: ['baseline', 'prompt-v2'], threshold: 0.75, }); }); diff --git a/apps/dashboard/src/components/run-eval-threshold.ts b/apps/dashboard/src/components/run-eval-threshold.ts index aa3ea60e0..9ca563599 100644 --- a/apps/dashboard/src/components/run-eval-threshold.ts +++ b/apps/dashboard/src/components/run-eval-threshold.ts @@ -6,7 +6,6 @@ interface BuildRunEvalRequestOptions { testIds: string[]; target: string; experiment: string; - tags: string[]; thresholdInput: string; studioThreshold?: number; workers: string; @@ -40,7 +39,6 @@ export function buildRunEvalRequest({ testIds, target, experiment, - tags, thresholdInput, studioThreshold, workers, @@ -51,7 +49,6 @@ export function buildRunEvalRequest({ if (testIds.length > 0) req.test_ids = testIds; if (target) req.target = target; if (experiment.trim()) req.experiment = experiment.trim(); - if (tags.length > 0) req.tags = tags; const resolvedThreshold = getDefaultThresholdInputValue(thresholdInput, studioThreshold); if (resolvedThreshold) req.threshold = Number.parseFloat(resolvedThreshold); diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index 80bc0bb2c..a7bdf2059 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -39,9 +39,10 @@ import type { RunDetailResponse, RunEvalRequest, RunListResponse, - RunTagsResponse, StudioConfigResponse, SuitesResponse, + TagGroupsResponse, + TagKeysResponse, TargetsResponse, TranscriptArtifactResponse, } from './types'; @@ -203,6 +204,43 @@ export const experimentsOptions = queryOptions({ queryFn: () => fetchJson('/api/experiments'), }); +/** Default tag key for the Tags-tab group-by selector. */ +export const DEFAULT_TAG_KEY = 'experiment'; + +/** `GET /api/tags` — available tag keys for the group-by selector. */ +export const tagKeysOptions = queryOptions({ + queryKey: ['tags', 'keys'], + queryFn: () => fetchJson('/api/tags'), +}); + +/** `GET /api/tags?key=` — per-value summaries for the selected tag key. */ +export function tagGroupsOptions(key: string) { + return queryOptions({ + queryKey: ['tags', 'groups', key], + queryFn: () => fetchJson(`/api/tags?key=${encodeURIComponent(key)}`), + enabled: !!key, + }); +} + +export function projectTagKeysOptions(projectId: string) { + return queryOptions({ + queryKey: ['projects', projectId, 'tags', 'keys'], + queryFn: () => fetchJson(`${projectApiBase(projectId)}/tags`), + enabled: !!projectId, + }); +} + +export function projectTagGroupsOptions(projectId: string, key: string) { + return queryOptions({ + queryKey: ['projects', projectId, 'tags', 'groups', key], + queryFn: () => + fetchJson( + `${projectApiBase(projectId)}/tags?key=${encodeURIComponent(key)}`, + ), + enabled: !!projectId && !!key, + }); +} + export function compareOptionsWithBaseline(baseline?: string) { return queryOptions({ queryKey: ['compare', 'baseline', baseline ?? ''], @@ -764,61 +802,6 @@ export async function deleteRunApi(runId: string, projectId?: string): Promise { - const url = projectId - ? `${projectApiBase(projectId)}/runs/${encodeURIComponent(runId)}/tags` - : `/api/runs/${encodeURIComponent(runId)}/tags`; - const res = await fetch(url, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - tags, - ...(expectedTagRevision ? { expected_tag_revision: expectedTagRevision } : {}), - }), - }); - if (!res.ok) { - const err = await res.json().catch(() => ({ error: res.statusText })); - throw new Error((err as { error?: string }).error ?? `Failed to save tags: ${res.status}`); - } - return res.json() as Promise; -} - -/** Clear the tags for a run, rejecting stale browser edits when a revision is provided. */ -export async function deleteRunTagsApi( - runId: string, - projectId?: string, - expectedTagRevision?: string, -): Promise { - const url = projectId - ? `${projectApiBase(projectId)}/runs/${encodeURIComponent(runId)}/tags` - : `/api/runs/${encodeURIComponent(runId)}/tags`; - const res = await fetch(url, { - method: 'DELETE', - ...(expectedTagRevision - ? { - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ expected_tag_revision: expectedTagRevision }), - } - : {}), - }); - if (!res.ok) { - const err = await res.json().catch(() => ({ error: res.statusText })); - throw new Error((err as { error?: string }).error ?? `Failed to delete tags: ${res.status}`); - } -} - export async function saveStudioConfig( config: Partial, ): Promise { diff --git a/apps/dashboard/src/lib/navigation.test.ts b/apps/dashboard/src/lib/navigation.test.ts index 73dbdf4d5..b275cc201 100644 --- a/apps/dashboard/src/lib/navigation.test.ts +++ b/apps/dashboard/src/lib/navigation.test.ts @@ -6,7 +6,6 @@ import { evalResultIdentityKey, evalResultPath, evalResultSearchParams, - experimentPath, initialProjectRedirectStorageKey, jobPath, matchesEvalResultIdentity, @@ -15,6 +14,7 @@ import { runPath, runsHomePath, suitePath, + tagValuePath, } from './navigation'; describe('resolveInitialProjectRedirect', () => { @@ -78,8 +78,11 @@ describe('route path helpers', () => { expect(suitePath('run::1', 'evals/smoke.eval.yaml', 'demo project')).toBe( '/projects/demo%20project/runs/run%3A%3A1/suite/evals%2Fsmoke.eval.yaml', ); - expect(experimentPath('prod-baseline', 'demo project')).toBe( - '/projects/demo%20project/experiments/prod-baseline', + expect(tagValuePath('experiment', 'prod-baseline', 'demo project')).toBe( + '/projects/demo%20project/tags/experiment/prod-baseline', + ); + expect(tagValuePath('team', 'core team', 'demo project')).toBe( + '/projects/demo%20project/tags/team/core%20team', ); expect(runsHomePath('wtg-ai-prompts')).toBe('/projects/wtg-ai-prompts?tab=runs'); }); diff --git a/apps/dashboard/src/lib/navigation.ts b/apps/dashboard/src/lib/navigation.ts index d8bb1488e..fe30a37a7 100644 --- a/apps/dashboard/src/lib/navigation.ts +++ b/apps/dashboard/src/lib/navigation.ts @@ -5,7 +5,7 @@ * breadcrumbs, and regression tests all agree on the canonical URLs. */ -export type StudioTabId = 'runs' | 'experiments' | 'analytics' | 'targets'; +export type StudioTabId = 'runs' | 'tags' | 'analytics' | 'targets'; export interface IndexRouteDecision { kind: 'dashboard' | 'single-project-home' | 'redirect'; @@ -85,10 +85,13 @@ export function matchesEvalResultIdentity( ); } -export function experimentPath(experimentName: string, projectId?: string): string { - return projectId - ? `/projects/${encodeURIComponent(projectId)}/experiments/${encodeURIComponent(experimentName)}` - : `/experiments/${encodeURIComponent(experimentName)}`; +/** + * Path to a tag-value detail view (`/tags//`), the generalization + * of the old experiment detail route. Group-by key defaults to `experiment`. + */ +export function tagValuePath(key: string, value: string, projectId?: string): string { + const suffix = `tags/${encodeURIComponent(key)}/${encodeURIComponent(value)}`; + return projectId ? `/projects/${encodeURIComponent(projectId)}/${suffix}` : `/${suffix}`; } export function jobPath(runId: string, projectId?: string): string { @@ -113,8 +116,8 @@ export function runsHomePath(projectId?: string): string { return projectId ? projectHomePath(projectId, 'runs') : '/?tab=runs'; } -export function experimentsHomePath(projectId?: string): string { - return projectId ? projectHomePath(projectId, 'experiments') : '/?tab=experiments'; +export function tagsHomePath(projectId?: string): string { + return projectId ? projectHomePath(projectId, 'tags') : '/?tab=tags'; } export function resolveInitialProjectRedirect( diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index fa094f83c..32571a674 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -49,18 +49,6 @@ export interface RunMeta { */ on_remote?: boolean; project_id?: string; - /** Optional user-assigned tags from the run's sidecar tags.json. */ - tags?: string[]; - /** Tags currently present in the remote results repo before local metadata overlays. */ - remote_tags?: string[]; - /** Locally edited tags waiting to sync back to the remote results repo. */ - pending_tags?: string[]; - /** True when local editable metadata differs from the fetched remote metadata. */ - metadata_dirty?: boolean; - /** Materialized final run state from the result bundle and tag sidecar. */ - final_state?: RunFinalState; - /** Optimistic-concurrency token for mutable tag state. */ - tag_revision?: string; /** * Live execution status. Only present for Dashboard-launched runs that are * still being tracked in-memory — used to render a spinner in RunList @@ -70,11 +58,6 @@ export interface RunMeta { status?: 'starting' | 'running' | 'finished' | 'failed'; } -export interface RunFinalState { - lifecycle: 'active' | 'hidden' | 'deleted'; - tags: string[]; -} - export interface RunListResponse { runs: RunMeta[]; next_cursor?: string; @@ -274,8 +257,6 @@ export interface RunDetailResponse { source: 'local' | 'remote'; source_label?: string; runtime_source?: RunRuntimeSource; - final_state?: RunFinalState; - tag_revision?: string; /** Live execution status when this run is still tracked in-memory by Dashboard. */ status?: 'starting' | 'running' | 'finished' | 'failed'; /** Path to the run workspace directory (relative to cwd when inside, otherwise absolute). Local runs only. */ @@ -376,6 +357,34 @@ export interface ExperimentsResponse { experiments: ExperimentSummary[]; } +/** + * Per-value summary for a selected tag key, as returned by + * `GET /api/tags?key=`. `name` is the tag value (e.g. a team, env, or + * experiment name); the remaining fields mirror {@link ExperimentSummary}. + */ +export interface TagGroupSummary { + name: string; + run_count: number; + target_count: number; + eval_count: number; + quality_count?: number; + passed_count: number; + execution_error_count?: number; + pass_rate: number; + last_run: string | null; +} + +/** `GET /api/tags` — the available tag keys for the group-by selector. */ +export interface TagKeysResponse { + keys: string[]; +} + +/** `GET /api/tags?key=` — per-value summaries for the selected tag key. */ +export interface TagGroupsResponse { + key: string; + groups: TagGroupSummary[]; +} + export interface CompareTestResult { test_id: string; /** Optional per-test category from the source eval result, when available. */ @@ -414,12 +423,6 @@ export interface CompareRunEntry { started_at: string; experiment: string; target: string; - tags?: string[]; - remote_tags?: string[]; - pending_tags?: string[]; - metadata_dirty?: boolean; - final_state?: RunFinalState; - tag_revision?: string; source: 'local' | 'remote'; eval_count: number; quality_count?: number; @@ -438,16 +441,6 @@ export interface CompareResponse { runs?: CompareRunEntry[]; } -export interface RunTagsResponse { - tags: string[]; - remote_tags?: string[]; - pending_tags?: string[]; - metadata_dirty?: boolean; - final_state?: RunFinalState; - tag_revision: string; - updated_at: string; -} - export interface CombineDuplicateConflict { key: string; test_id: string; @@ -466,7 +459,6 @@ export interface CombineRunsResponse { experiment: string; combined_from_run_ids: string[]; duplicate_conflicts?: CombineDuplicateConflict[]; - tags?: string[]; } export interface TargetSummary { @@ -676,7 +668,6 @@ export interface RunEvalRequest { test_ids?: string[]; target?: string; experiment?: string; - tags?: string[]; threshold?: number; workers?: number; /** Resume an interrupted run: skip already-completed tests and append to `output`. */ diff --git a/apps/dashboard/src/routeTree.gen.ts b/apps/dashboard/src/routeTree.gen.ts index e77db636c..7f148ee70 100644 --- a/apps/dashboard/src/routeTree.gen.ts +++ b/apps/dashboard/src/routeTree.gen.ts @@ -15,12 +15,14 @@ import { Route as RunsRunIdRouteImport } from './routes/runs/$runId' import { Route as ProjectsProjectIdRouteImport } from './routes/projects/$projectId' import { Route as JobsRunIdRouteImport } from './routes/jobs/$runId' import { Route as ExperimentsExperimentNameRouteImport } from './routes/experiments/$experimentName' +import { Route as TagsKeyValueRouteImport } from './routes/tags/$key.$value' import { Route as EvalsRunIdEvalIdRouteImport } from './routes/evals/$runId.$evalId' import { Route as RunsRunIdSuiteSuiteRouteImport } from './routes/runs/$runId_.suite.$suite' import { Route as RunsRunIdCategoryCategoryRouteImport } from './routes/runs/$runId_.category.$category' import { Route as ProjectsProjectIdRunsRunIdRouteImport } from './routes/projects/$projectId_/runs/$runId' import { Route as ProjectsProjectIdJobsRunIdRouteImport } from './routes/projects/$projectId_/jobs/$runId' import { Route as ProjectsProjectIdExperimentsExperimentNameRouteImport } from './routes/projects/$projectId_/experiments/$experimentName' +import { Route as ProjectsProjectIdTagsKeyValueRouteImport } from './routes/projects/$projectId_/tags/$key.$value' import { Route as ProjectsProjectIdEvalsRunIdEvalIdRouteImport } from './routes/projects/$projectId_/evals/$runId.$evalId' import { Route as ProjectsProjectIdRunsRunIdSuiteSuiteRouteImport } from './routes/projects/$projectId_/runs/$runId_.suite.$suite' import { Route as ProjectsProjectIdRunsRunIdCategoryCategoryRouteImport } from './routes/projects/$projectId_/runs/$runId_.category.$category' @@ -56,6 +58,11 @@ const ExperimentsExperimentNameRoute = path: '/experiments/$experimentName', getParentRoute: () => rootRouteImport, } as any) +const TagsKeyValueRoute = TagsKeyValueRouteImport.update({ + id: '/tags/$key/$value', + path: '/tags/$key/$value', + getParentRoute: () => rootRouteImport, +} as any) const EvalsRunIdEvalIdRoute = EvalsRunIdEvalIdRouteImport.update({ id: '/evals/$runId/$evalId', path: '/evals/$runId/$evalId', @@ -90,6 +97,12 @@ const ProjectsProjectIdExperimentsExperimentNameRoute = path: '/projects/$projectId/experiments/$experimentName', getParentRoute: () => rootRouteImport, } as any) +const ProjectsProjectIdTagsKeyValueRoute = + ProjectsProjectIdTagsKeyValueRouteImport.update({ + id: '/projects/$projectId_/tags/$key/$value', + path: '/projects/$projectId/tags/$key/$value', + getParentRoute: () => rootRouteImport, + } as any) const ProjectsProjectIdEvalsRunIdEvalIdRoute = ProjectsProjectIdEvalsRunIdEvalIdRouteImport.update({ id: '/projects/$projectId_/evals/$runId/$evalId', @@ -117,12 +130,14 @@ export interface FileRoutesByFullPath { '/projects/$projectId': typeof ProjectsProjectIdRoute '/runs/$runId': typeof RunsRunIdRoute '/evals/$runId/$evalId': typeof EvalsRunIdEvalIdRoute + '/tags/$key/$value': typeof TagsKeyValueRoute '/projects/$projectId/experiments/$experimentName': typeof ProjectsProjectIdExperimentsExperimentNameRoute '/projects/$projectId/jobs/$runId': typeof ProjectsProjectIdJobsRunIdRoute '/projects/$projectId/runs/$runId': typeof ProjectsProjectIdRunsRunIdRoute '/runs/$runId/category/$category': typeof RunsRunIdCategoryCategoryRoute '/runs/$runId/suite/$suite': typeof RunsRunIdSuiteSuiteRoute '/projects/$projectId/evals/$runId/$evalId': typeof ProjectsProjectIdEvalsRunIdEvalIdRoute + '/projects/$projectId/tags/$key/$value': typeof ProjectsProjectIdTagsKeyValueRoute '/projects/$projectId/runs/$runId/category/$category': typeof ProjectsProjectIdRunsRunIdCategoryCategoryRoute '/projects/$projectId/runs/$runId/suite/$suite': typeof ProjectsProjectIdRunsRunIdSuiteSuiteRoute } @@ -134,12 +149,14 @@ export interface FileRoutesByTo { '/projects/$projectId': typeof ProjectsProjectIdRoute '/runs/$runId': typeof RunsRunIdRoute '/evals/$runId/$evalId': typeof EvalsRunIdEvalIdRoute + '/tags/$key/$value': typeof TagsKeyValueRoute '/projects/$projectId/experiments/$experimentName': typeof ProjectsProjectIdExperimentsExperimentNameRoute '/projects/$projectId/jobs/$runId': typeof ProjectsProjectIdJobsRunIdRoute '/projects/$projectId/runs/$runId': typeof ProjectsProjectIdRunsRunIdRoute '/runs/$runId/category/$category': typeof RunsRunIdCategoryCategoryRoute '/runs/$runId/suite/$suite': typeof RunsRunIdSuiteSuiteRoute '/projects/$projectId/evals/$runId/$evalId': typeof ProjectsProjectIdEvalsRunIdEvalIdRoute + '/projects/$projectId/tags/$key/$value': typeof ProjectsProjectIdTagsKeyValueRoute '/projects/$projectId/runs/$runId/category/$category': typeof ProjectsProjectIdRunsRunIdCategoryCategoryRoute '/projects/$projectId/runs/$runId/suite/$suite': typeof ProjectsProjectIdRunsRunIdSuiteSuiteRoute } @@ -152,12 +169,14 @@ export interface FileRoutesById { '/projects/$projectId': typeof ProjectsProjectIdRoute '/runs/$runId': typeof RunsRunIdRoute '/evals/$runId/$evalId': typeof EvalsRunIdEvalIdRoute + '/tags/$key/$value': typeof TagsKeyValueRoute '/projects/$projectId_/experiments/$experimentName': typeof ProjectsProjectIdExperimentsExperimentNameRoute '/projects/$projectId_/jobs/$runId': typeof ProjectsProjectIdJobsRunIdRoute '/projects/$projectId_/runs/$runId': typeof ProjectsProjectIdRunsRunIdRoute '/runs/$runId_/category/$category': typeof RunsRunIdCategoryCategoryRoute '/runs/$runId_/suite/$suite': typeof RunsRunIdSuiteSuiteRoute '/projects/$projectId_/evals/$runId/$evalId': typeof ProjectsProjectIdEvalsRunIdEvalIdRoute + '/projects/$projectId_/tags/$key/$value': typeof ProjectsProjectIdTagsKeyValueRoute '/projects/$projectId_/runs/$runId_/category/$category': typeof ProjectsProjectIdRunsRunIdCategoryCategoryRoute '/projects/$projectId_/runs/$runId_/suite/$suite': typeof ProjectsProjectIdRunsRunIdSuiteSuiteRoute } @@ -171,12 +190,14 @@ export interface FileRouteTypes { | '/projects/$projectId' | '/runs/$runId' | '/evals/$runId/$evalId' + | '/tags/$key/$value' | '/projects/$projectId/experiments/$experimentName' | '/projects/$projectId/jobs/$runId' | '/projects/$projectId/runs/$runId' | '/runs/$runId/category/$category' | '/runs/$runId/suite/$suite' | '/projects/$projectId/evals/$runId/$evalId' + | '/projects/$projectId/tags/$key/$value' | '/projects/$projectId/runs/$runId/category/$category' | '/projects/$projectId/runs/$runId/suite/$suite' fileRoutesByTo: FileRoutesByTo @@ -188,12 +209,14 @@ export interface FileRouteTypes { | '/projects/$projectId' | '/runs/$runId' | '/evals/$runId/$evalId' + | '/tags/$key/$value' | '/projects/$projectId/experiments/$experimentName' | '/projects/$projectId/jobs/$runId' | '/projects/$projectId/runs/$runId' | '/runs/$runId/category/$category' | '/runs/$runId/suite/$suite' | '/projects/$projectId/evals/$runId/$evalId' + | '/projects/$projectId/tags/$key/$value' | '/projects/$projectId/runs/$runId/category/$category' | '/projects/$projectId/runs/$runId/suite/$suite' id: @@ -205,12 +228,14 @@ export interface FileRouteTypes { | '/projects/$projectId' | '/runs/$runId' | '/evals/$runId/$evalId' + | '/tags/$key/$value' | '/projects/$projectId_/experiments/$experimentName' | '/projects/$projectId_/jobs/$runId' | '/projects/$projectId_/runs/$runId' | '/runs/$runId_/category/$category' | '/runs/$runId_/suite/$suite' | '/projects/$projectId_/evals/$runId/$evalId' + | '/projects/$projectId_/tags/$key/$value' | '/projects/$projectId_/runs/$runId_/category/$category' | '/projects/$projectId_/runs/$runId_/suite/$suite' fileRoutesById: FileRoutesById @@ -223,12 +248,14 @@ export interface RootRouteChildren { ProjectsProjectIdRoute: typeof ProjectsProjectIdRoute RunsRunIdRoute: typeof RunsRunIdRoute EvalsRunIdEvalIdRoute: typeof EvalsRunIdEvalIdRoute + TagsKeyValueRoute: typeof TagsKeyValueRoute ProjectsProjectIdExperimentsExperimentNameRoute: typeof ProjectsProjectIdExperimentsExperimentNameRoute ProjectsProjectIdJobsRunIdRoute: typeof ProjectsProjectIdJobsRunIdRoute ProjectsProjectIdRunsRunIdRoute: typeof ProjectsProjectIdRunsRunIdRoute RunsRunIdCategoryCategoryRoute: typeof RunsRunIdCategoryCategoryRoute RunsRunIdSuiteSuiteRoute: typeof RunsRunIdSuiteSuiteRoute ProjectsProjectIdEvalsRunIdEvalIdRoute: typeof ProjectsProjectIdEvalsRunIdEvalIdRoute + ProjectsProjectIdTagsKeyValueRoute: typeof ProjectsProjectIdTagsKeyValueRoute ProjectsProjectIdRunsRunIdCategoryCategoryRoute: typeof ProjectsProjectIdRunsRunIdCategoryCategoryRoute ProjectsProjectIdRunsRunIdSuiteSuiteRoute: typeof ProjectsProjectIdRunsRunIdSuiteSuiteRoute } @@ -277,6 +304,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ExperimentsExperimentNameRouteImport parentRoute: typeof rootRouteImport } + '/tags/$key/$value': { + id: '/tags/$key/$value' + path: '/tags/$key/$value' + fullPath: '/tags/$key/$value' + preLoaderRoute: typeof TagsKeyValueRouteImport + parentRoute: typeof rootRouteImport + } '/evals/$runId/$evalId': { id: '/evals/$runId/$evalId' path: '/evals/$runId/$evalId' @@ -319,6 +353,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ProjectsProjectIdExperimentsExperimentNameRouteImport parentRoute: typeof rootRouteImport } + '/projects/$projectId_/tags/$key/$value': { + id: '/projects/$projectId_/tags/$key/$value' + path: '/projects/$projectId/tags/$key/$value' + fullPath: '/projects/$projectId/tags/$key/$value' + preLoaderRoute: typeof ProjectsProjectIdTagsKeyValueRouteImport + parentRoute: typeof rootRouteImport + } '/projects/$projectId_/evals/$runId/$evalId': { id: '/projects/$projectId_/evals/$runId/$evalId' path: '/projects/$projectId/evals/$runId/$evalId' @@ -351,6 +392,7 @@ const rootRouteChildren: RootRouteChildren = { ProjectsProjectIdRoute: ProjectsProjectIdRoute, RunsRunIdRoute: RunsRunIdRoute, EvalsRunIdEvalIdRoute: EvalsRunIdEvalIdRoute, + TagsKeyValueRoute: TagsKeyValueRoute, ProjectsProjectIdExperimentsExperimentNameRoute: ProjectsProjectIdExperimentsExperimentNameRoute, ProjectsProjectIdJobsRunIdRoute: ProjectsProjectIdJobsRunIdRoute, @@ -359,6 +401,7 @@ const rootRouteChildren: RootRouteChildren = { RunsRunIdSuiteSuiteRoute: RunsRunIdSuiteSuiteRoute, ProjectsProjectIdEvalsRunIdEvalIdRoute: ProjectsProjectIdEvalsRunIdEvalIdRoute, + ProjectsProjectIdTagsKeyValueRoute: ProjectsProjectIdTagsKeyValueRoute, ProjectsProjectIdRunsRunIdCategoryCategoryRoute: ProjectsProjectIdRunsRunIdCategoryCategoryRoute, ProjectsProjectIdRunsRunIdSuiteSuiteRoute: diff --git a/apps/dashboard/src/routes/experiments/$experimentName.tsx b/apps/dashboard/src/routes/experiments/$experimentName.tsx index b2be36c1a..02b9a6157 100644 --- a/apps/dashboard/src/routes/experiments/$experimentName.tsx +++ b/apps/dashboard/src/routes/experiments/$experimentName.tsx @@ -1,16 +1,16 @@ /** - * Experiment detail route for single-project mode. + * Legacy experiment detail route. Kept for one release so bookmarked + * `/experiments/` links survive; redirects to the generalized + * `/tags/experiment/` view. */ -import { createFileRoute } from '@tanstack/react-router'; - -import { ExperimentDetail } from '~/components/ExperimentDetail'; +import { createFileRoute, redirect } from '@tanstack/react-router'; export const Route = createFileRoute('/experiments/$experimentName')({ - component: ExperimentDetailPage, + beforeLoad: ({ params }) => { + throw redirect({ + to: '/tags/$key/$value', + params: { key: 'experiment', value: params.experimentName }, + }); + }, }); - -function ExperimentDetailPage() { - const { experimentName } = Route.useParams(); - return ; -} diff --git a/apps/dashboard/src/routes/index.tsx b/apps/dashboard/src/routes/index.tsx index 53b8939b6..509a3f883 100644 --- a/apps/dashboard/src/routes/index.tsx +++ b/apps/dashboard/src/routes/index.tsx @@ -12,13 +12,14 @@ import { useEffect, useState } from 'react'; import { useQueryClient } from '@tanstack/react-query'; import { AddProjectModal } from '~/components/AddProjectModal'; import { AnalyticsTab } from '~/components/AnalyticsTab'; -import { ExperimentsTab } from '~/components/ExperimentsTab'; import { ProjectCard } from '~/components/ProjectCard'; import { RunEvalModal } from '~/components/RunEvalModal'; import { RunList } from '~/components/RunList'; import { RunSourceToolbar } from '~/components/RunSourceToolbar'; +import { TagsTab } from '~/components/TagsTab'; import { TargetsTab } from '~/components/TargetsTab'; import { + DEFAULT_TAG_KEY, confirmRemoteResultsMergeApi, remoteStatusOptions, removeProjectApi, @@ -47,7 +48,7 @@ type TabId = StudioTabId; const tabs: { id: TabId; label: string }[] = [ { id: 'runs', label: '🏃 Recent Runs' }, - { id: 'experiments', label: '🧪 Experiments' }, + { id: 'tags', label: '🏷️ Tags' }, { id: 'analytics', label: '📊 Analytics' }, { id: 'targets', label: '🤖 Targets' }, ]; @@ -188,7 +189,7 @@ function ProjectsDashboard() {

No projects registered yet.

- Add a project path to start browsing runs, experiments, analytics, and targets. + Add a project path to start browsing runs, tags, analytics, and targets.

) : ( @@ -225,6 +226,7 @@ function SingleProjectHome() { const routerState = useRouterState(); const searchParams = routerState.location.search as Record; const tab = searchParams.tab as TabId | undefined; + const tagKey = searchParams.key?.trim() ? searchParams.key : DEFAULT_TAG_KEY; const navigate = useNavigate(); const queryClient = useQueryClient(); const { data, isLoading, error, hasNextPage, fetchNextPage, isFetchingNextPage } = @@ -258,6 +260,7 @@ function SingleProjectHome() { void Promise.all([ queryClient.invalidateQueries({ queryKey: ['runs'] }), queryClient.invalidateQueries({ queryKey: ['experiments'] }), + queryClient.invalidateQueries({ queryKey: ['tags'] }), queryClient.invalidateQueries({ queryKey: ['compare'] }), queryClient.invalidateQueries({ queryKey: ['targets'] }), queryClient.invalidateQueries({ queryKey: ['remote-status', ''] }), @@ -291,6 +294,7 @@ function SingleProjectHome() { void Promise.all([ queryClient.invalidateQueries({ queryKey: ['runs'] }), queryClient.invalidateQueries({ queryKey: ['experiments'] }), + queryClient.invalidateQueries({ queryKey: ['tags'] }), queryClient.invalidateQueries({ queryKey: ['compare'] }), queryClient.invalidateQueries({ queryKey: ['targets'] }), queryClient.invalidateQueries({ queryKey: ['remote-status', ''] }), @@ -332,7 +336,15 @@ function SingleProjectHome() {