diff --git a/mcp/benchmark/src/api.ts b/mcp/benchmark/src/api.ts index 06114fa28..c951855aa 100644 --- a/mcp/benchmark/src/api.ts +++ b/mcp/benchmark/src/api.ts @@ -1,4 +1,4 @@ -import type { ProductionRun, Annotation, AnnotationMarker } from "./types"; +import type { ProductionRun, SearchResult, Annotation, AnnotationMarker } from "./types"; const BASE = "/api"; @@ -18,6 +18,7 @@ export const api = { sessions: { list: () => req("/sessions"), get: (id: string) => req(`/sessions/${id}`), + search: (q: string) => req(`/sessions/search?q=${encodeURIComponent(q)}`), annotate: ( id: string, body: { target: "session" | "tool_call"; target_id?: string; marker: AnnotationMarker; note?: string; author?: string }, diff --git a/mcp/benchmark/src/components/Sessions.tsx b/mcp/benchmark/src/components/Sessions.tsx index 79ad4a10f..b6abd2160 100644 --- a/mcp/benchmark/src/components/Sessions.tsx +++ b/mcp/benchmark/src/components/Sessions.tsx @@ -11,6 +11,7 @@ export function Sessions() { runs={state.runs} filteredRuns={state.filteredRuns} selected={state.selected} + quickSearch={state.quickSearch} repoSearch={state.repoSearch} sourceFilter={state.sourceFilter} rangeFilter={state.rangeFilter} @@ -19,11 +20,14 @@ export function Sessions() { sourceOptions={state.sourceOptions} load={state.load} loadDetail={state.loadDetail} + setQuickSearch={state.setQuickSearch} setRepoSearch={state.setRepoSearch} setSourceFilter={state.setSourceFilter} setRangeFilter={state.setRangeFilter} setDayFilter={state.setDayFilter} clearFilters={state.clearFilters} + searching={state.searching} + searchMatchMap={state.searchMatchMap} /> void; loadDetail: (run: ProductionRun) => void; + setQuickSearch: (v: string) => void; setRepoSearch: (v: string) => void; setSourceFilter: (v: string) => void; setRangeFilter: (v: "24h" | "7d" | "30d" | "all") => void; setDayFilter: (v: string) => void; clearFilters: () => void; + searching: boolean; + searchMatchMap: Map; } export function SessionSidebar({ @@ -30,6 +34,7 @@ export function SessionSidebar({ runs, filteredRuns, selected, + quickSearch, repoSearch, sourceFilter, rangeFilter, @@ -38,11 +43,14 @@ export function SessionSidebar({ sourceOptions, load, loadDetail, + setQuickSearch, setRepoSearch, setSourceFilter, setRangeFilter, setDayFilter, clearFilters, + searching, + searchMatchMap, }: SessionSidebarProps) { const navigate = useNavigate(); const location = useLocation(); @@ -150,6 +158,17 @@ export function SessionSidebar({ gap: "6px", }} > + setQuickSearch(e.target.value)} + style={inputStyle} + /> + {searching && ( +

+ Searching traces\u2026 +

+ )} ))} - {(repoSearch || + {(quickSearch || + repoSearch || sourceFilter !== "all" || rangeFilter !== "all" || dayFilter) && ( @@ -318,6 +338,25 @@ export function SessionSidebar({ {run.user_prompt_preview}

)} + {searchMatchMap.has(run.id) && (() => { + const m = searchMatchMap.get(run.id)!; + return ( +

+ {m.first_match_at_call === 0 + ? "match in prompt" + : `match at call #${m.first_match_at_call}/${m.total_tool_calls}`} +

+ ); + })()} )) )} diff --git a/mcp/benchmark/src/hooks/useSessionsState.ts b/mcp/benchmark/src/hooks/useSessionsState.ts index c75da3040..b23b70073 100644 --- a/mcp/benchmark/src/hooks/useSessionsState.ts +++ b/mcp/benchmark/src/hooks/useSessionsState.ts @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback, useMemo } from "react"; +import { useState, useEffect, useCallback, useMemo, useRef } from "react"; import { useSearchParams } from "react-router-dom"; import { toast } from "sonner"; import { api } from "../api"; @@ -10,7 +10,7 @@ import { import { parseTrace } from "../trace/parse"; import { analyzeTrace } from "../trace/analyze"; import type { ParsedTrace, TraceAnalysis, IssueKind } from "../trace/types"; -import type { ProductionRun, TokenUsage } from "../types"; +import type { ProductionRun, TokenUsage, SearchResult, SearchMatchInfo } from "../types"; import type { Annotation, AnnotationMarker } from "../components/Annotations"; export interface SessionsState { @@ -19,6 +19,7 @@ export interface SessionsState { filteredRuns: ProductionRun[]; selected: ProductionRun | null; annotations: Annotation[]; + quickSearch: string; repoSearch: string; sourceFilter: string; rangeFilter: "24h" | "7d" | "30d" | "all"; @@ -42,6 +43,9 @@ export interface SessionsState { toolCallId?: string, ) => void; handleTurnToggle: (turnId: string) => void; + searching: boolean; + searchMatchMap: Map; + setQuickSearch: (v: string) => void; setRepoSearch: (v: string) => void; setSourceFilter: (v: string) => void; setRangeFilter: (v: "24h" | "7d" | "30d" | "all") => void; @@ -67,6 +71,12 @@ export function useSessionsState(): SessionsState { const [showSessionAnnotationForm, setShowSessionAnnotationForm] = useState(false); const [loading, setLoading] = useState(true); + const [searching, setSearching] = useState(false); + const [searchResults, setSearchResults] = useState(null); + const searchTimerRef = useRef | null>(null); + const [quickSearch, setQuickSearch] = useState( + searchParams.get("q") || "", + ); const [repoSearch, setRepoSearch] = useState( searchParams.get("repo") || "", ); @@ -166,6 +176,7 @@ export function useSessionsState(): SessionsState { ); useEffect(() => { + setQuickSearch(searchParams.get("q") || ""); setRepoSearch(searchParams.get("repo") || ""); setSourceFilter(searchParams.get("source") || "all"); setRangeFilter( @@ -176,12 +187,37 @@ export function useSessionsState(): SessionsState { useEffect(() => { const nextParams = new URLSearchParams(); + if (quickSearch) nextParams.set("q", quickSearch); if (repoSearch) nextParams.set("repo", repoSearch); if (sourceFilter !== "all") nextParams.set("source", sourceFilter); if (rangeFilter !== "all") nextParams.set("range", rangeFilter); if (dayFilter) nextParams.set("day", dayFilter); setSearchParams(nextParams, { replace: true }); - }, [dayFilter, repoSearch, rangeFilter, setSearchParams, sourceFilter]); + }, [dayFilter, quickSearch, repoSearch, rangeFilter, setSearchParams, sourceFilter]); + + useEffect(() => { + if (searchTimerRef.current) clearTimeout(searchTimerRef.current); + if (!quickSearch || quickSearch.trim().length < 2) { + setSearchResults(null); + setSearching(false); + return; + } + setSearching(true); + searchTimerRef.current = setTimeout(async () => { + try { + const results = await api.sessions.search(quickSearch.trim()); + setSearchResults(results); + } catch (e: any) { + toast.error(e.message); + setSearchResults([]); + } finally { + setSearching(false); + } + }, 300); + return () => { + if (searchTimerRef.current) clearTimeout(searchTimerRef.current); + }; + }, [quickSearch]); useEffect(() => { if (parsed.turns.length === 0) { @@ -195,7 +231,7 @@ export function useSessionsState(): SessionsState { const filteredRuns = useMemo( () => - runs.filter((r) => { + (searchResults ?? runs).filter((r) => { if (sourceFilter !== "all" && (r.source || "unknown") !== sourceFilter) return false; if ( @@ -213,10 +249,16 @@ export function useSessionsState(): SessionsState { return false; return true; }), - [runs, dayFilter, repoSearch, rangeFilter, sourceFilter], + [runs, searchResults, dayFilter, repoSearch, rangeFilter, sourceFilter], ); + const searchMatchMap = useMemo(() => { + if (!searchResults) return new Map(); + return new Map(searchResults.map((r) => [r.id, r.search_match])); + }, [searchResults]); + const clearFilters = () => { + setQuickSearch(""); setRepoSearch(""); setSourceFilter("all"); setRangeFilter("all"); @@ -229,6 +271,7 @@ export function useSessionsState(): SessionsState { filteredRuns, selected, annotations, + quickSearch, repoSearch, sourceFilter, rangeFilter, @@ -243,11 +286,14 @@ export function useSessionsState(): SessionsState { selectedUsage, prompt, answer, + searching, + searchMatchMap, showSessionAnnotationForm, load, loadDetail, handleAnnotate, handleTurnToggle, + setQuickSearch, setRepoSearch, setSourceFilter, setRangeFilter, diff --git a/mcp/benchmark/src/types.ts b/mcp/benchmark/src/types.ts index a94295c83..e9cbe7fec 100644 --- a/mcp/benchmark/src/types.ts +++ b/mcp/benchmark/src/types.ts @@ -79,3 +79,13 @@ export interface ProductionRun { annotations?: Annotation[]; trace?: unknown; } + +export interface SearchMatchInfo { + first_match_at_call: number; + total_tool_calls: number; + match_context: string; +} + +export interface SearchResult extends ProductionRun { + search_match: SearchMatchInfo; +} diff --git a/mcp/src/benchmark/router.ts b/mcp/src/benchmark/router.ts index b9db2ee53..438e45b8a 100644 --- a/mcp/src/benchmark/router.ts +++ b/mcp/src/benchmark/router.ts @@ -1,11 +1,12 @@ import { Router } from "express"; -import { list_sessions, get_session, session_stats, add_annotation } from "./sessions.js"; +import { list_sessions, get_session, session_stats, add_annotation, search_sessions } from "./sessions.js"; export function benchmarkRouter(): Router { const router = Router(); // Sessions router.get("/sessions", list_sessions); + router.get("/sessions/search", search_sessions); router.get("/sessions/stats", session_stats); router.get("/sessions/:id", get_session); router.post("/sessions/:id/annotations", add_annotation); diff --git a/mcp/src/benchmark/sessions.ts b/mcp/src/benchmark/sessions.ts index 1f00e9b70..aadb954e1 100644 --- a/mcp/src/benchmark/sessions.ts +++ b/mcp/src/benchmark/sessions.ts @@ -134,6 +134,104 @@ function parseSessionMessages(filePath: string): { }; } +interface SearchMatch { + first_match_at_call: number; + total_tool_calls: number; + match_context: string; +} + +function searchSessionContent(filePath: string, query: string): SearchMatch | null { + const q = query.toLowerCase(); + let toolCallCount = 0; + let firstMatch: { at_call: number; context: string } | null = null; + + try { + const content = readFileSync(filePath, "utf-8"); + const lines = content.split("\n").filter((l) => l.trim()); + for (const line of lines) { + try { + const msg = JSON.parse(line) as { + role?: string; + content?: unknown; + }; + const role = msg.role ?? ""; + const msgContent = msg.content; + + const searchTexts: string[] = []; + + if (role === "user" || role === "system") { + searchTexts.push(getText(msgContent)); + } + + if (role === "assistant") { + searchTexts.push(getText(msgContent)); + if (Array.isArray(msgContent)) { + for (const item of msgContent) { + if (!item || typeof item !== "object") continue; + const e = item as Record; + if (e.type === "tool-call") { + toolCallCount++; + searchTexts.push(String(e.toolName ?? "")); + try { + searchTexts.push(JSON.stringify(e.input ?? "")); + } catch { /* skip */ } + } + if (e.type === "tool-result") { + searchTexts.push(String(e.toolName ?? "")); + try { + searchTexts.push(JSON.stringify(e.output ?? e.result ?? e.content ?? "")); + } catch { /* skip */ } + } + } + } + } + + if (role === "tool") { + searchTexts.push(getText(msgContent)); + if (Array.isArray(msgContent)) { + for (const item of msgContent) { + if (!item || typeof item !== "object") continue; + const e = item as Record; + if (e.type === "tool-result") { + try { + searchTexts.push(JSON.stringify(e.output ?? e.result ?? e.value ?? e.content ?? "")); + } catch { /* skip */ } + } + } + } + } + + if (!firstMatch) { + for (const text of searchTexts) { + const lower = text.toLowerCase(); + const idx = lower.indexOf(q); + if (idx !== -1) { + const start = Math.max(0, idx - 40); + const end = Math.min(text.length, idx + query.length + 80); + firstMatch = { + at_call: toolCallCount, + context: (start > 0 ? "…" : "") + text.slice(start, end).trim() + (end < text.length ? "…" : ""), + }; + break; + } + } + } + } catch { + // skip malformed lines + } + } + } catch { + return null; + } + + if (!firstMatch) return null; + return { + first_match_at_call: firstMatch.at_call, + total_tool_calls: toolCallCount, + match_context: firstMatch.context, + }; +} + function toNum(v: any): number { if (v == null) return 0; if (typeof v === "object" && typeof v.toNumber === "function") @@ -542,3 +640,102 @@ export async function session_stats(req: Request, res: Response) { res.status(500).json({ error: "Failed to fetch stats" }); } } + +export async function search_sessions(req: Request, res: Response) { + const q = (req.query.q as string || "").trim(); + if (!q || q.length < 2) { + res.status(400).json({ error: "Query parameter 'q' is required (min 2 characters)" }); + return; + } + if (q.length > 200) { + res.status(400).json({ error: "Query too long (max 200 characters)" }); + return; + } + + const limit = Math.min(Math.max(parseInt(req.query.limit as string) || 50, 1), 200); + const dir = sessionsDir(); + + if (!existsSync(dir)) { + res.json([]); + return; + } + + const files = readdirSync(dir).filter( + (f) => + f.endsWith(".jsonl") && + !f.endsWith(".meta.jsonl") && + !f.endsWith(".provenance.jsonl") && + !f.endsWith(".annotations.jsonl"), + ); + + // Pre-load Neo4j metadata if available + let neo4jMap = new Map>(); + if (db) { + try { + const sessions = await db.list_agent_sessions(); + for (const s of sessions) { + const id = String(s.node_key ?? s.name ?? ""); + if (id) neo4jMap.set(id, s); + } + } catch { + // fall back to orphan metadata + } + } + + const results: Array> = []; + + for (const file of files) { + if (results.length >= limit) break; + const fullPath = path.join(dir, file); + const match = searchSessionContent(fullPath, q); + if (!match) continue; + + const id = file.replace(/\.jsonl$/, ""); + const s = neo4jMap.get(id); + + let run: Record; + if (s) { + const { userPromptPreview, answerPreview, toolSequence, toolCallCount } = + existsSync(fullPath) + ? parseSessionMessages(fullPath) + : { userPromptPreview: "", answerPreview: "", toolSequence: [] as string[], toolCallCount: 0 }; + const input = toNum(s.input_tokens); + const cache_read = toNum(s.cache_read_tokens); + const cache_write = toNum(s.cache_write_tokens); + const output = toNum(s.output_tokens); + const total = toNum(s.total_tokens); + const prov = String(s.provider ?? ""); + const mod = String(s.model ?? ""); + const startTimeMs = toNum(s.start_time); + run = { + id, + source: String(s.source ?? "unknown"), + repo: String(s.repo ?? ""), + provider: prov, + model: mod, + timestamp: startTimeMs ? new Date(startTimeMs).toISOString() : new Date().toISOString(), + duration_ms: toNum(s.duration_ms), + token_usage: { input, cache_read, cache_write, output, total }, + cost_usd: calcCost(mod, prov, input, cache_read, cache_write, output), + status: String(s.status ?? "success"), + error_message: String(s.error_message ?? ""), + tool_sequence: toolSequence, + tool_call_count: toolCallCount, + user_prompt_preview: userPromptPreview, + answer_preview: answerPreview, + }; + } else { + run = buildOrphanRun(dir, file); + } + + results.push({ ...run, search_match: match }); + } + + results.sort((a, b) => { + const ma = (a.search_match as SearchMatch).first_match_at_call; + const mb = (b.search_match as SearchMatch).first_match_at_call; + return ma - mb; + }); + + res.json(results); +}