From 12df81de0136df8a0edef9f5dd309c09f9c4bce0 Mon Sep 17 00:00:00 2001 From: tomsmith8 Date: Fri, 10 Jul 2026 12:28:46 +0000 Subject: [PATCH] Generated with Hive: Add model and judgeModel validation, credential resolution, and persistence to benchmark run endpoint --- .../unit/api/legal-benchmark.test.ts | 222 ++++++++++++++++++ .../unit/services/stakwork-run.test.ts | 147 ++++++++++++ .../[slug]/legal/benchmarks/run/route.ts | 80 +++++-- src/hooks/useLegalBenchmarkRunList.ts | 7 + src/services/stakwork-run.ts | 11 +- src/types/legal.ts | 6 +- src/utils/mockSetup.ts | 6 + 7 files changed, 461 insertions(+), 18 deletions(-) diff --git a/src/__tests__/unit/api/legal-benchmark.test.ts b/src/__tests__/unit/api/legal-benchmark.test.ts index b4ba455622..f6fbf418f8 100644 --- a/src/__tests__/unit/api/legal-benchmark.test.ts +++ b/src/__tests__/unit/api/legal-benchmark.test.ts @@ -14,6 +14,7 @@ const mockDbStakworkRunCreate = vi.hoisted(() => vi.fn()); const mockDbStakworkRunUpdate = vi.hoisted(() => vi.fn()); const mockDbStakworkRunDeleteMany = vi.hoisted(() => vi.fn()); const mockDbTransaction = vi.hoisted(() => vi.fn()); +const mockDbLlmModelFindMany = vi.hoisted(() => vi.fn()); const mockDbWorkspaceFindUnique = vi.hoisted(() => vi.fn()); const mockPusherTrigger = vi.hoisted(() => vi.fn()); @@ -37,6 +38,9 @@ vi.mock("@/lib/db", () => ({ workspace: { findUnique: mockDbWorkspaceFindUnique, }, + llmModel: { + findMany: mockDbLlmModelFindMany, + }, $transaction: mockDbTransaction, }, })); @@ -191,6 +195,14 @@ beforeEach(() => { mockAddNode.mockResolvedValue({ success: true, ref_id: "node-ref-1" }); mockAddEdge.mockResolvedValue({ success: true }); + // Default: LLM model validation — seed known Anthropic models + mockDbLlmModelFindMany.mockResolvedValue([ + { name: "claude-opus-4-5" }, + { name: "claude-sonnet-4-6" }, + { name: "claude-opus-4-6" }, + { name: "claude-haiku-4-5" }, + ]); + // Default: Bifrost disabled → falls back to env key mockGetBifrostForLLM.mockResolvedValue(undefined); mockGetApiKeyForModel.mockReturnValue("env-anthropic-key"); @@ -1093,3 +1105,213 @@ describe("POST /run — Bifrost LLM credential vars in Stakwork payload", () => expect(vars.workspace_id).toBe("ws-1"); }); }); + +// ═══════════════════════════════════════════════════════════════════════════ +// POST /run — model/judgeModel validation & defaults +// ═══════════════════════════════════════════════════════════════════════════ + +describe("POST /run — model/judgeModel validation & defaults", () => { + type VarsShape = Record; + + function captureVars(): Promise { + return new Promise((resolve) => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url: string, opts?: RequestInit) => { + if (String(url).includes("task.json") || String(url).includes("contents")) { + return Promise.resolve({ ok: false, status: 404 }); + } + const payload = opts?.body ? JSON.parse(opts.body as string) : {}; + resolve(payload.workflow_params.set_var.attributes.vars as VarsShape); + return Promise.resolve({ ok: true, json: async () => ({ data: { project_id: 99 } }) }); + }), + ); + }); + } + + beforeEach(() => { + (getWorkspaceSwarmAccess as Mock).mockResolvedValue(MOCK_SWARM_ACCESS); + setupTransactionMock({ runnerResult: { id: "runner-new" } }); + }); + + test("defaults to claude-opus-4-5 / claude-sonnet-4-6 when model/judgeModel omitted", async () => { + const varsPromise = captureVars(); + const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), { + params: Promise.resolve({ slug: "openlaw" }), + }); + expect(res.status).toBe(201); + const vars = await varsPromise; + expect(vars.model).toBe("claude-opus-4-5"); + expect(vars.judge_model).toBe("claude-sonnet-4-6"); + }); + + test("accepts valid provider/name form and sends bare name to runner", async () => { + const varsPromise = captureVars(); + const res = await postRun( + makeRunRequest({ + taskSlug: "task-a", + taskTitle: "Task A", + model: "anthropic/claude-opus-4-6", + judgeModel: "anthropic/claude-haiku-4-5", + }), + { params: Promise.resolve({ slug: "openlaw" }) }, + ); + expect(res.status).toBe(201); + const vars = await varsPromise; + expect(vars.model).toBe("claude-opus-4-6"); + expect(vars.judge_model).toBe("claude-haiku-4-5"); + }); + + test("returns 400 for unknown execution model", async () => { + const res = await postRun( + makeRunRequest({ + taskSlug: "task-a", + taskTitle: "Task A", + model: "anthropic/gpt-4o", + }), + { params: Promise.resolve({ slug: "openlaw" }) }, + ); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/gpt-4o/i); + }); + + test("returns 400 for unknown judge model", async () => { + const res = await postRun( + makeRunRequest({ + taskSlug: "task-a", + taskTitle: "Task A", + judgeModel: "anthropic/gemini-flash", + }), + { params: Promise.resolve({ slug: "openlaw" }) }, + ); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/gemini-flash/i); + }); + + test("persists bare model and requestedJudgeModel in runner result at creation", async () => { + const createdRows: Array> = []; + mockDbTransaction.mockImplementation(async (fn: (tx: unknown) => Promise) => { + const tx = { + stakworkRun: { + findFirst: vi.fn().mockResolvedValue(null), + create: vi.fn().mockImplementation((args: { data: Record }) => { + createdRows.push(args.data); + return Promise.resolve({ id: "runner-new" }); + }), + update: vi.fn().mockResolvedValue({}), + }, + }; + return fn(tx); + }); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ data: { project_id: 99 } }), + })); + + await postRun( + makeRunRequest({ + taskSlug: "task-a", + taskTitle: "Task A", + model: "anthropic/claude-opus-4-6", + judgeModel: "anthropic/claude-haiku-4-5", + }), + { params: Promise.resolve({ slug: "openlaw" }) }, + ); + + expect(createdRows).toHaveLength(1); + const resultJson = JSON.parse(createdRows[0].result as string) as Record; + expect(resultJson.model).toBe("claude-opus-4-6"); + expect(resultJson.judge_model).toBe("claude-haiku-4-5"); + expect(resultJson.requestedJudgeModel).toBe("claude-haiku-4-5"); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════ +// POST /run — credential reorder regression +// ═══════════════════════════════════════════════════════════════════════════ + +describe("POST /run — credential reorder regression", () => { + type VarsShape = Record; + + function captureVars(): Promise { + return new Promise((resolve) => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url: string, opts?: RequestInit) => { + if (String(url).includes("task.json") || String(url).includes("contents")) { + return Promise.resolve({ ok: false, status: 404 }); + } + const payload = opts?.body ? JSON.parse(opts.body as string) : {}; + resolve(payload.workflow_params.set_var.attributes.vars as VarsShape); + return Promise.resolve({ ok: true, json: async () => ({ data: { project_id: 99 } }) }); + }), + ); + }); + } + + beforeEach(() => { + (getWorkspaceSwarmAccess as Mock).mockResolvedValue(MOCK_SWARM_ACCESS); + setupTransactionMock({ runnerResult: { id: "runner-new" } }); + }); + + test("no model in body → Bifrost called with default provider/name, dispatches correctly", async () => { + mockGetBifrostForLLM.mockResolvedValue(undefined); + mockGetApiKeyForModel.mockReturnValue("env-anthropic-key"); + + const varsPromise = captureVars(); + const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), { + params: Promise.resolve({ slug: "openlaw" }), + }); + expect(res.status).toBe(201); + + const vars = await varsPromise; + expect(vars.model).toBe("claude-opus-4-5"); + expect(vars.apiKey).toBe("env-anthropic-key"); + // Bifrost was called — with the default model in provider/name form + expect(mockGetBifrostForLLM).toHaveBeenCalledWith( + expect.objectContaining({ workspaceId: "ws-1" }), + expect.objectContaining({ model: "anthropic/claude-opus-4-5" }), + ); + }); + + test("fails with 500 when both Bifrost and env key are unavailable (no silent empty dispatch)", async () => { + mockGetBifrostForLLM.mockResolvedValue(undefined); + mockGetApiKeyForModel.mockReturnValue(undefined); + + // No fetch should happen (no Stakwork dispatch) + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ data: { project_id: 99 } }), + }); + vi.stubGlobal("fetch", mockFetch); + + const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), { + params: Promise.resolve({ slug: "openlaw" }), + }); + expect(res.status).toBe(500); + const body = await res.json(); + expect(body.error).toMatch(/no llm api key/i); + + // Stakwork /projects must NOT have been called + const stakworkCalls = mockFetch.mock.calls.filter(([url]: [string]) => + String(url).includes("/projects"), + ); + expect(stakworkCalls).toHaveLength(0); + }); + + test("Bifrost throws + env key available → still dispatches (graceful fallback)", async () => { + mockGetBifrostForLLM.mockRejectedValue(new Error("Bifrost down")); + mockGetApiKeyForModel.mockReturnValue("env-anthropic-key"); + + const varsPromise = captureVars(); + const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), { + params: Promise.resolve({ slug: "openlaw" }), + }); + expect(res.status).toBe(201); + const vars = await varsPromise; + expect(vars.apiKey).toBe("env-anthropic-key"); + }); +}); diff --git a/src/__tests__/unit/services/stakwork-run.test.ts b/src/__tests__/unit/services/stakwork-run.test.ts index fbfd05f0cd..84a855292c 100644 --- a/src/__tests__/unit/services/stakwork-run.test.ts +++ b/src/__tests__/unit/services/stakwork-run.test.ts @@ -5029,3 +5029,150 @@ describe("extractDiagramData — payload format variants", () => { expect(relayoutDiagram).not.toHaveBeenCalled(); }); }); + +// ═══════════════════════════════════════════════════════════════════════════ +// processLegalBenchmarkRunnerWebhook — requestedJudgeModel preservation +// ═══════════════════════════════════════════════════════════════════════════ + +describe("processStakworkRunWebhook — LEGAL_BENCHMARK_RUNNER requestedJudgeModel preservation", () => { + const WORKSPACE_ID = "ws-legal"; + const RUN_ID = "legal-run-1"; + const WEBHOOK_SECRET = "test-secret"; + + // Compute the HMAC the same way the service does + function makeRunToken(runId: string) { + const { createHmac } = require("crypto") as typeof import("crypto"); + return createHmac("sha256", WEBHOOK_SECRET).update(runId).digest("hex"); + } + + const baseLegalRun = { + id: RUN_ID, + type: StakworkRunType.LEGAL_BENCHMARK_RUNNER, + featureId: null, + promptVersionId: null, + workspaceId: WORKSPACE_ID, + workspace: { slug: "openlaw", id: WORKSPACE_ID }, + status: WorkflowStatus.IN_PROGRESS, + }; + + function makeQueryContext() { + return { + type: "LEGAL_BENCHMARK_RUNNER" as const, + workspace_id: WORKSPACE_ID, + run_id: RUN_ID, + run_token: makeRunToken(RUN_ID), + }; + } + + beforeEach(() => { + vi.clearAllMocks(); + process.env.NEXTAUTH_SECRET = WEBHOOK_SECRET; + mockedDb.stakworkRun.findFirst = vi.fn().mockResolvedValue(baseLegalRun); + mockedDb.stakworkRun.updateMany = vi.fn().mockResolvedValue({ count: 1 }); + mockedDb.stakworkRun.update = vi.fn().mockResolvedValue({}); + mockedDb.stakworkRun.findUnique = vi.fn().mockResolvedValue(null); + vi.mocked(pusherServer.trigger).mockResolvedValue(undefined as never); + }); + + test("runner-echoed judge_model does NOT overwrite requestedJudgeModel stored at creation", async () => { + // DB row has creation-time JSON with operator-chosen requestedJudgeModel = "claude-haiku-4-5" + const runWithCreationResult = { + ...baseLegalRun, + result: JSON.stringify({ + taskSlug: "contracts/task-1", + taskTitle: "Task 1", + model: "claude-opus-4-5", + judge_model: "claude-haiku-4-5", + requestedJudgeModel: "claude-haiku-4-5", + }), + }; + mockedDb.stakworkRun.findFirst = vi.fn().mockResolvedValue(runWithCreationResult); + + // Simulate the normalized payload shape (as the webhook route sends to processStakworkRunWebhook + // after normalizeLegalBenchmarkPayload): Harvey fields nested under `result`, + // with project_status/project_id at top level. + const webhookPayload = { + result: { + final_output: "runner output", + output_s3_url: "https://stakwork-uploads.s3.us-east-1.amazonaws.com/output/result.txt", + score: 80, + n_passed: 4, + n_total: 5, + all_pass: false, + judge_model: "claude-sonnet-4-6", // different from operator-requested! + }, + project_status: "complete", + project_id: 1234, + }; + + const capturedUpdates: Array> = []; + mockedDb.stakworkRun.update = vi.fn().mockImplementation((args: { data: Record }) => { + capturedUpdates.push(args.data); + return Promise.resolve({}); + }); + + await processStakworkRunWebhook(webhookPayload, makeQueryContext()); + + // At least one update must have been made to the result + expect(capturedUpdates.length).toBeGreaterThan(0); + + // Find the update that sets the merged result JSON + const resultUpdate = capturedUpdates.find((u) => typeof u.result === "string"); + expect(resultUpdate).toBeDefined(); + + const parsedResult = JSON.parse(resultUpdate!.result as string) as Record; + + // requestedJudgeModel must still be the operator-selected value (not the runner echo) + expect(parsedResult.requestedJudgeModel).toBe("claude-haiku-4-5"); + // model must also be preserved + expect(parsedResult.model).toBe("claude-opus-4-5"); + // runner-echoed judge_model is kept for audit + expect(parsedResult.judge_model).toBe("claude-sonnet-4-6"); + }); + + test("requestedJudgeModel absent on legacy run → no field injected (graceful)", async () => { + // Legacy DB row without requestedJudgeModel + const legacyRun = { + ...baseLegalRun, + result: JSON.stringify({ + taskSlug: "contracts/task-legacy", + taskTitle: "Legacy Task", + }), + }; + mockedDb.stakworkRun.findFirst = vi.fn().mockResolvedValue(legacyRun); + + const webhookPayload = { + result: { + final_output: "output", + output_s3_url: "https://stakwork-uploads.s3.us-east-1.amazonaws.com/output/r.txt", + score: 60, + n_passed: 3, + n_total: 5, + all_pass: false, + judge_model: "claude-sonnet-4-6", + }, + project_status: "complete", + project_id: 5678, + }; + + const capturedUpdates: Array> = []; + mockedDb.stakworkRun.update = vi.fn().mockImplementation((args: { data: Record }) => { + capturedUpdates.push(args.data); + return Promise.resolve({}); + }); + + await processStakworkRunWebhook(webhookPayload, makeQueryContext()); + + const resultUpdate = capturedUpdates.find((u) => typeof u.result === "string"); + expect(resultUpdate).toBeDefined(); + + const parsedResult = JSON.parse(resultUpdate!.result as string) as Record; + + // Legacy run: requestedJudgeModel not present (graceful) + expect(parsedResult.requestedJudgeModel).toBeUndefined(); + // model similarly absent + expect(parsedResult.model).toBeUndefined(); + // judge_model from runner still present + expect(parsedResult.judge_model).toBe("claude-sonnet-4-6"); + }); +}); diff --git a/src/app/api/workspaces/[slug]/legal/benchmarks/run/route.ts b/src/app/api/workspaces/[slug]/legal/benchmarks/run/route.ts index 23fd889305..f754fa467d 100644 --- a/src/app/api/workspaces/[slug]/legal/benchmarks/run/route.ts +++ b/src/app/api/workspaces/[slug]/legal/benchmarks/run/route.ts @@ -79,12 +79,61 @@ export async function POST(request: NextRequest, { params }: RouteParams) { ); } - const BENCHMARK_MODEL = "claude-opus-4-5"; + // Parse + validate body BEFORE Bifrost resolution so we can use the + // operator-supplied model for credential resolution. + let body: { taskSlug?: string; taskTitle?: string; model?: string; judgeModel?: string }; + try { + body = await request.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const { taskSlug, taskTitle } = body; + if (!taskSlug || !taskTitle) { + return NextResponse.json( + { error: "taskSlug and taskTitle are required" }, + { status: 400 }, + ); + } + + // Defaults (provider/name form expected from client; bare form for runner vars) + const DEFAULT_MODEL_PROVIDER = "anthropic/claude-opus-4-5"; + const DEFAULT_JUDGE_MODEL_PROVIDER = "anthropic/claude-sonnet-4-6"; + + // Client sends provider/name form; strip prefix for runner vars + const stripProviderPrefix = (m: string) => m.includes("/") ? m.split("/").slice(1).join("/") : m; + + const rawModel = body.model ?? DEFAULT_MODEL_PROVIDER; + const rawJudgeModel = body.judgeModel ?? DEFAULT_JUDGE_MODEL_PROVIDER; + const bareModel = stripProviderPrefix(rawModel); + const bareJudgeModel = stripProviderPrefix(rawJudgeModel); + + // Validate both against known Anthropic public models in the DB + const allowedModels = await db.llmModel.findMany({ + where: { provider: "ANTHROPIC", isPublic: true }, + select: { name: true }, + }); + const allowedNames = new Set(allowedModels.map((m) => m.name)); + + if (!allowedNames.has(bareModel)) { + return NextResponse.json( + { error: `Unknown or non-Anthropic execution model: ${bareModel}` }, + { status: 400 }, + ); + } + if (!allowedNames.has(bareJudgeModel)) { + return NextResponse.json( + { error: `Unknown or non-Anthropic judge model: ${bareJudgeModel}` }, + { status: 400 }, + ); + } + + // Resolve credentials using the validated execution model (provider/name form) let bifrost: Awaited> | undefined; try { bifrost = await getBifrostForLLM( { workspaceId, workspaceSlug: slug, userId: userOrResponse.id }, - { agentName: "plan-agent", model: BENCHMARK_MODEL }, + { agentName: "plan-agent", model: rawModel }, ); } catch (err) { console.warn( @@ -93,19 +142,12 @@ export async function POST(request: NextRequest, { params }: RouteParams) { ); } - // Parse + validate body - let body: { taskSlug?: string; taskTitle?: string }; - try { - body = await request.json(); - } catch { - return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); - } - - const { taskSlug, taskTitle } = body; - if (!taskSlug || !taskTitle) { + // Fail clearly rather than dispatching with an empty apiKey + const resolvedApiKey = bifrost?.apiKey ?? getApiKeyForModel(rawModel); + if (!resolvedApiKey) { return NextResponse.json( - { error: "taskSlug and taskTitle are required" }, - { status: 400 }, + { error: "No LLM API key available for the selected execution model" }, + { status: 500 }, ); } @@ -202,6 +244,9 @@ export async function POST(request: NextRequest, { params }: RouteParams) { const runnerResultJson: Record = { taskSlug, taskTitle, + model: bareModel, + judge_model: bareJudgeModel, + requestedJudgeModel: bareJudgeModel, // evalTriggerRef will be added later (non-fatal Jarvis step) }; @@ -258,8 +303,9 @@ export async function POST(request: NextRequest, { params }: RouteParams) { graph_base_url: graphBaseUrl, swarm_secret_alias: swarmSecretAlias, secret: swarmSecretAlias, - model: BENCHMARK_MODEL, - apiKey: bifrost?.apiKey ?? getApiKeyForModel(BENCHMARK_MODEL) ?? "", + model: bareModel, + judge_model: bareJudgeModel, + apiKey: resolvedApiKey, baseUrl: bifrost?.baseUrl ?? "", ...(bifrost && Object.keys(bifrost.headers).length > 0 ? { headers: bifrost.headers } @@ -272,6 +318,8 @@ export async function POST(request: NextRequest, { params }: RouteParams) { }, }; + console.log(`[legal/benchmarks/run] dispatching model=${bareModel} judge_model=${bareJudgeModel}`); + const stakworkResponse = await fetch(`${optionalEnvVars.STAKWORK_BASE_URL}/projects`, { method: "POST", headers: { diff --git a/src/hooks/useLegalBenchmarkRunList.ts b/src/hooks/useLegalBenchmarkRunList.ts index 01cba324d1..f570a63066 100644 --- a/src/hooks/useLegalBenchmarkRunList.ts +++ b/src/hooks/useLegalBenchmarkRunList.ts @@ -14,6 +14,10 @@ export interface BenchmarkRunListRow { n_passed?: number; n_total?: number; all_pass?: boolean; + // Model fields stored at creation time + model?: string; + requestedJudgeModel?: string; + judge_model?: string; } interface UseLegalBenchmarkRunListResult { @@ -70,6 +74,9 @@ export function useLegalBenchmarkRunList( n_passed: parsed?.n_passed, n_total: parsed?.n_total, all_pass: parsed?.all_pass, + model: parsed?.model, + requestedJudgeModel: parsed?.requestedJudgeModel, + judge_model: parsed?.judge_model, }; }); diff --git a/src/services/stakwork-run.ts b/src/services/stakwork-run.ts index aee4348967..a51c8d3213 100644 --- a/src/services/stakwork-run.ts +++ b/src/services/stakwork-run.ts @@ -1553,7 +1553,10 @@ async function processLegalBenchmarkRunnerWebhook( } } - // Merge output + flat score fields into persisted result + // Merge output + flat score fields into persisted result. + // NOTE: `requestedJudgeModel` and `model` were stored at creation time (operator-selected values) + // and must NOT be overwritten by webhook-echoed fields. The runner-echoed `judge_model` is + // persisted as-is for audit purposes, but the operator's selection is preserved in `requestedJudgeModel`. const mergedResult: Record = { ...resultJson, ...(finalOutput !== undefined ? { runnerOutputText: finalOutput } : {}), @@ -1566,7 +1569,13 @@ async function processLegalBenchmarkRunnerWebhook( ...(scoreFields.pass_rate !== undefined ? { pass_rate: scoreFields.pass_rate } : {}), ...(scoreFields.all_pass !== undefined ? { all_pass: scoreFields.all_pass } : {}), ...(scoreFields.scores_s3_url !== undefined ? { scores_s3_url: scoreFields.scores_s3_url } : {}), + // Persist runner-echoed judge_model for audit, but preserve the operator-requested value + // stored at creation time under `requestedJudgeModel` — never overwrite it here. ...(scoreFields.judge_model !== undefined ? { judge_model: scoreFields.judge_model } : {}), + // Restore operator-selected values in case the spread above clobbered them + // (they are already in resultJson from creation, but be explicit for safety) + ...(resultJson.requestedJudgeModel !== undefined ? { requestedJudgeModel: resultJson.requestedJudgeModel } : {}), + ...(resultJson.model !== undefined ? { model: resultJson.model } : {}), }; await db.stakworkRun.update({ diff --git a/src/types/legal.ts b/src/types/legal.ts index 3e8ca20356..6c644a0a98 100644 --- a/src/types/legal.ts +++ b/src/types/legal.ts @@ -36,8 +36,12 @@ export interface BenchmarkRunResult { all_pass?: boolean; /** S3 URL to per-criterion score breakdown (out of scope for v1 display) */ scores_s3_url?: string; - /** Name of the judge model used for evaluation */ + /** Name of the judge model used for evaluation (echoed from runner webhook) */ judge_model?: string; + /** Execution model selected by the operator at run submission time (bare name, e.g. "claude-opus-4-5") */ + model?: string; + /** Judge model selected by the operator at run submission time; preserved and never clobbered by webhook echo */ + requestedJudgeModel?: string; } /** diff --git a/src/utils/mockSetup.ts b/src/utils/mockSetup.ts index 6e19342e39..b40d850768 100644 --- a/src/utils/mockSetup.ts +++ b/src/utils/mockSetup.ts @@ -1012,6 +1012,12 @@ export async function ensureMockLlmModels(): Promise { inputPricePer1M: 1, outputPricePer1M: 5, }, + { + name: "claude-opus-4-5", + provider: LlmProvider.ANTHROPIC, + inputPricePer1M: 15, + outputPricePer1M: 75, + }, ]; for (const m of models) {