Generated with Hive: Add model and judgeModel validation, credential resolution, and persistence to benchmark run endpoint

src/__tests__/unit/api/legal-benchmark.test.ts

@@ -15,6 +15,7 @@ const mockDbStakworkRunCreate = vi.hoisted(() => vi.fn());
 const mockDbStakworkRunUpdate = vi.hoisted(() => vi.fn());
 const mockDbStakworkRunDeleteMany = vi.hoisted(() => vi.fn());
 const mockDbTransaction = vi.hoisted(() => vi.fn());
+const mockDbLlmModelFindMany = vi.hoisted(() => vi.fn());
 
 const mockDbWorkspaceFindUnique = vi.hoisted(() => vi.fn());
 const mockPusherTrigger = vi.hoisted(() => vi.fn());
@@ -38,6 +39,9 @@ vi.mock("@/lib/db", () => ({
     workspace: {
       findUnique: mockDbWorkspaceFindUnique,
     },
+    llmModel: {
+      findMany: mockDbLlmModelFindMany,
+    },
     $transaction: mockDbTransaction,
   },
 }));
@@ -192,6 +196,14 @@ beforeEach(() => {
   mockAddNode.mockResolvedValue({ success: true, ref_id: "node-ref-1" });
   mockAddEdge.mockResolvedValue({ success: true });
 
+  // Default: LLM model validation — seed known Anthropic models
+  mockDbLlmModelFindMany.mockResolvedValue([
+    { name: "claude-opus-4-5" },
+    { name: "claude-sonnet-4-6" },
+    { name: "claude-opus-4-6" },
+    { name: "claude-haiku-4-5" },
+  ]);
+
   // Default: Bifrost disabled → falls back to env key
   mockGetBifrostForLLM.mockResolvedValue(undefined);
   mockGetApiKeyForModel.mockReturnValue("env-anthropic-key");
@@ -1145,3 +1157,213 @@ describe("POST /run — Bifrost LLM credential vars in Stakwork payload", () =>
     expect(vars.workspace_id).toBe("ws-1");
   });
 });
+
+// ═══════════════════════════════════════════════════════════════════════════
+// POST /run — model/judgeModel validation & defaults
+// ═══════════════════════════════════════════════════════════════════════════
+
+describe("POST /run — model/judgeModel validation & defaults", () => {
+  type VarsShape = Record<string, unknown>;
+
+  function captureVars(): Promise<VarsShape> {
+    return new Promise((resolve) => {
+      vi.stubGlobal(
+        "fetch",
+        vi.fn().mockImplementation((url: string, opts?: RequestInit) => {
+          if (String(url).includes("task.json") || String(url).includes("contents")) {
+            return Promise.resolve({ ok: false, status: 404 });
+          }
+          const payload = opts?.body ? JSON.parse(opts.body as string) : {};
+          resolve(payload.workflow_params.set_var.attributes.vars as VarsShape);
+          return Promise.resolve({ ok: true, json: async () => ({ data: { project_id: 99 } }) });
+        }),
+      );
+    });
+  }
+
+  beforeEach(() => {
+    (getWorkspaceSwarmAccess as Mock).mockResolvedValue(MOCK_SWARM_ACCESS);
+    setupTransactionMock({ runnerResult: { id: "runner-new" } });
+  });
+
+  test("defaults to claude-opus-4-5 / claude-sonnet-4-6 when model/judgeModel omitted", async () => {
+    const varsPromise = captureVars();
+    const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), {
+      params: Promise.resolve({ slug: "openlaw" }),
+    });
+    expect(res.status).toBe(201);
+    const vars = await varsPromise;
+    expect(vars.model).toBe("claude-opus-4-5");
+    expect(vars.judge_model).toBe("claude-sonnet-4-6");
+  });
+
+  test("accepts valid provider/name form and sends bare name to runner", async () => {
+    const varsPromise = captureVars();
+    const res = await postRun(
+      makeRunRequest({
+        taskSlug: "task-a",
+        taskTitle: "Task A",
+        model: "anthropic/claude-opus-4-6",
+        judgeModel: "anthropic/claude-haiku-4-5",
+      }),
+      { params: Promise.resolve({ slug: "openlaw" }) },
+    );
+    expect(res.status).toBe(201);
+    const vars = await varsPromise;
+    expect(vars.model).toBe("claude-opus-4-6");
+    expect(vars.judge_model).toBe("claude-haiku-4-5");
+  });
+
+  test("returns 400 for unknown execution model", async () => {
+    const res = await postRun(
+      makeRunRequest({
+        taskSlug: "task-a",
+        taskTitle: "Task A",
+        model: "anthropic/gpt-4o",
+      }),
+      { params: Promise.resolve({ slug: "openlaw" }) },
+    );
+    expect(res.status).toBe(400);
+    const body = await res.json();
+    expect(body.error).toMatch(/gpt-4o/i);
+  });
+
+  test("returns 400 for unknown judge model", async () => {
+    const res = await postRun(
+      makeRunRequest({
+        taskSlug: "task-a",
+        taskTitle: "Task A",
+        judgeModel: "anthropic/gemini-flash",
+      }),
+      { params: Promise.resolve({ slug: "openlaw" }) },
+    );
+    expect(res.status).toBe(400);
+    const body = await res.json();
+    expect(body.error).toMatch(/gemini-flash/i);
+  });
+
+  test("persists bare model and requestedJudgeModel in runner result at creation", async () => {
+    const createdRows: Array<Record<string, unknown>> = [];
+    mockDbTransaction.mockImplementation(async (fn: (tx: unknown) => Promise<unknown>) => {
+      const tx = {
+        stakworkRun: {
+          findFirst: vi.fn().mockResolvedValue(null),
+          create: vi.fn().mockImplementation((args: { data: Record<string, unknown> }) => {
+            createdRows.push(args.data);
+            return Promise.resolve({ id: "runner-new" });
+          }),
+          update: vi.fn().mockResolvedValue({}),
+        },
+      };
+      return fn(tx);
+    });
+
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ data: { project_id: 99 } }),
+    }));
+
+    await postRun(
+      makeRunRequest({
+        taskSlug: "task-a",
+        taskTitle: "Task A",
+        model: "anthropic/claude-opus-4-6",
+        judgeModel: "anthropic/claude-haiku-4-5",
+      }),
+      { params: Promise.resolve({ slug: "openlaw" }) },
+    );
+
+    expect(createdRows).toHaveLength(1);
+    const resultJson = JSON.parse(createdRows[0].result as string) as Record<string, unknown>;
+    expect(resultJson.model).toBe("claude-opus-4-6");
+    expect(resultJson.judge_model).toBe("claude-haiku-4-5");
+    expect(resultJson.requestedJudgeModel).toBe("claude-haiku-4-5");
+  });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// POST /run — credential reorder regression
+// ═══════════════════════════════════════════════════════════════════════════
+
+describe("POST /run — credential reorder regression", () => {
+  type VarsShape = Record<string, unknown>;
+
+  function captureVars(): Promise<VarsShape> {
+    return new Promise((resolve) => {
+      vi.stubGlobal(
+        "fetch",
+        vi.fn().mockImplementation((url: string, opts?: RequestInit) => {
+          if (String(url).includes("task.json") || String(url).includes("contents")) {
+            return Promise.resolve({ ok: false, status: 404 });
+          }
+          const payload = opts?.body ? JSON.parse(opts.body as string) : {};
+          resolve(payload.workflow_params.set_var.attributes.vars as VarsShape);
+          return Promise.resolve({ ok: true, json: async () => ({ data: { project_id: 99 } }) });
+        }),
+      );
+    });
+  }
+
+  beforeEach(() => {
+    (getWorkspaceSwarmAccess as Mock).mockResolvedValue(MOCK_SWARM_ACCESS);
+    setupTransactionMock({ runnerResult: { id: "runner-new" } });
+  });
+
+  test("no model in body → Bifrost called with default provider/name, dispatches correctly", async () => {
+    mockGetBifrostForLLM.mockResolvedValue(undefined);
+    mockGetApiKeyForModel.mockReturnValue("env-anthropic-key");
+
+    const varsPromise = captureVars();
+    const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), {
+      params: Promise.resolve({ slug: "openlaw" }),
+    });
+    expect(res.status).toBe(201);
+
+    const vars = await varsPromise;
+    expect(vars.model).toBe("claude-opus-4-5");
+    expect(vars.apiKey).toBe("env-anthropic-key");
+    // Bifrost was called — with the default model in provider/name form
+    expect(mockGetBifrostForLLM).toHaveBeenCalledWith(
+      expect.objectContaining({ workspaceId: "ws-1" }),
+      expect.objectContaining({ model: "anthropic/claude-opus-4-5" }),
+    );
+  });
+
+  test("fails with 500 when both Bifrost and env key are unavailable (no silent empty dispatch)", async () => {
+    mockGetBifrostForLLM.mockResolvedValue(undefined);
+    mockGetApiKeyForModel.mockReturnValue(undefined);
+
+    // No fetch should happen (no Stakwork dispatch)
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ data: { project_id: 99 } }),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), {
+      params: Promise.resolve({ slug: "openlaw" }),
+    });
+    expect(res.status).toBe(500);
+    const body = await res.json();
+    expect(body.error).toMatch(/no llm api key/i);
+
+    // Stakwork /projects must NOT have been called
+    const stakworkCalls = mockFetch.mock.calls.filter(([url]: [string]) =>
+      String(url).includes("/projects"),
+    );
+    expect(stakworkCalls).toHaveLength(0);
+  });
+
+  test("Bifrost throws + env key available → still dispatches (graceful fallback)", async () => {
+    mockGetBifrostForLLM.mockRejectedValue(new Error("Bifrost down"));
+    mockGetApiKeyForModel.mockReturnValue("env-anthropic-key");
+
+    const varsPromise = captureVars();
+    const res = await postRun(makeRunRequest({ taskSlug: "task-a", taskTitle: "Task A" }), {
+      params: Promise.resolve({ slug: "openlaw" }),
+    });
+    expect(res.status).toBe(201);
+    const vars = await varsPromise;
+    expect(vars.apiKey).toBe("env-anthropic-key");
+  });
+});

src/__tests__/unit/services/stakwork-run.test.ts

@@ -5070,3 +5070,150 @@ describe("extractDiagramData — payload format variants", () => {
     expect(relayoutDiagram).not.toHaveBeenCalled();
   });
 });
+
+// ═══════════════════════════════════════════════════════════════════════════
+// processLegalBenchmarkRunnerWebhook — requestedJudgeModel preservation
+// ═══════════════════════════════════════════════════════════════════════════
+
+describe("processStakworkRunWebhook — LEGAL_BENCHMARK_RUNNER requestedJudgeModel preservation", () => {
+  const WORKSPACE_ID = "ws-legal";
+  const RUN_ID = "legal-run-1";
+  const WEBHOOK_SECRET = "test-secret";
+
+  // Compute the HMAC the same way the service does
+  function makeRunToken(runId: string) {
+    const { createHmac } = require("crypto") as typeof import("crypto");
+    return createHmac("sha256", WEBHOOK_SECRET).update(runId).digest("hex");
+  }
+
+  const baseLegalRun = {
+    id: RUN_ID,
+    type: StakworkRunType.LEGAL_BENCHMARK_RUNNER,
+    featureId: null,
+    promptVersionId: null,
+    workspaceId: WORKSPACE_ID,
+    workspace: { slug: "openlaw", id: WORKSPACE_ID },
+    status: WorkflowStatus.IN_PROGRESS,
+  };
+
+  function makeQueryContext() {
+    return {
+      type: "LEGAL_BENCHMARK_RUNNER" as const,
+      workspace_id: WORKSPACE_ID,
+      run_id: RUN_ID,
+      run_token: makeRunToken(RUN_ID),
+    };
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    process.env.NEXTAUTH_SECRET = WEBHOOK_SECRET;
+    mockedDb.stakworkRun.findFirst = vi.fn().mockResolvedValue(baseLegalRun);
+    mockedDb.stakworkRun.updateMany = vi.fn().mockResolvedValue({ count: 1 });
+    mockedDb.stakworkRun.update = vi.fn().mockResolvedValue({});
+    mockedDb.stakworkRun.findUnique = vi.fn().mockResolvedValue(null);
+    vi.mocked(pusherServer.trigger).mockResolvedValue(undefined as never);
+  });
+
+  test("runner-echoed judge_model does NOT overwrite requestedJudgeModel stored at creation", async () => {
+    // DB row has creation-time JSON with operator-chosen requestedJudgeModel = "claude-haiku-4-5"
+    const runWithCreationResult = {
+      ...baseLegalRun,
+      result: JSON.stringify({
+        taskSlug: "contracts/task-1",
+        taskTitle: "Task 1",
+        model: "claude-opus-4-5",
+        judge_model: "claude-haiku-4-5",
+        requestedJudgeModel: "claude-haiku-4-5",
+      }),
+    };
+    mockedDb.stakworkRun.findFirst = vi.fn().mockResolvedValue(runWithCreationResult);
+
+    // Simulate the normalized payload shape (as the webhook route sends to processStakworkRunWebhook
+    // after normalizeLegalBenchmarkPayload): Harvey fields nested under `result`,
+    // with project_status/project_id at top level.
+    const webhookPayload = {
+      result: {
+        final_output: "runner output",
+        output_s3_url: "https://stakwork-uploads.s3.us-east-1.amazonaws.com/output/result.txt",
+        score: 80,
+        n_passed: 4,
+        n_total: 5,
+        all_pass: false,
+        judge_model: "claude-sonnet-4-6",  // different from operator-requested!
+      },
+      project_status: "complete",
+      project_id: 1234,
+    };
+
+    const capturedUpdates: Array<Record<string, unknown>> = [];
+    mockedDb.stakworkRun.update = vi.fn().mockImplementation((args: { data: Record<string, unknown> }) => {
+      capturedUpdates.push(args.data);
+      return Promise.resolve({});
+    });
+
+    await processStakworkRunWebhook(webhookPayload, makeQueryContext());
+
+    // At least one update must have been made to the result
+    expect(capturedUpdates.length).toBeGreaterThan(0);
+
+    // Find the update that sets the merged result JSON
+    const resultUpdate = capturedUpdates.find((u) => typeof u.result === "string");
+    expect(resultUpdate).toBeDefined();
+
+    const parsedResult = JSON.parse(resultUpdate!.result as string) as Record<string, unknown>;
+
+    // requestedJudgeModel must still be the operator-selected value (not the runner echo)
+    expect(parsedResult.requestedJudgeModel).toBe("claude-haiku-4-5");
+    // model must also be preserved
+    expect(parsedResult.model).toBe("claude-opus-4-5");
+    // runner-echoed judge_model is kept for audit
+    expect(parsedResult.judge_model).toBe("claude-sonnet-4-6");
+  });
+
+  test("requestedJudgeModel absent on legacy run → no field injected (graceful)", async () => {
+    // Legacy DB row without requestedJudgeModel
+    const legacyRun = {
+      ...baseLegalRun,
+      result: JSON.stringify({
+        taskSlug: "contracts/task-legacy",
+        taskTitle: "Legacy Task",
+      }),
+    };
+    mockedDb.stakworkRun.findFirst = vi.fn().mockResolvedValue(legacyRun);
+
+    const webhookPayload = {
+      result: {
+        final_output: "output",
+        output_s3_url: "https://stakwork-uploads.s3.us-east-1.amazonaws.com/output/r.txt",
+        score: 60,
+        n_passed: 3,
+        n_total: 5,
+        all_pass: false,
+        judge_model: "claude-sonnet-4-6",
+      },
+      project_status: "complete",
+      project_id: 5678,
+    };
+
+    const capturedUpdates: Array<Record<string, unknown>> = [];
+    mockedDb.stakworkRun.update = vi.fn().mockImplementation((args: { data: Record<string, unknown> }) => {
+      capturedUpdates.push(args.data);
+      return Promise.resolve({});
+    });
+
+    await processStakworkRunWebhook(webhookPayload, makeQueryContext());
+
+    const resultUpdate = capturedUpdates.find((u) => typeof u.result === "string");
+    expect(resultUpdate).toBeDefined();
+
+    const parsedResult = JSON.parse(resultUpdate!.result as string) as Record<string, unknown>;
+
+    // Legacy run: requestedJudgeModel not present (graceful)
+    expect(parsedResult.requestedJudgeModel).toBeUndefined();
+    // model similarly absent
+    expect(parsedResult.model).toBeUndefined();
+    // judge_model from runner still present
+    expect(parsedResult.judge_model).toBe("claude-sonnet-4-6");
+  });
+});