From 547325ca739ef2580da7938d579a3366ba47b511 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Mon, 24 Aug 2026 10:37:00 +0800 Subject: [PATCH] fix: inspect materialized demo citation assets Copy cited demo images into durable Notebook storage on demand so VLM inspection can succeed without a parsed-document sync. Co-authored-by: Cursor --- src/domains/chat/route-answer.ts | 50 +++++++++-- src/domains/chat/route-service.test.ts | 117 +++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 5 deletions(-) diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 612c256..73cb38f 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -29,6 +29,7 @@ import { CHAT_MODEL } from "@/lib/ai" import type { HardenChatAssetUrl } from "./media-assets" import { isAuthError } from "@/integrations/dashboard/api-key-service" import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" +import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { summarizeUnknownError } from "@/lib/format-log-value" import { logger } from "@/lib/logger" import { routeResult, type RouteResult } from "@/lib/route-result" @@ -527,14 +528,31 @@ async function hardenSingleChatAsset(input: { readonly assetUrl?: string | null readonly contentType?: string | null }): Promise { - if ( - input.source.status !== "ready" || - !input.source.knowhereDocumentId || - !input.source.knowhereJobId - ) { + if (input.source.status !== "ready" || !input.source.knowhereDocumentId) { return null } + if (input.source.demoKey) { + const demoAssetUrl = resolveDemoSourceAssetUrl({ + demoSourceId: input.source.demoKey, + sourcePath: input.sourcePath, + }) + if (!demoAssetUrl) return null + + return hardenChatAssetByDocument({ + workspaceId: input.workspaceId, + parsedStorage: input.parsedStorage, + documentId: input.source.knowhereDocumentId, + revisionKey: input.source.knowhereDocumentId, + sourcePath: input.sourcePath, + assetUrl: demoAssetUrl, + contentType: input.contentType, + sourceId: input.source.id, + }) + } + + if (!input.source.knowhereJobId) return null + return hardenChatAssetByDocument({ workspaceId: input.workspaceId, parsedStorage: input.parsedStorage, @@ -547,6 +565,28 @@ async function hardenSingleChatAsset(input: { }) } +function resolveDemoSourceAssetUrl(input: { + readonly demoSourceId: string + readonly sourcePath: string +}): string | null { + const normalizedPath = input.sourcePath + .replaceAll("\\", "/") + .replace(/^\.\/+/u, "") + const pathSegments = normalizedPath.split("/") + if ( + pathSegments.some( + (segment) => segment.length === 0 || segment === "." || segment === "..", + ) + ) { + return null + } + + const encodedPath = pathSegments.map(encodeURIComponent).join("/") + return knowhereDemoApi.resolveApiURL( + `/api/v1/demo/sources/${encodeURIComponent(input.demoSourceId)}/assets/${encodedPath}`, + ) +} + async function hardenChatAssetByDocument(input: { readonly workspaceId: string readonly parsedStorage: BlobParsedDocumentStorage diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 4eab67d..e71d38f 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -40,6 +40,12 @@ vi.mock("@/integrations/knowhere", () => ({ makeKnowhereClientWithParsedStorage: mocks.makeKnowhereClientWithParsedStorage, })) +vi.mock("@/integrations/knowhere-demo", () => ({ + knowhereDemoApi: { + resolveApiURL: (pathname: string) => `https://demo.example${pathname}`, + }, +})) + vi.mock("@/domains/chat", async (importOriginal) => { const original = await importOriginal() return { @@ -373,6 +379,117 @@ describe("chat route services", () => { expect(imagePart.mediaType).toBe("image/png") }) + it("hardens materialized demo page assets for image inspection", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const readySource = makeSource({ + title: "TSLA-Q4-2025-Update.pdf", + demoKey: "demo-tsla-q4-2025", + knowhereDocumentId: "doc_materialized_tesla", + knowhereJobId: null, + }) + const demoAssetUrl = + "https://demo.example/api/v1/demo/sources/demo-tsla-q4-2025/assets/page_citation_assets/page-6.png" + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_materialized_tesla/doc_materialized_tesla/page_citation_assets/page-6.png" + mocks.parsedStorageWriteAsset.mockResolvedValue({ url: durableUrl }) + mocks.generateObject.mockResolvedValue({ + object: { + analysis: "The total production row is visible.", + pages: [], + }, + }) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([readySource]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly inspectImages?: (request: { + readonly question: string + readonly assets: readonly { + readonly ref: string + readonly label: string + readonly sourcePath?: string | null + readonly source: { + readonly documentId?: string | null + readonly sourceFileName?: string | null + readonly sectionPath?: string | null + } + }[] + }) => Promise<{ + readonly analysis: string + readonly inspected: readonly { + readonly ref: string + readonly label: string + }[] + readonly skipped: readonly { + readonly ref: string + readonly reason: string + }[] + }> + }) => { + const inspection = await input.inspectImages?.({ + question: "Locate total production.", + assets: [ + { + ref: "asset:r1:result:1", + label: + "TSLA-Q4-2025-Update.pdf / page_citation_assets/page-6.png", + sourcePath: "page_citation_assets/page-6.png", + source: { + documentId: "doc_materialized_tesla", + sourceFileName: "TSLA-Q4-2025-Update.pdf", + sectionPath: "OPERATIONAL SUMMARY", + }, + }, + ], + }) + expect(inspection).toEqual({ + analysis: "The total production row is visible.", + inspected: [ + { + ref: "asset:r1:result:1", + label: + "TSLA-Q4-2025-Update.pdf / page_citation_assets/page-6.png", + }, + ], + skipped: [], + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Inspect it" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "What was Tesla total production?" }, + }) + + expect(result.status).toBe(200) + expect(mocks.parsedStorageGetAssetUrl).toHaveBeenCalledWith({ + documentId: "doc_materialized_tesla", + revisionKey: "doc_materialized_tesla", + sourcePath: "page_citation_assets/page-6.png", + }) + expect(fetch).toHaveBeenCalledWith(demoAssetUrl) + expect(fetch).toHaveBeenCalledWith(durableUrl) + expect(mocks.parsedStorageWriteAsset).toHaveBeenCalledWith({ + documentId: "doc_materialized_tesla", + revisionKey: "doc_materialized_tesla", + sourcePath: "page_citation_assets/page-6.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }) + }) + it("uses image inspection source paths for retrieved page assets", async () => { const workspace = makeWorkspace() const client = { retrieval: { query: vi.fn() } }