diff --git a/.github/workflows/create-tag.yml b/.github/workflows/create-tag.yml index 7b8d278b6..6242cc748 100644 --- a/.github/workflows/create-tag.yml +++ b/.github/workflows/create-tag.yml @@ -40,6 +40,7 @@ on: - memory-consolidate - opencode - openwiki + - pdf - pi - provider-anthropic - provider-claude-code diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d718f856..db3e324cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,6 +34,7 @@ on: - 'memory-consolidate/v*' - 'opencode/v*' - 'openwiki/v*' + - 'pdf/v*' - 'pi/v*' - 'provider-anthropic/v*' - 'provider-claude-code/v*' diff --git a/README.md b/README.md index fe941ac7f..ec6ec82a7 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ npx skills add iii-hq/iii --all | [`worktree`](worktree/) | Rust | Git worktree lifecycle for parallel agents — `worktree::*` mint, claim, and track isolated worktrees per repo, emit six lifecycle trigger types, and land branches back through a per-repo FIFO queue (rebase, test gate, ff-only merge). | | [`github`](github/) | Rust | GitHub CLI (`gh`) as an iii worker — typed `github::pr/issue/repo/run/workflow/release/search::*` functions plus `github::exec` argv passthrough and `github::api` for any GitHub REST endpoint. | | [`openwiki`](openwiki/) | Node | Source-grounded markdown wiki for any git repository — a lead agent plans the index and writer sub-agents store cited pages via `openwiki::write-page`, with router and heuristic fallback tiers, incremental refresh from git diffs on a per-wiki cron schedule, and a browser UI + JSON API under `/openwiki`. | +| [`pdf`](pdf/) | Rust | Read PDFs locally — `pdf::classify` routes text-based versus scanned in tens of milliseconds and names the pages that still need OCR, `pdf::to-markdown` converts with headings, lists and tables intact, and `pdf::extract-items` / `::extract-regions` expose positions and the text inside a box. Ships a console page. | ## SDK diff --git a/console/web/src/components/chat/AttachmentButton.tsx b/console/web/src/components/chat/AttachmentButton.tsx index c6eafe19c..ee3d3b618 100644 --- a/console/web/src/components/chat/AttachmentButton.tsx +++ b/console/web/src/components/chat/AttachmentButton.tsx @@ -41,6 +41,9 @@ export function AttachmentButton({ size: f.size, type: f.type || 'application/octet-stream', dataUrl: await readPreview(f), + // Kept so the send path can hand the bytes to a worker that reads this + // kind of file (PDFs go through `pdf::to-markdown`). Not persisted. + file: f, })), ) onAttach(attachments) diff --git a/console/web/src/components/chat/ChatView.tsx b/console/web/src/components/chat/ChatView.tsx index 16d680c74..79308ef00 100644 --- a/console/web/src/components/chat/ChatView.tsx +++ b/console/web/src/components/chat/ChatView.tsx @@ -33,6 +33,11 @@ import { useConversationsCtxOptional } from '@/lib/conversations-context' import { syncEditorWorkspace } from '@/lib/editor-sync' import { expandFileMentions, parseFileMentions } from '@/lib/file-mentions' import { formatStopReason } from '@/lib/format-stop-reason' +import { + expandPdfAttachments, + isPdfAttachment, + summaryLabel, +} from '@/lib/pdf-attachments' import { newMessageId } from '@/lib/session-id' import { cn } from '@/lib/utils' import { fetchDefaultWorkingDir, validateWorkspaceDir } from '@/lib/working-dir' @@ -500,6 +505,29 @@ export function ChatView({ ).blocks } } + // Same expansion as the live send path: a queued message's PDFs have + // to reach the agent as markdown too, or editing a queued message + // would silently drop the document it carried. + if ( + backend.id === 'real' && + payload.attachments.some(isPdfAttachment) + ) { + const expanded = await expandPdfAttachments(payload.attachments) + if (expanded.blocks.length > 0) { + attachedBlocks = [...(attachedBlocks ?? []), ...expanded.blocks] + } + // Same reporting as the live send path. Staying silent here would let + // an edited queued message lose its document with no explanation. + for (const failure of expanded.failures) { + onAppendMessage( + conversationId, + makeSystemNotice( + `could not read ${failure.name} — ${failure.reason}`, + 'warn', + ), + ) + } + } try { await backend.editQueued?.( conversationId, @@ -995,6 +1023,44 @@ export function ChatView({ } } + // A PDF is not text: read as bytes it reaches the model as noise, so the + // `pdf` worker converts it on this machine and the markdown is appended + // as another attachment block. Failures never block the send — an + // unreadable document becomes a placeholder block plus a warn notice, so + // the model knows it was handed something it could not read. + if (backend.id === 'real' && payload.attachments.some(isPdfAttachment)) { + const expanded = await expandPdfAttachments(payload.attachments) + if (expanded.blocks.length > 0) { + attachedBlocks = [...(attachedBlocks ?? []), ...expanded.blocks] + } + // Relabel the chip with what the worker made of the document. The + // expansion runs before the model is called, so it never shows up as a + // function call — without this a person has no way to tell the PDF was + // read at all. + if (expanded.read.length > 0 && !willQueue) { + const byId = new Map(expanded.read.map((r) => [r.id, r])) + onPatchMessage(conversationId, userMsg.id, { + // `file` is dropped here as well as relabelled. It has done its job + // by now, and keeping it would hold the whole document in memory + // for as long as the conversation stays open. + attachments: (userMsg.attachments ?? []).map(({ file, ...a }) => { + void file + const summary = byId.get(a.id) + return summary ? { ...a, name: summaryLabel(a.name, summary) } : a + }), + }) + } + for (const failure of expanded.failures) { + onAppendMessage( + conversationId, + makeSystemNotice( + `could not read ${failure.name} — ${failure.reason}`, + 'warn', + ), + ) + } + } + // Mid-stream send (MOT-3837): a turn is already streaming, so the // harness queues the message and delivers it when the stream ends. No // second stream loop — the live one keeps rendering. The draft chip diff --git a/console/web/src/lib/pdf-attachments.test.ts b/console/web/src/lib/pdf-attachments.test.ts new file mode 100644 index 000000000..ab948668c --- /dev/null +++ b/console/web/src/lib/pdf-attachments.test.ts @@ -0,0 +1,317 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { Attachment } from '@/types/chat' +import { + CLASSIFY_FUNCTION_ID, + expandPdfAttachments, + isPdfAttachment, + MAX_PDFS_PER_SEND, + summaryLabel, + TO_MARKDOWN_FUNCTION_ID, +} from './pdf-attachments' + +function pdf(name = 'report.pdf', bytes = 'hello'): Attachment { + return { + id: name, + name, + size: bytes.length, + type: 'application/pdf', + file: new File([bytes], name, { type: 'application/pdf' }), + } +} + +function textFile(): Attachment { + return { + id: 'notes.txt', + name: 'notes.txt', + size: 4, + type: 'text/plain', + file: new File(['note'], 'notes.txt', { type: 'text/plain' }), + } +} + +type TriggerFn = ( + functionId: string, + payload: Record, +) => Promise + +/** + * A fake bus: each function id maps to the value it returns, or to an `Error` + * it throws. The second element records call order, because "classify before + * extracting, and skip extracting on a scan" is behaviour worth asserting. + */ +function trigger( + responses: Record, +): [TriggerFn & { mock: { calls: unknown[][] } }, string[]] { + const calls: string[] = [] + const fn = vi.fn( + async (functionId: string, _payload: Record) => { + calls.push(functionId) + const value = responses[functionId] + if (value instanceof Error) throw value + return value + }, + ) + return [fn as unknown as TriggerFn & { mock: { calls: unknown[][] } }, calls] +} + +describe('isPdfAttachment', () => { + it('matches by declared type', () => { + expect(isPdfAttachment(pdf())).toBe(true) + }) + + it('matches by extension when the browser reports no type', () => { + expect( + isPdfAttachment({ ...pdf(), type: 'application/octet-stream' }), + ).toBe(true) + expect(isPdfAttachment({ ...pdf('REPORT.PDF'), type: '' })).toBe(true) + }) + + it('leaves other files alone', () => { + expect(isPdfAttachment(textFile())).toBe(false) + }) +}) + +describe('expandPdfAttachments', () => { + it('does nothing when there are no PDFs', async () => { + const [fn] = trigger({}) + const result = await expandPdfAttachments([textFile()], fn) + expect(result).toEqual({ blocks: [], read: [], failures: [] }) + expect(fn.mock.calls).toHaveLength(0) + }) + + it('classifies before extracting and inlines the markdown', async () => { + const [fn, calls] = trigger({ + [CLASSIFY_FUNCTION_ID]: { + document_type: 'text_based', + page_count: 8, + pages_needing_ocr: [], + }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { text: '# Prospectus', chars: 12, total_chars: 12 }, + page_count: 8, + }, + }) + + const { blocks, failures } = await expandPdfAttachments([pdf()], fn) + + expect(calls).toEqual([CLASSIFY_FUNCTION_ID, TO_MARKDOWN_FUNCTION_ID]) + expect(failures).toEqual([]) + expect(blocks).toHaveLength(1) + expect(blocks[0]).toContain('') + }) + + /** + * A scan must not silently become an empty attachment: the model has to be + * able to tell "I read it and it has no text" from "I was given nothing", + * because only the first is worth reporting back to the person. + */ + it('says a scan was read and found unreadable, and skips extraction', async () => { + const [fn, calls] = trigger({ + [CLASSIFY_FUNCTION_ID]: { + document_type: 'scanned', + page_count: 3, + pages_needing_ocr: [1, 2, 3], + ocr_reasons: [{ page: 1, reasons: ['scanned'] }], + }, + }) + + const { blocks } = await expandPdfAttachments([pdf()], fn) + + expect(calls).toEqual([CLASSIFY_FUNCTION_ID]) + expect(blocks[0]).toContain('needs-ocr="true"') + expect(blocks[0]).toContain('no extractable text') + expect(blocks[0]).toContain('scanned') + expect(blocks[0]).toContain('Do not claim the document is empty') + }) + + it('reports truncation with the size it withheld', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { document_type: 'text_based', page_count: 400 }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { + text: 'start of a very long report', + chars: 27, + total_chars: 900_000, + truncated: true, + }, + page_count: 400, + }, + }) + + const { blocks } = await expandPdfAttachments([pdf()], fn) + + expect(blocks[0]).toContain('truncated="true"') + expect(blocks[0]).toContain('total-chars="900000"') + expect(blocks[0]).toContain('pages filter') + }) + + it('carries the per-page OCR verdict of a mixed document', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { + document_type: 'mixed', + page_count: 10, + pages_needing_ocr: [4, 5], + }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { text: 'readable pages', chars: 14, total_chars: 14 }, + page_count: 10, + }, + }) + + const { blocks } = await expandPdfAttachments([pdf()], fn) + + expect(blocks[0]).toContain('pages-needing-ocr="4,5"') + expect(blocks[0]).toContain('Pages 4, 5 hold no readable text') + }) + + /** A missing worker is the failure a person can actually act on. */ + it('names the missing worker rather than surfacing a bus error', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: new Error( + "remote error (NOT_FOUND): Function 'pdf::classify' is not registered.", + ), + }) + + const { blocks, failures } = await expandPdfAttachments([pdf()], fn) + + expect(failures).toHaveLength(1) + expect(failures[0].reason).toContain('iii worker add pdf') + expect(blocks[0]).toContain('error=') + }) + + /** A failed read must still produce a block, so the send is never blocked. */ + it('turns a failure into a placeholder block instead of throwing', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: new Error('boom'), + }) + + const { blocks, failures } = await expandPdfAttachments([pdf()], fn) + + expect(blocks).toHaveLength(1) + expect(blocks[0]).toContain('path="report.pdf"') + expect(failures[0].reason).toBe('boom') + }) + + it('reports the documents it dropped over the per-send cap', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { document_type: 'text_based', page_count: 1 }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { text: 'x', chars: 1, total_chars: 1 }, + page_count: 1, + }, + }) + + const many = Array.from({ length: MAX_PDFS_PER_SEND + 2 }, (_, i) => + pdf(`doc-${i}.pdf`), + ) + const { blocks, failures } = await expandPdfAttachments(many, fn) + + expect(blocks).toHaveLength(MAX_PDFS_PER_SEND + 2) + expect(failures).toHaveLength(2) + expect(failures[0].reason).toContain('per message') + }) + + /** + * A conversation reloaded from history keeps the chip but not the bytes. + * Re-reading is not this function's job, and it must not error. + */ + it('skips an attachment whose file is gone', async () => { + const [fn] = trigger({}) + const withoutFile: Attachment = { + id: 'old', + name: 'old.pdf', + size: 10, + type: 'application/pdf', + } + const result = await expandPdfAttachments([withoutFile], fn) + expect(result).toEqual({ blocks: [], read: [], failures: [] }) + expect(fn.mock.calls).toHaveLength(0) + }) + + /** + * The expansion runs before the model is called, so it never appears as a + * function call in the transcript. This summary is the only place a person + * can see that the document was read. + */ + it('reports what it made of the document, for the chip', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { + document_type: 'text_based', + page_count: 8, + elapsed_ms: 15, + }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { text: 'x'.repeat(500), chars: 500, total_chars: 5932 }, + page_count: 8, + elapsed_ms: 72, + }, + }) + + const { read } = await expandPdfAttachments([pdf()], fn) + + expect(read).toHaveLength(1) + expect(read[0]).toMatchObject({ + id: 'report.pdf', + pages: 8, + chars: 5932, + elapsedMs: 87, + needsOcr: false, + truncated: false, + }) + expect(summaryLabel('report.pdf', read[0])).toBe( + 'report.pdf · 8 pages · 5,932 chars · 87 ms', + ) + }) + + it('summarizes a scan as unreadable rather than as zero characters', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { + document_type: 'scanned', + page_count: 3, + elapsed_ms: 9, + }, + }) + + const { read } = await expandPdfAttachments([pdf()], fn) + + expect(read[0].needsOcr).toBe(true) + expect(read[0].chars).toBeUndefined() + expect(summaryLabel('scan.pdf', read[0])).toBe( + 'scan.pdf · 3 pages · no readable text · 9 ms', + ) + }) + + it('marks a truncated extract so the count is not read as the whole document', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { document_type: 'text_based', page_count: 400 }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { text: 'y', chars: 1, total_chars: 900_000, truncated: true }, + page_count: 400, + elapsed_ms: 39_700, + }, + }) + + const { read } = await expandPdfAttachments([pdf()], fn) + + expect(read[0].truncated).toBe(true) + expect(summaryLabel('big.pdf', read[0])).toContain('900,000+ chars') + }) + + it('escapes quotes in a file name rather than breaking the header', async () => { + const [fn] = trigger({ + [CLASSIFY_FUNCTION_ID]: { document_type: 'text_based', page_count: 1 }, + [TO_MARKDOWN_FUNCTION_ID]: { + body: { text: 'x', chars: 1, total_chars: 1 }, + page_count: 1, + }, + }) + + const { blocks } = await expandPdfAttachments([pdf('a"b.pdf')], fn) + expect(blocks[0]).toContain('path="a"b.pdf"') + }) +}) diff --git a/console/web/src/lib/pdf-attachments.ts b/console/web/src/lib/pdf-attachments.ts new file mode 100644 index 000000000..83ac35164 --- /dev/null +++ b/console/web/src/lib/pdf-attachments.ts @@ -0,0 +1,341 @@ +/** + * PDF attachment expansion for the composer send path. + * + * A PDF attached in the composer used to reach the agent as nothing at all: + * `AttachmentButton` builds a preview only for small text and image files, and + * the only thing the send path forwards is text blocks. The agent then answered + * as though no document had been given to it. + * + * At send time each attached PDF is read by the `pdf` worker on the machine and + * appended as an `` text block — the same envelope + * `#file()` mentions use, so the transcript, the chip renderer, and the + * model all see a shape they already understand. + * + * Classification comes first because it decides whether extraction is worth + * doing: a scan has no text to extract, and a block saying so is far more useful + * to the model than an empty one. Failures never block the send. + */ + +import { getIiiClient } from '@/lib/iii-client' +import type { Attachment } from '@/types/chat' + +export const CLASSIFY_FUNCTION_ID = 'pdf::classify' +export const TO_MARKDOWN_FUNCTION_ID = 'pdf::to-markdown' + +/** Max PDFs expanded per send; extras are reported, never silently dropped. */ +export const MAX_PDFS_PER_SEND = 4 + +/** + * Characters of markdown inlined per document. A long report would otherwise + * consume the context the question needed. The block says when it stops short, + * and the agent can call `pdf::to-markdown` itself with a page filter. + */ +export const MAX_MARKDOWN_CHARS = 20_000 + +/** + * Largest document read from the composer. Encoding happens in the browser, so + * an enormous file is a frozen tab before the worker ever sees it, and its own + * ceiling would reject it anyway. Refuse it here with an explanation instead. + */ +export const MAX_PDF_BYTES = 64 * 1024 * 1024 + +const ATTACHED_FILE_PREFIX = '` block per expanded document, in input order. */ + blocks: string[] + /** One entry per document actually read, for the message chips. */ + read: PdfReadSummary[] + failures: PdfExpansionFailure[] +} + +/** Whether an attachment is a PDF, by declared type or by extension. */ +export function isPdfAttachment(attachment: Attachment): boolean { + return ( + attachment.type === 'application/pdf' || + attachment.name.toLowerCase().endsWith('.pdf') + ) +} + +// --- wire subset of the pdf worker --------------------------------------- + +interface ClassifyWire { + document_type?: 'text_based' | 'scanned' | 'image_based' | 'mixed' + page_count?: number + pages_needing_ocr?: number[] + ocr_reasons?: Array<{ page: number; reasons: string[] }> + elapsed_ms?: number +} + +interface MarkdownWire { + body?: { + text?: string + chars?: number + total_chars?: number + truncated?: boolean + } + page_count?: number + pages_with_tables?: number[] + has_encoding_issues?: boolean + elapsed_ms?: number +} + +type TriggerFn = ( + functionId: string, + payload: Record, +) => Promise + +/** + * Base64 without building one enormous argument list. + * + * `String.fromCharCode(...bytes)` overflows the call stack somewhere around a + * megabyte, which is a small PDF. Chunking keeps it linear and bounded. + */ +async function fileToBase64(file: File): Promise { + const bytes = new Uint8Array(await file.arrayBuffer()) + const CHUNK = 0x8000 + let binary = '' + for (let i = 0; i < bytes.length; i += CHUNK) { + binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)) + } + return btoa(binary) +} + +/** + * Read every attached PDF through the `pdf` worker and format the blocks. + * + * Attachments without their underlying `File` are skipped silently: a + * conversation reloaded from history carries the chip metadata but not the + * bytes, and re-reading a document the user attached in a previous session is + * not this function's job. + */ +export async function expandPdfAttachments( + attachments: Attachment[], + trigger?: TriggerFn, +): Promise { + const pdfs = attachments.filter((a) => isPdfAttachment(a) && a.file) + if (pdfs.length === 0) return { blocks: [], read: [], failures: [] } + + const call = + trigger ?? + (async (functionId: string, payload: Record) => { + const client = await getIiiClient() + return client.trigger(functionId, payload) + }) + + const blocks: string[] = [] + const read: PdfReadSummary[] = [] + const failures: PdfExpansionFailure[] = [] + + for (const attachment of pdfs.slice(0, MAX_PDFS_PER_SEND)) { + if (attachment.size > MAX_PDF_BYTES) { + const mb = Math.round(MAX_PDF_BYTES / (1024 * 1024)) + const reason = `larger than the ${mb} MB limit for reading a PDF in the composer` + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + continue + } + try { + const outcome = await expandOne(attachment, call) + blocks.push(outcome.block) + read.push(outcome.summary) + } catch (err) { + const reason = describeFailure(err) + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + } + } + + for (const dropped of pdfs.slice(MAX_PDFS_PER_SEND)) { + const reason = `only ${MAX_PDFS_PER_SEND} PDFs are read per message` + blocks.push(failureBlock(dropped.name, reason)) + failures.push({ name: dropped.name, reason }) + } + + return { blocks, read, failures } +} + +interface ExpandOutcome { + block: string + summary: PdfReadSummary +} + +async function expandOne( + attachment: Attachment, + call: TriggerFn, +): Promise { + const bytes_base64 = await fileToBase64(attachment.file as File) + + const classified = (await call(CLASSIFY_FUNCTION_ID, { + bytes_base64, + })) as ClassifyWire + const type = classified.document_type ?? 'mixed' + const pages = classified.page_count ?? 0 + const classifyMs = classified.elapsed_ms ?? 0 + + const unreadable = (): ExpandOutcome => ({ + block: scannedBlock(attachment.name, pages, classified), + summary: { + id: attachment.id, + pages, + elapsedMs: classifyMs, + needsOcr: true, + truncated: false, + }, + }) + + // A scan has nothing to extract. Say so in the block: the model needs to know + // the document was read and found unreadable, not that it was never given one. + if (type === 'scanned' || type === 'image_based') return unreadable() + + const converted = (await call(TO_MARKDOWN_FUNCTION_ID, { + bytes_base64, + max_chars: MAX_MARKDOWN_CHARS, + })) as MarkdownWire + + const body = converted.body ?? {} + const text = body.text ?? '' + if (text.trim().length === 0) return unreadable() + + const attrs = [ + `path="${escapeAttr(attachment.name)}"`, + `size="${attachment.size}"`, + `pages="${converted.page_count ?? pages}"`, + 'format="pdf-markdown"', + ] + if (body.truncated) { + attrs.push('truncated="true"') + attrs.push(`total-chars="${body.total_chars ?? text.length}"`) + } + if (converted.has_encoding_issues) attrs.push('encoding-issues="true"') + + const needsOcr = classified.pages_needing_ocr ?? [] + if (needsOcr.length > 0) { + attrs.push(`pages-needing-ocr="${needsOcr.join(',')}"`) + } + + const notes: string[] = [] + if (body.truncated) { + notes.push( + `This is the first ${body.chars ?? text.length} of ${ + body.total_chars ?? '?' + } characters. Call ${TO_MARKDOWN_FUNCTION_ID} with a pages filter for the rest.`, + ) + } + if (needsOcr.length > 0) { + notes.push( + `Pages ${needsOcr.join(', ')} hold no readable text and are missing from this extract.`, + ) + } + if (converted.has_encoding_issues) { + notes.push('Font encodings decoded badly; treat this text as unreliable.') + } + + const preamble = notes.length > 0 ? `${notes.join(' ')}\n\n` : '' + return { + block: `${ATTACHED_FILE_PREFIX}${attrs.join(' ')}>\n${preamble}${text}\n`, + summary: { + id: attachment.id, + pages: converted.page_count ?? pages, + chars: body.total_chars ?? text.length, + elapsedMs: classifyMs + (converted.elapsed_ms ?? 0), + needsOcr: needsOcr.length > 0, + truncated: body.truncated === true, + }, + } +} + +/** + * One line for the chip on the sent message: what the worker made of the + * document, and how fast. This is the only place a person can see that the PDF + * was read at all, because the expansion happens before the model is called and + * so never appears as a function call in the transcript. + */ +export function summaryLabel(name: string, summary: PdfReadSummary): string { + const parts = [`${summary.pages} page${summary.pages === 1 ? '' : 's'}`] + if (summary.needsOcr && summary.chars === undefined) { + parts.push('no readable text') + } else if (summary.chars !== undefined) { + parts.push( + `${summary.chars.toLocaleString('en-US')}${summary.truncated ? '+' : ''} chars`, + ) + } + parts.push(`${summary.elapsedMs} ms`) + return `${name} · ${parts.join(' · ')}` +} + +function scannedBlock( + name: string, + pages: number, + classified: ClassifyWire, +): string { + const reasons = [ + ...new Set((classified.ocr_reasons ?? []).flatMap((r) => r.reasons)), + ] + const why = reasons.length > 0 ? ` (${reasons.join(', ')})` : '' + const attrs = [ + `path="${escapeAttr(name)}"`, + `pages="${pages}"`, + 'format="pdf-markdown"', + 'needs-ocr="true"', + ] + return ( + `${ATTACHED_FILE_PREFIX}${attrs.join(' ')}>\n` + + `This PDF holds no extractable text${why}. It is a scan or an image, so ` + + `every one of its ${pages} pages would need OCR. Nothing was extracted. Do ` + + `not claim the document is empty — say it could not be read as text.\n` + + `` + ) +} + +function failureBlock(name: string, reason: string): string { + return `${ATTACHED_FILE_PREFIX}path="${escapeAttr(name)}" error="${escapeAttr(reason)}" />` +} + +/** + * The one failure worth naming precisely: the worker is not installed. Anything + * else surfaces as-is, trimmed. + */ +function describeFailure(err: unknown): string { + const message = err instanceof Error ? err.message : String(err) + if (/not registered|NOT_FOUND|function .* not found/i.test(message)) { + return 'the pdf worker is not running — install it with `iii worker add pdf`' + } + return message.length > 160 ? `${message.slice(0, 157)}…` : message +} + +/** + * `>` has to be escaped as well as `&` and `"`. The header is parsed by finding + * the first `>`, so a file name containing one would cut the header short and + * lose every attribute after it. + */ +function escapeAttr(value: string): string { + return value + .replaceAll('&', '&') + .replaceAll('"', '"') + .replaceAll('>', '>') +} diff --git a/console/web/src/types/chat.ts b/console/web/src/types/chat.ts index 7a87265f7..46062c667 100644 --- a/console/web/src/types/chat.ts +++ b/console/web/src/types/chat.ts @@ -49,6 +49,13 @@ export interface Attachment { type: string /** present only for previewable text/image attachments under ~1MB */ dataUrl?: string + /** + * The picked file, for attachment kinds a worker reads at send time (PDFs go + * through `pdf::to-markdown` — see `lib/pdf-attachments.ts`). Browser-only and + * deliberately not persisted: a conversation reloaded from history keeps the + * chip, not the bytes. + */ + file?: File } interface BaseMessage { diff --git a/iii-permissions.yaml b/iii-permissions.yaml index b706f2f00..af2b5179e 100644 --- a/iii-permissions.yaml +++ b/iii-permissions.yaml @@ -184,6 +184,10 @@ rules: - '!openwiki::on-turn-completed' - '!openwiki::on-config-change' + # pdf: internal target. The hot-reload hook follows the same pattern as the + # other on-config-change denies. + - '!pdf::on-config-change' + # Read-only / introspection (extend below for your tools). - state::get - state::list @@ -307,3 +311,14 @@ rules: # HTTP-trigger verification; it enforces size/timeout caps and server-side # SSRF protection, so it is allowed by default. - web::fetch + + # pdf: every function is a pure read of a document the agent could already + # reach through the filesystem scope, and reaching it any other way returns + # binary noise. Nothing here writes, spends, or leaves the machine, and each + # response is capped by the worker's own configuration. Gating these would + # mean an approval prompt to read a file the agent is already allowed to open. + - pdf::classify + - pdf::to-markdown + - pdf::extract-text + - pdf::extract-items + - pdf::extract-regions diff --git a/pdf/Cargo.lock b/pdf/Cargo.lock new file mode 100644 index 000000000..452574ad2 --- /dev/null +++ b/pdf/Cargo.lock @@ -0,0 +1,2844 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "iii-console-ui" +version = "0.1.0" +dependencies = [ + "iii-sdk", + "schemars", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "iii-helpers" +version = "0.21.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bdc7bbc3abfde934a62cdc5d3045adf52914dfc1ed6c20f8af691fc561dc55" +dependencies = [ + "futures-util", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "reqwest", + "schemars", + "serde", + "serde_json", + "sysinfo", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "iii-sdk" +version = "0.21.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07dd060fddcc9153b0dd07c038a14cf172ce15ce1d4edb98155563ed55b2caba" +dependencies = [ + "async-trait", + "futures-util", + "hostname", + "iii-helpers", + "reqwest", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lopdf" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "rand 0.10.2", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand 0.9.5", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "pdf" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "clap", + "iii-console-ui", + "iii-sdk", + "include_dir", + "pdf-inspector", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "pdf-inspector" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7475018de0880b394b7cc50f871fac0c010aa411dcd14c64074a3e640a4c05c" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.119", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/pdf/Cargo.toml b/pdf/Cargo.toml new file mode 100644 index 000000000..2a2e6d61f --- /dev/null +++ b/pdf/Cargo.toml @@ -0,0 +1,47 @@ +[workspace] + +[package] +name = "pdf" +version = "0.1.0" +edition = "2021" +description = "PDF worker for iii — local classification, markdown conversion, positioned text and table extraction, and per-page OCR routing (pdf::* functions)" +license = "Apache-2.0" +repository = "https://github.com/iii-hq/workers" +publish = false + +[lib] +name = "pdf" +path = "src/lib.rs" + +[[bin]] +name = "pdf" +path = "src/main.rs" + +[dependencies] +iii-sdk = "=0.21.6" +# Worker-side injectable console UI (content function + console:script/style +# triggers + hot-reload watcher) — direct link, never published. +iii-console-ui = { path = "../crates/console-ui" } +# The parser. Pure Rust, no C libraries, no subprocess, no network. +pdf-inspector = "=0.1.7" +# The CJK CMap payload the parser resolves at runtime; embedded from OUT_DIR so +# a cross-compiled binary keeps them (see src/cmaps.rs). +include_dir = "0.7" +base64 = "0.22" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +clap = { version = "4", features = ["derive", "env"] } +schemars = "0.8" + +[build-dependencies] +# `cargo metadata` parsing in build.rs, to locate the parser's CMap payload. +serde_json = "1" + +[dev-dependencies] +base64 = "0.22" +tempfile = "3" diff --git a/pdf/README.md b/pdf/README.md new file mode 100644 index 000000000..6ab5ba989 --- /dev/null +++ b/pdf/README.md @@ -0,0 +1,156 @@ +# pdf + +Read PDFs on the machine, with no OCR service and no API key. This worker +classifies a document in about twenty milliseconds — is this real text, or a +photograph of a page? — converts text-based documents to markdown that keeps +their headings, lists, links and tables, and reports exactly which pages still +need OCR and why. It also exposes the layout underneath: where every run of +characters sits, and what the text is inside a given box on a page. Nothing is +uploaded, and a long document is capped rather than dumped, so a report does not +swallow the context an agent needed for the answer. + +It ships a console page too. Drop a PDF in and see exactly what the agent sees: +the verdict, the per-page OCR decision, and the extracted markdown. + +## Install + +```bash +iii worker add pdf +``` + +## Quickstart + +Classify first. It is cheap, and it decides whether anything else is worth +doing: extraction on a scan returns nothing, and without the verdict an empty +result is indistinguishable from an empty document. + +```rust +use iii_sdk::{register_worker, InitOptions}; +use iii_sdk::protocol::TriggerRequest; +use serde_json::json; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let iii = register_worker("ws://localhost:49134", InitOptions::default()); + + let verdict = iii.trigger(TriggerRequest { + function_id: "pdf::classify".into(), + payload: json!({ "path": "/tmp/report.pdf" }), + action: None, + timeout_ms: Some(30_000), + }).await?; + // { "document_type": "text_based", "confidence": 1.0, "page_count": 40, + // "pages_needing_ocr": [], "ocr_reasons": [], "elapsed_ms": 18, … } + + let markdown = iii.trigger(TriggerRequest { + function_id: "pdf::to-markdown".into(), + payload: json!({ "path": "/tmp/report.pdf", "pages": [1, 2, 3] }), + action: None, + timeout_ms: Some(60_000), + }).await?; + // { "body": { "text": "# Quarterly Report\n…", "chars": 5693, + // "total_chars": 5693, "truncated": false }, … } + + println!("{markdown:#?}"); + Ok(()) +} +``` + +A document with no path goes in as `bytes_base64` instead. An encrypted one +takes a `password` on `pdf::classify` and `pdf::to-markdown`. + +### Reading the verdict + +`document_type` is `text_based`, `scanned`, `image_based` or `mixed`. The +document-level answer is not the whole story: a two-hundred-page report with a +scanned cover is not a scanned document, and treating it as one sends the whole +thing to an OCR service for the sake of one page. `pages_needing_ocr` and +`ocr_reasons` carry the per-page decision: + +| Reason | What it means | +|---|---| +| `scanned` | A raster page. It needs a vision model. | +| `no_text` | Nothing extractable and nothing to OCR. Often a blank page. | +| `vector_text` | Characters drawn as outlines rather than text. Unreadable as characters. | +| `suspected_garbled_text` | A text layer that decodes to nonsense. Do not trust it, whatever the document type says. | + +### Response caps + +Every text-bearing response is capped and says so. `truncated: true` with a +`total_chars` far above `chars` means you are holding a fragment. + +The cheap fix is `pages`, not a bigger cap: conversion cost scales with the +document, so narrowing to the pages you need is faster as well as smaller. A +four-hundred-page report takes tens of seconds to convert whole and +milliseconds a page at a time. + +`max_chars: 0` lifts the cap entirely. That belongs in a pipeline moving a +document to storage, not in a call whose result lands in a conversation. + +### Reading a box on a page + +When a vision model has located a region and you want the real characters +rather than its transcription: + +```json +{ + "path": "/tmp/invoice.pdf", + "regions": [{ "page": 1, "boxes": [[320.0, 640.0, 560.0, 700.0]] }], + "mode": "text" +} +``` + +`mode: "table"` runs table detection over the same box and returns a markdown +table instead. + +### Two conventions worth knowing + +Page numbers are 1-indexed everywhere on this surface, in requests and +responses. + +Coordinates are not uniform, and each response states which it used. +`pdf::extract-items` reports PDF points from the **bottom** left, the PDF +convention. `pdf::extract-regions` takes boxes in PDF points from the **top** +left, which is what a layout model produces. Getting this wrong is silent: text +comes back, just from the wrong end of the page. + +## Configuration + +Configuration lives in the `configuration` worker under the id `pdf` and every +field hot-reloads. Nothing here needs a restart. + +```yaml +max_input_bytes: 268435456 # largest document accepted, before parsing +max_chars: 40000 # default cap on returned text or markdown +preview_chars: 600 # leading characters shown alongside a capped body +max_items: 5000 # default cap on positioned items in one response +classify_sample_pages: 8 # pages sampled to classify; 0 scans everything +min_text_ops_per_page: 3 # text operators before a page counts as text +text_page_ratio_threshold: 0.6 # share of text pages to call a document text-based +``` + +The three detection fields are the ones worth understanding. Sampling is what +keeps classification at tens of milliseconds on a four-hundred-page file; it +also means the verdict comes from part of the document, which is why every +response reports `pages_sampled`. Raise `classify_sample_pages`, or set it to +`0`, when a borderline mixed document needs settling. + +Defaults live in [`src/config.rs`](src/config.rs). + +## Called on demand + +This worker registers no harness hook and injects nothing into any prompt. A +conversation that never touches a document never pays for it, and there is no +per-turn cost to having it installed. An agent finds it the ordinary way, +through the function registry and [`skills/SKILL.md`](skills/SKILL.md); a +person finds it through the console page. + +## What this worker does not do + +It does not rasterize pages, so it cannot OCR anything. Scanned and image-based +documents get classified and routed, not read. Image content is reported as a +placeholder with a real bounding box and no pixels. + +It is a parser, not a renderer: it walks the document's content streams and +reconstructs the geometry, which is why it is fast and why it needs no service +behind it. diff --git a/pdf/build.rs b/pdf/build.rs new file mode 100644 index 000000000..c78e8ad79 --- /dev/null +++ b/pdf/build.rs @@ -0,0 +1,292 @@ +//! Build script for the `pdf` worker. +//! +//! Three jobs: +//! +//! 1. Forward the build-time target triple to the binary as `env!("TARGET")` +//! (used by `manifest.rs` for the registry `supported_targets` field). +//! 2. Stage the parser's CJK CMap payload into `OUT_DIR` so `src/cmaps.rs` can +//! embed it with `include_dir!`. The parser resolves those files at runtime +//! relative to its OWN `CARGO_MANIFEST_DIR`, which for a dependency is a +//! path inside the build machine's cargo registry — a directory that does +//! not exist on the machine running a released binary. Without this staging +//! step CID fonts with no ToUnicode table silently decode to nothing. +//! 3. Ensure the injected console UI assets exist: `src/ui.rs` embeds +//! `ui/dist/page.js` and `ui/dist/styles.css` via `include_str!`, so if +//! either is missing or stale we run `pnpm install && pnpm build` inside +//! `ui/` first (the `state` worker's precedent). Set `SKIP_UI_BUILD=1` to +//! use the existing `ui/dist/` outputs as-is. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::SystemTime; + +fn main() { + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); + + stage_cmaps(); + build_ui(); +} + +// --------------------------------------------------------------------------- +// CMaps +// --------------------------------------------------------------------------- + +/// Copy `pdf-inspector`'s `external/bcmaps` into `$OUT_DIR/bcmaps`. +/// +/// The source directory is resolved through `cargo metadata` rather than by +/// guessing at a registry path, so a vendored, patched, or path-overridden +/// dependency stages the right files. +fn stage_cmaps() { + let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR is set by cargo")); + let dest = out_dir.join("bcmaps"); + + let src = locate_dependency_cmaps(); + println!("cargo:rerun-if-changed={}", src.display()); + + if dest.is_dir() && dir_file_count(&dest) == dir_file_count(&src) { + return; + } + + if dest.exists() { + std::fs::remove_dir_all(&dest) + .unwrap_or_else(|e| panic!("failed to clear {}: {e}", dest.display())); + } + std::fs::create_dir_all(&dest) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", dest.display())); + + let entries = + std::fs::read_dir(&src).unwrap_or_else(|e| panic!("failed to read {}: {e}", src.display())); + let mut copied = 0usize; + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_file() { + continue; + } + let name = path.file_name().expect("directory entry has a file name"); + std::fs::copy(&path, dest.join(name)) + .unwrap_or_else(|e| panic!("failed to copy {}: {e}", path.display())); + copied += 1; + } + + assert!( + copied > 0, + "staged zero CMap files from {} — the parser dependency layout changed", + src.display() + ); +} + +/// `/external/bcmaps`, via `cargo metadata`. +fn locate_dependency_cmaps() -> PathBuf { + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); + let output = Command::new(cargo) + .args(["metadata", "--format-version", "1"]) + .output() + .expect("failed to run `cargo metadata`"); + assert!( + output.status.success(), + "`cargo metadata` failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let metadata: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("cargo metadata emits json"); + let manifest = metadata["packages"] + .as_array() + .expect("metadata carries a packages array") + .iter() + .find(|p| p["name"] == "pdf-inspector") + .and_then(|p| p["manifest_path"].as_str()) + .unwrap_or_else(|| { + panic!("`pdf-inspector` is not in the dependency graph — cannot stage its CMaps") + }); + + let dir = Path::new(manifest) + .parent() + .expect("a manifest path has a parent") + .join("external") + .join("bcmaps"); + assert!( + dir.is_dir(), + "expected the parser's CMap directory at {} — the dependency layout changed", + dir.display() + ); + dir +} + +fn dir_file_count(dir: &Path) -> usize { + std::fs::read_dir(dir) + .map(|entries| entries.flatten().filter(|e| e.path().is_file()).count()) + .unwrap_or(0) +} + +// --------------------------------------------------------------------------- +// Console UI +// --------------------------------------------------------------------------- + +fn build_ui() { + // Both of these change what this script does, so a change in either has + // to invalidate it. + println!("cargo:rerun-if-env-changed=SKIP_UI_BUILD"); + println!("cargo:rerun-if-env-changed=PNPM"); + // `dist/` itself is not listed: include_str! reads it directly, and + // listing it would rebuild-loop on our own output. + println!("cargo:rerun-if-changed=ui/page.tsx"); + println!("cargo:rerun-if-changed=ui/styles.css"); + println!("cargo:rerun-if-changed=ui/src"); + println!("cargo:rerun-if-changed=ui/build.mjs"); + println!("cargo:rerun-if-changed=ui/package.json"); + // The lockfile lives at the workers-repo root (pnpm workspace: the ui + // project links @iii-dev/console-ui from packages/console-ui). + println!("cargo:rerun-if-changed=../pnpm-lock.yaml"); + println!("cargo:rerun-if-changed=ui/tsconfig.json"); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let ui_dir = manifest_dir.join("ui"); + let dist_assets = [ + ui_dir.join("dist").join("page.js"), + ui_dir.join("dist").join("styles.css"), + ]; + + if dist_assets + .iter() + .all(|a| a.exists() && dist_is_fresh(a, &ui_dir)) + { + return; + } + + if std::env::var_os("SKIP_UI_BUILD").is_some() { + for asset in &dist_assets { + if !asset.exists() { + panic!( + "SKIP_UI_BUILD set but {} is missing — build the UI manually \ + (cd ui && pnpm install && pnpm build) or unset the env var", + asset.display() + ); + } + } + return; + } + + let pnpm = locate_pnpm(); + + let status = Command::new(&pnpm) + .args(["install"]) + .current_dir(&ui_dir) + .status() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `pnpm install` in {}: {e}", + ui_dir.display() + ) + }); + if !status.success() { + panic!("`pnpm install` exited with {status} — see logs above"); + } + + let status = Command::new(&pnpm) + .args(["build"]) + .current_dir(&ui_dir) + .status() + .unwrap_or_else(|e| panic!("failed to spawn `pnpm build` in {}: {e}", ui_dir.display())); + if !status.success() { + panic!("`pnpm build` exited with {status} — see logs above"); + } + + for asset in &dist_assets { + if !asset.exists() { + panic!( + "`pnpm build` finished but {} is still missing — check the esbuild \ + output above", + asset.display() + ); + } + } +} + +/// `true` when the built asset is at least as new as every source that +/// contributes to it. Conservative: any I/O failure forces a rebuild. +fn dist_is_fresh(dist_asset: &Path, ui_dir: &Path) -> bool { + let Ok(dist_mtime) = dist_asset.metadata().and_then(|m| m.modified()) else { + return false; + }; + + let watched_files = [ + ui_dir.join("page.tsx"), + ui_dir.join("styles.css"), + ui_dir.join("build.mjs"), + ui_dir.join("package.json"), + ui_dir.join("../../pnpm-lock.yaml"), + ui_dir.join("tsconfig.json"), + ]; + for f in watched_files.iter() { + if !f.exists() { + continue; + } + let Ok(m) = f.metadata().and_then(|m| m.modified()) else { + return false; + }; + if m > dist_mtime { + return false; + } + } + + for dir in [ui_dir.join("src")] { + if dir.exists() && !subtree_older_than(&dir, dist_mtime) { + return false; + } + } + + true +} + +fn subtree_older_than(root: &Path, ceiling: SystemTime) -> bool { + let Ok(read) = std::fs::read_dir(root) else { + return false; + }; + for entry in read.flatten() { + let path = entry.path(); + let Ok(meta) = entry.metadata() else { + return false; + }; + if meta.is_dir() { + if !subtree_older_than(&path, ceiling) { + return false; + } + } else { + let Ok(m) = meta.modified() else { + return false; + }; + if m > ceiling { + return false; + } + } + } + true +} + +fn locate_pnpm() -> PathBuf { + if let Ok(explicit) = std::env::var("PNPM") { + return PathBuf::from(explicit); + } + let candidates = if cfg!(windows) { + ["pnpm.cmd", "pnpm.exe", "pnpm"].as_slice() + } else { + ["pnpm"].as_slice() + }; + let path = std::env::var_os("PATH").unwrap_or_default(); + for dir in std::env::split_paths(&path) { + for name in candidates { + let candidate = dir.join(name); + if candidate.is_file() { + return candidate; + } + } + } + panic!( + "pnpm not found on PATH — install Node + pnpm, or set SKIP_UI_BUILD=1 \ + after building the UI manually with `cd ui && pnpm install && pnpm build`" + ); +} diff --git a/pdf/examples/probe.rs b/pdf/examples/probe.rs new file mode 100644 index 000000000..faa74a1b0 --- /dev/null +++ b/pdf/examples/probe.rs @@ -0,0 +1,128 @@ +//! Local smoke probe: run every function against a real PDF and print the +//! shape of what comes back. +//! +//! cargo run --example probe -- [password] +//! +//! Not a test. Tests must be deterministic and fixture-committed; this is for +//! pointing the worker at a document on the machine and looking at the result. + +use pdf::config::WorkerConfig; +use pdf::functions::{classify, items, markdown, regions, text}; +use pdf::source::PdfSource; + +fn main() { + let mut args = std::env::args().skip(1); + let path = args.next().expect("usage: probe [password]"); + let password = args.next(); + + pdf::cmaps::materialize(); + let cfg = WorkerConfig::default(); + let src = || PdfSource { + path: Some(path.clone()), + bytes_base64: None, + fs_scope: None, + }; + + println!("== {path}"); + + let classified = classify::handle( + classify::Request { + source: src(), + password: password.clone(), + sample_pages: None, + }, + &cfg, + ) + .expect("classify"); + println!( + "classify: {:?} confidence {:.2} pages {} sampled {:?} need-ocr {:?} in {}ms", + classified.document_type, + classified.confidence, + classified.page_count, + classified.pages_sampled, + classified.pages_needing_ocr, + classified.elapsed_ms + ); + for reason in &classified.ocr_reasons { + println!(" page {} -> {:?}", reason.page, reason.reasons); + } + + let md = markdown::handle( + markdown::Request { + source: src(), + password: password.clone(), + pages: None, + max_chars: None, + profile: markdown::Profile::Fidelity, + include_images: false, + strip_headers_footers: true, + per_page: false, + }, + &cfg, + ) + .expect("to-markdown"); + println!( + "markdown: {} of {} chars (truncated {}) tables {:?} columns {:?} encoding-issues {} in {}ms", + md.body.chars, + md.body.total_chars, + md.body.truncated, + md.pages_with_tables, + md.pages_with_columns, + md.has_encoding_issues, + md.elapsed_ms + ); + let head: String = md.body.text.chars().take(400).collect(); + println!("--- first 400 chars ---\n{head}\n---"); + + let txt = text::handle( + text::Request { + source: src(), + max_chars: Some(0), + }, + &cfg, + ) + .expect("extract-text"); + println!("text: {} chars", txt.body.total_chars); + + let it = items::handle( + items::Request { + source: src(), + pages: Some(vec![1]), + max_items: Some(3), + }, + &cfg, + ) + .expect("extract-items"); + println!( + "items: page 1 has {} items ({} returned, {})", + it.total_count, it.count, it.coordinate_origin + ); + for item in &it.items { + println!( + " {:?} {:?} @ ({:.1},{:.1}) {}pt {}", + item.kind, item.text, item.x, item.y, item.font_size, item.font + ); + } + + let rg = regions::handle( + regions::Request { + source: src(), + regions: vec![regions::PageRegions { + page: 1, + boxes: vec![[0.0, 0.0, 612.0, 400.0]], + }], + mode: regions::Mode::Text, + }, + &cfg, + ) + .expect("extract-regions"); + let region_text = rg.pages[0].regions[0] + .text + .chars() + .take(120) + .collect::(); + println!( + "regions: {} boxes, {} unreliable ({})\n {:?}", + rg.region_count, rg.regions_needing_ocr, rg.coordinate_origin, region_text + ); +} diff --git a/pdf/iii.worker.yaml b/pdf/iii.worker.yaml new file mode 100644 index 000000000..99a4ef1ea --- /dev/null +++ b/pdf/iii.worker.yaml @@ -0,0 +1,10 @@ +iii: v1 +name: pdf +language: rust +deploy: binary +manifest: Cargo.toml +bin: pdf +tags: [pdf, document, markdown, text-extraction, ocr-routing, tables] +description: Read PDFs locally — classify text-based vs scanned, convert to markdown, extract positioned text and tables, and report which pages still need OCR. +dependencies: + configuration: "^0.21.6" diff --git a/pdf/skills/SKILL.md b/pdf/skills/SKILL.md new file mode 100644 index 000000000..1e82ccd39 --- /dev/null +++ b/pdf/skills/SKILL.md @@ -0,0 +1,81 @@ +--- +name: pdf +description: >- + Read PDFs locally without OCR or an API key — classify text-based versus + scanned in tens of milliseconds and name the pages that still need OCR, + convert to markdown with headings, lists and tables intact, and pull + positioned text or the exact characters inside a box on a page. +--- + +# pdf + +The pdf worker parses PDF documents on the machine. A PDF is not text: reading +one with a file-reading function returns binary noise and spends the context on +it, so every PDF goes through `pdf::*` instead. Parsing is local, needs no +credential, and sends nothing anywhere. + +Its first job is a routing decision. `pdf::classify` samples the document's +content streams and answers whether the pages hold real characters or are +photographs of pages, plus which individual pages cannot be read without a +vision model and why. That verdict decides whether the rest of the work is +worth doing at all, and it is what separates "this document is empty" from +"this document is a scan". + +Its second job is reading. Text-based documents convert to markdown that keeps +the shape of the original, because headings, lists and tables are recovered +from font sizes and page geometry rather than from any structure the file +promises. Underneath that sit the positions themselves, for callers that need +to know where text is and not only what it says. + +This worker is called on demand. It registers no harness hook and injects +nothing into any prompt, so a conversation that never touches a document never +pays for it. Reach for it when one appears. + +## When to Use + +- A conversation names a PDF path or hands one over: call `pdf::classify` + before anything else. Never read a PDF with a file-reading function; it + returns binary noise and spends the context on it. +- Read a document: `pdf::to-markdown`, narrowed with `pages` when it is long. +- Search or embed a document rather than read it: `pdf::extract-text`. +- Decide whether a document is worth sending to a vision model, and which of + its pages: `pdf::classify`, then read `pages_needing_ocr` and `ocr_reasons`. +- A vision model located a region and you want the real characters rather than + its transcription: `pdf::extract-regions`. +- Reason about layout, headings by size, or where a value sits on the page: + `pdf::extract-items`. + +## Boundaries + +- Nothing here rasterizes a page, so nothing here can OCR. Scanned and + image-based documents are classified and routed, never read. Image content + is reported as a placeholder with a real box and no pixels. +- `suspected_garbled_text` in `ocr_reasons` means the text layer decodes to + nonsense. Do not trust the extraction, whatever `document_type` says. +- Responses are capped. `truncated: true` with a much larger `total_chars` + means you hold a fragment and must not answer from it. Narrow with `pages` + rather than raising the cap: conversion cost scales with the document, so a + page filter is faster as well as smaller. `max_chars: 0` lifts the cap and + belongs in a pipeline moving a document to storage, not in a call whose + result lands in the conversation. +- Encrypted documents take a `password` on `pdf::classify` and + `pdf::to-markdown` only. The other three cannot decrypt and say so. +- Page numbers are 1-indexed everywhere, in requests and responses. +- Coordinates differ by function and every response states which it used: + `pdf::extract-items` reports PDF points from the bottom left, + `pdf::extract-regions` takes boxes in PDF points from the top left. Assuming + the wrong one returns text from the wrong end of the page with no error. + +## Functions + +- `pdf::classify` — the routing call. Document type, confidence, page count, + the 1-indexed pages needing OCR, and a machine-readable reason per page. +- `pdf::to-markdown` — markdown with headings, lists, links and tables + recovered; optional page filter, per-page output, and a fidelity or compact + profile. +- `pdf::extract-text` — plain text, no structure recovery. Cheaper than + markdown when the result will be searched or embedded. +- `pdf::extract-items` — every positioned run of characters with its box, + font, size, and recovered bold, italic, underline and strikeout. +- `pdf::extract-regions` — the text, or a markdown table, inside given boxes + on given pages. diff --git a/pdf/src/cmaps.rs b/pdf/src/cmaps.rs new file mode 100644 index 000000000..ca9be3aad --- /dev/null +++ b/pdf/src/cmaps.rs @@ -0,0 +1,168 @@ +//! CJK CMap payload, embedded in this binary. +//! +//! Why this module exists: the parser loads its built-in CJK CMaps from disk at +//! runtime, resolving them against the `CARGO_MANIFEST_DIR` recorded when the +//! parser crate itself was compiled. Consumed as a dependency and cross +//! compiled in CI, that is a path inside the build machine's cargo registry — +//! it does not exist on the machine running a released binary. The lookup then +//! finds nothing and CID fonts that carry no ToUnicode table decode to empty +//! text. Nothing crashes and nothing is logged, so a released worker would +//! quietly return an empty document for a class of Chinese, Japanese and Korean +//! PDFs while passing every test built from source. +//! +//! The fix, in three steps: `build.rs` stages the parser's CMap directory into +//! `OUT_DIR`, this module embeds that directory into the binary, and +//! [`materialize`] writes it to a cache directory on first use and points the +//! parser at it through the `PDF_INSPECTOR_BCMAPS_DIR` environment variable, +//! which the parser checks ahead of its compiled-in path. + +use std::path::{Path, PathBuf}; + +use include_dir::{include_dir, Dir}; + +/// Staged by `build.rs` from the parser's own `external/bcmaps`. +static CMAPS: Dir<'_> = include_dir!("$OUT_DIR/bcmaps"); + +/// The environment variable the parser consults before its compiled-in path. +const CMAP_DIR_ENV: &str = "PDF_INSPECTOR_BCMAPS_DIR"; + +/// Write the embedded CMaps to a cache directory and point the parser at them. +/// +/// Best effort by design: a read-only or full disk costs CJK fidelity, which is +/// worth a warning, not a failed boot. An operator who has already set +/// `PDF_INSPECTOR_BCMAPS_DIR` keeps their directory untouched. +pub fn materialize() { + if let Some(existing) = std::env::var_os(CMAP_DIR_ENV) { + tracing::info!( + dir = %Path::new(&existing).display(), + "{CMAP_DIR_ENV} already set; using the operator's CMap directory" + ); + return; + } + + let dir = cache_dir(); + match write_all(&dir) { + Ok(written) => { + std::env::set_var(CMAP_DIR_ENV, &dir); + tracing::info!( + dir = %dir.display(), + files = written, + "CJK CMaps materialized" + ); + } + Err(e) => { + tracing::warn!( + dir = %dir.display(), + error = %e, + "failed to materialize CJK CMaps; CID fonts without a ToUnicode table \ + will extract as empty text" + ); + } + } +} + +/// Per-version cache directory, so a worker upgrade cannot serve a previous +/// release's payload out of a warm cache. +fn cache_dir() -> PathBuf { + let root = std::env::var_os("XDG_CACHE_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".cache"))) + .unwrap_or_else(std::env::temp_dir); + root.join("iii") + .join("pdf") + .join(format!("bcmaps-{}", env!("CARGO_PKG_VERSION"))) +} + +/// Write every embedded file that is missing or the wrong size. Returns the +/// number of files written. +fn write_all(dir: &Path) -> std::io::Result { + std::fs::create_dir_all(dir)?; + let mut written = 0usize; + for file in CMAPS.files() { + let name = file + .path() + .file_name() + .expect("an embedded file has a file name"); + let dest = dir.join(name); + if dest + .metadata() + .is_ok_and(|m| m.len() == file.contents().len() as u64) + { + continue; + } + std::fs::write(&dest, file.contents())?; + written += 1; + } + Ok(written) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The whole point of the module. A staging regression shows up here rather + /// than as silently empty CJK text in production. + #[test] + fn cmaps_are_embedded() { + let count = CMAPS.files().count(); + assert!( + count > 100, + "expected the parser's full CMap payload, embedded {count} files" + ); + } + + #[test] + fn embedded_cmaps_are_nonempty() { + for file in CMAPS.files() { + assert!( + !file.contents().is_empty(), + "{} staged as an empty file", + file.path().display() + ); + } + } + + #[test] + fn known_cjk_cmaps_are_present() { + let names: Vec = CMAPS + .files() + .filter_map(|f| { + f.path() + .file_name() + .map(|n| n.to_string_lossy().to_string()) + }) + .collect(); + // One per CJK script: simplified Chinese, Japanese, Korean, traditional + // Chinese. A partial staging would still pass the count check above. + for expected in [ + "UniGB-UCS2-H.bcmap", + "UniJIS-UCS2-H.bcmap", + "UniKS-UCS2-H.bcmap", + "UniCNS-UCS2-H.bcmap", + ] { + assert!( + names.iter().any(|n| n == expected), + "missing {expected} — the CMap payload is incomplete" + ); + } + } + + #[test] + fn write_all_is_idempotent() { + let tmp = tempfile::tempdir().expect("temp dir"); + let first = write_all(tmp.path()).expect("first write"); + assert_eq!(first, CMAPS.files().count()); + let second = write_all(tmp.path()).expect("second write"); + assert_eq!(second, 0, "a warm cache must not be rewritten"); + } + + #[test] + fn cache_dir_is_version_scoped() { + let dir = cache_dir(); + assert!( + dir.ends_with(format!("bcmaps-{}", env!("CARGO_PKG_VERSION"))), + "cache dir must be version scoped: {}", + dir.display() + ); + } +} diff --git a/pdf/src/config.rs b/pdf/src/config.rs new file mode 100644 index 000000000..b134e9226 --- /dev/null +++ b/pdf/src/config.rs @@ -0,0 +1,303 @@ +//! Operator-facing runtime configuration. +//! +//! The authoritative value comes from the `configuration` worker at boot +//! (see [`crate::configuration`]); a `--config` YAML file, when passed, only +//! SEEDS the initial registration. Every field has a serde default so an empty +//! object yields a fully-populated config, and every field is a per-call +//! tuning knob read from the live snapshot — nothing here requires a restart. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Root config shape. Unknown keys are rejected so a typo'd field fails loudly +/// instead of silently running the default. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WorkerConfig { + /// Largest PDF accepted, in bytes. Guards against a path or a base64 blob + /// large enough to exhaust memory during parsing. + #[serde(default = "default_max_input_bytes")] + pub max_input_bytes: u64, + + /// Default cap on the characters of extracted text or markdown returned in + /// one response. A capped response still reports the true total, so the + /// caller knows what it did not receive. Per-call `max_chars` overrides + /// this; `0` means no cap. + #[serde(default = "default_max_chars")] + pub max_chars: usize, + + /// Characters of leading content included as a preview alongside a capped + /// or omitted body. + #[serde(default = "default_preview_chars")] + pub preview_chars: usize, + + /// Largest number of positioned text items returned by `pdf::extract-items` + /// in one response. A dense page carries thousands. + #[serde(default = "default_max_items")] + pub max_items: usize, + + /// Pages sampled when classifying a document. Sampling is what keeps + /// classification at tens of milliseconds on a large file. `0` scans every + /// page. + #[serde(default = "default_classify_sample_pages")] + pub classify_sample_pages: usize, + + /// Text-drawing operators a page needs before it counts as a text page. + #[serde(default = "default_min_text_ops_per_page")] + pub min_text_ops_per_page: usize, + + /// Share of sampled pages (0.0 to 1.0) that must look like text pages + /// before the whole document is called text-based. + #[serde(default = "default_text_page_ratio_threshold")] + pub text_page_ratio_threshold: f32, +} + +fn default_max_input_bytes() -> u64 { + 256 * 1024 * 1024 +} + +fn default_max_chars() -> usize { + 40_000 +} + +fn default_preview_chars() -> usize { + 600 +} + +fn default_max_items() -> usize { + 5_000 +} + +fn default_classify_sample_pages() -> usize { + 8 +} + +fn default_min_text_ops_per_page() -> usize { + 3 +} + +fn default_text_page_ratio_threshold() -> f32 { + 0.6 +} + +impl Default for WorkerConfig { + fn default() -> Self { + Self { + max_input_bytes: default_max_input_bytes(), + max_chars: default_max_chars(), + preview_chars: default_preview_chars(), + max_items: default_max_items(), + classify_sample_pages: default_classify_sample_pages(), + min_text_ops_per_page: default_min_text_ops_per_page(), + text_page_ratio_threshold: default_text_page_ratio_threshold(), + } + } +} + +impl WorkerConfig { + /// Parse a seed config from YAML, expanding `${NAME}` against the process + /// env FIRST (the seed file is the only path that needs expansion — values + /// fetched from `configuration::get` are already env-expanded by the + /// configuration worker), then deserializing. + pub fn from_yaml(yaml: &str) -> Result { + let expanded = expand_env(yaml); + let parsed: Self = + serde_yaml::from_str(&expanded).map_err(|e| format!("yaml parse: {e}"))?; + parsed.validate() + } + + /// Reject values that parse but cannot mean anything. + /// + /// `text_page_ratio_threshold` is a share, so a value outside 0.0 to 1.0 + /// silently makes every document classify the same way. Better to refuse + /// the config than to run on it. + fn validate(self) -> Result { + let ratio = self.text_page_ratio_threshold; + if !ratio.is_finite() || !(0.0..=1.0).contains(&ratio) { + return Err(format!( + "text_page_ratio_threshold must be a share between 0.0 and 1.0, got {ratio}" + )); + } + Ok(self) + } + + /// Read and parse a YAML seed file (env-expanded — see [`Self::from_yaml`]). + pub fn from_file(path: &str) -> Result { + let raw = std::fs::read_to_string(path).map_err(|e| format!("read {path}: {e}"))?; + Self::from_yaml(&raw) + } + + /// Parse a config from a JSON value already env-expanded by the + /// configuration worker. Does NOT run [`expand_env`] (double expansion + /// would be a bug) and tolerates a zero-field object (serde defaults fill + /// in). + pub fn from_json(value: &Value) -> Result { + let parsed: Self = + serde_json::from_value(value.clone()).map_err(|e| format!("json parse: {e}"))?; + parsed.validate() + } + + pub fn to_json(&self) -> Value { + serde_json::to_value(self).expect("WorkerConfig serializes") + } + + /// The JSON Schema registered with the `configuration` worker. Field + /// doc-comments become property descriptions; the shipped defaults are + /// attached as a top-level `example`. + pub fn json_schema() -> Value { + let root = schemars::schema_for!(WorkerConfig); + let mut schema = + serde_json::to_value(&root.schema).expect("WorkerConfig JSON Schema serializes"); + if let Some(obj) = schema.as_object_mut() { + if !root.definitions.is_empty() { + obj.insert( + "definitions".into(), + serde_json::to_value(&root.definitions).expect("definitions serialize"), + ); + } + obj.insert("example".into(), WorkerConfig::default().to_json()); + } + schema + } + + /// Effective character cap for one response: the per-call override when + /// present, else the configured default. `0` means uncapped. + pub fn effective_max_chars(&self, requested: Option) -> usize { + requested.unwrap_or(self.max_chars) + } +} + +/// Expand `${NAME}` and `${NAME:default}` against the process env. An unset +/// variable with no default expands to the empty string, matching the +/// configuration worker's own expansion. +fn expand_env(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut rest = input; + while let Some(start) = rest.find("${") { + out.push_str(&rest[..start]); + let after = &rest[start + 2..]; + match after.find('}') { + Some(end) => { + let spec = &after[..end]; + let (name, fallback) = match spec.split_once(':') { + Some((n, d)) => (n, Some(d)), + None => (spec, None), + }; + match (std::env::var(name), fallback) { + (Ok(v), _) => out.push_str(&v), + (Err(_), Some(d)) => out.push_str(d), + (Err(_), None) => { + tracing::warn!(var = %name, "config references undefined env var") + } + } + rest = &after[end + 1..]; + } + None => { + out.push_str("${"); + rest = after; + } + } + } + out.push_str(rest); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_yaml_yields_defaults() { + let cfg = WorkerConfig::from_yaml("{}").expect("empty object parses"); + assert_eq!(cfg, WorkerConfig::default()); + } + + #[test] + fn yaml_overrides_each_field() { + let cfg = WorkerConfig::from_yaml( + "max_input_bytes: 1024\n\ + max_chars: 10\n\ + preview_chars: 5\n\ + max_items: 7\n\ + classify_sample_pages: 3\n\ + min_text_ops_per_page: 9\n\ + text_page_ratio_threshold: 0.25\n", + ) + .expect("full object parses"); + assert_eq!(cfg.max_input_bytes, 1024); + assert_eq!(cfg.max_chars, 10); + assert_eq!(cfg.preview_chars, 5); + assert_eq!(cfg.max_items, 7); + assert_eq!(cfg.classify_sample_pages, 3); + assert_eq!(cfg.min_text_ops_per_page, 9); + assert!((cfg.text_page_ratio_threshold - 0.25).abs() < f32::EPSILON); + } + + #[test] + fn unknown_field_is_rejected() { + let err = WorkerConfig::from_yaml("max_charz: 10\n").expect_err("typo must fail loudly"); + assert!( + err.contains("max_charz"), + "error should name the field: {err}" + ); + } + + #[test] + fn a_ratio_outside_zero_to_one_is_rejected() { + for bad in ["-0.1", "1.5"] { + let err = WorkerConfig::from_yaml(&format!("text_page_ratio_threshold: {bad}\n")) + .expect_err("out of range"); + assert!(err.contains("between 0.0 and 1.0"), "{err}"); + } + // Both parse paths validate, not just the seed file. + let err = WorkerConfig::from_json(&serde_json::json!({ + "text_page_ratio_threshold": 2.0 + })) + .expect_err("out of range"); + assert!(err.contains("between 0.0 and 1.0"), "{err}"); + + for good in ["0.0", "0.6", "1.0"] { + assert!( + WorkerConfig::from_yaml(&format!("text_page_ratio_threshold: {good}\n")).is_ok(), + "{good} is a valid share" + ); + } + } + + #[test] + fn json_round_trips() { + let cfg = WorkerConfig { + max_chars: 123, + ..WorkerConfig::default() + }; + let back = WorkerConfig::from_json(&cfg.to_json()).expect("round trip"); + assert_eq!(cfg, back); + } + + #[test] + fn schema_carries_defaults_as_example() { + let schema = WorkerConfig::json_schema(); + assert_eq!(schema["example"], WorkerConfig::default().to_json()); + assert!(schema["properties"]["max_chars"]["description"].is_string()); + } + + #[test] + fn per_call_max_chars_overrides_the_default() { + let cfg = WorkerConfig::default(); + assert_eq!(cfg.effective_max_chars(None), cfg.max_chars); + assert_eq!(cfg.effective_max_chars(Some(7)), 7); + assert_eq!(cfg.effective_max_chars(Some(0)), 0); + } + + #[test] + fn env_expansion_applies_to_the_seed_only() { + std::env::set_var("PDF_TEST_CHARS", "99"); + let cfg = WorkerConfig::from_yaml("max_chars: ${PDF_TEST_CHARS}\n").expect("expands"); + assert_eq!(cfg.max_chars, 99); + std::env::remove_var("PDF_TEST_CHARS"); + + let cfg = WorkerConfig::from_yaml("max_chars: ${PDF_UNSET_VAR:42}\n").expect("falls back"); + assert_eq!(cfg.max_chars, 42); + } +} diff --git a/pdf/src/configuration.rs b/pdf/src/configuration.rs new file mode 100644 index 000000000..ef6cc088d --- /dev/null +++ b/pdf/src/configuration.rs @@ -0,0 +1,261 @@ +//! Integration with the `configuration` worker: register the schema, fetch the +//! authoritative value at boot, and hot-reload it when it changes. +//! +//! Every field here is a per-call tuning knob read from the live snapshot, so +//! there is nothing structural to rebuild and nothing that needs a restart. +//! +//! `configuration` is a REQUIRED boot dependency: a failed register or fetch +//! aborts startup rather than running on a guessed size ceiling. + +use std::sync::Arc; +use std::time::Duration; + +use iii_sdk::errors::Error; +use iii_sdk::protocol::{RegisterTriggerInput, TriggerRequest}; +use iii_sdk::{IIIClient, RegisterFunction}; +use serde_json::{json, Value}; +use tokio::sync::RwLock; + +use crate::config::WorkerConfig; + +/// Hot-swappable config snapshot shared with every handler. A handler takes a +/// `read().await`, clones the inner `Arc` out, and drops the lock before doing +/// any work; `apply_config` replaces the inner `Arc` under the write lock. +pub type ConfigCell = Arc>>; + +pub const CONFIG_ID: &str = "pdf"; +const CONFIG_FN_ID: &str = "pdf::on-config-change"; +const CONFIG_RETRIES: u32 = 3; +/// Base backoff between configuration RPC retries, multiplied by the attempt +/// number for a linear backoff. +const CONFIG_RETRY_BACKOFF_MS: u64 = 250; + +/// Register this worker's configuration schema. When `seed` is present its +/// value becomes `initial_value`; otherwise the built-in default is seeded only +/// when nothing is stored yet, so calling this every boot is safe. +pub async fn register_config(iii: &IIIClient, seed: Option<&WorkerConfig>) -> Result<(), String> { + let mut payload = json!({ + "id": CONFIG_ID, + "name": "PDF", + "description": "Limits and detection thresholds for reading PDFs: the size ceiling on an \ + accepted document, the caps on how much text, markdown and how many \ + positioned items one response returns, and how many pages classification \ + samples before deciding whether a document holds real text.", + "schema": WorkerConfig::json_schema(), + }); + if let Some(seed) = seed { + payload["initial_value"] = seed.to_json(); + } else if should_seed_default_value(iii).await? { + payload["initial_value"] = WorkerConfig::default().to_json(); + } + trigger_with_retry(iii, "configuration::register", payload).await?; + Ok(()) +} + +/// Read the live configuration (env-expanded by the configuration worker; +/// `from_json` does NOT re-expand). +pub async fn fetch_config(iii: &IIIClient) -> Result { + let value = get_config_value(iii).await?; + if value.is_null() { + tracing::info!("no configuration value found; using built-in defaults"); + return Ok(WorkerConfig::default()); + } + WorkerConfig::from_json(&value) +} + +async fn should_seed_default_value(iii: &IIIClient) -> Result { + match try_get_config_value(iii).await? { + None => Ok(true), + Some(value) if value.is_null() => Ok(true), + Some(_) => Ok(false), + } +} + +async fn get_config_value(iii: &IIIClient) -> Result { + try_get_config_value(iii) + .await? + .ok_or_else(|| format!("configuration `{CONFIG_ID}` not found")) +} + +/// `Ok(None)` when the entry does not exist. The engine's missing-entry codes +/// vary in case, so match case-insensitively. +async fn try_get_config_value(iii: &IIIClient) -> Result, String> { + match trigger_with_retry(iii, "configuration::get", json!({ "id": CONFIG_ID })).await { + Ok(resp) => Ok(resp.get("value").cloned()), + Err(e) if e.to_ascii_uppercase().contains("NOT_FOUND") => Ok(None), + Err(e) => Err(e), + } +} + +/// Swap the config snapshot under the write lock. +pub async fn apply_config(cell: &ConfigCell, cfg: WorkerConfig) { + *cell.write().await = Arc::new(cfg); +} + +/// Payload of the internal config-change handler. The handler re-fetches the +/// authoritative value, so this carries only the advisory id; a struct rather +/// than a `Value` keeps the request schema concrete. +#[derive(Debug, Default, serde::Deserialize, schemars::JsonSchema)] +pub struct OnConfigChangeEvent { + /// Configuration id that changed (advisory; the handler re-fetches). + #[serde(default)] + pub id: Option, +} + +/// Ack returned by the internal config-change handler. +#[derive(Debug, serde::Serialize, schemars::JsonSchema)] +pub struct OnConfigChangeResponse { + pub ok: bool, +} + +/// Register the internal config-change handler and bind a `configuration` +/// trigger. The handler re-fetches via `configuration::get` and ignores the +/// trigger payload, so a direct call can never inject config. +pub fn register_config_trigger(iii: &IIIClient, cell: ConfigCell) -> Result<(), Error> { + let cell_for_fn = cell.clone(); + let engine = iii.clone(); + iii.register_function( + CONFIG_FN_ID, + RegisterFunction::new_async(move |_event: OnConfigChangeEvent| { + let cell = cell_for_fn.clone(); + let engine = engine.clone(); + async move { + on_config_change(&engine, &cell).await; + Ok::(OnConfigChangeResponse { ok: true }) + } + }) + .description( + "Internal: hot-reload the pdf worker from the authoritative configuration when it \ + changes, swapping the per-call snapshot.", + ), + ); + + iii.register_trigger(RegisterTriggerInput { + trigger_type: "configuration".to_string(), + function_id: CONFIG_FN_ID.to_string(), + config: json!({ + "configuration_id": CONFIG_ID, + "event_types": ["configuration:updated"], + }), + metadata: None, + })?; + Ok(()) +} + +/// Reload from the AUTHORITATIVE configuration. +/// +/// The caller-supplied trigger payload is deliberately ignored: +/// `pdf::on-config-change` is a bus function, so trusting a `new_value` in the +/// payload would let any caller lift the size ceiling without touching +/// persisted state. +async fn on_config_change(iii: &IIIClient, cell: &ConfigCell) { + let cfg = match fetch_config(iii).await { + Ok(cfg) => cfg, + Err(e) => { + tracing::error!( + error = %e, + "config-change: failed to fetch authoritative configuration; keeping previous config" + ); + return; + } + }; + apply_config(cell, cfg).await; + tracing::info!("pdf configuration reloaded"); +} + +/// `true` for the one error that is an answer rather than a failure: the entry +/// does not exist yet. Retrying it wastes the backoff on every first boot and +/// logs two warnings for a completely normal state. +fn is_not_found(error: &str) -> bool { + error.to_ascii_uppercase().contains("NOT_FOUND") +} + +async fn trigger_with_retry( + iii: &IIIClient, + function_id: &str, + payload: Value, +) -> Result { + let mut last_err = String::new(); + for attempt in 1..=CONFIG_RETRIES { + match iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload: payload.clone(), + action: None, + timeout_ms: None, + }) + .await + { + Ok(v) => return Ok(v), + Err(e) => { + last_err = e.to_string(); + if is_not_found(&last_err) { + return Err(last_err); + } + if attempt < CONFIG_RETRIES { + tracing::warn!( + function_id, + attempt, + error = %last_err, + "configuration RPC failed; retrying" + ); + tokio::time::sleep(Duration::from_millis( + CONFIG_RETRY_BACKOFF_MS * u64::from(attempt), + )) + .await; + } + } + } + } + Err(format!( + "{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A missing entry is the normal first-boot state, not a transient + /// failure. Retrying it spends the whole backoff and logs warnings on + /// every clean install. + #[test] + fn a_missing_entry_is_not_retried() { + assert!(is_not_found( + "remote error (NOT_FOUND): configuration 'pdf' not found" + )); + assert!(is_not_found("STATEMENT_NOT_FOUND")); + assert!(!is_not_found("connection reset by peer")); + assert!(!is_not_found("timed out")); + } + + #[tokio::test] + async fn apply_config_swaps_the_snapshot() { + let cell: ConfigCell = Arc::new(RwLock::new(Arc::new(WorkerConfig::default()))); + assert_eq!( + cell.read().await.max_chars, + WorkerConfig::default().max_chars + ); + + apply_config( + &cell, + WorkerConfig { + max_chars: 7, + ..WorkerConfig::default() + }, + ) + .await; + assert_eq!(cell.read().await.max_chars, 7); + } + + /// The config-change handler must stay off the public catalog: it is + /// registered here, not in `functions::register_all`. + #[test] + fn the_reload_handler_is_not_on_the_public_catalog() { + let ids: Vec<&str> = crate::functions::catalog() + .iter() + .map(|s| s.function_id) + .collect(); + assert!(!ids.contains(&CONFIG_FN_ID)); + } +} diff --git a/pdf/src/functions/classify.rs b/pdf/src/functions/classify.rs new file mode 100644 index 000000000..4beb43228 --- /dev/null +++ b/pdf/src/functions/classify.rs @@ -0,0 +1,309 @@ +//! `pdf::classify` — the routing call. +//! +//! Answers one question in tens of milliseconds: does this document hold real +//! text, or is it pictures of text? Everything expensive downstream hangs off +//! that answer, so it samples content streams rather than extracting anything. +//! +//! The per-page verdict matters as much as the document-level one. A report +//! with a scanned cover and two hundred text pages is not a scanned document, +//! and treating it as one sends the whole thing to an OCR service for the sake +//! of one page. + +use pdf_inspector::{DetectionConfig, PdfType, ScanStrategy}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::source::{describe_error, PdfSource}; + +pub const ID: &str = "pdf::classify"; +pub const DESC: &str = "Classify a PDF as text-based, scanned, image-based or mixed, and report \ + which pages need OCR and why. Samples content streams rather than \ + extracting text, so it answers in tens of milliseconds. Call this before \ + any other pdf function."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: PdfSource, + + /// Password for an encrypted document. Never logged or echoed back. + #[serde(default)] + pub password: Option, + + /// Pages sampled for the verdict, overriding the configured default. `0` + /// scans every page, which is slower but settles a borderline mixed + /// document. + #[serde(default)] + pub sample_pages: Option, +} + +/// What a document is made of. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum DocumentType { + /// Real text throughout. Extract locally. + TextBased, + /// Pictures of pages. Every page needs OCR. + Scanned, + /// Images with little or no text layer. + ImageBased, + /// Some pages carry text, others do not. Read `pages_needing_ocr`. + Mixed, +} + +impl From for DocumentType { + fn from(value: PdfType) -> Self { + match value { + PdfType::TextBased => Self::TextBased, + PdfType::Scanned => Self::Scanned, + PdfType::ImageBased => Self::ImageBased, + PdfType::Mixed => Self::Mixed, + } + } +} + +/// Why one page cannot be read without OCR. +#[derive(Debug, Serialize, JsonSchema)] +pub struct PageOcrReason { + /// 1-indexed page number. + pub page: u32, + /// Machine-readable reasons: `scanned` (a raster page), `no_text` (nothing + /// extractable and nothing to OCR), `vector_text` (characters drawn as + /// outlines rather than text) or `suspected_garbled_text` (a text layer + /// that decodes to nonsense). + pub reasons: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The document-level verdict. + pub document_type: DocumentType, + + /// How much to trust the verdict, from 0.0 to 1.0. + pub confidence: f32, + + /// Pages in the document. + pub page_count: u32, + + /// Pages actually inspected. Lower than `page_count` when sampling, so a + /// verdict from a sample can be told apart from one that read everything. + /// Absent for an encrypted document, which takes a decryption path that + /// does not report the counters. + #[serde(skip_serializing_if = "Option::is_none")] + pub pages_sampled: Option, + + /// Inspected pages that carry text operators. Absent for an encrypted + /// document, for the same reason as `pages_sampled`. + #[serde(skip_serializing_if = "Option::is_none")] + pub pages_with_text: Option, + + /// 1-indexed pages that cannot be read without OCR. Empty for a clean + /// text-based document. + pub pages_needing_ocr: Vec, + + /// Per-page explanation for `pages_needing_ocr`. + pub ocr_reasons: Vec, + + /// `true` when the images carry meaning the text layer does not, so OCR + /// adds something even on a text-based document. Absent for an encrypted + /// document, for the same reason as `pages_sampled`. + #[serde(skip_serializing_if = "Option::is_none")] + pub ocr_recommended: Option, + + /// Document title from the PDF metadata, when it has one. + pub title: Option, + + /// `true` when font encodings decoded badly. Only known on the encrypted + /// path, which extracts far enough to notice; absent otherwise, where + /// `suspected_garbled_text` in `ocr_reasons` carries the same signal. + #[serde(skip_serializing_if = "Option::is_none")] + pub has_encoding_issues: Option, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the classification. + pub elapsed_ms: u64, +} + +/// Build the detection config for one call: worker defaults, with the per-call +/// sample override applied. +pub fn detection_config(cfg: &WorkerConfig, sample_pages: Option) -> DetectionConfig { + let sample = sample_pages.unwrap_or(cfg.classify_sample_pages); + DetectionConfig { + // A zero sample means "look at everything". `Full` says that precisely; + // `Sample(0)` would be a request to look at nothing. + strategy: if sample == 0 { + ScanStrategy::Full + } else { + // Saturate rather than cast: a `sample` above u32::MAX would wrap, + // and a wrap to 0 means Sample(0), which samples nothing at all. + ScanStrategy::Sample(u32::try_from(sample).unwrap_or(u32::MAX)) + }, + min_text_ops_per_page: cfg.min_text_ops_per_page as u32, + text_page_ratio_threshold: cfg.text_page_ratio_threshold, + } +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + let detection = detection_config(cfg, req.sample_pages); + + // Two paths, because the parser's fast detection entry point takes no + // password and simply errors on an encrypted document. The decrypting path + // runs the same detection under the same config, it just returns fewer + // counters, so those fields go absent rather than being invented. + let mut response = match &req.password { + Some(_) => classify_encrypted(&bytes, detection, req.password.clone())?, + None => classify_plain(&bytes, detection)?, + }; + + response.source = req.source.label(); + response.elapsed_ms = started.elapsed().as_millis() as u64; + Ok(response) +} + +fn classify_plain(bytes: &[u8], detection: DetectionConfig) -> Result { + let result = pdf_inspector::detect_pdf_type_mem_with_config(bytes, detection) + .map_err(|e| describe_error("classify", e, false, true))?; + + // Both `pages_needing_ocr` and the reason map are 1-indexed here, unlike + // the parser's lightweight classification entry point, which counts from + // zero. Covered by test. + let ocr_reasons = result + .ocr_reasons_by_page + .into_iter() + .map(|(page, reasons)| PageOcrReason { page, reasons }) + .collect(); + + Ok(Response { + document_type: result.pdf_type.into(), + confidence: result.confidence, + page_count: result.page_count, + pages_sampled: Some(result.pages_sampled), + pages_with_text: Some(result.pages_with_text), + pages_needing_ocr: result.pages_needing_ocr, + ocr_reasons, + ocr_recommended: Some(result.ocr_recommended), + title: result.title, + has_encoding_issues: None, + source: String::new(), + elapsed_ms: 0, + }) +} + +fn classify_encrypted( + bytes: &[u8], + detection: DetectionConfig, + password: Option, +) -> Result { + let result = pdf_inspector::process_pdf_mem_with_options( + bytes, + pdf_inspector::PdfOptions { + mode: pdf_inspector::ProcessMode::DetectOnly, + detection, + markdown: Default::default(), + page_filter: None, + password, + }, + ) + .map_err(|e| describe_error("classify", e, true, true))?; + + let ocr_reasons = result + .ocr_reasons_by_page + .into_iter() + .map(|r| PageOcrReason { + page: r.page, + reasons: r.reasons, + }) + .collect(); + + Ok(Response { + document_type: result.pdf_type.into(), + confidence: result.confidence, + page_count: result.page_count, + pages_sampled: None, + pages_with_text: None, + pages_needing_ocr: result.pages_needing_ocr, + ocr_reasons, + ocr_recommended: None, + title: result.title, + has_encoding_issues: Some(result.has_encoding_issues), + source: String::new(), + elapsed_ms: 0, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn zero_sample_pages_means_scan_everything() { + let cfg = WorkerConfig::default(); + assert!(matches!( + detection_config(&cfg, Some(0)).strategy, + ScanStrategy::Full + )); + } + + #[test] + fn sample_override_beats_the_configured_default() { + let cfg = WorkerConfig { + classify_sample_pages: 8, + ..WorkerConfig::default() + }; + assert!(matches!( + detection_config(&cfg, Some(3)).strategy, + ScanStrategy::Sample(3) + )); + assert!(matches!( + detection_config(&cfg, None).strategy, + ScanStrategy::Sample(8) + )); + } + + /// The parser's own default is `Sample(8)`, not the early-exit strategy its + /// documentation advertises. This worker's default must track the code, and + /// a dependency bump that changes it must break here rather than quietly + /// reclassifying every report with an image cover. + #[test] + fn worker_default_matches_the_parsers_actual_default() { + let parser_default = DetectionConfig::default(); + assert!(matches!(parser_default.strategy, ScanStrategy::Sample(8))); + + let cfg = WorkerConfig::default(); + let ours = detection_config(&cfg, None); + assert!(matches!(ours.strategy, ScanStrategy::Sample(8))); + assert_eq!( + ours.min_text_ops_per_page, + parser_default.min_text_ops_per_page + ); + assert!( + (ours.text_page_ratio_threshold - parser_default.text_page_ratio_threshold).abs() + < f32::EPSILON + ); + } + + #[test] + fn document_type_covers_every_parser_variant() { + assert_eq!( + DocumentType::from(PdfType::TextBased), + DocumentType::TextBased + ); + assert_eq!(DocumentType::from(PdfType::Scanned), DocumentType::Scanned); + assert_eq!( + DocumentType::from(PdfType::ImageBased), + DocumentType::ImageBased + ); + assert_eq!(DocumentType::from(PdfType::Mixed), DocumentType::Mixed); + } + + #[test] + fn document_type_serializes_in_snake_case() { + let json = serde_json::to_string(&DocumentType::TextBased).expect("serializes"); + assert_eq!(json, "\"text_based\""); + } +} diff --git a/pdf/src/functions/items.rs b/pdf/src/functions/items.rs new file mode 100644 index 000000000..18cda763b --- /dev/null +++ b/pdf/src/functions/items.rs @@ -0,0 +1,271 @@ +//! `pdf::extract-items` — where the text sits, not just what it says. +//! +//! Each item is one run of characters with its box on the page, its font, and +//! the styling the parser recovered. Underline and strikeout are geometric +//! findings: PDF has no flag for either, so they come from vector lines drawn +//! near the baseline. +//! +//! Coordinates are PDF points with the origin at the **bottom left** of the +//! page, which is the PDF convention and the opposite of every layout model's +//! output. The region functions use top-left instead, because that is what +//! their callers produce. Both are stated on the schema, because getting this +//! wrong is silent: the text comes back, it is just from the wrong end of the +//! page. + +use std::collections::HashSet; + +use pdf_inspector::types::ItemType; +use pdf_inspector::TextItem; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::source::{describe_error, PdfSource}; + +pub const ID: &str = "pdf::extract-items"; +pub const DESC: &str = "Extract positioned text items: the box, font, size and styling of every \ + run of characters on a page. Coordinates are PDF points with a \ + bottom-left origin. Use this for layout-aware reading; use \ + pdf::to-markdown to just read the document."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: PdfSource, + + /// 1-indexed pages to read. Omit for the whole document. + #[serde(default)] + pub pages: Option>, + + /// Items to return before truncating. Omit for the configured default; `0` + /// returns every item, which on a dense document is a very large response. + #[serde(default)] + pub max_items: Option, +} + +/// What one item is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ItemKind { + /// Ordinary text. + Text, + /// An image placeholder. The box is real; no pixels are decoded. + Image, + /// Text carrying a hyperlink; the target is in `link`. + Link, + /// A filled-in form field value. + FormField, +} + +/// One positioned run of characters. +#[derive(Debug, Serialize, JsonSchema)] +pub struct Item { + /// The characters. + pub text: String, + /// 1-indexed page number. + pub page: u32, + /// Left edge, PDF points from the left of the page. + pub x: f32, + /// Baseline, PDF points from the **bottom** of the page. + pub y: f32, + /// Width in PDF points. + pub width: f32, + /// Height in PDF points, approximated from the font size. + pub height: f32, + /// Font name as the document names it. + pub font: String, + /// Font size in points. + pub font_size: f32, + pub bold: bool, + pub italic: bool, + /// Recovered from vector lines near the baseline, not from a flag. + pub underline: bool, + /// Recovered from vector lines through the text, not from a flag. + pub strikeout: bool, + pub kind: ItemKind, + /// Link target, for a `link` item. + #[serde(skip_serializing_if = "Option::is_none")] + pub link: Option, + /// Marked-content id tying this item to the document's tagged structure + /// tree, when the document has one. + #[serde(skip_serializing_if = "Option::is_none")] + pub mcid: Option, +} + +impl From for Item { + fn from(item: TextItem) -> Self { + let (kind, link) = match &item.item_type { + ItemType::Text => (ItemKind::Text, None), + ItemType::Image => (ItemKind::Image, None), + ItemType::Link(url) => (ItemKind::Link, Some(url.clone())), + ItemType::FormField => (ItemKind::FormField, None), + }; + Self { + text: item.text, + page: item.page, + x: item.x, + y: item.y, + width: item.width, + height: item.height, + font: item.font, + font_size: item.font_size, + bold: item.is_bold, + italic: item.is_italic, + underline: item.is_underline, + strikeout: item.is_strikeout, + kind, + link, + mcid: item.mcid, + } + } +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The items, in document order, capped per `max_items`. + pub items: Vec, + + /// Items returned. + pub count: usize, + + /// Items the document holds for the requested pages. Equal to `count` when + /// nothing was dropped. + pub total_count: usize, + + /// `true` when `items` stops short. Narrow `pages`, or pass `max_items: 0`. + pub truncated: bool, + + /// Origin convention for `x` and `y`, always `pdf-points, bottom-left`. + /// Stated on every response because a caller that assumed the other + /// convention reads the wrong end of the page with no error. Note + /// `pdf::extract-regions` takes boxes with a top-left origin instead. + pub coordinate_origin: String, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the extraction. + pub elapsed_ms: u64, +} + +pub const COORDINATE_ORIGIN: &str = "pdf-points, bottom-left"; + +/// Validate a requested page list into the filter this entry point expects. +/// +/// This entry point already counts pages from one, so the numbers pass through +/// unchanged. Page 0 is still rejected: as a filter value it would silently +/// match nothing and return an empty document. +pub fn page_filter(pages: Option<&[u32]>) -> Result>, String> { + match pages { + None => Ok(None), + Some([]) => Err("`pages` was empty; omit it to read the whole document".to_string()), + Some(pages) if pages.contains(&0) => { + Err("page numbers are 1-indexed; 0 is not a page".to_string()) + } + Some(pages) => Ok(Some(pages.iter().copied().collect())), + } +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let filter = page_filter(req.pages.as_deref())?; + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + + let raw = + pdf_inspector::extractor::extract_text_with_positions_mem_pages(&bytes, filter.as_ref()) + .map_err(|e| describe_error("item extraction", e, false, false))?; + + let total_count = raw.len(); + let max_items = req.max_items.unwrap_or(cfg.max_items); + let truncated = max_items > 0 && total_count > max_items; + let items: Vec = if truncated { + raw.into_iter().take(max_items).map(Item::from).collect() + } else { + raw.into_iter().map(Item::from).collect() + }; + + Ok(Response { + count: items.len(), + items, + total_count, + truncated, + coordinate_origin: COORDINATE_ORIGIN.to_string(), + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample(item_type: ItemType) -> TextItem { + TextItem { + text: "hello".to_string(), + x: 1.0, + y: 2.0, + width: 3.0, + height: 4.0, + font: "Helvetica".to_string(), + font_size: 12.0, + page: 1, + is_bold: true, + is_italic: false, + is_underline: true, + is_strikeout: false, + item_type, + mcid: Some(7), + } + } + + #[test] + fn item_kinds_map_across() { + assert_eq!(Item::from(sample(ItemType::Text)).kind, ItemKind::Text); + assert_eq!(Item::from(sample(ItemType::Image)).kind, ItemKind::Image); + assert_eq!( + Item::from(sample(ItemType::FormField)).kind, + ItemKind::FormField + ); + } + + #[test] + fn a_link_carries_its_target() { + let item = Item::from(sample(ItemType::Link("https://example.com".to_string()))); + assert_eq!(item.kind, ItemKind::Link); + assert_eq!(item.link.as_deref(), Some("https://example.com")); + } + + #[test] + fn geometry_and_styling_survive_the_conversion() { + let item = Item::from(sample(ItemType::Text)); + assert_eq!( + (item.x, item.y, item.width, item.height), + (1.0, 2.0, 3.0, 4.0) + ); + assert!(item.bold); + assert!(item.underline); + assert!(!item.strikeout); + assert_eq!(item.mcid, Some(7)); + assert_eq!(item.page, 1); + } + + #[test] + fn page_zero_is_rejected_rather_than_matching_nothing() { + let err = page_filter(Some(&[1, 0])).expect_err("page 0"); + assert!(err.contains("1-indexed"), "{err}"); + } + + #[test] + fn an_empty_page_list_is_a_caller_mistake() { + let err = page_filter(Some(&[])).expect_err("empty list"); + assert!(err.contains("omit it"), "{err}"); + } + + #[test] + fn pages_pass_through_unconverted() { + let filter = page_filter(Some(&[1, 3])).expect("valid").expect("some"); + assert!(filter.contains(&1) && filter.contains(&3)); + assert_eq!(filter.len(), 2); + assert_eq!(page_filter(None), Ok(None)); + } +} diff --git a/pdf/src/functions/markdown.rs b/pdf/src/functions/markdown.rs new file mode 100644 index 000000000..55efe092d --- /dev/null +++ b/pdf/src/functions/markdown.rs @@ -0,0 +1,309 @@ +//! `pdf::to-markdown` — a text-based document as markdown that keeps its shape. +//! +//! Headings, lists, links and tables survive; the parser reconstructs them from +//! font sizes, geometry and ruled lines rather than from any structure the file +//! promises to have. +//! +//! The size cap is the other half of the job. A long report runs to hundreds of +//! thousands of characters, and handing that to a model wastes the context it +//! needed for the answer. Responses are capped by default and say so; a caller +//! that genuinely wants the whole document passes `max_chars: 0`, which is what +//! a worker-to-worker pipeline does when the document is going to storage +//! rather than to a model. + +use std::collections::HashSet; + +use pdf_inspector::{MarkdownOptions, MarkdownProfile, PdfOptions, ProcessMode}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::functions::classify::{detection_config, DocumentType, PageOcrReason}; +use crate::source::{describe_error, to_parser_pages, Body, PdfSource}; + +pub const ID: &str = "pdf::to-markdown"; +pub const DESC: &str = "Convert a text-based PDF to markdown, preserving headings, lists, links \ + and tables. Returns nothing for a scanned document — call pdf::classify \ + first. Responses are capped; pass max_chars 0 to take the whole document, \ + or pages to take a slice of it."; + +/// How faithful the markdown should be to the source characters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Profile { + /// Preserve the source text as written. + #[default] + Fidelity, + /// Prefer shorter output, collapsing runs like the dot leaders in a table + /// of contents. Not character-faithful to the source. + Compact, +} + +impl From for MarkdownProfile { + fn from(value: Profile) -> Self { + match value { + Profile::Fidelity => MarkdownProfile::Fidelity, + Profile::Compact => MarkdownProfile::Compact, + } + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: PdfSource, + + /// Password for an encrypted document. Never logged or echoed back. + #[serde(default)] + pub password: Option, + + /// 1-indexed pages to convert. Omit for the whole document. A page filter + /// is the cheap way to read a long report: take the pages you need rather + /// than the whole thing truncated. + #[serde(default)] + pub pages: Option>, + + /// Characters to return before truncating. Omit for the configured + /// default; `0` returns the whole document. + #[serde(default)] + pub max_chars: Option, + + /// Source fidelity versus token efficiency. + #[serde(default)] + pub profile: Profile, + + /// Include `[Image: …]` placeholders. Off by default: nothing here decodes + /// pixels, so a placeholder adds noise without adding information. + #[serde(default)] + pub include_images: bool, + + /// Strip repeated running headers and footers. + #[serde(default = "default_true")] + pub strip_headers_footers: bool, + + /// Return markdown per page as well as the joined document. Useful when a + /// caller wants to route some pages to OCR and keep the rest. + #[serde(default)] + pub per_page: bool, +} + +fn default_true() -> bool { + true +} + +/// One page of markdown, with its own OCR verdict. +#[derive(Debug, Serialize, JsonSchema)] +pub struct PageResult { + /// 1-indexed page number. + pub page: u32, + /// Markdown for this page. + pub markdown: String, + /// `true` when this page's text is not trustworthy and OCR would do better. + pub needs_ocr: bool, + /// Machine-readable reason, when the cause is known. + #[serde(skip_serializing_if = "Option::is_none")] + pub ocr_reason: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The document-level verdict, so a caller that skipped `pdf::classify` + /// still learns it got nothing because the document is a scan. + pub document_type: DocumentType, + + /// The markdown, capped per `max_chars`. + pub body: Body, + + /// Pages in the document. + pub page_count: u32, + + /// Pages actually converted. Equal to `page_count` unless `pages` was set. + pub pages_converted: u32, + + /// Per-page markdown, when `per_page` was requested. + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option>, + + /// 1-indexed pages holding a detected table. + pub pages_with_tables: Vec, + + /// 1-indexed pages laid out in multiple columns. + pub pages_with_columns: Vec, + + /// 1-indexed pages that need OCR. + pub pages_needing_ocr: Vec, + + /// Per-page explanation for `pages_needing_ocr`. + pub ocr_reasons: Vec, + + /// `true` when font encodings decoded badly. The markdown, if any, is not + /// to be trusted. + pub has_encoding_issues: bool, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the conversion. + pub elapsed_ms: u64, +} + +fn markdown_options(req: &Request) -> MarkdownOptions { + MarkdownOptions { + profile: req.profile.into(), + include_images: req.include_images, + strip_headers_footers: req.strip_headers_footers, + ..MarkdownOptions::default() + } +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + // The parser's per-page entry point takes no password, so this combination + // would extract the whole document and then fail only on the per-page pass, + // reporting a decryption error for a call that had already decrypted fine. + // Refuse it up front and say which half to drop. + if req.per_page && req.password.is_some() { + return Err( + "`per_page` cannot be combined with `password`: per-page extraction cannot decrypt. Call without `per_page`, or convert one page at a time with `pages`." + .to_string(), + ); + } + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + + let page_filter: Option> = match &req.pages { + Some(pages) if pages.is_empty() => { + return Err("`pages` was empty; omit it to convert the whole document".to_string()) + } + // The whole-document options take 1-indexed pages, unlike the per-page + // entry point below. Both are covered by tests. + Some(pages) => { + for &page in pages { + if page == 0 { + return Err("page numbers are 1-indexed; 0 is not a page".to_string()); + } + } + Some(pages.iter().copied().collect()) + } + None => None, + }; + + let options = PdfOptions { + mode: ProcessMode::Full, + detection: detection_config(cfg, None), + markdown: markdown_options(&req), + page_filter: page_filter.clone(), + password: req.password.clone(), + }; + + let result = pdf_inspector::process_pdf_mem_with_options(&bytes, options) + .map_err(|e| describe_error("markdown conversion", e, req.password.is_some(), true))?; + + let max_chars = cfg.effective_max_chars(req.max_chars); + let body = Body::new( + result.markdown.unwrap_or_default(), + max_chars, + cfg.preview_chars, + ); + + // Clamp to the document: a filter naming pages past the end converts + // fewer pages than it asked for, and reporting the request back would + // overstate what was read. + let pages_converted = page_filter + .as_ref() + .map(|f| (f.len() as u32).min(result.page_count)) + .unwrap_or(result.page_count); + + let per_page = if req.per_page { + Some(extract_per_page(&bytes, req.pages.as_deref())?) + } else { + None + }; + + let ocr_reasons = result + .ocr_reasons_by_page + .into_iter() + .map(|r| PageOcrReason { + page: r.page, + reasons: r.reasons, + }) + .collect(); + + Ok(Response { + document_type: result.pdf_type.into(), + body, + page_count: result.page_count, + pages_converted, + pages: per_page, + pages_with_tables: result.layout.pages_with_tables, + pages_with_columns: result.layout.pages_with_columns, + pages_needing_ocr: result.pages_needing_ocr, + ocr_reasons, + has_encoding_issues: result.has_encoding_issues, + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +/// Per-page markdown. The parser's per-page entry point counts pages from zero +/// on the way in and on the way out, so both directions are converted here. +fn extract_per_page(bytes: &[u8], pages: Option<&[u32]>) -> Result, String> { + let parser_pages = match pages { + Some(pages) => Some(to_parser_pages(pages)?), + None => None, + }; + let extracted = pdf_inspector::extract_pages_markdown_mem(bytes, parser_pages.as_deref()) + .map_err(|e| describe_error("per-page extraction", e, false, false))?; + + Ok(extracted + .pages + .into_iter() + .map(|p| PageResult { + page: crate::source::to_wire_page(p.page), + markdown: p.markdown, + needs_ocr: p.needs_ocr, + ocr_reason: p.ocr_reason, + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn profile_maps_to_the_parser() { + assert_eq!( + MarkdownProfile::from(Profile::Fidelity), + MarkdownProfile::Fidelity + ); + assert_eq!( + MarkdownProfile::from(Profile::Compact), + MarkdownProfile::Compact + ); + } + + #[test] + fn profile_defaults_to_fidelity() { + assert_eq!(Profile::default(), Profile::Fidelity); + } + + #[test] + fn images_are_excluded_by_default() { + let req: Request = serde_json::from_value(serde_json::json!({ "path": "x.pdf" })) + .expect("minimal request parses"); + assert!(!req.include_images); + assert!(req.strip_headers_footers); + assert!(!markdown_options(&req).include_images); + } + + #[test] + fn a_source_field_is_still_required_after_flattening() { + let req: Request = serde_json::from_value(serde_json::json!({})) + .expect("shape parses; validation is late"); + let err = req + .source + .load(&WorkerConfig::default()) + .expect_err("no source"); + assert!(err.contains("provide a `path`"), "{err}"); + } +} diff --git a/pdf/src/functions/mod.rs b/pdf/src/functions/mod.rs new file mode 100644 index 000000000..ecfd4ff7d --- /dev/null +++ b/pdf/src/functions/mod.rs @@ -0,0 +1,148 @@ +//! The worker's public surface: five functions over one parser. +//! +//! Every handler is synchronous CPU work over an owned buffer, so each one runs +//! on a blocking thread rather than on the async runtime. A two hundred page +//! document takes hundreds of milliseconds, which is long enough to stall the +//! executor and every other call sharing it. + +pub mod classify; +pub mod items; +pub mod markdown; +pub mod regions; +pub mod text; + +use std::sync::Arc; + +use iii_sdk::errors::Error; +use iii_sdk::{IIIClient, RegisterFunction}; + +use crate::configuration::ConfigCell; + +/// One entry of the wire surface: what a caller sees for one function. +pub struct FunctionSpec { + pub function_id: &'static str, + pub description: &'static str, + pub request_schema: schemars::schema::RootSchema, + pub response_schema: schemars::schema::RootSchema, +} + +/// Build a schema the same way iii-sdk does at registration, so the snapshot +/// equals what actually ships. +fn schema_of() -> schemars::schema::RootSchema { + schemars::r#gen::SchemaSettings::draft07() + .into_generator() + .into_root_schema_for::() +} + +fn spec(function_id: &'static str, description: &'static str) -> FunctionSpec +where + Req: schemars::JsonSchema, + Resp: schemars::JsonSchema, +{ + FunctionSpec { + function_id, + description, + request_schema: schema_of::(), + response_schema: schema_of::(), + } +} + +/// The full wire-surface catalog, in registration order. Golden-tested in +/// `tests/schemas.rs`; keep in lockstep with [`register_all`]. +pub fn catalog() -> Vec { + vec![ + spec::(classify::ID, classify::DESC), + spec::(markdown::ID, markdown::DESC), + spec::(text::ID, text::DESC), + spec::(items::ID, items::DESC), + spec::(regions::ID, regions::DESC), + ] +} + +/// Register one function whose handler is blocking CPU work over the live +/// config snapshot. +/// +/// The snapshot is read per call, so a configuration change takes effect on the +/// next invocation with no restart and no re-registration. +macro_rules! register_blocking { + ($iii:expr, $cell:expr, $module:ident) => {{ + let cell = $cell.clone(); + $iii.register_function( + $module::ID, + RegisterFunction::new_async(move |req: $module::Request| { + let cell = cell.clone(); + async move { + let cfg = cell.read().await.clone(); + tokio::task::spawn_blocking(move || $module::handle(req, &cfg)) + .await + .map_err(|e| Error::Handler(format!("{} panicked: {e}", $module::ID)))? + .map_err(Error::Handler) + } + }) + .description($module::DESC), + ); + }}; +} + +pub fn register_all(iii: &Arc, cell: &ConfigCell) { + register_blocking!(iii, cell, classify); + register_blocking!(iii, cell, markdown); + register_blocking!(iii, cell, text); + register_blocking!(iii, cell, items); + register_blocking!(iii, cell, regions); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn catalog_lists_every_function_in_registration_order() { + let ids: Vec<&str> = catalog().iter().map(|s| s.function_id).collect(); + assert_eq!( + ids, + vec![ + "pdf::classify", + "pdf::to-markdown", + "pdf::extract-text", + "pdf::extract-items", + "pdf::extract-regions", + ] + ); + } + + /// Function ids are the public wire surface: kebab-case in multi-word + /// segments, never snake_case, and always under this worker's namespace. + #[test] + fn function_ids_follow_the_naming_rule() { + for spec in catalog() { + assert!( + spec.function_id.starts_with("pdf::"), + "{} is outside the worker namespace", + spec.function_id + ); + assert!( + !spec.function_id.contains('_'), + "{} uses snake_case; multi-word segments are kebab-case", + spec.function_id + ); + assert_eq!( + spec.function_id.to_lowercase(), + spec.function_id, + "{} is not lowercase", + spec.function_id + ); + } + } + + #[test] + fn every_function_carries_a_description() { + for spec in catalog() { + assert!( + spec.description.len() > 40, + "{} needs a description a caller can act on", + spec.function_id + ); + } + } +} diff --git a/pdf/src/functions/regions.rs b/pdf/src/functions/regions.rs new file mode 100644 index 000000000..cbaf181b2 --- /dev/null +++ b/pdf/src/functions/regions.rs @@ -0,0 +1,253 @@ +//! `pdf::extract-regions` — read only what sits inside a box. +//! +//! This is the hybrid path. A vision model looks at a rendered page, finds the +//! invoice total or the table, and hands back a bounding box. Rather than trust +//! the model's transcription of the characters, ask the document: the real +//! text is already in the file, exact, with no chance of a misread digit. +//! +//! Two modes. `text` returns the characters inside the box. `table` runs table +//! detection over the items inside the box and returns a markdown table. +//! +//! Coordinates here are PDF points with the origin at the **top left**, which +//! is what a layout model produces. `pdf::extract-items` uses bottom-left, +//! which is the PDF convention. The two disagree deliberately, each matching +//! its own callers, and every response says which it used. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::source::{describe_error, to_parser_page, to_wire_page, PdfSource}; + +pub const ID: &str = "pdf::extract-regions"; +pub const DESC: &str = "Extract the real text, or a markdown table, from inside bounding boxes on \ + given pages. Built for the hybrid path where a vision model locates a \ + region and the exact characters come from the document rather than from a \ + transcription. Coordinates are PDF points with a top-left origin."; + +/// What to pull out of each box. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Mode { + /// The characters inside the box, as flat text. + #[default] + Text, + /// A markdown table, when the items inside the box form one. + Table, +} + +/// Boxes to read on one page. +#[derive(Debug, Deserialize, JsonSchema)] +pub struct PageRegions { + /// 1-indexed page number. + pub page: u32, + /// Boxes as `[x1, y1, x2, y2]` in PDF points, origin at the top left. + pub boxes: Vec<[f32; 4]>, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: PdfSource, + + /// One entry per page, each carrying the boxes to read on it. + pub regions: Vec, + + /// Flat text, or a markdown table. + #[serde(default)] + pub mode: Mode, +} + +/// What one box held. +#[derive(Debug, Serialize, JsonSchema)] +pub struct RegionResult { + /// The text, or the markdown table in `table` mode. + pub text: String, + /// `true` when the extraction is not trustworthy: an empty box, a font the + /// parser cannot decode, or text that decodes to nonsense. In `table` mode + /// it also means no table structure was found. + pub needs_ocr: bool, + /// Machine-readable reason, when the cause is known. + #[serde(skip_serializing_if = "Option::is_none")] + pub ocr_reason: Option, +} + +/// Results for one page, parallel to that page's requested boxes. +#[derive(Debug, Serialize, JsonSchema)] +pub struct PageResult { + /// 1-indexed page number. + pub page: u32, + /// One result per requested box, in the order they were given. + pub regions: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// One entry per requested page, in the order they were given. + pub pages: Vec, + + /// Boxes read across every page. + pub region_count: usize, + + /// Boxes whose result should not be trusted. + pub regions_needing_ocr: usize, + + /// Origin convention the requested boxes were read under, always + /// `pdf-points, top-left`. Stated on every response because a caller that + /// assumed the other convention gets text from the wrong end of the page + /// with no error. Note `pdf::extract-items` reports bottom-left instead. + pub coordinate_origin: String, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the extraction. + pub elapsed_ms: u64, +} + +pub const COORDINATE_ORIGIN: &str = "pdf-points, top-left"; + +/// Boxes to read on one page, in the shape the parser takes them: a 0-indexed +/// page number and its `[x1, y1, x2, y2]` boxes. +type ParserPageRegions = (u32, Vec<[f32; 4]>); + +/// Convert the wire's 1-indexed pages to the 0-indexed pages this parser entry +/// point expects, rejecting an empty request rather than doing no work quietly. +fn to_parser_regions(regions: &[PageRegions]) -> Result, String> { + if regions.is_empty() { + return Err("`regions` was empty; give at least one page and box".to_string()); + } + regions + .iter() + .map(|r| { + if r.boxes.is_empty() { + return Err(format!("page {} was given no boxes", r.page)); + } + for b in &r.boxes { + if b[2] <= b[0] || b[3] <= b[1] { + return Err(format!( + "page {}: box [{}, {}, {}, {}] is empty or inverted; expected \ + [x1, y1, x2, y2] with x2 > x1 and y2 > y1", + r.page, b[0], b[1], b[2], b[3] + )); + } + } + Ok((to_parser_page(r.page)?, r.boxes.clone())) + }) + .collect() +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let page_regions = to_parser_regions(&req.regions)?; + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + + let extracted = match req.mode { + Mode::Text => pdf_inspector::extract_text_in_regions_mem(&bytes, &page_regions), + Mode::Table => pdf_inspector::extract_tables_in_regions_mem(&bytes, &page_regions), + } + .map_err(|e| describe_error("region extraction", e, false, false))?; + + let mut region_count = 0usize; + let mut regions_needing_ocr = 0usize; + let pages: Vec = extracted + .into_iter() + .map(|p| { + let regions: Vec = p + .regions + .into_iter() + .map(|r| { + region_count += 1; + if r.needs_ocr { + regions_needing_ocr += 1; + } + RegionResult { + text: r.text, + needs_ocr: r.needs_ocr, + ocr_reason: r.ocr_reason, + } + }) + .collect(); + PageResult { + page: to_wire_page(p.page), + regions, + } + }) + .collect(); + + Ok(Response { + pages, + region_count, + regions_needing_ocr, + coordinate_origin: COORDINATE_ORIGIN.to_string(), + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn regions(page: u32) -> PageRegions { + PageRegions { + page, + boxes: vec![[10.0, 20.0, 110.0, 60.0]], + } + } + + #[test] + fn pages_are_converted_to_the_parsers_zero_indexed_numbering() { + let converted = to_parser_regions(&[regions(1), regions(5)]).expect("valid"); + assert_eq!(converted[0].0, 0); + assert_eq!(converted[1].0, 4); + } + + #[test] + fn page_zero_is_rejected_rather_than_wrapping() { + let err = to_parser_regions(&[regions(0)]).expect_err("page 0"); + assert!(err.contains("1-indexed"), "{err}"); + } + + #[test] + fn an_empty_request_is_a_caller_mistake() { + let err = to_parser_regions(&[]).expect_err("no regions"); + assert!(err.contains("at least one"), "{err}"); + + let err = to_parser_regions(&[PageRegions { + page: 1, + boxes: vec![], + }]) + .expect_err("no boxes"); + assert!(err.contains("no boxes"), "{err}"); + } + + /// An inverted box silently returns nothing, which reads as "the document + /// has no text there" rather than "the caller swapped two numbers". + #[test] + fn an_inverted_box_is_rejected() { + let err = to_parser_regions(&[PageRegions { + page: 1, + boxes: vec![[110.0, 60.0, 10.0, 20.0]], + }]) + .expect_err("inverted"); + assert!(err.contains("inverted"), "{err}"); + } + + #[test] + fn mode_defaults_to_text() { + assert_eq!(Mode::default(), Mode::Text); + } + + /// The two coordinate conventions in this worker must stay distinct and + /// explicit; collapsing them would be silently wrong, not loudly broken. + #[test] + fn region_and_item_origins_disagree_on_purpose() { + assert_ne!( + COORDINATE_ORIGIN, + crate::functions::items::COORDINATE_ORIGIN + ); + assert!(COORDINATE_ORIGIN.contains("top-left")); + assert!(crate::functions::items::COORDINATE_ORIGIN.contains("bottom-left")); + } +} diff --git a/pdf/src/functions/text.rs b/pdf/src/functions/text.rs new file mode 100644 index 000000000..5c78111f9 --- /dev/null +++ b/pdf/src/functions/text.rs @@ -0,0 +1,77 @@ +//! `pdf::extract-text` — the plain-text reading, with no attempt at structure. +//! +//! Cheaper than markdown and the right call when the caller is going to search +//! or embed the result rather than read it. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::source::{describe_error, Body, PdfSource}; + +pub const ID: &str = "pdf::extract-text"; +pub const DESC: &str = "Extract a PDF as plain text, with no structure recovery. Cheaper than \ + pdf::to-markdown and the right call when the text will be searched or \ + embedded rather than read."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: PdfSource, + + /// Characters to return before truncating. Omit for the configured + /// default; `0` returns the whole document. + #[serde(default)] + pub max_chars: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The text, capped per `max_chars`. + pub body: Body, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the extraction. + pub elapsed_ms: u64, +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + let text = pdf_inspector::extractor::extract_text_mem(&bytes) + .map_err(|e| describe_error("text extraction", e, false, false))?; + + Ok(Response { + body: Body::new( + text, + cfg.effective_max_chars(req.max_chars), + cfg.preview_chars, + ), + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_a_document_that_is_not_a_pdf() { + use base64::engine::general_purpose::STANDARD as BASE64; + use base64::Engine as _; + + let req = Request { + source: PdfSource { + path: None, + bytes_base64: Some(BASE64.encode(b"this is not a pdf")), + fs_scope: None, + }, + max_chars: None, + }; + let err = handle(req, &WorkerConfig::default()).expect_err("not a pdf"); + assert!(err.contains("extract"), "{err}"); + } +} diff --git a/pdf/src/lib.rs b/pdf/src/lib.rs new file mode 100644 index 000000000..c96e30e9b --- /dev/null +++ b/pdf/src/lib.rs @@ -0,0 +1,7 @@ +pub mod cmaps; +pub mod config; +pub mod configuration; +pub mod functions; +pub mod manifest; +pub mod source; +pub mod ui; diff --git a/pdf/src/main.rs b/pdf/src/main.rs new file mode 100644 index 000000000..85dd3253f --- /dev/null +++ b/pdf/src/main.rs @@ -0,0 +1,146 @@ +//! The pdf worker: read PDFs locally, and know which pages need OCR. +//! +//! Boot order, and why: +//! +//! 1. tracing, then the CLI +//! 2. `--manifest` prints and returns without connecting, because the registry +//! publish pipeline calls it and must not need an engine +//! 3. materialize the CJK CMaps, before any document is parsed +//! 4. connect +//! 5. register and fetch the configuration — a required boot dependency, so a +//! failure here aborts rather than running on guessed limits +//! 6. register the functions, then the console UI they drive +//! 7. bind the configuration trigger LAST, so its handler closes over fully +//! built state +//! 8. wait for a signal, then shut the SDK down cleanly + +use std::sync::Arc; + +use clap::Parser; +use iii_sdk::runtime::WorkerMetadata; +use iii_sdk::{register_worker, InitOptions}; +use tokio::sync::RwLock; +use tracing_subscriber::EnvFilter; + +use pdf::config::WorkerConfig; +use pdf::configuration::ConfigCell; +use pdf::{cmaps, configuration, functions, manifest, ui}; + +#[derive(Parser, Debug)] +#[command(name = "pdf", about = manifest::DESCRIPTION)] +struct Cli { + /// Optional one-time seed for the configuration entry on first + /// registration. Never overwrites a stored value. + #[arg(long)] + config: Option, + + /// Engine websocket URL. + #[arg(long, env = "III_URL", default_value = "ws://127.0.0.1:49134")] + url: String, + + /// Print the registry manifest and exit. + #[arg(long)] + manifest: bool, +} + +/// Wait for either interrupt or terminate. +/// +/// A managed worker is stopped with SIGTERM, and a process that only listens +/// for ctrl-c dies without running `shutdown_async`, which leaves its +/// Message-path triggers registered against a function that no longer exists. +#[cfg(unix)] +async fn wait_for_shutdown() -> anyhow::Result<()> { + use tokio::signal::unix::{signal, SignalKind}; + let mut terminate = signal(SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {} + } + Ok(()) +} + +#[cfg(not(unix))] +async fn wait_for_shutdown() -> anyhow::Result<()> { + tokio::signal::ctrl_c().await?; + Ok(()) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + let cli = Cli::parse(); + + if cli.manifest { + println!( + "{}", + serde_json::to_string_pretty(&manifest::build_manifest())? + ); + return Ok(()); + } + + // Before anything parses a document: without this, CID fonts with no + // ToUnicode table decode to empty text and nothing says so. + cmaps::materialize(); + + let iii = register_worker( + &cli.url, + InitOptions { + metadata: Some(WorkerMetadata { + runtime: "rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + name: "pdf".to_string(), + os: std::env::consts::OS.to_string(), + pid: Some(std::process::id()), + telemetry: None, + ..WorkerMetadata::default() + }), + ..InitOptions::default() + }, + ); + let iii = Arc::new(iii); + + // A malformed seed warns and falls through: the stored value or the + // built-in default still applies, and refusing to boot over a seed file + // would be worse than ignoring it. + let seed = cli + .config + .as_deref() + .and_then(|path| match WorkerConfig::from_file(path) { + Ok(cfg) => Some(cfg), + Err(e) => { + tracing::warn!(error = %e, path, "failed to parse config seed; ignoring it"); + None + } + }); + + configuration::register_config(&iii, seed.as_ref()) + .await + .map_err(|e| anyhow::anyhow!("configuration::register failed: {e}"))?; + let cfg = configuration::fetch_config(&iii) + .await + .map_err(|e| anyhow::anyhow!("configuration::get failed: {e}"))?; + tracing::info!( + max_input_bytes = cfg.max_input_bytes, + max_chars = cfg.max_chars, + classify_sample_pages = cfg.classify_sample_pages, + "configuration loaded" + ); + let cell: ConfigCell = Arc::new(RwLock::new(Arc::new(cfg))); + + functions::register_all(&iii, &cell); + ui::register(&iii); + + configuration::register_config_trigger(&iii, cell.clone()) + .map_err(|e| anyhow::anyhow!("configuration trigger registration failed: {e}"))?; + + tracing::info!(url = %cli.url, "pdf worker ready"); + + wait_for_shutdown().await?; + iii.shutdown_async().await; + Ok(()) +} diff --git a/pdf/src/manifest.rs b/pdf/src/manifest.rs new file mode 100644 index 000000000..2d1c49ddc --- /dev/null +++ b/pdf/src/manifest.rs @@ -0,0 +1,64 @@ +//! The `--manifest` payload the registry publish pipeline reads. +//! +//! Printed without connecting to the engine, so it stays fast and +//! side-effect-free. + +use serde::Serialize; + +use crate::config::WorkerConfig; + +#[derive(Debug, Serialize)] +pub struct ModuleManifest { + pub name: String, + pub version: String, + pub description: String, + pub default_config: serde_json::Value, + pub supported_targets: Vec, +} + +pub const DESCRIPTION: &str = + "Read PDFs locally: classify text-based versus scanned, convert to markdown, extract \ + positioned text and region text, and report which pages still need OCR."; + +pub fn build_manifest() -> ModuleManifest { + ModuleManifest { + name: env!("CARGO_PKG_NAME").to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + description: DESCRIPTION.to_string(), + default_config: WorkerConfig::default().to_json(), + supported_targets: vec![env!("TARGET").to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `POST /publish` rejects a manifest missing any of the five fields. + #[test] + fn manifest_carries_every_required_field() { + let json = serde_json::to_value(build_manifest()).expect("manifest serializes"); + assert_eq!(json["name"], "pdf"); + assert!(json["version"].as_str().is_some_and(|v| !v.is_empty())); + assert!(json["description"].as_str().is_some_and(|d| d.len() > 20)); + assert!(json["default_config"].is_object()); + assert!(json["supported_targets"] + .as_array() + .is_some_and(|t| !t.is_empty())); + } + + /// The manifest name is the folder name, the binary name, and the registry + /// key. A drift here breaks the release, not the build. + #[test] + fn manifest_name_matches_the_worker_name() { + assert_eq!(build_manifest().name, "pdf"); + } + + #[test] + fn default_config_mirrors_the_shipped_defaults() { + assert_eq!( + build_manifest().default_config, + WorkerConfig::default().to_json() + ); + } +} diff --git a/pdf/src/source.rs b/pdf/src/source.rs new file mode 100644 index 000000000..83f154380 --- /dev/null +++ b/pdf/src/source.rs @@ -0,0 +1,498 @@ +//! How a PDF reaches a handler, and the conventions every handler shares. +//! +//! Two shapes, one of them required: a filesystem `path`, or `bytes_base64` +//! for a document that only exists in memory. Both land as one owned buffer, +//! because the parser wants a slice and every function in this worker reads the +//! whole file anyway. +//! +//! Page numbers are the other shared convention. The parser is internally +//! inconsistent about them: some results count pages from one, some from zero, +//! and its own page filters disagree with each other. Every number crossing +//! this worker's wire is 1-indexed, and the conversions live here so a caller +//! never has to know which side of the boundary a number came from. + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +use crate::config::WorkerConfig; + +/// The filesystem jail a call runs under. +/// +/// The harness stamps this onto every function it dispatches, so a `path` an +/// agent supplies has to be checked against it. Without the check these +/// functions would read any document on the machine and hand back its text, +/// which is a way around the scope the session was granted. Mirrors the shape +/// the shell worker takes. +#[derive(Debug, Clone, Default, Deserialize, JsonSchema)] +pub struct FsScope { + /// The session's working directory. + pub root: String, + /// Additional directories or files explicitly granted to this session. + #[serde(default)] + pub grants: Vec, +} + +/// Where the PDF comes from. Exactly one of the two fields must be set. +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct PdfSource { + /// Filesystem path to the PDF. Mutually exclusive with `bytes_base64`. + #[serde(default)] + pub path: Option, + + /// Base64-encoded PDF bytes, for a document with no path. Mutually + /// exclusive with `path`. + #[serde(default)] + pub bytes_base64: Option, + + /// The filesystem jail this call runs under. Stamped by the harness on an + /// agent's call; absent on an operator or console call, which is already + /// user-initiated and not subject to the agent's scope. + #[serde(default)] + pub fs_scope: Option, +} + +impl PdfSource { + /// Read the document into memory, enforcing the configured size ceiling + /// before anything is parsed. + pub fn load(&self, cfg: &WorkerConfig) -> Result, String> { + match (&self.path, &self.bytes_base64) { + (Some(_), Some(_)) => { + Err("provide either `path` or `bytes_base64`, not both".to_string()) + } + (None, None) => Err("provide a `path` or `bytes_base64`".to_string()), + (Some(path), None) => Self::read_file(path, self.fs_scope.as_ref(), cfg), + (None, Some(encoded)) => Self::decode(encoded, cfg), + } + } + + /// A short label for logs and responses: the file name, or a note that the + /// document arrived inline. Never the full path, which may be sensitive. + pub fn label(&self) -> String { + match (&self.path, &self.bytes_base64) { + (Some(path), _) => std::path::Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| path.clone()), + _ => "".to_string(), + } + } + + fn read_file( + path: &str, + scope: Option<&FsScope>, + cfg: &WorkerConfig, + ) -> Result, String> { + // Resolve before checking. A path is only inside the jail once symlinks + // and `..` are gone, and `metadata` would follow a symlink out of it. + let resolved = std::fs::canonicalize(path).map_err(|e| format!("{path}: {e}"))?; + if let Some(scope) = scope { + authorize(&resolved, scope)?; + } + let meta = std::fs::metadata(&resolved).map_err(|e| format!("{path}: {e}"))?; + if !meta.is_file() { + return Err(format!("{path}: not a file")); + } + check_size(meta.len(), cfg)?; + std::fs::read(&resolved).map_err(|e| format!("{path}: {e}")) + } + + fn decode(encoded: &str, cfg: &WorkerConfig) -> Result, String> { + // Reject on the encoded length first: decoding a huge blob to find out + // it is too large defeats the ceiling. + check_size((encoded.len() as u64 / 4) * 3, cfg)?; + let bytes = BASE64 + .decode(encoded.as_bytes()) + .map_err(|e| format!("bytes_base64 is not valid base64: {e}"))?; + check_size(bytes.len() as u64, cfg)?; + Ok(bytes) + } +} + +/// Reject a resolved path that sits outside the session's jail. +/// +/// The comparison is on canonical paths and whole path components, so a +/// sibling directory whose name merely starts with the root (`/w/project-old` +/// against a root of `/w/project`) is not treated as inside it. +fn authorize(resolved: &Path, scope: &FsScope) -> Result<(), String> { + let allowed = std::iter::once(&scope.root).chain(scope.grants.iter()); + for entry in allowed { + // A grant that does not resolve is a stale grant, not a reason to fail + // the call: skip it and let the remaining ones decide. + let Ok(base) = std::fs::canonicalize(entry) else { + continue; + }; + if resolved == base || resolved.starts_with(&base) { + return Ok(()); + } + } + Err(format!( + "{} is outside this session's filesystem scope", + resolved.display() + )) +} + +fn check_size(bytes: u64, cfg: &WorkerConfig) -> Result<(), String> { + if cfg.max_input_bytes > 0 && bytes > cfg.max_input_bytes { + return Err(format!( + "document is {bytes} bytes, over the configured max_input_bytes of {}", + cfg.max_input_bytes + )); + } + Ok(()) +} + +/// A body that may have been shortened to fit one response, and the numbers a +/// caller needs to decide what to do about it. +/// +/// The cap is what keeps a long document from flooding a model's context. A +/// caller that genuinely wants the whole thing asks for `max_chars: 0`, which +/// is the shape a worker-to-worker pipeline uses to move a document without it +/// passing through anyone's context. +#[derive(Debug, Serialize, JsonSchema)] +pub struct Body { + /// The content, shortened to the effective character cap. + pub text: String, + + /// Characters returned in `text`. + pub chars: usize, + + /// Characters the document actually holds. Equal to `chars` when nothing + /// was dropped. + pub total_chars: usize, + + /// `true` when `text` stops short of the document. Ask again with + /// `max_chars: 0` to take everything, or on the functions that accept one, + /// narrow with a `pages` filter. + pub truncated: bool, + + /// Leading characters of the content. Present only when the body was + /// truncated, so a caller can see the shape of what it did not get without + /// re-reading the start of `text`. + #[serde(skip_serializing_if = "Option::is_none")] + pub preview: Option, +} + +impl Body { + /// Build a response body, applying `max_chars` (`0` means uncapped) on a + /// character boundary. + pub fn new(full: String, max_chars: usize, preview_chars: usize) -> Self { + let total_chars = full.chars().count(); + if max_chars == 0 || total_chars <= max_chars { + return Self { + chars: total_chars, + total_chars, + text: full, + truncated: false, + preview: None, + }; + } + let text: String = full.chars().take(max_chars).collect(); + let preview: String = full.chars().take(preview_chars).collect(); + Self { + chars: text.chars().count(), + total_chars, + text, + truncated: true, + preview: Some(preview), + } + } +} + +/// Turn a parser error into something the caller can act on. +/// +/// "PDF is encrypted" is the parser's answer to three different situations, and +/// the caller's next move differs in each: supply a password, supply a +/// different password, or stop asking this function. `password_supported` says +/// whether the calling function has a password parameter at all, because three +/// of the five do not and telling someone to pass one there wastes their turn. +pub fn describe_error( + what: &str, + err: impl std::fmt::Display, + supplied: bool, + password_supported: bool, +) -> String { + let text = err.to_string(); + if !text.to_lowercase().contains("encrypt") { + return format!("{what} failed: {text}"); + } + let advice = match (supplied, password_supported) { + (true, _) => "the supplied password did not open it", + (false, true) => "pass the document's password", + (false, false) => { + "this function cannot decrypt; use pdf::classify or pdf::to-markdown, which take a password" + } + }; + format!("{what} failed: the document is encrypted and {advice}") +} + +/// Convert a 0-indexed page number from the parser to the 1-indexed number this +/// worker puts on the wire. +pub fn to_wire_page(zero_indexed: u32) -> u32 { + zero_indexed + 1 +} + +/// Convert a 1-indexed page number from the wire to the 0-indexed number the +/// parser's per-page extraction expects. +pub fn to_parser_page(one_indexed: u32) -> Result { + one_indexed + .checked_sub(1) + .ok_or_else(|| "page numbers are 1-indexed; 0 is not a page".to_string()) +} + +/// Convert a whole 1-indexed page list for the parser, rejecting `0` rather +/// than silently wrapping it to the last page. +pub fn to_parser_pages(pages: &[u32]) -> Result, String> { + pages.iter().copied().map(to_parser_page).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg() -> WorkerConfig { + WorkerConfig::default() + } + + #[test] + fn requires_exactly_one_input() { + let err = PdfSource::default().load(&cfg()).expect_err("neither"); + assert!(err.contains("provide a `path`"), "{err}"); + + let both = PdfSource { + path: Some("a.pdf".into()), + bytes_base64: Some("AAAA".into()), + fs_scope: None, + }; + let err = both.load(&cfg()).expect_err("both"); + assert!(err.contains("not both"), "{err}"); + } + + #[test] + fn decodes_inline_bytes() { + let src = PdfSource { + path: None, + bytes_base64: Some(BASE64.encode(b"%PDF-1.4")), + fs_scope: None, + }; + assert_eq!(src.load(&cfg()).expect("decodes"), b"%PDF-1.4"); + } + + #[test] + fn rejects_malformed_base64() { + let src = PdfSource { + path: None, + bytes_base64: Some("not base64!!!".into()), + fs_scope: None, + }; + let err = src.load(&cfg()).expect_err("malformed"); + assert!(err.contains("not valid base64"), "{err}"); + } + + #[test] + fn enforces_the_size_ceiling_before_decoding() { + let cfg = WorkerConfig { + max_input_bytes: 4, + ..WorkerConfig::default() + }; + let src = PdfSource { + path: None, + bytes_base64: Some(BASE64.encode(vec![0u8; 1024])), + fs_scope: None, + }; + let err = src.load(&cfg).expect_err("over the ceiling"); + assert!(err.contains("max_input_bytes"), "{err}"); + } + + /// The harness stamps a scope on every call it dispatches. Without this + /// check an agent could read any document on the machine and get its text + /// back, which is a way around the scope its session was granted. + #[test] + fn a_path_outside_the_session_scope_is_refused() { + let dir = tempfile::tempdir().expect("temp dir"); + let inside = dir.path().join("report.pdf"); + std::fs::write(&inside, b"%PDF-1.4").expect("write"); + + let outside = tempfile::tempdir().expect("second temp dir"); + let secret = outside.path().join("payroll.pdf"); + std::fs::write(&secret, b"%PDF-1.4").expect("write"); + + let scope = FsScope { + root: dir.path().to_string_lossy().to_string(), + grants: vec![], + }; + + let allowed = PdfSource { + path: Some(inside.to_string_lossy().to_string()), + bytes_base64: None, + fs_scope: Some(scope.clone()), + }; + assert!(allowed.load(&cfg()).is_ok(), "a path inside the root reads"); + + let refused = PdfSource { + path: Some(secret.to_string_lossy().to_string()), + bytes_base64: None, + fs_scope: Some(scope), + }; + let err = refused.load(&cfg()).expect_err("outside the scope"); + assert!( + err.contains("outside this session's filesystem scope"), + "{err}" + ); + } + + #[test] + fn an_explicit_grant_widens_the_scope() { + let root = tempfile::tempdir().expect("temp dir"); + let granted = tempfile::tempdir().expect("granted dir"); + let doc = granted.path().join("statement.pdf"); + std::fs::write(&doc, b"%PDF-1.4").expect("write"); + + let source = PdfSource { + path: Some(doc.to_string_lossy().to_string()), + bytes_base64: None, + fs_scope: Some(FsScope { + root: root.path().to_string_lossy().to_string(), + grants: vec![granted.path().to_string_lossy().to_string()], + }), + }; + assert!(source.load(&cfg()).is_ok(), "an explicit grant is honoured"); + } + + /// A sibling whose name merely starts with the root is not inside it. + /// A prefix comparison on strings would let `/w/project-old` pass for a + /// root of `/w/project`. + #[test] + fn a_sibling_directory_with_a_shared_prefix_is_not_inside_the_scope() { + let parent = tempfile::tempdir().expect("temp dir"); + let root = parent.path().join("project"); + let sibling = parent.path().join("project-old"); + std::fs::create_dir_all(&root).expect("root"); + std::fs::create_dir_all(&sibling).expect("sibling"); + let doc = sibling.join("secret.pdf"); + std::fs::write(&doc, b"%PDF-1.4").expect("write"); + + let source = PdfSource { + path: Some(doc.to_string_lossy().to_string()), + bytes_base64: None, + fs_scope: Some(FsScope { + root: root.to_string_lossy().to_string(), + grants: vec![], + }), + }; + let err = source.load(&cfg()).expect_err("sibling is outside"); + assert!(err.contains("outside"), "{err}"); + } + + /// Inline bytes carry no path, so there is nothing to escape and the scope + /// does not apply. + #[test] + fn inline_bytes_are_unaffected_by_a_scope() { + let source = PdfSource { + path: None, + bytes_base64: Some(BASE64.encode(b"%PDF-1.4")), + fs_scope: Some(FsScope { + root: "/nowhere".to_string(), + grants: vec![], + }), + }; + assert!(source.load(&cfg()).is_ok()); + } + + #[test] + fn label_never_leaks_the_directory() { + let src = PdfSource { + path: Some("/home/someone/private/report.pdf".into()), + bytes_base64: None, + fs_scope: None, + }; + assert_eq!(src.label(), "report.pdf"); + assert_eq!(PdfSource::default().label(), ""); + } + + #[test] + fn body_reports_what_it_dropped() { + let body = Body::new("abcdefghij".to_string(), 4, 2); + assert_eq!(body.text, "abcd"); + assert_eq!(body.chars, 4); + assert_eq!(body.total_chars, 10); + assert!(body.truncated); + assert_eq!(body.preview.as_deref(), Some("ab")); + } + + #[test] + fn body_uncapped_when_max_chars_is_zero() { + let body = Body::new("abcdefghij".to_string(), 0, 2); + assert_eq!(body.text, "abcdefghij"); + assert!(!body.truncated); + assert!(body.preview.is_none()); + } + + /// Truncation must not split a multi-byte character. + #[test] + fn body_truncates_on_character_boundaries() { + let body = Body::new("日本語のテキスト".to_string(), 3, 2); + assert_eq!(body.text, "日本語"); + assert_eq!(body.total_chars, 8); + } + + /// Three situations behind one parser message, three different next moves. + #[test] + fn encryption_errors_say_what_to_do_next() { + let no_password = describe_error("classify", "PDF is encrypted", false, true); + assert!( + no_password.contains("pass the document's password"), + "{no_password}" + ); + + let wrong_password = describe_error("classify", "PDF is encrypted", true, true); + assert!( + wrong_password.contains("did not open it"), + "{wrong_password}" + ); + + let unsupported = describe_error("extract", "PDF is encrypted", false, false); + assert!(unsupported.contains("pdf::to-markdown"), "{unsupported}"); + } + + /// The password must not survive into an error string. The parser is what + /// produces this text, so the test feeds in an error that DOES carry the + /// password and asserts the rewrite drops it. The previous version passed a + /// message with no password in it, so it proved nothing. + #[test] + fn encryption_errors_never_echo_the_password() { + let leaky = "PDF is encrypted: bad password 'hunter2-secret'"; + let message = describe_error("classify", leaky, true, true); + assert!( + !message.contains("hunter2-secret"), + "the password survived into the error: {message}" + ); + assert!( + message.contains("did not open it"), + "the caller still needs to know the password was wrong: {message}" + ); + } + + #[test] + fn other_errors_pass_through_unchanged() { + let message = describe_error("extract", "not a PDF file", false, true); + assert_eq!(message, "extract failed: not a PDF file"); + } + + #[test] + fn page_conversions_round_trip() { + assert_eq!(to_wire_page(0), 1); + assert_eq!(to_parser_page(1), Ok(0)); + assert_eq!(to_parser_pages(&[1, 3, 5]), Ok(vec![0, 2, 4])); + } + + /// Page 0 is a caller mistake, and `0 - 1` on a u32 would wrap to the last + /// page of a very different document. + #[test] + fn page_zero_is_rejected() { + assert!(to_parser_page(0).is_err()); + assert!(to_parser_pages(&[1, 0]).is_err()); + } +} diff --git a/pdf/src/ui.rs b/pdf/src/ui.rs new file mode 100644 index 000000000..0ea12ff2c --- /dev/null +++ b/pdf/src/ui.rs @@ -0,0 +1,70 @@ +//! Injectable console UI for the pdf worker. +//! +//! Ships two assets into any running console: +//! +//! - `pdf/page.js` (`console:script`) — a page that takes a document and shows +//! what the agent sees: the classification verdict, the per-page OCR +//! decision, and the extracted markdown. +//! - `pdf/styles.css` (`console:style`) — the stylesheet, every rule scoped +//! under `[data-iii-ui="pdf"]`. +//! +//! The registration machinery (content function `pdf::ui-content`, one +//! Message-path trigger per asset, the `III_PDF_UI_WATCH` hot-reload watcher) +//! lives in the shared `iii-console-ui` crate; this module only names the +//! assets and embeds their bytes. +//! +//! The assets are compiled from `ui/` by esbuild (react and +//! `@iii-dev/console-ui` external, so they resolve through the console's import +//! map at runtime) and embedded at compile time, so the worker stays one +//! self-contained binary. + +use std::sync::Arc; + +use iii_console_ui::ConsoleUi; +use iii_sdk::IIIClient; + +pub const PAGE_PATH: &str = "pdf/page.js"; +pub const STYLES_PATH: &str = "pdf/styles.css"; + +/// Built by `build.rs` (esbuild over `ui/`). +const PAGE_JS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/page.js")); +const STYLES_CSS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/styles.css")); + +fn console_ui() -> ConsoleUi { + ConsoleUi::new("pdf") + .script(PAGE_PATH, PAGE_JS) + .style(STYLES_PATH, STYLES_CSS) +} + +/// Register the pdf worker's console UI. Call after the functions it drives. +pub fn register(iii: &Arc) { + console_ui().register(iii); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ui_builder_accepts_the_assets() { + // The builder panics on any path or kind the console would reject. + let _ = console_ui(); + } + + #[test] + fn embedded_page_is_nonempty_esm() { + assert!(PAGE_JS.contains("export"), "built page.js looks wrong"); + } + + /// An unscoped rule in injected CSS is unlayered and silently beats the + /// console's own styles document-wide. + #[test] + fn embedded_styles_are_scoped() { + // esbuild prints the attribute selector unquoted ([data-iii-ui=pdf]). + assert!( + STYLES_CSS.contains(r#"[data-iii-ui="pdf"]"#) + || STYLES_CSS.contains("[data-iii-ui=pdf]"), + "built styles.css must be scoped under the worker's data-iii-ui attribute" + ); + } +} diff --git a/pdf/tests/fixtures.rs b/pdf/tests/fixtures.rs new file mode 100644 index 000000000..66affad88 --- /dev/null +++ b/pdf/tests/fixtures.rs @@ -0,0 +1,546 @@ +//! Behaviour against real documents. +//! +//! The unit tests cover the pure logic; these run the actual parser over the +//! committed fixtures, which is where a dependency upgrade that changes what a +//! document produces will show up. +//! +//! Both fixtures are built by `tests/fixtures/make_fixtures.py` from raw PDF +//! syntax, so their content is known exactly and the assertions can be precise +//! rather than "returns something". + +use pdf::config::WorkerConfig; +use pdf::functions::classify::DocumentType; +use pdf::functions::{classify, items, markdown, regions, text}; +use pdf::source::PdfSource; + +fn fixture(name: &str) -> PdfSource { + PdfSource { + path: Some(format!( + "{}/tests/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )), + bytes_base64: None, + // Unstamped: these exercise the handlers, not the jail, which has its + // own tests in `src/source.rs`. + fs_scope: None, + } +} + +fn cfg() -> WorkerConfig { + WorkerConfig::default() +} + +// --------------------------------------------------------------------------- +// classify +// --------------------------------------------------------------------------- + +#[test] +fn a_text_document_classifies_as_text_based_and_needs_no_ocr() { + let result = classify::handle( + classify::Request { + source: fixture("text-two-page.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify"); + + assert_eq!(result.document_type, DocumentType::TextBased); + assert_eq!(result.page_count, 2); + assert!(result.pages_needing_ocr.is_empty()); + assert!(result.ocr_reasons.is_empty()); + assert_eq!(result.source, "text-two-page.pdf"); +} + +/// The document-level verdict is not enough on its own: the per-page reason is +/// what tells a caller whether to send pages to a vision model. +#[test] +fn a_document_with_no_text_is_flagged_page_by_page() { + let result = classify::handle( + classify::Request { + source: fixture("no-text.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify"); + + assert_ne!(result.document_type, DocumentType::TextBased); + assert_eq!(result.pages_needing_ocr, vec![1]); + assert_eq!(result.ocr_reasons.len(), 1); + assert_eq!(result.ocr_reasons[0].page, 1); + assert!( + result.ocr_reasons[0] + .reasons + .contains(&"no_text".to_string()), + "expected a no_text reason, got {:?}", + result.ocr_reasons[0].reasons + ); +} + +/// Pages needing OCR are reported 1-indexed. The parser has a second +/// classification entry point that counts from zero, so a refactor onto it +/// would turn page 1 into page 0 with no compile error. +#[test] +fn ocr_pages_are_reported_one_indexed() { + let result = classify::handle( + classify::Request { + source: fixture("no-text.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify"); + + assert!( + !result.pages_needing_ocr.contains(&0), + "page 0 is not a page; the numbering slipped to 0-indexed" + ); +} + +#[test] +fn the_sample_counters_are_present_for_an_unencrypted_document() { + let result = classify::handle( + classify::Request { + source: fixture("text-two-page.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify"); + + assert_eq!(result.pages_sampled, Some(2)); + assert_eq!(result.pages_with_text, Some(2)); + assert!(result.ocr_recommended.is_some()); +} + +/// `min_text_ops_per_page` is a real lever, not decoration: a page whose text +/// operators fall under it stops counting as a text page. The fixture draws +/// three operators per page, so raising the bar past three must empty the +/// count. +/// +/// The document-level verdict deliberately is NOT asserted here. A document +/// with no images is not called scanned merely because its operator counts are +/// low, so the type stays text-based while the counter goes to zero. That is +/// the parser's behaviour and worth pinning as a fact rather than assuming the +/// verdict tracks the counter. +#[test] +fn the_text_operator_threshold_changes_what_counts_as_a_text_page() { + let default = classify::handle( + classify::Request { + source: fixture("text-two-page.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify"); + assert_eq!(default.pages_with_text, Some(2)); + + let strict = WorkerConfig { + min_text_ops_per_page: 4, + ..WorkerConfig::default() + }; + let result = classify::handle( + classify::Request { + source: fixture("text-two-page.pdf"), + password: None, + sample_pages: None, + }, + &strict, + ) + .expect("classify"); + + assert_eq!( + result.pages_with_text, + Some(0), + "with the bar above the fixture's operator count, no page should qualify" + ); + assert!( + result.confidence < default.confidence, + "a document that no longer looks like text should be reported less confidently" + ); +} + +// --------------------------------------------------------------------------- +// to-markdown +// --------------------------------------------------------------------------- + +#[test] +fn markdown_recovers_structure_from_the_page() { + let result = markdown::handle( + markdown::Request { + source: fixture("text-two-page.pdf"), + password: None, + pages: None, + max_chars: None, + profile: markdown::Profile::Fidelity, + include_images: false, + strip_headers_footers: true, + per_page: false, + }, + &cfg(), + ) + .expect("to-markdown"); + + // The 24pt line becomes a heading; the 12pt lines stay body text. Nothing + // in the document says "heading" — it is recovered from the font size. + assert!( + result.body.text.contains("# Quarterly Report"), + "expected a recovered heading, got:\n{}", + result.body.text + ); + assert!(result.body.text.contains("Revenue rose to 4.2 million")); + assert!(result.body.text.contains("# Appendix")); + assert!(!result.body.truncated); + assert_eq!(result.page_count, 2); + assert!(!result.has_encoding_issues); +} + +/// A page filter is the cheap way to read a long document, and the numbers are +/// 1-indexed on the wire while the parser's per-page entry point counts from +/// zero. Asking for page 2 must return page 2's content, not page 1's. +#[test] +fn a_page_filter_selects_that_page_and_not_its_neighbour() { + let result = markdown::handle( + markdown::Request { + source: fixture("text-two-page.pdf"), + password: None, + pages: Some(vec![2]), + max_chars: None, + profile: markdown::Profile::Fidelity, + include_images: false, + strip_headers_footers: true, + per_page: true, + }, + &cfg(), + ) + .expect("to-markdown"); + + assert!( + result.body.text.contains("Appendix"), + "page 2 should hold the appendix, got:\n{}", + result.body.text + ); + assert!( + !result.body.text.contains("Quarterly Report"), + "page 1 leaked into a page-2 request:\n{}", + result.body.text + ); + assert_eq!(result.pages_converted, 1); + + let pages = result.pages.expect("per_page requested"); + assert_eq!(pages.len(), 1); + assert_eq!(pages[0].page, 2, "per-page numbering must stay 1-indexed"); + assert!(pages[0].markdown.contains("Appendix")); +} + +/// Truncation must report what it dropped, or a caller answers from a fragment +/// believing it has the document. +#[test] +fn a_capped_response_says_how_much_it_withheld() { + let result = markdown::handle( + markdown::Request { + source: fixture("text-two-page.pdf"), + password: None, + pages: None, + max_chars: Some(20), + profile: markdown::Profile::Fidelity, + include_images: false, + strip_headers_footers: true, + per_page: false, + }, + &cfg(), + ) + .expect("to-markdown"); + + assert!(result.body.truncated); + assert_eq!(result.body.chars, 20); + assert!(result.body.total_chars > 20); + assert!(result.body.preview.is_some()); +} + +#[test] +fn max_chars_zero_returns_the_whole_document() { + let result = markdown::handle( + markdown::Request { + source: fixture("text-two-page.pdf"), + password: None, + pages: None, + max_chars: Some(0), + profile: markdown::Profile::Fidelity, + include_images: false, + strip_headers_footers: true, + per_page: false, + }, + &cfg(), + ) + .expect("to-markdown"); + + assert!(!result.body.truncated); + assert_eq!(result.body.chars, result.body.total_chars); +} + +/// A scan produces no markdown. The response must still carry the verdict, so +/// an empty body is distinguishable from a document that is genuinely blank. +#[test] +fn a_document_with_no_text_yields_no_markdown_but_still_reports_why() { + let result = markdown::handle( + markdown::Request { + source: fixture("no-text.pdf"), + password: None, + pages: None, + max_chars: None, + profile: markdown::Profile::Fidelity, + include_images: false, + strip_headers_footers: true, + per_page: false, + }, + &cfg(), + ) + .expect("to-markdown"); + + assert!(result.body.text.trim().is_empty()); + assert_ne!(result.document_type, DocumentType::TextBased); + assert_eq!(result.pages_needing_ocr, vec![1]); +} + +// --------------------------------------------------------------------------- +// extract-text +// --------------------------------------------------------------------------- + +#[test] +fn plain_text_extraction_returns_the_words_without_the_structure() { + let result = text::handle( + text::Request { + source: fixture("text-two-page.pdf"), + max_chars: Some(0), + }, + &cfg(), + ) + .expect("extract-text"); + + assert!(result.body.text.contains("Quarterly Report")); + assert!(result.body.text.contains("Appendix")); + assert!( + !result.body.text.contains("# "), + "plain text should carry no markdown markers" + ); +} + +// --------------------------------------------------------------------------- +// extract-items +// --------------------------------------------------------------------------- + +/// The positions are the reason this function exists, so assert the actual +/// numbers the fixture was drawn with rather than that items came back. +#[test] +fn items_carry_the_geometry_the_page_was_drawn_with() { + let result = items::handle( + items::Request { + source: fixture("text-two-page.pdf"), + pages: Some(vec![1]), + max_items: None, + }, + &cfg(), + ) + .expect("extract-items"); + + assert_eq!(result.total_count, 3); + assert!(!result.truncated); + assert_eq!(result.coordinate_origin, "pdf-points, bottom-left"); + + let heading = result + .items + .iter() + .find(|i| i.text.contains("Quarterly Report")) + .expect("the heading is on page 1"); + assert_eq!(heading.page, 1); + assert_eq!(heading.x, 72.0); + assert_eq!(heading.y, 700.0); + assert_eq!(heading.font_size, 24.0); + + let body = result + .items + .iter() + .find(|i| i.text.contains("Revenue rose")) + .expect("the body line is on page 1"); + assert_eq!(body.font_size, 12.0); + // Bottom-left origin: the heading sits higher on the page, so its y is + // LARGER. Under a top-left origin this comparison would invert. + assert!( + heading.y > body.y, + "with a bottom-left origin the heading must have the larger y" + ); +} + +#[test] +fn an_item_cap_reports_what_it_left_behind() { + let result = items::handle( + items::Request { + source: fixture("text-two-page.pdf"), + pages: Some(vec![1]), + max_items: Some(1), + }, + &cfg(), + ) + .expect("extract-items"); + + assert_eq!(result.count, 1); + assert_eq!(result.total_count, 3); + assert!(result.truncated); +} + +#[test] +fn a_page_filter_limits_which_items_come_back() { + let page_two = items::handle( + items::Request { + source: fixture("text-two-page.pdf"), + pages: Some(vec![2]), + max_items: None, + }, + &cfg(), + ) + .expect("extract-items"); + + assert!(page_two.items.iter().all(|i| i.page == 2)); + assert!(page_two.items.iter().any(|i| i.text.contains("Appendix"))); +} + +// --------------------------------------------------------------------------- +// extract-regions +// --------------------------------------------------------------------------- + +/// Region boxes use a TOP-left origin while items use bottom-left. A box over +/// the top of the page must therefore return the heading, and the same numbers +/// read as bottom-left would return the wrong end of the page. +#[test] +fn a_region_over_the_top_of_the_page_returns_the_heading() { + let result = regions::handle( + regions::Request { + source: fixture("text-two-page.pdf"), + regions: vec![regions::PageRegions { + page: 1, + boxes: vec![[0.0, 0.0, 612.0, 200.0]], + }], + mode: regions::Mode::Text, + }, + &cfg(), + ) + .expect("extract-regions"); + + assert_eq!(result.coordinate_origin, "pdf-points, top-left"); + assert_eq!(result.region_count, 1); + assert_eq!( + result.pages[0].page, 1, + "page numbering must stay 1-indexed" + ); + assert!( + result.pages[0].regions[0].text.contains("Quarterly Report"), + "a top-left box over the first 200 points should hold the heading, got {:?}", + result.pages[0].regions[0].text + ); +} + +#[test] +fn an_empty_region_is_reported_as_unreliable_rather_than_as_empty_text() { + let result = regions::handle( + regions::Request { + source: fixture("text-two-page.pdf"), + regions: vec![regions::PageRegions { + page: 1, + // The lower half of the page carries nothing. + boxes: vec![[0.0, 600.0, 612.0, 790.0]], + }], + mode: regions::Mode::Text, + }, + &cfg(), + ) + .expect("extract-regions"); + + assert_eq!(result.regions_needing_ocr, 1); + assert!(result.pages[0].regions[0].needs_ocr); +} + +// --------------------------------------------------------------------------- +// input handling +// --------------------------------------------------------------------------- + +/// A document handed over inline must produce the same result as the same +/// document read from disk. +#[test] +fn inline_bytes_and_a_path_agree() { + use base64::engine::general_purpose::STANDARD as BASE64; + use base64::Engine as _; + + let path = format!( + "{}/tests/fixtures/text-two-page.pdf", + env!("CARGO_MANIFEST_DIR") + ); + let bytes = std::fs::read(&path).expect("fixture readable"); + + let from_path = classify::handle( + classify::Request { + source: fixture("text-two-page.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify from path"); + + let from_bytes = classify::handle( + classify::Request { + source: PdfSource { + path: None, + bytes_base64: Some(BASE64.encode(&bytes)), + fs_scope: None, + }, + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect("classify from bytes"); + + assert_eq!(from_path.document_type, from_bytes.document_type); + assert_eq!(from_path.page_count, from_bytes.page_count); + assert_eq!(from_path.pages_needing_ocr, from_bytes.pages_needing_ocr); + assert_eq!(from_bytes.source, ""); +} + +#[test] +fn a_document_over_the_size_ceiling_is_refused_before_parsing() { + let tiny = WorkerConfig { + max_input_bytes: 16, + ..WorkerConfig::default() + }; + let err = classify::handle( + classify::Request { + source: fixture("text-two-page.pdf"), + password: None, + sample_pages: None, + }, + &tiny, + ) + .expect_err("over the ceiling"); + assert!(err.contains("max_input_bytes"), "{err}"); +} + +#[test] +fn a_missing_file_fails_with_the_path_in_the_message() { + let err = classify::handle( + classify::Request { + source: fixture("does-not-exist.pdf"), + password: None, + sample_pages: None, + }, + &cfg(), + ) + .expect_err("missing file"); + assert!(err.contains("does-not-exist.pdf"), "{err}"); +} diff --git a/pdf/tests/fixtures/README.md b/pdf/tests/fixtures/README.md new file mode 100644 index 000000000..89fb52b32 --- /dev/null +++ b/pdf/tests/fixtures/README.md @@ -0,0 +1,20 @@ +# Test fixtures + +Hand-built PDFs, generated by `make_fixtures.py` in this directory. They are +written from raw PDF syntax rather than exported from an application, so they +stay small, their content is known exactly, and they carry no third-party +licensing. + +| File | What it exercises | +|---|---| +| `text-two-page.pdf` | A text-based document. Two pages, known strings, so page filters, character caps and item positions can be asserted exactly. | +| `no-text.pdf` | Pages that draw only a filled rectangle. Nothing to extract, so classification must not call it text-based and must flag the pages for OCR. | + +Regenerate with: + +```bash +python3 tests/fixtures/make_fixtures.py +``` + +The generated files are committed. A parser upgrade that changes what these +documents produce should show up as a failing assertion, which is the point. diff --git a/pdf/tests/fixtures/make_fixtures.py b/pdf/tests/fixtures/make_fixtures.py new file mode 100644 index 000000000..6d7ebdf79 --- /dev/null +++ b/pdf/tests/fixtures/make_fixtures.py @@ -0,0 +1,90 @@ +"""Generate the committed test PDFs from raw PDF syntax. + +Written by hand rather than exported from an application: the documents stay +under two kilobytes, their content is known exactly, and nothing here is +third-party. Run from the worker root: + + python3 tests/fixtures/make_fixtures.py +""" + +from pathlib import Path + +HERE = Path(__file__).parent + + +def build(objects: list[bytes]) -> bytes: + """Assemble numbered objects into a PDF with a correct xref table.""" + out = bytearray(b"%PDF-1.4\n") + offsets = [0] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n".encode() + body + b"\nendobj\n" + + xref_at = len(out) + out += f"xref\n0 {len(objects) + 1}\n".encode() + out += b"0000000000 65535 f \n" + for offset in offsets[1:]: + out += f"{offset:010d} 00000 n \n".encode() + out += ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{xref_at}\n%%EOF\n" + ).encode() + return bytes(out) + + +def stream(content: bytes) -> bytes: + return b"<< /Length %d >>\nstream\n" % len(content) + content + b"\nendstream" + + +def text_two_page() -> bytes: + """Two pages of real text, at known positions and sizes.""" + page_one = ( + b"BT /F1 24 Tf 72 700 Td (Quarterly Report) Tj ET\n" + b"BT /F1 12 Tf 72 660 Td (Revenue rose to 4.2 million in the period.) Tj ET\n" + b"BT /F1 12 Tf 72 640 Td (Costs held flat against the prior quarter.) Tj ET\n" + ) + # Three text operators per page, deliberately: the default + # min_text_ops_per_page is 3, and a page under it does not count as a text + # page. A two-operator page here would make the fixture's own confidence + # score an artefact of the threshold rather than of the content. + page_two = ( + b"BT /F1 24 Tf 72 700 Td (Appendix) Tj ET\n" + b"BT /F1 12 Tf 72 660 Td (Figures are unaudited and stated in euro.) Tj ET\n" + b"BT /F1 12 Tf 72 640 Td (Comparatives have been restated where needed.) Tj ET\n" + ) + return build([ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 7 0 R >> >> /Contents 4 0 R >>", + stream(page_one), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 7 0 R >> >> /Contents 6 0 R >>", + stream(page_two), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ]) + + +def no_text() -> bytes: + """A page that draws a filled rectangle and nothing else. + + There is no text operator anywhere, which is what a scanned page looks like + to a parser that reads content streams. + """ + page = b"0.2 0.2 0.2 rg 72 500 468 250 re f\n" + return build([ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << >> /Contents 4 0 R >>", + stream(page), + ]) + + +if __name__ == "__main__": + for name, data in [ + ("text-two-page.pdf", text_two_page()), + ("no-text.pdf", no_text()), + ]: + (HERE / name).write_bytes(data) + print(f"{name}: {len(data)} bytes") diff --git a/pdf/tests/fixtures/no-text.pdf b/pdf/tests/fixtures/no-text.pdf new file mode 100644 index 000000000..1c387db52 Binary files /dev/null and b/pdf/tests/fixtures/no-text.pdf differ diff --git a/pdf/tests/fixtures/text-two-page.pdf b/pdf/tests/fixtures/text-two-page.pdf new file mode 100644 index 000000000..13919e4d3 Binary files /dev/null and b/pdf/tests/fixtures/text-two-page.pdf differ diff --git a/pdf/tests/golden/schemas/pdf.classify.json b/pdf/tests/golden/schemas/pdf.classify.json new file mode 100644 index 000000000..2d78c07f7 --- /dev/null +++ b/pdf/tests/golden/schemas/pdf.classify.json @@ -0,0 +1,238 @@ +{ + "description": "Classify a PDF as text-based, scanned, image-based or mixed, and report which pages need OCR and why. Samples content streams rather than extracting text, so it answers in tens of milliseconds. Call this before any other pdf function.", + "function_id": "pdf::classify", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell worker takes.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the PDF comes from. Exactly one of the two fields must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded PDF bytes, for a document with no path. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "password": { + "default": null, + "description": "Password for an encrypted document. Never logged or echoed back.", + "type": [ + "string", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the PDF. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + }, + "sample_pages": { + "default": null, + "description": "Pages sampled for the verdict, overriding the configured default. `0` scans every page, which is slower but settles a borderline mixed document.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "DocumentType": { + "description": "What a document is made of.", + "oneOf": [ + { + "description": "Real text throughout. Extract locally.", + "enum": [ + "text_based" + ], + "type": "string" + }, + { + "description": "Pictures of pages. Every page needs OCR.", + "enum": [ + "scanned" + ], + "type": "string" + }, + { + "description": "Images with little or no text layer.", + "enum": [ + "image_based" + ], + "type": "string" + }, + { + "description": "Some pages carry text, others do not. Read `pages_needing_ocr`.", + "enum": [ + "mixed" + ], + "type": "string" + } + ] + }, + "PageOcrReason": { + "description": "Why one page cannot be read without OCR.", + "properties": { + "page": { + "description": "1-indexed page number.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "reasons": { + "description": "Machine-readable reasons: `scanned` (a raster page), `no_text` (nothing extractable and nothing to OCR), `vector_text` (characters drawn as outlines rather than text) or `suspected_garbled_text` (a text layer that decodes to nonsense).", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "page", + "reasons" + ], + "type": "object" + } + }, + "properties": { + "confidence": { + "description": "How much to trust the verdict, from 0.0 to 1.0.", + "format": "float", + "type": "number" + }, + "document_type": { + "allOf": [ + { + "$ref": "#/definitions/DocumentType" + } + ], + "description": "The document-level verdict." + }, + "elapsed_ms": { + "description": "Wall-clock time for the classification.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "has_encoding_issues": { + "description": "`true` when font encodings decoded badly. Only known on the encrypted path, which extracts far enough to notice; absent otherwise, where `suspected_garbled_text` in `ocr_reasons` carries the same signal.", + "type": [ + "boolean", + "null" + ] + }, + "ocr_reasons": { + "description": "Per-page explanation for `pages_needing_ocr`.", + "items": { + "$ref": "#/definitions/PageOcrReason" + }, + "type": "array" + }, + "ocr_recommended": { + "description": "`true` when the images carry meaning the text layer does not, so OCR adds something even on a text-based document. Absent for an encrypted document, for the same reason as `pages_sampled`.", + "type": [ + "boolean", + "null" + ] + }, + "page_count": { + "description": "Pages in the document.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "pages_needing_ocr": { + "description": "1-indexed pages that cannot be read without OCR. Empty for a clean text-based document.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + }, + "pages_sampled": { + "description": "Pages actually inspected. Lower than `page_count` when sampling, so a verdict from a sample can be told apart from one that read everything. Absent for an encrypted document, which takes a decryption path that does not report the counters.", + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "pages_with_text": { + "description": "Inspected pages that carry text operators. Absent for an encrypted document, for the same reason as `pages_sampled`.", + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + }, + "title": { + "description": "Document title from the PDF metadata, when it has one.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "confidence", + "document_type", + "elapsed_ms", + "ocr_reasons", + "page_count", + "pages_needing_ocr", + "source" + ], + "title": "Response", + "type": "object" + } +} diff --git a/pdf/tests/golden/schemas/pdf.extract-items.json b/pdf/tests/golden/schemas/pdf.extract-items.json new file mode 100644 index 000000000..f69fbe946 --- /dev/null +++ b/pdf/tests/golden/schemas/pdf.extract-items.json @@ -0,0 +1,265 @@ +{ + "description": "Extract positioned text items: the box, font, size and styling of every run of characters on a page. Coordinates are PDF points with a bottom-left origin. Use this for layout-aware reading; use pdf::to-markdown to just read the document.", + "function_id": "pdf::extract-items", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell worker takes.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the PDF comes from. Exactly one of the two fields must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded PDF bytes, for a document with no path. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "max_items": { + "default": null, + "description": "Items to return before truncating. Omit for the configured default; `0` returns every item, which on a dense document is a very large response.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "pages": { + "default": null, + "description": "1-indexed pages to read. Omit for the whole document.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": [ + "array", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the PDF. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Item": { + "description": "One positioned run of characters.", + "properties": { + "bold": { + "type": "boolean" + }, + "font": { + "description": "Font name as the document names it.", + "type": "string" + }, + "font_size": { + "description": "Font size in points.", + "format": "float", + "type": "number" + }, + "height": { + "description": "Height in PDF points, approximated from the font size.", + "format": "float", + "type": "number" + }, + "italic": { + "type": "boolean" + }, + "kind": { + "$ref": "#/definitions/ItemKind" + }, + "link": { + "description": "Link target, for a `link` item.", + "type": [ + "string", + "null" + ] + }, + "mcid": { + "description": "Marked-content id tying this item to the document's tagged structure tree, when the document has one.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "page": { + "description": "1-indexed page number.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "strikeout": { + "description": "Recovered from vector lines through the text, not from a flag.", + "type": "boolean" + }, + "text": { + "description": "The characters.", + "type": "string" + }, + "underline": { + "description": "Recovered from vector lines near the baseline, not from a flag.", + "type": "boolean" + }, + "width": { + "description": "Width in PDF points.", + "format": "float", + "type": "number" + }, + "x": { + "description": "Left edge, PDF points from the left of the page.", + "format": "float", + "type": "number" + }, + "y": { + "description": "Baseline, PDF points from the **bottom** of the page.", + "format": "float", + "type": "number" + } + }, + "required": [ + "bold", + "font", + "font_size", + "height", + "italic", + "kind", + "page", + "strikeout", + "text", + "underline", + "width", + "x", + "y" + ], + "type": "object" + }, + "ItemKind": { + "description": "What one item is.", + "oneOf": [ + { + "description": "Ordinary text.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "An image placeholder. The box is real; no pixels are decoded.", + "enum": [ + "image" + ], + "type": "string" + }, + { + "description": "Text carrying a hyperlink; the target is in `link`.", + "enum": [ + "link" + ], + "type": "string" + }, + { + "description": "A filled-in form field value.", + "enum": [ + "form_field" + ], + "type": "string" + } + ] + } + }, + "properties": { + "coordinate_origin": { + "description": "Origin convention for `x` and `y`, always `pdf-points, bottom-left`. Stated on every response because a caller that assumed the other convention reads the wrong end of the page with no error. Note `pdf::extract-regions` takes boxes with a top-left origin instead.", + "type": "string" + }, + "count": { + "description": "Items returned.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "elapsed_ms": { + "description": "Wall-clock time for the extraction.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "items": { + "description": "The items, in document order, capped per `max_items`.", + "items": { + "$ref": "#/definitions/Item" + }, + "type": "array" + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + }, + "total_count": { + "description": "Items the document holds for the requested pages. Equal to `count` when nothing was dropped.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "truncated": { + "description": "`true` when `items` stops short. Narrow `pages`, or pass `max_items: 0`.", + "type": "boolean" + } + }, + "required": [ + "coordinate_origin", + "count", + "elapsed_ms", + "items", + "source", + "total_count", + "truncated" + ], + "title": "Response", + "type": "object" + } +} diff --git a/pdf/tests/golden/schemas/pdf.extract-regions.json b/pdf/tests/golden/schemas/pdf.extract-regions.json new file mode 100644 index 000000000..fac8db494 --- /dev/null +++ b/pdf/tests/golden/schemas/pdf.extract-regions.json @@ -0,0 +1,227 @@ +{ + "description": "Extract the real text, or a markdown table, from inside bounding boxes on given pages. Built for the hybrid path where a vision model locates a region and the exact characters come from the document rather than from a transcription. Coordinates are PDF points with a top-left origin.", + "function_id": "pdf::extract-regions", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell worker takes.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + }, + "Mode": { + "description": "What to pull out of each box.", + "oneOf": [ + { + "description": "The characters inside the box, as flat text.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "A markdown table, when the items inside the box form one.", + "enum": [ + "table" + ], + "type": "string" + } + ] + }, + "PageRegions": { + "description": "Boxes to read on one page.", + "properties": { + "boxes": { + "description": "Boxes as `[x1, y1, x2, y2]` in PDF points, origin at the top left.", + "items": { + "items": { + "format": "float", + "type": "number" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + "type": "array" + }, + "page": { + "description": "1-indexed page number.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "boxes", + "page" + ], + "type": "object" + } + }, + "description": "Where the PDF comes from. Exactly one of the two fields must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded PDF bytes, for a document with no path. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "mode": { + "allOf": [ + { + "$ref": "#/definitions/Mode" + } + ], + "default": "text", + "description": "Flat text, or a markdown table." + }, + "path": { + "default": null, + "description": "Filesystem path to the PDF. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + }, + "regions": { + "description": "One entry per page, each carrying the boxes to read on it.", + "items": { + "$ref": "#/definitions/PageRegions" + }, + "type": "array" + } + }, + "required": [ + "regions" + ], + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "PageResult": { + "description": "Results for one page, parallel to that page's requested boxes.", + "properties": { + "page": { + "description": "1-indexed page number.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "regions": { + "description": "One result per requested box, in the order they were given.", + "items": { + "$ref": "#/definitions/RegionResult" + }, + "type": "array" + } + }, + "required": [ + "page", + "regions" + ], + "type": "object" + }, + "RegionResult": { + "description": "What one box held.", + "properties": { + "needs_ocr": { + "description": "`true` when the extraction is not trustworthy: an empty box, a font the parser cannot decode, or text that decodes to nonsense. In `table` mode it also means no table structure was found.", + "type": "boolean" + }, + "ocr_reason": { + "description": "Machine-readable reason, when the cause is known.", + "type": [ + "string", + "null" + ] + }, + "text": { + "description": "The text, or the markdown table in `table` mode.", + "type": "string" + } + }, + "required": [ + "needs_ocr", + "text" + ], + "type": "object" + } + }, + "properties": { + "coordinate_origin": { + "description": "Origin convention the requested boxes were read under, always `pdf-points, top-left`. Stated on every response because a caller that assumed the other convention gets text from the wrong end of the page with no error. Note `pdf::extract-items` reports bottom-left instead.", + "type": "string" + }, + "elapsed_ms": { + "description": "Wall-clock time for the extraction.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "pages": { + "description": "One entry per requested page, in the order they were given.", + "items": { + "$ref": "#/definitions/PageResult" + }, + "type": "array" + }, + "region_count": { + "description": "Boxes read across every page.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "regions_needing_ocr": { + "description": "Boxes whose result should not be trusted.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + } + }, + "required": [ + "coordinate_origin", + "elapsed_ms", + "pages", + "region_count", + "regions_needing_ocr", + "source" + ], + "title": "Response", + "type": "object" + } +} diff --git a/pdf/tests/golden/schemas/pdf.extract-text.json b/pdf/tests/golden/schemas/pdf.extract-text.json new file mode 100644 index 000000000..c1a70e30b --- /dev/null +++ b/pdf/tests/golden/schemas/pdf.extract-text.json @@ -0,0 +1,143 @@ +{ + "description": "Extract a PDF as plain text, with no structure recovery. Cheaper than pdf::to-markdown and the right call when the text will be searched or embedded rather than read.", + "function_id": "pdf::extract-text", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell worker takes.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the PDF comes from. Exactly one of the two fields must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded PDF bytes, for a document with no path. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "max_chars": { + "default": null, + "description": "Characters to return before truncating. Omit for the configured default; `0` returns the whole document.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the PDF. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Body": { + "description": "A body that may have been shortened to fit one response, and the numbers a caller needs to decide what to do about it.\n\nThe cap is what keeps a long document from flooding a model's context. A caller that genuinely wants the whole thing asks for `max_chars: 0`, which is the shape a worker-to-worker pipeline uses to move a document without it passing through anyone's context.", + "properties": { + "chars": { + "description": "Characters returned in `text`.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "preview": { + "description": "Leading characters of the content. Present only when the body was truncated, so a caller can see the shape of what it did not get without re-reading the start of `text`.", + "type": [ + "string", + "null" + ] + }, + "text": { + "description": "The content, shortened to the effective character cap.", + "type": "string" + }, + "total_chars": { + "description": "Characters the document actually holds. Equal to `chars` when nothing was dropped.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "truncated": { + "description": "`true` when `text` stops short of the document. Ask again with `max_chars: 0` to take everything, or on the functions that accept one, narrow with a `pages` filter.", + "type": "boolean" + } + }, + "required": [ + "chars", + "text", + "total_chars", + "truncated" + ], + "type": "object" + } + }, + "properties": { + "body": { + "allOf": [ + { + "$ref": "#/definitions/Body" + } + ], + "description": "The text, capped per `max_chars`." + }, + "elapsed_ms": { + "description": "Wall-clock time for the extraction.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + } + }, + "required": [ + "body", + "elapsed_ms", + "source" + ], + "title": "Response", + "type": "object" + } +} diff --git a/pdf/tests/golden/schemas/pdf.to-markdown.json b/pdf/tests/golden/schemas/pdf.to-markdown.json new file mode 100644 index 000000000..1f25c4a66 --- /dev/null +++ b/pdf/tests/golden/schemas/pdf.to-markdown.json @@ -0,0 +1,371 @@ +{ + "description": "Convert a text-based PDF to markdown, preserving headings, lists, links and tables. Returns nothing for a scanned document — call pdf::classify first. Responses are capped; pass max_chars 0 to take the whole document, or pages to take a slice of it.", + "function_id": "pdf::to-markdown", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell worker takes.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + }, + "Profile": { + "description": "How faithful the markdown should be to the source characters.", + "oneOf": [ + { + "description": "Preserve the source text as written.", + "enum": [ + "fidelity" + ], + "type": "string" + }, + { + "description": "Prefer shorter output, collapsing runs like the dot leaders in a table of contents. Not character-faithful to the source.", + "enum": [ + "compact" + ], + "type": "string" + } + ] + } + }, + "description": "Where the PDF comes from. Exactly one of the two fields must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded PDF bytes, for a document with no path. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "include_images": { + "default": false, + "description": "Include `[Image: …]` placeholders. Off by default: nothing here decodes pixels, so a placeholder adds noise without adding information.", + "type": "boolean" + }, + "max_chars": { + "default": null, + "description": "Characters to return before truncating. Omit for the configured default; `0` returns the whole document.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "pages": { + "default": null, + "description": "1-indexed pages to convert. Omit for the whole document. A page filter is the cheap way to read a long report: take the pages you need rather than the whole thing truncated.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": [ + "array", + "null" + ] + }, + "password": { + "default": null, + "description": "Password for an encrypted document. Never logged or echoed back.", + "type": [ + "string", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the PDF. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + }, + "per_page": { + "default": false, + "description": "Return markdown per page as well as the joined document. Useful when a caller wants to route some pages to OCR and keep the rest.", + "type": "boolean" + }, + "profile": { + "allOf": [ + { + "$ref": "#/definitions/Profile" + } + ], + "default": "fidelity", + "description": "Source fidelity versus token efficiency." + }, + "strip_headers_footers": { + "default": true, + "description": "Strip repeated running headers and footers.", + "type": "boolean" + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Body": { + "description": "A body that may have been shortened to fit one response, and the numbers a caller needs to decide what to do about it.\n\nThe cap is what keeps a long document from flooding a model's context. A caller that genuinely wants the whole thing asks for `max_chars: 0`, which is the shape a worker-to-worker pipeline uses to move a document without it passing through anyone's context.", + "properties": { + "chars": { + "description": "Characters returned in `text`.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "preview": { + "description": "Leading characters of the content. Present only when the body was truncated, so a caller can see the shape of what it did not get without re-reading the start of `text`.", + "type": [ + "string", + "null" + ] + }, + "text": { + "description": "The content, shortened to the effective character cap.", + "type": "string" + }, + "total_chars": { + "description": "Characters the document actually holds. Equal to `chars` when nothing was dropped.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "truncated": { + "description": "`true` when `text` stops short of the document. Ask again with `max_chars: 0` to take everything, or on the functions that accept one, narrow with a `pages` filter.", + "type": "boolean" + } + }, + "required": [ + "chars", + "text", + "total_chars", + "truncated" + ], + "type": "object" + }, + "DocumentType": { + "description": "What a document is made of.", + "oneOf": [ + { + "description": "Real text throughout. Extract locally.", + "enum": [ + "text_based" + ], + "type": "string" + }, + { + "description": "Pictures of pages. Every page needs OCR.", + "enum": [ + "scanned" + ], + "type": "string" + }, + { + "description": "Images with little or no text layer.", + "enum": [ + "image_based" + ], + "type": "string" + }, + { + "description": "Some pages carry text, others do not. Read `pages_needing_ocr`.", + "enum": [ + "mixed" + ], + "type": "string" + } + ] + }, + "PageOcrReason": { + "description": "Why one page cannot be read without OCR.", + "properties": { + "page": { + "description": "1-indexed page number.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "reasons": { + "description": "Machine-readable reasons: `scanned` (a raster page), `no_text` (nothing extractable and nothing to OCR), `vector_text` (characters drawn as outlines rather than text) or `suspected_garbled_text` (a text layer that decodes to nonsense).", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "page", + "reasons" + ], + "type": "object" + }, + "PageResult": { + "description": "One page of markdown, with its own OCR verdict.", + "properties": { + "markdown": { + "description": "Markdown for this page.", + "type": "string" + }, + "needs_ocr": { + "description": "`true` when this page's text is not trustworthy and OCR would do better.", + "type": "boolean" + }, + "ocr_reason": { + "description": "Machine-readable reason, when the cause is known.", + "type": [ + "string", + "null" + ] + }, + "page": { + "description": "1-indexed page number.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "markdown", + "needs_ocr", + "page" + ], + "type": "object" + } + }, + "properties": { + "body": { + "allOf": [ + { + "$ref": "#/definitions/Body" + } + ], + "description": "The markdown, capped per `max_chars`." + }, + "document_type": { + "allOf": [ + { + "$ref": "#/definitions/DocumentType" + } + ], + "description": "The document-level verdict, so a caller that skipped `pdf::classify` still learns it got nothing because the document is a scan." + }, + "elapsed_ms": { + "description": "Wall-clock time for the conversion.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "has_encoding_issues": { + "description": "`true` when font encodings decoded badly. The markdown, if any, is not to be trusted.", + "type": "boolean" + }, + "ocr_reasons": { + "description": "Per-page explanation for `pages_needing_ocr`.", + "items": { + "$ref": "#/definitions/PageOcrReason" + }, + "type": "array" + }, + "page_count": { + "description": "Pages in the document.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "pages": { + "description": "Per-page markdown, when `per_page` was requested.", + "items": { + "$ref": "#/definitions/PageResult" + }, + "type": [ + "array", + "null" + ] + }, + "pages_converted": { + "description": "Pages actually converted. Equal to `page_count` unless `pages` was set.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "pages_needing_ocr": { + "description": "1-indexed pages that need OCR.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + }, + "pages_with_columns": { + "description": "1-indexed pages laid out in multiple columns.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + }, + "pages_with_tables": { + "description": "1-indexed pages holding a detected table.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + } + }, + "required": [ + "body", + "document_type", + "elapsed_ms", + "has_encoding_issues", + "ocr_reasons", + "page_count", + "pages_converted", + "pages_needing_ocr", + "pages_with_columns", + "pages_with_tables", + "source" + ], + "title": "Response", + "type": "object" + } +} diff --git a/pdf/tests/integration.rs b/pdf/tests/integration.rs new file mode 100644 index 000000000..f550851a6 --- /dev/null +++ b/pdf/tests/integration.rs @@ -0,0 +1,261 @@ +//! The worker's surface driven over a real engine. +//! +//! These assert at the wire, not in Rust. A response type can be correct and +//! still lose a field on the way out, because serde skips what it is told to +//! skip and a rename never fails to compile. Every assertion here reads the +//! JSON a caller would actually receive. +//! +//! Self-skips when no `iii` binary is available. + +mod support; + +use serde_json::json; +use support::engine::{fixture_path, with_stack}; + +#[tokio::test(flavor = "multi_thread")] +async fn classify_answers_over_the_bus() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::classify", + json!({ "path": fixture_path("text-two-page.pdf") }), + ) + .await + .expect("pdf::classify"); + + assert_eq!(out["document_type"], json!("text_based")); + assert_eq!(out["page_count"], json!(2)); + assert_eq!(out["source"], json!("text-two-page.pdf")); + assert_eq!(out["pages_needing_ocr"], json!([])); + assert!(out["confidence"].is_number()); + assert!(out["elapsed_ms"].is_number()); + }) + .await; +} + +/// The per-page OCR verdict is the routing signal, and it has to survive +/// serialization intact — including the reason strings, which the console and +/// the agent guidance both key off. +#[tokio::test(flavor = "multi_thread")] +async fn the_ocr_verdict_reaches_the_caller_with_its_reasons() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::classify", + json!({ "path": fixture_path("no-text.pdf") }), + ) + .await + .expect("pdf::classify"); + + assert_eq!(out["pages_needing_ocr"], json!([1])); + let reasons = out["ocr_reasons"].as_array().expect("ocr_reasons array"); + assert_eq!(reasons.len(), 1); + assert_eq!(reasons[0]["page"], json!(1)); + assert_eq!(reasons[0]["reasons"], json!(["no_text"])); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn markdown_crosses_the_wire_with_its_body_envelope() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::to-markdown", + json!({ "path": fixture_path("text-two-page.pdf"), "max_chars": 0 }), + ) + .await + .expect("pdf::to-markdown"); + + let text = out["body"]["text"].as_str().expect("body.text"); + assert!(text.contains("# Quarterly Report"), "{text}"); + assert_eq!(out["body"]["truncated"], json!(false)); + assert_eq!(out["body"]["chars"], out["body"]["total_chars"]); + // Nothing was withheld, so there is nothing to preview. + assert!(out["body"].get("preview").is_none()); + }) + .await; +} + +/// A truncated body must carry the numbers that let a caller notice. If +/// `total_chars` were dropped on the wire, a fragment would look like a whole +/// document. +#[tokio::test(flavor = "multi_thread")] +async fn a_truncated_body_reports_the_full_size_over_the_wire() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::to-markdown", + json!({ "path": fixture_path("text-two-page.pdf"), "max_chars": 15 }), + ) + .await + .expect("pdf::to-markdown"); + + assert_eq!(out["body"]["truncated"], json!(true)); + assert_eq!(out["body"]["chars"], json!(15)); + assert!(out["body"]["total_chars"].as_u64().unwrap() > 15); + assert!(out["body"]["preview"].is_string()); + }) + .await; +} + +/// Page numbers are 1-indexed on the wire in both directions. The parser's own +/// per-page entry point counts from zero, so this is the assertion that catches +/// a conversion going missing in a refactor. +#[tokio::test(flavor = "multi_thread")] +async fn page_numbers_stay_one_indexed_across_the_wire() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::to-markdown", + json!({ + "path": fixture_path("text-two-page.pdf"), + "pages": [2], + "per_page": true, + "max_chars": 0 + }), + ) + .await + .expect("pdf::to-markdown"); + + let pages = out["pages"].as_array().expect("per-page results"); + assert_eq!(pages.len(), 1); + assert_eq!(pages[0]["page"], json!(2)); + assert!(pages[0]["markdown"].as_str().unwrap().contains("Appendix")); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn items_carry_their_geometry_and_state_their_origin() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::extract-items", + json!({ "path": fixture_path("text-two-page.pdf"), "pages": [1] }), + ) + .await + .expect("pdf::extract-items"); + + assert_eq!(out["coordinate_origin"], json!("pdf-points, bottom-left")); + assert_eq!(out["total_count"], json!(3)); + assert_eq!(out["truncated"], json!(false)); + + let items = out["items"].as_array().expect("items array"); + let heading = items + .iter() + .find(|i| i["text"].as_str().unwrap_or_default().contains("Quarterly")) + .expect("the heading survives serialization"); + assert_eq!(heading["page"], json!(1)); + assert_eq!(heading["x"], json!(72.0)); + assert_eq!(heading["y"], json!(700.0)); + assert_eq!(heading["font_size"], json!(24.0)); + assert_eq!(heading["kind"], json!("text")); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn regions_read_the_box_and_state_the_opposite_origin() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::extract-regions", + json!({ + "path": fixture_path("text-two-page.pdf"), + "regions": [{ "page": 1, "boxes": [[0.0, 0.0, 612.0, 200.0]] }] + }), + ) + .await + .expect("pdf::extract-regions"); + + assert_eq!(out["coordinate_origin"], json!("pdf-points, top-left")); + assert_eq!(out["region_count"], json!(1)); + let pages = out["pages"].as_array().expect("pages array"); + assert_eq!(pages[0]["page"], json!(1)); + assert!(pages[0]["regions"][0]["text"] + .as_str() + .unwrap() + .contains("Quarterly Report")); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn plain_text_extraction_crosses_the_wire() { + with_stack(|stack| async move { + let out = stack + .call( + "pdf::extract-text", + json!({ "path": fixture_path("text-two-page.pdf"), "max_chars": 0 }), + ) + .await + .expect("pdf::extract-text"); + + let text = out["body"]["text"].as_str().expect("body.text"); + assert!(text.contains("Quarterly Report")); + assert!(text.contains("Appendix")); + }) + .await; +} + +/// A caller mistake must come back as an error the caller can act on, not as an +/// empty success that reads like a document with nothing in it. +#[tokio::test(flavor = "multi_thread")] +async fn caller_mistakes_come_back_as_errors() { + with_stack(|stack| async move { + let err = stack + .call("pdf::classify", json!({})) + .await + .expect_err("no source given"); + assert!(err.contains("path"), "{err}"); + + let err = stack + .call( + "pdf::extract-items", + json!({ "path": fixture_path("text-two-page.pdf"), "pages": [0] }), + ) + .await + .expect_err("page 0"); + assert!(err.contains("1-indexed"), "{err}"); + + let err = stack + .call( + "pdf::extract-regions", + json!({ + "path": fixture_path("text-two-page.pdf"), + "regions": [{ "page": 1, "boxes": [[300.0, 300.0, 10.0, 10.0]] }] + }), + ) + .await + .expect_err("inverted box"); + assert!(err.contains("inverted"), "{err}"); + }) + .await; +} + +/// This worker is called on demand, so it must cost a turn that has nothing to +/// do with a document exactly nothing. It registers no harness hook and binds no +/// trigger type: an agent finds it through the function registry, and a person +/// finds it through the console page. A hook here would run on every single +/// generation, which is what a turn-loop worker is for, not this one. +#[tokio::test(flavor = "multi_thread")] +async fn the_worker_never_runs_on_a_turn_that_does_not_ask_for_it() { + with_stack(|stack| async move { + let out = stack + .call("engine::functions::list", json!({ "search": "pdf" })) + .await + .expect("engine::functions::list"); + + let listed = serde_json::to_string(&out).expect("serializes"); + assert!( + listed.contains("pdf::classify"), + "an agent must be able to discover this worker: {listed}" + ); + assert!( + !listed.contains("inject-guidance"), + "this worker must register no harness hook: {listed}" + ); + }) + .await; +} diff --git a/pdf/tests/manifest.rs b/pdf/tests/manifest.rs new file mode 100644 index 000000000..496233ef8 --- /dev/null +++ b/pdf/tests/manifest.rs @@ -0,0 +1,96 @@ +//! The `--manifest` contract, exercised through the real binary. +//! +//! The unit tests in `src/manifest.rs` cover the struct; this covers the CLI +//! path the registry publish pipeline actually calls, including the part that +//! matters most: it must print and exit without touching the engine. + +use std::process::Command; + +fn manifest_json() -> serde_json::Value { + let output = Command::new(env!("CARGO_BIN_EXE_pdf")) + .arg("--manifest") + .output() + .expect("the worker binary runs"); + + assert!( + output.status.success(), + "--manifest exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + + serde_json::from_slice(&output.stdout).unwrap_or_else(|e| { + panic!( + "--manifest did not print JSON ({e}): {}", + String::from_utf8_lossy(&output.stdout) + ) + }) +} + +/// `POST /publish` rejects a manifest missing any of these five. +#[test] +fn manifest_prints_every_field_the_registry_requires() { + let json = manifest_json(); + + assert_eq!(json["name"], "pdf"); + assert!( + json["version"].as_str().is_some_and(|v| !v.is_empty()), + "version missing" + ); + assert!( + json["description"].as_str().is_some_and(|d| d.len() > 20), + "description missing or too short to be useful in the registry" + ); + assert!(json["default_config"].is_object(), "default_config missing"); + assert!( + json["supported_targets"] + .as_array() + .is_some_and(|t| !t.is_empty()), + "supported_targets missing or empty" + ); +} + +/// The build script forwards the build-time triple. A manifest advertising a +/// target the binary was not built for would hand consumers the wrong artifact. +#[test] +fn supported_targets_carries_a_real_triple() { + let json = manifest_json(); + let target = json["supported_targets"][0] + .as_str() + .expect("a target triple"); + assert!( + target.contains('-'), + "{target} does not look like a target triple" + ); +} + +/// The manifest path must not need an engine: the publish pipeline runs it on a +/// bare runner with nothing listening. +#[test] +fn manifest_needs_no_engine() { + let output = Command::new(env!("CARGO_BIN_EXE_pdf")) + .arg("--manifest") + // A URL nothing is listening on. Connecting would hang or fail; the + // manifest path must return before it ever tries. + .args(["--url", "ws://127.0.0.1:1"]) + .output() + .expect("the worker binary runs"); + + assert!( + output.status.success(), + "--manifest must not depend on an engine, got {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); +} + +/// The default config the registry publishes has to be the config the worker +/// actually boots with, or an operator reading the registry is misled. +#[test] +fn published_default_config_matches_the_shipped_defaults() { + let json = manifest_json(); + assert_eq!( + json["default_config"], + pdf::config::WorkerConfig::default().to_json() + ); +} diff --git a/pdf/tests/schemas.rs b/pdf/tests/schemas.rs new file mode 100644 index 000000000..794850058 --- /dev/null +++ b/pdf/tests/schemas.rs @@ -0,0 +1,137 @@ +//! Wire-schema snapshots for the five `pdf::*` functions. +//! +//! `pdf::functions::catalog()` is the single source of truth for each +//! function's id, registration description, and schemars-derived request and +//! response schemas, generated with the same construction iii-sdk uses at +//! registration, from the same input and output structs. Each entry is +//! serialized to pretty JSON and compared against +//! `tests/golden/schemas/.json` (`::` maps to `.` in filenames). +//! +//! These snapshots ARE the product surface consumed by callers and agents, so +//! any schema or description change must land as an explicit golden diff. +//! Regenerate with `UPDATE_GOLDENS=1 cargo test`. + +mod support; + +use pdf::functions::{catalog, FunctionSpec}; + +fn golden_file_name(function_id: &str) -> String { + format!("schemas/{}.json", function_id.replace("::", ".")) +} + +fn spec_to_pretty_json(spec: &FunctionSpec) -> String { + let value = serde_json::json!({ + "function_id": spec.function_id, + "description": spec.description, + "request_schema": spec.request_schema, + "response_schema": spec.response_schema, + }); + let mut pretty = serde_json::to_string_pretty(&value).expect("spec serializes"); + pretty.push('\n'); + pretty +} + +/// The catalog must cover exactly the registered functions, in registration +/// order (kept in lockstep with `register_all`). +#[test] +fn catalog_lists_all_five_functions_in_registration_order() { + let ids: Vec<&str> = catalog().iter().map(|s| s.function_id).collect(); + assert_eq!( + ids, + vec![ + "pdf::classify", + "pdf::to-markdown", + "pdf::extract-text", + "pdf::extract-items", + "pdf::extract-regions", + ] + ); +} + +/// Every catalog entry matches its committed golden. Mismatches are collected +/// across ALL functions before failing, so one run shows the full drift. +#[test] +fn wire_schema_snapshots_match_goldens() { + let mut failures = Vec::new(); + for spec in catalog() { + let rel = golden_file_name(spec.function_id); + let actual = spec_to_pretty_json(&spec); + if let Err(msg) = support::check_golden(&rel, &actual) { + failures.push(msg); + } + } + assert!( + failures.is_empty(), + "{} wire-schema golden(s) drifted:\n\n{}", + failures.len(), + failures.join("\n") + ); +} + +/// No function may ship the permissive `AnyValue` schema — the deploy-time +/// "unknown" request/response schema this convention exists to prevent. +#[test] +fn every_function_has_typed_request_and_response_schemas() { + for spec in catalog() { + support::assert_typed_schema( + &format!("{} request_schema", spec.function_id), + &spec.request_schema, + ); + support::assert_typed_schema( + &format!("{} response_schema", spec.function_id), + &spec.response_schema, + ); + } +} + +/// Field doc comments become schema descriptions, and callers rely on them. +/// Losing them is a silent documentation regression that still compiles. +#[test] +fn schemas_carry_field_descriptions() { + for spec in catalog() { + let rendered = serde_json::to_string(&spec.request_schema).expect("schema serializes"); + assert!( + rendered.contains("description"), + "{}: request schema lost its field descriptions", + spec.function_id + ); + } +} + +/// The page-numbering convention is the easiest thing on this surface to get +/// silently wrong, so it must be stated in the schema an agent reads, not only +/// in the worker README. +#[test] +fn page_fields_state_their_indexing() { + for spec in catalog() { + let rendered = serde_json::to_string(&spec.response_schema).expect("schema serializes"); + if rendered.contains("\"page\"") || rendered.contains("pages_needing_ocr") { + assert!( + rendered.contains("1-indexed"), + "{}: response mentions pages without stating the indexing", + spec.function_id + ); + } + } +} + +/// Both coordinate conventions must be spelled out where they are used. The two +/// functions disagree on the origin, and a caller that assumes wrong gets text +/// from the wrong end of the page with no error. +#[test] +fn coordinate_conventions_are_documented_on_the_wire() { + for spec in catalog() { + let rendered = serde_json::to_string(&spec.response_schema).expect("schema serializes"); + match spec.function_id { + "pdf::extract-items" => assert!( + rendered.contains("bottom"), + "extract-items must document its bottom-left origin" + ), + "pdf::extract-regions" => assert!( + rendered.contains("top left") || rendered.contains("top-left"), + "extract-regions must document its top-left origin" + ), + _ => {} + } + } +} diff --git a/pdf/tests/support/engine.rs b/pdf/tests/support/engine.rs new file mode 100644 index 000000000..b7d8d56cb --- /dev/null +++ b/pdf/tests/support/engine.rs @@ -0,0 +1,203 @@ +//! Engine-backed test bootstrap. Self-skips when no `iii` binary is on PATH +//! and `III_ENGINE_BIN` is unset, so CI and casual local runs stay green. +//! +//! The worker's functions are registered in process against a real engine +//! rather than by spawning the binary. That is the point of these tests: they +//! exercise the actual wire path, where serde silently drops a field a unit +//! test would never notice. + +#![allow(dead_code)] + +use std::io::Write as _; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::{register_worker, IIIClient, InitOptions}; +use serde_json::{json, Value}; +use tokio::sync::RwLock; + +use pdf::config::WorkerConfig; +use pdf::configuration::ConfigCell; + +pub struct Engine { + pub url: String, + child: std::process::Child, + dir: std::path::PathBuf, +} + +impl Drop for Engine { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn engine_bin() -> Option { + if let Ok(p) = std::env::var("III_ENGINE_BIN") { + return Some(p.into()); + } + let on_path = std::process::Command::new("iii") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); + on_path.then(|| "iii".into()) +} + +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("local addr") + .port() +} + +/// `true` once the configuration worker answers at all. A not-found for a +/// nonexistent id still proves it is serving, which is what we need to know. +async fn configuration_serving(probe: &IIIClient) -> bool { + match probe + .trigger(TriggerRequest { + function_id: "configuration::get".into(), + payload: json!({ "id": "__readiness_probe__" }), + action: None, + timeout_ms: Some(1000), + }) + .await + { + Ok(_) => true, + Err(e) => e.to_string().to_ascii_uppercase().contains("NOT_FOUND"), + } +} + +async fn spawn_engine() -> Option { + let bin = engine_bin()?; + let port = free_port(); + let dir = std::env::temp_dir().join(format!("pdf-it-{}-{port}", std::process::id())); + std::fs::create_dir_all(&dir).ok()?; + // The configuration worker's fs adapter writes one YAML file per id here; + // create it up front so register and get never race a missing path. + std::fs::create_dir_all(dir.join("configuration")).ok()?; + + let config = format!( + r#"workers: + - name: iii-worker-manager + config: + port: {port} + - name: configuration + config: + adapter: + name: fs + config: + directory: "{dir}/configuration" + ttl_seconds: 0 +"#, + port = port, + dir = dir.display(), + ); + let config_path = dir.join("config.yaml"); + std::fs::File::create(&config_path) + .and_then(|mut f| f.write_all(config.as_bytes())) + .ok()?; + + let child = std::process::Command::new(&bin) + .arg("--no-update-check") + .arg("--config") + .arg(&config_path) + .current_dir(&dir) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .ok()?; + + let url = format!("ws://127.0.0.1:{port}"); + let probe = register_worker(&url, InitOptions::default()); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + // The engine core must be up AND the configuration worker serving: + // this worker treats configuration as a required boot dependency, so + // starting before it is ready would fail spuriously. Any response to + // the probe id counts, including a not-found. + let core_ready = probe + .trigger(TriggerRequest { + function_id: "engine::workers::list".into(), + payload: json!({}), + action: None, + timeout_ms: Some(1000), + }) + .await + .is_ok(); + + if core_ready && configuration_serving(&probe).await { + break; + } + if Instant::now() > deadline { + probe.shutdown_async().await; + return None; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + probe.shutdown_async().await; + + Some(Engine { url, child, dir }) +} + +pub struct Stack { + pub iii: Arc, + _engine: Engine, +} + +impl Stack { + /// Invoke one of this worker's functions over the bus, the way a caller + /// would. + pub async fn call(&self, function_id: &str, payload: Value) -> Result { + self.iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: Some(30_000), + }) + .await + .map_err(|e| e.to_string()) + } +} + +/// Boot an engine and register this worker against it, or `None` when no +/// engine binary is available. +pub async fn boot() -> Option { + let engine = spawn_engine().await?; + let iii = Arc::new(register_worker(&engine.url, InitOptions::default())); + + let cell: ConfigCell = Arc::new(RwLock::new(Arc::new(WorkerConfig::default()))); + pdf::functions::register_all(&iii, &cell); + + // Let the registrations land before the first call. + tokio::time::sleep(Duration::from_millis(500)).await; + + Some(Stack { + iii, + _engine: engine, + }) +} + +/// Run `f` against a freshly booted stack; skip when no engine is available. +pub async fn with_stack(f: F) +where + F: FnOnce(Stack) -> Fut, + Fut: std::future::Future, +{ + let Some(stack) = boot().await else { + eprintln!("skipping: no iii engine (set III_ENGINE_BIN or put `iii` on PATH)"); + return; + }; + f(stack).await; +} + +/// Absolute path to a committed fixture. +pub fn fixture_path(name: &str) -> String { + format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) +} diff --git a/pdf/tests/support/mod.rs b/pdf/tests/support/mod.rs new file mode 100644 index 000000000..a4ac4c981 --- /dev/null +++ b/pdf/tests/support/mod.rs @@ -0,0 +1,124 @@ +//! Shared test support. +//! +//! - [`engine`] boots a real iii engine and registers this worker against it. +//! - The golden-file helpers below back `tests/schemas.rs`. +//! +//! Hand-rolled golden harness (deliberately no snapshot dependency). Goldens +//! live under `tests/golden/` and are committed; any wire-surface change must +//! show up as an explicit, reviewed diff. +//! +//! Workflow: +//! - `cargo test` compares actual output against the committed goldens. +//! - `UPDATE_GOLDENS=1 cargo test` regenerates the files; review the git diff, +//! then commit the new goldens alongside the change that caused them. + +#![allow(dead_code)] + +pub mod engine; + +use std::fs; +use std::path::PathBuf; + +/// Root of the committed golden files. +pub fn golden_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/golden") +} + +fn update_mode() -> bool { + std::env::var("UPDATE_GOLDENS") + .map(|v| v == "1") + .unwrap_or(false) +} + +/// Compare `actual` against the golden file at `tests/golden/`. Returns +/// `Err(readable diff hint)` on mismatch or missing golden; with +/// `UPDATE_GOLDENS=1` the file is (re)written and the check passes. +pub fn check_golden(rel: &str, actual: &str) -> Result<(), String> { + let path = golden_root().join(rel); + if update_mode() { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; + } + fs::write(&path, actual).map_err(|e| format!("write {}: {e}", path.display()))?; + return Ok(()); + } + let expected = fs::read_to_string(&path).map_err(|e| { + format!( + "golden file {} unreadable ({e}).\n\ + Run `UPDATE_GOLDENS=1 cargo test` to (re)generate, then review and \ + commit the diff.", + path.display() + ) + })?; + if expected == actual { + return Ok(()); + } + Err(diff_hint(rel, &expected, actual)) +} + +/// Readable first-divergence diff hint: line number, expected versus actual +/// around the mismatch, and the regeneration instructions. +fn diff_hint(rel: &str, expected: &str, actual: &str) -> String { + let exp_lines: Vec<&str> = expected.lines().collect(); + let act_lines: Vec<&str> = actual.lines().collect(); + let first_diff = exp_lines + .iter() + .zip(act_lines.iter()) + .position(|(e, a)| e != a) + .unwrap_or_else(|| exp_lines.len().min(act_lines.len())); + + const CONTEXT: usize = 3; + let lo = first_diff.saturating_sub(CONTEXT); + let hi = (first_diff + CONTEXT + 1).max(first_diff + 1); + + let mut out = format!( + "golden mismatch: tests/golden/{rel}\n\ + first divergence at line {} (expected {} lines, actual {} lines)\n", + first_diff + 1, + exp_lines.len(), + act_lines.len() + ); + out.push_str("--- expected (golden) ---\n"); + for (i, line) in exp_lines.iter().enumerate().skip(lo).take(hi - lo) { + let marker = if i == first_diff { ">" } else { " " }; + out.push_str(&format!("{marker} {:>4} | {line}\n", i + 1)); + } + out.push_str("--- actual ---\n"); + for (i, line) in act_lines.iter().enumerate().skip(lo).take(hi - lo) { + let marker = if i == first_diff { ">" } else { " " }; + out.push_str(&format!("{marker} {:>4} | {line}\n", i + 1)); + } + out.push_str( + "If this change is intentional, run `UPDATE_GOLDENS=1 cargo test`, review \ + the git diff, and commit the updated goldens.\n", + ); + out +} + +/// Assert a schemars-derived request or response schema is a real schema and +/// not the permissive `AnyValue` schema a `Value` handler emits (the "unknown" +/// schema this convention exists to prevent). A real schema carries at least +/// one schema-defining keyword. +pub fn assert_typed_schema(label: &str, schema: &schemars::schema::RootSchema) { + let value = serde_json::to_value(schema).expect("schema serializes"); + let obj = value + .as_object() + .unwrap_or_else(|| panic!("{label}: schema is not a JSON object")); + const DEFINING: [&str; 8] = [ + "type", + "properties", + "$ref", + "allOf", + "anyOf", + "oneOf", + "enum", + "items", + ]; + let has_defining = DEFINING.iter().any(|k| obj.contains_key(*k)); + assert!( + has_defining, + "{label}: schema is the permissive AnyValue/empty schema (no \ + type/properties/$ref/…). The handler is registered with `Value` — give it \ + a typed struct deriving JsonSchema. Got: {value}" + ); +} diff --git a/pdf/ui/build.mjs b/pdf/ui/build.mjs new file mode 100644 index 000000000..4326612b2 --- /dev/null +++ b/pdf/ui/build.mjs @@ -0,0 +1,36 @@ +/** + * Build the worker's two console assets: + * + * page.tsx → dist/page.js (injected over `console:script`) + * styles.css → dist/styles.css (injected over `console:style`) + * + * The five shared specifiers stay EXTERNAL — they resolve at runtime through + * the console's import map. A bundled React copy surfaces as a cryptic + * "Invalid hook call" with nothing pointing at the cause. `--watch` pairs with + * the worker's III_PDF_UI_WATCH poller for the hot-reload dev loop. + */ + +import esbuild from 'esbuild' + +const options = { + entryPoints: ['page.tsx', 'styles.css'], + bundle: true, + format: 'esm', + jsx: 'automatic', + outdir: 'dist', + external: [ + 'react', + 'react-dom', + 'react-dom/client', + 'react/jsx-runtime', + '@iii-dev/console-ui', + ], + logLevel: 'info', +} + +if (process.argv.includes('--watch')) { + const ctx = await esbuild.context(options) + await ctx.watch() +} else { + await esbuild.build(options) +} diff --git a/pdf/ui/package.json b/pdf/ui/package.json new file mode 100644 index 000000000..6648d932c --- /dev/null +++ b/pdf/ui/package.json @@ -0,0 +1,18 @@ +{ + "name": "@iii-workers/pdf-ui", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "build": "tsc --noEmit && node build.mjs", + "watch": "node build.mjs --watch" + }, + "dependencies": { + "@iii-dev/console-ui": "workspace:*" + }, + "devDependencies": { + "@types/react": "^19.2.14", + "esbuild": "^0.25.0", + "typescript": "^5.9.2" + } +} diff --git a/pdf/ui/page.tsx b/pdf/ui/page.tsx new file mode 100644 index 000000000..8b6003a50 --- /dev/null +++ b/pdf/ui/page.tsx @@ -0,0 +1,28 @@ +/** + * Entry for the pdf worker's injected console UI — compiled by esbuild (react + * and @iii-dev/console-ui external) into dist/page.js and served over the + * `console:script` trigger (see src/ui.rs). The stylesheet is its own asset: + * ./styles.css ships over `console:style` as pdf/styles.css. + * + * `setup(host)` composes two contributions: + * + * - src/page/ — the PDF page (#/ext/pdf-reader) + * - src/function-trigger-message/ — how pdf::* calls render in chat and traces + * + * Registrations go through `host` so the loader disposes them on hot reload and + * worker disconnect. + */ + +import type { Host } from '@iii-dev/console-ui' +import { createPdfTriggerRenderer } from './src/function-trigger-message' +import { PdfPage } from './src/page' + +export default function setup(host: Host) { + host.pages.register({ + id: 'pdf-reader', + title: 'pdf', + render: () => , + }) + + host.functionTriggers.register(createPdfTriggerRenderer(host)) +} diff --git a/pdf/ui/src/function-trigger-message/index.tsx b/pdf/ui/src/function-trigger-message/index.tsx new file mode 100644 index 000000000..82b137ae2 --- /dev/null +++ b/pdf/ui/src/function-trigger-message/index.tsx @@ -0,0 +1,215 @@ +/** + * How `pdf::*` calls render in chat and traces. + * + * Without this they render as raw JSON, and the one number that matters — is + * this document readable, and how much of it did the agent actually get — is + * buried in it. Each renderer surfaces the decision, not the payload. + * + * Match narrowly and return `null` freely: `null` falls through to the console's + * own cards, which already handle errors and pending approvals better than a + * worker renderer should try to. + */ + +import { Badge, type FunctionTriggerMessage, type FunctionTriggerRenderer, type Host } from '@iii-dev/console-ui' + +import { + documentTypeLabel, + ocrReasonLabel, + type ClassifyResponse, + type MarkdownResponse, +} from '../lib/api' + +const HANDLED = new Set([ + 'pdf::classify', + 'pdf::to-markdown', + 'pdf::extract-text', + 'pdf::extract-items', + 'pdf::extract-regions', +]) + +export function createPdfTriggerRenderer(_host: Host): FunctionTriggerRenderer { + return { + id: 'pdf/page.js#renderer', + isMatch: (functionId) => HANDLED.has(functionId), + tryRender: (message) => render(message), + tryRenderPreview: (message) => render(message), + } +} + +/** + * A function result reaches the console wrapped by the harness as + * `{ content: [...], details: }`, not as the response + * itself. Reading the raw value looks like it works right up until every field + * is undefined and the renderer quietly falls through to an empty card, which + * is exactly what happened the first time this shipped. + * + * The console has its own `unwrapEnvelope`, but injected assets can only import + * from `@iii-dev/console-ui`, so the same two-line rule lives here. + */ +function unwrapEnvelope(value: unknown): unknown { + if (!value || typeof value !== 'object' || Array.isArray(value)) return value + const obj = value as Record + if (Array.isArray(obj.content) && 'details' in obj) return obj.details + return value +} + +function render(message: FunctionTriggerMessage) { + const output = unwrapEnvelope(message.output) + if (!output || typeof output !== 'object') return null + + switch (message.functionId) { + case 'pdf::classify': + return + case 'pdf::to-markdown': + return + case 'pdf::extract-text': + return + case 'pdf::extract-items': + return + case 'pdf::extract-regions': + return + default: + return null + } +} + +interface BodyLike { + chars: number + total_chars: number + truncated: boolean +} + +interface ItemsLike { + count?: number + total_count?: number + truncated?: boolean + source?: string +} + +interface RegionsLike { + region_count?: number + regions_needing_ocr?: number + source?: string +} + +function ClassifyCard({ result }: { result: ClassifyResponse }) { + if (!result.document_type) return null + const needing = result.pages_needing_ocr?.length ?? 0 + const reasons = new Set( + (result.ocr_reasons ?? []).flatMap((r) => r.reasons).map(ocrReasonLabel), + ) + return ( +
+
+ = result.page_count ? 'alert' : 'warn'}> + {documentTypeLabel(result.document_type)} + + {result.source} + + {result.page_count} pages · {Math.round((result.confidence ?? 0) * 100)}% confident ·{' '} + {result.elapsed_ms} ms + +
+

+ {needing === 0 + ? 'Every page is readable without OCR.' + : `${needing} of ${result.page_count} pages need OCR${ + reasons.size > 0 ? `: ${[...reasons].join('; ')}` : '' + }.`} +

+
+ ) +} + +function MarkdownCard({ result }: { result: MarkdownResponse }) { + const body = result.body + if (!body) return null + return ( +
+
+ + {body.truncated ? 'truncated' : 'complete'} + + {result.source} + + {result.pages_converted} of {result.page_count} pages · {result.elapsed_ms} ms + +
+

+ {body.truncated + ? `Returned ${format(body.chars)} of ${format(body.total_chars)} characters. The rest was not read.` + : `${format(body.chars)} characters of markdown.`} + {result.has_encoding_issues && ' Font encodings decoded badly; do not trust this text.'} +

+
+ ) +} + +function BodyCard({ + result, + label, +}: { + result: { body?: BodyLike; source?: string } + label: string +}) { + const body = result.body + if (!body) return null + return ( +
+
+ + {body.truncated ? 'truncated' : 'complete'} + + {result.source} +
+

+ {body.truncated + ? `Returned ${format(body.chars)} of ${format(body.total_chars)} characters of ${label}.` + : `${format(body.chars)} characters of ${label}.`} +

+
+ ) +} + +function ItemsCard({ result }: { result: ItemsLike }) { + if (typeof result.count !== 'number') return null + return ( +
+
+ + {result.count} items + + {result.source} +
+ {result.truncated && ( +

+ {format(result.total_count ?? 0)} items on those pages; the rest were not returned. +

+ )} +
+ ) +} + +function RegionsCard({ result }: { result: RegionsLike }) { + if (typeof result.region_count !== 'number') return null + const suspect = result.regions_needing_ocr ?? 0 + return ( +
+
+ + {result.region_count} regions + + {result.source} +
+ {suspect > 0 && ( +

+ {suspect} of {result.region_count} regions came back unreliable and would need OCR. +

+ )} +
+ ) +} + +function format(n: number): string { + return n.toLocaleString('en-US') +} diff --git a/pdf/ui/src/lib/api.ts b/pdf/ui/src/lib/api.ts new file mode 100644 index 000000000..2fc283425 --- /dev/null +++ b/pdf/ui/src/lib/api.ts @@ -0,0 +1,159 @@ +/** + * The slice of the worker's wire surface this page uses. + * + * Hand-modeled against `pdf/src/functions/*.rs`. The golden schema snapshots in + * `pdf/tests/golden/schemas/` are the source of truth; these types are the + * console's compile-time view of them. + */ + +import type { ExtensionIii } from '@iii-dev/console-ui' + +export type DocumentType = + | 'text_based' + | 'scanned' + | 'image_based' + | 'mixed' + +export interface PageOcrReason { + page: number + reasons: string[] +} + +export interface ClassifyResponse { + document_type: DocumentType + confidence: number + page_count: number + /** Absent for an encrypted document, which takes the decrypting path. */ + pages_sampled?: number + pages_with_text?: number + pages_needing_ocr: number[] + ocr_reasons: PageOcrReason[] + ocr_recommended: boolean + title: string | null + source: string + elapsed_ms: number +} + +export interface Body { + text: string + chars: number + total_chars: number + truncated: boolean + preview?: string +} + +export interface PageMarkdown { + page: number + markdown: string + needs_ocr: boolean + ocr_reason?: string +} + +export interface MarkdownResponse { + document_type: DocumentType + body: Body + page_count: number + pages_converted: number + pages?: PageMarkdown[] + pages_with_tables: number[] + pages_with_columns: number[] + pages_needing_ocr: number[] + ocr_reasons: PageOcrReason[] + has_encoding_issues: boolean + source: string + elapsed_ms: number +} + +/** + * Read a File as base64 without building one enormous argument list. + * + * `String.fromCharCode(...bytes)` overflows the call stack somewhere around a + * megabyte, which is a small PDF. Chunking keeps it linear and bounded. + */ +export async function fileToBase64(file: File): Promise { + const buffer = new Uint8Array(await file.arrayBuffer()) + const CHUNK = 0x8000 + let binary = '' + for (let i = 0; i < buffer.length; i += CHUNK) { + binary += String.fromCharCode(...buffer.subarray(i, i + CHUNK)) + } + return btoa(binary) +} + +/** Human-readable label for a document type. */ +export function documentTypeLabel(type: DocumentType): string { + switch (type) { + case 'text_based': + return 'text based' + case 'image_based': + return 'image based' + default: + return type + } +} + +/** What a document type means for the caller, in one sentence. */ +export function documentTypeMeaning(type: DocumentType): string { + switch (type) { + case 'text_based': + return 'Real text throughout. Extract it locally.' + case 'scanned': + return 'Pictures of pages. Every page needs OCR.' + case 'image_based': + return 'Images with little or no text layer.' + case 'mixed': + return 'Some pages carry text and some do not.' + } +} + +/** Plain-language expansion of a machine-readable OCR reason. */ +export function ocrReasonLabel(reason: string): string { + switch (reason) { + case 'scanned': + return 'a raster page' + case 'no_text': + return 'nothing extractable, and nothing to OCR' + case 'vector_text': + return 'characters drawn as outlines, not text' + case 'suspected_garbled_text': + return 'a text layer that decodes to nonsense' + default: + return reason + } +} + +export interface Inspection { + classify: ClassifyResponse + markdown: MarkdownResponse | null +} + +/** + * Classify first, then convert only if there is something to convert. + * + * This is the routing the worker's guidance asks an agent to do, made visible: + * a scan gets classified and stops, rather than spending a second producing an + * empty document. + */ +export async function inspect( + iii: ExtensionIii, + file: File, +): Promise { + const bytes_base64 = await fileToBase64(file) + + const classify = await iii.trigger( + 'pdf::classify', + { bytes_base64 }, + { timeoutMs: 60_000 }, + ) + + if (classify.document_type === 'scanned' || classify.document_type === 'image_based') { + return { classify, markdown: null } + } + + const markdown = await iii.trigger( + 'pdf::to-markdown', + { bytes_base64, max_chars: 0, per_page: true }, + { timeoutMs: 120_000 }, + ) + return { classify, markdown } +} diff --git a/pdf/ui/src/page/index.tsx b/pdf/ui/src/page/index.tsx new file mode 100644 index 000000000..8eb261309 --- /dev/null +++ b/pdf/ui/src/page/index.tsx @@ -0,0 +1,352 @@ +/** + * The PDF page: drop a document in and see exactly what an agent sees. + * + * The layout follows the decision an agent makes. The verdict comes first, + * because it determines whether anything else is worth doing. The per-page OCR + * grid comes next, because a mixed document is the interesting case and a + * document-level verdict hides it. The markdown comes last, since by then you + * already know whether to trust it. + */ + +import { + Badge, + Button, + CodeEditor, + EmptyState, + MarkdownPreview, + StatusPanel, + Tabs, + TabsContent, + TabsList, + TabsTrigger, + Tooltip, + TooltipContent, + TooltipTrigger, + type Host, +} from '@iii-dev/console-ui' +import { useCallback, useMemo, useRef, useState } from 'react' + +import { + documentTypeLabel, + documentTypeMeaning, + inspect, + ocrReasonLabel, + type Inspection, +} from '../lib/api' + +type State = + | { status: 'idle' } + | { status: 'reading'; name: string } + | { status: 'done'; name: string; result: Inspection } + | { status: 'failed'; name: string; error: string } + +export function PdfPage({ host }: { host: Host }) { + const [state, setState] = useState({ status: 'idle' }) + const [dragging, setDragging] = useState(false) + const input = useRef(null) + + const run = useCallback( + async (file: File) => { + setState({ status: 'reading', name: file.name }) + try { + const result = await inspect(host.iii, file) + setState({ status: 'done', name: file.name, result }) + } catch (error) { + setState({ + status: 'failed', + name: file.name, + error: error instanceof Error ? error.message : String(error), + }) + } + }, + [host.iii], + ) + + const onDrop = useCallback( + (event: React.DragEvent) => { + event.preventDefault() + setDragging(false) + const file = event.dataTransfer.files?.[0] + if (file) void run(file) + }, + [run], + ) + + // A failed read still counts as loaded: the error panel is the result, and + // reverting to a full-height drop zone above it would bury the explanation. + const hasResult = state.status === 'done' || state.status === 'failed' + const loadedName = hasResult ? state.name : null + + return ( +
+
+

pdf

+

+ Classify a document, see which pages need OCR, and read the markdown an + agent would get. Parsing runs in the worker on this machine. +

+
+ + {/* The drop zone owns the page until a document is loaded, then shrinks + to a bar: once there are results, they are what the page is for. */} +
{ + e.preventDefault() + setDragging(true) + }} + onDragLeave={() => setDragging(false)} + onDrop={onDrop} + > +

+ {state.status === 'reading' ? ( + `reading ${state.name}` + ) : hasResult ? ( + <> + {loadedName} + + {dragging ? 'drop to replace' : 'or drop another here'} + + + ) : ( + 'Drop a PDF here' + )} +

+ + { + const file = e.target.files?.[0] + if (file) void run(file) + e.target.value = '' + }} + /> +
+ + {state.status === 'failed' && ( + + )} + + {state.status === 'idle' && ( + + )} + + {state.status === 'done' && } +
+ ) +} + +function Result({ name, result }: { name: string; result: Inspection }) { + const { classify, markdown } = result + + const ocrByPage = useMemo(() => { + const map = new Map() + for (const entry of classify.ocr_reasons) map.set(entry.page, entry.reasons) + return map + }, [classify.ocr_reasons]) + + const tables = new Set(markdown?.pages_with_tables ?? []) + const columns = new Set(markdown?.pages_with_columns ?? []) + const totalMs = classify.elapsed_ms + (markdown?.elapsed_ms ?? 0) + const chars = markdown?.body.total_chars ?? 0 + // The speed claim is the reason to parse locally rather than pay an OCR + // service, so show the rate, not only the duration. + const charsPerSecond = + markdown && markdown.elapsed_ms > 0 + ? Math.round(chars / (markdown.elapsed_ms / 1000)) + : null + + return ( +
+
+ + {documentTypeLabel(classify.document_type)} + + + {documentTypeMeaning(classify.document_type)} + +
+ +
+ + + + + + +
+ +

+ {markdown ? ( + <> + {chars.toLocaleString('en-US')} characters in {totalMs} ms + {charsPerSecond + ? `, about ${charsPerSecond.toLocaleString('en-US')} characters a second` + : ''} + . Parsed on this machine, with nothing uploaded. + + ) : ( + <>Classified in {totalMs} ms on this machine, with nothing uploaded. + )} +

+ + {classify.title &&

{classify.title}

} + + {markdown?.has_encoding_issues && ( + + )} + + {classify.pages_needing_ocr.length > 0 && ( +
+

Pages needing OCR

+
    + {classify.pages_needing_ocr.map((page) => ( +
  • + page {page} + + {(ocrByPage.get(page) ?? []).map(ocrReasonLabel).join('; ') || + 'no reason recorded'} + +
  • + ))} +
+
+ )} + + {markdown ? ( + + + document + + pages{markdown.pages ? ` (${markdown.pages.length})` : ''} + + markdown source + + + + + + + + {(markdown.pages ?? []).map((page) => ( +
+
+ page {page.page} + {page.needs_ocr && ( + + {page.ocr_reason ? ocrReasonLabel(page.ocr_reason) : 'needs ocr'} + + )} + {tables.has(page.page) && table} + {columns.has(page.page) && columns} +
+ +
+ ))} +
+ + + {/* The console's one code editor, read-only. Never bundle another. */} + {}} + language="markdown" + readOnly + aria-label="Extracted markdown source" + className="pdf-ui__editor" + /> + +
+ ) : ( + + )} +
+ ) +} + +function Stat({ + label, + value, + hint, +}: { + label: string + value: string + hint?: string +}) { + const tile = ( +
+
{label}
+
{value}
+
+ ) + if (!hint) return tile + return ( + + {tile} + {hint} + + ) +} + +function badgeVariant(type: string): 'default' | 'accent' | 'warn' | 'alert' { + switch (type) { + case 'text_based': + return 'accent' + case 'mixed': + return 'warn' + case 'scanned': + case 'image_based': + return 'alert' + default: + return 'default' + } +} diff --git a/pdf/ui/styles.css b/pdf/ui/styles.css new file mode 100644 index 000000000..e778ecaae --- /dev/null +++ b/pdf/ui/styles.css @@ -0,0 +1,274 @@ +/** + * Every rule is scoped under [data-iii-ui="pdf"]. + * + * Injected CSS is unlayered, so one unscoped selector silently beats the + * console's own fully-layered styles across the whole document. Colors come + * from the console's design tokens, so both themes work without a media query. + */ + +[data-iii-ui='pdf'] .pdf-ui { + display: flex; + flex-direction: column; + gap: 1.25rem; + padding: 1.5rem; + max-width: 64rem; + margin: 0 auto; +} + +[data-iii-ui='pdf'] .pdf-ui__head { + display: flex; + flex-direction: column; + gap: 0.35rem; +} + +[data-iii-ui='pdf'] .pdf-ui__title { + font-size: 1.35rem; + font-weight: 600; + color: var(--color-ink); + margin: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__lede { + color: var(--color-ink-faint); + font-size: 0.875rem; + line-height: 1.5; + margin: 0; + max-width: 46rem; +} + +[data-iii-ui='pdf'] .pdf-ui__drop { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 0.75rem; + padding: 2.25rem 1.5rem; + border: 1px dashed var(--color-rule); + border-radius: 0.5rem; + background: var(--color-panel); + transition: border-color 120ms ease, background-color 120ms ease; +} + +[data-iii-ui='pdf'] .pdf-ui__drop--over { + border-color: var(--color-accent); + background: var(--color-paper-2); +} + +/* Once a document is loaded the zone becomes a one-line bar: still a drop + target, no longer the centre of the page. */ +[data-iii-ui='pdf'] .pdf-ui__drop--compact { + flex-direction: row; + align-items: center; + justify-content: space-between; + gap: 1rem; + padding: 0.6rem 0.75rem; + border-style: solid; + border-color: var(--color-rule-2); +} + +[data-iii-ui='pdf'] .pdf-ui__drop-label { + color: var(--color-ink-faint); + font-size: 0.875rem; + margin: 0; + display: flex; + align-items: baseline; + gap: 0.6rem; + min-width: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__drop-file { + color: var(--color-ink); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +[data-iii-ui='pdf'] .pdf-ui__drop-hint { + color: var(--color-ink-ghost); + font-size: 0.75rem; + white-space: nowrap; +} + +[data-iii-ui='pdf'] .pdf-ui__file { + display: none; +} + +[data-iii-ui='pdf'] .pdf-ui__result { + display: flex; + flex-direction: column; + gap: 1rem; +} + +[data-iii-ui='pdf'] .pdf-ui__verdict { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.6rem; +} + +[data-iii-ui='pdf'] .pdf-ui__verdict-meaning { + color: var(--color-ink-faint); + font-size: 0.875rem; +} + +[data-iii-ui='pdf'] .pdf-ui__doc-title { + color: var(--color-ink); + font-size: 0.95rem; + font-weight: 500; + margin: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(7.5rem, 1fr)); + gap: 0.75rem; + margin: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__stat { + display: flex; + flex-direction: column; + gap: 0.15rem; + padding: 0.6rem 0.75rem; + border: 1px solid var(--color-rule-2); + border-radius: 0.375rem; + background: var(--color-panel); +} + +[data-iii-ui='pdf'] .pdf-ui__stat-label { + color: var(--color-ink-ghost); + font-size: 0.7rem; + letter-spacing: 0.04em; + text-transform: uppercase; + margin: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__stat-value { + color: var(--color-ink); + font-size: 1.05rem; + font-variant-numeric: tabular-nums; + margin: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__section { + color: var(--color-ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0 0 0.5rem; +} + +[data-iii-ui='pdf'] .pdf-ui__ocr-list { + list-style: none; + display: flex; + flex-direction: column; + gap: 0.25rem; + margin: 0; + padding: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__ocr-row { + display: flex; + align-items: baseline; + gap: 0.6rem; + padding: 0.35rem 0.6rem; + border-radius: 0.3rem; + background: var(--color-panel); + font-size: 0.85rem; +} + +[data-iii-ui='pdf'] .pdf-ui__ocr-page { + color: var(--color-ink); + font-variant-numeric: tabular-nums; + min-width: 5rem; +} + +[data-iii-ui='pdf'] .pdf-ui__ocr-why { + color: var(--color-ink-faint); +} + +[data-iii-ui='pdf'] .pdf-ui__tabs { + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +[data-iii-ui='pdf'] .pdf-ui__pane { + max-height: 34rem; + overflow: auto; + border: 1px solid var(--color-rule-2); + border-radius: 0.375rem; + padding: 1rem; + background: var(--color-bg); +} + +[data-iii-ui='pdf'] .pdf-ui__page { + padding-bottom: 1rem; + margin-bottom: 1rem; + border-bottom: 1px solid var(--color-rule-2); +} + +[data-iii-ui='pdf'] .pdf-ui__page:last-child { + border-bottom: none; + margin-bottom: 0; + padding-bottom: 0; +} + +[data-iii-ui='pdf'] .pdf-ui__page-head { + display: flex; + align-items: center; + gap: 0.5rem; + margin-bottom: 0.5rem; +} + +[data-iii-ui='pdf'] .pdf-ui__page-no { + color: var(--color-ink-ghost); + font-size: 0.75rem; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +/* The shared Monaco editor grows with its content, so the pane scrolls. */ +[data-iii-ui='pdf'] .pdf-ui__editor { + min-height: 20rem; + width: 100%; +} + +[data-iii-ui='pdf'] .pdf-ui__timing { + color: var(--color-ink-ghost); + font-size: 0.78rem; + font-variant-numeric: tabular-nums; + margin: 0; +} + +/* The chat and trace renderer for pdf::* calls. */ + +[data-iii-ui='pdf'] .pdf-trigger { + display: flex; + flex-direction: column; + gap: 0.3rem; +} + +[data-iii-ui='pdf'] .pdf-trigger__row { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.5rem; +} + +[data-iii-ui='pdf'] .pdf-trigger__file { + color: var(--color-ink); + font-size: 0.85rem; +} + +[data-iii-ui='pdf'] .pdf-trigger__meta { + color: var(--color-ink-ghost); + font-size: 0.75rem; + font-variant-numeric: tabular-nums; +} + +[data-iii-ui='pdf'] .pdf-trigger__line { + color: var(--color-ink-faint); + font-size: 0.85rem; + line-height: 1.5; + margin: 0; +} diff --git a/pdf/ui/tsconfig.json b/pdf/ui/tsconfig.json new file mode 100644 index 000000000..e5ac60540 --- /dev/null +++ b/pdf/ui/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": true, + "noEmit": true, + "types": [] + }, + "include": ["page.tsx", "src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89d682eda..3538af1e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -321,6 +321,22 @@ importers: specifier: ^19.2.14 version: 19.2.17 + pdf/ui: + dependencies: + '@iii-dev/console-ui': + specifier: workspace:* + version: link:../../packages/console-ui + devDependencies: + '@types/react': + specifier: ^19.2.14 + version: 19.2.17 + esbuild: + specifier: ^0.25.0 + version: 0.25.12 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + state/ui: dependencies: '@iii-dev/console-ui': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cf84811de..139931d87 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,6 +15,7 @@ packages: - editor/ui - eval/ui - memory/ui + - pdf/ui - state/ui - iii-directory/ui - worktree/ui