diff --git a/DESIGN.md b/DESIGN.md index 6ed19b3..ea4d187 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -201,6 +201,7 @@ A default `countTokens` ships with the core (word-level + unicode CJK tokenizer, | prune (range → summary block) | **core** `processTurn` (pure) | | boundary resolution / search | **core** `resolveBoundaries` (pure) | | block allocation / state mutation / tiers | **core** `applyCompression` (pure) | +| compress **argument parsing** (lenient: fences, trailing commas, truncated-array salvage; field-name variants) | **core** `parseCompressArgs` (pure; diagnostics are data — adapters emit them) | | young→old promotion / batch merge | **core** `sync-blocks` node (`advanceSurvival`) + `merge-blocks` node (pure) | | emergency truncation (context near full) | **core** `emergency-truncate` node — the LAST safety valve; no age-based GC | | protected-tools filtering logic | **core** (pure: message + config → bool) | diff --git a/src/index.ts b/src/index.ts index c3772b9..a1ddde6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -54,6 +54,8 @@ export { hideConsumedCompressCalls } from "./hide-consumed.js"; export type { HideConsumedResult } from "./hide-consumed.js"; export { rebuildCompressionState } from "./rebuild.js"; export type { RebuildResult, RebuildPorts } from "./rebuild.js"; +export { parseCompressArgs } from "./parse-compress-input.js"; +export type { CompressParseDiagnostics, CompressParseKind, ParsedCompressInput } from "./parse-compress-input.js"; export { renderVisibleRefs, renderRefsNode, createRenderRefsNode } from "./render-refs.js"; export type { RenderStrategy } from "./render-refs.js"; export { resolveTransformChannel } from "./transform-channel.js"; diff --git a/src/parse-compress-input.ts b/src/parse-compress-input.ts new file mode 100644 index 0000000..374086f --- /dev/null +++ b/src/parse-compress-input.ts @@ -0,0 +1,394 @@ +// Single lenient parser for compress tool arguments, with structured +// diagnostics for the failure shapes that actually occur in production. +// +// Models and LLM gateways do not always emit strict JSON for the compress +// tool call. Observed shapes: +// - fenced JSON: "```json ... ```" +// - trailing commas +// - raw newlines inside string values (line-wrapped summaries) +// - the whole arguments object stringified by the gateway (vLLM, +// billion-context#176) +// - the stream cut off mid-arguments, leaving a truncated JSON prefix +// +// Hosts parse this on their own today: rebuild.ts (strict, silent skip), +// the billion-context proxy (strict, silent {}), billion-context-pi +// (strict, throw), billion-context-omp (strict, silent null). This module +// is the shared implementation the adapters converge on (acp-kernel#108). +// +// Salvage semantics: for truncated input, the complete entries of the +// `content` array are recovered from the surviving prefix. A partially +// written entry is dropped, never guessed. Diagnostics are data, not logs: +// adapters decide where to emit them (log line, debug event, tool text). + +import type { CompressRangeSpec } from "./types.js"; + +export type CompressParseKind = + | "ok" + | "empty-input" + | "not-object" + | "missing-content" + | "content-not-array" + | "malformed-json" + | "truncated" + | "no-valid-ranges"; + +export interface CompressParseDiagnostics { + /** true only when at least one range was recovered. */ + ok: boolean; + /** Why the input parsed the way it did. */ + kind: CompressParseKind; + /** First 800 chars of the raw string input (string inputs only). */ + rawPrefix?: string; + /** Raw string input length (string inputs only). */ + length?: number; + /** Top-level keys of the parsed object — catches `content` vs `ranges` drift. */ + keys?: string[]; + /** Entries dropped because they were not valid ranges. */ + invalidItems: number; +} + +export interface ParsedCompressInput { + ranges: CompressRangeSpec[]; + diagnostics: CompressParseDiagnostics; +} + +/** + * Parse compress tool arguments in any host wire shape. + * + * Accepts a decoded object, a JSON string (possibly fenced, trailing-comma, + * raw-newline, or double-stringified), or a truncated JSON prefix (salvage + * mode). Invalid entries are skipped, never fatal; the reason is in + * `diagnostics.kind`. + */ +export function parseCompressArgs(input: unknown, opts?: { callId?: string }): ParsedCompressInput { + const callId = opts?.callId; + const diag: CompressParseDiagnostics = { ok: false, kind: "ok", invalidItems: 0 }; + + if (input === null || input === undefined) { + diag.kind = "empty-input"; + return finish([], diag); + } + + if (typeof input === "string") { + return parseStringInput(input, callId, diag); + } + + if (typeof input !== "object" || Array.isArray(input)) { + diag.kind = "not-object"; + return finish([], diag); + } + + return parseObjectValue(input as Record, callId, diag); +} + +function parseStringInput(raw: string, callId: string | undefined, diag: CompressParseDiagnostics): ParsedCompressInput { + diag.rawPrefix = raw.slice(0, 800); + diag.length = raw.length; + const cleaned = stripFence(raw.trim()); + if (cleaned === "") { + diag.kind = "empty-input"; + return finish([], diag); + } + + let value: unknown = tryParseLenient(cleaned); + // One level of double-stringification: the host wrapped an already + // stringified argument in another JSON string. + if (typeof value === "string") { + const inner = tryParseLenient(stripFence(value)); + if (inner !== undefined) value = inner; + } + + if (value !== null && typeof value === "object" && !Array.isArray(value)) { + return parseObjectValue(value as Record, callId, diag); + } + if (value !== undefined) { + // Parsed, but not to an object: bare array, number, boolean, null. + diag.kind = "not-object"; + return finish([], diag); + } + + // Unparseable prefix: salvage the complete content-array entries. + const entries = salvageContentEntries(cleaned); + return finishSalvage(entries, callId, diag, looksTruncated(cleaned)); +} + +function parseObjectValue(value: Record, callId: string | undefined, diag: CompressParseDiagnostics): ParsedCompressInput { + diag.keys = Object.keys(value); + const content = value["content"]; + if (content === undefined) { + // Model drift: a single range at the top level (no content array). + // The proxy defends against this shape; the kernel now owns it. + const single = validateEntry(value, callId); + if (single) { + diag.kind = "ok"; + return finish([single], diag); + } + diag.kind = "missing-content"; + return finish([], diag); + } + + let entries: unknown[]; + let salvaged = false; + + if (Array.isArray(content)) { + entries = content; + } else if (typeof content === "string") { + // Stringified content array: vLLM-style gateways stringify nested + // arrays, so `content` arrives as a JSON string of the array. + const parsed = parseContentArray(content); + if (parsed === null) { + diag.kind = "content-not-array"; + return finish([], diag); + } + entries = parsed.entries; + salvaged = parsed.salvaged; + } else { + diag.kind = "content-not-array"; + return finish([], diag); + } + + const { ranges, invalid } = validateEntries(entries, callId); + // Top-level fallbacks: topic and summaryMaxChars apply to every range + // that does not specify its own (omp/pi schemas define both at the top + // level; per-entry values win). + const topTopic = stringOr(value["topic"]); + const topMaxChars = value["summaryMaxChars"]; + const hasTopMaxChars = typeof topMaxChars === "number" && Number.isFinite(topMaxChars); + if (topTopic !== undefined || hasTopMaxChars) { + for (const r of ranges) { + if (r.topic === undefined && topTopic !== undefined) r.topic = topTopic; + if (r.summaryMaxChars === undefined && hasTopMaxChars) r.summaryMaxChars = topMaxChars; + } + } + diag.invalidItems = invalid; + diag.kind = salvaged ? "truncated" : ranges.length > 0 ? "ok" : "no-valid-ranges"; + return finish(ranges, diag); +} + +function parseContentArray(s: string): { entries: unknown[]; salvaged: boolean } | null { + const cleaned = stripFence(s.trim()); + let value: unknown = cleaned === "" ? undefined : tryParseLenient(cleaned); + if (typeof value === "string") { + value = tryParseLenient(stripFence(value)); + } + if (Array.isArray(value)) { + return { entries: value, salvaged: false }; + } + if (value === undefined) { + // Unparseable (usually truncated): recover the complete entries. + const entries = salvageContentEntries('{"content": ' + cleaned); + return { entries, salvaged: entries.length > 0 }; + } + return null; +} + +function finish(ranges: CompressRangeSpec[], diag: CompressParseDiagnostics): ParsedCompressInput { + diag.ok = ranges.length > 0; + return { ranges, diagnostics: diag }; +} + +function finishSalvage(entries: unknown[], callId: string | undefined, diag: CompressParseDiagnostics, truncatedShape: boolean): ParsedCompressInput { + const { ranges, invalid } = validateEntries(entries, callId); + diag.invalidItems = invalid; + diag.kind = entries.length > 0 || truncatedShape ? "truncated" : "malformed-json"; + return finish(ranges, diag); +} + +function validateEntries(entries: unknown[], callId: string | undefined): { ranges: CompressRangeSpec[]; invalid: number } { + const ranges: CompressRangeSpec[] = []; + let invalid = 0; + for (const entry of entries) { + const range = validateEntry(entry, callId); + if (range !== null) ranges.push(range); + else invalid++; + } + return { ranges, invalid }; +} + +function validateEntry(entry: unknown, callId: string | undefined): CompressRangeSpec | null { + if (entry === null || typeof entry !== "object" || Array.isArray(entry)) return null; + const e = entry as Record; + // Field-name variants: startRef/endRef are canonical; startId/endId is + // model drift (and the legacy rebuild.ts spelling); messageId is the + // startId-less messageRef from the historical API. + const start = stringOr(e["startRef"]) ?? stringOr(e["startId"]) ?? stringOr(e["messageId"]); + const end = stringOr(e["endRef"]) ?? stringOr(e["endId"]) ?? stringOr(e["messageId"]); + if (start === undefined || end === undefined) return null; + const summary = stringOr(e["summary"]); + if (summary === undefined) return null; + const range: CompressRangeSpec = { startRef: start, endRef: end, summary }; + const topic = stringOr(e["topic"]); + if (topic !== undefined) range.topic = topic; + const maxChars = e["summaryMaxChars"]; + if (typeof maxChars === "number" && Number.isFinite(maxChars)) range.summaryMaxChars = maxChars; + if (callId !== undefined) range.compressCallId = callId; + return range; +} + +function stringOr(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +// --- tolerant JSON parsing ------------------------------------------------- + +function tryParseLenient(s: string): unknown { + if (s === "") return undefined; + try { + return JSON.parse(s); + } catch { + // keep trying the repaired variants below + } + const noTrailingCommas = stripTrailingCommas(s); + if (noTrailingCommas !== s) { + try { + return JSON.parse(noTrailingCommas); + } catch { + // keep trying + } + } + const fixed = escapeRawNewlinesInStrings(noTrailingCommas); + if (fixed !== noTrailingCommas) { + try { + return JSON.parse(fixed); + } catch { + // fall through to salvage + } + } + return undefined; +} + +function stripFence(s: string): string { + if (!s.startsWith("```")) return s; + const firstNewline = s.indexOf("\n"); + if (firstNewline === -1) return s; + const bodyStart = firstNewline + 1; + const end = s.lastIndexOf("```"); + return end > bodyStart ? s.slice(bodyStart, end).trim() : s.slice(bodyStart).trim(); +} + +// Drop commas that sit immediately before a closing brace/bracket +// (outside string literals). +function stripTrailingCommas(s: string): string { + let out = ""; + let inString = false; + let escaped = false; + for (let i = 0; i < s.length; i++) { + const ch = s.charAt(i); + if (inString) { + out += ch; + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') { + inString = true; + out += ch; + continue; + } + if (ch === ",") { + let j = i + 1; + while (j < s.length && (s.charAt(j) === " " || s.charAt(j) === "\t" || s.charAt(j) === "\n" || s.charAt(j) === "\r")) j++; + if (j < s.length && (s.charAt(j) === "}" || s.charAt(j) === "]")) continue; + } + out += ch; + } + return out; +} + +// Raw \n, \r, \t inside string literals are invalid JSON; escape them. +// Outside strings they are legal whitespace and left alone. +function escapeRawNewlinesInStrings(s: string): string { + let out = ""; + let inString = false; + let escaped = false; + for (let i = 0; i < s.length; i++) { + const ch = s.charAt(i); + if (!inString) { + if (ch === '"') inString = true; + out += ch; + continue; + } + if (escaped) { + out += ch; + escaped = false; + continue; + } + if (ch === "\\") { + out += ch; + escaped = true; + continue; + } + if (ch === "\n") { out += "\\n"; continue; } + if (ch === "\r") { out += "\\r"; continue; } + if (ch === "\t") { out += "\\t"; continue; } + if (ch === '"') inString = false; + out += ch; + } + return out; +} + +// Unbalanced brackets or an unterminated string at end of input is the +// signature of a mid-stream cutoff (as opposed to balanced garbage). +function looksTruncated(s: string): boolean { + let depth = 0; + let inString = false; + let escaped = false; + for (let i = 0; i < s.length; i++) { + const ch = s.charAt(i); + if (inString) { + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') { inString = true; continue; } + if (ch === "{" || ch === "[") depth++; + else if (ch === "}" || ch === "]") depth--; + } + return depth > 0 || inString; +} + +/** + * Recover the complete entries of the `content` array from a truncated JSON + * prefix. Walks the prefix with a small state machine (string / escape / + * bracket depth); every object that opens and closes at depth 1 is + * re-parsed leniently and kept only if it still parses. Partial entries are + * dropped — the parser never invents model content. + */ +function salvageContentEntries(raw: string): unknown[] { + const match = /"content"\s*:\s*\[/.exec(raw); + if (match === null) return []; + const arrayStart = match.index + match[0].length - 1; + const entries: unknown[] = []; + let depth = 0; // brackets nested inside the content array + let inString = false; + let escaped = false; + let entryStart = -1; + for (let i = arrayStart + 1; i < raw.length; i++) { + const ch = raw.charAt(i); + if (inString) { + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') { inString = true; continue; } + if (ch === "{" || ch === "[") { + if (depth === 0 && ch === "{" && entryStart === -1) entryStart = i; + depth++; + continue; + } + if (ch === "}" || ch === "]") { + depth--; + if (depth < 0) break; // the content array itself closed + if (depth === 0 && entryStart !== -1) { + const entrySlice = raw.slice(entryStart, i + 1); + entryStart = -1; + const parsed = tryParseLenient(entrySlice); + if (parsed !== undefined) entries.push(parsed); + } + } + } + return entries; +} diff --git a/src/rebuild.ts b/src/rebuild.ts index 2768191..6fe10c3 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -1,16 +1,9 @@ import { createCore } from "./compress.js"; +import { parseCompressArgs } from "./parse-compress-input.js"; import { assignRefs, highestUsedIndex } from "./refs.js"; import { defaultCountTokens } from "./tokenize.js"; import type { CompressionState, CoreMessage } from "./types.js"; -export interface CompressInputEntry { - startId?: string; - endId?: string; - messageId?: string; - summary: string; - topic?: string; -} - export interface RebuildResult { state: CompressionState; blocksRebuilt: number; @@ -26,7 +19,9 @@ export interface RebuildPorts { * message order, so they are fork-stable — a ref in a historical compress * input points to the same logical message after a fork regenerates IDs. * The rebuilt state is an approximation: only raw model summaries are - * replayed (no protected-content enrichments). + * replayed (no protected-content enrichments). Arguments are parsed with + * the lenient parseCompressArgs, so truncated or stringified historical + * inputs are salvaged instead of silently dropped. */ export function rebuildCompressionState( state: CompressionState, @@ -45,7 +40,7 @@ export function rebuildCompressionState( let blocksRebuilt = 0; for (const invocation of invocations) { - const ranges = extractRanges(invocation.input, invocation.callId); + const { ranges } = parseCompressArgs(invocation.raw, { callId: invocation.callId }); if (ranges.length === 0) continue; const result = core.applyCompression({ ranges, messages, state: working, config }); working = result.state; @@ -57,51 +52,14 @@ export function rebuildCompressionState( interface CompressInvocation { callId: string | undefined; - input: unknown; + raw: string; } function collectCompressInvocations(messages: CoreMessage[]): CompressInvocation[] { const invocations: CompressInvocation[] = []; for (const message of messages) { if (message.toolName !== "compress" || message.contentType !== "tool-call") continue; - let input: unknown; - try { - input = JSON.parse(message.text ?? ""); - } catch { - continue; - } - invocations.push({ callId: message.toolCallId, input }); + invocations.push({ callId: message.toolCallId, raw: message.text ?? "" }); } return invocations; } - -function extractRanges( - input: unknown, - callId: string | undefined, -): Array<{ - startRef: string; - endRef: string; - summary: string; - topic?: string; - compressCallId?: string; -}> { - const content = (input as { content?: unknown[] })?.content; - if (!Array.isArray(content)) return []; - const ranges = []; - for (const entry of content) { - if (!entry || typeof entry !== "object") continue; - const e = entry as CompressInputEntry; - if (typeof e.summary !== "string") continue; - const start = e.startId ?? e.messageId; - const end = e.endId ?? e.messageId; - if (typeof start !== "string" || typeof end !== "string") continue; - ranges.push({ - startRef: start, - endRef: end, - summary: e.summary, - topic: typeof e.topic === "string" ? e.topic : undefined, - compressCallId: callId, - }); - } - return ranges; -} diff --git a/tests/parse-compress-input.test.ts b/tests/parse-compress-input.test.ts new file mode 100644 index 0000000..58a2079 --- /dev/null +++ b/tests/parse-compress-input.test.ts @@ -0,0 +1,434 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseCompressArgs } from "../src/parse-compress-input.js"; +import { rebuildCompressionState } from "../src/rebuild.js"; +import { createInitialState } from "../src/state.js"; +import { defaultConfig } from "../src/config.js"; +import type { CompressRangeSpec, CoreMessage } from "../src/types.js"; + +// --------------------------------------------------------------------------- +// Input-shape normalization +// --------------------------------------------------------------------------- + +test("parseCompressArgs parses a valid object with all fields", () => { + const input = { + content: [ + { startRef: "m00001", endRef: "m00002", summary: "intro", topic: "setup", summaryMaxChars: 5000 }, + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(diagnostics.ok, true); + assert.equal(diagnostics.invalidItems, 0); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00002"); + assert.equal(ranges[0]?.summary, "intro"); + assert.equal(ranges[0]?.topic, "setup"); + assert.equal(ranges[0]?.summaryMaxChars, 5000); +}); + +test("parseCompressArgs parses a raw JSON document string", () => { + const input = JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "intro" }], + }); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "intro"); + assert.equal(diagnostics.length, input.length); + assert.equal(diagnostics.rawPrefix, input.slice(0, 800)); +}); + +test("parseCompressArgs unwraps one level of double-stringification", () => { + const input = JSON.stringify( + JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "doubled" }], + }), + ); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "doubled"); +}); + +test("parseCompressArgs accepts a stringified content array (vLLM shape)", () => { + const input = { + content: JSON.stringify([{ startRef: "m00001", endRef: "m00002", summary: "vllm" }]), + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.summary, "vllm"); +}); + +test("parseCompressArgs salvages a stringified content array truncated mid-entry", () => { + const full = [ + { startRef: "m00001", endRef: "m00002", summary: "first entry complete" }, + { startRef: "m00003", endRef: "m00004", summary: "second entry cut off mid" }, + ]; + const cut = JSON.stringify(full).slice(0, JSON.stringify(full).length - 12); + const input = { content: cut }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first entry complete"); + assert.equal(diagnostics.invalidItems, 0); +}); + +test("parseCompressArgs strips markdown fences (with and without language tag)", () => { + const doc = JSON.stringify({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "fenced" }], + }); + for (const input of ["```json\n" + doc + "\n```", "```\n" + doc + "\n```"]) { + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok", "expected ok for: " + input.slice(0, 20)); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "fenced"); + } +}); + +test("parseCompressArgs repairs trailing commas", () => { + const input = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "S",}]}'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "S"); +}); + +test("parseCompressArgs escapes raw newlines inside JSON string values", () => { + // A provider serialized a real newline inside the summary string. + const input = + '{"content": [{"startRef": "m00001", "endRef": "m00002", "summary": "line1\nline2"}]}'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "line1\nline2"); +}); + +test("parseCompressArgs preserves escaped quotes in salvaged entries", () => { + const entry = { startRef: "m00001", endRef: "m00002", summary: 'has "quotes" inside' }; + const doc = JSON.stringify({ content: [entry, { startRef: "m00003", endRef: "m00004", summary: "gone" }] }); + // Cut after the first entry's closing brace (the comma follows it). + const firstClose = doc.indexOf("}") + 1; + const input = doc.slice(0, firstClose) + ', {"startRef": "m00003", "endRef": "m0'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, 'has "quotes" inside'); +}); + +// --------------------------------------------------------------------------- +// Truncation salvage +// --------------------------------------------------------------------------- + +test("parseCompressArgs salvages complete entries from a truncated JSON prefix", () => { + const doc = JSON.stringify({ + content: [ + { startRef: "m00001", endRef: "m00002", summary: "first entry complete" }, + { startRef: "m00003", endRef: "m00004", summary: "second entry cut off mid" }, + ], + }); + const input = doc.slice(0, doc.length - 12); + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "first entry complete"); +}); + +test("parseCompressArgs salvages entries when the closing brackets are missing entirely", () => { + const entry = '{"startRef": "m00001", "endRef": "m00002", "summary": "no brackets after me"}'; + const input = '{"content": [' + entry + ", {\"startRef\": \"m00003\""; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, true); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "no brackets after me"); +}); + +test("parseCompressArgs reports truncated with no ranges when the first entry is cut", () => { + const input = '{"content": [{"startRef": "m00001", "endRef": "m0'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); +}); + +test("parseCompressArgs does not fabricate partial entries", () => { + // startRef present, endRef/summary missing and cut: entry is incomplete, must be dropped. + const input = '{"content": [{"startRef": "m00001", "summary": "par'; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "truncated"); + assert.equal(ranges.length, 0); +}); + +// --------------------------------------------------------------------------- +// Error classification +// --------------------------------------------------------------------------- + +test("parseCompressArgs classifies empty inputs", () => { + for (const input of [null, undefined, "", " "]) { + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "empty-input", JSON.stringify(input)); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + } +}); + +test("parseCompressArgs classifies non-object parsed values", () => { + for (const input of [[], 42, '"just a string"', "true"]) { + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "not-object", JSON.stringify(input)); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + } +}); + +test("parseCompressArgs classifies an object without a content key", () => { + const { diagnostics } = parseCompressArgs({ foo: 1, bar: "x" }); + assert.equal(diagnostics.kind, "missing-content"); + assert.equal(diagnostics.ok, false); + assert.deepEqual(diagnostics.keys, ["foo", "bar"]); +}); + +test("parseCompressArgs classifies a content value that is neither array nor array-string", () => { + const { diagnostics } = parseCompressArgs({ content: "123" }); + assert.equal(diagnostics.kind, "content-not-array"); + assert.equal(diagnostics.ok, false); +}); + +test("parseCompressArgs classifies balanced garbage as malformed-json", () => { + const { ranges, diagnostics } = parseCompressArgs("hello"); + assert.equal(diagnostics.kind, "malformed-json"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); +}); + +test("parseCompressArgs counts invalid entries and keeps the valid ones", () => { + const input = { + content: [ + { startRef: "m00001", endRef: "m00002", summary: "good" }, + { summary: "missing refs" }, + { startRef: "m00003", endRef: "m00004" }, + { startRef: "m00005", endRef: "m00006", summary: 42 }, + "garbage", + null, + ], + }; + const { ranges, diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]?.summary, "good"); + assert.equal(diagnostics.invalidItems, 5); +}); + +test("parseCompressArgs reports no-valid-ranges for an empty content array", () => { + const { ranges, diagnostics } = parseCompressArgs({ content: [] }); + assert.equal(diagnostics.kind, "no-valid-ranges"); + assert.equal(diagnostics.ok, false); + assert.equal(ranges.length, 0); + assert.equal(diagnostics.invalidItems, 0); +}); + +test("parseCompressArgs reports no-valid-ranges for an empty stringified content", () => { + const { ranges, diagnostics } = parseCompressArgs({ content: "" }); + assert.equal(diagnostics.kind, "no-valid-ranges"); + assert.equal(ranges.length, 0); +}); + +// --------------------------------------------------------------------------- +// Field-name variants and call-id stamping +// --------------------------------------------------------------------------- + +test("parseCompressArgs accepts startId/endId and messageId name variants", () => { + const { ranges, diagnostics } = parseCompressArgs({ + content: [ + { startId: "m00001", endId: "m00002", summary: "variant-a" }, + { messageId: "m00003", summary: "variant-b" }, + ], + }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 2); + assert.equal(ranges[0]?.startRef, "m00001"); + assert.equal(ranges[0]?.endRef, "m00002"); + assert.equal(ranges[1]?.startRef, "m00003"); + assert.equal(ranges[1]?.endRef, "m00003"); +}); + +test("parseCompressArgs stamps compressCallId when a callId is provided", () => { + const input = { content: [{ startRef: "m00001", endRef: "m00002", summary: "stamped" }] }; + const stamped = parseCompressArgs(input, { callId: "call-9" }); + assert.equal(stamped.ranges[0]?.compressCallId, "call-9"); + const plain = parseCompressArgs(input); + assert.equal(plain.ranges[0]?.compressCallId, undefined); +}); + +// --------------------------------------------------------------------------- +// Diagnostics shape +// --------------------------------------------------------------------------- + +test("parseCompressArgs caps rawPrefix at 800 chars and reports full length", () => { + const input = "x".repeat(1000); + const { diagnostics } = parseCompressArgs(input); + assert.equal(diagnostics.kind, "malformed-json"); + assert.equal(diagnostics.length, 1000); + assert.equal(diagnostics.rawPrefix?.length, 800); +}); + +test("parseCompressArgs leaves rawPrefix/length undefined for object input", () => { + const { diagnostics } = parseCompressArgs({ content: [] }); + assert.equal(diagnostics.rawPrefix, undefined); + assert.equal(diagnostics.length, undefined); +}); + +// --------------------------------------------------------------------------- +// rebuildCompressionState regression (fork-recovery path) +// --------------------------------------------------------------------------- + +function rebuildMessages(toolCallText: string, toolCallId: string): CoreMessage[] { + const messages: CoreMessage[] = [ + { id: "raw1", role: "user", contentType: "text", text: "first message" }, + { id: "raw2", role: "assistant", contentType: "text", text: "second message" }, + ]; + if (toolCallText !== "") { + messages.push({ + id: "raw3", + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId, + text: toolCallText, + }); + } + return messages; +} + +function rebuildConfig() { + return defaultConfig(200000, { + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + }); +} + +test("rebuildCompressionState rebuilds from a strict JSON object (pre-existing behavior)", () => { + const args = JSON.stringify({ + content: [{ startId: "m00001", endId: "m00002", summary: "strict object" }], + }); + const result = rebuildCompressionState(createInitialState(), rebuildMessages(args, "call1"), rebuildConfig()); + assert.equal(result.blocksRebuilt, 1); + const block = result.state.blocks.find((b) => b.summary.includes("strict object")); + assert.ok(block); + assert.equal(block.compressCallId, "call1"); +}); + +test("rebuildCompressionState recovers vLLM stringified content (fork-recovery gap fix)", () => { + // vLLM hosts stringify the nested content array; the old strict + // Array.isArray check dropped these silently. + const args = JSON.stringify({ + content: JSON.stringify([{ startId: "m00001", endId: "m00002", summary: "vllm stringified" }]), + }); + const result = rebuildCompressionState(createInitialState(), rebuildMessages(args, "call1"), rebuildConfig()); + assert.equal(result.blocksRebuilt, 1); + const block = result.state.blocks.find((b) => b.summary.includes("vllm stringified")); + assert.ok(block); + assert.equal(block.compressCallId, "call1"); +}); + +test("rebuildCompressionState salvages a truncated compress tool-call text", () => { + const doc = JSON.stringify({ + content: [ + { startId: "m00001", endId: "m00002", summary: "survives truncation" }, + { startId: "m00003", endId: "m00004", summary: "cut off here" }, + ], + }); + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages(doc.slice(0, doc.length - 12), "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 1); + assert.ok(result.state.blocks.find((b) => b.summary.includes("survives truncation"))); +}); + +test("rebuildCompressionState survives garbage tool-call text without rebuilding", () => { + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages("total garbage not json", "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 0); + assert.equal(result.state.blocks.length, 0); +}); + +test("rebuildCompressionState skips compress calls with no valid ranges", () => { + const result = rebuildCompressionState( + createInitialState(), + rebuildMessages(JSON.stringify({ content: [] }), "call1"), + rebuildConfig(), + ); + assert.equal(result.blocksRebuilt, 0); + assert.equal(result.state.blocks.length, 0); +}); + +test("parseCompressArgs returns ranges typed as CompressRangeSpec", () => { + // Compile-time contract: ranges feed core.applyCompression directly. + const { ranges } = parseCompressArgs({ + content: [{ startRef: "m00001", endRef: "m00002", summary: "typed" }], + }); + const spec: CompressRangeSpec | undefined = ranges[0]; + assert.ok(spec); + assert.equal(typeof spec.startRef, "string"); +}); + +// --- top-level shapes the adapters observe (single range, topic, summaryMaxChars) --- + +test("parseCompressArgs accepts a single range at the top level (no content array)", () => { + const { ranges, diagnostics } = parseCompressArgs({ startRef: "m00001", endRef: "m00002", summary: "single" }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startRef, "m00001"); + assert.equal(ranges[0]!.endRef, "m00002"); + assert.equal(ranges[0]!.summary, "single"); +}); + +test("parseCompressArgs accepts a top-level single range with startId/endId variants", () => { + const { ranges, diagnostics } = parseCompressArgs({ startId: "m00001", endId: "m00002", summary: "single" }); + assert.equal(diagnostics.kind, "ok"); + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startRef, "m00001"); +}); + +test("parseCompressArgs reports missing-content when there is no content and no valid single range", () => { + const { ranges, diagnostics } = parseCompressArgs({ foo: "bar" }); + assert.equal(diagnostics.kind, "missing-content"); + assert.equal(ranges.length, 0); +}); + +test("parseCompressArgs applies a top-level topic to ranges without their own", () => { + const { ranges } = parseCompressArgs({ + topic: "Top", + content: [ + { startRef: "m00001", endRef: "m00002", summary: "a" }, + { startRef: "m00003", endRef: "m00004", summary: "b", topic: "Own" }, + ], + }); + assert.equal(ranges[0]!.topic, "Top"); + assert.equal(ranges[1]!.topic, "Own"); +}); + +test("parseCompressArgs applies a top-level summaryMaxChars to ranges without their own", () => { + const { ranges } = parseCompressArgs({ + summaryMaxChars: 5000, + content: [ + { startRef: "m00001", endRef: "m00002", summary: "a" }, + { startRef: "m00003", endRef: "m00004", summary: "b", summaryMaxChars: 999 }, + ], + }); + assert.equal(ranges[0]!.summaryMaxChars, 5000); + assert.equal(ranges[1]!.summaryMaxChars, 999); +});