diff --git a/docs/claude-instructions-cache-stabilize/PAPER_OCXFIX.pdf b/docs/claude-instructions-cache-stabilize/PAPER_OCXFIX.pdf new file mode 100644 index 0000000000..5206d30d1f Binary files /dev/null and b/docs/claude-instructions-cache-stabilize/PAPER_OCXFIX.pdf differ diff --git a/docs/claude-instructions-cache-stabilize/README.md b/docs/claude-instructions-cache-stabilize/README.md new file mode 100644 index 0000000000..44cf88a995 --- /dev/null +++ b/docs/claude-instructions-cache-stabilize/README.md @@ -0,0 +1,28 @@ +# Claude instructions cache stabilize (OCXFIX) + +OpenCodex inbound conversion puts Claude Code system text into OpenAI Responses +`instructions`. Claude Code then appends a growing `…` +footer (and occasional TaskCreate nudges) on every turn. That prefix churn +causes prompt-cache misses on Muse/Go. + +This change strips those dynamic footers from `instructions` and reattaches the +latest notice as a trailing `input` message so the cacheable prefix stays stable. + +## Paper + +See [PAPER_OCXFIX.pdf](./PAPER_OCXFIX.pdf) (Warexpor). + +Measured cache-hit rates on Muse Spark 1.3 via OpenCode Go / OpenCodex: + +| Slice | Baseline mean | OCXFIX mean | +| --- | ---: | ---: | +| Claude Code (n=75) | 0.168384 | 0.864374 | +| Claude S/T4 (n=5) | 0.134922 | 0.982700 | +| Grok Build (n=75) | 0.966526 | 0.941345 | + +Cause: Anthropic→Responses conversion stores Claude system text in `instructions`; +Claude Code appends growing `` (and rare TaskCreate nudges), so +`instructions_sha` changes every turn. Grok traffic has no `instructions` field +and is the control. + +Code: `src/claude/inbound-cache-stabilize.ts`, wired from `src/claude/inbound.ts`. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 0ff7d4861c..ef04b588e2 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -304,6 +304,7 @@ "claude-desktop-remote-hub.test.ts": "claude-integration", "claude-dotenv-provenance-transport.test.ts": "claude-integration", "claude-gateway-cache.test.ts": "claude-integration", + "claude-inbound-cache-stabilize.test.ts": "claude-integration", "claude-inbound-debug.test.ts": "claude-integration", "claude-inbound.test.ts": "claude-integration", "claude-management-api.test.ts": "claude-integration", diff --git a/src/claude/inbound-cache-stabilize.ts b/src/claude/inbound-cache-stabilize.ts new file mode 100644 index 0000000000..e5e3e2aedf --- /dev/null +++ b/src/claude/inbound-cache-stabilize.ts @@ -0,0 +1,48 @@ +/** + * Claude Code appends growing `…` footers (and + * occasional TaskCreate nudges) into system text that becomes Responses + * `instructions`. That churn breaks Muse/Go prefix cache on the instructions + * prefix even when tools stay stable. Strip dynamics from instructions; + * surface the latest notice on `input` instead. + * + * Only canonical harness notices are stripped: a standalone integer + * `` line, and the exact TaskCreate reminder paragraph. + * Inline documentation of those tags/tools is left in `instructions`. + */ + +/** Standalone harness footer: own line, integer payload, not mid-sentence docs. */ +const TOTAL_TOKENS_RE = + /(?:^|\r?\n)[ \t]*(\d+<\/total_tokens>)[ \t]*(?=\r?\n|$)/g; + +/** Canonical Claude Code reminder: distinctive opening sentence through closing reminder. */ +const TASKCREATE_NUDGE_RE = + /(?:^|\r?\n)[ \t]*(The task tools haven't been used recently\.\s+If you're working on tasks that would benefit from tracking, consider using TaskCreate to add them\.\s+Only use these if relevant to the current work\.\s+This is just a gentle reminder - ignore if not applicable\.)[ \t]*(?=\r?\n|$)/g; + +export function stabilizeClaudeInstructionsForPromptCache( + instructions: string, +): { instructions: string; dynamicNotice: string | null } { + if (!instructions) { + return { instructions: "", dynamicNotice: null }; + } + + let latestTotal: string | null = null; + for (const m of instructions.matchAll(TOTAL_TOKENS_RE)) { + latestTotal = m[1] ?? m[0]; + } + + let latestNudge: string | null = null; + for (const m of instructions.matchAll(TASKCREATE_NUDGE_RE)) { + latestNudge = (m[1] ?? m[0]).trim(); + } + + let cleaned = instructions.replace(TOTAL_TOKENS_RE, ""); + cleaned = cleaned.replace(TASKCREATE_NUDGE_RE, ""); + cleaned = cleaned.replace(/\n{3,}/g, "\n\n").trim(); + + const noticeParts: string[] = []; + if (latestTotal) noticeParts.push(latestTotal); + if (latestNudge) noticeParts.push(latestNudge); + const dynamicNotice = noticeParts.length > 0 ? noticeParts.join("\n\n") : null; + + return { instructions: cleaned, dynamicNotice }; +} diff --git a/src/claude/inbound.ts b/src/claude/inbound.ts index c2e3ded9b2..57336fca03 100644 --- a/src/claude/inbound.ts +++ b/src/claude/inbound.ts @@ -17,6 +17,7 @@ export { resolveInboundModel, effortForThinkingBudget, effortFromOutputConfig, e import { AnthropicRequestError, isRec, type Rec } from "./inbound-records"; import { resolveInboundModel, effortForThinkingBudget, effortFromOutputConfig, formatFromOutputConfig } from "./inbound-model-options"; import { systemToInstructions, toolsToResponses, toolChoiceToResponses } from "./inbound-content-options"; +import { stabilizeClaudeInstructionsForPromptCache } from "./inbound-cache-stabilize"; import { decodeReasoningEnvelope, encodeReasoningEnvelope, OCX_REASONING_PREFIX } from "../responses/reasoning-envelope"; import { createTranslatorBudget, type TranslatorBudget } from "../lib/translator-budget"; @@ -345,7 +346,23 @@ function translateAnthropicRequest(raw: unknown, cc: OcxClaudeCodeConfig | undef stream: raw.stream === true, }; - if (systemParts.length > 0) body.instructions = systemParts.join("\n\n"); + let stabilizedInstructions = ""; + if (systemParts.length > 0) { + // Claude Code appends growing footers (and occasional + // TaskCreate nudges) into system text. That churn breaks Muse/Go prefix + // cache on the Responses instructions prefix even when tools stay stable. + // Strip dynamics from instructions; surface the latest notice on input. + const stabilized = stabilizeClaudeInstructionsForPromptCache(systemParts.join("\n\n")); + if (stabilized.instructions) body.instructions = stabilized.instructions; + if (stabilized.dynamicNotice) { + input.push({ + type: "message", + role: "user", + content: [{ type: "input_text", text: stabilized.dynamicNotice }], + }); + } + stabilizedInstructions = stabilized.instructions; + } const tools = toolsToResponses(raw.tools); if (tools) body.tools = tools; @@ -383,11 +400,12 @@ function translateAnthropicRequest(raw: unknown, cc: OcxClaudeCodeConfig | undef // Exact-prefix matching still isolates content; the key only steers routing // affinity. Callers must NOT synthesize a session_id header from this fallback // (audit 133 R2#3). + // Claude Code uses metadata.user_id session key; Desktop fallback must hash stabilized instructions so key tracks the cacheable prefix. body.prompt_cache_key = createHash("sha256") .update(canonicalJson({ version: 2, model: body.model, - system: systemParts, + system: stabilizedInstructions, tools: Array.isArray(body.tools) ? body.tools : [], })) .digest("hex").slice(0, 32); diff --git a/tests/claude-integration/claude-inbound-cache-stabilize.test.ts b/tests/claude-integration/claude-inbound-cache-stabilize.test.ts new file mode 100644 index 0000000000..d6f32ece69 --- /dev/null +++ b/tests/claude-integration/claude-inbound-cache-stabilize.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, test } from "bun:test"; +import { stabilizeClaudeInstructionsForPromptCache } from "../../src/claude/inbound-cache-stabilize"; +import { anthropicToResponsesTranslation } from "../../src/claude/inbound"; + +const TASKCREATE_NUDGE = [ + "The task tools haven't been used recently. If you're working on tasks that would benefit from tracking, consider using TaskCreate to add them.", + "Only use these if relevant to the current work. This is just a gentle reminder - ignore if not applicable.", +].join(" "); + +function footer(used: number): string { + return `${used}`; +} + +describe("stabilizeClaudeInstructionsForPromptCache", () => { + test("empty input is a no-op", () => { + expect(stabilizeClaudeInstructionsForPromptCache("")).toEqual({ + instructions: "", + dynamicNotice: null, + }); + }); + + test("stable instructions without dynamics pass through", () => { + const instructions = "You are Claude Code.\n\nPrefer terse answers."; + expect(stabilizeClaudeInstructionsForPromptCache(instructions)).toEqual({ + instructions, + dynamicNotice: null, + }); + }); + + test("three total_tokens footers keep only the latest in the notice", () => { + const stable = "You are Claude Code."; + const first = footer(1000); + const second = footer(4000); + const third = footer(8000); + const result = stabilizeClaudeInstructionsForPromptCache( + [stable, first, second, third].join("\n\n"), + ); + expect(result.instructions).toBe(stable); + expect(result.instructions).not.toContain(""); + expect(result.dynamicNotice).toBe(third); + }); + + test("instructions contain zero total_tokens after stripping", () => { + const result = stabilizeClaudeInstructionsForPromptCache( + `System.\n${footer(1)}\nMore system.\n${footer(3)}`, + ); + expect(result.instructions).not.toMatch(//); + expect(result.instructions).toContain("System."); + expect(result.instructions).toContain("More system."); + expect(result.dynamicNotice).toBe(footer(3)); + }); + + test("TaskCreate nudge is stripped from instructions and kept in the notice", () => { + const stable = "You are Claude Code."; + const result = stabilizeClaudeInstructionsForPromptCache( + `${stable}\n\n${TASKCREATE_NUDGE}`, + ); + expect(result.instructions).toBe(stable); + expect(result.instructions).not.toContain("TaskCreate"); + expect(result.dynamicNotice).toBe(TASKCREATE_NUDGE); + }); + + test("latest footer and latest nudge both surface in the notice", () => { + const stable = "Stay stable."; + const older = footer(10); + const latest = footer(50); + const result = stabilizeClaudeInstructionsForPromptCache( + [stable, older, TASKCREATE_NUDGE, latest].join("\n\n"), + ); + expect(result.instructions).toBe(stable); + expect(result.dynamicNotice).toBe(`${latest}\n\n${TASKCREATE_NUDGE}`); + }); + + test("inline documentation of total_tokens tags stays in instructions", () => { + const docs = "The harness may emit a 123 footer; do not invent one."; + const result = stabilizeClaudeInstructionsForPromptCache(docs); + expect(result.instructions).toBe(docs); + expect(result.dynamicNotice).toBeNull(); + }); + + test("TaskCreate mentioned in docs is not treated as the harness nudge", () => { + const docs = "The task tools haven't been used recently. You may mention TaskCreate in docs without the reminder."; + const result = stabilizeClaudeInstructionsForPromptCache(docs); + expect(result.instructions).toBe(docs); + expect(result.dynamicNotice).toBeNull(); + }); + + test("docs plus a real footer keep the docs and move only the latest footer", () => { + const docs = "Describe 0 in the protocol guide."; + const latest = footer(8000); + const result = stabilizeClaudeInstructionsForPromptCache( + [docs, footer(1), latest].join("\n\n"), + ); + expect(result.instructions).toBe(docs); + expect(result.dynamicNotice).toBe(latest); + }); +}); + +describe("anthropicToResponsesTranslation cache-stabilize wire-in", () => { + test("moves the latest total_tokens footer onto a trailing input user message", () => { + const first = footer(1000); + const latest = footer(8000); + const { body } = anthropicToResponsesTranslation({ + model: "m", + max_tokens: 1, + system: ["You are Claude Code.", first, latest].join("\n\n"), + messages: [{ role: "user", content: "hi" }], + }); + expect(body.instructions).toBe("You are Claude Code."); + expect(String(body.instructions)).not.toContain(""); + const input = body.input as Array>; + const last = input[input.length - 1]!; + expect(last).toEqual({ + type: "message", + role: "user", + content: [{ type: "input_text", text: latest }], + }); + expect(input.some(item => item.role === "user" && item !== last)).toBe(true); + }); + + test("Desktop prompt_cache_key fallback hashes stabilized instructions, not total_tokens footers", () => { + const stable = "You are Claude Code."; + const keyOf = (system: string) => + anthropicToResponsesTranslation({ + model: "m", + max_tokens: 1, + system, + messages: [{ role: "user", content: "hi" }], + }).body.prompt_cache_key as string; + const stableKey = keyOf(stable); + expect(stableKey).toMatch(/^[0-9a-f]{32}$/); + expect(keyOf([stable, footer(1000), footer(8000)].join("\n\n"))).toBe(stableKey); + expect(keyOf([stable, footer(99999)].join("\n\n"))).toBe(stableKey); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 98907bd26a..a213efebf0 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -139,6 +139,7 @@ "claude-desktop-remote-hub.test.ts": "claude-integration", "claude-dotenv-provenance-transport.test.ts": "claude-integration", "claude-gateway-cache.test.ts": "claude-integration", + "claude-inbound-cache-stabilize.test.ts": "claude-integration", "claude-inbound-debug.test.ts": "claude-integration", "claude-inbound.test.ts": "claude-integration", "claude-management-api.test.ts": "claude-integration",