From 38c9fb1401c134abbd4ab57553e5830182bf5944 Mon Sep 17 00:00:00 2001 From: Peyton-Spencer Date: Mon, 10 Aug 2026 05:00:58 -0400 Subject: [PATCH] fix: validate batch translations and heal keys missing from target locales Two failure modes let `translate` report success while writing empty locale files and committing a fully-populated lock (poisoned state): 1. translateBatch used generateObject with a Record schema, which compiles to additionalProperties-only JSON schema. Gemini (via OpenRouter) silently returns {} for such schemas and OpenAI strict mode rejects them outright. Now uses generateText with robust JSON extraction and strict validation: every requested key must come back as a non-empty string, with one corrective retry for missing keys before the batch hard-fails (which the existing failure path then reports and keeps out of the lock). 2. syncLocaleFiles only diffed source against the lock, so keys the lock considered translated but absent from a target locale file were never re-sent. Target files now self-heal: per-locale missing keys are included in that locale's batches regardless of the lock state. Co-Authored-By: Claude Fable 5 --- src/lock.ts | 61 ++++++-- src/translate.ts | 123 ++++++++++++++-- tests/translate-validation.test.ts | 228 +++++++++++++++++++++++++++++ 3 files changed, 383 insertions(+), 29 deletions(-) create mode 100644 tests/translate-validation.test.ts diff --git a/src/lock.ts b/src/lock.ts index acda072..da28e8b 100644 --- a/src/lock.ts +++ b/src/lock.ts @@ -160,7 +160,20 @@ export async function syncLocaleFiles( const changedCount = Object.keys(changedKeys).length; - if (changedCount === 0 && deletedKeys.length === 0) { + // Self-heal: keys the lock considers translated but that are absent from a + // target locale file (e.g. after an interrupted or previously-buggy run) + // still need translation for that locale, regardless of the lock diff. + const missingByLocale: Record = {}; + for (const targetLocale of targetLocales) { + const existing = readTargetFile(join(localesDir, `${targetLocale}.json`)); + const missing = Object.keys(sourceDict).filter( + (key) => !(key in changedKeys) && !(key in existing), + ); + if (missing.length > 0) missingByLocale[targetLocale] = missing; + } + const missingLocaleCount = Object.keys(missingByLocale).length; + + if (changedCount === 0 && deletedKeys.length === 0 && missingLocaleCount === 0) { log("No changes detected in locale files."); return { status: "no-changes", @@ -170,7 +183,7 @@ export async function syncLocaleFiles( }; } - if (changedCount === 0) { + if (changedCount === 0 && missingLocaleCount === 0) { // Deletions only — prune target files and the lock, no AI calls needed for (const targetLocale of targetLocales) { const targetFilePath = join(localesDir, `${targetLocale}.json`); @@ -185,17 +198,26 @@ export async function syncLocaleFiles( return { status: "synced", translatedKeys: [], deletedKeys, failures: [] }; } - log( - `Translating ${changedCount} key${changedCount > 1 ? "s" : ""} to ${targetLocales.length} locale${targetLocales.length > 1 ? "s" : ""}...`, - ); - - // Context hints for the changed keys, passed to the translator - const changedContexts: Record = {}; - for (const key of Object.keys(changedKeys)) { - const ctx = pendingEntries[key]?.context; - if (ctx) changedContexts[key] = ctx; + if (changedCount > 0) { + log( + `Translating ${changedCount} key${changedCount > 1 ? "s" : ""} to ${targetLocales.length} locale${targetLocales.length > 1 ? "s" : ""}...`, + ); + } + if (missingLocaleCount > 0) { + const healTotal = Object.values(missingByLocale).reduce( + (sum, keys) => sum + keys.length, + 0, + ); + log( + `Healing ${healTotal} key${healTotal > 1 ? "s" : ""} missing from ${missingLocaleCount} locale file${missingLocaleCount > 1 ? "s" : ""}...`, + ); } + // Context hints, passed to the translator. Changed keys carry their pending + // entry's context; healed keys fall back to the committed lock entry. + const contextFor = (key: string): string | undefined => + pendingEntries[key]?.context ?? lock.keys[key]?.context; + const failures: SyncFailure[] = []; const failedKeys = new Set(); @@ -205,15 +227,26 @@ export async function syncLocaleFiles( // Load existing translations to preserve unchanged keys const existing = readTargetFile(targetFilePath); - // Batch translate changed keys - const entries = Object.entries(changedKeys); + // Changed keys for every locale, plus this locale's healed keys + const localeEntries: Record = { ...changedKeys }; + for (const key of missingByLocale[targetLocale] ?? []) { + localeEntries[key] = sourceDict[key]!; + } + const localeContexts: Record = {}; + for (const key of Object.keys(localeEntries)) { + const ctx = contextFor(key); + if (ctx) localeContexts[key] = ctx; + } + + // Batch translate + const entries = Object.entries(localeEntries); for (let i = 0; i < entries.length; i += batchSize) { const batch = Object.fromEntries(entries.slice(i, i + batchSize)); try { const translated = await translate( batch, targetLocale, - changedContexts, + localeContexts, ); Object.assign(existing, translated); } catch (err) { diff --git a/src/translate.ts b/src/translate.ts index 0e4e1c8..583e304 100644 --- a/src/translate.ts +++ b/src/translate.ts @@ -11,6 +11,64 @@ async function loadGenerateObject() { return generateObject; } +async function loadGenerateText() { + const { generateText } = await import("ai"); + return generateText; +} + +/** + * Pull a JSON object out of a model text response. Tolerates markdown code + * fences and prose around the object; takes the outermost `{...}` span. + * Exported for tests. + */ +export function extractJsonObject(text: string): Record { + const start = text.indexOf("{"); + const end = text.lastIndexOf("}"); + if (start === -1 || end <= start) { + throw new Error("model response contained no JSON object"); + } + const parsed: unknown = JSON.parse(text.slice(start, end + 1)); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new Error("model response was not a JSON object"); + } + return parsed as Record; +} + +/** + * Normalize a parsed model response into a translations dictionary limited to + * the requested keys, and report which requested keys are missing (absent, + * non-string, or empty). Unwraps a `{ "translations": { ... } }` envelope if + * the model added one. Exported for tests. + */ +export function collectBatchTranslations( + parsed: Record, + requestedKeys: string[], +): { translations: Record; missing: string[] } { + let dict = parsed; + const inner = parsed["translations"]; + if ( + typeof inner === "object" && + inner !== null && + !Array.isArray(inner) && + // Only unwrap when the envelope key is not itself a requested key + !requestedKeys.includes("translations") + ) { + dict = inner as Record; + } + + const translations: Record = {}; + const missing: string[] = []; + for (const key of requestedKeys) { + const value = dict[key]; + if (typeof value === "string" && value.length > 0) { + translations[key] = value; + } else { + missing.push(key); + } + } + return { translations, missing }; +} + /** * Translate a batch of key-value pairs from one locale to another using AI. * Supports optional per-key context hints for disambiguation. @@ -53,22 +111,57 @@ export async function translateBatch( } } - const generateObject = await loadGenerateObject(); - const { object } = await generateObject({ - model, - schema: z.object({ - translations: z.record(z.string(), z.string()), - }), - system: systemPrompt || defaultSystem, - prompt: [ - `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`, - `Return a JSON object with the exact same keys and the translated values.`, - contextSection, - JSON.stringify(entries, null, 2), - ].join("\n"), - }); + // generateText + manual JSON parsing instead of generateObject: a + // Record compiles to a JSON schema made only of + // `additionalProperties`, which several providers' structured-output modes + // handle badly — Gemini (via OpenRouter) silently returns `{}` and OpenAI's + // strict mode rejects the schema outright. Free-form JSON with strict + // post-validation works across every provider. + const generateText = await loadGenerateText(); + const basePrompt = [ + `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`, + `Respond with ONLY a JSON object — no prose, no code fences — containing the exact same keys and the translated values.`, + contextSection, + JSON.stringify(entries, null, 2), + ].join("\n"); + + const attempt = async (prompt: string) => { + const { text } = await generateText({ + model, + system: systemPrompt || defaultSystem, + prompt, + }); + return collectBatchTranslations(extractJsonObject(text), keys); + }; + + let { translations, missing } = await attempt(basePrompt); + + if (missing.length > 0) { + // One corrective retry for just the missing keys, then hard-fail so the + // caller records the batch as failed instead of committing a poisoned + // lock over silently-untranslated keys. + const retryEntries: Record = {}; + for (const key of missing) retryEntries[key] = entries[key]!; + const retry = await attempt( + [ + `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`, + `Respond with ONLY a JSON object — no prose, no code fences — containing the exact same keys and the translated values.`, + contextSection, + JSON.stringify(retryEntries, null, 2), + ].join("\n"), + ); + translations = { ...translations, ...retry.translations }; + missing = retry.missing; + } + + if (missing.length > 0) { + const sample = missing.slice(0, 3).join('", "'); + throw new Error( + `model returned no translation for ${missing.length} of ${keys.length} keys (e.g. "${sample}")`, + ); + } - return object.translations; + return translations; } /** diff --git a/tests/translate-validation.test.ts b/tests/translate-validation.test.ts new file mode 100644 index 0000000..44f7f21 --- /dev/null +++ b/tests/translate-validation.test.ts @@ -0,0 +1,228 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + extractJsonObject, + collectBatchTranslations, +} from "../src/translate"; +import { syncLocaleFiles, type TranslateFn } from "../src/lock"; +import { hashContent } from "../src/hash"; +import type { LockFile } from "../src/types"; + +describe("extractJsonObject", () => { + test("parses a bare JSON object", () => { + expect(extractJsonObject('{"a": "b"}')).toEqual({ a: "b" }); + }); + + test("strips markdown fences and prose", () => { + const text = 'Here you go:\n```json\n{"a": "b"}\n```\nDone!'; + expect(extractJsonObject(text)).toEqual({ a: "b" }); + }); + + test("throws when no object is present", () => { + expect(() => extractJsonObject("sorry, I cannot help")).toThrow( + /no JSON object/, + ); + }); + + test("throws on a JSON array", () => { + // indexOf("{") only matches inside a nested object — a top-level array + // with object items parses to the first item, which is an object; a pure + // scalar array has no braces at all. + expect(() => extractJsonObject("[1, 2, 3]")).toThrow(); + }); +}); + +describe("collectBatchTranslations", () => { + test("collects requested keys directly", () => { + const { translations, missing } = collectBatchTranslations( + { Hello: "Hola", Bye: "Adiós" }, + ["Hello", "Bye"], + ); + expect(translations).toEqual({ Hello: "Hola", Bye: "Adiós" }); + expect(missing).toEqual([]); + }); + + test("unwraps a translations envelope", () => { + const { translations, missing } = collectBatchTranslations( + { translations: { Hello: "Hola" } }, + ["Hello"], + ); + expect(translations).toEqual({ Hello: "Hola" }); + expect(missing).toEqual([]); + }); + + test("reports absent, empty, and non-string values as missing", () => { + const { translations, missing } = collectBatchTranslations( + { Hello: "Hola", Empty: "", Num: 42 as unknown as string }, + ["Hello", "Empty", "Num", "Absent"], + ); + expect(translations).toEqual({ Hello: "Hola" }); + expect(missing).toEqual(["Empty", "Num", "Absent"]); + }); + + test("an empty object reports every key missing (Gemini regression)", () => { + const { translations, missing } = collectBatchTranslations({}, [ + "Hello", + "Bye", + ]); + expect(translations).toEqual({}); + expect(missing).toEqual(["Hello", "Bye"]); + }); + + test("drops extra keys the model invented", () => { + const { translations } = collectBatchTranslations( + { Hello: "Hola", Invented: "???" }, + ["Hello"], + ); + expect(translations).toEqual({ Hello: "Hola" }); + }); + + test("does not unwrap when 'translations' is itself a requested key", () => { + const { translations, missing } = collectBatchTranslations( + { translations: "traducciones" as unknown as string }, + ["translations"], + ); + expect(translations).toEqual({ translations: "traducciones" }); + expect(missing).toEqual([]); + }); +}); + +describe("syncLocaleFiles self-heal", () => { + let dir: string; + + const fakeTranslate: TranslateFn = async (batch, targetLocale) => { + const out: Record = {}; + for (const [key, value] of Object.entries(batch)) { + out[key] = `${targetLocale}:${value.toUpperCase()}`; + } + return out; + }; + + const fullLock = (dict: Record): LockFile => ({ + version: 1, + sourceLocale: "en", + keys: Object.fromEntries( + Object.entries(dict).map(([key, value]) => [ + key, + { hash: hashContent(value), source: value }, + ]), + ), + }); + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "st-heal-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + test("translates keys missing from a target even when the lock is current", async () => { + const source = { Hello: "Hello", Bye: "Bye" }; + writeFileSync(join(dir, "en.json"), JSON.stringify(source)); + // Lock says everything is translated (poisoned state) … + writeFileSync( + join(dir, ".solid-translate.lock"), + JSON.stringify(fullLock(source)), + ); + // … but the target file is empty. + writeFileSync(join(dir, "es.json"), "{}"); + + const result = await syncLocaleFiles({ + localesDir: dir, + sourceLocale: "en", + targetLocales: ["es"], + batchSize: 50, + translate: fakeTranslate, + }); + + expect(result.status).toBe("synced"); + expect(result.failures).toEqual([]); + const es = JSON.parse(readFileSync(join(dir, "es.json"), "utf-8")); + expect(es).toEqual({ Hello: "es:HELLO", Bye: "es:BYE" }); + }); + + test("heals only the locale that is missing keys", async () => { + const source = { Hello: "Hello" }; + writeFileSync(join(dir, "en.json"), JSON.stringify(source)); + writeFileSync( + join(dir, ".solid-translate.lock"), + JSON.stringify(fullLock(source)), + ); + writeFileSync(join(dir, "es.json"), "{}"); + writeFileSync( + join(dir, "fr.json"), + JSON.stringify({ Hello: "fr:preexisting" }), + ); + + const calls: string[] = []; + const trackingTranslate: TranslateFn = async (batch, locale) => { + calls.push(locale); + return fakeTranslate(batch, locale, undefined); + }; + + const result = await syncLocaleFiles({ + localesDir: dir, + sourceLocale: "en", + targetLocales: ["es", "fr"], + batchSize: 50, + translate: trackingTranslate, + }); + + expect(result.status).toBe("synced"); + expect(calls).toEqual(["es"]); + const fr = JSON.parse(readFileSync(join(dir, "fr.json"), "utf-8")); + expect(fr).toEqual({ Hello: "fr:preexisting" }); + }); + + test("still reports no-changes when targets are complete", async () => { + const source = { Hello: "Hello" }; + writeFileSync(join(dir, "en.json"), JSON.stringify(source)); + writeFileSync( + join(dir, ".solid-translate.lock"), + JSON.stringify(fullLock(source)), + ); + writeFileSync(join(dir, "es.json"), JSON.stringify({ Hello: "Hola" })); + + const result = await syncLocaleFiles({ + localesDir: dir, + sourceLocale: "en", + targetLocales: ["es"], + batchSize: 50, + translate: async () => { + throw new Error("translate should not have been called"); + }, + }); + + expect(result.status).toBe("no-changes"); + }); + + test("a failed heal keeps the missing state for the next run", async () => { + const source = { Hello: "Hello" }; + writeFileSync(join(dir, "en.json"), JSON.stringify(source)); + const lock = fullLock(source); + writeFileSync(join(dir, ".solid-translate.lock"), JSON.stringify(lock)); + writeFileSync(join(dir, "es.json"), "{}"); + + const result = await syncLocaleFiles({ + localesDir: dir, + sourceLocale: "en", + targetLocales: ["es"], + batchSize: 50, + translate: async () => { + throw new Error("provider down"); + }, + }); + + expect(result.failures.length).toBe(1); + const es = JSON.parse(readFileSync(join(dir, "es.json"), "utf-8")); + expect(es).toEqual({}); + // Lock entry is untouched, and the key is still detected as missing next run + const relock = JSON.parse( + readFileSync(join(dir, ".solid-translate.lock"), "utf-8"), + ); + expect(relock.keys.Hello).toEqual(lock.keys.Hello); + }); +});