[fix/translation-validation] rejects corrupted model output

src/commands/translate.js

@@ -27,6 +27,7 @@ import { loadTranslationProvider } from '../lib/provider.js'
 import { loadCache } from '../lib/cache.js'
 import { shutdown } from '../shutdown.js'
 import { loadReferenceFile } from '../lib/reference-loader.js'
+import { validateTranslation } from '../lib/translation-validation.js'
 
 // Anything that looks like %%var%%, {{var}}, %s/%d/%i/%f/%F/%o/%x, or {0}.
 // Used to strip out interpolation tokens before deciding whether a source
@@ -301,19 +302,6 @@ export async function runTranslation({ appState, options, log }) {
 				const referenceValueHash = calculateHash(`${refValue}${refContextValue?.length ? `_${refContextValue}` : ''}`)	// If either of the ref value or the context value change, we'll update
 				const curValue = (key in outputData) ? outputData[key] : null
 
-				// If a prior run marked this (lang, key) untranslatable AGAINST the
-				// current source hash, skip — re-asking the same provider with the
-				// same input gets the same English-back result. We override on
-				// --force, or if the user has hand-added a target value (curValue
-				// non-empty and different from the source), so a manual fix can
-				// retake the slot.
-				const untranslatableHash = readOnlyCache?.untranslatable?.[targetLang]?.[key]
-				const userProvidedTranslation = typeof curValue === 'string' && curValue.length > 0 && curValue !== refValue
-				if (untranslatableHash && untranslatableHash === referenceValueHash && !options.force && !userProvidedTranslation) {
-					log.D(`Skip ${targetLang}/${key}: cached as untranslatable for current source hash`)
-					continue
-				}
-
 				// Skip non-string values (objects, arrays, etc.)
 				const refValueType = typeof refValue
 				if (refValueType !== 'string') {
@@ -340,6 +328,23 @@ export async function runTranslation({ appState, options, log }) {
 					continue
 				}
 
+				const existingValidation = curValue === null || targetLang === sourceLang
+					? { valid: true }
+					: validateTranslation({ source: refValue, translated: curValue })
+
+				// If a prior run marked this (lang, key) untranslatable AGAINST the
+				// current source hash, skip — re-asking the same provider with the
+				// same input gets the same English-back result. We override on
+				// --force, or if the user has hand-added a target value (curValue
+				// non-empty and different from the source), so a manual fix can
+				// retake the slot.
+				const untranslatableHash = readOnlyCache?.untranslatable?.[targetLang]?.[key]
+				const userProvidedTranslation = typeof curValue === 'string' && curValue.length > 0 && curValue !== refValue
+				if (untranslatableHash && untranslatableHash === referenceValueHash && !options.force && !userProvidedTranslation && existingValidation.valid) {
+					log.D(`Skip ${targetLang}/${key}: cached as untranslatable for current source hash`)
+					continue
+				}
+
 				const currentValueHash = curValue?.length ? calculateHash(curValue) : null
 
 				// Check if translation needs update
@@ -364,6 +369,7 @@ export async function runTranslation({ appState, options, log }) {
 				// Map reason key => true/false
 				const possibleReasonsForTranslationMap = {
 					forced: options.force,
+					invalidExistingTranslation: !existingValidation.valid,
 					outputFileDidNotExist,
 					userMissingReferenceValueHash,
 					userModifiedReferenceValue,
@@ -706,8 +712,13 @@ async function translateKeyForLanguage({
 				await sleep(backoffInterval)
 			}
 		} else {
-			newValue = translateResult.translated
-			result.success = true
+			const validation = validateTranslation({ source: refValue, translated: translateResult.translated })
+			if (validation.valid) {
+				newValue = translateResult.translated
+				result.success = true
+			} else {
+				log.W(`[${targetLang}] rejected translation for "${key}": ${validation.reason}`)
+			}
 		}
 	}
 
@@ -923,6 +934,11 @@ async function processLanguageBatched({ appState, taskList, batchSize, options,
 			for (const [ key, newValue ] of Object.entries(batchResult.translations)) {
 				const t = tasksByKey.get(key)
 				if (!t) continue
+				const validation = validateTranslation({ source: t.state.refValue, translated: newValue })
+				if (!validation.valid) {
+					log.W(`[${targetLang}] rejected translation for "${key}": ${validation.reason}`)
+					continue
+				}
 				// Provider returned source verbatim and source has real translatable
 				// content → mark untranslatable instead of writing English back into
 				// the target file. See processTranslationTask for the same rule on
@@ -1044,4 +1060,3 @@ async function runEndOfRunAudit({ appState, workQueue, writableCache, referenceD
 		log.D(`[audit] all expected keys present in all output files`)
 	}
 }
-

src/lib/translation-validation.js

@@ -0,0 +1,44 @@
+const PLACEHOLDER_RE = /(%%[^%]+%%|\{\{[^}]+\}\}|%[sdifFox]|\{\d+\})/g
+
+// Deliberately narrow, high-confidence signs that a model returned commentary
+// instead of a translation. This is not intended to judge translation quality.
+const MODEL_COMMENTARY_PATTERNS = [
+	/\b(?:I (?:apologize|cannot|can't|do not|don't|am unable|recommend|need to correct|should clarify)|Unable to translate)\b/i,
+	/\b(?:Here is the correct translation|Please (?:provide|clarify) the target language|professional translation service|consult(?:ing)? (?:with )?a native speaker)\b/i,
+	/(?:^|\n)\s*(?:Note|Translation):\s/i,
+	/(?:^|\n)\s*AI:\s*(?:Human|I)\b/i,
+]
+
+const MAX_TRANSLATION_LENGTH_MULTIPLIER = 8
+const MAX_TRANSLATION_LENGTH_FLOOR = 500
+
+export function extractPlaceholders(value) {
+	if (typeof value !== 'string') return []
+	return (value.match(PLACEHOLDER_RE) || []).sort()
+}
+
+export function validateTranslation({ source, translated }) {
+	if (typeof translated !== 'string' || !translated.trim()) {
+		return { valid: false, reason: 'translation is empty or not a string' }
+	}
+
+	const sourcePlaceholders = extractPlaceholders(source)
+	const translatedPlaceholders = extractPlaceholders(translated)
+	if (sourcePlaceholders.join('\0') !== translatedPlaceholders.join('\0')) {
+		return {
+			valid: false,
+			reason: `placeholder mismatch (expected ${JSON.stringify(sourcePlaceholders)}, got ${JSON.stringify(translatedPlaceholders)})`,
+		}
+	}
+
+	if (MODEL_COMMENTARY_PATTERNS.some(pattern => pattern.test(translated))) {
+		return { valid: false, reason: 'translation contains model commentary or a refusal' }
+	}
+
+	const maxLength = Math.max(String(source ?? '').length * MAX_TRANSLATION_LENGTH_MULTIPLIER, MAX_TRANSLATION_LENGTH_FLOOR)
+	if (translated.length > maxLength) {
+		return { valid: false, reason: `translation is implausibly long (${translated.length} characters; maximum ${maxLength})` }
+	}
+
+	return { valid: true, reason: null }
+}

test/translation-validation.test.js

@@ -0,0 +1,46 @@
+import { expect } from 'chai'
+import { extractPlaceholders, validateTranslation } from '../src/lib/translation-validation.js'
+
+describe('translation validation', () => {
+	it('extracts and sorts supported placeholders', () => {
+		expect(extractPlaceholders('Hi %%name%%, {{count}} {0} %s')).to.deep.equal([
+			'%%name%%',
+			'%s',
+			'{{count}}',
+			'{0}',
+		].sort())
+	})
+
+	it('accepts a normal translation with preserved placeholders', () => {
+		const result = validateTranslation({
+			source: 'Failed to load %%count%% tracks',
+			translated: 'Impossible de charger %%count%% pistes',
+		})
+		expect(result.valid).to.equal(true)
+	})
+
+	it('rejects missing, duplicated, and translated placeholders', () => {
+		for (const translated of [
+			'Impossible de charger les pistes',
+			'Impossible %%count%% %%count%%',
+			'Impossible de charger %%nombre%% pistes',
+		]) {
+			expect(validateTranslation({ source: 'Failed %%count%%', translated }).valid).to.equal(false)
+		}
+	})
+
+	it('rejects model refusals and commentary', () => {
+		for (const translated of [
+			'I cannot provide an accurate translation without a native speaker.',
+			'Texte traduit\n\nNote: this term is normally kept in English.',
+			'AI: Human, here is your translation.',
+		]) {
+			expect(validateTranslation({ source: 'Account', translated }).valid).to.equal(false)
+		}
+	})
+
+	it('rejects implausibly long output', () => {
+		const translated = 'x'.repeat(501)
+		expect(validateTranslation({ source: 'Copy', translated }).valid).to.equal(false)
+	})
+})