diff --git a/src/commands/translate.js b/src/commands/translate.js index 28bac1a..c94d90f 100644 --- a/src/commands/translate.js +++ b/src/commands/translate.js @@ -27,6 +27,7 @@ import { loadTranslationProvider } from '../lib/provider.js' import { loadCache } from '../lib/cache.js' import { shutdown } from '../shutdown.js' import { loadReferenceFile } from '../lib/reference-loader.js' +import { validateTranslation } from '../lib/translation-validation.js' // Anything that looks like %%var%%, {{var}}, %s/%d/%i/%f/%F/%o/%x, or {0}. // Used to strip out interpolation tokens before deciding whether a source @@ -301,19 +302,6 @@ export async function runTranslation({ appState, options, log }) { const referenceValueHash = calculateHash(`${refValue}${refContextValue?.length ? `_${refContextValue}` : ''}`) // If either of the ref value or the context value change, we'll update const curValue = (key in outputData) ? outputData[key] : null - // If a prior run marked this (lang, key) untranslatable AGAINST the - // current source hash, skip — re-asking the same provider with the - // same input gets the same English-back result. We override on - // --force, or if the user has hand-added a target value (curValue - // non-empty and different from the source), so a manual fix can - // retake the slot. - const untranslatableHash = readOnlyCache?.untranslatable?.[targetLang]?.[key] - const userProvidedTranslation = typeof curValue === 'string' && curValue.length > 0 && curValue !== refValue - if (untranslatableHash && untranslatableHash === referenceValueHash && !options.force && !userProvidedTranslation) { - log.D(`Skip ${targetLang}/${key}: cached as untranslatable for current source hash`) - continue - } - // Skip non-string values (objects, arrays, etc.) const refValueType = typeof refValue if (refValueType !== 'string') { @@ -340,6 +328,23 @@ export async function runTranslation({ appState, options, log }) { continue } + const existingValidation = curValue === null || targetLang === sourceLang + ? { valid: true } + : validateTranslation({ source: refValue, translated: curValue }) + + // If a prior run marked this (lang, key) untranslatable AGAINST the + // current source hash, skip — re-asking the same provider with the + // same input gets the same English-back result. We override on + // --force, or if the user has hand-added a target value (curValue + // non-empty and different from the source), so a manual fix can + // retake the slot. + const untranslatableHash = readOnlyCache?.untranslatable?.[targetLang]?.[key] + const userProvidedTranslation = typeof curValue === 'string' && curValue.length > 0 && curValue !== refValue + if (untranslatableHash && untranslatableHash === referenceValueHash && !options.force && !userProvidedTranslation && existingValidation.valid) { + log.D(`Skip ${targetLang}/${key}: cached as untranslatable for current source hash`) + continue + } + const currentValueHash = curValue?.length ? calculateHash(curValue) : null // Check if translation needs update @@ -364,6 +369,7 @@ export async function runTranslation({ appState, options, log }) { // Map reason key => true/false const possibleReasonsForTranslationMap = { forced: options.force, + invalidExistingTranslation: !existingValidation.valid, outputFileDidNotExist, userMissingReferenceValueHash, userModifiedReferenceValue, @@ -706,8 +712,13 @@ async function translateKeyForLanguage({ await sleep(backoffInterval) } } else { - newValue = translateResult.translated - result.success = true + const validation = validateTranslation({ source: refValue, translated: translateResult.translated }) + if (validation.valid) { + newValue = translateResult.translated + result.success = true + } else { + log.W(`[${targetLang}] rejected translation for "${key}": ${validation.reason}`) + } } } @@ -923,6 +934,11 @@ async function processLanguageBatched({ appState, taskList, batchSize, options, for (const [ key, newValue ] of Object.entries(batchResult.translations)) { const t = tasksByKey.get(key) if (!t) continue + const validation = validateTranslation({ source: t.state.refValue, translated: newValue }) + if (!validation.valid) { + log.W(`[${targetLang}] rejected translation for "${key}": ${validation.reason}`) + continue + } // Provider returned source verbatim and source has real translatable // content → mark untranslatable instead of writing English back into // the target file. See processTranslationTask for the same rule on @@ -1044,4 +1060,3 @@ async function runEndOfRunAudit({ appState, workQueue, writableCache, referenceD log.D(`[audit] all expected keys present in all output files`) } } - diff --git a/src/lib/translation-validation.js b/src/lib/translation-validation.js new file mode 100644 index 0000000..2596a8f --- /dev/null +++ b/src/lib/translation-validation.js @@ -0,0 +1,44 @@ +const PLACEHOLDER_RE = /(%%[^%]+%%|\{\{[^}]+\}\}|%[sdifFox]|\{\d+\})/g + +// Deliberately narrow, high-confidence signs that a model returned commentary +// instead of a translation. This is not intended to judge translation quality. +const MODEL_COMMENTARY_PATTERNS = [ + /\b(?:I (?:apologize|cannot|can't|do not|don't|am unable|recommend|need to correct|should clarify)|Unable to translate)\b/i, + /\b(?:Here is the correct translation|Please (?:provide|clarify) the target language|professional translation service|consult(?:ing)? (?:with )?a native speaker)\b/i, + /(?:^|\n)\s*(?:Note|Translation):\s/i, + /(?:^|\n)\s*AI:\s*(?:Human|I)\b/i, +] + +const MAX_TRANSLATION_LENGTH_MULTIPLIER = 8 +const MAX_TRANSLATION_LENGTH_FLOOR = 500 + +export function extractPlaceholders(value) { + if (typeof value !== 'string') return [] + return (value.match(PLACEHOLDER_RE) || []).sort() +} + +export function validateTranslation({ source, translated }) { + if (typeof translated !== 'string' || !translated.trim()) { + return { valid: false, reason: 'translation is empty or not a string' } + } + + const sourcePlaceholders = extractPlaceholders(source) + const translatedPlaceholders = extractPlaceholders(translated) + if (sourcePlaceholders.join('\0') !== translatedPlaceholders.join('\0')) { + return { + valid: false, + reason: `placeholder mismatch (expected ${JSON.stringify(sourcePlaceholders)}, got ${JSON.stringify(translatedPlaceholders)})`, + } + } + + if (MODEL_COMMENTARY_PATTERNS.some(pattern => pattern.test(translated))) { + return { valid: false, reason: 'translation contains model commentary or a refusal' } + } + + const maxLength = Math.max(String(source ?? '').length * MAX_TRANSLATION_LENGTH_MULTIPLIER, MAX_TRANSLATION_LENGTH_FLOOR) + if (translated.length > maxLength) { + return { valid: false, reason: `translation is implausibly long (${translated.length} characters; maximum ${maxLength})` } + } + + return { valid: true, reason: null } +} diff --git a/test/translation-validation.test.js b/test/translation-validation.test.js new file mode 100644 index 0000000..409a792 --- /dev/null +++ b/test/translation-validation.test.js @@ -0,0 +1,46 @@ +import { expect } from 'chai' +import { extractPlaceholders, validateTranslation } from '../src/lib/translation-validation.js' + +describe('translation validation', () => { + it('extracts and sorts supported placeholders', () => { + expect(extractPlaceholders('Hi %%name%%, {{count}} {0} %s')).to.deep.equal([ + '%%name%%', + '%s', + '{{count}}', + '{0}', + ].sort()) + }) + + it('accepts a normal translation with preserved placeholders', () => { + const result = validateTranslation({ + source: 'Failed to load %%count%% tracks', + translated: 'Impossible de charger %%count%% pistes', + }) + expect(result.valid).to.equal(true) + }) + + it('rejects missing, duplicated, and translated placeholders', () => { + for (const translated of [ + 'Impossible de charger les pistes', + 'Impossible %%count%% %%count%%', + 'Impossible de charger %%nombre%% pistes', + ]) { + expect(validateTranslation({ source: 'Failed %%count%%', translated }).valid).to.equal(false) + } + }) + + it('rejects model refusals and commentary', () => { + for (const translated of [ + 'I cannot provide an accurate translation without a native speaker.', + 'Texte traduit\n\nNote: this term is normally kept in English.', + 'AI: Human, here is your translation.', + ]) { + expect(validateTranslation({ source: 'Account', translated }).valid).to.equal(false) + } + }) + + it('rejects implausibly long output', () => { + const translated = 'x'.repeat(501) + expect(validateTranslation({ source: 'Copy', translated }).valid).to.equal(false) + }) +})