From ee14836add7b393b84268fe61c00fad0a57b8a11 Mon Sep 17 00:00:00 2001 From: Jonathan Yang <14588641+Joilence@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:02:02 +0200 Subject: [PATCH 1/2] fix(narration): match atWord anchors outside ASCII MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `normalizeWord` strips a token with `[^\w']`, and JavaScript's `\w` is `[A-Za-z0-9_]` without the `u` flag. Every Cyrillic, Han, Devanagari, Greek, Hebrew and Arabic token therefore normalises to the empty string and they all compare equal, so `atWord` returns whichever word comes first in the scene for any anchor. It returns a plausible number rather than null, so nothing reports it. Accented Latin is mangled rather than erased: "Anträge" only matches a caller who writes "antrge". `\p{L}\p{M}\p{N}` with the `u` flag. `\p{M}` matters as much as the rest: combining marks carry the sound in Devanagari, Thai and Arabic, so without them "काल" still collapses onto "कल" and the anchor still lands on the wrong word. Keeping marks means a decomposed spelling stops equalling a precomposed one, so normalise to NFC first, otherwise an NFD "Anträge" never matches Whisper's NFC "Anträge" and the effect silently never fires. Also refuse an anchor that strips to nothing rather than matching the first token that also strips to nothing, which returned a confident 1000ms for punctuation. 10 of the 12 new cases fail against the previous expression. The one caller in demos/showcase.demo.ts reads `atWord(...) ?? fb`, so a stricter miss falls back to its fixed timeout. --- src/narration.ts | 28 +++++++- tests/narration-atword.test.ts | 124 +++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 tests/narration-atword.test.ts diff --git a/src/narration.ts b/src/narration.ts index c4eeec4..0301f51 100644 --- a/src/narration.ts +++ b/src/narration.ts @@ -647,6 +647,10 @@ export class NarrationTimeline { if (!words?.length) return null; const normalized = normalizeWord(target); + // An anchor of pure punctuation strips to nothing, and so does any token + // of pure punctuation, so without this the first such token in the scene + // matches and the caller gets a confident wrong time. + if (!normalized) return null; for (const w of words) { if (normalizeWord(w.text) !== normalized) continue; const elapsedInSceneMs = (Date.now() - this.startTime) - markMs; @@ -657,7 +661,29 @@ export class NarrationTimeline { } } -const normalizeWord = (s: string): string => s.toLowerCase().replace(/[^\w']/g, ''); +/** + * Strip a transcribed token down to what two spellings of the same word share: + * case and surrounding punctuation, which Whisper attaches inconsistently. + * + * `\p{L}\p{N}` with the `u` flag rather than `\w`, which is ASCII-only: under + * `\w` every Cyrillic, Han, Devanagari, Greek, Hebrew and Arabic token stripped + * to the empty string, so they all compared equal and any anchor matched + * whichever word came first in the scene. + * + * `\p{M}` belongs with them. Combining marks are not decoration in every + * script: Devanagari matras and the virama, Thai vowel and tone marks and + * Arabic harakat carry the sound, so dropping them collapses distinct words + * onto each other and "काल" matches "कल". + * + * NFC first, because keeping marks means a decomposed spelling stops equalling + * a precomposed one, and the two are visually identical. + * + * Errors here do not announce themselves: a wrong match returns a plausible + * number, a missed match returns null, and `atWord` already returns null for + * several legitimate reasons. + */ +const normalizeWord = (s: string): string => + s.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{M}\p{N}']/gu, ''); export interface WordTiming { text: string; diff --git a/tests/narration-atword.test.ts b/tests/narration-atword.test.ts new file mode 100644 index 0000000..8b66904 --- /dev/null +++ b/tests/narration-atword.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { NarrationTimeline } from '../src/narration.js'; + +/** + * `atWord` matched on a token stripped by `[^\w']`, which is ASCII-only. Every + * Cyrillic, Han and Devanagari word stripped to the empty string, so they all + * compared equal: asking for any anchor returned the first word in the scene, + * at a time that looked reasonable and was wrong. Accented Latin was mangled + * rather than erased, so "Anträge" only matched a caller who also wrote it + * without the umlaut. + * + * The transcript is cached per process on first read, so every case here shares + * one file with a scene per language. + */ +const WORDS = (...pairs: Array<[string, number]>) => + pairs.map(([text, start]) => ({ text, start, end: start + 0.4 })); + +const TRANSCRIPT = { + version: 1, + model: 'test', + scenes: { + ru: WORDS(['Две', 1], ['заявки,', 2], ['одобрены.', 3]), + zh: WORDS(['两', 1], ['个', 2], ['申请', 3]), + hi: WORDS(['दो', 1], ['अनुरोध,', 2], ['स्वीकृत।', 3]), + de: WORDS(['Zwei', 1], ['Anträge,', 2], ['genehmigt.', 3]), + en: WORDS(['Two', 1], ['requests,', 2], ['approved.', 3]), + + // Pairs that differ only by a combining mark. These are the cases that + // survived the first fix: dropping \p{M} collapses each pair onto one + // spelling, so the anchor lands on whichever came first. + hiMarks: WORDS(['कल', 1], ['है', 2], ['काल', 3]), + thMarks: WORDS(['ไม้', 1], ['คือ', 2], ['ไม่', 3]), + + // A leading token with no letters or digits at all. + punct: WORDS(['...', 1], ['hello', 2], ['world', 3]), + }, +}; + +let dir: string; + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), 'argo-atword-')); + const path = join(dir, 'transcript.json'); + writeFileSync(path, JSON.stringify(TRANSCRIPT), 'utf-8'); + process.env.ARGO_TRANSCRIPT_PATH = path; +}); + +afterAll(() => { + delete process.env.ARGO_TRANSCRIPT_PATH; + rmSync(dir, { recursive: true, force: true }); +}); + +/** A timeline sitting at the very start of `scene`. */ +function atStartOf(scene: string): NarrationTimeline { + const timeline = new NarrationTimeline(); + timeline.start(); + timeline.mark(scene); + return timeline; +} + +describe('atWord across scripts', () => { + // Each case asks for the *third* word. Under the old normalisation every + // non-Latin token was equal, so the first word matched and the answer came + // back near 1000ms instead of near 3000ms. + it.each([ + ['ru', 'одобрены'], + ['zh', '申请'], + ['hi', 'स्वीकृत'], + ['de', 'genehmigt'], + ['en', 'approved'], + ])('finds the third word in %s and not the first', (scene, anchor) => { + const ms = atStartOf(scene).atWord(scene, anchor); + expect(ms).not.toBeNull(); + expect(ms).toBeGreaterThan(2500); + expect(ms).toBeLessThanOrEqual(3000); + }); + + it('keeps a diacritic significant instead of stripping it', () => { + // "Anträge" must be reachable as written. It also must not be reachable by + // an ASCII-folded spelling, which the old behaviour accidentally allowed. + expect(atStartOf('de').atWord('de', 'Anträge')).toBeGreaterThan(1500); + expect(atStartOf('de').atWord('de', 'antrge')).toBeNull(); + }); + + it('still ignores case and trailing punctuation', () => { + expect(atStartOf('en').atWord('en', 'REQUESTS')).toBeGreaterThan(1500); + expect(atStartOf('ru').atWord('ru', 'заявки')).toBeGreaterThan(1500); + }); + + it('returns null for a word the scene does not contain', () => { + expect(atStartOf('ru').atWord('ru', 'отклонены')).toBeNull(); + expect(atStartOf('zh').atWord('zh', '批准')).toBeNull(); + }); +}); + +describe('atWord and combining marks', () => { + it.each([ + ['hiMarks', 'काल', 'कल'], + ['thMarks', 'ไม่', 'ไม้'], + ])('%s: anchors on %s rather than the mark-stripped %s', (scene, anchor) => { + const ms = atStartOf(scene).atWord(scene, anchor); + expect(ms).toBeGreaterThan(2500); + expect(ms).toBeLessThanOrEqual(3000); + }); + + it('matches a decomposed anchor against a precomposed transcript', () => { + // Visually identical, and a script pulled through a filesystem path or a + // copy-paste can carry either. Without an NFC pass this returns null + // forever, which is indistinguishable from "the word is already spoken". + expect(atStartOf('de').atWord('de', 'Anträge'.normalize('NFD'))).toBeGreaterThan(1500); + }); + + it('refuses an anchor that strips to nothing', () => { + // Both the anchor and the scene's first token normalise to '', so without + // a guard they compare equal and the caller gets 1000ms for punctuation. + expect(atStartOf('punct').atWord('punct', '...')).toBeNull(); + expect(atStartOf('punct').atWord('punct', '?!')).toBeNull(); + // The real words in that scene stay reachable. + expect(atStartOf('punct').atWord('punct', 'world')).toBeGreaterThan(2500); + }); +}); From 6aa41160feec158c11159a85ff517f71557b7895 Mon Sep 17 00:00:00 2001 From: Jonathan Yang <14588641+Joilence@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:03:29 +0200 Subject: [PATCH 2/2] docs(narration): tighten and correct comments --- src/narration.ts | 28 ++++++---------------------- tests/narration-atword.test.ts | 33 +++++++-------------------------- 2 files changed, 13 insertions(+), 48 deletions(-) diff --git a/src/narration.ts b/src/narration.ts index 0301f51..45c99f5 100644 --- a/src/narration.ts +++ b/src/narration.ts @@ -647,9 +647,8 @@ export class NarrationTimeline { if (!words?.length) return null; const normalized = normalizeWord(target); - // An anchor of pure punctuation strips to nothing, and so does any token - // of pure punctuation, so without this the first such token in the scene - // matches and the caller gets a confident wrong time. + // Pure punctuation strips to nothing on both sides, so without this the + // first such token in the scene matches and the caller gets a wrong time. if (!normalized) return null; for (const w of words) { if (normalizeWord(w.text) !== normalized) continue; @@ -662,25 +661,10 @@ export class NarrationTimeline { } /** - * Strip a transcribed token down to what two spellings of the same word share: - * case and surrounding punctuation, which Whisper attaches inconsistently. - * - * `\p{L}\p{N}` with the `u` flag rather than `\w`, which is ASCII-only: under - * `\w` every Cyrillic, Han, Devanagari, Greek, Hebrew and Arabic token stripped - * to the empty string, so they all compared equal and any anchor matched - * whichever word came first in the scene. - * - * `\p{M}` belongs with them. Combining marks are not decoration in every - * script: Devanagari matras and the virama, Thai vowel and tone marks and - * Arabic harakat carry the sound, so dropping them collapses distinct words - * onto each other and "काल" matches "कल". - * - * NFC first, because keeping marks means a decomposed spelling stops equalling - * a precomposed one, and the two are visually identical. - * - * Errors here do not announce themselves: a wrong match returns a plausible - * number, a missed match returns null, and `atWord` already returns null for - * several legitimate reasons. + * `\w` is ASCII-only and strips a non-Latin token to '', so every such + * token compares equal. `\p{M}` is needed too: combining marks carry the + * sound in Devanagari, Thai and Arabic. NFC first, so decomposed equals + * precomposed. */ const normalizeWord = (s: string): string => s.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{M}\p{N}']/gu, ''); diff --git a/tests/narration-atword.test.ts b/tests/narration-atword.test.ts index 8b66904..573b023 100644 --- a/tests/narration-atword.test.ts +++ b/tests/narration-atword.test.ts @@ -4,20 +4,13 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { NarrationTimeline } from '../src/narration.js'; +const WORDS = (...pairs: Array<[string, number]>) => + pairs.map(([text, start]) => ({ text, start, end: start + 0.4 })); + /** - * `atWord` matched on a token stripped by `[^\w']`, which is ASCII-only. Every - * Cyrillic, Han and Devanagari word stripped to the empty string, so they all - * compared equal: asking for any anchor returned the first word in the scene, - * at a time that looked reasonable and was wrong. Accented Latin was mangled - * rather than erased, so "Anträge" only matched a caller who also wrote it - * without the umlaut. - * * The transcript is cached per process on first read, so every case here shares * one file with a scene per language. */ -const WORDS = (...pairs: Array<[string, number]>) => - pairs.map(([text, start]) => ({ text, start, end: start + 0.4 })); - const TRANSCRIPT = { version: 1, model: 'test', @@ -28,13 +21,11 @@ const TRANSCRIPT = { de: WORDS(['Zwei', 1], ['Anträge,', 2], ['genehmigt.', 3]), en: WORDS(['Two', 1], ['requests,', 2], ['approved.', 3]), - // Pairs that differ only by a combining mark. These are the cases that - // survived the first fix: dropping \p{M} collapses each pair onto one - // spelling, so the anchor lands on whichever came first. + // Pairs differing only by a combining mark: without \p{M} each collapses + // onto one spelling and the anchor lands on whichever came first. hiMarks: WORDS(['कल', 1], ['है', 2], ['काल', 3]), thMarks: WORDS(['ไม้', 1], ['คือ', 2], ['ไม่', 3]), - // A leading token with no letters or digits at all. punct: WORDS(['...', 1], ['hello', 2], ['world', 3]), }, }; @@ -62,9 +53,6 @@ function atStartOf(scene: string): NarrationTimeline { } describe('atWord across scripts', () => { - // Each case asks for the *third* word. Under the old normalisation every - // non-Latin token was equal, so the first word matched and the answer came - // back near 1000ms instead of near 3000ms. it.each([ ['ru', 'одобрены'], ['zh', '申请'], @@ -73,14 +61,11 @@ describe('atWord across scripts', () => { ['en', 'approved'], ])('finds the third word in %s and not the first', (scene, anchor) => { const ms = atStartOf(scene).atWord(scene, anchor); - expect(ms).not.toBeNull(); expect(ms).toBeGreaterThan(2500); expect(ms).toBeLessThanOrEqual(3000); }); it('keeps a diacritic significant instead of stripping it', () => { - // "Anträge" must be reachable as written. It also must not be reachable by - // an ASCII-folded spelling, which the old behaviour accidentally allowed. expect(atStartOf('de').atWord('de', 'Anträge')).toBeGreaterThan(1500); expect(atStartOf('de').atWord('de', 'antrge')).toBeNull(); }); @@ -107,18 +92,14 @@ describe('atWord and combining marks', () => { }); it('matches a decomposed anchor against a precomposed transcript', () => { - // Visually identical, and a script pulled through a filesystem path or a - // copy-paste can carry either. Without an NFC pass this returns null - // forever, which is indistinguishable from "the word is already spoken". + // A null here is indistinguishable from "the word is already spoken". expect(atStartOf('de').atWord('de', 'Anträge'.normalize('NFD'))).toBeGreaterThan(1500); }); it('refuses an anchor that strips to nothing', () => { - // Both the anchor and the scene's first token normalise to '', so without - // a guard they compare equal and the caller gets 1000ms for punctuation. + // Anchor and first token both normalise to '', so unguarded they match. expect(atStartOf('punct').atWord('punct', '...')).toBeNull(); expect(atStartOf('punct').atWord('punct', '?!')).toBeNull(); - // The real words in that scene stay reachable. expect(atStartOf('punct').atWord('punct', 'world')).toBeGreaterThan(2500); }); });