diff --git a/CLAUDE.md b/CLAUDE.md index 8f66f34..2a80498 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -316,6 +316,7 @@ Custom `test` fixture extends Playwright's `test` with a `narration` fixture tha - ~~`zoomTo` transforms `documentElement`~~ — FIXED: post-export camera moves (`narration` option) use ffmpeg `zoompan` — overlays are already burned into the video and unaffected. Legacy browser-side `zoomTo` (without `narration`) still has this issue. - OpenAI engine requests raw PCM (`response_format: 'pcm'`) and converts to Float32 directly — do not use `convertToWav` (ffmpeg pipe introduces 0xFFFFFFFF data size artifacts). - `convertToWav` (ffmpeg pipe to stdout) writes WAV with `0xFFFFFFFF` data size — `parseWavHeader` falls back to actual buffer length. All engines using `convertToWav` are affected. +- Gemini TTS needs a TTS model; general models answer an AUDIO request with 400. It returns headerless PCM `ffmpeg -i pipe:0` cannot sniff, spelled `audio/L16;codec=pcm;rate=24000` by 2.5 models and `audio/l16; rate=24000; channels=1` by 3.1: pass `parseRawAudioMime(mimeType)` as `convertToWav`'s third argument. Little-endian despite RFC 2586, matching what Google sends. - Showcase demo video hosted via GitHub gist comment upload: https://gist.github.com/shreyaskarnik/6a0996942a96528a984010f36de76079 - `tsc` build may silently fail if `tsconfig.json` is missing — verify it exists before trusting `npm run build` output - `dissolve` transition is a shorter dip-to-black, not a true crossfade blend. A real crossfade would require ffmpeg `xfade` with re-encoded segment pairs — impractical for continuous recordings. diff --git a/src/tts/engine.ts b/src/tts/engine.ts index f7db023..9615879 100644 --- a/src/tts/engine.ts +++ b/src/tts/engine.ts @@ -266,6 +266,68 @@ export function buildAtempoChain(speed: number): string[] { return ['-filter:a', stages.map(s => `atempo=${fmt(s)}`).join(',')]; } +/** A headerless audio stream. ffmpeg cannot sniff one, so it has to be told. */ +export interface RawAudioFormat { + /** ffmpeg demuxer name, passed to `-f`. `s16le` for signed 16-bit + * little-endian. Not a codec: `-c:a` would reject it. */ + format: string; + sampleRate: number; + channels: number; +} + +/** One parameter's value, tolerating the spacing and quoting RFC 2045 allows. */ +function mimeParam(params: string[], name: string): string | undefined { + const pattern = new RegExp(`^${name}\\s*=\\s*(.*)$`, 'i'); + for (const param of params) { + const match = pattern.exec(param); + if (match) return match[1].trim().replace(/^"(.*)"$/s, '$1'); + } + return undefined; +} + +/** + * Read a raw-PCM media type into the arguments ffmpeg needs to open it. + * + * Raw PCM carries no header, so ffmpeg cannot open it without being told the + * format. Gemini's TTS models send `audio/L16;codec=pcm;rate=24000`. + * + * Little-endian contradicts RFC 2586 section 3, which defines L16 as network + * byte order, but it is what Google sends. A conforming provider needs `s16be`. + * + * Returns null for self-describing formats, which ffmpeg can probe itself. + * Throws for an L16 type carrying no readable rate, which nothing can recover. + */ +export function parseRawAudioMime(mimeType: string | undefined): RawAudioFormat | null { + if (!mimeType) return null; + const [type, ...params] = mimeType.split(';').map(part => part.trim()); + // L16 is the only raw encoding the engines here emit. `audio/L8` (RFC 3551) + // and `audio/L24` (RFC 3190) exist but nothing returns them, so they are + // left unhandled rather than guessed at. + if (type.toLowerCase() !== 'audio/l16') return null; + + const rawRate = mimeParam(params, 'rate'); + const rate = Number(rawRate); + if (!Number.isFinite(rate) || rate <= 0) { + // Refusing beats guessing. Declaring 24000 for a stream that is really + // 16000 does not fail: it returns a clip a third shorter at 1.5x pitch, + // exit code 0, and argo derives scene durations from clip length, so every + // wait in the recording shortens and nothing reports a problem. RFC 2586 + // lists `rate` as required, so a missing one is malformed input. + throw new Error( + `cannot read a sample rate from "${mimeType}". Raw PCM carries no header, ` + + 'so the rate has to come from the media type.', + ); + } + // Unlike `rate`, the channel default is the RFC's own: "channels ... defaults + // to 1" in the L16 registration. + const channels = Number(mimeParam(params, 'channels')); + return { + format: 's16le', + sampleRate: rate, + channels: Number.isFinite(channels) && channels > 0 ? channels : 1, + }; +} + /** * Convert arbitrary audio (MP3, OGG, PCM, etc.) to Argo's WAV format * (mono, Float32, 24kHz) using ffmpeg. @@ -273,10 +335,21 @@ export function buildAtempoChain(speed: number): string[] { * `speed` is applied here because engines that render server-side (ElevenLabs, * Gemini) have no native rate control — this conversion is the only place the * rate can change. Engines with their own speed parameter must not use it. + * + * `inputFormat` describes a headerless stream. Pass it whenever the source is + * raw PCM; omit it and ffmpeg probes the container itself. */ -export function convertToWav(audioBuffer: Buffer, speed = 1): Buffer { +export function convertToWav( + audioBuffer: Buffer, + speed = 1, + inputFormat?: RawAudioFormat | null, +): Buffer { const { execFileSync } = childProcess; + const inputArgs = inputFormat + ? ['-f', inputFormat.format, '-ar', String(inputFormat.sampleRate), '-ac', String(inputFormat.channels)] + : []; const result = execFileSync('ffmpeg', [ + ...inputArgs, '-i', 'pipe:0', ...buildAtempoChain(speed), '-f', 'wav', diff --git a/src/tts/engines/gemini.ts b/src/tts/engines/gemini.ts index f3e7b2f..acf61f4 100644 --- a/src/tts/engines/gemini.ts +++ b/src/tts/engines/gemini.ts @@ -12,7 +12,10 @@ export class GeminiEngine implements TTSEngine { constructor(options?: GeminiEngineOptions) { this.apiKey = options?.apiKey ?? ''; - this.model = options?.model ?? 'gemini-2.5-flash'; + // Must be a TTS model: the general ones answer an AUDIO request with 400, + // "This model only supports text output". The `native-audio` models are no + // use either, they speak only the Live API socket. + this.model = options?.model ?? 'gemini-3.1-flash-tts-preview'; } private resolveApiKey(): string { @@ -68,7 +71,8 @@ export class GeminiEngine implements TTSEngine { // Convert to Argo WAV format. Gemini has no speed parameter, so the rate // change rides along with the conversion. - const { convertToWav } = await import('../engine.js'); - return convertToWav(audioBuffer, options.speed ?? 1); + const { convertToWav, parseRawAudioMime } = await import('../engine.js'); + const inputFormat = parseRawAudioMime(audioPart.inlineData.mimeType); + return convertToWav(audioBuffer, options.speed ?? 1, inputFormat); } } diff --git a/tests/tts/gemini.test.ts b/tests/tts/gemini.test.ts new file mode 100644 index 0000000..ac8c99a --- /dev/null +++ b/tests/tts/gemini.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { GeminiEngine } from '../../src/tts/engines/gemini.js'; + +/** + * The bug this guards was "GeminiEngine.generate never produced audio", and + * nothing in the suite called that method: `config.test.ts` only asserts the + * constructor does not throw, so the whole suite stayed green while the engine + * was dead. Covering `parseRawAudioMime` and `convertToWav` separately does not + * help either, because the defect was the wiring between them. + * + * Pattern borrowed from `mlx-audio.test.ts`: mock the transport, let the + * downstream ffmpeg call be a stub, and assert on the argv it was handed. + */ +const generateContent = vi.fn(); + +vi.mock('@google/genai', () => ({ + GoogleGenAI: class { + models = { generateContent }; + }, +})); + +vi.mock('node:child_process', async importOriginal => ({ + ...(await importOriginal()), + execFileSync: vi.fn(() => Buffer.from('fake-wav')), +})); + +/** A response shaped like the one Gemini's TTS models actually return. */ +function audioResponse(mimeType: string) { + return { + candidates: [ + { content: { parts: [{ inlineData: { mimeType, data: Buffer.from('pcm').toString('base64') } }] } }, + ], + }; +} + +/** The ffmpeg argv from the conversion that followed. */ +function ffmpegArgs(): string[] { + return vi.mocked(execFileSync).mock.calls[0][1] as string[]; +} + +describe('GeminiEngine.generate', () => { + beforeEach(() => { + vi.mocked(execFileSync).mockClear(); + generateContent.mockReset(); + }); + + it('tells ffmpeg the format of the raw PCM it was sent', async () => { + generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm;rate=24000')); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await engine.generate('hello', { voice: 'Kore' }); + + const argv = ffmpegArgs(); + const i = argv.indexOf('-i'); + expect(argv.slice(0, i)).toEqual(['-f', 's16le', '-ar', '24000', '-ac', '1']); + }); + + it('takes the rate from the response rather than assuming one', async () => { + // A wrong rate is silently wrong, so the engine has to read what it was sent. + generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm;rate=16000')); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await engine.generate('hello', { voice: 'Kore' }); + + expect(ffmpegArgs()[ffmpegArgs().indexOf('-ar') + 1]).toBe('16000'); + }); + + it('carries speed through to the conversion', async () => { + generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm;rate=24000')); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await engine.generate('hello', { voice: 'Kore', speed: 1.5 }); + + expect(ffmpegArgs()).toContain('atempo=1.5'); + }); + + it('refuses a response it cannot read the rate from', async () => { + generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm')); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await expect(engine.generate('hello', { voice: 'Kore' })).rejects.toThrow(/sample rate/); + }); + + it('defaults to a model that can actually return audio', async () => { + // The previous default was `gemini-2.5-flash`, which answers an AUDIO + // request with 400 "This model only supports text output". Nothing in the + // suite caught it because the transport is mocked here, so pin the model + // name: it is the only part of that failure visible without a live call. + generateContent.mockResolvedValue(audioResponse('audio/l16; rate=24000; channels=1')); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await engine.generate('hello', { voice: 'Kore' }); + + expect(generateContent.mock.calls[0][0].model).toBe('gemini-3.1-flash-tts-preview'); + expect(engine.describe().model).toBe('gemini-3.1-flash-tts-preview'); + }); + + it('reads the media type spelling the 3.1 models use', async () => { + // Same audio, different spelling: lowercase `l16`, spaces after the + // semicolons, no `codec`, and an explicit `channels`. Both spellings are + // captured from live responses, 2.5 and 3.1 respectively. + generateContent.mockResolvedValue(audioResponse('audio/l16; rate=24000; channels=1')); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await engine.generate('hello', { voice: 'Kore' }); + + const argv = ffmpegArgs(); + expect(argv.slice(0, argv.indexOf('-i'))).toEqual(['-f', 's16le', '-ar', '24000', '-ac', '1']); + }); + + it('raises a useful error when the response carries no audio', async () => { + generateContent.mockResolvedValue({ candidates: [{ content: { parts: [{ text: 'sorry' }] } }] }); + const engine = new GeminiEngine({ apiKey: 'test' }); + + await expect(engine.generate('hello', { voice: 'Kore' })).rejects.toThrow(/did not return audio/); + }); +}); diff --git a/tests/tts/raw-pcm-roundtrip.test.ts b/tests/tts/raw-pcm-roundtrip.test.ts new file mode 100644 index 0000000..60623eb --- /dev/null +++ b/tests/tts/raw-pcm-roundtrip.test.ts @@ -0,0 +1,99 @@ +/** + * The one thing `raw-pcm.test.ts` cannot check: whether the demuxer argv it + * asserts actually decodes the bytes Gemini sends. Every test there stubs + * `execFileSync`, so they compare the string 's16le' against the source that + * produced it and would pass just as happily if the answer were 's16be'. This + * file decodes a synthesized sine through real ffmpeg instead, where a byte + * swap lands near full scale rather than at the 0.25 peak it was handed. + */ +import { it, expect } from 'vitest'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { convertToWav, parseRawAudioMime, parseWavHeader } from '../../src/tts/engine.js'; +import { describeWithCapability } from '../helpers/capability.js'; + +const execFileP = promisify(execFile); + +// CI installs ffmpeg deliberately, so a miss there means the workflow drifted +// rather than that the host is bare. +let hasFfmpeg = false; +try { + await execFileP('ffmpeg', ['-version']); + hasFfmpeg = true; +} catch { + hasFfmpeg = false; +} + +/** A mono s16le sine, the shape Gemini's TTS models return. */ +function sineS16LE(samples: number, sampleRate: number, hz: number): Buffer { + const buf = Buffer.alloc(samples * 2); + for (let i = 0; i < samples; i++) { + // 0.25 full scale, chosen so a byte swap lands far outside it. + const v = Math.round(Math.sin((2 * Math.PI * hz * i) / sampleRate) * 0.25 * 32767); + buf.writeInt16LE(v, i * 2); + } + return buf; +} + +/** Peak absolute amplitude of a mono float32 WAV, as a fraction of full scale. */ +function peakAmplitude(wav: Buffer): number { + const { dataOffset, dataSize } = parseWavHeader(wav); + // convertToWav writes 0xFFFFFFFF as the data size (a known ffmpeg-pipe + // quirk recorded in CLAUDE.md), so trust the buffer rather than the header. + const end = Math.min(wav.length, dataOffset + dataSize); + let peak = 0; + for (let i = dataOffset; i + 4 <= end; i += 4) { + peak = Math.max(peak, Math.abs(wav.readFloatLE(i))); + } + return peak; +} + +describeWithCapability(hasFfmpeg, 'an ffmpeg binary')('raw PCM survives a real ffmpeg conversion', () => { + const RATE = 16000; + const SAMPLES = RATE / 2; // 0.5s, deliberately not the 24kHz output rate + + it('decodes a Gemini-shaped L16 response to the amplitude it was given', () => { + const pcm = sineS16LE(SAMPLES, RATE, 440); + const format = parseRawAudioMime(`audio/L16;codec=pcm;rate=${RATE}`); + + const wav = convertToWav(pcm, 1, format); + const header = parseWavHeader(wav); + + // Output contract: mono float32 at 24kHz regardless of what came in. + // ffmpeg 8 tags float32 as IEEE_FLOAT (3) and ffmpeg 6 as EXTENSIBLE + // (0xfffe), so the tag is not the contract. The peak check below reads + // the samples as float32 and is what actually pins the format. + expect(header.sampleRate).toBe(24000); + expect(header.numChannels).toBe(1); + expect(header.bitsPerSample).toBe(32); + expect([3, 0xfffe]).toContain(header.audioFormat); + + const peak = peakAmplitude(wav); + expect(peak).toBeGreaterThan(0.2); + expect(peak).toBeLessThan(0.35); + }); + + it('reads the RFC byte order as near full-scale noise', () => { + // Pins the deviation as a deliberate one. If a future edit "corrects" + // s16le to s16be to match the RFC, the test above goes red and this one + // says why: the same bytes read big-endian are not quietly wrong, they + // are loud. + const pcm = sineS16LE(SAMPLES, RATE, 440); + + const asBigEndian = convertToWav(pcm, 1, { format: 's16be', sampleRate: RATE, channels: 1 }); + + expect(peakAmplitude(asBigEndian)).toBeGreaterThan(0.9); + }); + + it('honours the rate from the media type rather than assuming 24kHz', () => { + // Duration is the observable: a wrong rate resamples rather than erroring. + const pcm = sineS16LE(SAMPLES, RATE, 440); + + const correct = convertToWav(pcm, 1, parseRawAudioMime(`audio/L16;rate=${RATE}`)); + const wrong = convertToWav(pcm, 1, parseRawAudioMime('audio/L16;rate=24000')); + + expect(parseWavHeader(correct).durationMs).toBeGreaterThan(450); + expect(parseWavHeader(correct).durationMs).toBeLessThan(550); + expect(parseWavHeader(wrong).durationMs).toBeLessThan(400); + }); +}); diff --git a/tests/tts/raw-pcm.test.ts b/tests/tts/raw-pcm.test.ts new file mode 100644 index 0000000..13b0707 --- /dev/null +++ b/tests/tts/raw-pcm.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { convertToWav, parseRawAudioMime } from '../../src/tts/engine.js'; +import { execFileSync } from 'node:child_process'; + +// `engine.ts` reads `execFileSync` off the namespace at call time, so the whole +// module has to be replaced. An ESM namespace object cannot be spied on. +vi.mock('node:child_process', async importOriginal => ({ + ...(await importOriginal()), + execFileSync: vi.fn(() => Buffer.from('fake-wav')), +})); + +/** + * Gemini's TTS models return RFC 2586 linear PCM: sample data with no RIFF + * header and no magic bytes. `ffmpeg -i pipe:0` cannot sniff that and dies with + * "Invalid data found when processing input", so every Gemini clip failed to + * convert. The format has to be read off the media type and handed to ffmpeg. + * + * Self-describing containers must keep going through ffmpeg's own probing: + * forcing a demuxer on an MP3 would decode noise. + */ +describe('parseRawAudioMime', () => { + it('reads codec, rate and channels out of a Gemini media type', () => { + expect(parseRawAudioMime('audio/L16;codec=pcm;rate=24000')).toEqual({ + format: 's16le', + sampleRate: 24000, + channels: 1, + }); + }); + + it('accepts an explicit channel count', () => { + expect(parseRawAudioMime('audio/L16;codec=pcm;rate=16000;channels=2')).toEqual({ + format: 's16le', + sampleRate: 16000, + channels: 2, + }); + }); + + it('tolerates case, and spacing on both sides of the equals', () => { + // RFC 2045 allows both, and a gateway that normalises the header is enough + // to produce either. + expect(parseRawAudioMime('AUDIO/L16; CODEC=pcm; RATE=48000')?.sampleRate).toBe(48000); + expect(parseRawAudioMime('audio/L16; rate = 16000')?.sampleRate).toBe(16000); + }); + + it('reads a quoted value', () => { + // `Number('"16000"')` is NaN, which used to land in the silent fallback. + expect(parseRawAudioMime('audio/L16;codec=pcm;rate="16000"')?.sampleRate).toBe(16000); + }); + + it('refuses a rate it cannot read instead of guessing one', () => { + // Throwing is the point: a guessed rate is silently wrong, never an error. + for (const mime of ['audio/L16;codec=pcm', 'audio/L16;rate=abc', 'audio/L16;rate=0', 'audio/L16;rate=']) { + expect(() => parseRawAudioMime(mime)).toThrow(/sample rate/); + } + }); + + it('leaves self-describing containers to ffmpeg', () => { + expect(parseRawAudioMime('audio/mpeg')).toBeNull(); + expect(parseRawAudioMime('audio/wav')).toBeNull(); + expect(parseRawAudioMime('audio/ogg;codecs=opus')).toBeNull(); + expect(parseRawAudioMime(undefined)).toBeNull(); + }); +}); + +describe('convertToWav input format', () => { + beforeEach(() => { + vi.mocked(execFileSync).mockClear(); + }); + + /** The ffmpeg argv as it would actually be spawned. */ + function args(): string[] { + return vi.mocked(execFileSync).mock.calls[0][1] as string[]; + } + + it('declares the demuxer before the input when given a raw format', () => { + convertToWav(Buffer.from('pcm'), 1, { format: 's16le', sampleRate: 24000, channels: 1 }); + const argv = args(); + const i = argv.indexOf('-i'); + // Position relative to `-i` is the property that matters; pinning absolute + // indices would redden on a harmless `-hide_banner`. + expect(argv.slice(0, i)).toEqual(['-f', 's16le', '-ar', '24000', '-ac', '1']); + }); + + it('leaves the command untouched when no raw format is given', () => { + convertToWav(Buffer.from('mp3')); + expect(args()[0]).toBe('-i'); + expect(args()).not.toContain('s16le'); + }); + + it('treats a null format the same as an absent one', () => { + convertToWav(Buffer.from('mp3'), 1, null); + expect(args()[0]).toBe('-i'); + }); + + it('still normalises the output to mono 24kHz float', () => { + // `-ar`/`-ac` now appear on both sides of `-i`, which is exactly the shape + // a later edit collapses by mistake. The output side is the contract. + convertToWav(Buffer.from('pcm'), 1, { format: 's16le', sampleRate: 16000, channels: 2 }); + const argv = args(); + const i = argv.indexOf('-i'); + const out = argv.slice(i); + expect(out).toEqual(expect.arrayContaining(['-acodec', 'pcm_f32le'])); + expect(out[out.indexOf('-ac') + 1]).toBe('1'); + expect(out[out.indexOf('-ar') + 1]).toBe('24000'); + }); + + it('still applies speed to a raw stream', () => { + convertToWav(Buffer.from('pcm'), 1.5, { format: 's16le', sampleRate: 24000, channels: 1 }); + const argv = args(); + expect(argv).toContain('atempo=1.5'); + // The demuxer applies to the input and the tempo filter to the output, so + // ffmpeg reads them by position: swapping the two changes what they act on. + expect(argv.indexOf('-f')).toBeLessThan(argv.indexOf('-i')); + expect(argv.indexOf('-filter:a')).toBeGreaterThan(argv.indexOf('-i')); + }); +});