fix(tts): decode the raw PCM Gemini returns

CLAUDE.md

@@ -316,6 +316,7 @@ Custom `test` fixture extends Playwright's `test` with a `narration` fixture tha
 - ~~`zoomTo` transforms `documentElement`~~ — FIXED: post-export camera moves (`narration` option) use ffmpeg `zoompan` — overlays are already burned into the video and unaffected. Legacy browser-side `zoomTo` (without `narration`) still has this issue.
 - OpenAI engine requests raw PCM (`response_format: 'pcm'`) and converts to Float32 directly — do not use `convertToWav` (ffmpeg pipe introduces 0xFFFFFFFF data size artifacts).
 - `convertToWav` (ffmpeg pipe to stdout) writes WAV with `0xFFFFFFFF` data size — `parseWavHeader` falls back to actual buffer length. All engines using `convertToWav` are affected.
+- Gemini TTS needs a TTS model; general models answer an AUDIO request with 400. It returns headerless PCM `ffmpeg -i pipe:0` cannot sniff, spelled `audio/L16;codec=pcm;rate=24000` by 2.5 models and `audio/l16; rate=24000; channels=1` by 3.1: pass `parseRawAudioMime(mimeType)` as `convertToWav`'s third argument. Little-endian despite RFC 2586, matching what Google sends.
 - Showcase demo video hosted via GitHub gist comment upload: https://gist.github.com/shreyaskarnik/6a0996942a96528a984010f36de76079
 - `tsc` build may silently fail if `tsconfig.json` is missing — verify it exists before trusting `npm run build` output
 - `dissolve` transition is a shorter dip-to-black, not a true crossfade blend. A real crossfade would require ffmpeg `xfade` with re-encoded segment pairs — impractical for continuous recordings.

src/tts/engine.ts

@@ -266,17 +266,90 @@ export function buildAtempoChain(speed: number): string[] {
   return ['-filter:a', stages.map(s => `atempo=${fmt(s)}`).join(',')];
 }
 
+/** A headerless audio stream. ffmpeg cannot sniff one, so it has to be told. */
+export interface RawAudioFormat {
+  /** ffmpeg demuxer name, passed to `-f`. `s16le` for signed 16-bit
+   *  little-endian. Not a codec: `-c:a` would reject it. */
+  format: string;
+  sampleRate: number;
+  channels: number;
+}
+
+/** One parameter's value, tolerating the spacing and quoting RFC 2045 allows. */
+function mimeParam(params: string[], name: string): string | undefined {
+  const pattern = new RegExp(`^${name}\\s*=\\s*(.*)$`, 'i');
+  for (const param of params) {
+    const match = pattern.exec(param);
+    if (match) return match[1].trim().replace(/^"(.*)"$/s, '$1');
+  }
+  return undefined;
+}
+
+/**
+ * Read a raw-PCM media type into the arguments ffmpeg needs to open it.
+ *
+ * Raw PCM carries no header, so ffmpeg cannot open it without being told the
+ * format. Gemini's TTS models send `audio/L16;codec=pcm;rate=24000`.
+ *
+ * Little-endian contradicts RFC 2586 section 3, which defines L16 as network
+ * byte order, but it is what Google sends. A conforming provider needs `s16be`.
+ *
+ * Returns null for self-describing formats, which ffmpeg can probe itself.
+ * Throws for an L16 type carrying no readable rate, which nothing can recover.
+ */
+export function parseRawAudioMime(mimeType: string | undefined): RawAudioFormat | null {
+  if (!mimeType) return null;
+  const [type, ...params] = mimeType.split(';').map(part => part.trim());
+  // L16 is the only raw encoding the engines here emit. `audio/L8` (RFC 3551)
+  // and `audio/L24` (RFC 3190) exist but nothing returns them, so they are
+  // left unhandled rather than guessed at.
+  if (type.toLowerCase() !== 'audio/l16') return null;
+
+  const rawRate = mimeParam(params, 'rate');
+  const rate = Number(rawRate);
+  if (!Number.isFinite(rate) || rate <= 0) {
+    // Refusing beats guessing. Declaring 24000 for a stream that is really
+    // 16000 does not fail: it returns a clip a third shorter at 1.5x pitch,
+    // exit code 0, and argo derives scene durations from clip length, so every
+    // wait in the recording shortens and nothing reports a problem. RFC 2586
+    // lists `rate` as required, so a missing one is malformed input.
+    throw new Error(
+      `cannot read a sample rate from "${mimeType}". Raw PCM carries no header, ` +
+        'so the rate has to come from the media type.',
+    );
+  }
+  // Unlike `rate`, the channel default is the RFC's own: "channels ... defaults
+  // to 1" in the L16 registration.
+  const channels = Number(mimeParam(params, 'channels'));
+  return {
+    format: 's16le',
+    sampleRate: rate,
+    channels: Number.isFinite(channels) && channels > 0 ? channels : 1,
+  };
+}
+
 /**
  * Convert arbitrary audio (MP3, OGG, PCM, etc.) to Argo's WAV format
  * (mono, Float32, 24kHz) using ffmpeg.
  *
  * `speed` is applied here because engines that render server-side (ElevenLabs,
  * Gemini) have no native rate control — this conversion is the only place the
  * rate can change. Engines with their own speed parameter must not use it.
+ *
+ * `inputFormat` describes a headerless stream. Pass it whenever the source is
+ * raw PCM; omit it and ffmpeg probes the container itself.
  */
-export function convertToWav(audioBuffer: Buffer, speed = 1): Buffer {
+export function convertToWav(
+  audioBuffer: Buffer,
+  speed = 1,
+  inputFormat?: RawAudioFormat | null,
+): Buffer {
   const { execFileSync } = childProcess;
+  const inputArgs = inputFormat
+    ? ['-f', inputFormat.format, '-ar', String(inputFormat.sampleRate), '-ac', String(inputFormat.channels)]
+    : [];
   const result = execFileSync('ffmpeg', [
+    ...inputArgs,
     '-i', 'pipe:0',
     ...buildAtempoChain(speed),
     '-f', 'wav',

src/tts/engines/gemini.ts

@@ -12,7 +12,10 @@ export class GeminiEngine implements TTSEngine {
 
   constructor(options?: GeminiEngineOptions) {
     this.apiKey = options?.apiKey ?? '';
-    this.model = options?.model ?? 'gemini-2.5-flash';
+    // Must be a TTS model: the general ones answer an AUDIO request with 400,
+    // "This model only supports text output". The `native-audio` models are no
+    // use either, they speak only the Live API socket.
+    this.model = options?.model ?? 'gemini-3.1-flash-tts-preview';
   }
 
   private resolveApiKey(): string {
@@ -68,7 +71,8 @@ export class GeminiEngine implements TTSEngine {
 
     // Convert to Argo WAV format. Gemini has no speed parameter, so the rate
     // change rides along with the conversion.
-    const { convertToWav } = await import('../engine.js');
-    return convertToWav(audioBuffer, options.speed ?? 1);
+    const { convertToWav, parseRawAudioMime } = await import('../engine.js');
+    const inputFormat = parseRawAudioMime(audioPart.inlineData.mimeType);
+    return convertToWav(audioBuffer, options.speed ?? 1, inputFormat);
   }
 }

tests/tts/gemini.test.ts

@@ -0,0 +1,118 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { execFileSync } from 'node:child_process';
+import { GeminiEngine } from '../../src/tts/engines/gemini.js';
+
+/**
+ * The bug this guards was "GeminiEngine.generate never produced audio", and
+ * nothing in the suite called that method: `config.test.ts` only asserts the
+ * constructor does not throw, so the whole suite stayed green while the engine
+ * was dead. Covering `parseRawAudioMime` and `convertToWav` separately does not
+ * help either, because the defect was the wiring between them.
+ *
+ * Pattern borrowed from `mlx-audio.test.ts`: mock the transport, let the
+ * downstream ffmpeg call be a stub, and assert on the argv it was handed.
+ */
+const generateContent = vi.fn();
+
+vi.mock('@google/genai', () => ({
+  GoogleGenAI: class {
+    models = { generateContent };
+  },
+}));
+
+vi.mock('node:child_process', async importOriginal => ({
+  ...(await importOriginal<typeof import('node:child_process')>()),
+  execFileSync: vi.fn(() => Buffer.from('fake-wav')),
+}));
+
+/** A response shaped like the one Gemini's TTS models actually return. */
+function audioResponse(mimeType: string) {
+  return {
+    candidates: [
+      { content: { parts: [{ inlineData: { mimeType, data: Buffer.from('pcm').toString('base64') } }] } },
+    ],
+  };
+}
+
+/** The ffmpeg argv from the conversion that followed. */
+function ffmpegArgs(): string[] {
+  return vi.mocked(execFileSync).mock.calls[0][1] as string[];
+}
+
+describe('GeminiEngine.generate', () => {
+  beforeEach(() => {
+    vi.mocked(execFileSync).mockClear();
+    generateContent.mockReset();
+  });
+
+  it('tells ffmpeg the format of the raw PCM it was sent', async () => {
+    generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm;rate=24000'));
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await engine.generate('hello', { voice: 'Kore' });
+
+    const argv = ffmpegArgs();
+    const i = argv.indexOf('-i');
+    expect(argv.slice(0, i)).toEqual(['-f', 's16le', '-ar', '24000', '-ac', '1']);
+  });
+
+  it('takes the rate from the response rather than assuming one', async () => {
+    // A wrong rate is silently wrong, so the engine has to read what it was sent.
+    generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm;rate=16000'));
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await engine.generate('hello', { voice: 'Kore' });
+
+    expect(ffmpegArgs()[ffmpegArgs().indexOf('-ar') + 1]).toBe('16000');
+  });
+
+  it('carries speed through to the conversion', async () => {
+    generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm;rate=24000'));
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await engine.generate('hello', { voice: 'Kore', speed: 1.5 });
+
+    expect(ffmpegArgs()).toContain('atempo=1.5');
+  });
+
+  it('refuses a response it cannot read the rate from', async () => {
+    generateContent.mockResolvedValue(audioResponse('audio/L16;codec=pcm'));
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await expect(engine.generate('hello', { voice: 'Kore' })).rejects.toThrow(/sample rate/);
+  });
+
+  it('defaults to a model that can actually return audio', async () => {
+    // The previous default was `gemini-2.5-flash`, which answers an AUDIO
+    // request with 400 "This model only supports text output". Nothing in the
+    // suite caught it because the transport is mocked here, so pin the model
+    // name: it is the only part of that failure visible without a live call.
+    generateContent.mockResolvedValue(audioResponse('audio/l16; rate=24000; channels=1'));
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await engine.generate('hello', { voice: 'Kore' });
+
+    expect(generateContent.mock.calls[0][0].model).toBe('gemini-3.1-flash-tts-preview');
+    expect(engine.describe().model).toBe('gemini-3.1-flash-tts-preview');
+  });
+
+  it('reads the media type spelling the 3.1 models use', async () => {
+    // Same audio, different spelling: lowercase `l16`, spaces after the
+    // semicolons, no `codec`, and an explicit `channels`. Both spellings are
+    // captured from live responses, 2.5 and 3.1 respectively.
+    generateContent.mockResolvedValue(audioResponse('audio/l16; rate=24000; channels=1'));
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await engine.generate('hello', { voice: 'Kore' });
+
+    const argv = ffmpegArgs();
+    expect(argv.slice(0, argv.indexOf('-i'))).toEqual(['-f', 's16le', '-ar', '24000', '-ac', '1']);
+  });
+
+  it('raises a useful error when the response carries no audio', async () => {
+    generateContent.mockResolvedValue({ candidates: [{ content: { parts: [{ text: 'sorry' }] } }] });
+    const engine = new GeminiEngine({ apiKey: 'test' });
+
+    await expect(engine.generate('hello', { voice: 'Kore' })).rejects.toThrow(/did not return audio/);
+  });
+});

tests/tts/raw-pcm-roundtrip.test.ts

@@ -0,0 +1,99 @@
+/**
+ * The one thing `raw-pcm.test.ts` cannot check: whether the demuxer argv it
+ * asserts actually decodes the bytes Gemini sends. Every test there stubs
+ * `execFileSync`, so they compare the string 's16le' against the source that
+ * produced it and would pass just as happily if the answer were 's16be'. This
+ * file decodes a synthesized sine through real ffmpeg instead, where a byte
+ * swap lands near full scale rather than at the 0.25 peak it was handed.
+ */
+import { it, expect } from 'vitest';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { convertToWav, parseRawAudioMime, parseWavHeader } from '../../src/tts/engine.js';
+import { describeWithCapability } from '../helpers/capability.js';
+
+const execFileP = promisify(execFile);
+
+// CI installs ffmpeg deliberately, so a miss there means the workflow drifted
+// rather than that the host is bare.
+let hasFfmpeg = false;
+try {
+  await execFileP('ffmpeg', ['-version']);
+  hasFfmpeg = true;
+} catch {
+  hasFfmpeg = false;
+}
+
+/** A mono s16le sine, the shape Gemini's TTS models return. */
+function sineS16LE(samples: number, sampleRate: number, hz: number): Buffer {
+  const buf = Buffer.alloc(samples * 2);
+  for (let i = 0; i < samples; i++) {
+    // 0.25 full scale, chosen so a byte swap lands far outside it.
+    const v = Math.round(Math.sin((2 * Math.PI * hz * i) / sampleRate) * 0.25 * 32767);
+    buf.writeInt16LE(v, i * 2);
+  }
+  return buf;
+}
+
+/** Peak absolute amplitude of a mono float32 WAV, as a fraction of full scale. */
+function peakAmplitude(wav: Buffer): number {
+  const { dataOffset, dataSize } = parseWavHeader(wav);
+  // convertToWav writes 0xFFFFFFFF as the data size (a known ffmpeg-pipe
+  // quirk recorded in CLAUDE.md), so trust the buffer rather than the header.
+  const end = Math.min(wav.length, dataOffset + dataSize);
+  let peak = 0;
+  for (let i = dataOffset; i + 4 <= end; i += 4) {
+    peak = Math.max(peak, Math.abs(wav.readFloatLE(i)));
+  }
+  return peak;
+}
+
+describeWithCapability(hasFfmpeg, 'an ffmpeg binary')('raw PCM survives a real ffmpeg conversion', () => {
+  const RATE = 16000;
+  const SAMPLES = RATE / 2; // 0.5s, deliberately not the 24kHz output rate
+
+  it('decodes a Gemini-shaped L16 response to the amplitude it was given', () => {
+    const pcm = sineS16LE(SAMPLES, RATE, 440);
+    const format = parseRawAudioMime(`audio/L16;codec=pcm;rate=${RATE}`);
+
+    const wav = convertToWav(pcm, 1, format);
+    const header = parseWavHeader(wav);
+
+    // Output contract: mono float32 at 24kHz regardless of what came in.
+    // ffmpeg 8 tags float32 as IEEE_FLOAT (3) and ffmpeg 6 as EXTENSIBLE
+    // (0xfffe), so the tag is not the contract. The peak check below reads
+    // the samples as float32 and is what actually pins the format.
+    expect(header.sampleRate).toBe(24000);
+    expect(header.numChannels).toBe(1);
+    expect(header.bitsPerSample).toBe(32);
+    expect([3, 0xfffe]).toContain(header.audioFormat);
+
+    const peak = peakAmplitude(wav);
+    expect(peak).toBeGreaterThan(0.2);
+    expect(peak).toBeLessThan(0.35);
+  });
+
+  it('reads the RFC byte order as near full-scale noise', () => {
+    // Pins the deviation as a deliberate one. If a future edit "corrects"
+    // s16le to s16be to match the RFC, the test above goes red and this one
+    // says why: the same bytes read big-endian are not quietly wrong, they
+    // are loud.
+    const pcm = sineS16LE(SAMPLES, RATE, 440);
+
+    const asBigEndian = convertToWav(pcm, 1, { format: 's16be', sampleRate: RATE, channels: 1 });
+
+    expect(peakAmplitude(asBigEndian)).toBeGreaterThan(0.9);
+  });
+
+  it('honours the rate from the media type rather than assuming 24kHz', () => {
+    // Duration is the observable: a wrong rate resamples rather than erroring.
+    const pcm = sineS16LE(SAMPLES, RATE, 440);
+
+    const correct = convertToWav(pcm, 1, parseRawAudioMime(`audio/L16;rate=${RATE}`));
+    const wrong = convertToWav(pcm, 1, parseRawAudioMime('audio/L16;rate=24000'));
+
+    expect(parseWavHeader(correct).durationMs).toBeGreaterThan(450);
+    expect(parseWavHeader(correct).durationMs).toBeLessThan(550);
+    expect(parseWavHeader(wrong).durationMs).toBeLessThan(400);
+  });
+});