From dfce75f948a77919a0ccdbeaa588dfdb1cd8cf11 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Apr 2026 19:53:22 +0000 Subject: [PATCH 1/2] fix: produce valid audio chunks when transcribing files >25MB The old chunking logic split raw file bytes at arbitrary boundaries, producing invalid audio files that Whisper rejects. Web used blob.slice() on WebM containers; native split base64-encoded M4A at random offsets. Fix: Web now decodes via AudioContext to PCM then encodes valid WAV chunks. Native parses the M4A (MP4) container to find sample boundaries and builds valid M4A files for each chunk with proper atoms. https://claude.ai/code/session_01AALsZMcA3oGi5w3xEcR71C --- .../processing.transcribeRecording.test.ts | 110 ++--- services/audio-splitter.ts | 456 ++++++++++++++++++ services/processing.ts | 40 +- 3 files changed, 503 insertions(+), 103 deletions(-) create mode 100644 services/audio-splitter.ts diff --git a/__tests__/processing.transcribeRecording.test.ts b/__tests__/processing.transcribeRecording.test.ts index 4f54982..264a047 100644 --- a/__tests__/processing.transcribeRecording.test.ts +++ b/__tests__/processing.transcribeRecording.test.ts @@ -5,6 +5,7 @@ * - supabase.auth → authenticated session * - fetch → fake Whisper endpoint returning "chunk-" * - expo-file-system/legacy → in-memory file store + * - audio-splitter → returns pre-built chunks (web blobs / native temp URIs) * - react-native Platform → overridable per describe block */ @@ -33,39 +34,32 @@ jest.mock('expo-file-system/legacy', () => ({ copyAsync: jest.fn(), })); +jest.mock('../services/audio-splitter', () => ({ + splitAudioWeb: jest.fn(), + splitAudioNative: jest.fn(), +})); + // ─── Imports (after mocks) ──────────────────────────────────────────────────── import * as FileSystem from 'expo-file-system/legacy'; import { Platform } from 'react-native'; import { transcribeRecording } from '../services/processing'; +import { splitAudioWeb, splitAudioNative } from '../services/audio-splitter'; // ─── Constants ──────────────────────────────────────────────────────────────── const MB = 1024 * 1024; -const CHUNK_BYTES = 24 * MB; - -// ─── Helpers ───────────────────────────────────────────────────────────────── - -/** Build a base64 string that represents `bytes` bytes of binary data. */ -function makeBase64(bytes: number): string { - return 'A'.repeat(Math.ceil(bytes / 3) * 4); -} - -// ─── In-memory file store ───────────────────────────────────────────────────── - -const fileStore: Record = {}; -const deletedUris: string[] = []; // ─── Global fetch mock ──────────────────────────────────────────────────────── let whisperCallCount = 0; const mockFetch = jest.fn(); +const deletedUris: string[] = []; beforeEach(() => { jest.clearAllMocks(); whisperCallCount = 0; deletedUris.length = 0; - Object.keys(fileStore).forEach(k => delete fileStore[k]); // Default: native (Platform as any).OS = 'native'; @@ -81,17 +75,9 @@ beforeEach(() => { }); global.fetch = mockFetch as any; - // FileSystem: in-memory file store - (FileSystem.writeAsStringAsync as jest.Mock).mockImplementation((uri: string, data: string) => { - fileStore[uri] = data; - return Promise.resolve(); - }); - (FileSystem.readAsStringAsync as jest.Mock).mockImplementation((uri: string) => - Promise.resolve(fileStore[uri] ?? '') - ); + // FileSystem: track deletions (FileSystem.deleteAsync as jest.Mock).mockImplementation((uri: string) => { deletedUris.push(uri); - delete fileStore[uri]; return Promise.resolve(); }); }); @@ -105,23 +91,26 @@ describe('transcribeRecording — small file (≤ 25 MB)', () => { const result = await transcribeRecording('file://recording.m4a'); expect(mockFetch).toHaveBeenCalledTimes(1); - expect(result).toBe('chunk-1'); + expect(result).toEqual({ + transcription: 'chunk-1', + uncertainTerms: [], + }); }); }); describe('transcribeRecording — large file (> 25 MB, native)', () => { - // 50 MB → ceil(50/24) = 3 chunks (24 MB + 24 MB + 2 MB) - const FILE_SIZE = 50 * MB; - const EXPECTED_CHUNKS = Math.ceil(FILE_SIZE / CHUNK_BYTES); // 3 + const EXPECTED_CHUNKS = 3; + const tempUris = ['cache://chunk-0.m4a', 'cache://chunk-1.m4a', 'cache://chunk-2.m4a']; beforeEach(() => { - (FileSystem.getInfoAsync as jest.Mock).mockResolvedValue({ exists: true, size: FILE_SIZE }); - fileStore['file://recording.m4a'] = makeBase64(FILE_SIZE); + (FileSystem.getInfoAsync as jest.Mock).mockResolvedValue({ exists: true, size: 50 * MB }); + (splitAudioNative as jest.Mock).mockResolvedValue(tempUris); }); - it('splits into the correct number of chunks', async () => { + it('calls splitAudioNative and sends one request per chunk', async () => { await transcribeRecording('file://recording.m4a'); + expect(splitAudioNative).toHaveBeenCalledWith('file://recording.m4a'); expect(mockFetch).toHaveBeenCalledTimes(EXPECTED_CHUNKS); }); @@ -129,23 +118,20 @@ describe('transcribeRecording — large file (> 25 MB, native)', () => { const result = await transcribeRecording('file://recording.m4a'); const expected = Array.from({ length: EXPECTED_CHUNKS }, (_, i) => `chunk-${i + 1}`).join(' '); - expect(result).toBe(expected); + expect(result).toEqual({ + transcription: expected, + uncertainTerms: [], + }); }); - it('writes one temp file per chunk then deletes all of them', async () => { + it('deletes all temp files after transcription', async () => { await transcribeRecording('file://recording.m4a'); - expect(FileSystem.writeAsStringAsync).toHaveBeenCalledTimes(EXPECTED_CHUNKS); expect(FileSystem.deleteAsync).toHaveBeenCalledTimes(EXPECTED_CHUNKS); - - const writtenUris = (FileSystem.writeAsStringAsync as jest.Mock).mock.calls.map( - (c: any[]) => c[0] as string - ); - expect(deletedUris.sort()).toEqual(writtenUris.sort()); + expect(deletedUris.sort()).toEqual(tempUris.sort()); }); it('deletes all temp files even when Whisper throws', async () => { - // Make the second Whisper call fail let calls = 0; mockFetch.mockImplementation(() => { calls += 1; @@ -155,47 +141,29 @@ describe('transcribeRecording — large file (> 25 MB, native)', () => { await expect(transcribeRecording('file://recording.m4a')).rejects.toThrow(); - // Cleanup must still have run for all written temp files - const writtenUris = (FileSystem.writeAsStringAsync as jest.Mock).mock.calls.map( - (c: any[]) => c[0] as string - ); - expect(writtenUris.length).toBeGreaterThan(0); - expect(deletedUris.sort()).toEqual(writtenUris.sort()); - }); - - it('each chunk is the correct base64 size', async () => { - await transcribeRecording('file://recording.m4a'); - - const charsPerChunk = (CHUNK_BYTES / 3) * 4; - const chunkSizes = (FileSystem.writeAsStringAsync as jest.Mock).mock.calls.map( - (c: any[]) => (c[1] as string).length - ); - - // All chunks except the last must be exactly charsPerChunk - chunkSizes.slice(0, -1).forEach(len => expect(len).toBe(charsPerChunk)); - // Last chunk is smaller (the remainder) - expect(chunkSizes[chunkSizes.length - 1]).toBeLessThan(charsPerChunk); - // All chunks together re-assemble to the full file - const totalChars = chunkSizes.reduce((a, b) => a + b, 0); - expect(totalChars).toBe(makeBase64(FILE_SIZE).length); + // Cleanup must still have run for all temp files + expect(deletedUris.sort()).toEqual(tempUris.sort()); }); }); describe('transcribeRecording — large file (> 25 MB, web)', () => { - // 30 MB → ceil(30/24) = 2 chunks - const FILE_SIZE = 30 * MB; - const EXPECTED_CHUNKS = Math.ceil(FILE_SIZE / CHUNK_BYTES); // 2 + const EXPECTED_CHUNKS = 2; + const fakeBlobs = [ + new Blob(['chunk-a'], { type: 'audio/wav' }), + new Blob(['chunk-b'], { type: 'audio/wav' }), + ]; beforeEach(() => { (Platform as any).OS = 'web'; + (splitAudioWeb as jest.Mock).mockResolvedValue(fakeBlobs); - const blob = new Blob(['x'.repeat(FILE_SIZE)], { type: 'audio/webm' }); + const originalBlob = new Blob(['x'.repeat(30 * MB)], { type: 'audio/webm' }); let calls = 0; mockFetch.mockImplementation(() => { calls += 1; if (calls === 1) { // First call is fetch(localUri) to get the blob - return Promise.resolve({ blob: () => Promise.resolve(blob) }); + return Promise.resolve({ blob: () => Promise.resolve(originalBlob) }); } const idx = calls - 1; // Whisper call index return Promise.resolve({ @@ -205,9 +173,10 @@ describe('transcribeRecording — large file (> 25 MB, web)', () => { }); }); - it('splits into the correct number of Whisper calls', async () => { + it('calls splitAudioWeb and sends one Whisper request per chunk', async () => { await transcribeRecording('blob:recording'); + expect(splitAudioWeb).toHaveBeenCalled(); // 1 (blob fetch) + EXPECTED_CHUNKS (Whisper) expect(mockFetch).toHaveBeenCalledTimes(1 + EXPECTED_CHUNKS); }); @@ -216,7 +185,10 @@ describe('transcribeRecording — large file (> 25 MB, web)', () => { const result = await transcribeRecording('blob:recording'); const expected = Array.from({ length: EXPECTED_CHUNKS }, (_, i) => `chunk-${i + 1}`).join(' '); - expect(result).toBe(expected); + expect(result).toEqual({ + transcription: expected, + uncertainTerms: [], + }); }); it('does not touch the native filesystem', async () => { diff --git a/services/audio-splitter.ts b/services/audio-splitter.ts new file mode 100644 index 0000000..1def0d6 --- /dev/null +++ b/services/audio-splitter.ts @@ -0,0 +1,456 @@ +/** + * Audio file splitter for files exceeding the 25 MB Whisper API limit. + * + * Web: AudioContext → decode to PCM → split → encode as WAV chunks + * Native: Parse M4A (MP4) container → split at sample boundaries → valid M4A chunks + */ + +import { Platform } from 'react-native'; +import * as FileSystem from 'expo-file-system/legacy'; + +const CHUNK_BYTES = 24 * 1024 * 1024; // 24 MB per chunk (1 MB safety margin) + +// ─── Shared helpers ────────────────────────────────────────────────────────── + +function concatBytes(...arrays: Uint8Array[]): Uint8Array { + const total = arrays.reduce((s, a) => s + a.length, 0); + const result = new Uint8Array(total); + let off = 0; + for (const a of arrays) { + result.set(a, off); + off += a.length; + } + return result; +} + +// ─── Web: AudioContext → WAV ───────────────────────────────────────────────── + +function encodeWav(pcm: Float32Array, sampleRate: number): Blob { + const n = pcm.length; + const buf = new ArrayBuffer(44 + n * 2); + const v = new DataView(buf); + const w = (off: number, s: string) => { + for (let i = 0; i < s.length; i++) v.setUint8(off + i, s.charCodeAt(i)); + }; + + w(0, 'RIFF'); + v.setUint32(4, 36 + n * 2, true); + w(8, 'WAVE'); + w(12, 'fmt '); + v.setUint32(16, 16, true); // fmt chunk size + v.setUint16(20, 1, true); // PCM + v.setUint16(22, 1, true); // mono + v.setUint32(24, sampleRate, true); // sample rate + v.setUint32(28, sampleRate * 2, true); // byte rate (mono, 16-bit) + v.setUint16(32, 2, true); // block align + v.setUint16(34, 16, true); // bits per sample + w(36, 'data'); + v.setUint32(40, n * 2, true); + + for (let i = 0; i < n; i++) { + const s = Math.max(-1, Math.min(1, pcm[i])); + v.setInt16(44 + i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + } + + return new Blob([buf], { type: 'audio/wav' }); +} + +export async function splitAudioWeb(blob: Blob): Promise { + const arrayBuffer = await blob.arrayBuffer(); + const ctx = new (globalThis.AudioContext || (globalThis as any).webkitAudioContext)(); + try { + const audioBuffer = await ctx.decodeAudioData(arrayBuffer); + const sr = audioBuffer.sampleRate; + const len = audioBuffer.length; + const nCh = audioBuffer.numberOfChannels; + + // Mix to mono + const mono = new Float32Array(len); + for (let ch = 0; ch < nCh; ch++) { + const chData = audioBuffer.getChannelData(ch); + for (let i = 0; i < len; i++) mono[i] += chData[i] / nCh; + } + + // Max samples per chunk: (CHUNK_BYTES - 44 byte WAV header) / 2 bytes per sample + const samplesPerChunk = Math.floor((CHUNK_BYTES - 44) / 2); + const blobs: Blob[] = []; + for (let off = 0; off < len; off += samplesPerChunk) { + blobs.push(encodeWav(mono.slice(off, Math.min(off + samplesPerChunk, len)), sr)); + } + return blobs; + } finally { + await ctx.close(); + } +} + +// ─── Native: M4A (MP4) parse & mux ────────────────────────────────────────── + +interface M4AInfo { + ftyp: Uint8Array; + timescale: number; + stsd: Uint8Array; // complete stsd atom (with header) + sampleSizes: number[]; + sampleDeltas: { count: number; delta: number }[]; + audioDataOffset: number; // file-level offset of first audio sample +} + +const CONTAINERS = new Set(['moov', 'trak', 'mdia', 'minf', 'stbl', 'dinf', 'udta']); + +function readType(d: Uint8Array, pos: number): string { + return String.fromCharCode(d[pos], d[pos + 1], d[pos + 2], d[pos + 3]); +} + +function parseM4A(data: Uint8Array): M4AInfo { + const view = new DataView(data.buffer, data.byteOffset, data.byteLength); + const info: Partial = {}; + + const walk = (start: number, end: number) => { + let pos = start; + while (pos + 8 <= end) { + let size = view.getUint32(pos); + const type = readType(data, pos + 4); + let headerSize = 8; + + if (size === 1 && pos + 16 <= end) { + // 64-bit extended size + const hi = view.getUint32(pos + 8); + const lo = view.getUint32(pos + 12); + size = hi * 0x100000000 + lo; + headerSize = 16; + } else if (size === 0) { + size = end - pos; // atom extends to end of parent + } + if (size < headerSize || pos + size > end) break; + + const dataStart = pos + headerSize; + const atomEnd = pos + size; + + switch (type) { + case 'ftyp': + info.ftyp = data.slice(pos, atomEnd); + break; + + case 'mdhd': { + // version(1)+flags(3) then fields differ by version + const ver = data[dataStart]; + // v0: create(4)+mod(4)+timescale(4) → timescale at +12 + // v1: create(8)+mod(8)+timescale(4) → timescale at +20 + const tsOff = ver === 0 ? 12 : 20; + info.timescale = view.getUint32(dataStart + tsOff); + break; + } + + case 'stsd': + info.stsd = data.slice(pos, atomEnd); + break; + + case 'stsz': { + // version+flags(4), sample_size(4), sample_count(4), entries… + const constantSize = view.getUint32(dataStart + 4); + const count = view.getUint32(dataStart + 8); + const sizes: number[] = []; + if (constantSize === 0) { + for (let i = 0; i < count; i++) sizes.push(view.getUint32(dataStart + 12 + i * 4)); + } else { + for (let i = 0; i < count; i++) sizes.push(constantSize); + } + info.sampleSizes = sizes; + break; + } + + case 'stts': { + // version+flags(4), entry_count(4), entries(8 each) + const entryCount = view.getUint32(dataStart + 4); + const deltas: { count: number; delta: number }[] = []; + for (let i = 0; i < entryCount; i++) { + deltas.push({ + count: view.getUint32(dataStart + 8 + i * 8), + delta: view.getUint32(dataStart + 12 + i * 8), + }); + } + info.sampleDeltas = deltas; + break; + } + + case 'stco': { + // version+flags(4), entry_count(4), offsets(4 each) + const n = view.getUint32(dataStart + 4); + if (n > 0) info.audioDataOffset = view.getUint32(dataStart + 8); + break; + } + + case 'co64': { + const n = view.getUint32(dataStart + 4); + if (n > 0) { + const hi = view.getUint32(dataStart + 8); + const lo = view.getUint32(dataStart + 12); + info.audioDataOffset = hi * 0x100000000 + lo; + } + break; + } + + default: + if (CONTAINERS.has(type)) walk(dataStart, atomEnd); + break; + } + + pos = atomEnd; + } + }; + + walk(0, data.length); + + if (!info.ftyp) throw new Error('Invalid M4A: missing ftyp atom'); + if (info.timescale == null) throw new Error('Invalid M4A: missing mdhd atom'); + if (!info.stsd) throw new Error('Invalid M4A: missing stsd atom'); + if (!info.sampleSizes) throw new Error('Invalid M4A: missing stsz atom'); + if (!info.sampleDeltas) throw new Error('Invalid M4A: missing stts atom'); + if (info.audioDataOffset == null) throw new Error('Invalid M4A: missing stco/co64 atom'); + + return info as M4AInfo; +} + +// ─── M4A muxer helpers ────────────────────────────────────────────────────── + +function buildAtom(type: string, payload: Uint8Array): Uint8Array { + const atom = new Uint8Array(8 + payload.length); + new DataView(atom.buffer).setUint32(0, 8 + payload.length); + atom[4] = type.charCodeAt(0); + atom[5] = type.charCodeAt(1); + atom[6] = type.charCodeAt(2); + atom[7] = type.charCodeAt(3); + atom.set(payload, 8); + return atom; +} + +/** Extract stts entries for samples [startSample, endSample). */ +function sttsForRange( + allDeltas: { count: number; delta: number }[], + startSample: number, + endSample: number, +): { count: number; delta: number }[] { + const result: { count: number; delta: number }[] = []; + let idx = 0; + for (const { count, delta } of allDeltas) { + const entryEnd = idx + count; + if (entryEnd <= startSample) { idx = entryEnd; continue; } + if (idx >= endSample) break; + const n = Math.min(entryEnd, endSample) - Math.max(idx, startSample); + if (n > 0) { + if (result.length > 0 && result[result.length - 1].delta === delta) { + result[result.length - 1].count += n; + } else { + result.push({ count: n, delta }); + } + } + idx = entryEnd; + } + return result; +} + +/** + * Build a minimal valid M4A file containing the given audio samples. + * + * Atom layout: ftyp | moov (mvhd, trak(tkhd, mdia(mdhd, hdlr, minf(smhd, dinf, stbl)))) | mdat + */ +function buildM4AFile( + ftyp: Uint8Array, + stsd: Uint8Array, + timescale: number, + sampleSizes: number[], + sttsEntries: { count: number; delta: number }[], + audioData: Uint8Array, +): Uint8Array { + const n = sampleSizes.length; + const duration = sttsEntries.reduce((s, e) => s + e.count * e.delta, 0); + + // ── stbl children ── + + // stts: version+flags(4) + entry_count(4) + entries(8 each) + const sttsP = new Uint8Array(4 + 4 + sttsEntries.length * 8); + const sttsV = new DataView(sttsP.buffer); + sttsV.setUint32(4, sttsEntries.length); + sttsEntries.forEach((e, i) => { + sttsV.setUint32(8 + i * 8, e.count); + sttsV.setUint32(12 + i * 8, e.delta); + }); + + // stsz: version+flags(4) + sample_size(4) + count(4) + entries(4 each) + const stszP = new Uint8Array(4 + 4 + 4 + n * 4); + const stszV = new DataView(stszP.buffer); + stszV.setUint32(8, n); + sampleSizes.forEach((sz, i) => stszV.setUint32(12 + i * 4, sz)); + + // stsc: version+flags(4) + entry_count(4) + one entry(12) + const stscP = new Uint8Array(4 + 4 + 12); + const stscV = new DataView(stscP.buffer); + stscV.setUint32(4, 1); + stscV.setUint32(8, 1); // first_chunk (1-based) + stscV.setUint32(12, n); // samples_per_chunk + stscV.setUint32(16, 1); // sample_description_index + + // stco: version+flags(4) + entry_count(4) + offset(4) — placeholder, patched later + const stcoP = new Uint8Array(4 + 4 + 4); + new DataView(stcoP.buffer).setUint32(4, 1); + + const stbl = buildAtom('stbl', concatBytes( + stsd, + buildAtom('stts', sttsP), + buildAtom('stsz', stszP), + buildAtom('stsc', stscP), + buildAtom('stco', stcoP), + )); + + // ── minf ── + + const smhd = buildAtom('smhd', new Uint8Array(8)); // version+flags(4) + balance(2) + reserved(2) + + const urlP = new Uint8Array(4); + urlP[3] = 1; // self-contained flag + const drefP = new Uint8Array(8 + 12); // version+flags(4) + count(4) + url atom(12) + new DataView(drefP.buffer).setUint32(4, 1); + drefP.set(buildAtom('url ', urlP), 8); + + const minf = buildAtom('minf', concatBytes( + smhd, + buildAtom('dinf', buildAtom('dref', drefP)), + stbl, + )); + + // ── mdia ── + + // mdhd v0: version+flags(4) + create(4) + mod(4) + timescale(4) + duration(4) + lang(2) + pre(2) = 24 + const mdhdP = new Uint8Array(24); + const mdhdV = new DataView(mdhdP.buffer); + mdhdV.setUint32(12, timescale); + mdhdV.setUint32(16, duration); + mdhdP[20] = 0x55; mdhdP[21] = 0xC4; // undetermined language + + // hdlr: version+flags(4) + pre_defined(4) + handler_type(4) + reserved(12) + name + const handlerName = new TextEncoder().encode('SoundHandler\0'); + const hdlrP = new Uint8Array(4 + 4 + 4 + 12 + handlerName.length); + // handler_type = 'soun' at offset 8 + hdlrP[8] = 0x73; hdlrP[9] = 0x6F; hdlrP[10] = 0x75; hdlrP[11] = 0x6E; + hdlrP.set(handlerName, 24); + + const mdia = buildAtom('mdia', concatBytes( + buildAtom('mdhd', mdhdP), + buildAtom('hdlr', hdlrP), + minf, + )); + + // ── trak ── + + // tkhd v0: version+flags(4) + create(4) + mod(4) + trackID(4) + reserved(4) + // + duration(4) + reserved(8) + layer(2) + altGroup(2) + volume(2) + // + reserved(2) + matrix(36) + width(4) + height(4) = 84 + const tkhdP = new Uint8Array(84); + const tkhdV = new DataView(tkhdP.buffer); + tkhdP[3] = 3; // flags: enabled + in_movie + tkhdV.setUint32(12, 1); // track_ID + tkhdV.setUint32(20, duration); // duration + tkhdV.setUint16(36, 0x0100); // volume = 1.0 + tkhdV.setUint32(40, 0x00010000); // matrix[0] + tkhdV.setUint32(56, 0x00010000); // matrix[4] + tkhdV.setUint32(72, 0x40000000); // matrix[8] + + const trak = buildAtom('trak', concatBytes(buildAtom('tkhd', tkhdP), mdia)); + + // ── moov ── + + // mvhd v0: version+flags(4) + create(4) + mod(4) + timescale(4) + duration(4) + // + rate(4) + volume(2) + reserved(10) + matrix(36) + pre_defined(24) + // + next_track_id(4) = 100 + const mvhdP = new Uint8Array(100); + const mvhdV = new DataView(mvhdP.buffer); + mvhdV.setUint32(12, timescale); // timescale + mvhdV.setUint32(16, duration); // duration + mvhdV.setUint32(20, 0x00010000); // rate = 1.0 + mvhdV.setUint16(24, 0x0100); // volume = 1.0 + mvhdV.setUint32(36, 0x00010000); // matrix[0] + mvhdV.setUint32(52, 0x00010000); // matrix[4] + mvhdV.setUint32(68, 0x40000000); // matrix[8] + mvhdV.setUint32(96, 2); // next_track_id + + const moov = buildAtom('moov', concatBytes(buildAtom('mvhd', mvhdP), trak)); + + // ── mdat ── + const mdat = buildAtom('mdat', audioData); + + // ── Patch stco offset ── + // Audio data starts at: ftyp.length + moov.length + 8 (mdat atom header) + const audioOffset = ftyp.length + moov.length + 8; + // Find 'stco' atom in moov and write the offset + for (let i = 0; i < moov.length - 20; i++) { + if (moov[i + 4] === 0x73 && moov[i + 5] === 0x74 && + moov[i + 6] === 0x63 && moov[i + 7] === 0x6F) { // 'stco' + new DataView(moov.buffer, moov.byteOffset).setUint32(i + 16, audioOffset); + break; + } + } + + return concatBytes(ftyp, moov, mdat); +} + +// ─── Native: split M4A into valid chunk files ──────────────────────────────── + +function base64ToBytes(b64: string): Uint8Array { + const raw = atob(b64); + const bytes = new Uint8Array(raw.length); + for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i); + return bytes; +} + +function bytesToBase64(bytes: Uint8Array): string { + const parts: string[] = []; + for (let i = 0; i < bytes.length; i += 8192) { + const slice = bytes.subarray(i, Math.min(i + 8192, bytes.length)); + parts.push(String.fromCharCode.apply(null, Array.from(slice))); + } + return btoa(parts.join('')); +} + +export async function splitAudioNative(localUri: string): Promise { + const base64 = await FileSystem.readAsStringAsync(localUri, { + encoding: FileSystem.EncodingType.Base64, + }); + const data = base64ToBytes(base64); + const info = parseM4A(data); + + // Group samples into chunks of ≤ CHUNK_BYTES of audio data + const groups: { start: number; end: number }[] = []; + let grpStart = 0; + let grpSize = 0; + for (let i = 0; i < info.sampleSizes.length; i++) { + if (grpSize + info.sampleSizes[i] > CHUNK_BYTES && grpSize > 0) { + groups.push({ start: grpStart, end: i }); + grpStart = i; + grpSize = 0; + } + grpSize += info.sampleSizes[i]; + } + if (grpSize > 0) groups.push({ start: grpStart, end: info.sampleSizes.length }); + + // Build a valid M4A file for each group and write to a temp file + const tempUris: string[] = []; + let dataOffset = info.audioDataOffset; + + for (let g = 0; g < groups.length; g++) { + const { start, end } = groups[g]; + const chunkSizes = info.sampleSizes.slice(start, end); + const chunkDataSize = chunkSizes.reduce((s, sz) => s + sz, 0); + const audioBytes = data.slice(dataOffset, dataOffset + chunkDataSize); + dataOffset += chunkDataSize; + + const entries = sttsForRange(info.sampleDeltas, start, end); + const m4aBytes = buildM4AFile(info.ftyp, info.stsd, info.timescale, chunkSizes, entries, audioBytes); + + const tempUri = `${FileSystem.cacheDirectory}chunk-${Date.now()}-${g}.m4a`; + await FileSystem.writeAsStringAsync(tempUri, bytesToBase64(m4aBytes), { + encoding: FileSystem.EncodingType.Base64, + }); + tempUris.push(tempUri); + } + + return tempUris; +} diff --git a/services/processing.ts b/services/processing.ts index 4388f9b..c88f66d 100644 --- a/services/processing.ts +++ b/services/processing.ts @@ -2,6 +2,7 @@ import { Platform } from 'react-native'; import * as FileSystem from 'expo-file-system/legacy'; import { supabase } from './supabase'; import { ProcessingResult, FormatType, UncertainTerm } from '../types'; +import { splitAudioWeb, splitAudioNative } from './audio-splitter'; export interface TranscribeResult { transcription: string; @@ -9,17 +10,6 @@ export interface TranscribeResult { } const WHISPER_MAX_BYTES = 25 * 1024 * 1024; // 25 MB — OpenAI Whisper API limit -const CHUNK_BYTES = 24 * 1024 * 1024; // 24 MB per chunk (1 MB safety margin) - -function sliceBlob(blob: Blob): Blob[] { - const chunks: Blob[] = []; - let offset = 0; - while (offset < blob.size) { - chunks.push(blob.slice(offset, offset + CHUNK_BYTES, blob.type)); - offset += CHUNK_BYTES; - } - return chunks; -} export const transcribeRecording = async ( localUri: string, @@ -73,13 +63,13 @@ export const transcribeRecording = async ( return sendChunk(formData); } - // File exceeds 25 MB: split, transcribe each chunk, join results + // File exceeds 25 MB: decode audio, split into valid WAV chunks, transcribe each // Uncertain terms from fragment chunks are meaningless — skip them - const chunks = sliceBlob(blob); + const chunks = await splitAudioWeb(blob); const parts: string[] = []; for (let i = 0; i < chunks.length; i++) { const formData = new FormData(); - formData.append('audio', chunks[i] as any, `recording-part${i + 1}.webm`); + formData.append('audio', chunks[i] as any, `recording-part${i + 1}.wav`); formData.append('mode', 'transcribe_only'); const { transcription } = await sendChunk(formData); parts.push(transcription); @@ -99,26 +89,8 @@ export const transcribeRecording = async ( return sendChunk(formData); } - // File exceeds 25 MB: read as base64, split into chunks, write temp files - // Base64 chars per chunk: (CHUNK_BYTES / 3) * 4 — always a multiple of 4 for valid base64 - const charsPerChunk = (CHUNK_BYTES / 3) * 4; - const base64Full = await FileSystem.readAsStringAsync(localUri, { - encoding: FileSystem.EncodingType.Base64, - }); - - const tempUris: string[] = []; - let offset = 0; - let i = 0; - while (offset < base64Full.length) { - const chunkB64 = base64Full.slice(offset, offset + charsPerChunk); - const tempUri = `${FileSystem.cacheDirectory}chunk-${Date.now()}-${i}.m4a`; - await FileSystem.writeAsStringAsync(tempUri, chunkB64, { - encoding: FileSystem.EncodingType.Base64, - }); - tempUris.push(tempUri); - offset += charsPerChunk; - i++; - } + // File exceeds 25 MB: parse M4A container, split at sample boundaries into valid M4A chunks + const tempUris = await splitAudioNative(localUri); try { const parts: string[] = []; From 668b33b877f7779f5f852800c179a148052cca78 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Apr 2026 09:30:00 +0000 Subject: [PATCH 2/2] fix: rewrite M4A muxer with pre-calculated sizes to fix stco offset bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous buildM4AFile used a byte-search to find 'stco' in the moov and patch its offset. This search could find false matches in atom type fields, causing the stco to point into moov instead of mdat — producing files Whisper rejects as "format not supported". Fix: pre-calculate all atom sizes, compute the correct stco offset upfront, then write the entire file sequentially into a single buffer. No search-and-patch needed. Also: - Parse ALL stco entries + stsc for proper multi-chunk M4A support (common in Android MediaRecorder files) - Compute per-sample file offsets instead of assuming contiguous layout - Add roundtrip tests (build → parse → verify audio data at offsets) https://claude.ai/code/session_01AALsZMcA3oGi5w3xEcR71C --- __tests__/audio-splitter.test.ts | 210 +++++++++++++ services/audio-splitter.ts | 506 +++++++++++++++++++------------ 2 files changed, 519 insertions(+), 197 deletions(-) create mode 100644 __tests__/audio-splitter.test.ts diff --git a/__tests__/audio-splitter.test.ts b/__tests__/audio-splitter.test.ts new file mode 100644 index 0000000..de28c44 --- /dev/null +++ b/__tests__/audio-splitter.test.ts @@ -0,0 +1,210 @@ +/** + * Roundtrip tests for the M4A parser and muxer. + * + * Builds a synthetic M4A → parses it → verifies parsed values. + * Then rebuilds from parsed data → re-parses → verifies roundtrip consistency. + * This catches any structural issues in the atom layout. + */ + +jest.mock('expo-file-system/legacy', () => ({ + EncodingType: { Base64: 'base64' }, + cacheDirectory: 'cache://', + readAsStringAsync: jest.fn(), + writeAsStringAsync: jest.fn(), + deleteAsync: jest.fn(), +})); + +import { parseM4A, buildM4AFile, sttsForRange, M4AInfo } from '../services/audio-splitter'; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +/** Build a minimal ftyp atom for M4A. */ +function makeFtyp(): Uint8Array { + // ftyp: size(4) + 'ftyp'(4) + major_brand(4) + minor_version(4) + compatible(4) = 20 bytes + const ftyp = new Uint8Array(20); + const v = new DataView(ftyp.buffer); + v.setUint32(0, 20); + ftyp.set([0x66, 0x74, 0x79, 0x70], 4); // 'ftyp' + ftyp.set([0x4D, 0x34, 0x41, 0x20], 8); // 'M4A ' + ftyp.set([0x69, 0x73, 0x6F, 0x6D], 16); // 'isom' + return ftyp; +} + +/** Build a minimal stsd atom for AAC audio (opaque — we just need valid bytes). */ +function makeStsd(): Uint8Array { + // Minimal stsd: header(8) + version+flags(4) + entry_count(4) + mp4a entry + // mp4a: header(8) + reserved(6) + data_ref_idx(2) + reserved(8) + + // channels(2) + sample_size(2) + reserved(4) + sample_rate(4) = 36 bytes + const mp4aSize = 36; + const stsdSize = 8 + 4 + 4 + mp4aSize; + const stsd = new Uint8Array(stsdSize); + const v = new DataView(stsd.buffer); + + // stsd header + v.setUint32(0, stsdSize); + stsd.set([0x73, 0x74, 0x73, 0x64], 4); // 'stsd' + // version + flags = 0 + v.setUint32(12, 1); // entry_count + + // mp4a + const mp4aOff = 16; + v.setUint32(mp4aOff, mp4aSize); + stsd.set([0x6D, 0x70, 0x34, 0x61], mp4aOff + 4); // 'mp4a' + // reserved (6 bytes) = 0 + v.setUint16(mp4aOff + 14, 1); // data_reference_index = 1 + // reserved (8 bytes) = 0 + v.setUint16(mp4aOff + 24, 1); // channel_count = 1 (mono) + v.setUint16(mp4aOff + 26, 16); // sample_size = 16 bits + // reserved (4 bytes) = 0 + v.setUint32(mp4aOff + 32, 44100 << 16); // sample_rate = 44100 as 16.16 fixed + + return stsd; +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +describe('parseM4A', () => { + const FTYP = makeFtyp(); + const STSD = makeStsd(); + const TIMESCALE = 44100; + const SAMPLE_SIZES = [256, 300, 280, 310, 290]; + const STTS_ENTRIES = [{ count: 5, delta: 1024 }]; + const AUDIO_DATA = new Uint8Array(256 + 300 + 280 + 310 + 290); // 1436 bytes + for (let i = 0; i < AUDIO_DATA.length; i++) AUDIO_DATA[i] = i & 0xFF; + + let m4a: Uint8Array; + let parsed: M4AInfo; + + beforeAll(() => { + // Use the production muxer to create the test file — its correctness is + // verified independently by the buildM4AFile roundtrip tests below. + m4a = buildM4AFile(FTYP, STSD, TIMESCALE, SAMPLE_SIZES, STTS_ENTRIES, AUDIO_DATA); + parsed = parseM4A(m4a); + }); + + it('extracts correct timescale', () => { + expect(parsed.timescale).toBe(TIMESCALE); + }); + + it('extracts correct sample sizes', () => { + expect(parsed.sampleSizes).toEqual(SAMPLE_SIZES); + }); + + it('extracts correct stts entries', () => { + expect(parsed.sampleDeltas).toEqual(STTS_ENTRIES); + }); + + it('extracts ftyp matching the original', () => { + expect(Array.from(parsed.ftyp)).toEqual(Array.from(FTYP)); + }); + + it('extracts stsd matching the original', () => { + expect(Array.from(parsed.stsd)).toEqual(Array.from(STSD)); + }); + + it('computes sequential per-sample offsets consistent with sample sizes', () => { + expect(parsed.sampleOffsets).toHaveLength(5); + for (let i = 1; i < parsed.sampleOffsets.length; i++) { + expect(parsed.sampleOffsets[i]).toBe(parsed.sampleOffsets[i - 1] + SAMPLE_SIZES[i - 1]); + } + }); + + it('audio data at parsed offsets matches the original', () => { + let expectedOff = 0; + for (let i = 0; i < SAMPLE_SIZES.length; i++) { + const fileOff = parsed.sampleOffsets[i]; + const sz = SAMPLE_SIZES[i]; + const actual = Array.from(m4a.slice(fileOff, fileOff + sz)); + const expected = Array.from(AUDIO_DATA.slice(expectedOff, expectedOff + sz)); + expect(actual).toEqual(expected); + expectedOff += sz; + } + }); +}); + +describe('buildM4AFile roundtrip', () => { + const FTYP = makeFtyp(); + const STSD = makeStsd(); + const TIMESCALE = 44100; + const SAMPLE_SIZES = [256, 300, 280, 310, 290, 320, 270, 305, 295, 260]; + const STTS_ENTRIES = [{ count: 10, delta: 1024 }]; + const totalAudioBytes = SAMPLE_SIZES.reduce((s, sz) => s + sz, 0); + const AUDIO_DATA = new Uint8Array(totalAudioBytes); + for (let i = 0; i < totalAudioBytes; i++) AUDIO_DATA[i] = (i * 7 + 13) & 0xFF; + + it('produces a file that parseM4A can parse with matching values', () => { + const built = buildM4AFile(FTYP, STSD, TIMESCALE, SAMPLE_SIZES, STTS_ENTRIES, AUDIO_DATA); + const parsed = parseM4A(built); + + expect(parsed.timescale).toBe(TIMESCALE); + expect(parsed.sampleSizes).toEqual(SAMPLE_SIZES); + expect(parsed.sampleDeltas).toEqual(STTS_ENTRIES); + expect(Array.from(parsed.ftyp)).toEqual(Array.from(FTYP)); + expect(Array.from(parsed.stsd)).toEqual(Array.from(STSD)); + }); + + it('stco offset points to the correct audio data position', () => { + const built = buildM4AFile(FTYP, STSD, TIMESCALE, SAMPLE_SIZES, STTS_ENTRIES, AUDIO_DATA); + const parsed = parseM4A(built); + + // Read audio data from the file at the parsed offsets + let srcOff = 0; + for (let i = 0; i < SAMPLE_SIZES.length; i++) { + const fileOff = parsed.sampleOffsets[i]; + const sz = SAMPLE_SIZES[i]; + const fromFile = Array.from(built.slice(fileOff, fileOff + sz)); + const fromOriginal = Array.from(AUDIO_DATA.slice(srcOff, srcOff + sz)); + expect(fromFile).toEqual(fromOriginal); + srcOff += sz; + } + }); + + it('double roundtrip: build → parse → build → parse gives identical results', () => { + const built1 = buildM4AFile(FTYP, STSD, TIMESCALE, SAMPLE_SIZES, STTS_ENTRIES, AUDIO_DATA); + const parsed1 = parseM4A(built1); + + // Reconstruct audio data from parsed offsets + const audioFromFile = new Uint8Array(totalAudioBytes); + let dst = 0; + for (let i = 0; i < parsed1.sampleSizes.length; i++) { + const off = parsed1.sampleOffsets[i]; + const sz = parsed1.sampleSizes[i]; + audioFromFile.set(built1.subarray(off, off + sz), dst); + dst += sz; + } + + const built2 = buildM4AFile(parsed1.ftyp, parsed1.stsd, parsed1.timescale, parsed1.sampleSizes, parsed1.sampleDeltas, audioFromFile); + const parsed2 = parseM4A(built2); + + expect(parsed2.timescale).toBe(parsed1.timescale); + expect(parsed2.sampleSizes).toEqual(parsed1.sampleSizes); + expect(parsed2.sampleDeltas).toEqual(parsed1.sampleDeltas); + expect(parsed2.sampleOffsets.length).toBe(parsed1.sampleOffsets.length); + }); +}); + +describe('sttsForRange', () => { + const ENTRIES = [ + { count: 100, delta: 1024 }, + { count: 50, delta: 960 }, + ]; + + it('returns full first entry for range [0, 100)', () => { + expect(sttsForRange(ENTRIES, 0, 100)).toEqual([{ count: 100, delta: 1024 }]); + }); + + it('returns partial first entry for range [10, 50)', () => { + expect(sttsForRange(ENTRIES, 10, 50)).toEqual([{ count: 40, delta: 1024 }]); + }); + + it('spans both entries for range [90, 120)', () => { + expect(sttsForRange(ENTRIES, 90, 120)).toEqual([ + { count: 10, delta: 1024 }, + { count: 20, delta: 960 }, + ]); + }); + + it('returns full second entry for range [100, 150)', () => { + expect(sttsForRange(ENTRIES, 100, 150)).toEqual([{ count: 50, delta: 960 }]); + }); +}); diff --git a/services/audio-splitter.ts b/services/audio-splitter.ts index 1def0d6..12b4f04 100644 --- a/services/audio-splitter.ts +++ b/services/audio-splitter.ts @@ -5,7 +5,6 @@ * Native: Parse M4A (MP4) container → split at sample boundaries → valid M4A chunks */ -import { Platform } from 'react-native'; import * as FileSystem from 'expo-file-system/legacy'; const CHUNK_BYTES = 24 * 1024 * 1024; // 24 MB per chunk (1 MB safety margin) @@ -37,13 +36,13 @@ function encodeWav(pcm: Float32Array, sampleRate: number): Blob { v.setUint32(4, 36 + n * 2, true); w(8, 'WAVE'); w(12, 'fmt '); - v.setUint32(16, 16, true); // fmt chunk size - v.setUint16(20, 1, true); // PCM - v.setUint16(22, 1, true); // mono - v.setUint32(24, sampleRate, true); // sample rate - v.setUint32(28, sampleRate * 2, true); // byte rate (mono, 16-bit) - v.setUint16(32, 2, true); // block align - v.setUint16(34, 16, true); // bits per sample + v.setUint32(16, 16, true); + v.setUint16(20, 1, true); + v.setUint16(22, 1, true); + v.setUint32(24, sampleRate, true); + v.setUint32(28, sampleRate * 2, true); + v.setUint16(32, 2, true); + v.setUint16(34, 16, true); w(36, 'data'); v.setUint32(40, n * 2, true); @@ -71,7 +70,6 @@ export async function splitAudioWeb(blob: Blob): Promise { for (let i = 0; i < len; i++) mono[i] += chData[i] / nCh; } - // Max samples per chunk: (CHUNK_BYTES - 44 byte WAV header) / 2 bytes per sample const samplesPerChunk = Math.floor((CHUNK_BYTES - 44) / 2); const blobs: Blob[] = []; for (let off = 0; off < len; off += samplesPerChunk) { @@ -85,13 +83,14 @@ export async function splitAudioWeb(blob: Blob): Promise { // ─── Native: M4A (MP4) parse & mux ────────────────────────────────────────── -interface M4AInfo { +export interface M4AInfo { ftyp: Uint8Array; timescale: number; stsd: Uint8Array; // complete stsd atom (with header) sampleSizes: number[]; sampleDeltas: { count: number; delta: number }[]; - audioDataOffset: number; // file-level offset of first audio sample + /** File-level byte offset of each audio sample (computed from stco + stsc + stsz). */ + sampleOffsets: number[]; } const CONTAINERS = new Set(['moov', 'trak', 'mdia', 'minf', 'stbl', 'dinf', 'udta']); @@ -100,9 +99,19 @@ function readType(d: Uint8Array, pos: number): string { return String.fromCharCode(d[pos], d[pos + 1], d[pos + 2], d[pos + 3]); } -function parseM4A(data: Uint8Array): M4AInfo { +/** + * Parse an M4A file and extract the metadata needed for splitting. + * Computes per-sample file offsets from stco + stsc + stsz. + */ +export function parseM4A(data: Uint8Array): M4AInfo { const view = new DataView(data.buffer, data.byteOffset, data.byteLength); - const info: Partial = {}; + let ftyp: Uint8Array | undefined; + let timescale: number | undefined; + let stsd: Uint8Array | undefined; + let sampleSizes: number[] | undefined; + let sampleDeltas: { count: number; delta: number }[] | undefined; + let chunkOffsets: number[] | undefined; + let stscEntries: { firstChunk: number; samplesPerChunk: number; descIdx: number }[] | undefined; const walk = (start: number, end: number) => { let pos = start; @@ -112,119 +121,153 @@ function parseM4A(data: Uint8Array): M4AInfo { let headerSize = 8; if (size === 1 && pos + 16 <= end) { - // 64-bit extended size const hi = view.getUint32(pos + 8); const lo = view.getUint32(pos + 12); size = hi * 0x100000000 + lo; headerSize = 16; } else if (size === 0) { - size = end - pos; // atom extends to end of parent + size = end - pos; } if (size < headerSize || pos + size > end) break; - const dataStart = pos + headerSize; + const ds = pos + headerSize; // data start const atomEnd = pos + size; switch (type) { case 'ftyp': - info.ftyp = data.slice(pos, atomEnd); + ftyp = data.slice(pos, atomEnd); break; case 'mdhd': { - // version(1)+flags(3) then fields differ by version - const ver = data[dataStart]; - // v0: create(4)+mod(4)+timescale(4) → timescale at +12 - // v1: create(8)+mod(8)+timescale(4) → timescale at +20 - const tsOff = ver === 0 ? 12 : 20; - info.timescale = view.getUint32(dataStart + tsOff); + const ver = data[ds]; + // v0: ver+flags(4) + create(4) + mod(4) + timescale(4) + // v1: ver+flags(4) + create(8) + mod(8) + timescale(4) + timescale = view.getUint32(ds + (ver === 0 ? 12 : 20)); break; } case 'stsd': - info.stsd = data.slice(pos, atomEnd); + stsd = data.slice(pos, atomEnd); break; case 'stsz': { - // version+flags(4), sample_size(4), sample_count(4), entries… - const constantSize = view.getUint32(dataStart + 4); - const count = view.getUint32(dataStart + 8); + const constSz = view.getUint32(ds + 4); + const count = view.getUint32(ds + 8); const sizes: number[] = []; - if (constantSize === 0) { - for (let i = 0; i < count; i++) sizes.push(view.getUint32(dataStart + 12 + i * 4)); + if (constSz === 0) { + for (let i = 0; i < count; i++) sizes.push(view.getUint32(ds + 12 + i * 4)); } else { - for (let i = 0; i < count; i++) sizes.push(constantSize); + for (let i = 0; i < count; i++) sizes.push(constSz); } - info.sampleSizes = sizes; + sampleSizes = sizes; break; } case 'stts': { - // version+flags(4), entry_count(4), entries(8 each) - const entryCount = view.getUint32(dataStart + 4); - const deltas: { count: number; delta: number }[] = []; - for (let i = 0; i < entryCount; i++) { - deltas.push({ - count: view.getUint32(dataStart + 8 + i * 8), - delta: view.getUint32(dataStart + 12 + i * 8), - }); + const ec = view.getUint32(ds + 4); + const d: { count: number; delta: number }[] = []; + for (let i = 0; i < ec; i++) { + d.push({ count: view.getUint32(ds + 8 + i * 8), delta: view.getUint32(ds + 12 + i * 8) }); } - info.sampleDeltas = deltas; + sampleDeltas = d; break; } case 'stco': { - // version+flags(4), entry_count(4), offsets(4 each) - const n = view.getUint32(dataStart + 4); - if (n > 0) info.audioDataOffset = view.getUint32(dataStart + 8); + const ec = view.getUint32(ds + 4); + chunkOffsets = []; + for (let i = 0; i < ec; i++) chunkOffsets.push(view.getUint32(ds + 8 + i * 4)); break; } case 'co64': { - const n = view.getUint32(dataStart + 4); - if (n > 0) { - const hi = view.getUint32(dataStart + 8); - const lo = view.getUint32(dataStart + 12); - info.audioDataOffset = hi * 0x100000000 + lo; + const ec = view.getUint32(ds + 4); + chunkOffsets = []; + for (let i = 0; i < ec; i++) { + const hi = view.getUint32(ds + 8 + i * 8); + const lo = view.getUint32(ds + 12 + i * 8); + chunkOffsets.push(hi * 0x100000000 + lo); + } + break; + } + + case 'stsc': { + const ec = view.getUint32(ds + 4); + stscEntries = []; + for (let i = 0; i < ec; i++) { + stscEntries.push({ + firstChunk: view.getUint32(ds + 8 + i * 12), + samplesPerChunk: view.getUint32(ds + 12 + i * 12), + descIdx: view.getUint32(ds + 16 + i * 12), + }); } break; } default: - if (CONTAINERS.has(type)) walk(dataStart, atomEnd); + if (CONTAINERS.has(type)) walk(ds, atomEnd); break; } - pos = atomEnd; } }; walk(0, data.length); - if (!info.ftyp) throw new Error('Invalid M4A: missing ftyp atom'); - if (info.timescale == null) throw new Error('Invalid M4A: missing mdhd atom'); - if (!info.stsd) throw new Error('Invalid M4A: missing stsd atom'); - if (!info.sampleSizes) throw new Error('Invalid M4A: missing stsz atom'); - if (!info.sampleDeltas) throw new Error('Invalid M4A: missing stts atom'); - if (info.audioDataOffset == null) throw new Error('Invalid M4A: missing stco/co64 atom'); + if (!ftyp) throw new Error('Invalid M4A: missing ftyp atom'); + if (timescale == null) throw new Error('Invalid M4A: missing mdhd atom'); + if (!stsd) throw new Error('Invalid M4A: missing stsd atom'); + if (!sampleSizes) throw new Error('Invalid M4A: missing stsz atom'); + if (!sampleDeltas) throw new Error('Invalid M4A: missing stts atom'); + if (!chunkOffsets) throw new Error('Invalid M4A: missing stco/co64 atom'); + if (!stscEntries) throw new Error('Invalid M4A: missing stsc atom'); + + // Compute per-sample file offsets from stco + stsc + stsz + const sampleOffsets = computeSampleOffsets(sampleSizes, stscEntries, chunkOffsets); - return info as M4AInfo; + return { ftyp, timescale, stsd, sampleSizes, sampleDeltas, sampleOffsets }; } -// ─── M4A muxer helpers ────────────────────────────────────────────────────── - -function buildAtom(type: string, payload: Uint8Array): Uint8Array { - const atom = new Uint8Array(8 + payload.length); - new DataView(atom.buffer).setUint32(0, 8 + payload.length); - atom[4] = type.charCodeAt(0); - atom[5] = type.charCodeAt(1); - atom[6] = type.charCodeAt(2); - atom[7] = type.charCodeAt(3); - atom.set(payload, 8); - return atom; +/** + * Map each sample to its file-level byte offset using stsc + stco + stsz. + * + * stsc tells us how many samples are in each chunk (from stco). + * Within a chunk, samples are contiguous, so we accumulate sizes. + */ +function computeSampleOffsets( + sampleSizes: number[], + stscEntries: { firstChunk: number; samplesPerChunk: number; descIdx: number }[], + chunkOffsets: number[], +): number[] { + const offsets: number[] = []; + let sampleIdx = 0; + + for (let chunkIdx = 0; chunkIdx < chunkOffsets.length && sampleIdx < sampleSizes.length; chunkIdx++) { + // Find how many samples are in this chunk (stsc uses 1-based chunk indices) + let samplesInChunk = stscEntries[0].samplesPerChunk; + for (const entry of stscEntries) { + if (entry.firstChunk - 1 <= chunkIdx) { + samplesInChunk = entry.samplesPerChunk; + } else { + break; + } + } + + let byteOff = chunkOffsets[chunkIdx]; + for (let s = 0; s < samplesInChunk && sampleIdx < sampleSizes.length; s++) { + offsets.push(byteOff); + byteOff += sampleSizes[sampleIdx]; + sampleIdx++; + } + } + + return offsets; } +// ─── stts range helper ─────────────────────────────────────────────────────── + /** Extract stts entries for samples [startSample, endSample). */ -function sttsForRange( +export function sttsForRange( allDeltas: { count: number; delta: number }[], startSample: number, endSample: number, @@ -248,12 +291,18 @@ function sttsForRange( return result; } +// ─── M4A muxer (sequential write, no patching) ────────────────────────────── + /** * Build a minimal valid M4A file containing the given audio samples. * - * Atom layout: ftyp | moov (mvhd, trak(tkhd, mdia(mdhd, hdlr, minf(smhd, dinf, stbl)))) | mdat + * All atom sizes are pre-calculated and the file is written sequentially into + * a single pre-allocated buffer. The stco offset is computed directly — no + * byte-search or patching is needed. + * + * Atom layout: ftyp | moov(mvhd, trak(tkhd, mdia(mdhd, hdlr, minf(smhd, dinf(dref), stbl(stsd,stts,stsz,stsc,stco))))) | mdat */ -function buildM4AFile( +export function buildM4AFile( ftyp: Uint8Array, stsd: Uint8Array, timescale: number, @@ -264,132 +313,176 @@ function buildM4AFile( const n = sampleSizes.length; const duration = sttsEntries.reduce((s, e) => s + e.count * e.delta, 0); - // ── stbl children ── + // ── Pre-calculate every atom size (content + 8-byte header) ── - // stts: version+flags(4) + entry_count(4) + entries(8 each) - const sttsP = new Uint8Array(4 + 4 + sttsEntries.length * 8); - const sttsV = new DataView(sttsP.buffer); - sttsV.setUint32(4, sttsEntries.length); - sttsEntries.forEach((e, i) => { - sttsV.setUint32(8 + i * 8, e.count); - sttsV.setUint32(12 + i * 8, e.delta); - }); + const sttsSize = 8 + 4 + 4 + sttsEntries.length * 8; + const stszSize = 8 + 4 + 4 + 4 + n * 4; + const stscSize = 8 + 4 + 4 + 12; // one entry + const stcoSize = 8 + 4 + 4 + 4; // one entry + const stblSize = 8 + stsd.length + sttsSize + stszSize + stscSize + stcoSize; + + const smhdSize = 8 + 8; + const urlSize = 8 + 4; + const drefSize = 8 + 4 + 4 + urlSize; + const dinfSize = 8 + drefSize; + const minfSize = 8 + smhdSize + dinfSize + stblSize; + + const mdhdSize = 8 + 24; + const HANDLER_NAME = [83,111,117,110,100,72,97,110,100,108,101,114,0]; // 'SoundHandler\0' + const hdlrSize = 8 + 4 + 4 + 4 + 12 + HANDLER_NAME.length; + const mdiaSize = 8 + mdhdSize + hdlrSize + minfSize; + + const tkhdSize = 8 + 84; + const trakSize = 8 + tkhdSize + mdiaSize; + + const mvhdSize = 8 + 100; + const moovSize = 8 + mvhdSize + trakSize; + + const mdatSize = 8 + audioData.length; + const totalSize = ftyp.length + moovSize + mdatSize; - // stsz: version+flags(4) + sample_size(4) + count(4) + entries(4 each) - const stszP = new Uint8Array(4 + 4 + 4 + n * 4); - const stszV = new DataView(stszP.buffer); - stszV.setUint32(8, n); - sampleSizes.forEach((sz, i) => stszV.setUint32(12 + i * 4, sz)); - - // stsc: version+flags(4) + entry_count(4) + one entry(12) - const stscP = new Uint8Array(4 + 4 + 12); - const stscV = new DataView(stscP.buffer); - stscV.setUint32(4, 1); - stscV.setUint32(8, 1); // first_chunk (1-based) - stscV.setUint32(12, n); // samples_per_chunk - stscV.setUint32(16, 1); // sample_description_index - - // stco: version+flags(4) + entry_count(4) + offset(4) — placeholder, patched later - const stcoP = new Uint8Array(4 + 4 + 4); - new DataView(stcoP.buffer).setUint32(4, 1); - - const stbl = buildAtom('stbl', concatBytes( - stsd, - buildAtom('stts', sttsP), - buildAtom('stsz', stszP), - buildAtom('stsc', stscP), - buildAtom('stco', stcoP), - )); - - // ── minf ── - - const smhd = buildAtom('smhd', new Uint8Array(8)); // version+flags(4) + balance(2) + reserved(2) - - const urlP = new Uint8Array(4); - urlP[3] = 1; // self-contained flag - const drefP = new Uint8Array(8 + 12); // version+flags(4) + count(4) + url atom(12) - new DataView(drefP.buffer).setUint32(4, 1); - drefP.set(buildAtom('url ', urlP), 8); - - const minf = buildAtom('minf', concatBytes( - smhd, - buildAtom('dinf', buildAtom('dref', drefP)), - stbl, - )); - - // ── mdia ── - - // mdhd v0: version+flags(4) + create(4) + mod(4) + timescale(4) + duration(4) + lang(2) + pre(2) = 24 - const mdhdP = new Uint8Array(24); - const mdhdV = new DataView(mdhdP.buffer); - mdhdV.setUint32(12, timescale); - mdhdV.setUint32(16, duration); - mdhdP[20] = 0x55; mdhdP[21] = 0xC4; // undetermined language - - // hdlr: version+flags(4) + pre_defined(4) + handler_type(4) + reserved(12) + name - const handlerName = new TextEncoder().encode('SoundHandler\0'); - const hdlrP = new Uint8Array(4 + 4 + 4 + 12 + handlerName.length); - // handler_type = 'soun' at offset 8 - hdlrP[8] = 0x73; hdlrP[9] = 0x6F; hdlrP[10] = 0x75; hdlrP[11] = 0x6E; - hdlrP.set(handlerName, 24); - - const mdia = buildAtom('mdia', concatBytes( - buildAtom('mdhd', mdhdP), - buildAtom('hdlr', hdlrP), - minf, - )); - - // ── trak ── - - // tkhd v0: version+flags(4) + create(4) + mod(4) + trackID(4) + reserved(4) - // + duration(4) + reserved(8) + layer(2) + altGroup(2) + volume(2) - // + reserved(2) + matrix(36) + width(4) + height(4) = 84 - const tkhdP = new Uint8Array(84); - const tkhdV = new DataView(tkhdP.buffer); - tkhdP[3] = 3; // flags: enabled + in_movie - tkhdV.setUint32(12, 1); // track_ID - tkhdV.setUint32(20, duration); // duration - tkhdV.setUint16(36, 0x0100); // volume = 1.0 - tkhdV.setUint32(40, 0x00010000); // matrix[0] - tkhdV.setUint32(56, 0x00010000); // matrix[4] - tkhdV.setUint32(72, 0x40000000); // matrix[8] - - const trak = buildAtom('trak', concatBytes(buildAtom('tkhd', tkhdP), mdia)); + // stco offset: audio data starts right after ftyp + moov + mdat header (8 bytes) + const audioOffset = ftyp.length + moovSize + 8; + + // ── Allocate and write sequentially ── + + const out = new Uint8Array(totalSize); + const dv = new DataView(out.buffer); + let p = 0; + + // Helper: write atom header (size + fourCC) + const hdr = (size: number, type: string) => { + dv.setUint32(p, size); p += 4; + out[p++] = type.charCodeAt(0); + out[p++] = type.charCodeAt(1); + out[p++] = type.charCodeAt(2); + out[p++] = type.charCodeAt(3); + }; + + // ── ftyp ── + out.set(ftyp, p); p += ftyp.length; // ── moov ── + hdr(moovSize, 'moov'); + + // mvhd (version 0, 100-byte payload) + hdr(mvhdSize, 'mvhd'); + p += 4; // version+flags + p += 8; // creation_time + modification_time + dv.setUint32(p, timescale); p += 4; // timescale + dv.setUint32(p, duration); p += 4; // duration + dv.setUint32(p, 0x00010000); p += 4; // rate = 1.0 + dv.setUint16(p, 0x0100); p += 2; // volume = 1.0 + p += 10; // reserved + // identity matrix (36 bytes) + dv.setUint32(p, 0x00010000); p += 4; // a + p += 12; // b, u, c + dv.setUint32(p, 0x00010000); p += 4; // d + p += 12; // v, tx, ty + dv.setUint32(p, 0x40000000); p += 4; // w + p += 24; // pre_defined + dv.setUint32(p, 2); p += 4; // next_track_id + + // trak + hdr(trakSize, 'trak'); + + // tkhd (version 0, 84-byte payload) + hdr(tkhdSize, 'tkhd'); + out[p + 3] = 3; p += 4; // version=0, flags=3 (enabled+in_movie) + p += 8; // creation + modification + dv.setUint32(p, 1); p += 4; // track_ID + p += 4; // reserved + dv.setUint32(p, duration); p += 4; // duration + p += 8; // reserved + p += 4; // layer + alternate_group + dv.setUint16(p, 0x0100); p += 2; // volume + p += 2; // reserved + dv.setUint32(p, 0x00010000); p += 4; // matrix[0] + p += 12; + dv.setUint32(p, 0x00010000); p += 4; // matrix[4] + p += 12; + dv.setUint32(p, 0x40000000); p += 4; // matrix[8] + p += 8; // width + height + + // mdia + hdr(mdiaSize, 'mdia'); + + // mdhd (version 0, 24-byte payload) + hdr(mdhdSize, 'mdhd'); + p += 4; // version+flags + p += 8; // creation + modification + dv.setUint32(p, timescale); p += 4; // timescale + dv.setUint32(p, duration); p += 4; // duration + dv.setUint16(p, 0x55C4); p += 2; // language = 'und' + p += 2; // pre_defined + + // hdlr + hdr(hdlrSize, 'hdlr'); + p += 4; // version+flags + p += 4; // pre_defined + out[p] = 0x73; out[p+1] = 0x6F; out[p+2] = 0x75; out[p+3] = 0x6E; p += 4; // 'soun' + p += 12; // reserved + for (const b of HANDLER_NAME) out[p++] = b; + + // minf + hdr(minfSize, 'minf'); + + // smhd + hdr(smhdSize, 'smhd'); + p += 8; // version+flags + balance + reserved + + // dinf + hdr(dinfSize, 'dinf'); + + // dref + hdr(drefSize, 'dref'); + p += 4; // version+flags + dv.setUint32(p, 1); p += 4; // entry_count + // url (self-contained) + hdr(urlSize, 'url '); + out[p + 3] = 1; p += 4; // version=0, flags=1 (self-contained) + + // stbl + hdr(stblSize, 'stbl'); + + // stsd (copy from original) + out.set(stsd, p); p += stsd.length; + + // stts + hdr(sttsSize, 'stts'); + p += 4; // version+flags + dv.setUint32(p, sttsEntries.length); p += 4; + for (const e of sttsEntries) { + dv.setUint32(p, e.count); p += 4; + dv.setUint32(p, e.delta); p += 4; + } - // mvhd v0: version+flags(4) + create(4) + mod(4) + timescale(4) + duration(4) - // + rate(4) + volume(2) + reserved(10) + matrix(36) + pre_defined(24) - // + next_track_id(4) = 100 - const mvhdP = new Uint8Array(100); - const mvhdV = new DataView(mvhdP.buffer); - mvhdV.setUint32(12, timescale); // timescale - mvhdV.setUint32(16, duration); // duration - mvhdV.setUint32(20, 0x00010000); // rate = 1.0 - mvhdV.setUint16(24, 0x0100); // volume = 1.0 - mvhdV.setUint32(36, 0x00010000); // matrix[0] - mvhdV.setUint32(52, 0x00010000); // matrix[4] - mvhdV.setUint32(68, 0x40000000); // matrix[8] - mvhdV.setUint32(96, 2); // next_track_id - - const moov = buildAtom('moov', concatBytes(buildAtom('mvhd', mvhdP), trak)); + // stsz + hdr(stszSize, 'stsz'); + p += 4; // version+flags + p += 4; // sample_size = 0 (variable) + dv.setUint32(p, n); p += 4; // sample_count + for (const sz of sampleSizes) { dv.setUint32(p, sz); p += 4; } + + // stsc (one entry: all samples in one chunk) + hdr(stscSize, 'stsc'); + p += 4; // version+flags + dv.setUint32(p, 1); p += 4; // entry_count + dv.setUint32(p, 1); p += 4; // first_chunk + dv.setUint32(p, n); p += 4; // samples_per_chunk + dv.setUint32(p, 1); p += 4; // sample_description_index + + // stco (one entry: offset to audio data in mdat) + hdr(stcoSize, 'stco'); + p += 4; // version+flags + dv.setUint32(p, 1); p += 4; // entry_count + dv.setUint32(p, audioOffset); p += 4; // chunk_offset — pre-calculated, no patching! // ── mdat ── - const mdat = buildAtom('mdat', audioData); - - // ── Patch stco offset ── - // Audio data starts at: ftyp.length + moov.length + 8 (mdat atom header) - const audioOffset = ftyp.length + moov.length + 8; - // Find 'stco' atom in moov and write the offset - for (let i = 0; i < moov.length - 20; i++) { - if (moov[i + 4] === 0x73 && moov[i + 5] === 0x74 && - moov[i + 6] === 0x63 && moov[i + 7] === 0x6F) { // 'stco' - new DataView(moov.buffer, moov.byteOffset).setUint32(i + 16, audioOffset); - break; - } - } + hdr(mdatSize, 'mdat'); + out.set(audioData, p); p += audioData.length; - return concatBytes(ftyp, moov, mdat); + return out; } // ─── Native: split M4A into valid chunk files ──────────────────────────────── @@ -417,7 +510,7 @@ export async function splitAudioNative(localUri: string): Promise { const data = base64ToBytes(base64); const info = parseM4A(data); - // Group samples into chunks of ≤ CHUNK_BYTES of audio data + // Group samples into chunks of ≤ CHUNK_BYTES of raw audio data const groups: { start: number; end: number }[] = []; let grpStart = 0; let grpSize = 0; @@ -431,19 +524,38 @@ export async function splitAudioNative(localUri: string): Promise { } if (grpSize > 0) groups.push({ start: grpStart, end: info.sampleSizes.length }); - // Build a valid M4A file for each group and write to a temp file + // Build a valid M4A file for each group const tempUris: string[] = []; - let dataOffset = info.audioDataOffset; for (let g = 0; g < groups.length; g++) { const { start, end } = groups[g]; - const chunkSizes = info.sampleSizes.slice(start, end); - const chunkDataSize = chunkSizes.reduce((s, sz) => s + sz, 0); - const audioBytes = data.slice(dataOffset, dataOffset + chunkDataSize); - dataOffset += chunkDataSize; + const chunkSampleSizes = info.sampleSizes.slice(start, end); + + // Gather audio bytes using per-sample offsets (handles non-contiguous layouts) + const firstOff = info.sampleOffsets[start]; + const lastOff = info.sampleOffsets[end - 1]; + const lastSize = info.sampleSizes[end - 1]; + const totalSpan = lastOff + lastSize - firstOff; + const totalDataSize = chunkSampleSizes.reduce((s, sz) => s + sz, 0); + + let audioBytes: Uint8Array; + if (totalSpan === totalDataSize) { + // Contiguous — fast path (single slice) + audioBytes = data.slice(firstOff, firstOff + totalDataSize); + } else { + // Non-contiguous — gather per sample + audioBytes = new Uint8Array(totalDataSize); + let dst = 0; + for (let s = start; s < end; s++) { + const src = info.sampleOffsets[s]; + const sz = info.sampleSizes[s]; + audioBytes.set(data.subarray(src, src + sz), dst); + dst += sz; + } + } const entries = sttsForRange(info.sampleDeltas, start, end); - const m4aBytes = buildM4AFile(info.ftyp, info.stsd, info.timescale, chunkSizes, entries, audioBytes); + const m4aBytes = buildM4AFile(info.ftyp, info.stsd, info.timescale, chunkSampleSizes, entries, audioBytes); const tempUri = `${FileSystem.cacheDirectory}chunk-${Date.now()}-${g}.m4a`; await FileSystem.writeAsStringAsync(tempUri, bytesToBase64(m4aBytes), {