diff --git a/static/js/audioContext.js b/static/js/audioContext.js new file mode 100644 index 0000000..fa30c29 --- /dev/null +++ b/static/js/audioContext.js @@ -0,0 +1,50 @@ +// Creating the AudioContext playback runs in. +// +// An AudioContext built with no options runs at whatever rate the operating +// system's output device is set to, and every node downstream inherits it. That +// is the right default up to a point, and past that point it is pure waste: +// every stem StemDeck produces is 44.1 kHz (`-ar 44100` in the ffmpeg call in +// app/pipeline/runner.py), so a 192 kHz context upsamples on decode and then +// does several times the work on interpolated samples carrying nothing the +// originals did not (#578). +// +// The pitch chain's WSOLA search is O(overlap * seek) per sequence and both +// windows are set in milliseconds, so its cost per second of audio is quadratic +// in the context rate. One active chain, measured offline against the real +// processor: 2.1% of real time at 44.1 kHz, 2.4% at 48, 7.3% at 96 and 28.2% at +// 192. Decoded buffers scale linearly over the same range, from 606 MB to +// 2.6 GB for a five minute six stem track. +// +// So: a cap, not a pin. At 44.1 and 48 kHz the graph is left exactly as it is +// today, because there is nothing to gain and a needless resample to lose. Only +// a device asking for more resolution than the material has gets capped. +const MAX_GRAPH_RATE = 48000; + +/** + * An AudioContext at a sane rate for 44.1 kHz material. + * + * The device rate can only be read from a context, and a context's rate is + * fixed once it exists, so a throwaway one is opened to ask and closed again. + * If that fails, or the browser declines the rate we ask for, we fall back to + * the plain constructor: playing at the device's rate is the current behaviour + * and is never worse than not playing. + */ +export function createPlaybackContext(AudioCtx) { + if (!AudioCtx) return null; + let deviceRate = 0; + try { + const probe = new AudioCtx(); + deviceRate = probe.sampleRate; + // Nothing is ever routed into it, and close() is a promise we do not need + // to wait on. Safari rejects close() on a context that never started. + Promise.resolve(probe.close()).catch(() => {}); + } catch { + return new AudioCtx(); + } + if (!(deviceRate > MAX_GRAPH_RATE)) return new AudioCtx(); + try { + return new AudioCtx({ sampleRate: MAX_GRAPH_RATE }); + } catch { + return new AudioCtx(); + } +} diff --git a/static/js/audioEngine.js b/static/js/audioEngine.js index 33d3daf..3e4a21d 100644 --- a/static/js/audioEngine.js +++ b/static/js/audioEngine.js @@ -22,18 +22,26 @@ const AudioCtx = window.AudioContext || window.webkitAudioContext; import { INPUT_COUNT, ZERO_INPUT, clampPitch, effectivePitch, inputForPitch, } from "./pitchBus.js"; +import { createPlaybackContext } from "./audioContext.js"; export function createAudioEngine(stems, { onTime, onEnded, context } = {}) { // Mobile/iOS only starts audio from a context resumed inside a user gesture. // Callers can pass a shared, gesture-unlocked `context` (the mobile UI does); // desktop passes none and we own a fresh one. We only close contexts we own. - const ctx = context || new AudioCtx(); + const ctx = context || createPlaybackContext(AudioCtx); const ownsCtx = !context; const master = ctx.createGain(); // One bus per semitone the worklet offers. A lane's transpose is expressed // as which bus it is connected to, so several lanes can sit in different // keys at once while still sharing the worklet's single tempo stage. const buses = Array.from({ length: INPUT_COUNT }, () => ctx.createGain()); + // How many tracks sit on each bus. A pitch bus is wired into the worklet only + // while this is non-zero; the unpitched bus is wired for good. See the same + // field in chunkedAudioEngine.js for why: a bus wired with nothing playing + // into it still reaches the processor as a channel of silence, which it took + // as a lane to transpose, so twelve pitch chains ran on silence for every + // track at zero transpose (#576, #575). + const busLanes = new Array(INPUT_COUNT).fill(0); master.connect(ctx.destination); let _playbackRate = 1.0; @@ -57,7 +65,12 @@ export function createAudioEngine(stems, { onTime, onEnded, context } = {}) { // The worklet loads asynchronously, so anything set before it arrived // would otherwise be dropped. Re-apply the current value now. stNode.parameters.get('tempo').value = _playbackRate; - for (let k = 0; k < INPUT_COUNT; k++) buses[k].connect(stNode, 0, k); + // Tracks decoded before the worklet arrived are already routed, so + // wire whichever pitch buses they occupy. + buses[ZERO_INPUT].connect(stNode, 0, ZERO_INPUT); + for (let k = 0; k < INPUT_COUNT; k++) { + if (k !== ZERO_INPUT && busLanes[k] > 0) buses[k].connect(stNode, 0, k); + } stNode.connect(master); }).catch((err) => { console.warn('[audioEngine] SoundTouch worklet load failed, using tape-effect fallback:', err); @@ -314,6 +327,24 @@ export function createAudioEngine(stems, { onTime, onEnded, context } = {}) { const ROUTE_FADE = 0.006; const ROUTE_HOLD = 0.02; + // A track arriving on, or leaving, a bus. Mirrors chunkedAudioEngine.js: + // only the first arrival and the last departure touch the worklet, the + // unpitched bus stays wired whatever its count because the click is + // scheduled onto it directly, and arrival is announced before the track + // connects so the bus is live by the time anything reaches it. No worklet + // means the tape-effect fallback, where every bus already feeds master. + function _laneArriving(bus) { + const k = buses.indexOf(bus); + if (busLanes[k]++ === 0 && k !== ZERO_INPUT && stNode) buses[k].connect(stNode, 0, k); + } + function _laneLeft(bus) { + if (!bus) return; + const k = buses.indexOf(bus); + if (--busLanes[k] === 0 && k !== ZERO_INPUT && stNode) { + try { buses[k].disconnect(stNode, 0, k); } catch { /* worklet already gone */ } + } + } + /** Connect a track to the bus for its current transpose. */ function routeTrack(t, immediate = false) { if (t.visualOnly) return; @@ -321,9 +352,12 @@ export function createAudioEngine(stems, { onTime, onEnded, context } = {}) { if (t.bus === target) return; const swap = () => { if (destroyed) return; + const previous = t.bus; + _laneArriving(target); try { t.analyser.disconnect(); } catch { /* was not connected yet */ } t.analyser.connect(target); t.bus = target; + _laneLeft(previous); }; if (immediate || !playing) { swap(); return; } const g = t.gain.gain; diff --git a/static/js/chunkedAudioEngine.js b/static/js/chunkedAudioEngine.js index 0545b14..54ab72f 100644 --- a/static/js/chunkedAudioEngine.js +++ b/static/js/chunkedAudioEngine.js @@ -188,16 +188,36 @@ function _pcmToAudioBuffer(ctx, pcmData, header) { import { INPUT_COUNT, ZERO_INPUT, clampPitch, effectivePitch, inputForPitch, } from "./pitchBus.js"; +import { createPlaybackContext } from "./audioContext.js"; export function createChunkedAudioEngine(stems, { onTime, onEnded, context } = {}) { const AC = window.AudioContext || window.webkitAudioContext; - const ctx = context || new AC(); + const ctx = context || createPlaybackContext(AC); const ownsCtx = !context; const master = ctx.createGain(); // One bus per semitone the worklet offers. A lane's transpose is expressed // as which bus it is connected to, so several lanes can sit in different // keys at once while still sharing the worklet's single tempo stage. const buses = Array.from({ length: INPUT_COUNT }, () => ctx.createGain()); + // How many lanes sit on each bus. A pitch bus is wired into the worklet only + // while this is non-zero; the unpitched bus is wired for good. + // + // The processor decides whether a semitone is in use by whether its input has + // any channels, and takes its bypass path (a straight copy) only when none + // do. Wiring every bus up front defeated that: a bus with nothing playing + // into it still arrives as one channel of silence in Chrome, so the processor + // built a pitch chain for all twelve semitones and ran WSOLA, the anti-alias + // cascade and the resampler on silence for the whole track, at zero transpose + // (#576). + // + // The windows are set in milliseconds, so the wasted work per second of audio + // scales with the AudioContext's rate, which follows the system output device. + // Measured offline, seven stems, no transpose: 0.2% of real time with only + // the unpitched input wired against 17.8% with all thirteen at 44.1 kHz, 0.8% + // against 78% at 96 kHz, and over real time at 192 kHz. That is the crackle + // on a laptop and the silence at 192 kHz in #575. Connecting a bus only while + // a lane is on it hands the processor the empty input its contract describes. + const busLanes = new Array(INPUT_COUNT).fill(0); master.connect(ctx.destination); let stNode = null; @@ -218,7 +238,12 @@ export function createChunkedAudioEngine(stems, { onTime, onEnded, context } = { // The worklet loads asynchronously, so anything set before it arrived // would otherwise be dropped. Re-apply the current value now. stNode.parameters.get('tempo').value = _playbackRate; - for (let k = 0; k < INPUT_COUNT; k++) buses[k].connect(stNode, 0, k); + // Lanes were routed before the worklet arrived (see the stem loop + // below), so wire whichever pitch buses they already occupy. + buses[ZERO_INPUT].connect(stNode, 0, ZERO_INPUT); + for (let k = 0; k < INPUT_COUNT; k++) { + if (k !== ZERO_INPUT && busLanes[k] > 0) buses[k].connect(stNode, 0, k); + } stNode.connect(master); }).catch((err) => { console.warn('[chunkedEngine] SoundTouch worklet failed, tape-effect fallback:', err); @@ -320,15 +345,41 @@ export function createChunkedAudioEngine(stems, { onTime, onEnded, context } = { const ROUTE_FADE = 0.006; const ROUTE_HOLD = 0.02; + // A lane arriving on, or leaving, a bus. Only the first arrival and the last + // departure touch the worklet: the bus is wired in while occupied and taken + // out again once empty, so the processor sees exactly the inputs that carry a + // lane. The unpitched bus stays wired whatever its count, because the click is + // scheduled onto it directly. In the tape-effect fallback there is no worklet + // and every bus already feeds master, so there is nothing to do. + // + // Arrival is announced before the lane connects, so a bus is live in the + // worklet by the time anything reaches it, and a lane never lands on a bus + // that leads nowhere. A chain whose input goes empty is fed silence and kept + // for CHAIN_LINGER_BLOCKS by the processor, which is how its tail drains. + function _laneArriving(bus) { + const k = buses.indexOf(bus); + if (busLanes[k]++ === 0 && k !== ZERO_INPUT && stNode) buses[k].connect(stNode, 0, k); + } + function _laneLeft(bus) { + if (!bus) return; + const k = buses.indexOf(bus); + if (--busLanes[k] === 0 && k !== ZERO_INPUT && stNode) { + try { buses[k].disconnect(stNode, 0, k); } catch { /* worklet already gone */ } + } + } + /** Connect a stem to the bus for its current transpose. */ function routeStem(stem, immediate = false) { const target = buses[inputForPitch(effectivePitch(stem.name, stem.pitch, stem.pitchable))]; if (stem.bus === target) return; const swap = () => { if (destroyed) return; + const previous = stem.bus; + _laneArriving(target); try { stem.analyser.disconnect(); } catch { /* was not connected yet */ } stem.analyser.connect(target); stem.bus = target; + _laneLeft(previous); }; if (immediate || !playing) { swap(); return; } const g = stem.gain.gain; diff --git a/static/js/player.js b/static/js/player.js index f3bfec4..6e1877c 100644 --- a/static/js/player.js +++ b/static/js/player.js @@ -29,6 +29,7 @@ import { } from "./state.js"; import { createAudioEngine, estimateDecodedBytes } from "./audioEngine.js"; import { createChunkedAudioEngine } from "./chunkedAudioEngine.js"; +import { createPlaybackContext } from "./audioContext.js"; import { addVisualOnlyStems, buildPlaybackStems } from "./playbackStems.js"; import { vuLevel } from "./vuScale.js"; import { createMetronome } from "./metronome.js"; @@ -1701,7 +1702,10 @@ async function initFooterWaveform(stemUrl) { const AudioCtx = window.AudioContext || window.webkitAudioContext; if (!AudioCtx) return; try { - visualAudioContext ??= new AudioCtx(); + // Decode-only, for the footer's peaks. It never plays anything, so a + // 192 kHz device would cost it four times the buffer for four times the + // samples per peak bar and not one pixel of extra detail (#578). + visualAudioContext ??= createPlaybackContext(AudioCtx); const res = await fetch(stemUrl, { cache: "force-cache" }); if (!res.ok) return; const buf = await visualAudioContext.decodeAudioData(await res.arrayBuffer()); diff --git a/static/mobile/app.js b/static/mobile/app.js index e67401f..a7cc418 100644 --- a/static/mobile/app.js +++ b/static/mobile/app.js @@ -6,6 +6,7 @@ import { fetchJobs, jobToCard } from "../js/shared/jobs.js"; import { createChunkedAudioEngine } from "../js/chunkedAudioEngine.js"; import { PITCH_MAX, PITCH_MIN, clampPitch } from "../js/pitchBus.js"; +import { createPlaybackContext } from "../js/audioContext.js"; // Per-stem label + color, keyed by the backend stem name. Unknown names fall // back to a rotating palette so non-standard models still render sensibly. const STEM_META = { @@ -48,7 +49,7 @@ let audioCtx = null; function ensureAudioCtx() { const AC = window.AudioContext || window.webkitAudioContext; if (!AC) return null; - if (!audioCtx) audioCtx = new AC(); + if (!audioCtx) audioCtx = createPlaybackContext(AC); if (audioCtx.state === "suspended") audioCtx.resume().catch(() => {}); return audioCtx; } diff --git a/tests/js/audio-context.test.mjs b/tests/js/audio-context.test.mjs new file mode 100644 index 0000000..3956186 --- /dev/null +++ b/tests/js/audio-context.test.mjs @@ -0,0 +1,103 @@ +// The rate the audio graph runs at. +// +// Every stem is 44.1 kHz, so a context inheriting a 192 kHz output device does +// several times the DSP and holds four times the buffers for nothing (#578). +// These checks pin down that the cap only ever fires above 48 kHz, and that a +// browser refusing the requested rate still gets a working context. +import { createPlaybackContext } from '../../static/js/audioContext.js'; + +let passed = 0; +let failed = 0; + +function check(name, condition, detail = '') { + if (condition) { + passed++; + console.log(`PASS ${name}`); + } else { + failed++; + console.log(`FAIL ${name}${detail ? ` -- ${detail}` : ''}`); + } +} + +/** A stand-in AudioContext that records how each instance was constructed. */ +function fakeCtor(deviceRate, { probeThrows = false, cappedThrows = false } = {}) { + const built = []; + class FakeCtx { + constructor(options) { + built.push(options); + if (probeThrows && built.length === 1) throw new Error('no audio device'); + if (cappedThrows && options?.sampleRate) throw new Error('rate not available'); + this.options = options; + // The real constructor grants the rate it is asked for, or the device's. + this.sampleRate = options?.sampleRate || deviceRate; + this.closed = false; + } + + close() { this.closed = true; return Promise.resolve(); } + } + return { FakeCtx, built }; +} + +// Below the cap the graph is left exactly as it is today. Pinning here would +// only add a resample that nothing asked for. +for (const rate of [44100, 48000]) { + const { FakeCtx, built } = fakeCtor(rate); + const ctx = createPlaybackContext(FakeCtx); + check( + `a ${rate} Hz device is left alone`, + built.length === 2 && built[1] === undefined && ctx.sampleRate === rate, + `built ${JSON.stringify(built)} at ${ctx.sampleRate}`, + ); +} + +// Above it, the device is asking for more resolution than the material has. +for (const rate of [88200, 96000, 176400, 192000]) { + const { FakeCtx, built } = fakeCtor(rate); + const ctx = createPlaybackContext(FakeCtx); + check( + `a ${rate} Hz device is capped to 48000`, + built[1]?.sampleRate === 48000 && ctx.sampleRate === 48000, + `built ${JSON.stringify(built)} at ${ctx.sampleRate}`, + ); +} + +{ + // The probe exists only to read the device rate, and nothing is ever routed + // into it. Leaving it open would hold a hardware stream open for the session. + const { FakeCtx } = fakeCtor(192000); + let probe = null; + const Wrapped = class extends FakeCtx { + constructor(options) { super(options); if (!probe) probe = this; } + }; + createPlaybackContext(Wrapped); + check('the probe context is closed again', probe?.closed === true); +} + +{ + // A browser that will not build a context at all is not a reason to give up + // on playback: the device's own rate is what we would have used anyway. + const { FakeCtx, built } = fakeCtor(192000, { probeThrows: true }); + const ctx = createPlaybackContext(FakeCtx); + check( + 'a failed probe falls back to the plain constructor', + ctx.sampleRate === 192000 && built.length === 2 && built[1] === undefined, + `built ${JSON.stringify(built)}`, + ); +} + +{ + // Safari has historically refused rates its device cannot produce. Playing at + // the device's rate is worse than capping and far better than not playing. + const { FakeCtx, built } = fakeCtor(192000, { cappedThrows: true }); + const ctx = createPlaybackContext(FakeCtx); + check( + 'a refused rate falls back to the plain constructor', + ctx.sampleRate === 192000 && built.length === 3 && built[2] === undefined, + `built ${JSON.stringify(built)}`, + ); +} + +check('no constructor means no context', createPlaybackContext(null) === null); + +console.log(`\n${passed} passed, ${failed} failed`); +if (failed) process.exit(1); diff --git a/tests/js/audio-routing.test.mjs b/tests/js/audio-routing.test.mjs index bed6af4..deee3a0 100644 --- a/tests/js/audio-routing.test.mjs +++ b/tests/js/audio-routing.test.mjs @@ -78,7 +78,18 @@ class FakeNode { return destination; } - disconnect() { this.connections = []; } + // Matches the real overloads closely enough to tell them apart: the engines + // drop a whole lane with `disconnect()` and take a single bus back out of the + // worklet with `disconnect(node, output, input)`. A mock that cleared + // everything either way would pass whichever one the code used. + disconnect(destination, output = 0, input = 0) { + if (destination === undefined) { this.connections = []; return; } + const before = this.connections.length; + this.connections = this.connections.filter((edge) => + !(edge.destination === destination && edge.output === output && edge.input === input)); + // The real API throws when asked to remove a connection that is not there. + if (this.connections.length === before) throw new Error('InvalidAccessError'); + } } class FakeGain extends FakeNode { @@ -243,6 +254,18 @@ async function verifyEngine(name, create) { }; check(`${name} exposes the unpitched bus for the metronome`, engine.getMasterNode() === bus(ZERO_INPUT)); + // Wiring a bus into the worklet is what tells the processor that semitone is + // in use, and a wired bus with nothing on it still reaches the processor as a + // channel of silence, which it builds a pitch chain for (#576). So at start + // only the unpitched bus may be wired, and the checks below watch pitch buses + // come and go with the lanes on them. + const wired = () => { + const out = []; + for (let k = 0; k < INPUT_COUNT; k++) if (bus(k)) out.push(k - ZERO_INPUT); + return out.join(','); + }; + check(`${name} wires only the unpitched bus while nothing is transposed`, wired() === '0', `wired ${wired()}`); + const originalAnalysers = engine.getAnalysers('original'); check(`${name} groups all complement analysers under original`, originalAnalysers.length === 2); const drumAnalyser = originalAnalysers[0]; @@ -266,6 +289,7 @@ async function verifyEngine(name, create) { check(`${name} puts a lane in the key it was given`, vocalBus === -2, `landed on ${vocalBus}`); check(`${name} reports the key it was given`, engine.getStemPitch('vocals') === -2); check(`${name} leaves other lanes where they were`, busOf(melodicAnalyser) === 0); + check(`${name} wires a bus when a lane arrives on it`, wired() === '-2,0', `wired ${wired()}`); // A key past the range the DSP is measured over stops at the edge rather // than landing somewhere unusable. @@ -274,6 +298,7 @@ async function verifyEngine(name, create) { `${name} clamps a lane to the offered range`, busOf(engine.getAnalysers('vocals')[0]) === 6, ); + check(`${name} unwires a bus once its last lane has left`, wired() === '0,6', `wired ${wired()}`); // One control drives both sources in the `original` group, and only one of // them is allowed to move. This is the whole reason the unpitched input @@ -291,6 +316,17 @@ async function verifyEngine(name, create) { ); check(`${name} reports drums as not pitchable`, engine.isStemPitchable('drums') === false); + // Two lanes share the +3 bus, so it must survive one of them leaving and go + // only when the other does too. The unpitched bus is never unwired: the click + // is scheduled onto it whether or not any lane sits there. + engine.setStemPitch('vocals', 3); + check(`${name} shares one bus between lanes in the same key`, wired() === '0,3', `wired ${wired()}`); + engine.setStemPitch('vocals', 0); + check(`${name} keeps a bus wired while another lane is still on it`, wired() === '0,3', `wired ${wired()}`); + engine.setStemPitch('original', 0); + check(`${name} unwires a shared bus only when it empties`, wired() === '0', `wired ${wired()}`); + engine.setStemPitch('original', 3); + engine.setStemPitch('vocals', 3); if (name === 'full-decode engine') {