diff --git a/local/interp.py b/local/interp.py index d8eed1d..42828e6 100644 --- a/local/interp.py +++ b/local/interp.py @@ -1,12 +1,18 @@ -"""Frame interpolation via RIFE (rife-ncnn-vulkan, Apple GPU through MoltenVK) — 2x a video's frame rate. +"""Frame interpolation via RIFE (rife-ncnn-vulkan, Apple GPU through MoltenVK) — Nx a video's frame rate. -Used to render Wan clips at HALF the frames (faster denoise + VAE) then interpolate back up to the target -fps with real motion. RIFE at 2x (small frame gap) is near-ground-truth on cinematic pans/walking. MIT -licensed (wrapper + weights). Falls back to nothing here — the engine handles the ffmpeg fallback. +Used to render Wan clips at a fraction of the timeline frames (faster denoise + VAE) then interpolate back +up with real motion. RIFE v4.x is timestep-conditioned, so an Nx pass synthesizes N-1 evenly spaced frames +per gap in ONE pass — near-ground-truth on cinematic pans/walking at these small gaps. MIT licensed +(wrapper + weights). Falls back to nothing here — the engine handles the ffmpeg fallback. -run_interpolate(req): {video, out, [model], [gpuid], [out_fps]} -> {ok, frames_in, frames_out}. Doubles the -frame count (a midpoint between each pair + the last frame repeated once = 2n frames), muxed at out_fps -(pass 2x the source fps) so the clip keeps exactly its source duration with real in-between motion. +run_interpolate(req): {video, out, [factor], [model], [gpuid], [out_fps]} -> {ok, frames_in, frames_out}. +Multiplies the frame count by `factor` (default 2, clamped to 2..4): N-1 midpoints between each pair, plus +the last frame repeated factor-1 times = n*factor frames, muxed at out_fps (pass factor x the source fps) +so the clip keeps exactly its source duration with real in-between motion. + +Why the caller may ask for 3x rather than 2x: the engine's timeline is 24fps and the 14B saves at 16fps. +2x lands on 32fps, which conforms to 24 by dropping 1 frame in 4 at uneven phase (visible cadence break); +3x lands on 48fps, an exact 2:1 decimation to 24. See rifeFactor() in src/engine/localVideo.ts. """ import os import shutil @@ -48,6 +54,9 @@ def run_interpolate(req: dict) -> dict: out = req["out"] model = req.get("model") or _default_model() gpuid = int(req.get("gpuid", 0)) + # Clamped: 1 would be a no-op pass, and past 4 the synthesized-to-real frame ratio stops buying + # smoothness while the per-clip cost keeps growing. + factor = max(2, min(4, int(req.get("factor", 2) or 2))) work = video + "_interp" # A previous FAILED/killed run leaves stale frames here; ffmpeg's image2 demuxer would happily append @@ -67,19 +76,23 @@ def run_interpolate(req: dict) -> dict: # No caching: the sidecar runs each job in a fresh subprocess (MoltenVK isolation), so nothing persists. rife = Rife(gpuid=gpuid, model=model, scale=2, width=w, height=h) - # interleave: f0, mid(f0,f1), f1, mid(f1,f2), f2, ..., fN, fN -> 2n frames. The final frame repeats - # once so the clip keeps EXACTLY its source duration at 2x fps (2n-1 frames would run 1/(2*fps) short - # per clip — enough to break the chained-total >= scene-window invariant and drift the timeline). + # interleave: f0, [factor-1 midpoints], f1, [factor-1 midpoints], f2, ..., fN, then fN repeated + # factor-1 times -> exactly n*factor frames. The tail repeat keeps the clip at EXACTLY its source + # duration at factor x fps (n*factor-(factor-1) frames would run short per clip — enough to break the + # chained-total >= scene-window invariant and drift the timeline). oi = 0 prev = first Image.open(os.path.join(fin, frames[0])).convert("RGB").save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 for i in range(1, n): cur = Image.open(os.path.join(fin, frames[i])).convert("RGB") - mid = rife.process(prev, cur, timestep=0.5) - mid.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 + for j in range(1, factor): + # RIFE v4.x takes an arbitrary timestep, so 3x is one pass with two midpoints per gap — not a + # recursive 2x-of-2x (which would interpolate already-synthesized frames and compound error). + rife.process(prev, cur, timestep=j / factor).save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 cur.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 prev = cur - prev.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 + for _ in range(factor - 1): + prev.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 fps = float(req.get("out_fps", 24)) subprocess.run([ diff --git a/src/engine/backends/local/video.ts b/src/engine/backends/local/video.ts index f07f8ab..d9886a3 100644 --- a/src/engine/backends/local/video.ts +++ b/src/engine/backends/local/video.ts @@ -6,7 +6,7 @@ // per native sub-clip, chained from the previous clip's real last frame; the clip is TRIMMED to the frame // grid (never stretched). See DUAL_BACKEND_PLAN.md §1.7. import { env, envInt, envBool } from '../../config'; -import { lastFrame, concatClips } from '../../ffmpeg'; +import { lastFrame, concatClips, matchLevels } from '../../ffmpeg'; import * as P from '../../stages'; import * as S from '../../storage'; import { genVideoLocal, localNativeFps, localMaxFrames } from '../../localVideo'; @@ -22,13 +22,14 @@ import { workers, kfWorkers, } from '../sceneShared'; +import { TIMELINE_RES } from '../../../shared/videoRes'; import type { VideoBackend, SceneRenderCtx, Emit, Cancelled } from '../types'; /** Render a local scene as ONE continuous shot: chain native-length sub-clips — each i2v from the previous * clip's last frame (the first from the keyframe) — to fill the scene, then concatenate. This fills a long * scene with real motion instead of stretching one short clip (slow-motion), so we can use fewer/longer * scenes (fewer cuts). A scene that already fits in one native clip just renders directly. */ -async function renderLocalScene(pid: string, k: number, kfFirst: string, clipPrompt: string, wdur: number, raw: string, seed: number, emit: Emit): Promise<[boolean, string]> { +async function renderLocalScene(pid: string, k: number, kfFirst: string, clipPrompt: string, wdur: number, raw: string, seed: number, emit: Emit, anchor: string | null): Promise<[boolean, string]> { // Native clip budget from the SELECTED model (the 14B is 16fps — assuming 24 here used to overestimate // nativeSec, so chained totals could come out SHORTER than the scene window). const fps = Math.max(1, localNativeFps()); @@ -52,14 +53,21 @@ async function renderLocalScene(pid: string, k: number, kfFirst: string, clipPro subs.push(subOut); emit({ event: 'subclip', index: k, sub: i + 1, total: nSub }); if (i < nSub - 1) { - const lf = await lastFrame(subOut, S.tmp(`lf_${pid}_${k}_${i}.png`)); + let lf = await lastFrame(subOut, S.tmp(`lf_${pid}_${k}_${i}.png`)); + // Anti-drift: this frame was GENERATED (and re-encoded), and the next sub-clip is generated from it — + // exposure/saturation error compounds link by link with nothing bounding it (VB_LOCAL_CHAIN_MAX caps + // the chain across SCENES, never within one). A clamped pull back toward the scene's anchor keyframe + // is a no-op while there is no drift, so it costs nothing on short scenes. + if (lf && anchor && envBool('VB_LOCAL_CHAIN_MATCH', true)) { + lf = await matchLevels(lf, anchor, S.tmp(`lfm_${pid}_${k}_${i}.png`)); + } if (lf) startImg = lf; // continue the motion from the last frame } } return (await concatClips(subs, raw)) ? [true, ''] : [false, 'failed to assemble chained sub-clips']; } -async function renderClip(pid: string, k: number, p: any, kfFirst: string, emit: Emit, seed = 42): Promise<[boolean, string]> { +async function renderClip(pid: string, k: number, p: any, kfFirst: string, emit: Emit, seed = 42, anchor: string | null = null): Promise<[boolean, string]> { const sc = S.getScene(pid, k) || {}; const wdur = Math.max(0.4, Number(sc.endSec || 0) - Number(sc.startSec || 0) || 4); // The storyboard's per-scene motion direction (explicit camera move + chained subject action) leads the @@ -74,7 +82,9 @@ async function renderClip(pid: string, k: number, p: any, kfFirst: string, emit: const clipPrompt = stripWrittenText(`${motion}. ${sc.prompt || ''}, cinematic${vstyle ? ', ' + vstyle : ''}`); // Wan renders each scene as one continuous shot: a single start frame per native clip, chained into a // long shot (renderLocalScene) so a long scene has real motion instead of one stretched slow-mo clip. - const [ok, err] = await renderLocalScene(pid, k, kfFirst, clipPrompt, wdur, raw, seed, emit); + // The drift anchor is the scene's own keyframe when it has one; a chained 'continue' scene inherits the + // anchor of the CUT scene that started its run (passed in), so the whole run is pulled toward one look. + const [ok, err] = await renderLocalScene(pid, k, kfFirst, clipPrompt, wdur, raw, seed, emit, anchor || kfFirst); if (!ok) { const reason = P.isContentBlock(err) ? "This scene was blocked by the model's safety filter." : `Scene render failed: ${(err || '').slice(0, 160)}`; putSceneMerged(pid, k, sc, { status: 'failed', error: reason }); @@ -115,10 +125,14 @@ async function renderScenesLocalChained(pid: string, p: any, toRender: number[], // clip pass — SEQUENTIAL: a continue scene starts from the previous scene's last frame. emit({ event: 'stage', stage: 'clips', total: toRender.length }); let prevLast: string | null = null; + // The look every scene in the current run is pulled back toward: the keyframe of the CUT scene that + // opened the run. Reset at each cut, so a deliberate new look never gets dragged toward the previous one. + let anchor: string | null = null; for (const k of toRender) { checkCancel(cancelled); let start = cut[k] ? kfPaths[k] : prevLast; if (!start) start = await buildKeyframe(pid, k, p, toon); // chain broke (or keyframe failed) → fresh keyframe + if (cut[k]) anchor = start || null; const sc = S.getScene(pid, k) || {}; if (!start) { putSceneMerged(pid, k, sc, { status: 'failed', error: 'Could not get a start frame for this scene.' }); @@ -126,11 +140,15 @@ async function renderScenesLocalChained(pid: string, p: any, toRender: number[], prevLast = null; continue; } - const [ok] = await renderClip(pid, k, p, start, emit); + const [ok] = await renderClip(pid, k, p, start, emit, 42, anchor); prevLast = null; if (ok) { const clipKey = `${pid}/clips/scene_${k}.mp4`; if (S.mediaExists(clipKey)) prevLast = await lastFrame(S.mediaPath(clipKey), S.tmp(`chain_last_${pid}_${k}.png`)); + // Same clamped anti-drift correction as the within-scene chain, at the scene seam. + if (prevLast && anchor && envBool('VB_LOCAL_CHAIN_MATCH', true)) { + prevLast = await matchLevels(prevLast, anchor, S.tmp(`chain_lastm_${pid}_${k}.png`)); + } } const d = S.listScenes(pid).filter((s) => s.status === 'done').length; S.updateProject(pid, { scenesDone: d, progress: Math.round((0.3 + (0.6 * Math.min(d, target)) / Math.max(1, target)) * 1000) / 1000 }); @@ -208,7 +226,7 @@ export const localVideo: VideoBackend = { return [true, '']; }, - timelineRes: () => ({ w: 832, h: 480 }), + timelineRes: () => TIMELINE_RES.local, // Real-ESRGAN → 1080 finish. VB_LOCAL_UPSCALE=0 opts a power user out (preserves the pre-C4 gate). needsUpscale: () => envBool('VB_LOCAL_UPSCALE', true), needsGpu: () => true, diff --git a/src/engine/cloud/video.ts b/src/engine/cloud/video.ts index 18af784..7697caf 100644 --- a/src/engine/cloud/video.ts +++ b/src/engine/cloud/video.ts @@ -20,6 +20,7 @@ import { checkCancel, workers, } from '../backends/sceneShared'; +import { TIMELINE_RES } from '../../shared/videoRes'; import type { VideoBackend, SceneRenderCtx, Emit } from '../backends/types'; const CLIP_MIN_SEC = 3; @@ -199,7 +200,7 @@ export const cloudVideo: VideoBackend = { return [true, '']; }, - timelineRes: () => ({ w: 1280, h: 720 }), + timelineRes: () => TIMELINE_RES.cloud, needsUpscale: () => false, needsGpu: () => false, }; diff --git a/src/engine/config.ts b/src/engine/config.ts index 58edfda..da1bdd2 100644 --- a/src/engine/config.ts +++ b/src/engine/config.ts @@ -3,8 +3,21 @@ // values are also usable in dev/tests. const CFG: Record = {}; -/** Merge an injected env map (decrypted keys + settings) into the runtime config. Last write wins. */ +/** REPLACE the runtime config with an injected env map (decrypted keys + settings), once per engine op. + * + * Clearing first is required, not cosmetic. The resolver emits several keys only for one model/backend + * combination, and some ops inject a one-shot override — so a merge lets a previous op's value survive into + * the next one and silently change how the model runs: + * - `VB_LOCAL_WAN_STEPS` is emitted only for the 5B (autoconfig) and injected only by `rerender-clips` + * (main/index.ts). Leaking it into a later 14B run forces that step count over the Lightning 4-step + * distillation AND makes `isHd` read false, so the HD path silently drops to a few-step, no-LoRA, + * tiny-VAE render — strictly worse than both Fast and HD. + * - `VB_LOCAL_WAN_DIR` / `VB_LOCAL_QUALITY` likewise persisted after the setting that produced them was + * cleared. + * Every op passes the full resolved env (main/index.ts `streamOp` spreads `sidecarEnv()`), so a replace is + * always complete. Reads still fall back to `process.env`, which stays the power-user override channel. */ export function setEnv(map: Record): void { + for (const k of Object.keys(CFG)) delete CFG[k]; for (const [k, v] of Object.entries(map)) { if (v != null && v !== '') CFG[k] = String(v); } diff --git a/src/engine/ffmpeg.ts b/src/engine/ffmpeg.ts index 8bea255..d6001f4 100644 --- a/src/engine/ffmpeg.ts +++ b/src/engine/ffmpeg.ts @@ -5,7 +5,7 @@ import { spawn } from 'node:child_process'; import fs from 'node:fs'; import ffmpegStatic from 'ffmpeg-static'; import ffprobeStatic from 'ffprobe-static'; -import { env, envInt } from './config'; +import { env, envInt, envBool } from './config'; import { tmp, copyIn, fileExists, fileSize } from './storage'; // Resolve a bundled binary path; when packaged inside app.asar the real file lives in app.asar.unpacked. @@ -103,12 +103,86 @@ export async function lastFrame(video: string, out: string): Promise { - if (!clips.length) return null; - const list = tmp(`concat_${Math.abs(out.split('').reduce((a, c) => (a * 31 + c.charCodeAt(0)) | 0, 7))}.txt`); +/** Mean luma + saturation of a still (signalstats), or null if unreadable. */ +export async function frameLevels(img: string): Promise<{ y: number; sat: number } | null> { + const r = await run(FFMPEG, ['-v', 'error', '-i', img, '-vf', 'signalstats,metadata=print:file=-', '-f', 'null', '-']); + const s = r.stdout.toString(); + const y = parseFloat((s.match(/lavfi\.signalstats\.YAVG=([\d.]+)/) || [])[1] || ''); + const sat = parseFloat((s.match(/lavfi\.signalstats\.SATAVG=([\d.]+)/) || [])[1] || ''); + return Number.isFinite(y) && Number.isFinite(sat) ? { y, sat } : null; +} + +// How far a single level-match may push exposure / saturation. Deliberately small: the correction must be +// able to cancel accumulated drift without ever overriding a scene's INTENTIONAL lighting change. +const MATCH_MAX_BRIGHT = 0.08; // eq brightness units (~±20 of 255) +const MATCH_MAX_SAT = 0.06; // ±6% saturation + +/** Nudge a chained start frame's exposure/saturation back toward its anchor keyframe, by a CLAMPED amount. + * + * Chained i2v drifts: each sub-clip is generated FROM the previous clip's last frame, so per-generation + * exposure/saturation error compounds with nothing pulling it back — a 12s scene is 6 links at the 14B's + * ~2.3s native clip, and the cross-scene chain adds more on top. The clamp is what makes this safe to apply + * unconditionally: with no drift the correction rounds to nothing and the source frame is returned + * untouched, and it can never move a frame far enough to fight a deliberate lighting change. + * Best-effort — any probe/encode failure returns the source frame. */ +export async function matchLevels(src: string, ref: string, out: string): Promise { + const [a, b] = await Promise.all([frameLevels(src), frameLevels(ref)]); + if (!a || !b || !(a.sat > 0)) return src; + const bright = Math.max(-MATCH_MAX_BRIGHT, Math.min(MATCH_MAX_BRIGHT, (b.y - a.y) / 255)); + const sat = Math.max(1 - MATCH_MAX_SAT, Math.min(1 + MATCH_MAX_SAT, b.sat / a.sat)); + if (Math.abs(bright) < 0.004 && Math.abs(sat - 1) < 0.01) return src; // below visibility — don't re-encode + const ok = await ffmpeg(['-i', src, '-vf', `eq=brightness=${bright.toFixed(4)}:saturation=${sat.toFixed(4)}`, out]); + return ok && fileSize(out) > 0 ? out : src; +} + +/** Write a concat-demuxer list file for `clips` and return its path. */ +function concatList(clips: string[], tag: string): string { + const list = tmp(`concat_${tag}.txt`); fs.writeFileSync(list, clips.map((c) => `file '${c.replace(/\\/g, '/')}'`).join('\n') + '\n'); - const ok = await ffmpeg(['-f', 'concat', '-safe', '0', '-i', list, ...x264(), '-an', out]); + return list; +} + +/** Concatenate clips into one video, audio stripped — STREAM-COPY when the inputs are compatible, else + * re-encode. + * + * Saving this hop matters: a local clip already goes through sub-clip concat -> window conform -> timeline + * concat -> upscale -> grade, and every x264 generation erodes the 480p texture that the upscaler then + * amplifies. Sub-clips of one scene come from the same model at the same settings, so `-c copy` is the + * normal case and it is bit-exact. + * + * The copy is VERIFIED, not assumed: the concat demuxer can exit 0 while producing a broken or truncated + * file when the inputs' codec parameters differ subtly, so the result is only accepted when its duration + * matches the sum of the inputs. Anything else falls back to the re-encode that was always here. */ +export async function concatClips(clips: string[], out: string, opts: { fps?: number } = {}): Promise { + if (!clips.length) return null; + const tag = String(Math.abs(out.split('').reduce((a, c) => (a * 31 + c.charCodeAt(0)) | 0, 7))); + const list = concatList(clips, tag); + if (envBool('VB_CONCAT_COPY', true)) { + const durs = await Promise.all(clips.map((c) => probeDuration(c))); + if (durs.every((d) => d != null && d > 0)) { + const want = (durs as number[]).reduce((a, b) => a + b, 0); + const copyOut = out.replace(/\.mp4$/, '') + '_copy.mp4'; + if (await ffmpeg(['-f', 'concat', '-safe', '0', '-i', list, '-c', 'copy', '-an', copyOut])) { + const got = await probeDuration(copyOut); + // 1 frame of slack: container rounding, not a dropped segment. + if (got != null && Math.abs(got - want) < 1 / FPS && fileSize(copyOut) > 0) { + try { + fs.renameSync(copyOut, out); + return out; + } catch { + /* cross-device or locked — fall through to the re-encode */ + } + } + } + try { + fs.rmSync(copyOut, { force: true }); + } catch { + /* best-effort cleanup */ + } + } + } + const rate = opts.fps ? ['-r', String(Math.trunc(opts.fps))] : []; + const ok = await ffmpeg(['-f', 'concat', '-safe', '0', '-i', list, ...x264(), ...rate, '-an', out]); return ok && fileExists(out) ? out : null; } @@ -135,7 +209,26 @@ export async function fitToWindow(raw: string, startSec: number, endSec: number, const rawDur = (await probeDuration(raw)) || target; let factor = rawDur && rawDur > 0 ? target / rawDur : 1.0; factor = Math.min(8.0, Math.max(0.05, factor)); - await ffmpeg(['-i', raw, '-vf', `setpts=${factor.toFixed(6)}*PTS,fps=${fps}`, '-frames:v', String(nFrames), ...x264(), '-an', out]); + const setpts = `setpts=${factor.toFixed(6)}*PTS`; + + // `setpts,fps=` resamples time by DUPLICATING or DROPPING whole frames — no motion compensation. The + // cloud provider only accepts whole-second durations, so a scene window is almost never an exact match + // and this retime happens on essentially every cloud clip: at factor 1.13 that is one duplicated frame in + // eight, which reads as steady judder. When the deviation is big enough to see, synthesize the in-between + // frames instead (same job RIFE does for the local path, which has no cloud equivalent). + // Gated: minterpolate is CPU-heavy at 720p, so it is skipped for deviations too small to notice. + const dev = Math.abs(factor - 1); + if (envBool('VB_SMOOTH_RETIME', true) && dev > (parseFloat(env('VB_SMOOTH_RETIME_MIN', '0.06')) || 0.06)) { + const mci = `${setpts},minterpolate=fps=${fps}:mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1`; + if (await ffmpeg(['-i', raw, '-vf', mci, '-frames:v', String(nFrames), ...x264(), '-an', out])) { + // Accept ONLY at the exact frame-grid length. The whole point of this conform is that scene lengths + // telescope with no drift, so a clip one frame short would desync the song — better to spend the + // encode twice than to let motion smoothing cost frame accuracy. + const got = await probeDuration(out); + if (got != null && fileSize(out) > 0 && Math.abs(got - target) < 0.5 / fps) return out; + } + } + await ffmpeg(['-i', raw, '-vf', `${setpts},fps=${fps}`, '-frames:v', String(nFrames), ...x264(), '-an', out]); return out; } diff --git a/src/engine/localVideo.ts b/src/engine/localVideo.ts index 86d98e8..03e7693 100644 --- a/src/engine/localVideo.ts +++ b/src/engine/localVideo.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import fs from 'node:fs'; import { env, envInt, envBool } from './config'; -import { vW, vH } from './ffmpeg'; +import { vW, vH, FPS } from './ffmpeg'; import { ensureSidecar, sidecarPost, readMarker } from './sidecar'; /** Which local i2v model (the Fast/Quality choice in Settings): @@ -60,7 +60,11 @@ export async function genVideoLocal(img: string, prompt: string, outMp4: string, // ── Wan 2.2 (5B / 14B) ───────────────────────────────────────────────────────────────────────── const is5b = model === '5b'; const nativeFps = localNativeFps(); - const isHd = !is5b && env('VB_LOCAL_QUALITY', 'fast') === 'hd' && !env('VB_LOCAL_WAN_STEPS'); + // HD is a pure QUALITY-setting question. It used to also require "no explicit step override", which made + // any injected VB_LOCAL_WAN_STEPS (rerender-clips, or a 5B run before setEnv gained replace semantics) + // silently demote an HD render to the fast VAE + the short deadline while ALSO skipping the Lightning + // LoRA — the worst of both paths. The step override still applies below; it just no longer redefines HD. + const isHd = !is5b && env('VB_LOCAL_QUALITY', 'fast') === 'hd'; const payload: Record = { model_dir: md, image: img, @@ -121,19 +125,46 @@ export async function genVideoLocal(img: string, prompt: string, outMp4: string, } } -/** RIFE 2x a sub-24fps clip (the 14B saves at Wan's native 16fps) so the 24fps timeline conform DECIMATES - * (32→24 drops 1 frame in 4) instead of duplicating every other frame — the duplication reads as constant - * judder. ~3s per clip on the Apple GPU (ncnn/MoltenVK). Best-effort: on any failure the raw clip stands. +/** Smallest interpolation factor that lands a sub-timeline clip on an EXACT multiple of the timeline fps. + * + * This is the difference between smooth and merely "less bad". The 14B saves at Wan's native 16fps; a plain + * 2x lands on 32fps, and the timeline conform (`fps=24`) then has to drop 1 frame in 4 at uneven phase — a + * repeating 4-frame cadence break that reads as micro-judder. 3x lands on 48fps, which decimates to 24 as a + * clean 2:1 (every second frame, perfectly even). RIFE v4.x is timestep-conditioned, so a 3x pass costs one + * extra synthesized frame per gap, not a second full pass. + * Returns 1 when the clip is already at/above the timeline rate (no interpolation needed). Falls back to 2 + * when no clean factor exists within `maxFactor` — still better than leaving it at native. */ +export function rifeFactor(nativeFps: number, timelineFps: number = FPS, maxFactor = 4): number { + if (!(nativeFps > 0) || nativeFps >= timelineFps) return 1; + for (let f = 2; f <= maxFactor; f++) if ((nativeFps * f) % timelineFps === 0) return f; + return 2; +} + +/** Interpolate a sub-timeline clip up to an exact multiple of the 24fps timeline (see rifeFactor), so the + * conform decimates evenly instead of duplicating or dropping frames at an uneven phase. ~3-5s per clip on + * the Apple GPU (ncnn/MoltenVK). Best-effort: on any failure the raw clip stands — but the failure is now + * LOGGED, because a silent fallback means every clip in that render conforms 16→24 by duplicating one frame + * in two (constant judder) with nothing in the UI or logs to explain why the result looks worse. * Per-clip ONLY — interpolating an assembled timeline would synthesize morph frames across scene cuts. */ async function rifeSmooth(clip: string, nativeFps: number, deadlineMs: number): Promise { - if (nativeFps >= 24 || !envBool('VB_LOCAL_RIFE', true)) return; + if (!envBool('VB_LOCAL_RIFE', true)) return; + const factor = rifeFactor(nativeFps); + if (factor < 2) return; const interp = clip.replace(/\.mp4$/, '_rife.mp4'); try { // timeout_sec: the sidecar kills its worker subprocess just before our HTTP deadline, so an abandoned // job can never sit on the GPU lock after we've given up (that would starve the next clip's /i2v). - const r = await sidecarPost('/interp', { video: clip, out: interp, out_fps: nativeFps * 2, timeout_sec: Math.max(60, Math.floor(deadlineMs / 1000) - 60) }, deadlineMs); - if (r.ok && fs.existsSync(interp) && fs.statSync(interp).size > 0) fs.renameSync(interp, clip); - } catch { - /* interpolation is optional polish — the 16fps clip still plays */ + const r = await sidecarPost( + '/interp', + { video: clip, out: interp, factor, out_fps: nativeFps * factor, timeout_sec: Math.max(60, Math.floor(deadlineMs / 1000) - 60) }, + deadlineMs, + ); + if (r.ok && fs.existsSync(interp) && fs.statSync(interp).size > 0) { + fs.renameSync(interp, clip); + return; + } + console.error(`[rife] interpolation did not produce output (${nativeFps}→${nativeFps * factor}fps): ${r?.error || 'unknown'} — clip stays at ${nativeFps}fps and will judder on the ${FPS}fps timeline`); + } catch (e: any) { + console.error(`[rife] interpolation failed (${nativeFps}→${nativeFps * factor}fps): ${e?.message || e} — clip stays at ${nativeFps}fps and will judder on the ${FPS}fps timeline`); } } diff --git a/src/engine/pipeline.ts b/src/engine/pipeline.ts index 649748c..345c37e 100644 --- a/src/engine/pipeline.ts +++ b/src/engine/pipeline.ts @@ -4,7 +4,7 @@ // Progress is reported through an `emit(event)` callback the engine turns into IPC events. import crypto from 'node:crypto'; import fs from 'node:fs'; -import { env, envInt } from './config'; +import { env, envInt, envBool } from './config'; import { costTotal } from './cost'; import * as S from './storage'; @@ -23,7 +23,7 @@ function fsReadHeadTail(path: string, size: number): Buffer { fs.closeSync(fd); } } -import { FPS, probeDuration, toPng, putThumb, stillClip, ffmpeg, toWav, x264, conformClip } from './ffmpeg'; +import { FPS, probeDuration, toPng, putThumb, stillClip, ffmpeg, toWav, x264, conformClip, concatClips } from './ffmpeg'; import * as P from './stages'; import { assertCastExists } from './backends/sceneShared'; import { ensureSidecar, sidecarPost } from './sidecar'; @@ -510,13 +510,16 @@ async function assemble(pid: string, emit: Emit): Promise<{ projectId: string; v // different native resolution — concatenating mixed dimensions corrupts the output, so normalize first. const normalized: string[] = []; for (let i = 0; i < clips.length; i++) normalized.push(await conformClip(clips[i], `${work}/clips/norm_${i}.mp4`)); - const concat = `${work}/concat.txt`; - S.writeText(concat, normalized.map((c) => `file '${c.replace(/\\/g, '/')}'`).join('\n') + '\n'); const silent = `${work}/output/silent.mp4`; - await ffmpeg(['-f', 'concat', '-safe', '0', '-i', concat, ...x264(), '-r', String(FPS), silent]); + // Stream-copy the timeline when the conformed clips are compatible — they all come out of the same + // window conform at the same size and rate, so this is the normal case and it removes one full x264 + // generation from every clip's path (render -> sub-clip concat -> conform -> HERE -> upscale -> grade). + // concatClips verifies the copy against the summed input duration and re-encodes at FPS if it doesn't fit. + await concatClips(normalized, silent, { fps: FPS }); // ── finish chain (both steps best-effort — on any failure the plain concat plays) ────────────── let master = silent; + let upscaled = false; // 1) Upscale: the local backend emits 832×480/896×512 — watched fullscreen that reads soft no matter how // good the denoise was. One Real-ESRGAN pass (sidecar /upscale, Apple GPU — idle by assemble time) // to 1080p. Runs once on the whole timeline: every clip + failed-scene fill shares one size here. The @@ -527,7 +530,10 @@ async function assemble(pid: string, emit: Emit): Promise<{ projectId: string; v const up = `${work}/output/upscaled.mp4`; const upSec = envInt('VB_UPSCALE_DEADLINE_SEC', 5400); const r = await sidecarPost('/upscale', { video: silent, out: up, target_h: envInt('VB_UPSCALE_H', 1080), timeout_sec: Math.max(60, upSec - 60) }, upSec * 1000); - if (r.ok && S.fileSize(up) > 0) master = up; + if (r.ok && S.fileSize(up) > 0) { + master = up; + upscaled = true; + } } catch { /* upscale is polish, never fail the render for it */ } @@ -538,11 +544,18 @@ async function assemble(pid: string, emit: Emit): Promise<{ projectId: string; v const finish = env('VB_FINISH', 'subtle'); if (finish !== 'off') { const graded = `${work}/output/graded.mp4`; + // On an UPSCALED master the two motion-hostile steps come off. hqdn3d's last two numbers are its + // TEMPORAL strength: it averages a pixel against the same pixel in neighbouring frames, which on a + // timeline whose in-between frames RIFE just synthesized smears real motion and can ghost fast pans — + // exactly the fluidity the interpolation pass was there to buy. And Real-ESRGAN already resolves edge + // detail, so a second unsharp on top rings them. Spatial denoise, grade and grain still run. + // VB_FINISH_TEMPORAL=1 / VB_FINISH_SHARPEN=1 restore the old chain. + const temporal = !upscaled || envBool('VB_FINISH_TEMPORAL', false); const vf = [ - 'hqdn3d=1.5:1.5:3:3', + temporal ? 'hqdn3d=1.5:1.5:3:3' : 'hqdn3d=1.5:1.5:0:0', "curves=master='0/0 0.25/0.22 0.5/0.5 0.75/0.79 1/1'", 'eq=saturation=1.06', - 'unsharp=5:5:0.35:5:5:0.0', + ...(!upscaled || envBool('VB_FINISH_SHARPEN', false) ? ['unsharp=5:5:0.35:5:5:0.0'] : []), ...(finish === 'filmic' ? ['vignette=PI/5'] : []), `noise=c0s=${finish === 'filmic' ? 7 : 4}:c0f=t+u`, ].join(','); diff --git a/src/main/autoconfig.ts b/src/main/autoconfig.ts index 022f0c4..5b16a7d 100644 --- a/src/main/autoconfig.ts +++ b/src/main/autoconfig.ts @@ -4,8 +4,9 @@ // I1 no key ⇒ local (first, un-overridable) I2 key ⇒ still local by default (cloud is explicit opt-in) // I3 tier never enables cloud I4 zero-key users stay fully local // Types are imported type-only, so this module has no runtime deps (no electron) and is unit-testable. -// Not yet wired into sidecarEnv — that lands at C5. +// Wired into sidecarEnv() in main/index.ts — every engine op is spawned with the env this resolver emits. import { STAGES } from './settingsSchema'; +import { TIMELINE_RES } from '../shared/videoRes'; import type { Settings, Stage, StageSelection, Backend } from './settingsSchema'; import type { LocalCapabilities } from './localModels'; @@ -64,10 +65,12 @@ function toEnv(stages: Record, settings: Settings): Record env.VB_OR_VIDEO_MODEL = settings.cloud.videoModel; env.VB_VLM_MODEL = settings.cloud.vlmModel; env.VB_MODERATION_MODEL = settings.cloud.moderationModel; - // Timeline resolution follows the VIDEO backend (local Wan 832×480; cloud Kling 1280×720). + // Timeline resolution follows the VIDEO backend. The numbers live in shared/videoRes.ts, which the engine + // backends' timelineRes() reads too — the two must not drift apart. const videoLocal = stages.VIDEO.backend === 'local'; - env.VB_W = videoLocal ? '832' : '1280'; - env.VB_H = videoLocal ? '480' : '720'; + const res = videoLocal ? TIMELINE_RES.local : TIMELINE_RES.cloud; + env.VB_W = String(res.w); + env.VB_H = String(res.h); // GPU serialization only when the video stage runs locally (one Wan run saturates unified memory). env.VB_WORKERS = videoLocal ? '1' : String(settings.workers || 4); // A local keyframe matches the timeline so it isn't resized into the clip. diff --git a/src/main/index.ts b/src/main/index.ts index 8764aa6..2fc2e80 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -341,9 +341,15 @@ function registerIpc() { guardRender(o.pid) ?? streamOp('render:' + o.pid, 'render', ['--project', o.pid, ...(o.preview ? ['--preview'] : []), ...(o.regenStory ? ['--regen-story'] : [])], undefined, renderNeedsGpu())); ipcMain.handle('render:resume', (_e, pid: string) => guardRender(pid) ?? streamOp('render:' + pid, 'resume', ['--project', pid], undefined, renderNeedsGpu())); - // Re-render the existing clips at quality (20 steps), reusing storyboard + keyframes — only the video step. + // Re-render the existing clips at higher quality, reusing storyboard + keyframes — only the video step. + // The step bump is 5B-ONLY. The 5B runs native steps (10 by default), so 20 is a real quality gain. The + // 14B's fast path is the Wan2.2-Lightning 4-step DISTILLATION: forcing 20 steps through it over-denoises + // into flat, slow-motion movement (see localVideo.ts) — the opposite of a quality re-render — and its HD + // path already runs the full 40-step model-config schedule. So on the 14B this op keeps the resolved + // settings and simply re-renders. ipcMain.handle('render:requality', (_e, pid: string) => - guardRender(pid) ?? streamOp('render:' + pid, 'rerender-clips', ['--project', pid], { VB_LOCAL_WAN_STEPS: '20' }, renderNeedsGpu())); + guardRender(pid) ?? + streamOp('render:' + pid, 'rerender-clips', ['--project', pid], getSettings().localVideoModel === '5b' ? { VB_LOCAL_WAN_STEPS: '20' } : undefined, renderNeedsGpu())); ipcMain.handle('scene:regenerate', (_e, o: { pid: string; index: number }) => guardRender(o.pid) ?? streamOp('render:' + o.pid, 'regenerate-scene', ['--project', o.pid, '--index', String(o.index)], undefined, renderNeedsGpu())); ipcMain.handle('op:cancel', (_e, opId: string) => { RUNS.get(opId)?.cancel(); return true; }); diff --git a/src/shared/videoRes.ts b/src/shared/videoRes.ts new file mode 100644 index 0000000..8610e38 --- /dev/null +++ b/src/shared/videoRes.ts @@ -0,0 +1,15 @@ +// The timeline resolution each VIDEO backend renders at — ONE definition, imported by both sides that +// need it, because they have to agree exactly: +// 1. main/autoconfig.ts turns it into VB_W / VB_H (what the engine actually generates and conforms to); +// 2. engine/backends/*/video.ts reports it as `timelineRes()` on the backend interface. +// These used to be two hand-copied pairs of literals with nothing tying them together, so a change to one +// silently desynced the other — and a mismatch is not loud: clips still render, they just get scaled and +// padded by conformClip at assemble time, costing resolution for no visible reason. +export const TIMELINE_RES = { + // Wan 2.2 i2v on MLX: the 48GB Metal working-set ceiling. Finished to 1080p by the ESRGAN upscale pass. + local: { w: 832, h: 480 }, + // Kling returns near-HD already, so the cloud path skips the upscale entirely. + cloud: { w: 1280, h: 720 }, +} as const; + +export type TimelineRes = { w: number; h: number }; diff --git a/test/render-quality.test.ts b/test/render-quality.test.ts new file mode 100644 index 0000000..a85d155 --- /dev/null +++ b/test/render-quality.test.ts @@ -0,0 +1,109 @@ +// Unit tests for the render-quality/fluidity invariants (pure — no electron, no ffmpeg, no sidecar). +// Each test here pins a defect that was silent in production: nothing crashed, the video just came out +// worse, which is exactly the class of bug a unit test has to catch instead of a human eye. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { setEnv, env, envInt } from '../src/engine/config'; +import { rifeFactor } from '../src/engine/localVideo'; +import { FPS } from '../src/engine/ffmpeg'; +import { TIMELINE_RES } from '../src/shared/videoRes'; +import { DEFAULTS, type Settings } from '../src/main/settingsSchema'; +import { resolveConfig, type KeyState } from '../src/main/autoconfig'; +import type { LocalCapabilities } from '../src/main/localModels'; + +const mk = (over: Partial = {}): Settings => ({ ...structuredClone(DEFAULTS), ...over }); +const CAPS = (supported: boolean): LocalCapabilities => ({ supported, depsInstalled: true } as LocalCapabilities); +const NO_KEYS: KeyState = { openrouter: false, replicate: false }; +const BOTH: KeyState = { openrouter: true, replicate: true }; + +// ── setEnv: per-op REPLACE, not merge ────────────────────────────────────────── +// The bug this pins: CFG was merge-only, so any key emitted for one model/op survived into the next one. +// `rerender-clips` injects VB_LOCAL_WAN_STEPS=20 and the resolver emits it for the 5B — either one used to +// poison every later 14B render in the same process: 20 steps forced through a 4-step Lightning +// distillation (flat, slow-motion output) and, worse, `isHd` read the leftover key and silently demoted an +// HD render to the fast VAE + short deadline while ALSO skipping the LoRA. + +test('setEnv replaces the previous op env — a one-shot override cannot leak into the next render', () => { + setEnv({ VB_LOCAL_VIDEO_MODEL: '5b', VB_LOCAL_WAN_STEPS: '20' }); + assert.equal(env('VB_LOCAL_WAN_STEPS'), '20'); + // Next op resolves a 14B render and emits no step override at all. + setEnv({ VB_LOCAL_VIDEO_MODEL: '14b', VB_LOCAL_QUALITY: 'hd' }); + assert.equal(env('VB_LOCAL_WAN_STEPS'), '', 'stale step override must not survive into the next op'); + assert.equal(env('VB_LOCAL_VIDEO_MODEL'), '14b'); + assert.equal(env('VB_LOCAL_QUALITY'), 'hd'); +}); + +test('setEnv clears every key, not just the ones the new map mentions', () => { + setEnv({ VB_LOCAL_WAN_DIR: '/old/model', VB_STT_LANG: 'it', VB_W: '832' }); + setEnv({ VB_W: '1280' }); + assert.equal(env('VB_LOCAL_WAN_DIR'), '', 'a cleared setting must not keep its old value'); + assert.equal(env('VB_STT_LANG'), ''); + assert.equal(env('VB_W'), '1280'); +}); + +test('setEnv drops empty values rather than storing them (defaults must win)', () => { + setEnv({ VB_WORKERS: '4' }); + setEnv({ VB_WORKERS: '' }); + assert.equal(envInt('VB_WORKERS', 7), 7, 'an empty injected value falls through to the caller default'); +}); + +// ── RIFE factor: land on an EXACT multiple of the timeline rate ──────────────── +// The bug this pins: a fixed 2x took the 14B's native 16fps to 32fps, and the 24fps conform then had to +// drop 1 frame in 4 at uneven phase — a repeating cadence break. 3x lands on 48fps, an exact 2:1. + +test('rifeFactor lands sub-timeline rates on an exact multiple of the timeline fps', () => { + for (const native of [8, 12, 16]) { + const f = rifeFactor(native); + assert.ok(f >= 2, `${native}fps must be interpolated (got factor ${f})`); + assert.equal((native * f) % FPS, 0, `${native}fps x${f} = ${native * f} must divide evenly into ${FPS}`); + } +}); + +test('rifeFactor picks 3 for the 14B native 16fps (48→24 is a clean 2:1, 32→24 is not)', () => { + assert.equal(rifeFactor(16), 3); + assert.notEqual(rifeFactor(16), 2, 'the old fixed 2x left an uneven 32→24 decimation'); +}); + +test('rifeFactor is a no-op at or above the timeline rate (the 5B is already 24fps)', () => { + assert.equal(rifeFactor(24), 1); + assert.equal(rifeFactor(30), 1); +}); + +test('rifeFactor never returns a factor that would cost more than one extra frame per gap', () => { + for (const native of [8, 10, 12, 15, 16, 20, 23]) assert.ok(rifeFactor(native) <= 4, `${native}fps`); +}); + +test('rifeFactor falls back to 2 (never 0/1) when no clean factor exists in range', () => { + const f = rifeFactor(23); // 23*n is never a multiple of 24 for n<=4 + assert.equal(f, 2, 'still interpolate — a sub-timeline clip must never be left to duplicate frames'); +}); + +test('rifeFactor tolerates a nonsense native rate instead of dividing by zero', () => { + assert.equal(rifeFactor(0), 1); + assert.equal(rifeFactor(-5), 1); +}); + +// ── timeline resolution: ONE definition ─────────────────────────────────────── +// The bug this pins: the resolver and the backend interface each carried their own hand-copied literals. +// A mismatch is silent — clips still render, conformClip just scales and pads them at assemble time, so +// the render quietly loses resolution with nothing to point at. + +test('the resolver emits the shared timeline resolution for a local VIDEO stage', () => { + const e = resolveConfig(CAPS(true), mk(), NO_KEYS).toEnv(); + assert.equal(e.VB_VIDEO_BACKEND, 'local'); + assert.equal(e.VB_W, String(TIMELINE_RES.local.w)); + assert.equal(e.VB_H, String(TIMELINE_RES.local.h)); +}); + +test('the resolver emits the shared timeline resolution for a cloud VIDEO stage', () => { + const e = resolveConfig(CAPS(true), mk({ backendPreference: 'prefer-cloud' }), BOTH).toEnv(); + assert.equal(e.VB_VIDEO_BACKEND, 'cloud'); + assert.equal(e.VB_W, String(TIMELINE_RES.cloud.w)); + assert.equal(e.VB_H, String(TIMELINE_RES.cloud.h)); +}); + +test('a local keyframe is generated at the timeline size (never resized into the clip)', () => { + const e = resolveConfig(CAPS(true), mk(), NO_KEYS).toEnv(); + assert.equal(e.VB_LOCAL_KEYFRAME_W, e.VB_W); + assert.equal(e.VB_LOCAL_KEYFRAME_H, e.VB_H); +});