From c03e8c207c5551c63ca9c06f83994c7722c6fbc5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 16:25:48 +0000 Subject: [PATCH] fix(render): quality + fluidity defects across the local and cloud paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight fixes to the render chain. Every one of them was silent in production — nothing crashed, the video just came out worse than the code was capable of. 1. setEnv is now a per-op REPLACE, not a merge (engine/config.ts). CFG only ever accumulated, so any key emitted for one model/op survived into the next. `render:requality` injects VB_LOCAL_WAN_STEPS=20 and the resolver emits it for the 5B — either one then poisoned every later 14B render in the same process: 20 steps forced through a 4-step Lightning distillation (flat, slow-motion output) and, worse, `isHd` read the leftover key and silently demoted an HD render to the fast VAE + short deadline while also skipping the LoRA — strictly worse than both Fast and HD. 2. isHd no longer depends on the absence of a step override (localVideo.ts). Quality is a settings question; a step override must not redefine it. 3. render:requality bumps steps for the 5B only (main/index.ts). The 5B runs native steps, so 20 is a real gain. Pushing 20 through the 14B's Lightning distillation over-denoises into flat motion, and its HD path already runs the full 40-step schedule. 4. RIFE targets an exact multiple of the timeline rate (localVideo.ts, interp.py). A fixed 2x took the 14B's native 16fps to 32, and the 24fps conform then dropped 1 frame in 4 at uneven phase — a repeating cadence break on the DEFAULT model. 3x lands on 48fps, an exact 2:1. interp.py generalizes to an arbitrary factor using RIFE v4's timestep conditioning (one pass, not a recursive 2x-of-2x that would interpolate synthesized frames). 5. A failed interpolation is logged instead of swallowed (localVideo.ts). The fallback conforms 16->24 by duplicating one frame in two — constant judder, previously with nothing in the logs to explain it. 6. The cloud retime is motion-compensated when the deviation is visible (ffmpeg.ts). The provider only accepts whole-second durations, so nearly every cloud clip is retimed, and setpts+fps resamples by duplicating or dropping whole frames with no local-style RIFE equivalent. Accepted only at the exact frame-grid length, so smoothing can never cost frame accuracy. 7. Clamped anti-drift on chained start frames (ffmpeg.ts, backends/local/video.ts). Chained i2v compounds exposure/saturation error per link with nothing bounding it — VB_LOCAL_CHAIN_MAX caps the chain across scenes, never within one, so a 12s scene is 6 unbounded links. The clamp makes it a no-op when there is no drift and keeps it from ever fighting deliberate lighting. 8. One x264 generation removed, and the grade stops fighting the render (ffmpeg.ts, pipeline.ts). Timeline concat now stream-copies when the inputs are compatible, verified against the summed input duration. On an upscaled master the temporal denoise (which smears the frames RIFE just synthesized) and the second unsharp (over an already-sharpened ESRGAN output) come off. Also: timeline resolution gets one definition in shared/videoRes.ts instead of two hand-copied literal pairs, and autoconfig's stale "not yet wired" comment is corrected — it has been wired into sidecarEnv all along. Every new behavior is env-gated back to the old one. Verified: typecheck, 29 unit tests (12 new), check-no-cloud, build, plus the ffmpeg helpers executed against synthetic clips to prove the filtergraphs are valid and frame-exact rather than silently falling back. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TD7w2Asmu4Ga5HBGK99261 --- local/interp.py | 39 +++++++---- src/engine/backends/local/video.ts | 32 +++++++-- src/engine/cloud/video.ts | 3 +- src/engine/config.ts | 15 +++- src/engine/ffmpeg.ts | 107 ++++++++++++++++++++++++++-- src/engine/localVideo.ts | 51 +++++++++++--- src/engine/pipeline.ts | 29 +++++--- src/main/autoconfig.ts | 11 +-- src/main/index.ts | 10 ++- src/shared/videoRes.ts | 15 ++++ test/render-quality.test.ts | 109 +++++++++++++++++++++++++++++ 11 files changed, 368 insertions(+), 53 deletions(-) create mode 100644 src/shared/videoRes.ts create mode 100644 test/render-quality.test.ts diff --git a/local/interp.py b/local/interp.py index d8eed1d..42828e6 100644 --- a/local/interp.py +++ b/local/interp.py @@ -1,12 +1,18 @@ -"""Frame interpolation via RIFE (rife-ncnn-vulkan, Apple GPU through MoltenVK) — 2x a video's frame rate. +"""Frame interpolation via RIFE (rife-ncnn-vulkan, Apple GPU through MoltenVK) — Nx a video's frame rate. -Used to render Wan clips at HALF the frames (faster denoise + VAE) then interpolate back up to the target -fps with real motion. RIFE at 2x (small frame gap) is near-ground-truth on cinematic pans/walking. MIT -licensed (wrapper + weights). Falls back to nothing here — the engine handles the ffmpeg fallback. +Used to render Wan clips at a fraction of the timeline frames (faster denoise + VAE) then interpolate back +up with real motion. RIFE v4.x is timestep-conditioned, so an Nx pass synthesizes N-1 evenly spaced frames +per gap in ONE pass — near-ground-truth on cinematic pans/walking at these small gaps. MIT licensed +(wrapper + weights). Falls back to nothing here — the engine handles the ffmpeg fallback. -run_interpolate(req): {video, out, [model], [gpuid], [out_fps]} -> {ok, frames_in, frames_out}. Doubles the -frame count (a midpoint between each pair + the last frame repeated once = 2n frames), muxed at out_fps -(pass 2x the source fps) so the clip keeps exactly its source duration with real in-between motion. +run_interpolate(req): {video, out, [factor], [model], [gpuid], [out_fps]} -> {ok, frames_in, frames_out}. +Multiplies the frame count by `factor` (default 2, clamped to 2..4): N-1 midpoints between each pair, plus +the last frame repeated factor-1 times = n*factor frames, muxed at out_fps (pass factor x the source fps) +so the clip keeps exactly its source duration with real in-between motion. + +Why the caller may ask for 3x rather than 2x: the engine's timeline is 24fps and the 14B saves at 16fps. +2x lands on 32fps, which conforms to 24 by dropping 1 frame in 4 at uneven phase (visible cadence break); +3x lands on 48fps, an exact 2:1 decimation to 24. See rifeFactor() in src/engine/localVideo.ts. """ import os import shutil @@ -48,6 +54,9 @@ def run_interpolate(req: dict) -> dict: out = req["out"] model = req.get("model") or _default_model() gpuid = int(req.get("gpuid", 0)) + # Clamped: 1 would be a no-op pass, and past 4 the synthesized-to-real frame ratio stops buying + # smoothness while the per-clip cost keeps growing. + factor = max(2, min(4, int(req.get("factor", 2) or 2))) work = video + "_interp" # A previous FAILED/killed run leaves stale frames here; ffmpeg's image2 demuxer would happily append @@ -67,19 +76,23 @@ def run_interpolate(req: dict) -> dict: # No caching: the sidecar runs each job in a fresh subprocess (MoltenVK isolation), so nothing persists. rife = Rife(gpuid=gpuid, model=model, scale=2, width=w, height=h) - # interleave: f0, mid(f0,f1), f1, mid(f1,f2), f2, ..., fN, fN -> 2n frames. The final frame repeats - # once so the clip keeps EXACTLY its source duration at 2x fps (2n-1 frames would run 1/(2*fps) short - # per clip — enough to break the chained-total >= scene-window invariant and drift the timeline). + # interleave: f0, [factor-1 midpoints], f1, [factor-1 midpoints], f2, ..., fN, then fN repeated + # factor-1 times -> exactly n*factor frames. The tail repeat keeps the clip at EXACTLY its source + # duration at factor x fps (n*factor-(factor-1) frames would run short per clip — enough to break the + # chained-total >= scene-window invariant and drift the timeline). oi = 0 prev = first Image.open(os.path.join(fin, frames[0])).convert("RGB").save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 for i in range(1, n): cur = Image.open(os.path.join(fin, frames[i])).convert("RGB") - mid = rife.process(prev, cur, timestep=0.5) - mid.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 + for j in range(1, factor): + # RIFE v4.x takes an arbitrary timestep, so 3x is one pass with two midpoints per gap — not a + # recursive 2x-of-2x (which would interpolate already-synthesized frames and compound error). + rife.process(prev, cur, timestep=j / factor).save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 cur.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 prev = cur - prev.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 + for _ in range(factor - 1): + prev.save(os.path.join(fout, f"o_{oi:05d}.png")); oi += 1 fps = float(req.get("out_fps", 24)) subprocess.run([ diff --git a/src/engine/backends/local/video.ts b/src/engine/backends/local/video.ts index f07f8ab..d9886a3 100644 --- a/src/engine/backends/local/video.ts +++ b/src/engine/backends/local/video.ts @@ -6,7 +6,7 @@ // per native sub-clip, chained from the previous clip's real last frame; the clip is TRIMMED to the frame // grid (never stretched). See DUAL_BACKEND_PLAN.md §1.7. import { env, envInt, envBool } from '../../config'; -import { lastFrame, concatClips } from '../../ffmpeg'; +import { lastFrame, concatClips, matchLevels } from '../../ffmpeg'; import * as P from '../../stages'; import * as S from '../../storage'; import { genVideoLocal, localNativeFps, localMaxFrames } from '../../localVideo'; @@ -22,13 +22,14 @@ import { workers, kfWorkers, } from '../sceneShared'; +import { TIMELINE_RES } from '../../../shared/videoRes'; import type { VideoBackend, SceneRenderCtx, Emit, Cancelled } from '../types'; /** Render a local scene as ONE continuous shot: chain native-length sub-clips — each i2v from the previous * clip's last frame (the first from the keyframe) — to fill the scene, then concatenate. This fills a long * scene with real motion instead of stretching one short clip (slow-motion), so we can use fewer/longer * scenes (fewer cuts). A scene that already fits in one native clip just renders directly. */ -async function renderLocalScene(pid: string, k: number, kfFirst: string, clipPrompt: string, wdur: number, raw: string, seed: number, emit: Emit): Promise<[boolean, string]> { +async function renderLocalScene(pid: string, k: number, kfFirst: string, clipPrompt: string, wdur: number, raw: string, seed: number, emit: Emit, anchor: string | null): Promise<[boolean, string]> { // Native clip budget from the SELECTED model (the 14B is 16fps — assuming 24 here used to overestimate // nativeSec, so chained totals could come out SHORTER than the scene window). const fps = Math.max(1, localNativeFps()); @@ -52,14 +53,21 @@ async function renderLocalScene(pid: string, k: number, kfFirst: string, clipPro subs.push(subOut); emit({ event: 'subclip', index: k, sub: i + 1, total: nSub }); if (i < nSub - 1) { - const lf = await lastFrame(subOut, S.tmp(`lf_${pid}_${k}_${i}.png`)); + let lf = await lastFrame(subOut, S.tmp(`lf_${pid}_${k}_${i}.png`)); + // Anti-drift: this frame was GENERATED (and re-encoded), and the next sub-clip is generated from it — + // exposure/saturation error compounds link by link with nothing bounding it (VB_LOCAL_CHAIN_MAX caps + // the chain across SCENES, never within one). A clamped pull back toward the scene's anchor keyframe + // is a no-op while there is no drift, so it costs nothing on short scenes. + if (lf && anchor && envBool('VB_LOCAL_CHAIN_MATCH', true)) { + lf = await matchLevels(lf, anchor, S.tmp(`lfm_${pid}_${k}_${i}.png`)); + } if (lf) startImg = lf; // continue the motion from the last frame } } return (await concatClips(subs, raw)) ? [true, ''] : [false, 'failed to assemble chained sub-clips']; } -async function renderClip(pid: string, k: number, p: any, kfFirst: string, emit: Emit, seed = 42): Promise<[boolean, string]> { +async function renderClip(pid: string, k: number, p: any, kfFirst: string, emit: Emit, seed = 42, anchor: string | null = null): Promise<[boolean, string]> { const sc = S.getScene(pid, k) || {}; const wdur = Math.max(0.4, Number(sc.endSec || 0) - Number(sc.startSec || 0) || 4); // The storyboard's per-scene motion direction (explicit camera move + chained subject action) leads the @@ -74,7 +82,9 @@ async function renderClip(pid: string, k: number, p: any, kfFirst: string, emit: const clipPrompt = stripWrittenText(`${motion}. ${sc.prompt || ''}, cinematic${vstyle ? ', ' + vstyle : ''}`); // Wan renders each scene as one continuous shot: a single start frame per native clip, chained into a // long shot (renderLocalScene) so a long scene has real motion instead of one stretched slow-mo clip. - const [ok, err] = await renderLocalScene(pid, k, kfFirst, clipPrompt, wdur, raw, seed, emit); + // The drift anchor is the scene's own keyframe when it has one; a chained 'continue' scene inherits the + // anchor of the CUT scene that started its run (passed in), so the whole run is pulled toward one look. + const [ok, err] = await renderLocalScene(pid, k, kfFirst, clipPrompt, wdur, raw, seed, emit, anchor || kfFirst); if (!ok) { const reason = P.isContentBlock(err) ? "This scene was blocked by the model's safety filter." : `Scene render failed: ${(err || '').slice(0, 160)}`; putSceneMerged(pid, k, sc, { status: 'failed', error: reason }); @@ -115,10 +125,14 @@ async function renderScenesLocalChained(pid: string, p: any, toRender: number[], // clip pass — SEQUENTIAL: a continue scene starts from the previous scene's last frame. emit({ event: 'stage', stage: 'clips', total: toRender.length }); let prevLast: string | null = null; + // The look every scene in the current run is pulled back toward: the keyframe of the CUT scene that + // opened the run. Reset at each cut, so a deliberate new look never gets dragged toward the previous one. + let anchor: string | null = null; for (const k of toRender) { checkCancel(cancelled); let start = cut[k] ? kfPaths[k] : prevLast; if (!start) start = await buildKeyframe(pid, k, p, toon); // chain broke (or keyframe failed) → fresh keyframe + if (cut[k]) anchor = start || null; const sc = S.getScene(pid, k) || {}; if (!start) { putSceneMerged(pid, k, sc, { status: 'failed', error: 'Could not get a start frame for this scene.' }); @@ -126,11 +140,15 @@ async function renderScenesLocalChained(pid: string, p: any, toRender: number[], prevLast = null; continue; } - const [ok] = await renderClip(pid, k, p, start, emit); + const [ok] = await renderClip(pid, k, p, start, emit, 42, anchor); prevLast = null; if (ok) { const clipKey = `${pid}/clips/scene_${k}.mp4`; if (S.mediaExists(clipKey)) prevLast = await lastFrame(S.mediaPath(clipKey), S.tmp(`chain_last_${pid}_${k}.png`)); + // Same clamped anti-drift correction as the within-scene chain, at the scene seam. + if (prevLast && anchor && envBool('VB_LOCAL_CHAIN_MATCH', true)) { + prevLast = await matchLevels(prevLast, anchor, S.tmp(`chain_lastm_${pid}_${k}.png`)); + } } const d = S.listScenes(pid).filter((s) => s.status === 'done').length; S.updateProject(pid, { scenesDone: d, progress: Math.round((0.3 + (0.6 * Math.min(d, target)) / Math.max(1, target)) * 1000) / 1000 }); @@ -208,7 +226,7 @@ export const localVideo: VideoBackend = { return [true, '']; }, - timelineRes: () => ({ w: 832, h: 480 }), + timelineRes: () => TIMELINE_RES.local, // Real-ESRGAN → 1080 finish. VB_LOCAL_UPSCALE=0 opts a power user out (preserves the pre-C4 gate). needsUpscale: () => envBool('VB_LOCAL_UPSCALE', true), needsGpu: () => true, diff --git a/src/engine/cloud/video.ts b/src/engine/cloud/video.ts index 18af784..7697caf 100644 --- a/src/engine/cloud/video.ts +++ b/src/engine/cloud/video.ts @@ -20,6 +20,7 @@ import { checkCancel, workers, } from '../backends/sceneShared'; +import { TIMELINE_RES } from '../../shared/videoRes'; import type { VideoBackend, SceneRenderCtx, Emit } from '../backends/types'; const CLIP_MIN_SEC = 3; @@ -199,7 +200,7 @@ export const cloudVideo: VideoBackend = { return [true, '']; }, - timelineRes: () => ({ w: 1280, h: 720 }), + timelineRes: () => TIMELINE_RES.cloud, needsUpscale: () => false, needsGpu: () => false, }; diff --git a/src/engine/config.ts b/src/engine/config.ts index 58edfda..da1bdd2 100644 --- a/src/engine/config.ts +++ b/src/engine/config.ts @@ -3,8 +3,21 @@ // values are also usable in dev/tests. const CFG: Record = {}; -/** Merge an injected env map (decrypted keys + settings) into the runtime config. Last write wins. */ +/** REPLACE the runtime config with an injected env map (decrypted keys + settings), once per engine op. + * + * Clearing first is required, not cosmetic. The resolver emits several keys only for one model/backend + * combination, and some ops inject a one-shot override — so a merge lets a previous op's value survive into + * the next one and silently change how the model runs: + * - `VB_LOCAL_WAN_STEPS` is emitted only for the 5B (autoconfig) and injected only by `rerender-clips` + * (main/index.ts). Leaking it into a later 14B run forces that step count over the Lightning 4-step + * distillation AND makes `isHd` read false, so the HD path silently drops to a few-step, no-LoRA, + * tiny-VAE render — strictly worse than both Fast and HD. + * - `VB_LOCAL_WAN_DIR` / `VB_LOCAL_QUALITY` likewise persisted after the setting that produced them was + * cleared. + * Every op passes the full resolved env (main/index.ts `streamOp` spreads `sidecarEnv()`), so a replace is + * always complete. Reads still fall back to `process.env`, which stays the power-user override channel. */ export function setEnv(map: Record): void { + for (const k of Object.keys(CFG)) delete CFG[k]; for (const [k, v] of Object.entries(map)) { if (v != null && v !== '') CFG[k] = String(v); } diff --git a/src/engine/ffmpeg.ts b/src/engine/ffmpeg.ts index 8bea255..d6001f4 100644 --- a/src/engine/ffmpeg.ts +++ b/src/engine/ffmpeg.ts @@ -5,7 +5,7 @@ import { spawn } from 'node:child_process'; import fs from 'node:fs'; import ffmpegStatic from 'ffmpeg-static'; import ffprobeStatic from 'ffprobe-static'; -import { env, envInt } from './config'; +import { env, envInt, envBool } from './config'; import { tmp, copyIn, fileExists, fileSize } from './storage'; // Resolve a bundled binary path; when packaged inside app.asar the real file lives in app.asar.unpacked. @@ -103,12 +103,86 @@ export async function lastFrame(video: string, out: string): Promise { - if (!clips.length) return null; - const list = tmp(`concat_${Math.abs(out.split('').reduce((a, c) => (a * 31 + c.charCodeAt(0)) | 0, 7))}.txt`); +/** Mean luma + saturation of a still (signalstats), or null if unreadable. */ +export async function frameLevels(img: string): Promise<{ y: number; sat: number } | null> { + const r = await run(FFMPEG, ['-v', 'error', '-i', img, '-vf', 'signalstats,metadata=print:file=-', '-f', 'null', '-']); + const s = r.stdout.toString(); + const y = parseFloat((s.match(/lavfi\.signalstats\.YAVG=([\d.]+)/) || [])[1] || ''); + const sat = parseFloat((s.match(/lavfi\.signalstats\.SATAVG=([\d.]+)/) || [])[1] || ''); + return Number.isFinite(y) && Number.isFinite(sat) ? { y, sat } : null; +} + +// How far a single level-match may push exposure / saturation. Deliberately small: the correction must be +// able to cancel accumulated drift without ever overriding a scene's INTENTIONAL lighting change. +const MATCH_MAX_BRIGHT = 0.08; // eq brightness units (~±20 of 255) +const MATCH_MAX_SAT = 0.06; // ±6% saturation + +/** Nudge a chained start frame's exposure/saturation back toward its anchor keyframe, by a CLAMPED amount. + * + * Chained i2v drifts: each sub-clip is generated FROM the previous clip's last frame, so per-generation + * exposure/saturation error compounds with nothing pulling it back — a 12s scene is 6 links at the 14B's + * ~2.3s native clip, and the cross-scene chain adds more on top. The clamp is what makes this safe to apply + * unconditionally: with no drift the correction rounds to nothing and the source frame is returned + * untouched, and it can never move a frame far enough to fight a deliberate lighting change. + * Best-effort — any probe/encode failure returns the source frame. */ +export async function matchLevels(src: string, ref: string, out: string): Promise { + const [a, b] = await Promise.all([frameLevels(src), frameLevels(ref)]); + if (!a || !b || !(a.sat > 0)) return src; + const bright = Math.max(-MATCH_MAX_BRIGHT, Math.min(MATCH_MAX_BRIGHT, (b.y - a.y) / 255)); + const sat = Math.max(1 - MATCH_MAX_SAT, Math.min(1 + MATCH_MAX_SAT, b.sat / a.sat)); + if (Math.abs(bright) < 0.004 && Math.abs(sat - 1) < 0.01) return src; // below visibility — don't re-encode + const ok = await ffmpeg(['-i', src, '-vf', `eq=brightness=${bright.toFixed(4)}:saturation=${sat.toFixed(4)}`, out]); + return ok && fileSize(out) > 0 ? out : src; +} + +/** Write a concat-demuxer list file for `clips` and return its path. */ +function concatList(clips: string[], tag: string): string { + const list = tmp(`concat_${tag}.txt`); fs.writeFileSync(list, clips.map((c) => `file '${c.replace(/\\/g, '/')}'`).join('\n') + '\n'); - const ok = await ffmpeg(['-f', 'concat', '-safe', '0', '-i', list, ...x264(), '-an', out]); + return list; +} + +/** Concatenate clips into one video, audio stripped — STREAM-COPY when the inputs are compatible, else + * re-encode. + * + * Saving this hop matters: a local clip already goes through sub-clip concat -> window conform -> timeline + * concat -> upscale -> grade, and every x264 generation erodes the 480p texture that the upscaler then + * amplifies. Sub-clips of one scene come from the same model at the same settings, so `-c copy` is the + * normal case and it is bit-exact. + * + * The copy is VERIFIED, not assumed: the concat demuxer can exit 0 while producing a broken or truncated + * file when the inputs' codec parameters differ subtly, so the result is only accepted when its duration + * matches the sum of the inputs. Anything else falls back to the re-encode that was always here. */ +export async function concatClips(clips: string[], out: string, opts: { fps?: number } = {}): Promise { + if (!clips.length) return null; + const tag = String(Math.abs(out.split('').reduce((a, c) => (a * 31 + c.charCodeAt(0)) | 0, 7))); + const list = concatList(clips, tag); + if (envBool('VB_CONCAT_COPY', true)) { + const durs = await Promise.all(clips.map((c) => probeDuration(c))); + if (durs.every((d) => d != null && d > 0)) { + const want = (durs as number[]).reduce((a, b) => a + b, 0); + const copyOut = out.replace(/\.mp4$/, '') + '_copy.mp4'; + if (await ffmpeg(['-f', 'concat', '-safe', '0', '-i', list, '-c', 'copy', '-an', copyOut])) { + const got = await probeDuration(copyOut); + // 1 frame of slack: container rounding, not a dropped segment. + if (got != null && Math.abs(got - want) < 1 / FPS && fileSize(copyOut) > 0) { + try { + fs.renameSync(copyOut, out); + return out; + } catch { + /* cross-device or locked — fall through to the re-encode */ + } + } + } + try { + fs.rmSync(copyOut, { force: true }); + } catch { + /* best-effort cleanup */ + } + } + } + const rate = opts.fps ? ['-r', String(Math.trunc(opts.fps))] : []; + const ok = await ffmpeg(['-f', 'concat', '-safe', '0', '-i', list, ...x264(), ...rate, '-an', out]); return ok && fileExists(out) ? out : null; } @@ -135,7 +209,26 @@ export async function fitToWindow(raw: string, startSec: number, endSec: number, const rawDur = (await probeDuration(raw)) || target; let factor = rawDur && rawDur > 0 ? target / rawDur : 1.0; factor = Math.min(8.0, Math.max(0.05, factor)); - await ffmpeg(['-i', raw, '-vf', `setpts=${factor.toFixed(6)}*PTS,fps=${fps}`, '-frames:v', String(nFrames), ...x264(), '-an', out]); + const setpts = `setpts=${factor.toFixed(6)}*PTS`; + + // `setpts,fps=` resamples time by DUPLICATING or DROPPING whole frames — no motion compensation. The + // cloud provider only accepts whole-second durations, so a scene window is almost never an exact match + // and this retime happens on essentially every cloud clip: at factor 1.13 that is one duplicated frame in + // eight, which reads as steady judder. When the deviation is big enough to see, synthesize the in-between + // frames instead (same job RIFE does for the local path, which has no cloud equivalent). + // Gated: minterpolate is CPU-heavy at 720p, so it is skipped for deviations too small to notice. + const dev = Math.abs(factor - 1); + if (envBool('VB_SMOOTH_RETIME', true) && dev > (parseFloat(env('VB_SMOOTH_RETIME_MIN', '0.06')) || 0.06)) { + const mci = `${setpts},minterpolate=fps=${fps}:mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1`; + if (await ffmpeg(['-i', raw, '-vf', mci, '-frames:v', String(nFrames), ...x264(), '-an', out])) { + // Accept ONLY at the exact frame-grid length. The whole point of this conform is that scene lengths + // telescope with no drift, so a clip one frame short would desync the song — better to spend the + // encode twice than to let motion smoothing cost frame accuracy. + const got = await probeDuration(out); + if (got != null && fileSize(out) > 0 && Math.abs(got - target) < 0.5 / fps) return out; + } + } + await ffmpeg(['-i', raw, '-vf', `${setpts},fps=${fps}`, '-frames:v', String(nFrames), ...x264(), '-an', out]); return out; } diff --git a/src/engine/localVideo.ts b/src/engine/localVideo.ts index 86d98e8..03e7693 100644 --- a/src/engine/localVideo.ts +++ b/src/engine/localVideo.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import fs from 'node:fs'; import { env, envInt, envBool } from './config'; -import { vW, vH } from './ffmpeg'; +import { vW, vH, FPS } from './ffmpeg'; import { ensureSidecar, sidecarPost, readMarker } from './sidecar'; /** Which local i2v model (the Fast/Quality choice in Settings): @@ -60,7 +60,11 @@ export async function genVideoLocal(img: string, prompt: string, outMp4: string, // ── Wan 2.2 (5B / 14B) ───────────────────────────────────────────────────────────────────────── const is5b = model === '5b'; const nativeFps = localNativeFps(); - const isHd = !is5b && env('VB_LOCAL_QUALITY', 'fast') === 'hd' && !env('VB_LOCAL_WAN_STEPS'); + // HD is a pure QUALITY-setting question. It used to also require "no explicit step override", which made + // any injected VB_LOCAL_WAN_STEPS (rerender-clips, or a 5B run before setEnv gained replace semantics) + // silently demote an HD render to the fast VAE + the short deadline while ALSO skipping the Lightning + // LoRA — the worst of both paths. The step override still applies below; it just no longer redefines HD. + const isHd = !is5b && env('VB_LOCAL_QUALITY', 'fast') === 'hd'; const payload: Record = { model_dir: md, image: img, @@ -121,19 +125,46 @@ export async function genVideoLocal(img: string, prompt: string, outMp4: string, } } -/** RIFE 2x a sub-24fps clip (the 14B saves at Wan's native 16fps) so the 24fps timeline conform DECIMATES - * (32→24 drops 1 frame in 4) instead of duplicating every other frame — the duplication reads as constant - * judder. ~3s per clip on the Apple GPU (ncnn/MoltenVK). Best-effort: on any failure the raw clip stands. +/** Smallest interpolation factor that lands a sub-timeline clip on an EXACT multiple of the timeline fps. + * + * This is the difference between smooth and merely "less bad". The 14B saves at Wan's native 16fps; a plain + * 2x lands on 32fps, and the timeline conform (`fps=24`) then has to drop 1 frame in 4 at uneven phase — a + * repeating 4-frame cadence break that reads as micro-judder. 3x lands on 48fps, which decimates to 24 as a + * clean 2:1 (every second frame, perfectly even). RIFE v4.x is timestep-conditioned, so a 3x pass costs one + * extra synthesized frame per gap, not a second full pass. + * Returns 1 when the clip is already at/above the timeline rate (no interpolation needed). Falls back to 2 + * when no clean factor exists within `maxFactor` — still better than leaving it at native. */ +export function rifeFactor(nativeFps: number, timelineFps: number = FPS, maxFactor = 4): number { + if (!(nativeFps > 0) || nativeFps >= timelineFps) return 1; + for (let f = 2; f <= maxFactor; f++) if ((nativeFps * f) % timelineFps === 0) return f; + return 2; +} + +/** Interpolate a sub-timeline clip up to an exact multiple of the 24fps timeline (see rifeFactor), so the + * conform decimates evenly instead of duplicating or dropping frames at an uneven phase. ~3-5s per clip on + * the Apple GPU (ncnn/MoltenVK). Best-effort: on any failure the raw clip stands — but the failure is now + * LOGGED, because a silent fallback means every clip in that render conforms 16→24 by duplicating one frame + * in two (constant judder) with nothing in the UI or logs to explain why the result looks worse. * Per-clip ONLY — interpolating an assembled timeline would synthesize morph frames across scene cuts. */ async function rifeSmooth(clip: string, nativeFps: number, deadlineMs: number): Promise { - if (nativeFps >= 24 || !envBool('VB_LOCAL_RIFE', true)) return; + if (!envBool('VB_LOCAL_RIFE', true)) return; + const factor = rifeFactor(nativeFps); + if (factor < 2) return; const interp = clip.replace(/\.mp4$/, '_rife.mp4'); try { // timeout_sec: the sidecar kills its worker subprocess just before our HTTP deadline, so an abandoned // job can never sit on the GPU lock after we've given up (that would starve the next clip's /i2v). - const r = await sidecarPost('/interp', { video: clip, out: interp, out_fps: nativeFps * 2, timeout_sec: Math.max(60, Math.floor(deadlineMs / 1000) - 60) }, deadlineMs); - if (r.ok && fs.existsSync(interp) && fs.statSync(interp).size > 0) fs.renameSync(interp, clip); - } catch { - /* interpolation is optional polish — the 16fps clip still plays */ + const r = await sidecarPost( + '/interp', + { video: clip, out: interp, factor, out_fps: nativeFps * factor, timeout_sec: Math.max(60, Math.floor(deadlineMs / 1000) - 60) }, + deadlineMs, + ); + if (r.ok && fs.existsSync(interp) && fs.statSync(interp).size > 0) { + fs.renameSync(interp, clip); + return; + } + console.error(`[rife] interpolation did not produce output (${nativeFps}→${nativeFps * factor}fps): ${r?.error || 'unknown'} — clip stays at ${nativeFps}fps and will judder on the ${FPS}fps timeline`); + } catch (e: any) { + console.error(`[rife] interpolation failed (${nativeFps}→${nativeFps * factor}fps): ${e?.message || e} — clip stays at ${nativeFps}fps and will judder on the ${FPS}fps timeline`); } } diff --git a/src/engine/pipeline.ts b/src/engine/pipeline.ts index 649748c..345c37e 100644 --- a/src/engine/pipeline.ts +++ b/src/engine/pipeline.ts @@ -4,7 +4,7 @@ // Progress is reported through an `emit(event)` callback the engine turns into IPC events. import crypto from 'node:crypto'; import fs from 'node:fs'; -import { env, envInt } from './config'; +import { env, envInt, envBool } from './config'; import { costTotal } from './cost'; import * as S from './storage'; @@ -23,7 +23,7 @@ function fsReadHeadTail(path: string, size: number): Buffer { fs.closeSync(fd); } } -import { FPS, probeDuration, toPng, putThumb, stillClip, ffmpeg, toWav, x264, conformClip } from './ffmpeg'; +import { FPS, probeDuration, toPng, putThumb, stillClip, ffmpeg, toWav, x264, conformClip, concatClips } from './ffmpeg'; import * as P from './stages'; import { assertCastExists } from './backends/sceneShared'; import { ensureSidecar, sidecarPost } from './sidecar'; @@ -510,13 +510,16 @@ async function assemble(pid: string, emit: Emit): Promise<{ projectId: string; v // different native resolution — concatenating mixed dimensions corrupts the output, so normalize first. const normalized: string[] = []; for (let i = 0; i < clips.length; i++) normalized.push(await conformClip(clips[i], `${work}/clips/norm_${i}.mp4`)); - const concat = `${work}/concat.txt`; - S.writeText(concat, normalized.map((c) => `file '${c.replace(/\\/g, '/')}'`).join('\n') + '\n'); const silent = `${work}/output/silent.mp4`; - await ffmpeg(['-f', 'concat', '-safe', '0', '-i', concat, ...x264(), '-r', String(FPS), silent]); + // Stream-copy the timeline when the conformed clips are compatible — they all come out of the same + // window conform at the same size and rate, so this is the normal case and it removes one full x264 + // generation from every clip's path (render -> sub-clip concat -> conform -> HERE -> upscale -> grade). + // concatClips verifies the copy against the summed input duration and re-encodes at FPS if it doesn't fit. + await concatClips(normalized, silent, { fps: FPS }); // ── finish chain (both steps best-effort — on any failure the plain concat plays) ────────────── let master = silent; + let upscaled = false; // 1) Upscale: the local backend emits 832×480/896×512 — watched fullscreen that reads soft no matter how // good the denoise was. One Real-ESRGAN pass (sidecar /upscale, Apple GPU — idle by assemble time) // to 1080p. Runs once on the whole timeline: every clip + failed-scene fill shares one size here. The @@ -527,7 +530,10 @@ async function assemble(pid: string, emit: Emit): Promise<{ projectId: string; v const up = `${work}/output/upscaled.mp4`; const upSec = envInt('VB_UPSCALE_DEADLINE_SEC', 5400); const r = await sidecarPost('/upscale', { video: silent, out: up, target_h: envInt('VB_UPSCALE_H', 1080), timeout_sec: Math.max(60, upSec - 60) }, upSec * 1000); - if (r.ok && S.fileSize(up) > 0) master = up; + if (r.ok && S.fileSize(up) > 0) { + master = up; + upscaled = true; + } } catch { /* upscale is polish, never fail the render for it */ } @@ -538,11 +544,18 @@ async function assemble(pid: string, emit: Emit): Promise<{ projectId: string; v const finish = env('VB_FINISH', 'subtle'); if (finish !== 'off') { const graded = `${work}/output/graded.mp4`; + // On an UPSCALED master the two motion-hostile steps come off. hqdn3d's last two numbers are its + // TEMPORAL strength: it averages a pixel against the same pixel in neighbouring frames, which on a + // timeline whose in-between frames RIFE just synthesized smears real motion and can ghost fast pans — + // exactly the fluidity the interpolation pass was there to buy. And Real-ESRGAN already resolves edge + // detail, so a second unsharp on top rings them. Spatial denoise, grade and grain still run. + // VB_FINISH_TEMPORAL=1 / VB_FINISH_SHARPEN=1 restore the old chain. + const temporal = !upscaled || envBool('VB_FINISH_TEMPORAL', false); const vf = [ - 'hqdn3d=1.5:1.5:3:3', + temporal ? 'hqdn3d=1.5:1.5:3:3' : 'hqdn3d=1.5:1.5:0:0', "curves=master='0/0 0.25/0.22 0.5/0.5 0.75/0.79 1/1'", 'eq=saturation=1.06', - 'unsharp=5:5:0.35:5:5:0.0', + ...(!upscaled || envBool('VB_FINISH_SHARPEN', false) ? ['unsharp=5:5:0.35:5:5:0.0'] : []), ...(finish === 'filmic' ? ['vignette=PI/5'] : []), `noise=c0s=${finish === 'filmic' ? 7 : 4}:c0f=t+u`, ].join(','); diff --git a/src/main/autoconfig.ts b/src/main/autoconfig.ts index 022f0c4..5b16a7d 100644 --- a/src/main/autoconfig.ts +++ b/src/main/autoconfig.ts @@ -4,8 +4,9 @@ // I1 no key ⇒ local (first, un-overridable) I2 key ⇒ still local by default (cloud is explicit opt-in) // I3 tier never enables cloud I4 zero-key users stay fully local // Types are imported type-only, so this module has no runtime deps (no electron) and is unit-testable. -// Not yet wired into sidecarEnv — that lands at C5. +// Wired into sidecarEnv() in main/index.ts — every engine op is spawned with the env this resolver emits. import { STAGES } from './settingsSchema'; +import { TIMELINE_RES } from '../shared/videoRes'; import type { Settings, Stage, StageSelection, Backend } from './settingsSchema'; import type { LocalCapabilities } from './localModels'; @@ -64,10 +65,12 @@ function toEnv(stages: Record, settings: Settings): Record env.VB_OR_VIDEO_MODEL = settings.cloud.videoModel; env.VB_VLM_MODEL = settings.cloud.vlmModel; env.VB_MODERATION_MODEL = settings.cloud.moderationModel; - // Timeline resolution follows the VIDEO backend (local Wan 832×480; cloud Kling 1280×720). + // Timeline resolution follows the VIDEO backend. The numbers live in shared/videoRes.ts, which the engine + // backends' timelineRes() reads too — the two must not drift apart. const videoLocal = stages.VIDEO.backend === 'local'; - env.VB_W = videoLocal ? '832' : '1280'; - env.VB_H = videoLocal ? '480' : '720'; + const res = videoLocal ? TIMELINE_RES.local : TIMELINE_RES.cloud; + env.VB_W = String(res.w); + env.VB_H = String(res.h); // GPU serialization only when the video stage runs locally (one Wan run saturates unified memory). env.VB_WORKERS = videoLocal ? '1' : String(settings.workers || 4); // A local keyframe matches the timeline so it isn't resized into the clip. diff --git a/src/main/index.ts b/src/main/index.ts index 8764aa6..2fc2e80 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -341,9 +341,15 @@ function registerIpc() { guardRender(o.pid) ?? streamOp('render:' + o.pid, 'render', ['--project', o.pid, ...(o.preview ? ['--preview'] : []), ...(o.regenStory ? ['--regen-story'] : [])], undefined, renderNeedsGpu())); ipcMain.handle('render:resume', (_e, pid: string) => guardRender(pid) ?? streamOp('render:' + pid, 'resume', ['--project', pid], undefined, renderNeedsGpu())); - // Re-render the existing clips at quality (20 steps), reusing storyboard + keyframes — only the video step. + // Re-render the existing clips at higher quality, reusing storyboard + keyframes — only the video step. + // The step bump is 5B-ONLY. The 5B runs native steps (10 by default), so 20 is a real quality gain. The + // 14B's fast path is the Wan2.2-Lightning 4-step DISTILLATION: forcing 20 steps through it over-denoises + // into flat, slow-motion movement (see localVideo.ts) — the opposite of a quality re-render — and its HD + // path already runs the full 40-step model-config schedule. So on the 14B this op keeps the resolved + // settings and simply re-renders. ipcMain.handle('render:requality', (_e, pid: string) => - guardRender(pid) ?? streamOp('render:' + pid, 'rerender-clips', ['--project', pid], { VB_LOCAL_WAN_STEPS: '20' }, renderNeedsGpu())); + guardRender(pid) ?? + streamOp('render:' + pid, 'rerender-clips', ['--project', pid], getSettings().localVideoModel === '5b' ? { VB_LOCAL_WAN_STEPS: '20' } : undefined, renderNeedsGpu())); ipcMain.handle('scene:regenerate', (_e, o: { pid: string; index: number }) => guardRender(o.pid) ?? streamOp('render:' + o.pid, 'regenerate-scene', ['--project', o.pid, '--index', String(o.index)], undefined, renderNeedsGpu())); ipcMain.handle('op:cancel', (_e, opId: string) => { RUNS.get(opId)?.cancel(); return true; }); diff --git a/src/shared/videoRes.ts b/src/shared/videoRes.ts new file mode 100644 index 0000000..8610e38 --- /dev/null +++ b/src/shared/videoRes.ts @@ -0,0 +1,15 @@ +// The timeline resolution each VIDEO backend renders at — ONE definition, imported by both sides that +// need it, because they have to agree exactly: +// 1. main/autoconfig.ts turns it into VB_W / VB_H (what the engine actually generates and conforms to); +// 2. engine/backends/*/video.ts reports it as `timelineRes()` on the backend interface. +// These used to be two hand-copied pairs of literals with nothing tying them together, so a change to one +// silently desynced the other — and a mismatch is not loud: clips still render, they just get scaled and +// padded by conformClip at assemble time, costing resolution for no visible reason. +export const TIMELINE_RES = { + // Wan 2.2 i2v on MLX: the 48GB Metal working-set ceiling. Finished to 1080p by the ESRGAN upscale pass. + local: { w: 832, h: 480 }, + // Kling returns near-HD already, so the cloud path skips the upscale entirely. + cloud: { w: 1280, h: 720 }, +} as const; + +export type TimelineRes = { w: number; h: number }; diff --git a/test/render-quality.test.ts b/test/render-quality.test.ts new file mode 100644 index 0000000..a85d155 --- /dev/null +++ b/test/render-quality.test.ts @@ -0,0 +1,109 @@ +// Unit tests for the render-quality/fluidity invariants (pure — no electron, no ffmpeg, no sidecar). +// Each test here pins a defect that was silent in production: nothing crashed, the video just came out +// worse, which is exactly the class of bug a unit test has to catch instead of a human eye. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { setEnv, env, envInt } from '../src/engine/config'; +import { rifeFactor } from '../src/engine/localVideo'; +import { FPS } from '../src/engine/ffmpeg'; +import { TIMELINE_RES } from '../src/shared/videoRes'; +import { DEFAULTS, type Settings } from '../src/main/settingsSchema'; +import { resolveConfig, type KeyState } from '../src/main/autoconfig'; +import type { LocalCapabilities } from '../src/main/localModels'; + +const mk = (over: Partial = {}): Settings => ({ ...structuredClone(DEFAULTS), ...over }); +const CAPS = (supported: boolean): LocalCapabilities => ({ supported, depsInstalled: true } as LocalCapabilities); +const NO_KEYS: KeyState = { openrouter: false, replicate: false }; +const BOTH: KeyState = { openrouter: true, replicate: true }; + +// ── setEnv: per-op REPLACE, not merge ────────────────────────────────────────── +// The bug this pins: CFG was merge-only, so any key emitted for one model/op survived into the next one. +// `rerender-clips` injects VB_LOCAL_WAN_STEPS=20 and the resolver emits it for the 5B — either one used to +// poison every later 14B render in the same process: 20 steps forced through a 4-step Lightning +// distillation (flat, slow-motion output) and, worse, `isHd` read the leftover key and silently demoted an +// HD render to the fast VAE + short deadline while ALSO skipping the LoRA. + +test('setEnv replaces the previous op env — a one-shot override cannot leak into the next render', () => { + setEnv({ VB_LOCAL_VIDEO_MODEL: '5b', VB_LOCAL_WAN_STEPS: '20' }); + assert.equal(env('VB_LOCAL_WAN_STEPS'), '20'); + // Next op resolves a 14B render and emits no step override at all. + setEnv({ VB_LOCAL_VIDEO_MODEL: '14b', VB_LOCAL_QUALITY: 'hd' }); + assert.equal(env('VB_LOCAL_WAN_STEPS'), '', 'stale step override must not survive into the next op'); + assert.equal(env('VB_LOCAL_VIDEO_MODEL'), '14b'); + assert.equal(env('VB_LOCAL_QUALITY'), 'hd'); +}); + +test('setEnv clears every key, not just the ones the new map mentions', () => { + setEnv({ VB_LOCAL_WAN_DIR: '/old/model', VB_STT_LANG: 'it', VB_W: '832' }); + setEnv({ VB_W: '1280' }); + assert.equal(env('VB_LOCAL_WAN_DIR'), '', 'a cleared setting must not keep its old value'); + assert.equal(env('VB_STT_LANG'), ''); + assert.equal(env('VB_W'), '1280'); +}); + +test('setEnv drops empty values rather than storing them (defaults must win)', () => { + setEnv({ VB_WORKERS: '4' }); + setEnv({ VB_WORKERS: '' }); + assert.equal(envInt('VB_WORKERS', 7), 7, 'an empty injected value falls through to the caller default'); +}); + +// ── RIFE factor: land on an EXACT multiple of the timeline rate ──────────────── +// The bug this pins: a fixed 2x took the 14B's native 16fps to 32fps, and the 24fps conform then had to +// drop 1 frame in 4 at uneven phase — a repeating cadence break. 3x lands on 48fps, an exact 2:1. + +test('rifeFactor lands sub-timeline rates on an exact multiple of the timeline fps', () => { + for (const native of [8, 12, 16]) { + const f = rifeFactor(native); + assert.ok(f >= 2, `${native}fps must be interpolated (got factor ${f})`); + assert.equal((native * f) % FPS, 0, `${native}fps x${f} = ${native * f} must divide evenly into ${FPS}`); + } +}); + +test('rifeFactor picks 3 for the 14B native 16fps (48→24 is a clean 2:1, 32→24 is not)', () => { + assert.equal(rifeFactor(16), 3); + assert.notEqual(rifeFactor(16), 2, 'the old fixed 2x left an uneven 32→24 decimation'); +}); + +test('rifeFactor is a no-op at or above the timeline rate (the 5B is already 24fps)', () => { + assert.equal(rifeFactor(24), 1); + assert.equal(rifeFactor(30), 1); +}); + +test('rifeFactor never returns a factor that would cost more than one extra frame per gap', () => { + for (const native of [8, 10, 12, 15, 16, 20, 23]) assert.ok(rifeFactor(native) <= 4, `${native}fps`); +}); + +test('rifeFactor falls back to 2 (never 0/1) when no clean factor exists in range', () => { + const f = rifeFactor(23); // 23*n is never a multiple of 24 for n<=4 + assert.equal(f, 2, 'still interpolate — a sub-timeline clip must never be left to duplicate frames'); +}); + +test('rifeFactor tolerates a nonsense native rate instead of dividing by zero', () => { + assert.equal(rifeFactor(0), 1); + assert.equal(rifeFactor(-5), 1); +}); + +// ── timeline resolution: ONE definition ─────────────────────────────────────── +// The bug this pins: the resolver and the backend interface each carried their own hand-copied literals. +// A mismatch is silent — clips still render, conformClip just scales and pads them at assemble time, so +// the render quietly loses resolution with nothing to point at. + +test('the resolver emits the shared timeline resolution for a local VIDEO stage', () => { + const e = resolveConfig(CAPS(true), mk(), NO_KEYS).toEnv(); + assert.equal(e.VB_VIDEO_BACKEND, 'local'); + assert.equal(e.VB_W, String(TIMELINE_RES.local.w)); + assert.equal(e.VB_H, String(TIMELINE_RES.local.h)); +}); + +test('the resolver emits the shared timeline resolution for a cloud VIDEO stage', () => { + const e = resolveConfig(CAPS(true), mk({ backendPreference: 'prefer-cloud' }), BOTH).toEnv(); + assert.equal(e.VB_VIDEO_BACKEND, 'cloud'); + assert.equal(e.VB_W, String(TIMELINE_RES.cloud.w)); + assert.equal(e.VB_H, String(TIMELINE_RES.cloud.h)); +}); + +test('a local keyframe is generated at the timeline size (never resized into the clip)', () => { + const e = resolveConfig(CAPS(true), mk(), NO_KEYS).toEnv(); + assert.equal(e.VB_LOCAL_KEYFRAME_W, e.VB_W); + assert.equal(e.VB_LOCAL_KEYFRAME_H, e.VB_H); +});