From c50eb3ae2720cb196a003a6a55542f47b72a4706 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Thu, 18 Jun 2026 17:13:19 -0700 Subject: [PATCH 1/3] feat(ui): cached source-frame thumbnail endpoint for the filmstrip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/media/thumb?mediaId=&t=&h= extracts one small still from a SOURCE media file (not the composited timeline) via a fast input-seek frame grab (buildThumbnailArgs). Each thumb is keyed by (mediaId, t, h) and cached under the workspace, generated to a temp path then atomically renamed so concurrent filmstrip requests never serve a half-written file. Unknown media → 404; a non-thumbnailable source (e.g. audio-only) → 422, so a clip just falls back to its gradient block. --- src/ffmpeg/args/preview.ts | 32 ++++++++++++++++++++++++ src/ui/server.ts | 50 +++++++++++++++++++++++++++++++++++++- tests/preview.test.ts | 27 +++++++++++++++++++- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/src/ffmpeg/args/preview.ts b/src/ffmpeg/args/preview.ts index b1030c4..14550ba 100644 --- a/src/ffmpeg/args/preview.ts +++ b/src/ffmpeg/args/preview.ts @@ -16,3 +16,35 @@ export function buildPreviewFrameArgs({ input, output, atSec }: PreviewFrameArgs // -an skips audio decoding — irrelevant for a still frame. return ['-y', '-ss', String(atSec), '-i', input, '-vframes', '1', '-q:v', '2', '-an', output]; } + +export interface ThumbnailArgs { + input: string; + output: string; + /** Source timecode (seconds) to grab the thumbnail from. */ + atSec: number; + /** Target height in px; width follows the source aspect (even, for yuv420p). */ + height: number; +} + +/** + * A small, scaled still for the timeline filmstrip. Same fast input-seek as the + * preview frame, but scaled down (`-2` keeps the aspect with an even width) and + * a slightly cheaper JPEG — these are tiny and there are many per clip. + */ +export function buildThumbnailArgs({ input, output, atSec, height }: ThumbnailArgs): string[] { + return [ + '-y', + '-ss', + String(atSec), + '-i', + input, + '-vframes', + '1', + '-vf', + `scale=-2:${height}`, + '-q:v', + '4', + '-an', + output, + ]; +} diff --git a/src/ui/server.ts b/src/ui/server.ts index bf194e1..e600144 100644 --- a/src/ui/server.ts +++ b/src/ui/server.ts @@ -1,6 +1,6 @@ import { randomBytes } from 'node:crypto'; import { createWriteStream, existsSync } from 'node:fs'; -import { mkdir, readdir, readFile, stat, unlink } from 'node:fs/promises'; +import { mkdir, readdir, readFile, rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, resolve } from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; @@ -11,7 +11,9 @@ import getPort from 'get-port'; import { Hono } from 'hono'; import open from 'open'; import { ZodError, z } from 'zod'; +import { buildThumbnailArgs } from '../ffmpeg/args/preview.js'; import { probe } from '../ffmpeg/probe.js'; +import { runFfmpeg } from '../ffmpeg/run.js'; import { appendOp, readSession, SessionCorruptError, snapshotsDir } from '../session/store.js'; import type { SessionEntry } from '../session/types.js'; import { @@ -31,6 +33,7 @@ import { import { buildMediaMap } from '../timeline/media-registry.js'; import { CompositionOpError } from '../timeline/ops.js'; import { runPlan } from '../timeline/run-plan.js'; +import type { MediaId } from '../timeline/schema.js'; import { CompositionVerbSchema } from '../timeline/verbs.js'; import { ingest } from '../tools/ingest.js'; import { preview } from '../tools/preview.js'; @@ -252,6 +255,51 @@ export async function startUiServer(options: UiServerOptions = {}): Promise { + const mediaId = c.req.query('mediaId') ?? ''; + const t = Number(c.req.query('t') ?? '0'); + const h = Math.min(Math.max(Math.round(Number(c.req.query('h') ?? '48')), 16), 240); + if (!mediaId || Number.isNaN(t) || t < 0) + return c.json({ error: 'mediaId and t≥0 required' }, 400); + + const thumbsDir = resolve(getWorkspace(), '.thumbs'); + const safeKey = `${mediaId}-${t.toFixed(2)}-${h}`.replace(/[^\w.-]/g, '_'); + const out = resolve(thumbsDir, `${safeKey}.jpg`); + + const serve = async () => + c.body(await readFile(out), 200, { + 'Content-Type': 'image/jpeg', + 'Cache-Control': 'public, max-age=31536000, immutable', + }); + + // Cache hit — the file's existence means this mediaId was already validated. + if (existsSync(out)) return serve(); + + const info = (await buildMediaMap()).get(mediaId as MediaId); + if (!info) return c.json({ error: `Unknown media: ${mediaId}` }, 404); + try { + await mkdir(thumbsDir, { recursive: true }); + // Generate to a unique temp path then atomically rename, so concurrent + // requests for the same thumb never serve a half-written file. + const tmp = resolve(thumbsDir, `${safeKey}.${randomBytes(4).toString('hex')}.tmp.jpg`); + await runFfmpeg(buildThumbnailArgs({ input: info.path, output: tmp, atSec: t, height: h })); + await rename(tmp, out); + return serve(); + } catch (err) { + // A non-thumbnailable source (e.g. audio-only) or a transient FFmpeg failure + // — the clip just falls back to its gradient block. Clean error, not a 500. + const message = err instanceof Error ? err.message : String(err); + return c.json({ error: message }, 422); + } + }); + /** * POST /api/timeline/export — compile the whole document to a rendered file and * log it to the session so it appears in Outputs and is playable. Returns the diff --git a/tests/preview.test.ts b/tests/preview.test.ts index 32432ad..5542bba 100644 --- a/tests/preview.test.ts +++ b/tests/preview.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { buildPreviewFrameArgs } from '../src/ffmpeg/args/preview.js'; +import { buildPreviewFrameArgs, buildThumbnailArgs } from '../src/ffmpeg/args/preview.js'; import { PreviewInput } from '../src/tools/preview.js'; describe('buildPreviewFrameArgs', () => { @@ -53,6 +53,31 @@ describe('buildPreviewFrameArgs', () => { }); }); +describe('buildThumbnailArgs', () => { + it('scales to the requested height (even width) with a single fast-seek frame', () => { + expect(buildThumbnailArgs({ input: 'in.mp4', output: 't.jpg', atSec: 2, height: 48 })).toEqual([ + '-y', + '-ss', + '2', + '-i', + 'in.mp4', + '-vframes', + '1', + '-vf', + 'scale=-2:48', + '-q:v', + '4', + '-an', + 't.jpg', + ]); + }); + + it('honors a different height', () => { + const args = buildThumbnailArgs({ input: 'a', output: 'b.jpg', atSec: 0, height: 96 }); + expect(args[args.indexOf('-vf') + 1]).toBe('scale=-2:96'); + }); +}); + describe('PreviewInput', () => { it('accepts a valid input', () => { expect(() => PreviewInput.parse({ input: 'video.mp4', atSec: 1.5 })).not.toThrow(); From 8b96d5f30fc5a34231e3381710f0f16af5df544e Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Thu, 18 Jun 2026 17:13:19 -0700 Subject: [PATCH 2/3] feat(ui): render frame-thumbnail filmstrips on timeline clips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Media clips now show a row of source-frame thumbnails across their trimmed window — a filmstrip — instead of a flat gradient block. The clip samples one /api/media/thumb still per ~72px slot (capped at 16), at the middle of each slot; the gradient shows underneath until the frames load, a scrim keeps the label/duration legible, and text/color clips keep their solid block. Thumbs are lazy-loaded so long timelines don't fetch off-screen frames. --- src/ui/web/src/components/DocTimeline.tsx | 36 +++++++++++++++++++- src/ui/web/src/styles.css | 40 +++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/ui/web/src/components/DocTimeline.tsx b/src/ui/web/src/components/DocTimeline.tsx index cab9989..dc471c7 100644 --- a/src/ui/web/src/components/DocTimeline.tsx +++ b/src/ui/web/src/components/DocTimeline.tsx @@ -1,10 +1,13 @@ import { useEffect, useState } from 'react'; import { clipDuration, clipEndSec, clipLabel, compositionDuration } from '../lib/composition.js'; -import type { Clip, Composition } from '../types.js'; +import type { Clip, Composition, MediaClip } from '../types.js'; const PX_PER_SEC = 90; const GUTTER = 128; // left lane-label column width (px) const MIN_SPAN_SEC = 6; // keep an empty/short timeline from collapsing +const THUMB_H = 48; // filmstrip thumbnail height (px) — the clip's inner height +const TARGET_THUMB_W = 72; // desired filmstrip frame slot width (px) +const MAX_THUMBS = 16; // cap thumbnails per clip to bound requests /** * Read view of the CompositionDoc: a seconds ruler, one lane per track, clips @@ -189,8 +192,39 @@ function ClipBlock({ onClick={() => onSelect(selected ? null : clip.id)} title={`${clipLabel(clip)} · ${clip.startSec.toFixed(2)}–${clipEndSec(clip).toFixed(2)}s`} > + {clip.kind === 'media' ? : null} {clipLabel(clip)} {clipDuration(clip).toFixed(1)}s ); } + +/** + * A filmstrip: a row of source-frame thumbnails across the clip's trimmed window. + * Each frame is a cached `/api/media/thumb` still sampled at the middle of its + * slot; the clip's gradient shows underneath until they load. + */ +function Filmstrip({ clip, width }: { clip: MediaClip; width: number }) { + const n = Math.max(1, Math.min(MAX_THUMBS, Math.round(width / TARGET_THUMB_W))); + const { mediaId, sourceInSec, sourceOutSec } = clip; + const slotW = width / n; + return ( + - {hasClips ? ( - onScrub(Number(e.target.value))} - aria-label="Playhead position (seconds)" - /> - ) : null} ▶ {playhead.toFixed(2)}s