diff --git a/.gitignore b/.gitignore index 33cf7a9d2f..422b1b9bff 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ result-* # Auto-caption model + ORT wasm — regenerated at build by scripts/fetch-caption-model.mjs /caption-assets/ +public/caption-assets diff --git a/README.md b/README.md index 6d098d97c2..433ac1055e 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,19 @@ The goal of this continuation is to keep OpenScreen alive as a fully open-source - Export to MP4 or GIF in multiple aspect ratios and resolutions. - Languages supported: Arabic, English, Spanish, French, Italian, Japanese, Korean, Portuguese (Brazil), Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. +## Command-line interface (headless) + +OpenScreen ships a CLI for scripts, CI, and AI coding agents: record the screen +headlessly, edit the `.openscreen` project JSON programmatically (zooms, +annotations, trims), and render MP4/GIF with the full export pipeline — no +visible windows, NDJSON output with `--json`. + +```bash +openscreen record --duration 20 --project demo.openscreen --json +openscreen export demo.openscreen -o demo.mp4 --json +``` + +See [docs/cli.md](./docs/cli.md). ## Installation diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000000..adba6470a7 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,288 @@ +# OpenScreen CLI + +Headless command-line interface for recording the screen and exporting `.openscreen` +projects — no visible windows, machine-readable output. Designed so scripts, CI +pipelines, and AI coding agents can produce polished product demos automatically: + +```text +record → edit the project JSON programmatically → export → MP4/GIF +``` + +## Running + +Development (after `npm run build-vite` and, for recording, the native helper build): + +```bash +npm run cli -- [options] +# or directly: +./node_modules/.bin/electron . [options] +``` + +Packaged app (the CLI ships inside the normal binary): + +```bash +# macOS +/Applications/Openscreen.app/Contents/MacOS/Openscreen export demo.openscreen -o demo.mp4 +# Windows +"C:\Program Files\Openscreen\Openscreen.exe" export demo.openscreen -o demo.mp4 +``` + +CLI runs skip the single-instance lock, so they work while the GUI app is open. + +## Commands + +### `openscreen record` + +Records headlessly through the same pipeline as the GUI: the native +ScreenCaptureKit helper on macOS, the WGC helper on Windows (browser capture as +fallback). Recordings land in the app's recordings directory +(`/recordings/`), exactly like GUI recordings — including the +`.cursor.json` cursor-telemetry sidecar used for editable cursors and auto-zoom. + +```bash +openscreen record --duration 30 --project demo.openscreen --json +openscreen record --window "My App" --mic --system-audio +openscreen record --display 1 --cursor system +``` + +| Option | Meaning | +|---|---| +| `--display ` | Screen index to record (default 0) | +| `--window ` | Record the first window whose title contains `<title>` | +| `--mic` / `--mic-device <name>` | Capture microphone (optionally by device-label substring) | +| `--system-audio` | Capture system audio | +| `--cursor <editable-overlay\|system>` | Hide the system cursor and record telemetry (default), or bake it into the video | +| `--duration <seconds>` | Stop automatically | +| `--project <out.openscreen>` | Write a ready-to-export project file when done | +| `--json` | NDJSON events on stdout | + +Stopping without `--duration`: send SIGINT/SIGTERM to the process, or type +`stop` + Enter on its stdin. + +Platform notes: + +- **macOS**: requires the Swift helper (`npm run build:native:mac`, needs Xcode) + and the Screen Recording permission for whatever binary hosts Electron + (your terminal during development). Webcam capture is not available in CLI + recording on macOS (same limitation as the native helper). +- **Windows**: requires the WGC helper (`npm run build:native:win`). SIGTERM + does not exist on Windows — stop recordings with Ctrl+C, stdin `stop`, or + `--duration` (a hard `taskkill` loses the recording). Microphone access is + gated by Settings → Privacy → Microphone; there is no programmatic prompt. +- **Linux**: uses the browser capture pipeline; cursor options are limited, + matching the GUI. On Wayland, capture goes through the PipeWire portal, + which may show a system picker dialog and requires a desktop session + (headless/SSH sessions without a portal cannot record). + +### Multi-window demos: `--windows` (clean capture + slide transitions) + +The flagship mode for product demos. Every listed window is captured +**continuously and cleanly in parallel** (ScreenCaptureKit window capture — no +desktop background, no overlap, works even while occluded), together with a +focus timeline. The export step then composites one video that *switches to +whichever window you focused*, with the incoming window sliding in from the +side it sat on: + +```bash +openscreen record --windows "Terminal,Code,Chrome" --project demo.openscreen +# ...work normally across the three windows... +openscreen export demo.openscreen -o demo.mp4 --follow-windows +``` + +- Focus held ≥1.2 s switches the screen; brief flickers are ignored; focusing + an unlisted window keeps the current one on screen. +- Transitions are 450 ms eased slides; the direction follows screen geometry + (a window to the right slides in from the right). +- All windows must live in one capture process: concurrent helper *processes* + interrupt each other's streams (SCStreamErrorDomain -3805), so the Swift + helper gained a `multiWindows` mode running N streams in-process. +- macOS only; no audio yet (add narration with `export --audio`). Composes + with wallpaper/padding/annotations and `--audio` like any export. + +### Display recording: `--follow-windows` zoom mode + +One window is rarely enough for a product demo — a real workflow moves between +a terminal, an editor, a browser. `--follow-windows` records the whole display +**plus a window-focus timeline** (a `<video>.focus.json` sidecar produced by a +native helper, macOS only for now), and the export step turns that timeline +into zoom regions that cinematically pan/zoom to whichever window you focused: + +```bash +openscreen record --display 0 --follow-windows --project demo.openscreen +# ...switch between your terminal, editor, browser while recording... +openscreen export demo.openscreen -o demo.mp4 --follow-windows +``` + +Behavior: + +- Each window focused for ≥1.5 s becomes a zoom region framing that window + (6% margin, zoom capped at 5×); the renderer's zoom spring animates the + camera between windows instead of hard-cutting. +- Near-fullscreen windows produce no zoom — the camera pulls back to the full + frame. +- Brief detours (<1.5 s) are ignored, and regions never overlap zooms already + in the project, so manual edits win. +- Combines with `--auto-zoom` (cursor-dwell zooms fill the gaps between + window switches) and all other export options. + +### `openscreen sources` + +Lists capturable displays, windows, and microphones — the same enumeration the +GUI picker uses — so scripts and agents can choose `--display`, `--window`, and +`--mic-device` values without guesswork. + +```bash +openscreen sources # human-readable +openscreen sources --json +``` + +`--json` emits the payload on the final `done` event: + +```json +{ + "event": "done", + "success": true, + "sources": { + "displays": [{ "index": 0, "id": "screen:1:0", "name": "Entire screen" }], + "windows": [{ "id": "window:210:0", "name": "My App" }], + "microphones": [{ "label": "MacBook Pro Microphone (Built-in)" }], + "microphoneLabelsUnavailable": false + } +} +``` + +### `openscreen export` + +Renders a project to MP4 or GIF using the app's real export pipeline (WebCodecs + +PixiJS, faster than realtime) in a hidden window. Falls back to SwiftShader when +no GPU is available (CI), and applies everything the editor would: zooms, trims, +speed regions, wallpaper/padding, annotations, cursor rendering, webcam layouts. + +```bash +openscreen export demo.openscreen # format/quality from the project +openscreen export demo.openscreen -o out.mp4 --quality source +openscreen export demo.openscreen -o out.gif --gif-fps 20 --gif-size large +openscreen export demo.openscreen --json | while read line; do ...; done +``` + +| Option | Meaning | +|---|---| +| `-o, --out <path>` | Output file; extension picks the format. Default: next to the project | +| `--format <mp4\|gif>` | Override the project's stored format | +| `--quality <medium\|good\|source>` | MP4 quality | +| `--gif-fps <15\|20\|25\|30>`, `--gif-size <medium\|large\|original>` | GIF settings | +| `--preview-size <WxH>` | Reference preview box (default `1280x720`), see below | +| `--auto-zoom` | Add automatic zooms from cursor telemetry before rendering — the same dwell-detection engine as the editor's magic wand. Existing zoom regions are kept; suggestions never overlap them | +| `--audio <file>` | Mix a voiceover file into the MP4 (mp3/wav/m4a — anything Chromium can decode; AIFF is not supported) | +| `--audio-mode <mix\|replace>` | Layer the voiceover over the recording's audio (default `mix`) or replace it | +| `--audio-offset <seconds>` | Delay before the voiceover starts (default 0) | +| `--json` | NDJSON progress + result on stdout | + +`--audio` re-muxes after the render: video packets are copied untouched, and the +audio is mixed offline (OfflineAudioContext) and re-encoded to AAC. MP4 only. +In `mix` mode the original audio is ducked to 40% under the voiceover so the +sum cannot clip; use `replace` to drop the original entirely. + +**Media path rule**: for safety, a project's referenced media is only auto-approved +when it lives in the app's recordings directory or **next to the project file**. +Keep `.openscreen` files beside their media (or record via the CLI, which uses +the recordings directory). + +**`--preview-size`**: annotation font sizes and border radii are stored in +preview-pixel space and scaled by `export size / preview size` — in the GUI the +"preview" is the editor window, so results depend on window size. The CLI uses a +deterministic reference box instead (the composition fitted into 1280×720). +Authoring tip for scripts: treat annotation `fontSize` as "pixels in a +1280-wide preview". + +### `openscreen pack` + +Copies a project and everything it references (screen/webcam video, cursor +telemetry sidecar) into one portable folder and rewrites the project's media +paths: + +```bash +openscreen pack demo.openscreen --out bundle/ +``` + +The folder survives being moved or shipped as a CI artifact: when the stored +absolute paths go stale, the loader falls back to files with the same basename +next to the project file. + +### `openscreen captions` + +Transcribes the project's audio with the app's on-device Whisper model (no +upload; language auto-detected) and writes the resulting caption annotations +into the project. Re-running replaces earlier auto-captions; manual annotations +are preserved. + +```bash +openscreen captions demo.openscreen --min-words 2 --max-words 7 +openscreen export demo.openscreen -o demo.mp4 # subtitles are burned in +``` + +Requires an audio track in the project's video (e.g. `record --mic`, or a +voiceover mixed in with a re-recorded source). Accuracy reflects the bundled +whisper-tiny model: strong for English, rough for other languages. + +### `openscreen info` + +Prints a project summary (referenced media and whether it exists, format, +region counts). Exits non-zero if the referenced video is missing. + +```bash +openscreen info demo.openscreen --json +``` + +## Machine-readable output (`--json`) + +One JSON object per line on stdout (NDJSON). stderr carries diagnostics only. + +```jsonl +{"event":"started","command":"export"} +{"event":"progress","percentage":42,"currentFrame":50,"totalFrames":120,"estimatedTimeRemaining":3} +{"event":"done","success":true,"outputPath":"/path/out.mp4","format":"mp4","width":1920,"height":1080} +``` + +Record emits `log` events (`Recording started`, …), `stopping`, and a final +`done` carrying `screenVideoPath`, `cursorDataPath`, `durationMs`, and +`projectPath` when `--project` was used. Exit code is 0 on success, 1 on +failure, 2 on bad arguments. + +## Example: automated product demo (for scripts/agents) + +```bash +# 1. Record 20 seconds of the running app +openscreen record --window "MyProduct" --duration 20 --project demo.openscreen --json + +# 2. Edit the project: add a zoom and a caption (plain JSON) +node -e ' + const fs = require("fs"); + const p = JSON.parse(fs.readFileSync("demo.openscreen", "utf8")); + p.editor.zoomRegions.push({ id: "z1", startMs: 2000, endMs: 6000, depth: 3, + focus: { cx: 0.5, cy: 0.4 }, focusMode: "manual", source: "manual" }); + p.editor.annotationRegions.push({ id: "a1", startMs: 500, endMs: 4000, + type: "text", content: "One-click setup", textContent: "One-click setup", + position: { x: 8, y: 6 }, size: { width: 40, height: 12 }, + style: { fontSize: 24, color: "#fff" }, zIndex: 1 }); + fs.writeFileSync("demo.openscreen", JSON.stringify(p, null, 2)); +' + +# 3. Narrate with any TTS (macOS `say` shown; any engine producing mp3/wav/m4a works) +say -o voice.m4a --file-format=m4af "Welcome to my product. Here's a quick tour." + +# 4. Render with auto-zooms; the voiceover replaces the recording's own audio +# (drop --audio-mode replace to duck the original under the narration instead) +openscreen export demo.openscreen -o demo.mp4 --auto-zoom --audio voice.m4a --audio-mode replace --json +``` + +## Architecture + +- `electron/cli/args.ts` — pure argv parser (unit-tested in `args.test.ts`). +- `electron/cli/cliMain.ts` — headless boot: no HUD/tray/menu/dock, stdio + protocol, signal handling, exit codes. Registers the same IPC surface as the + GUI (`registerIpcHandlers`) with inert window callbacks. +- `src/cli/CliExportRunner.tsx` / `src/cli/CliRecordRunner.tsx` — hidden-window + runners (`?windowType=cli-export|cli-record`) that drive the existing + exporter classes and `useScreenRecorder` hook. +- Contracts shared between main and renderer: `src/lib/cliContracts.ts`. diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts new file mode 100644 index 0000000000..3c00bc2002 --- /dev/null +++ b/electron/cli/args.test.ts @@ -0,0 +1,203 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { parseCliArgs } from "./args"; + +const CWD = path.resolve("/work"); +const inCwd = (name: string) => path.resolve(CWD, name); + +function parse(args: string[]) { + return parseCliArgs(["electron", "app-path", ...args], 2, CWD); +} + +describe("parseCliArgs", () => { + it("returns null when no subcommand is present (GUI launch)", () => { + expect(parse([])).toBeNull(); + expect(parse(["--some-chromium-flag"])).toBeNull(); + }); + + it("skips leading Chromium switches before the subcommand (AppImage --no-sandbox)", () => { + expect(parse(["--no-sandbox", "export", "demo.openscreen"])).toMatchObject({ + kind: "export", + projectPath: inCwd("demo.openscreen"), + }); + expect(parse(["--no-sandbox", "--enable-unsafe-swiftshader", "record"])).toMatchObject({ + kind: "record", + }); + expect(parse(["--no-sandbox", "--help"])).toMatchObject({ kind: "help" }); + expect(parse(["--no-sandbox"])).toBeNull(); + }); + + it("parses a minimal export command and resolves relative paths", () => { + const cmd = parse(["export", "demo.openscreen"]); + expect(cmd).toMatchObject({ + kind: "export", + projectPath: inCwd("demo.openscreen"), + outPath: null, + format: null, + }); + }); + + it("parses export options and infers format from --out extension", () => { + const cmd = parse([ + "export", + "/p/demo.openscreen", + "-o", + "out.gif", + "--gif-fps", + "20", + "--preview-size", + "1600x900", + "--json", + ]); + expect(cmd).toMatchObject({ + kind: "export", + outPath: inCwd("out.gif"), + format: "gif", + gifFrameRate: 20, + previewWidth: 1600, + previewHeight: 900, + json: true, + }); + }); + + it("rejects a --format that conflicts with the --out extension", () => { + const cmd = parse(["export", "a.openscreen", "-o", "x.mp4", "--format", "gif"]); + expect(cmd).toMatchObject({ kind: "error" }); + }); + + it("rejects export without a project path", () => { + expect(parse(["export"])).toMatchObject({ kind: "error" }); + }); + + it("parses voiceover audio options", () => { + const cmd = parse([ + "export", + "a.openscreen", + "--audio", + "voice.mp3", + "--audio-mode", + "replace", + "--audio-offset", + "1.5", + ]); + expect(cmd).toMatchObject({ + kind: "export", + audioPath: inCwd("voice.mp3"), + audioMode: "replace", + audioOffsetSec: 1.5, + }); + }); + + it("defaults audio mode to mix and rejects --audio with gif", () => { + expect(parse(["export", "a.openscreen", "--audio", "v.mp3"])).toMatchObject({ + audioMode: "mix", + audioOffsetSec: 0, + }); + expect(parse(["export", "a.openscreen", "--audio", "v.mp3", "--format", "gif"])).toMatchObject({ + kind: "error", + }); + expect(parse(["export", "a.openscreen", "--audio-offset", "-1"])).toMatchObject({ + kind: "error", + }); + }); + + it("parses record defaults", () => { + expect(parse(["record"])).toMatchObject({ + kind: "record", + displayIndex: 0, + windowTitle: null, + mic: false, + systemAudio: false, + cursorMode: "editable-overlay", + durationMs: null, + }); + }); + + it("parses record options", () => { + const cmd = parse([ + "record", + "--display", + "1", + "--mic-device", + "MacBook", + "--system-audio", + "--duration", + "12.5", + "--project", + "demo.openscreen", + ]); + expect(cmd).toMatchObject({ + kind: "record", + displayIndex: 1, + mic: true, + micDevice: "MacBook", + systemAudio: true, + durationMs: 12500, + projectOut: inCwd("demo.openscreen"), + }); + }); + + it("rejects invalid record values", () => { + expect(parse(["record", "--duration", "0"])).toMatchObject({ kind: "error" }); + expect(parse(["record", "--cursor", "off"])).toMatchObject({ kind: "error" }); + expect(parse(["record", "--project", "demo.json"])).toMatchObject({ kind: "error" }); + }); + + it("parses --auto-zoom", () => { + expect(parse(["export", "a.openscreen", "--auto-zoom"])).toMatchObject({ + kind: "export", + autoZoom: true, + }); + expect(parse(["export", "a.openscreen"])).toMatchObject({ autoZoom: false }); + }); + + it("parses sources", () => { + expect(parse(["sources", "--json"])).toMatchObject({ kind: "sources", json: true }); + expect(parse(["sources", "--bogus"])).toMatchObject({ kind: "error" }); + expect(parse(["sources", "extra-arg"])).toMatchObject({ kind: "error" }); + }); + + it("parses pack", () => { + expect(parse(["pack", "demo.openscreen", "--out", "bundle"])).toMatchObject({ + kind: "pack", + projectPath: inCwd("demo.openscreen"), + outDir: inCwd("bundle"), + }); + expect(parse(["pack", "demo.openscreen"])).toMatchObject({ kind: "error" }); + expect(parse(["pack", "--out", "bundle"])).toMatchObject({ kind: "error" }); + expect(parse(["pack", "demo.openscreen", "--out", "bundle", "--bogus"])).toMatchObject({ + kind: "error", + }); + expect(parse(["pack", "a.openscreen", "b.openscreen", "--out", "bundle"])).toMatchObject({ + kind: "error", + }); + }); + + it("parses captions", () => { + expect( + parse(["captions", "demo.openscreen", "--min-words", "1", "--max-words", "5"]), + ).toMatchObject({ + kind: "captions", + projectPath: inCwd("demo.openscreen"), + minWordsPerCaption: 1, + maxWordsPerCaption: 5, + }); + expect(parse(["captions", "demo.openscreen"])).toMatchObject({ + minWordsPerCaption: 2, + maxWordsPerCaption: 7, + }); + expect( + parse(["captions", "a.openscreen", "--min-words", "9", "--max-words", "3"]), + ).toMatchObject({ kind: "error" }); + }); + + it("parses info and help", () => { + expect(parse(["info", "demo.openscreen", "--json"])).toMatchObject({ + kind: "info", + projectPath: inCwd("demo.openscreen"), + json: true, + }); + expect(parse(["help"])).toMatchObject({ kind: "help" }); + expect(parse(["--help"])).toMatchObject({ kind: "help" }); + }); +}); diff --git a/electron/cli/args.ts b/electron/cli/args.ts new file mode 100644 index 0000000000..179f9a2b3b --- /dev/null +++ b/electron/cli/args.ts @@ -0,0 +1,491 @@ +// Pure argv parser for the OpenScreen CLI. No Electron imports so it can be +// unit-tested under plain vitest. + +import path from "node:path"; +import type { CliExportRequest, CliRecordRequest, CliRequest } from "../../src/lib/cliContracts"; + +export interface CliInfoCommand { + kind: "info"; + projectPath: string; +} + +export interface CliHelpCommand { + kind: "help"; +} + +export interface CliErrorCommand { + kind: "error"; + message: string; +} + +export type CliCommand = ( + | CliRequest + | CliInfoCommand + | CliPackCommand + | CliHelpCommand + | CliErrorCommand +) & { + /** Machine-readable NDJSON output on stdout instead of human progress. */ + json?: boolean; +}; + +const SUBCOMMANDS = new Set([ + "export", + "record", + "sources", + "pack", + "captions", + "info", + "help", + "--help", + "-h", +]); + +export const CLI_USAGE = `OpenScreen CLI + +Usage: + openscreen export <project.openscreen> [options] Render a project to MP4/GIF + openscreen record [options] Record the screen headlessly + openscreen sources [--json] List displays, windows and microphones + openscreen pack <project.openscreen> --out <dir> Copy project + media into one portable folder + openscreen captions <project.openscreen> Add auto-captions (on-device Whisper) to a project + [--min-words <n>] [--max-words <n>] + openscreen info <project.openscreen> [--json] Inspect a project file + openscreen help Show this help + +Export options: + -o, --out <path> Output file (.mp4 or .gif). Default: next to the project file + --format <mp4|gif> Override the format stored in the project + --quality <medium|good|source> + MP4 quality (default: from project) + --gif-fps <15|20|25|30> GIF frame rate (default: from project) + --gif-size <medium|large|original> + GIF size preset (default: from project) + --preview-size <WxH> Reference preview box for annotation scaling (default 1280x720) + --auto-zoom Add automatic zooms from cursor telemetry (editor's magic wand) + --follow-windows Pan/zoom to the focused window using telemetry from + "record --follow-windows" + --audio <file> Mix a voiceover audio file into the MP4 (mp3/wav/m4a) + --audio-mode <mix|replace> + Layer over the recording's audio (default) or replace it + --audio-offset <seconds> Delay before the voiceover starts (default 0) + --json NDJSON progress/result on stdout + +Record options (recording is saved into the app's recordings directory): + --display <n> Screen index to record (default 0) + --window <title> Record the first window whose title contains <title> + --mic Capture the default microphone + --mic-device <name> Microphone device name (implies --mic) + --system-audio Capture system audio + --cursor <editable-overlay|system> + Cursor capture mode (default editable-overlay) + --duration <seconds> Stop automatically after this long + --project <out.openscreen> + Write a ready-to-export project file when done + --follow-windows Also record which window has focus (macOS); pair with + "export --follow-windows" for automatic window switching + --windows <t1,t2,...> Multi-window capture (macOS): record every matched window + in parallel; "export --follow-windows" switches between + them with slide transitions as focus changes + --json NDJSON events on stdout + +Stopping a recording: send SIGINT/SIGTERM, type "stop" + Enter on stdin, +or pass --duration. +`; + +function takeValue(argv: string[], i: number, flag: string): [string, number] { + const next = argv[i + 1]; + if (next === undefined || next.startsWith("--")) { + throw new Error(`${flag} requires a value`); + } + return [next, i + 1]; +} + +function resolvePath(p: string, cwd: string): string { + return path.isAbsolute(p) ? p : path.resolve(cwd, p); +} + +/** + * Extracts CLI arguments from process.argv. Returns null when no subcommand is + * present (normal GUI launch). `firstArgIndex` should be 1 for packaged builds + * and 2 when running via `electron <app-path> ...`. + */ +export function parseCliArgs( + argv: string[], + firstArgIndex: number, + cwd: string = process.cwd(), +): CliCommand | null { + const rawArgs = argv.slice(firstArgIndex).filter((a) => !a.startsWith("--inspect")); + + // Skip *leading* Chromium/Electron switches (e.g. the AppImage's required + // `--no-sandbox`) so `Openscreen --no-sandbox export demo.openscreen` still + // enters CLI mode. Only leading dash-tokens are skipped — everything after + // the subcommand belongs to the subcommand parser. `--help`/`-h` are ours. + let subIndex = 0; + while ( + subIndex < rawArgs.length && + rawArgs[subIndex].startsWith("-") && + rawArgs[subIndex] !== "--help" && + rawArgs[subIndex] !== "-h" + ) { + subIndex++; + } + + const args = rawArgs.slice(subIndex); + const sub = args[0]; + if (!sub || !SUBCOMMANDS.has(sub)) return null; + if (sub === "help" || sub === "--help" || sub === "-h") return { kind: "help" }; + + try { + if (sub === "export") return parseExport(args.slice(1), cwd); + if (sub === "record") return parseRecord(args.slice(1), cwd); + if (sub === "sources") return parseSources(args.slice(1)); + if (sub === "pack") return parsePack(args.slice(1), cwd); + if (sub === "captions") return parseCaptions(args.slice(1), cwd); + return parseInfo(args.slice(1), cwd); + } catch (error) { + return { kind: "error", message: error instanceof Error ? error.message : String(error) }; + } +} + +function parseExport(args: string[], cwd: string): CliCommand { + const request: CliExportRequest & { json?: boolean } = { + kind: "export", + projectPath: "", + outPath: null, + format: null, + quality: null, + gifFrameRate: null, + gifSizePreset: null, + previewWidth: null, + previewHeight: null, + autoZoom: false, + followWindows: false, + audioPath: null, + audioMode: "mix", + audioOffsetSec: 0, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + switch (arg) { + case "-o": + case "--out": { + const [value, next] = takeValue(args, i, arg); + request.outPath = resolvePath(value, cwd); + i = next; + break; + } + case "--format": { + const [value, next] = takeValue(args, i, arg); + if (value !== "mp4" && value !== "gif") { + throw new Error(`--format must be mp4 or gif, got "${value}"`); + } + request.format = value; + i = next; + break; + } + case "--quality": { + const [value, next] = takeValue(args, i, arg); + if (value !== "medium" && value !== "good" && value !== "source") { + throw new Error(`--quality must be medium, good or source, got "${value}"`); + } + request.quality = value; + i = next; + break; + } + case "--gif-fps": { + const [value, next] = takeValue(args, i, arg); + const fps = Number(value); + if (fps !== 15 && fps !== 20 && fps !== 25 && fps !== 30) { + throw new Error(`--gif-fps must be 15, 20, 25 or 30, got "${value}"`); + } + request.gifFrameRate = fps; + i = next; + break; + } + case "--gif-size": { + const [value, next] = takeValue(args, i, arg); + if (value !== "medium" && value !== "large" && value !== "original") { + throw new Error(`--gif-size must be medium, large or original, got "${value}"`); + } + request.gifSizePreset = value; + i = next; + break; + } + case "--preview-size": { + const [value, next] = takeValue(args, i, arg); + const match = /^(\d+)x(\d+)$/.exec(value); + if (!match) throw new Error(`--preview-size must look like 1280x720, got "${value}"`); + const previewWidth = Number(match[1]); + const previewHeight = Number(match[2]); + if (previewWidth <= 0 || previewHeight <= 0) { + throw new Error(`--preview-size dimensions must be positive, got "${value}"`); + } + request.previewWidth = previewWidth; + request.previewHeight = previewHeight; + i = next; + break; + } + case "--auto-zoom": + request.autoZoom = true; + break; + case "--follow-windows": + request.followWindows = true; + break; + case "--audio": { + const [value, next] = takeValue(args, i, arg); + request.audioPath = resolvePath(value, cwd); + i = next; + break; + } + case "--audio-mode": { + const [value, next] = takeValue(args, i, arg); + if (value !== "mix" && value !== "replace") { + throw new Error(`--audio-mode must be mix or replace, got "${value}"`); + } + request.audioMode = value; + i = next; + break; + } + case "--audio-offset": { + const [value, next] = takeValue(args, i, arg); + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds < 0) { + throw new Error( + `--audio-offset must be a non-negative number of seconds, got "${value}"`, + ); + } + request.audioOffsetSec = seconds; + i = next; + break; + } + case "--json": + request.json = true; + break; + default: + if (arg.startsWith("-")) throw new Error(`Unknown export option: ${arg}`); + if (request.projectPath) throw new Error(`Unexpected extra argument: ${arg}`); + request.projectPath = resolvePath(arg, cwd); + } + } + + if (!request.projectPath) throw new Error("export requires a <project.openscreen> path"); + if (request.audioPath && request.format === "gif") { + throw new Error("--audio is only supported for MP4 exports"); + } + if (request.outPath) { + const ext = path.extname(request.outPath).toLowerCase(); + if (ext !== ".mp4" && ext !== ".gif") { + throw new Error(`--out must end in .mp4 or .gif, got "${request.outPath}"`); + } + const extFormat = ext === ".gif" ? "gif" : "mp4"; + if (request.format && request.format !== extFormat) { + throw new Error(`--format ${request.format} conflicts with --out extension ${ext}`); + } + request.format = extFormat; + } + return request; +} + +function parseRecord(args: string[], cwd: string): CliCommand { + const request: CliRecordRequest & { json?: boolean } = { + kind: "record", + displayIndex: 0, + windowTitle: null, + mic: false, + micDevice: null, + systemAudio: false, + cursorMode: "editable-overlay", + durationMs: null, + projectOut: null, + followWindows: false, + windows: null, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + switch (arg) { + case "--display": { + const [value, next] = takeValue(args, i, arg); + const index = Number(value); + if (!Number.isInteger(index) || index < 0) { + throw new Error(`--display must be a non-negative integer, got "${value}"`); + } + request.displayIndex = index; + i = next; + break; + } + case "--window": { + const [value, next] = takeValue(args, i, arg); + request.windowTitle = value; + i = next; + break; + } + case "--mic": + request.mic = true; + break; + case "--mic-device": { + const [value, next] = takeValue(args, i, arg); + request.mic = true; + request.micDevice = value; + i = next; + break; + } + case "--system-audio": + request.systemAudio = true; + break; + case "--cursor": { + const [value, next] = takeValue(args, i, arg); + if (value !== "editable-overlay" && value !== "system") { + throw new Error(`--cursor must be editable-overlay or system, got "${value}"`); + } + request.cursorMode = value; + i = next; + break; + } + case "--duration": { + const [value, next] = takeValue(args, i, arg); + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds <= 0) { + throw new Error(`--duration must be a positive number of seconds, got "${value}"`); + } + request.durationMs = Math.round(seconds * 1000); + i = next; + break; + } + case "--project": { + const [value, next] = takeValue(args, i, arg); + if (!value.endsWith(".openscreen")) { + throw new Error(`--project must end in .openscreen, got "${value}"`); + } + request.projectOut = resolvePath(value, cwd); + i = next; + break; + } + case "--follow-windows": + request.followWindows = true; + break; + case "--windows": { + const [value, next] = takeValue(args, i, arg); + const titles = value + .split(",") + .map((title) => title.trim()) + .filter(Boolean); + if (titles.length < 2) { + throw new Error( + '--windows needs at least two comma-separated titles, e.g. "Terminal,Chrome"', + ); + } + request.windows = titles; + i = next; + break; + } + case "--json": + request.json = true; + break; + default: + throw new Error(`Unknown record option: ${arg}`); + } + } + if (request.followWindows && request.windowTitle) { + throw new Error( + "--follow-windows records the whole display and cannot be combined with --window", + ); + } + return request; +} + +function parseSources(args: string[]): CliCommand { + let json = false; + for (const arg of args) { + if (arg === "--json") { + json = true; + } else { + throw new Error(`Unknown sources option: ${arg}`); + } + } + return { kind: "sources", json }; +} + +export interface CliPackCommand { + kind: "pack"; + projectPath: string; + outDir: string; +} + +function parsePack(args: string[], cwd: string): CliCommand { + let projectPath = ""; + let outDir = ""; + let json = false; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "-o" || arg === "--out") { + const [value, next] = takeValue(args, i, arg); + outDir = resolvePath(value, cwd); + i = next; + } else if (arg === "--json") { + json = true; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown pack option: ${arg}`); + } else if (projectPath) { + throw new Error(`Unexpected extra argument: ${arg}`); + } else { + projectPath = resolvePath(arg, cwd); + } + } + if (!projectPath) throw new Error("pack requires a <project.openscreen> path"); + if (!outDir) throw new Error("pack requires --out <directory>"); + return { kind: "pack", projectPath, outDir, json }; +} + +function parseCaptions(args: string[], cwd: string): CliCommand { + let projectPath = ""; + let minWordsPerCaption = 2; + let maxWordsPerCaption = 7; + let json = false; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--min-words" || arg === "--max-words") { + const [value, next] = takeValue(args, i, arg); + const count = Number(value); + if (!Number.isInteger(count) || count < 1) { + throw new Error(`${arg} must be a positive integer, got "${value}"`); + } + if (arg === "--min-words") minWordsPerCaption = count; + else maxWordsPerCaption = count; + i = next; + } else if (arg === "--json") { + json = true; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown captions option: ${arg}`); + } else if (projectPath) { + throw new Error(`Unexpected extra argument: ${arg}`); + } else { + projectPath = resolvePath(arg, cwd); + } + } + if (!projectPath) throw new Error("captions requires a <project.openscreen> path"); + if (minWordsPerCaption > maxWordsPerCaption) { + throw new Error("--min-words cannot exceed --max-words"); + } + return { kind: "captions", projectPath, minWordsPerCaption, maxWordsPerCaption, json }; +} + +function parseInfo(args: string[], cwd: string): CliCommand { + let projectPath = ""; + let json = false; + for (const arg of args) { + if (arg === "--json") { + json = true; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown info option: ${arg}`); + } else if (projectPath) { + throw new Error(`Unexpected extra argument: ${arg}`); + } else { + projectPath = resolvePath(arg, cwd); + } + } + if (!projectPath) throw new Error("info requires a <project.openscreen> path"); + return { kind: "info", projectPath, json }; +} diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts new file mode 100644 index 0000000000..f085d394d9 --- /dev/null +++ b/electron/cli/cliMain.ts @@ -0,0 +1,732 @@ +// Headless CLI mode: boots Electron without HUD/tray/menu, drives a hidden +// renderer window (windowType=cli-export | cli-record) that reuses the app's +// existing export and recording pipelines, and reports progress on stdio. + +import fs from "node:fs/promises"; +import path from "node:path"; +import readline from "node:readline"; +import { fileURLToPath } from "node:url"; +import { app, BrowserWindow, ipcMain, session, systemPreferences } from "electron"; +import type { + CliDoneResult, + CliProgressEvent, + CliRequest, + CliSourcesResult, +} from "../../src/lib/cliContracts"; +import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; +import { ASSET_BASE_URL_ARG } from "../windows"; +import { CLI_USAGE, type CliCommand } from "./args"; +import { FocusSampler, isFocusSamplingAvailable, resolveRecordedDisplayId } from "./focusSampler"; +import { startMultiWindowRecording } from "./multiWindowRecorder"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"]; +const RENDERER_DIST = path.join(__dirname, "..", "dist"); + +interface CliOutput { + json: boolean; + event(event: string, data?: Record<string, unknown>): void; + info(message: string): void; + error(message: string): void; + progress(p: CliProgressEvent): void; +} + +// stdout/stderr may be a closed pipe (`openscreen export | head`); writes must +// never take the process down — Electron would show a GUI error dialog. +function safeWrite(stream: NodeJS.WriteStream, text: string): void { + try { + stream.write(text); + } catch { + // EPIPE or closed stream; drop the output. + } +} + +function createOutput(json: boolean): CliOutput { + const isTty = process.stdout.isTTY === true; + let progressLineActive = false; + let lastProgressText = ""; + const clearProgressLine = () => { + if (progressLineActive) { + safeWrite(process.stdout, "\n"); + progressLineActive = false; + } + }; + return { + json, + event(event, data = {}) { + if (json) { + safeWrite(process.stdout, `${JSON.stringify({ event, ...data })}\n`); + } + }, + info(message) { + if (json) return; + clearProgressLine(); + safeWrite(process.stdout, `${message}\n`); + }, + error(message) { + clearProgressLine(); + if (json) { + safeWrite(process.stdout, `${JSON.stringify({ event: "error", message })}\n`); + } else { + safeWrite(process.stderr, `Error: ${message}\n`); + } + }, + progress(p) { + if (json) { + safeWrite(process.stdout, `${JSON.stringify({ event: "progress", ...p })}\n`); + return; + } + const frames = + p.currentFrame !== undefined && p.totalFrames + ? ` frame ${p.currentFrame}/${p.totalFrames}` + : ""; + const eta = + p.estimatedTimeRemaining !== undefined && Number.isFinite(p.estimatedTimeRemaining) + ? ` ETA ${Math.max(0, Math.round(p.estimatedTimeRemaining))}s` + : ""; + const phase = p.phase ? ` [${p.phase}]` : ""; + const text = `Exporting ${Math.round(p.percentage)}%${frames}${eta}${phase}`; + if (isTty) { + if (text === lastProgressText) return; + lastProgressText = text; + safeWrite(process.stdout, `\r\x1b[2K${text}`); + progressLineActive = true; + } else { + // Piped/non-TTY consumers get one line per whole-percent (or phase) change. + const coarse = `${Math.round(p.percentage)}%${phase}`; + if (coarse === lastProgressText) return; + lastProgressText = coarse; + safeWrite(process.stdout, `${text}\n`); + } + }, + }; +} + +function loadRunnerWindow(windowType: string): BrowserWindow { + const win = new BrowserWindow({ + width: 1280, + height: 720, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + // Same relaxation as the editor window: exporters load recording media + // via file:// URLs. + webSecurity: false, + backgroundThrottling: false, + }, + }); + + if (VITE_DEV_SERVER_URL) { + win.loadURL(`${VITE_DEV_SERVER_URL}?windowType=${windowType}`); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { query: { windowType } }); + } + return win; +} + +function registerAppHandlersForCli(cliWindowRef: () => BrowserWindow | null) { + // The recording/export pipelines are driven through the same IPC surface the + // GUI uses. Window-management callbacks become no-ops; "switch-to-editor" + // (fired by the recorder hook after it stores a finished session) is handled + // by the runner itself, so an inert factory is enough. + const noop = () => { + // Intentionally empty: CLI mode has no HUD/tray/editor windows to manage. + }; + const notAvailable = () => { + throw new Error("Window not available in CLI mode"); + }; + registerIpcHandlers( + noop, // createEditorWindow: recording finished; runner drives completion + notAvailable, // createSourceSelectorWindow + notAvailable, // createCountdownOverlayWindow + notAvailable, // createNotesWindow + cliWindowRef, + () => null, + () => null, + () => null, + noop, // onRecordingStateChange: no tray to update + noop, // switchToHud + ); +} + +function printSources(output: CliOutput, sources: CliSourcesResult): void { + if (output.json) { + // The "done" event already carries the payload; nothing extra to print. + return; + } + const lines: string[] = []; + lines.push("Displays:"); + for (const display of sources.displays) { + lines.push(` ${display.index} ${display.name} (${display.id})`); + } + lines.push("Windows:"); + if (sources.windows.length === 0) { + lines.push(" (none)"); + } + for (const win of sources.windows) { + lines.push(` - ${win.name}`); + } + lines.push("Microphones:"); + if (sources.microphoneLabelsUnavailable) { + lines.push(" (labels unavailable — grant microphone permission to see device names)"); + } else if (sources.microphones.length === 0) { + lines.push(" (none)"); + } + for (const mic of sources.microphones) { + lines.push(` - ${mic.label}`); + } + output.info(lines.join("\n")); +} + +async function writeProjectFile(projectOut: string, projectData: unknown): Promise<void> { + await fs.mkdir(path.dirname(projectOut), { recursive: true }); + await fs.writeFile(projectOut, JSON.stringify(projectData, null, 2), "utf8"); +} + +function setupRecordStopSignals(stop: (reason: string) => void): void { + // SIGINT covers Ctrl+C everywhere; SIGTERM never fires on Windows, where + // stdin "stop" or --duration are the graceful alternatives (see docs/cli.md). + process.on("SIGINT", () => stop("SIGINT")); + process.on("SIGTERM", () => stop("SIGTERM")); + try { + // Touching process.stdin can throw on Windows GUI-subsystem builds when + // spawned with stdio "ignore"; signals and --duration still work then. + const rl = readline.createInterface({ input: process.stdin }); + rl.on("line", (line) => { + const trimmed = line.trim().toLowerCase(); + if (trimmed === "stop" || trimmed === "q" || trimmed === "quit") { + stop("stdin"); + } + }); + rl.on("close", () => { + // stdin EOF is not a stop signal: agents may spawn the CLI with + // stdin closed and stop it via SIGINT/--duration instead. + }); + } catch { + // stdin unavailable; signals and --duration still work. + } +} + +interface PackedProjectData { + version?: number; + media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; + videoPath?: string; + editor?: Record<string, unknown>; +} + +/** Copies a project and everything it references into one portable folder. */ +async function runPackCommand(projectPath: string, outDir: string, json: boolean): Promise<number> { + const emit = (message: string) => { + if (!json) safeWrite(process.stdout, `${message}\n`); + }; + + const raw = await fs.readFile(projectPath, "utf8"); + const data = JSON.parse(raw) as PackedProjectData; + const media = data.media ?? (data.videoPath ? { screenVideoPath: data.videoPath } : undefined); + const screenVideoPath = media?.screenVideoPath; + if (!screenVideoPath) { + throw new Error("Project file does not reference a screen video"); + } + + const projectDir = path.dirname(path.resolve(projectPath)); + const resolveSource = async (mediaPath: string): Promise<string> => { + const exists = await fs + .stat(mediaPath) + .then((stats) => stats.isFile()) + .catch(() => false); + if (exists) return mediaPath; + const sibling = path.join(projectDir, path.basename(mediaPath)); + const siblingExists = await fs + .stat(sibling) + .then((stats) => stats.isFile()) + .catch(() => false); + if (siblingExists) return sibling; + throw new Error(`Referenced media not found: ${mediaPath}`); + }; + + await fs.mkdir(outDir, { recursive: true }); + + const copied: string[] = []; + const copyIn = async (sourcePath: string): Promise<string> => { + const destination = path.join(outDir, path.basename(sourcePath)); + if (path.resolve(sourcePath) !== path.resolve(destination)) { + await fs.copyFile(sourcePath, destination); + } + copied.push(destination); + return destination; + }; + + const screenSource = await resolveSource(screenVideoPath); + const newScreenPath = await copyIn(screenSource); + + let newWebcamPath: string | undefined; + if (media.webcamVideoPath) { + newWebcamPath = await copyIn(await resolveSource(media.webcamVideoPath)); + } + + // Cursor telemetry sidecar sits at "<video path>.cursor.json". + const cursorSidecar = `${screenSource}.cursor.json`; + const hasCursorData = await fs + .stat(cursorSidecar) + .then((stats) => stats.isFile()) + .catch(() => false); + if (hasCursorData) { + await copyIn(cursorSidecar); + } + + const packedProject: PackedProjectData = { + ...data, + media: { + ...media, + screenVideoPath: newScreenPath, + ...(newWebcamPath ? { webcamVideoPath: newWebcamPath } : {}), + }, + }; + delete packedProject.videoPath; + const packedProjectPath = path.join(outDir, path.basename(projectPath)); + await fs.writeFile(packedProjectPath, JSON.stringify(packedProject, null, 2), "utf8"); + + if (json) { + safeWrite( + process.stdout, + `${JSON.stringify({ + event: "done", + success: true, + projectPath: packedProjectPath, + files: [packedProjectPath, ...copied], + cursorData: hasCursorData, + })}\n`, + ); + } else { + emit(`Packed project → ${packedProjectPath}`); + for (const file of copied) { + emit(` + ${path.basename(file)}`); + } + if (!hasCursorData) { + emit(" (no cursor telemetry sidecar found)"); + } + emit( + "The folder is self-contained: if the stored paths go stale after moving it, the loader falls back to files next to the project.", + ); + } + return 0; +} + +async function runMultiWindowRecord( + command: import("../../src/lib/cliContracts").CliRecordRequest & { json?: boolean }, + output: CliOutput, +): Promise<void> { + const titles = command.windows ?? []; + output.info(`Starting multi-window capture (${titles.length} windows)…`); + + let handle: Awaited<ReturnType<typeof startMultiWindowRecording>>; + try { + handle = await startMultiWindowRecording(titles, command.displayIndex); + await handle.allStarted; + } catch (error) { + output.error(error instanceof Error ? error.message : String(error)); + app.exit(1); + return; + } + for (const summary of handle.windowSummaries) { + output.info(`Recording window: ${summary.title}`); + } + output.info("Recording started (all windows)"); + output.event("recording-started", { windows: handle.windowSummaries.map((w) => w.title) }); + + let stopping = false; + const finish = async (reason: string) => { + if (stopping) return; + stopping = true; + output.info(`Stopping recording (${reason})…`); + output.event("stopping", { reason }); + try { + const result = await handle.stop(); + if (command.projectOut) { + await writeProjectFile(command.projectOut, { + version: 2, + media: { screenVideoPath: result.primaryVideoPath, cursorCaptureMode: "system" }, + editor: {}, + }); + } + output.info(`Recorded ${result.videoPaths.length} windows → ${result.primaryVideoPath}`); + output.info(`Manifest → ${result.manifestPath}`); + if (command.projectOut) output.info(`Project → ${command.projectOut}`); + output.event("done", { + success: true, + screenVideoPath: result.primaryVideoPath, + videoPaths: result.videoPaths, + multiWindowManifestPath: result.manifestPath, + durationMs: result.durationMs, + focusSamples: result.focusSampleCount, + ...(command.projectOut ? { projectPath: command.projectOut } : {}), + }); + app.exit(0); + } catch (error) { + output.error(error instanceof Error ? error.message : String(error)); + output.event("done", { success: false }); + app.exit(1); + } + }; + + setupRecordStopSignals((reason) => void finish(reason)); + if (command.durationMs) { + setTimeout(() => void finish(`duration ${command.durationMs}ms`), command.durationMs); + } +} + +async function runInfoCommand(projectPath: string, json: boolean): Promise<number> { + const raw = await fs.readFile(projectPath, "utf8"); + const data = JSON.parse(raw) as { + version?: number; + media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; + videoPath?: string; + editor?: Record<string, unknown>; + }; + const editor = data.editor ?? {}; + const count = (key: string) => + Array.isArray(editor[key]) ? (editor[key] as unknown[]).length : 0; + const screenVideoPath = data.media?.screenVideoPath ?? data.videoPath ?? null; + const mediaExists = screenVideoPath + ? await fs + .access(screenVideoPath) + .then(() => true) + .catch(() => false) + : false; + + const summary = { + projectPath, + version: data.version ?? null, + screenVideoPath, + screenVideoExists: mediaExists, + webcamVideoPath: data.media?.webcamVideoPath ?? null, + cursorCaptureMode: data.media?.cursorCaptureMode ?? null, + exportFormat: (editor.exportFormat as string) ?? null, + exportQuality: (editor.exportQuality as string) ?? null, + aspectRatio: (editor.aspectRatio as string) ?? null, + zoomRegions: count("zoomRegions"), + trimRegions: count("trimRegions"), + speedRegions: count("speedRegions"), + annotationRegions: count("annotationRegions"), + }; + + if (json) { + safeWrite(process.stdout, `${JSON.stringify(summary)}\n`); + } else { + safeWrite( + process.stdout, + [ + `Project: ${summary.projectPath} (version ${summary.version ?? "?"})`, + `Video: ${summary.screenVideoPath ?? "(none)"}${mediaExists ? "" : " [MISSING]"}`, + `Webcam: ${summary.webcamVideoPath ?? "(none)"}`, + `Cursor: ${summary.cursorCaptureMode ?? "(unknown)"}`, + `Export: ${summary.exportFormat ?? "?"} / ${summary.exportQuality ?? "?"} / ${summary.aspectRatio ?? "?"}`, + `Timeline: ${summary.zoomRegions} zooms, ${summary.trimRegions} trims, ${summary.speedRegions} speed regions, ${summary.annotationRegions} annotations`, + ].join("\n") + "\n", + ); + } + return summary.screenVideoPath && !mediaExists ? 1 : 0; +} + +export function runCli(command: CliCommand): void { + if (command.kind === "help") { + process.stdout.write(CLI_USAGE); + app.exit(0); + return; + } + if (command.kind === "error") { + process.stderr.write(`Error: ${command.message}\n\n${CLI_USAGE}`); + app.exit(2); + return; + } + + const output = createOutput(command.json === true); + + // stdout belongs to the CLI protocol (NDJSON / progress); reroute the app's + // own console chatter (e.g. "[native-sck] starting…") to stderr. + const stringifyArg = (value: unknown): string => { + if (typeof value === "string") return value; + if (value instanceof Error) return value.stack ?? value.message; + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } + }; + for (const level of ["log", "info", "warn", "error", "debug"] as const) { + console[level] = (...args: unknown[]) => { + safeWrite(process.stderr, `${args.map(stringifyArg).join(" ")}\n`); + }; + } + + // A consumer closing the pipe (`openscreen export | head`) must not crash the + // process, and main-process exceptions must never surface as Electron's GUI + // error dialog — report on stderr and exit non-zero instead. + const ignoreStreamError = () => { + // Intentionally empty: EPIPE from a closed consumer is not an error here. + }; + process.stdout.on("error", ignoreStreamError); + process.stderr.on("error", ignoreStreamError); + process.on("uncaughtException", (error) => { + safeWrite(process.stderr, `Fatal: ${error?.stack ?? String(error)}\n`); + app.exit(1); + }); + process.on("unhandledRejection", (reason) => { + safeWrite( + process.stderr, + `Fatal (unhandled rejection): ${reason instanceof Error ? (reason.stack ?? reason.message) : String(reason)}\n`, + ); + app.exit(1); + }); + + // Set once cli-done has been received; suppresses the window-all-closed + // failure path during the normal teardown race after a successful run. + let finished = false; + + // GPU may be unavailable in CI/servers; let Chromium fall back to SwiftShader + // so the WebGL-based export renderer still works. + app.commandLine.appendSwitch("enable-unsafe-swiftshader"); + + // Never show the dock icon for CLI runs. + if (process.platform === "darwin") { + app.dock?.hide(); + } + + app.on("window-all-closed", () => { + // Completion is signalled via cli-done; a vanished window is a failure + // only while the run is still in flight. + if (finished) return; + output.error("Renderer window closed unexpectedly"); + app.exit(1); + }); + + void app + .whenReady() + .then(async () => { + if (command.kind === "info") { + const code = await runInfoCommand(command.projectPath, command.json === true); + app.exit(code); + return; + } + + if (command.kind === "record" && command.windows) { + await runMultiWindowRecord(command, output); + return; + } + + if (command.kind === "pack") { + const code = await runPackCommand( + command.projectPath, + command.outDir, + command.json === true, + ); + app.exit(code); + return; + } + + await fs.mkdir(path.join(app.getPath("userData"), "recordings"), { recursive: true }); + + // Media/screen permissions for the renderer (mic metering, future browser + // capture paths). Mirrors the GUI allowlist. + const allowed = [ + "media", + "audioCapture", + "microphone", + "videoCapture", + "camera", + "screen", + "display-capture", + ]; + session.defaultSession.setPermissionCheckHandler((_wc, permission) => + allowed.includes(permission), + ); + session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => + callback(allowed.includes(permission)), + ); + + // Browser-pipeline recording fallback (e.g. Linux, missing native helper) + // resolves the pre-selected source exactly like the GUI does. + session.defaultSession.setDisplayMediaRequestHandler( + (request, callback) => { + const source = getSelectedDesktopSource(); + if (!request.videoRequested || !source) { + callback({}); + return; + } + callback({ + video: source, + ...(request.audioRequested && process.platform === "win32" + ? { audio: "loopback" as const } + : {}), + }); + }, + { useSystemPicker: false }, + ); + + if (command.kind === "record" && command.mic && process.platform === "darwin") { + const micStatus = systemPreferences.getMediaAccessStatus("microphone"); + if (micStatus !== "granted") { + await systemPreferences.askForMediaAccess("microphone"); + } + } + + let cliWindow: BrowserWindow | null = null; + registerAppHandlersForCli(() => cliWindow); + + // Registered by the GUI boot path (main.ts) rather than registerIpcHandlers; + // the renderer's i18n init invokes it unconditionally. + ipcMain.handle("set-locale", () => { + // Locale only affects GUI menus/tray, which do not exist in CLI mode. + }); + ipcMain.handle("update-global-shortcut", () => ({ success: false })); + + const request: CliRequest = command; + ipcMain.handle("cli-get-request", () => request); + ipcMain.on("cli-log", (_event, level: string, message: string) => { + if (level === "error") { + output.error(message); + } else { + output.info(message); + output.event("log", { message }); + } + }); + ipcMain.on("cli-progress", (_event, progress: CliProgressEvent) => { + output.progress(progress); + }); + + ipcMain.handle("cli-done", async (_event, result: CliDoneResult) => { + if (finished) return; + finished = true; + + try { + if (result.success && command.kind === "record" && command.projectOut) { + if (result.projectData !== undefined) { + await writeProjectFile(command.projectOut, result.projectData); + result.projectPath = command.projectOut; + } + } + if (result.success && command.kind === "captions" && result.projectData !== undefined) { + await writeProjectFile(command.projectPath, result.projectData); + } + if ( + result.success && + command.kind === "record" && + command.followWindows && + focusSampler && + result.screenVideoPath + ) { + const focusData = focusSampler.stop(resolveRecordedDisplayId(command.displayIndex)); + if (focusData && focusData.samples.length > 0) { + const focusDataPath = `${result.screenVideoPath}.focus.json`; + await fs.writeFile(focusDataPath, JSON.stringify(focusData), "utf8"); + result.focusDataPath = focusDataPath; + } else { + output.error( + `Focus sampling produced no samples (helper diagnostics: ${focusSampler.diagnostics()})`, + ); + } + } + } catch (error) { + result.success = false; + result.error = `Run succeeded but writing the project file failed: ${String(error)}`; + } + + if (result.success) { + for (const warning of result.warnings ?? []) { + output.info(`Warning: ${warning}`); + output.event("warning", { message: warning }); + } + if (command.kind === "sources" && result.sources) { + printSources(output, result.sources); + } else if (command.kind === "captions") { + output.info( + `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`, + ); + } else if (command.kind === "export") { + output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`); + } else { + output.info(`Recording saved → ${result.screenVideoPath}`); + if (result.cursorDataPath) output.info(`Cursor data → ${result.cursorDataPath}`); + if (result.focusDataPath) output.info(`Focus data → ${result.focusDataPath}`); + if (result.projectPath) output.info(`Project → ${result.projectPath}`); + } + output.event("done", { ...result }); + } else { + output.error(result.error ?? "Unknown failure"); + output.event("done", { success: false, error: result.error }); + } + + // Give the renderer a beat to resolve the invoke before exiting. + setTimeout(() => app.exit(result.success ? 0 : 1), 50); + }); + + let focusSampler: FocusSampler | null = null; + if (command.kind === "record" && command.followWindows) { + if (!isFocusSamplingAvailable()) { + output.error( + process.platform === "darwin" + ? "--follow-windows needs the focus helper; build it with: npm run build:native:mac" + : "--follow-windows is currently macOS-only", + ); + app.exit(2); + return; + } + focusSampler = new FocusSampler(); + ipcMain.on("cli-recording-started", () => { + try { + focusSampler?.start(); + output.event("focus-sampling-started"); + } catch (error) { + output.error(`Focus sampling failed to start: ${String(error)}`); + } + }); + } else { + ipcMain.on("cli-recording-started", () => { + // No focus sampling requested; nothing to do. + }); + } + + if (command.kind === "record") { + const stop = (reason: string) => { + output.info(`Stopping recording (${reason})…`); + output.event("stopping", { reason }); + cliWindow?.webContents.send("cli-stop-recording"); + }; + setupRecordStopSignals(stop); + } + + const windowType = { + export: "cli-export", + record: "cli-record", + sources: "cli-sources", + captions: "cli-captions", + }[command.kind]; + cliWindow = loadRunnerWindow(windowType); + + // Surface renderer console errors/warnings on stderr — the hidden window + // has no other way to show what went wrong (toasts are invisible). + cliWindow.webContents.on("console-message", (details) => { + if (details.level === "error" || details.level === "warning") { + safeWrite(process.stderr, `[renderer] ${details.message}\n`); + } + }); + + cliWindow.webContents.on("did-fail-load", (_e, code, description) => { + output.error(`Failed to load runner window: ${description} (${code})`); + app.exit(1); + }); + cliWindow.webContents.on("render-process-gone", (_e, details) => { + output.error(`Renderer crashed: ${details.reason}`); + app.exit(1); + }); + + output.event("started", { command: command.kind }); + }) + .catch((error) => { + output.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); + app.exit(1); + }); +} diff --git a/electron/cli/focusSampler.ts b/electron/cli/focusSampler.ts new file mode 100644 index 0000000000..2bbb33a04e --- /dev/null +++ b/electron/cli/focusSampler.ts @@ -0,0 +1,180 @@ +// Spawns the macOS focus-telemetry helper during `record --follow-windows` +// and buffers its NDJSON samples. The CLI controller starts it when the +// capture begins and writes the collected timeline as a `<video>.focus.json` +// sidecar when the recording finishes. + +import { type ChildProcess, spawn } from "node:child_process"; +import { accessSync, constants as fsConstants } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { screen } from "electron"; +import type { + FocusDisplayInfo, + FocusRecordingData, + FocusSample, +} from "../../src/lib/windowFocus/contracts"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const FOCUS_HELPER_NAME = "openscreen-macos-focus-helper"; +const SAMPLE_INTERVAL_MS = 200; + +interface RawHelperSample { + type?: string; + timestampMs?: number; + windowNumber?: number; + appName?: string; + windowTitle?: string; + x?: number; + y?: number; + width?: number; + height?: number; + displayId?: number; +} + +function resolveFocusHelperPath(): string | null { + const archTag = `darwin-${process.arch === "arm64" ? "arm64" : "x64"}`; + const candidates = [ + process.env.OPENSCREEN_MAC_FOCUS_HELPER_EXE?.trim(), + path.join( + __dirname, + "..", + "electron", + "native", + "screencapturekit", + "build", + FOCUS_HELPER_NAME, + ), + path.join(__dirname, "..", "electron", "native", "bin", archTag, FOCUS_HELPER_NAME), + process.resourcesPath + ? path.join(process.resourcesPath, "electron", "native", "bin", archTag, FOCUS_HELPER_NAME) + : undefined, + ]; + for (const candidate of candidates) { + if (!candidate) continue; + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next location. + } + } + return null; +} + +export function isFocusSamplingAvailable(): boolean { + return process.platform === "darwin" && resolveFocusHelperPath() !== null; +} + +export class FocusSampler { + private child: ChildProcess | null = null; + private rawSamples: RawHelperSample[] = []; + private startedAtEpochMs: number | null = null; + private stdoutRemainder = ""; + private lastError = ""; + private sawReady = false; + private exitCode: number | string | null = null; + + diagnostics(): string { + return JSON.stringify({ + started: this.startedAtEpochMs !== null, + sawReady: this.sawReady, + rawSamples: this.rawSamples.length, + exitCode: this.exitCode, + lastError: this.lastError, + }); + } + + start(): void { + if (this.child) return; + const helperPath = resolveFocusHelperPath(); + if (!helperPath) { + throw new Error("Focus helper is not available. Build it with: npm run build:native:mac"); + } + + this.startedAtEpochMs = Date.now(); + this.child = spawn(helperPath, [JSON.stringify({ sampleIntervalMs: SAMPLE_INTERVAL_MS })], { + stdio: ["ignore", "pipe", "pipe"], + }); + this.child.stdout?.setEncoding("utf8"); + this.child.stdout?.on("data", (chunk: string) => { + this.stdoutRemainder += chunk; + const lines = this.stdoutRemainder.split("\n"); + this.stdoutRemainder = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + const parsed = JSON.parse(trimmed) as RawHelperSample; + if (parsed.type === "ready") { + this.sawReady = true; + } else if (parsed.type === "sample") { + this.rawSamples.push(parsed); + } + } catch { + // Ignore malformed helper output lines. + } + } + }); + this.child.stderr?.setEncoding("utf8"); + this.child.stderr?.on("data", (chunk: string) => { + this.lastError = chunk.slice(0, 500); + }); + this.child.on("exit", (code, signal) => { + this.exitCode = code ?? signal ?? "unknown"; + }); + this.child.on("error", (error) => { + this.lastError = String(error); + this.child = null; + }); + } + + /** Stops the helper and converts the buffer into sidecar data. */ + stop(recordedDisplayId: number): FocusRecordingData | null { + const startedAt = this.startedAtEpochMs; + if (this.child) { + this.child.kill("SIGTERM"); + this.child = null; + } + if (startedAt === null) return null; + + const displays: FocusDisplayInfo[] = screen.getAllDisplays().map((display) => ({ + id: display.id, + bounds: display.bounds, + scaleFactor: display.scaleFactor, + isPrimary: display.id === screen.getPrimaryDisplay().id, + })); + + const samples: FocusSample[] = []; + for (const raw of this.rawSamples) { + if ( + typeof raw.timestampMs !== "number" || + typeof raw.width !== "number" || + typeof raw.height !== "number" + ) { + continue; + } + samples.push({ + timeMs: Math.max(0, raw.timestampMs - startedAt), + windowNumber: raw.windowNumber ?? 0, + appName: raw.appName ?? "", + windowTitle: raw.windowTitle ?? "", + x: raw.x ?? 0, + y: raw.y ?? 0, + width: raw.width, + height: raw.height, + displayId: raw.displayId ?? 0, + }); + } + + return { version: 1, recordedDisplayId, displays, samples }; + } +} + +export function resolveRecordedDisplayId(displayIndex: number): number { + // The CLI's --display index maps to desktopCapturer's screen ordering, + // which matches Electron's display list order on macOS. + const displays = screen.getAllDisplays(); + const display = displays[displayIndex] ?? screen.getPrimaryDisplay(); + return display.id; +} diff --git a/electron/cli/multiWindowRecorder.ts b/electron/cli/multiWindowRecorder.ts new file mode 100644 index 0000000000..d5a2c42053 --- /dev/null +++ b/electron/cli/multiWindowRecorder.ts @@ -0,0 +1,331 @@ +// Orchestrates `record --windows "a,b,c"`: one ScreenCaptureKit helper per +// matched window (all captured continuously and cleanly, even when occluded) +// plus the focus sampler. Runs entirely in the main process — no renderer. +// +// Output: one MP4 per window in the recordings directory, and a +// `<primary>.multiwindow.json` manifest binding the videos to the focus +// timeline for the export step's window-switch compositor. + +import { type ChildProcess, spawn } from "node:child_process"; +import { accessSync, constants as fsConstants } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { app, desktopCapturer } from "electron"; +import type { CapturedWindow, MultiWindowManifest } from "../../src/lib/windowSwitch/contracts"; +import { FocusSampler, resolveRecordedDisplayId } from "./focusSampler"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const CAPTURE_HELPER_NAME = "openscreen-screencapturekit-helper"; +const TARGET_FPS = 60; +const TARGET_WIDTH = 3840; +const TARGET_HEIGHT = 2160; +const START_TIMEOUT_MS = 15_000; +const STOP_TIMEOUT_MS = 20_000; + +function resolveCaptureHelperPath(): string | null { + const archTag = `darwin-${process.arch === "arm64" ? "arm64" : "x64"}`; + const candidates = [ + process.env.OPENSCREEN_SCK_CAPTURE_EXE?.trim(), + path.join( + __dirname, + "..", + "electron", + "native", + "screencapturekit", + "build", + CAPTURE_HELPER_NAME, + ), + path.join(__dirname, "..", "electron", "native", "bin", archTag, CAPTURE_HELPER_NAME), + process.resourcesPath + ? path.join(process.resourcesPath, "electron", "native", "bin", archTag, CAPTURE_HELPER_NAME) + : undefined, + ]; + for (const candidate of candidates) { + if (!candidate) continue; + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next location. + } + } + return null; +} + +function parseWindowIdFromSourceId(sourceId: string): number | null { + // macOS desktopCapturer window source ids look like "window:<CGWindowID>:0". + const match = /^window:(\d+)/.exec(sourceId); + return match ? Number(match[1]) : null; +} + +interface MultiHelperSession { + child: ChildProcess; + /** Resolves once every window has emitted recording-started. */ + allStarted: Promise<void>; + /** Resolves with the finalized paths once every window has stopped. */ + allStopped: Promise<string[]>; +} + +// One helper process captures every window: a second helper *process* would +// interrupt the first stream (SCStreamErrorDomain -3805), but multiple +// SCStreams coexist happily inside a single process. +function spawnMultiWindowCapture( + helperPath: string, + recordingsDir: string, + targets: { windowId: number; sourceId: string; screenPath: string }[], +): MultiHelperSession { + const request = { + schemaVersion: 1, + recordingId: Date.now(), + source: { + type: "window", + sourceId: targets[0].sourceId, + windowId: targets[0].windowId, + }, + video: { + fps: TARGET_FPS, + width: TARGET_WIDTH, + height: TARGET_HEIGHT, + hideSystemCursor: false, + }, + audio: { + system: { enabled: false }, + microphone: { enabled: false, gain: 1 }, + }, + webcam: { enabled: false, width: 0, height: 0, fps: 30 }, + cursor: { mode: "system" }, + outputs: { screenPath: targets[0].screenPath }, + multiWindows: targets, + }; + + const child = spawn(helperPath, [JSON.stringify(request)], { + cwd: recordingsDir, + stdio: ["pipe", "pipe", "pipe"], + }); + + let startedCount = 0; + const stoppedPaths: string[] = []; + let resolveAllStarted: () => void; + let rejectAllStarted: (error: Error) => void; + const allStarted = new Promise<void>((resolve, reject) => { + resolveAllStarted = resolve; + rejectAllStarted = reject; + }); + let resolveAllStopped: (paths: string[]) => void; + let rejectAllStopped: (error: Error) => void; + const allStopped = new Promise<string[]>((resolve, reject) => { + resolveAllStopped = resolve; + rejectAllStopped = reject; + }); + + let remainder = ""; + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + remainder += chunk; + const lines = remainder.split("\n"); + remainder = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) continue; + try { + const event = JSON.parse(trimmed) as { + event?: string; + message?: string; + screenPath?: string; + }; + if (event.event === "recording-started") { + startedCount++; + if (startedCount === targets.length) resolveAllStarted(); + } else if (event.event === "recording-stopped") { + if (event.screenPath) stoppedPaths.push(event.screenPath); + if (stoppedPaths.length === targets.length) { + // Preserve the requested window order regardless of finalize order. + const ordered = targets.map((target) => { + const match = stoppedPaths.find((candidate) => candidate === target.screenPath); + return match ?? target.screenPath; + }); + resolveAllStopped(ordered); + } + } else if (event.event === "error") { + const error = new Error(event.message ?? "Native capture error"); + rejectAllStarted(error); + rejectAllStopped(error); + } + } catch { + // Non-JSON helper chatter. + } + } + }); + let stderrTail = ""; + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderrTail = (stderrTail + chunk).slice(-500); + }); + child.on("exit", (code) => { + const error = new Error( + `Capture helper exited (${code})${stderrTail ? `: ${stderrTail}` : ""}`, + ); + rejectAllStarted(error); + rejectAllStopped(error); + }); + child.on("error", (error) => { + rejectAllStarted(error as Error); + rejectAllStopped(error as Error); + }); + + return { child, allStarted, allStopped }; +} + +function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> { + return Promise.race([ + promise, + new Promise<T>((_, reject) => + setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms), + ), + ]); +} + +export interface MultiWindowRecordingHandle { + /** Resolves when every window capture has emitted recording-started. */ + allStarted: Promise<void>; + windowSummaries: { appName: string; title: string }[]; + stop: () => Promise<{ + manifestPath: string; + primaryVideoPath: string; + videoPaths: string[]; + durationMs: number; + focusSampleCount: number; + }>; +} + +export async function startMultiWindowRecording( + titleFilters: string[], + displayIndex: number, +): Promise<MultiWindowRecordingHandle> { + if (process.platform !== "darwin") { + throw new Error("--windows capture is currently macOS-only"); + } + const helperPath = resolveCaptureHelperPath(); + if (!helperPath) { + throw new Error("Native capture helper missing; build it with: npm run build:native:mac"); + } + + const sources = await desktopCapturer.getSources({ + types: ["window"], + thumbnailSize: { width: 0, height: 0 }, + fetchWindowIcons: false, + }); + + const matched: { windowId: number; sourceId: string; title: string }[] = []; + for (const filter of titleFilters) { + const needle = filter.toLowerCase(); + const source = sources.find( + (candidate) => + candidate.name.toLowerCase().includes(needle) && + parseWindowIdFromSourceId(candidate.id) !== null && + !matched.some((m) => m.sourceId === candidate.id), + ); + if (!source) { + const available = sources.map((candidate) => ` - ${candidate.name}`).join("\n"); + throw new Error(`No window title contains "${filter}". Open windows:\n${available}`); + } + matched.push({ + windowId: parseWindowIdFromSourceId(source.id) as number, + sourceId: source.id, + title: source.name, + }); + } + if (matched.length < 2) { + throw new Error("--windows needs at least two comma-separated window titles"); + } + + const recordingsDir = path.join(app.getPath("userData"), "recordings"); + await fs.mkdir(recordingsDir, { recursive: true }); + const recordingId = Date.now(); + + const focusSampler = new FocusSampler(); + focusSampler.start(); + const startedAt = Date.now(); + + const session = spawnMultiWindowCapture( + helperPath, + recordingsDir, + matched.map((window, index) => ({ + windowId: window.windowId, + sourceId: window.sourceId, + screenPath: path.join(recordingsDir, `recording-${recordingId}-w${index}.mp4`), + })), + ); + + const allStarted = withTimeout( + session.allStarted, + START_TIMEOUT_MS, + "Window capture start", + ).catch(async (error) => { + session.child.kill("SIGKILL"); + focusSampler.stop(0); + throw error; + }); + + const stop = async () => { + const durationMs = Date.now() - startedAt; + session.child.stdin?.write("stop\n"); + const videoPaths = await withTimeout( + session.allStopped, + STOP_TIMEOUT_MS, + "Window capture stop", + ); + const focusData = focusSampler.stop(resolveRecordedDisplayId(displayIndex)); + + const windows: CapturedWindow[] = matched.map((window, index) => { + const focusMatch = focusData?.samples.find( + (sample) => sample.windowNumber === window.windowId, + ); + return { + windowId: window.windowId, + appName: focusMatch?.appName ?? "", + title: window.title, + videoPath: videoPaths[index], + bounds: focusMatch + ? { + x: focusMatch.x, + y: focusMatch.y, + width: focusMatch.width, + height: focusMatch.height, + } + : { x: 0, y: 0, width: 0, height: 0 }, + }; + }); + + const manifest: MultiWindowManifest = { + version: 1, + windows, + focus: focusData ?? { + version: 1, + recordedDisplayId: 0, + displays: [], + samples: [], + }, + durationMs, + }; + const primaryVideoPath = videoPaths[0]; + const manifestPath = `${primaryVideoPath}.multiwindow.json`; + await fs.writeFile(manifestPath, JSON.stringify(manifest), "utf8"); + + return { + manifestPath, + primaryVideoPath, + videoPaths, + durationMs, + focusSampleCount: focusData?.samples.length ?? 0, + }; + }; + + return { + allStarted, + windowSummaries: matched.map((window) => ({ appName: "", title: window.title })), + stop, + }; +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index f954fc99e2..da95c01b18 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -311,6 +311,13 @@ interface Window { onRequestSaveBeforeClose: (callback: () => Promise<boolean> | boolean) => () => void; onRequestCloseConfirm: (callback: () => void) => () => void; sendCloseConfirmResponse: (choice: "save" | "discard" | "cancel") => void; + // CLI mode (hidden runner windows; see electron/cli/) + cliGetRequest: () => Promise<import("../src/lib/cliContracts").CliRequest>; + cliProgress: (progress: import("../src/lib/cliContracts").CliProgressEvent) => void; + cliLog: (level: "info" | "error", message: string) => void; + cliRecordingStarted: () => void; + cliDone: (result: import("../src/lib/cliContracts").CliDoneResult) => Promise<void>; + onCliStopRecording: (callback: () => void) => () => void; setLocale: (locale: string) => Promise<void>; saveDiagnostic: (payload: { error: string; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 852250ec4c..b8ea2ef7cb 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -321,13 +321,40 @@ async function getApprovedProjectSession( trustedDirs.push(path.dirname(path.resolve(projectFilePath))); } - const screenVideoPath = await approveReadableVideoPath(media.screenVideoPath, trustedDirs); + // Packed/portable projects: when the stored absolute path no longer exists + // (project moved to another machine or directory), fall back to a file with + // the same basename next to the project file (see `openscreen pack`). + const resolveWithSiblingFallback = async (mediaPath: string): Promise<string> => { + if (!projectFilePath) return mediaPath; + const exists = await fs + .stat(mediaPath) + .then((stats) => stats.isFile()) + .catch(() => false); + if (exists) return mediaPath; + const sibling = path.join( + path.dirname(path.resolve(projectFilePath)), + path.basename(mediaPath), + ); + const siblingExists = await fs + .stat(sibling) + .then((stats) => stats.isFile()) + .catch(() => false); + return siblingExists ? sibling : mediaPath; + }; + + const screenVideoPath = await approveReadableVideoPath( + await resolveWithSiblingFallback(media.screenVideoPath), + trustedDirs, + ); if (!screenVideoPath) { throw new Error("Project references an invalid or unsupported screen video path"); } const webcamVideoPath = media.webcamVideoPath - ? await approveReadableVideoPath(media.webcamVideoPath, trustedDirs) + ? await approveReadableVideoPath( + await resolveWithSiblingFallback(media.webcamVideoPath), + trustedDirs, + ) : undefined; if (media.webcamVideoPath && !webcamVideoPath) { throw new Error("Project references an invalid or unsupported webcam video path"); diff --git a/electron/main.ts b/electron/main.ts index 12ce817fc8..1c4b7d71d6 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -12,6 +12,8 @@ import { Tray, } from "electron"; import { ShortcutBinding } from "../src/lib/shortcuts"; +import { parseCliArgs } from "./cli/args"; +import { runCli } from "./cli/cliMain"; import { isDiagnosticModeEnabled, mainLogBuffer } from "./diagnostics/main-log-buffer"; import { loadAndRegisterGlobalShortcut, @@ -31,6 +33,10 @@ import { const __dirname = path.dirname(fileURLToPath(import.meta.url)); +// CLI mode: `openscreen export|record|info|help ...` runs headless without +// HUD/tray/menu. Parsed before any GUI side effects; see electron/cli/. +const cliCommand = parseCliArgs(process.argv, app.isPackaged ? 1 : 2); + // Use Screen & System Audio Recording permissions instead of the CoreAudio Tap API on macOS. // Tap needs NSAudioCaptureUsageDescription in the parent app's Info.plist, which breaks when // running from a terminal/IDE during dev. @@ -122,11 +128,15 @@ function showMainWindow() { createWindow(); } -const stableInstanceLock = acquireStableInstanceLock(); -const hasElectronSingleInstanceLock = app.requestSingleInstanceLock(); +// CLI runs skip the single-instance lock so `openscreen export/record` works +// while the GUI app is open (they share nothing but the recordings directory). +const stableInstanceLock = cliCommand ? null : acquireStableInstanceLock(); +const hasElectronSingleInstanceLock = cliCommand ? false : app.requestSingleInstanceLock(); const hasSingleInstanceLock = Boolean(stableInstanceLock && hasElectronSingleInstanceLock); -if (hasSingleInstanceLock) { +if (cliCommand) { + runCli(cliCommand); +} else if (hasSingleInstanceLock) { app.on("second-instance", () => { showMainWindow(); }); @@ -478,11 +488,15 @@ function createCountdownOverlayWindowWrapper() { // Closing every window quits the app (tray goes too). The in-app "Return to Recorder" // button covers the editor-to-HUD round-trip, so closing the last window means "I'm done". -app.on("window-all-closed", () => { - app.quit(); -}); +// CLI mode owns its own lifecycle (see electron/cli/cliMain.ts). +if (!cliCommand) { + app.on("window-all-closed", () => { + app.quit(); + }); +} app.on("activate", () => { + if (cliCommand) return; // On macOS, re-open a window when the dock icon is clicked and none are open. const hasVisibleWindow = BrowserWindow.getAllWindows().some((window) => { if (window.isDestroyed() || !window.isVisible()) { @@ -503,7 +517,7 @@ app.on("will-quit", () => { stableInstanceLock?.release(); }); -const appReady = hasSingleInstanceLock ? app.whenReady() : null; +const appReady = !cliCommand && hasSingleInstanceLock ? app.whenReady() : null; appReady?.then(async () => { if (isDiagnosticModeEnabled()) { diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift index ec3b1d98dd..b6388cf277 100644 --- a/electron/native/screencapturekit/Package.swift +++ b/electron/native/screencapturekit/Package.swift @@ -15,6 +15,10 @@ let package = Package( .executable( name: "openscreen-macos-cursor-helper", targets: ["OpenScreenMacOSCursorHelper"] + ), + .executable( + name: "openscreen-macos-focus-helper", + targets: ["OpenScreenMacOSFocusHelper"] ) ], targets: [ @@ -25,6 +29,10 @@ let package = Package( .executableTarget( name: "OpenScreenMacOSCursorHelper", path: "Sources/OpenScreenMacOSCursorHelper" + ), + .executableTarget( + name: "OpenScreenMacOSFocusHelper", + path: "Sources/OpenScreenMacOSFocusHelper" ) ] ) diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSFocusHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSFocusHelper/main.swift new file mode 100644 index 0000000000..e29326dd8e --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSFocusHelper/main.swift @@ -0,0 +1,160 @@ +// openscreen-macos-focus-helper +// +// Samples the frontmost application's focused window (owner, title, bounds) +// and emits newline-delimited JSON on stdout. Used by the CLI's +// `record --follow-windows` to build a window-focus timeline that the export +// step turns into automatic pan/zoom regions. +// +// Protocol (mirrors the cursor helper): +// argv[1] optional JSON: {"sampleIntervalMs": 200} +// stdout {"type":"ready","timestampMs":...} +// {"type":"sample","timestampMs":...,"appName":"...","windowTitle":"...", +// "x":...,"y":...,"width":...,"height":...,"displayId":...} +// Samples are emitted when the focused window changes (app, window, or +// bounds beyond a small delta) plus a 1 s heartbeat. +// Stop with SIGTERM/SIGINT; EOF is ignored (parent may close stdin). +// +// Window enumeration uses CGWindowListCopyWindowInfo, which reports other +// apps' window names/bounds only when Screen Recording permission is granted — +// the same permission the capture pipeline already requires. + +import AppKit +import CoreGraphics +import Foundation + +struct FocusSample: Equatable { + var pid: pid_t + var appName: String + var windowNumber: Int + var windowTitle: String + var bounds: CGRect + var displayId: UInt32 +} + +func nowEpochMs() -> Int64 { + Int64(Date().timeIntervalSince1970 * 1000) +} + +func emit(_ object: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: object), + let line = String(data: data, encoding: .utf8) + else { return } + print(line) + fflush(stdout) +} + +func displayId(for rect: CGRect) -> UInt32 { + var count: UInt32 = 0 + var ids = [CGDirectDisplayID](repeating: 0, count: 16) + guard CGGetDisplaysWithRect(rect, 16, &ids, &count) == .success, count > 0 else { + return CGMainDisplayID() + } + // Prefer the display containing the window's center. + let center = CGPoint(x: rect.midX, y: rect.midY) + for index in 0..<Int(count) { + if CGDisplayBounds(ids[index]).contains(center) { + return ids[index] + } + } + return ids[0] +} + +/// Frontmost app's topmost on-screen, layer-0 window. +func currentFocusSample() -> FocusSample? { + guard let app = NSWorkspace.shared.frontmostApplication else { return nil } + let pid = app.processIdentifier + + guard + let windowInfos = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] + else { return nil } + + // CGWindowList returns windows in front-to-back order; the first layer-0 + // window owned by the frontmost app is its focused window. + for info in windowInfos { + guard let ownerPid = info[kCGWindowOwnerPID as String] as? pid_t, ownerPid == pid, + let layer = info[kCGWindowLayer as String] as? Int, layer == 0, + let boundsDict = info[kCGWindowBounds as String] as? [String: CGFloat] + else { continue } + + let bounds = CGRect( + x: boundsDict["X"] ?? 0, + y: boundsDict["Y"] ?? 0, + width: boundsDict["Width"] ?? 0, + height: boundsDict["Height"] ?? 0 + ) + // Skip tiny utility windows (tooltips, status items). + if bounds.width < 120 || bounds.height < 90 { continue } + + let windowNumber = info[kCGWindowNumber as String] as? Int ?? 0 + let title = info[kCGWindowName as String] as? String ?? "" + return FocusSample( + pid: pid, + appName: app.localizedName ?? "", + windowNumber: windowNumber, + windowTitle: title, + bounds: bounds, + displayId: displayId(for: bounds) + ) + } + return nil +} + +func boundsRoughlyEqual(_ a: CGRect, _ b: CGRect) -> Bool { + let delta: CGFloat = 8 + return abs(a.origin.x - b.origin.x) < delta && abs(a.origin.y - b.origin.y) < delta + && abs(a.width - b.width) < delta && abs(a.height - b.height) < delta +} + +// --- Entry point --- + +var sampleIntervalMs = 200 +if CommandLine.arguments.count > 1, + let data = CommandLine.arguments[1].data(using: .utf8), + let config = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let interval = config["sampleIntervalMs"] as? Int, interval >= 50 +{ + sampleIntervalMs = interval +} + +signal(SIGINT) { _ in exit(0) } +signal(SIGTERM) { _ in exit(0) } + +emit(["type": "ready", "timestampMs": nowEpochMs()]) + +var lastEmitted: FocusSample? +var lastEmitTimeMs: Int64 = 0 +let heartbeatMs: Int64 = 1000 + +let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.main) +timer.schedule( + deadline: .now(), repeating: .milliseconds(sampleIntervalMs), leeway: .milliseconds(20)) +timer.setEventHandler { + guard let sample = currentFocusSample() else { return } + let now = nowEpochMs() + let changed = + lastEmitted.map { previous in + previous.pid != sample.pid || previous.windowNumber != sample.windowNumber + || !boundsRoughlyEqual(previous.bounds, sample.bounds) + } ?? true + if !changed && now - lastEmitTimeMs < heartbeatMs { return } + lastEmitted = sample + lastEmitTimeMs = now + emit([ + "type": "sample", + "timestampMs": now, + "windowNumber": sample.windowNumber, + "appName": sample.appName, + "windowTitle": sample.windowTitle, + "x": Int(sample.bounds.origin.x), + "y": Int(sample.bounds.origin.y), + "width": Int(sample.bounds.width), + "height": Int(sample.bounds.height), + "displayId": Int(sample.displayId), + ]) +} +timer.resume() + +// Drain the main dispatch queue; RunLoop.main.run() does not reliably service +// dispatch timers in a bare command-line process. +dispatchMain() diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift index 9525b717bd..f061374fe4 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift @@ -62,6 +62,12 @@ struct RecordingRequest: Decodable { let manifestPath: String? } + struct WindowTarget: Decodable { + let windowId: UInt32 + let sourceId: String + let screenPath: String + } + let schemaVersion: Int? let recordingId: Int? let source: Source @@ -70,6 +76,11 @@ struct RecordingRequest: Decodable { let webcam: Webcam let cursor: Cursor let outputs: Outputs + /// Multi-window mode: capture every listed window concurrently in this + /// process (one SCStream + writer each). A second helper *process* would + /// interrupt the first stream (SCStreamErrorDomain -3805), so concurrent + /// window capture must live in a single process. + let multiWindows: [WindowTarget]? } enum HelperError: Error, CustomStringConvertible { @@ -647,6 +658,65 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } +@available(macOS 13.0, *) +func runMultiWindowCapture( + request: RecordingRequest, targets: [RecordingRequest.WindowTarget] +) async throws { + let recorders = targets.map { target in + ScreenCaptureRecorder( + request: RecordingRequest( + schemaVersion: request.schemaVersion, + recordingId: request.recordingId, + source: RecordingRequest.Source( + type: "window", + sourceId: target.sourceId, + displayId: nil, + windowId: target.windowId, + bounds: nil + ), + video: request.video, + audio: RecordingRequest.Audio( + system: RecordingRequest.Audio.SystemAudio(enabled: false), + microphone: RecordingRequest.Audio.Microphone( + enabled: false, deviceId: nil, deviceName: nil, gain: 1) + ), + webcam: RecordingRequest.Webcam( + enabled: false, deviceId: nil, deviceName: nil, width: 0, height: 0, fps: 30), + cursor: request.cursor, + outputs: RecordingRequest.Outputs(screenPath: target.screenPath, manifestPath: nil), + multiWindows: nil + )) + } + + let stopTask = Task.detached { + while let line = readLine() { + let command = line.trimmingCharacters(in: .whitespacesAndNewlines) + switch command { + case "stop": + for recorder in recorders { + await recorder.stop() + } + exit(0) + case "pause": + for recorder in recorders { + recorder.pause() + } + case "resume": + for recorder in recorders { + recorder.resume() + } + default: + break + } + } + } + + for recorder in recorders { + try await recorder.start() + } + await stopTask.value +} + @main struct OpenScreenScreenCaptureKitHelper { static func main() async { @@ -662,6 +732,12 @@ struct OpenScreenScreenCaptureKitHelper { let requestData = Data(CommandLine.arguments[1].utf8) let decoder = JSONDecoder() let request = try decoder.decode(RecordingRequest.self, from: requestData) + + if let targets = request.multiWindows, !targets.isEmpty { + try await runMultiWindowCapture(request: request, targets: targets) + return + } + let recorder = ScreenCaptureRecorder(request: request) let stopTask = Task.detached { while let line = readLine() { diff --git a/electron/preload.ts b/electron/preload.ts index f02a2ccd29..de9189d05c 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -302,4 +302,25 @@ contextBridge.exposeInMainWorld("electronAPI", { sendCloseConfirmResponse: (choice: "save" | "discard" | "cancel") => { ipcRenderer.send("close-confirm-response", choice); }, + // --- CLI mode (hidden runner windows; see electron/cli/) --- + cliGetRequest: (): Promise<import("../src/lib/cliContracts").CliRequest> => { + return ipcRenderer.invoke("cli-get-request"); + }, + cliProgress: (progress: import("../src/lib/cliContracts").CliProgressEvent) => { + ipcRenderer.send("cli-progress", progress); + }, + cliLog: (level: "info" | "error", message: string) => { + ipcRenderer.send("cli-log", level, message); + }, + cliRecordingStarted: () => { + ipcRenderer.send("cli-recording-started"); + }, + cliDone: (result: import("../src/lib/cliContracts").CliDoneResult) => { + return ipcRenderer.invoke("cli-done", result); + }, + onCliStopRecording: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("cli-stop-recording", listener); + return () => ipcRenderer.removeListener("cli-stop-recording", listener); + }, }); diff --git a/electron/windows.ts b/electron/windows.ts index 60dbc9df41..b330e514f3 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -14,7 +14,7 @@ const HEADLESS = process.env["HEADLESS"] === "true"; const ASSET_BASE_DIR = process.defaultApp ? path.join(__dirname, "..", "public") : process.resourcesPath; -const ASSET_BASE_URL_ARG = `--asset-base-url=${pathToFileURL(`${ASSET_BASE_DIR}${path.sep}`).toString()}`; +export const ASSET_BASE_URL_ARG = `--asset-base-url=${pathToFileURL(`${ASSET_BASE_DIR}${path.sep}`).toString()}`; let hudOverlayWindow: BrowserWindow | null = null; diff --git a/package.json b/package.json index 02f778efcc..00e9af3b53 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "diagnostic:run": "node scripts/diagnostic-tool/diagnostic.mjs", "diagnostic:smoke:win": "node scripts/diagnostic-tool/diagnostic.mjs --duration 3", "build-vite": "tsc && vite build", + "cli": "electron .", "test:browser": "vitest --config vitest.browser.config.ts --run", "test:browser:install": "playwright install --with-deps chromium-headless-shell", "test:e2e": "playwright test", diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs index b94836cfff..f29cb1ff93 100644 --- a/scripts/build-macos-screencapturekit-helper.mjs +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -15,11 +15,13 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(__dirname, ".."); const helperName = "openscreen-screencapturekit-helper"; const cursorHelperName = "openscreen-macos-cursor-helper"; +const focusHelperName = "openscreen-macos-focus-helper"; const packageDir = path.join(root, "electron", "native", "screencapturekit"); const buildDir = path.join(packageDir, "build"); const swiftBuildDir = path.join(buildDir, "swiftpm"); const localHelperPath = path.join(buildDir, helperName); const localCursorHelperPath = path.join(buildDir, cursorHelperName); +const localFocusHelperPath = path.join(buildDir, focusHelperName); // Build a separate single-arch binary per requested arch and place each in its own // electron/native/bin/darwin-<arch> folder (the runtime resolves that folder by the running app's @@ -125,6 +127,7 @@ for (const { swift, tag } of archs) { for (const [name, localPath] of [ [helperName, localHelperPath], [cursorHelperName, localCursorHelperPath], + [focusHelperName, localFocusHelperPath], ]) { const exe = findExecutable(archBuildDir, swift, name); if (!exe) { diff --git a/src/App.tsx b/src/App.tsx index 35749408c3..3dcecf6a75 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,6 +10,10 @@ import { ShortcutsProvider } from "./contexts/ShortcutsContext"; import { loadAllCustomFonts } from "./lib/customFonts"; const VideoEditor = lazy(() => import("./components/video-editor/VideoEditor")); +const CliExportRunner = lazy(() => import("./cli/CliExportRunner")); +const CliRecordRunner = lazy(() => import("./cli/CliRecordRunner")); +const CliSourcesRunner = lazy(() => import("./cli/CliSourcesRunner")); +const CliCaptionsRunner = lazy(() => import("./cli/CliCaptionsRunner")); const ShortcutsConfigDialog = lazy(() => import("./components/video-editor/ShortcutsConfigDialog").then((module) => ({ default: module.ShortcutsConfigDialog, @@ -66,6 +70,30 @@ export default function App() { return <SourceSelector />; case "countdown-overlay": return <CountdownOverlay />; + case "cli-export": + return ( + <Suspense fallback={null}> + <CliExportRunner /> + </Suspense> + ); + case "cli-record": + return ( + <Suspense fallback={null}> + <CliRecordRunner /> + </Suspense> + ); + case "cli-sources": + return ( + <Suspense fallback={null}> + <CliSourcesRunner /> + </Suspense> + ); + case "cli-captions": + return ( + <Suspense fallback={null}> + <CliCaptionsRunner /> + </Suspense> + ); case "editor": return ( <ShortcutsProvider> diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx new file mode 100644 index 0000000000..32d01ca119 --- /dev/null +++ b/src/cli/CliCaptionsRunner.tsx @@ -0,0 +1,168 @@ +// Hidden-window runner for `openscreen captions`: transcribes the project's +// audio with the on-device Whisper worker and writes the resulting caption +// annotations back into the project. Mirrors VideoEditor.generateAutoCaptions. + +import { useEffect, useRef, useState } from "react"; +import { + normalizeProjectEditor, + resolveProjectMedia, + toFileUrl, + validateProjectData, +} from "@/components/video-editor/projectPersistence"; +import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types"; +import { captionSegmentsToAnnotationRegions } from "@/lib/captioning/annotationsFromCaptions"; +import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k"; +import { + shiftTrimRegionsMsForCaptionBuffer, + trimLeadingSilenceMono16k, +} from "@/lib/captioning/leadingSilence"; +import { transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; +import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; +import { nativeBridgeClient } from "@/native"; + +/** Highest trailing number across existing region ids, so new ids never collide. */ +function nextNumericIdFrom(regions: { id: string }[]): number { + let max = 0; + for (const region of regions) { + const match = /(\d+)$/.exec(region.id); + if (match) max = Math.max(max, Number(match[1])); + } + return max + 1; +} + +async function runCaptions(request: CliCaptionsRequest): Promise<CliDoneResult> { + const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath); + if (!loaded.success || loaded.project === undefined) { + throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); + } + if (!validateProjectData(loaded.project)) { + throw new Error("Project file is not a valid .openscreen project"); + } + const project = loaded.project; + const media = resolveProjectMedia(project); + if (!media) { + throw new Error("Project file does not reference any recorded media"); + } + const editor = normalizeProjectEditor(project.editor ?? {}); + const trimRegions: TrimRegion[] = editor.trimRegions; + + window.electronAPI.cliLog("info", "Extracting audio…"); + const videoUrl = toFileUrl(media.screenVideoPath); + const { samples, durationSec } = await extractMono16kFromVideoUrl(videoUrl); + if (!Number.isFinite(durationSec) || durationSec <= 0 || samples.length < 800) { + throw new Error("The project's video has no usable audio track to transcribe"); + } + + const { samples: speechSamples, trimSec } = trimLeadingSilenceMono16k(samples); + if (speechSamples.length < 800) { + throw new Error("No speech detected in the project's audio"); + } + + const trimMs = Math.round(trimSec * 1000); + const trimRegionsForTranscribe = shiftTrimRegionsMsForCaptionBuffer(trimRegions, trimMs); + + const transcribeOptions = { + onStatus: (phase: "model" | "transcribe") => { + window.electronAPI.cliLog( + "info", + phase === "model" ? "Loading caption model…" : "Transcribing…", + ); + }, + }; + + let { segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(speechSamples, { + trimRegions: trimRegionsForTranscribe, + ...transcribeOptions, + }); + let transcribedFromTrimmedBuffer = true; + + // Leading-silence trimming can return empty even when the full source has + // speech. Retry once against the untrimmed buffer before giving up. + if (segmentsRaw.length === 0 && trimSec > 0) { + ({ segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(samples, { + trimRegions, + ...transcribeOptions, + })); + transcribedFromTrimmedBuffer = false; + } + + const segments = + transcribedFromTrimmedBuffer && trimSec > 0 + ? segmentsRaw.map((segment) => ({ + ...segment, + startSec: segment.startSec + trimSec, + endSec: segment.endSec + trimSec, + })) + : segmentsRaw; + + // Re-running the command replaces earlier auto-captions instead of stacking + // duplicates; manually added annotations are preserved. + const manualAnnotations: AnnotationRegion[] = editor.annotationRegions.filter( + (annotation) => annotation.annotationSource !== "auto-caption", + ); + const startNumericId = nextNumericIdFrom([...editor.annotationRegions, ...editor.zoomRegions]); + const startZIndex = manualAnnotations.reduce((max, a) => Math.max(max, a.zIndex + 1), 1); + + let { regions } = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { + minWordsPerCaption: request.minWordsPerCaption, + maxWordsPerCaption: request.maxWordsPerCaption, + timestampGranularity: granularity, + }); + if (regions.length === 0 && segments.length > 0) { + ({ regions } = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { + minWordsPerCaption: 1, + maxWordsPerCaption: Number.MAX_SAFE_INTEGER, + timestampGranularity: granularity, + })); + } + if (regions.length === 0) { + throw new Error("Transcription produced no caption segments"); + } + + const updatedProject = { + ...project, + editor: { + ...editor, + annotationRegions: [...manualAnnotations, ...regions], + }, + }; + + return { + success: true, + projectPath: request.projectPath, + projectData: updatedProject, + captionCount: regions.length, + }; +} + +export function CliCaptionsRunner() { + const startedRef = useRef(false); + const [status] = useState("Generating captions…"); + + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + const request = await window.electronAPI.cliGetRequest(); + if (request.kind !== "captions") { + throw new Error(`cli-captions window received a ${request.kind} request`); + } + const result = await runCaptions(request); + await window.electronAPI.cliDone(result); + } catch (error) { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + } + })(); + }, []); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliCaptionsRunner; diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx new file mode 100644 index 0000000000..ae95ab69b4 --- /dev/null +++ b/src/cli/CliExportRunner.tsx @@ -0,0 +1,461 @@ +// Hidden-window runner for `openscreen export`. Loads an .openscreen project, +// rebuilds the same exporter configuration the editor's export dialog would, +// and streams progress back to the CLI controller in the main process. + +import { useEffect, useRef, useState } from "react"; +import { + DEFAULT_CURSOR_SETTINGS, + DEFAULT_SOURCE_DIMENSIONS, +} from "@/components/video-editor/editorDefaults"; +import { + normalizeProjectEditor, + resolveProjectMedia, + toFileUrl, + validateProjectData, +} from "@/components/video-editor/projectPersistence"; +import { buildAutoZoomSuggestions } from "@/components/video-editor/timeline/zoomSuggestionUtils"; +import type { CursorTelemetryPoint, ZoomRegion } from "@/components/video-editor/types"; +import { + clampFocusToDepth, + DEFAULT_ZOOM_DEPTH, + ZOOM_DEPTH_SCALES, +} from "@/components/video-editor/types"; +import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; +import { hasNativeCursorRecordingData } from "@/lib/cursor/nativeCursor"; +import { calculateOutputDimensions, GifExporter } from "@/lib/exporter/gifExporter"; +import { + calculateEffectiveSourceDimensions, + calculateMp4ExportSettings, +} from "@/lib/exporter/mp4ExportSettings"; +import type { ExportProgress } from "@/lib/exporter/types"; +import { GIF_SIZE_PRESETS } from "@/lib/exporter/types"; +import { VideoExporter } from "@/lib/exporter/videoExporter"; +import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; +import type { FocusRecordingData } from "@/lib/windowFocus/contracts"; +import { FOCUS_SIDECAR_SUFFIX } from "@/lib/windowFocus/contracts"; +import { focusTelemetryToZoomRegions } from "@/lib/windowFocus/focusToZoomRegions"; +import type { MultiWindowManifest } from "@/lib/windowSwitch/contracts"; +import { MULTIWINDOW_SIDECAR_SUFFIX } from "@/lib/windowSwitch/contracts"; +import { nativeBridgeClient } from "@/native"; +import type { CursorRecordingData, NativePlatform } from "@/native/contracts"; +import { getAspectRatioValue, getNativeAspectRatioValue } from "@/utils/aspectRatioUtils"; +import { composeMultiWindowVideo } from "./multiWindowCompositor"; + +// Mirrors the private helper in VideoEditor.tsx. +function isClickInteractionType(interactionType: string | null | undefined) { + return ( + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click" + ); +} + +function probeVideoDimensions( + url: string, +): Promise<{ width: number; height: number; durationMs: number }> { + return new Promise((resolve, reject) => { + const video = document.createElement("video"); + video.preload = "metadata"; + video.muted = true; + const cleanup = () => { + clearTimeout(timer); + video.removeAttribute("src"); + video.load(); + }; + // A stalled load fires neither event; without a deadline the CLI hangs. + const timer = setTimeout(() => { + cleanup(); + reject(new Error(`Timed out reading video metadata: ${url}`)); + }, 30_000); + video.onloadedmetadata = () => { + const width = video.videoWidth; + const height = video.videoHeight; + const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0; + cleanup(); + resolve({ width, height, durationMs }); + }; + video.onerror = () => { + cleanup(); + reject(new Error(`Failed to load video metadata: ${url}`)); + }; + video.src = url; + }); +} + +/** Fit the composition aspect ratio into the reference preview box, mirroring + * how the editor sizes its on-screen preview container. */ +function fitPreviewBox(aspectRatioValue: number, boxWidth: number, boxHeight: number) { + let width = boxWidth; + let height = boxWidth / aspectRatioValue; + if (height > boxHeight) { + height = boxHeight; + width = boxHeight * aspectRatioValue; + } + return { width: Math.round(width), height: Math.round(height) }; +} + +/** Mirrors the editor's buildAutoZoomRegions: cursor-dwell suggestions that + * follow the cursor (focusMode "auto") and never overlap existing regions. */ +function buildAutoZoomRegions( + cursorTelemetry: CursorTelemetryPoint[], + totalMs: number, + existingRegions: ZoomRegion[], +): ZoomRegion[] { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry, + totalMs, + existingRegions, + defaultDurationMs: Math.max(1000, Math.round(totalMs * 0.05)), + }); + let nextId = 1; + return suggestions.map((suggestion) => ({ + id: `cli-auto-zoom-${nextId++}`, + startMs: Math.round(suggestion.span.start), + endMs: Math.round(suggestion.span.end), + depth: DEFAULT_ZOOM_DEPTH, + customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], + focus: clampFocusToDepth(suggestion.focus, DEFAULT_ZOOM_DEPTH), + focusMode: "auto" as const, + source: "auto" as const, + })); +} + +function replaceExtension(filePath: string, newExtension: string): string { + return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; +} + +async function runExport(request: CliExportRequest): Promise<CliDoneResult> { + const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath); + if (!loaded.success || loaded.project === undefined) { + throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); + } + if (!validateProjectData(loaded.project)) { + throw new Error("Project file is not a valid .openscreen project"); + } + const project = loaded.project; + const media = resolveProjectMedia(project); + if (!media) { + throw new Error("Project file does not reference any recorded media"); + } + // Prefer the main process's approved session paths: they carry the + // packed-project sibling fallback when the stored absolute paths are stale. + try { + const sessionResult = await window.electronAPI.getCurrentRecordingSession(); + const session = sessionResult?.session; + if (session?.screenVideoPath) { + media.screenVideoPath = session.screenVideoPath; + if (media.webcamVideoPath && session.webcamVideoPath) { + media.webcamVideoPath = session.webcamVideoPath; + } + } + } catch { + // Fall back to the paths stored in the project file. + } + const editor = normalizeProjectEditor(project.editor ?? {}); + + const format = request.format ?? editor.exportFormat; + if (request.audioPath && format === "gif") { + throw new Error( + "--audio is only supported for MP4 exports (this project's stored format is gif; pass --format mp4)", + ); + } + const quality = request.quality ?? editor.exportQuality; + const gifFrameRate = request.gifFrameRate ?? editor.gifFrameRate; + const gifSizePreset = request.gifSizePreset ?? editor.gifSizePreset; + const outPath = + request.outPath ?? replaceExtension(request.projectPath, format === "gif" ? ".gif" : ".mp4"); + + let videoUrl = toFileUrl(media.screenVideoPath); + const webcamVideoUrl = media.webcamVideoPath ? toFileUrl(media.webcamVideoPath) : undefined; + let isMultiWindow = false; + + // Multi-window captures: composite the per-window videos into one switcher + // video (slide transitions on focus hand-offs) before the normal pipeline. + if (request.followWindows) { + const manifestPath = `${media.screenVideoPath}${MULTIWINDOW_SIDECAR_SUFFIX}`; + const manifestResponse = await fetch(toFileUrl(manifestPath)).catch(() => null); + if (manifestResponse?.ok) { + isMultiWindow = true; + const manifest = (await manifestResponse.json()) as MultiWindowManifest; + window.electronAPI.cliLog( + "info", + `Multi-window capture: compositing ${manifest.windows.length} windows…`, + ); + const composed = await composeMultiWindowVideo( + manifest, + manifest.windows.map((captured) => toFileUrl(captured.videoPath)), + (progress) => + window.electronAPI.cliProgress({ + percentage: progress.percentage, + currentFrame: progress.currentFrame, + totalFrames: progress.totalFrames, + phase: "compositing-windows", + }), + ); + videoUrl = URL.createObjectURL(composed.blob); + window.electronAPI.cliLog( + "info", + `Window switcher composed: ${composed.timeline.segments.length} segment(s), ${composed.timeline.transitions.length} transition(s)`, + ); + } + } + + // Cursor sidecar data (native recordings). Both lookups tolerate missing files. + let cursorTelemetry: CursorTelemetryPoint[] = []; + let cursorRecordingData: CursorRecordingData | null = null; + try { + cursorTelemetry = await nativeBridgeClient.cursor.getTelemetry(media.screenVideoPath); + } catch { + cursorTelemetry = []; + } + try { + cursorRecordingData = await nativeBridgeClient.cursor.getRecordingData(media.screenVideoPath); + } catch { + cursorRecordingData = null; + } + + const recordingClicks = + cursorRecordingData?.samples + .filter((sample) => isClickInteractionType(sample.interactionType)) + .map((sample) => sample.timeMs) ?? []; + const cursorClickTimestamps = + recordingClicks.length > 0 + ? recordingClicks + : cursorTelemetry + .filter((sample) => isClickInteractionType(sample.interactionType)) + .map((sample) => sample.timeMs); + + let platform: NativePlatform | null = null; + try { + platform = await nativeBridgeClient.system.getPlatform(); + } catch { + platform = null; + } + const hasEditableCursorRecording = + (media.cursorCaptureMode ?? "editable-overlay") === "editable-overlay" && + (platform === "win32" || platform === "darwin") && + hasNativeCursorRecordingData(cursorRecordingData); + const effectiveShowCursor = DEFAULT_CURSOR_SETTINGS.show && hasEditableCursorRecording; + + const probed = await probeVideoDimensions(videoUrl); + const sourceWidth = probed.width || DEFAULT_SOURCE_DIMENSIONS.width; + const sourceHeight = probed.height || DEFAULT_SOURCE_DIMENSIONS.height; + + if (request.followWindows && !isMultiWindow) { + const sidecarPath = `${media.screenVideoPath}${FOCUS_SIDECAR_SUFFIX}`; + const response = await fetch(toFileUrl(sidecarPath)).catch(() => null); + if (!response?.ok) { + throw new Error( + `--follow-windows needs the focus telemetry sidecar (${sidecarPath}); record with "record --follow-windows"`, + ); + } + const focusData = (await response.json()) as FocusRecordingData; + const followRegions = focusTelemetryToZoomRegions(focusData, { + totalMs: probed.durationMs, + existingRegions: editor.zoomRegions, + }); + if (followRegions.length > 0) { + editor.zoomRegions = [...editor.zoomRegions, ...followRegions].sort( + (a, b) => a.startMs - b.startMs, + ); + } + window.electronAPI.cliLog( + "info", + `Follow-windows: added ${followRegions.length} region(s) from focus telemetry`, + ); + } + + if (request.autoZoom) { + const autoRegions = buildAutoZoomRegions( + cursorTelemetry, + probed.durationMs, + editor.zoomRegions, + ); + if (autoRegions.length > 0) { + editor.zoomRegions = [...editor.zoomRegions, ...autoRegions]; + } + window.electronAPI.cliLog( + "info", + `Auto-zoom: added ${autoRegions.length} region(s) from cursor telemetry`, + ); + } + const effectiveSourceDimensions = calculateEffectiveSourceDimensions( + sourceWidth, + sourceHeight, + editor.cropRegion, + ); + const aspectRatioValue = + editor.aspectRatio === "native" + ? getNativeAspectRatioValue(sourceWidth, sourceHeight, editor.cropRegion) + : getAspectRatioValue(editor.aspectRatio); + + const preview = fitPreviewBox( + aspectRatioValue, + request.previewWidth ?? 1280, + request.previewHeight ?? 720, + ); + + const onProgress = (progress: ExportProgress) => { + window.electronAPI.cliProgress({ + percentage: progress.percentage, + currentFrame: progress.currentFrame, + totalFrames: progress.totalFrames, + estimatedTimeRemaining: progress.estimatedTimeRemaining, + phase: progress.phase, + }); + }; + + const sharedConfig = { + videoUrl, + webcamVideoUrl, + wallpaper: editor.wallpaper, + zoomRegions: editor.zoomRegions, + cameraFullscreenRegions: editor.cameraFullscreenRegions, + trimRegions: editor.trimRegions, + speedRegions: editor.speedRegions, + showShadow: editor.shadowIntensity > 0, + shadowIntensity: editor.shadowIntensity, + showBlur: editor.showBlur, + motionBlurAmount: editor.motionBlurAmount, + borderRadius: editor.borderRadius, + padding: editor.padding, + cropRegion: editor.cropRegion, + cursorRecordingData, + cursorScale: effectiveShowCursor ? DEFAULT_CURSOR_SETTINGS.size : 0, + cursorSmoothing: DEFAULT_CURSOR_SETTINGS.smoothing, + cursorMotionBlur: DEFAULT_CURSOR_SETTINGS.motionBlur, + cursorClickBounce: DEFAULT_CURSOR_SETTINGS.clickBounce, + cursorClipToBounds: DEFAULT_CURSOR_SETTINGS.clipToBounds, + cursorTheme: editor.cursorTheme, + annotationRegions: editor.annotationRegions, + webcamLayoutPreset: editor.webcamLayoutPreset, + webcamMaskShape: editor.webcamMaskShape, + webcamMirrored: editor.webcamMirrored, + webcamReactiveZoom: editor.webcamReactiveZoom, + webcamSizePreset: editor.webcamSizePreset, + webcamPosition: editor.webcamPosition, + previewWidth: preview.width, + previewHeight: preview.height, + cursorTelemetry, + cursorClickTimestamps, + onProgress, + }; + + let blob: Blob; + let warnings: string[] | undefined; + let outWidth: number; + let outHeight: number; + + if (format === "gif") { + const gifDimensions = calculateOutputDimensions( + effectiveSourceDimensions.width, + effectiveSourceDimensions.height, + gifSizePreset, + GIF_SIZE_PRESETS, + aspectRatioValue, + ); + outWidth = gifDimensions.width; + outHeight = gifDimensions.height; + const gifExporter = new GifExporter({ + ...sharedConfig, + width: gifDimensions.width, + height: gifDimensions.height, + frameRate: gifFrameRate, + loop: editor.gifLoop, + sizePreset: gifSizePreset, + videoPadding: editor.padding, + }); + const result = await gifExporter.export(); + if (!result.success || !result.blob) { + throw new Error(result.error ?? "GIF export failed"); + } + blob = result.blob; + warnings = result.warnings; + } else { + const mp4Settings = calculateMp4ExportSettings({ + quality, + sourceWidth: effectiveSourceDimensions.width, + sourceHeight: effectiveSourceDimensions.height, + aspectRatioValue, + }); + outWidth = mp4Settings.width; + outHeight = mp4Settings.height; + const exporter = new VideoExporter({ + ...sharedConfig, + width: mp4Settings.width, + height: mp4Settings.height, + frameRate: 60, + bitrate: mp4Settings.bitrate, + codec: "avc1.640033", + }); + const result = await exporter.export(); + if (!result.success || !result.blob) { + throw new Error(result.error ?? "MP4 export failed"); + } + blob = result.blob; + warnings = result.warnings; + } + + if (request.audioPath && format === "mp4") { + window.electronAPI.cliProgress({ percentage: 100, phase: "mixing-voiceover" }); + const audioResponse = await fetch(toFileUrl(request.audioPath)); + if (!audioResponse.ok) { + throw new Error(`Failed to read voiceover file: ${request.audioPath}`); + } + const voiceoverData = await audioResponse.arrayBuffer(); + blob = await mixVoiceoverIntoVideo(blob, { + voiceoverData, + mode: request.audioMode, + offsetSec: request.audioOffsetSec, + }); + } + + const arrayBuffer = await blob.arrayBuffer(); + const saveResult = await window.electronAPI.writeExportToPath(arrayBuffer, outPath); + if (!saveResult.success || !saveResult.path) { + throw new Error(saveResult.message ?? `Failed to write output to ${outPath}`); + } + + return { + success: true, + outputPath: saveResult.path, + format, + width: outWidth, + height: outHeight, + warnings, + }; +} + +export function CliExportRunner() { + const startedRef = useRef(false); + const [status, setStatus] = useState("Starting export…"); + + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + const request = (await window.electronAPI.cliGetRequest()) as CliExportRequest; + if (request.kind !== "export") { + throw new Error(`cli-export window received a ${request.kind} request`); + } + setStatus(`Exporting ${request.projectPath}…`); + const result = await runExport(request); + await window.electronAPI.cliDone(result); + } catch (error) { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + } + })(); + }, []); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliExportRunner; diff --git a/src/cli/CliRecordRunner.tsx b/src/cli/CliRecordRunner.tsx new file mode 100644 index 0000000000..387a9ef321 --- /dev/null +++ b/src/cli/CliRecordRunner.tsx @@ -0,0 +1,302 @@ +// Hidden-window runner for `openscreen record`. Reuses the full recording +// pipeline via useScreenRecorder (native macOS/Windows helpers with browser +// fallback), driven by the CLI controller instead of the HUD. + +import { useEffect, useRef, useState } from "react"; +import { + normalizeProjectEditor, + PROJECT_VERSION, +} from "@/components/video-editor/projectPersistence"; +import { useScreenRecorder } from "@/hooks/useScreenRecorder"; +import type { CliRecordRequest } from "@/lib/cliContracts"; + +type Phase = "init" | "recording" | "stopping" | "done"; + +async function pickSource(request: CliRecordRequest): Promise<ProcessedDesktopSource> { + const sources = await window.electronAPI.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 32, height: 18 }, + }); + + if (request.windowTitle) { + const needle = request.windowTitle.toLowerCase(); + const match = sources.find( + (source) => source.id.startsWith("window:") && source.name.toLowerCase().includes(needle), + ); + if (!match) { + const windows = sources + .filter((s) => s.id.startsWith("window:")) + .map((s) => ` - ${s.name}`) + .join("\n"); + throw new Error( + `No window title contains "${request.windowTitle}". Open windows:\n${windows}`, + ); + } + return match; + } + + const screens = sources.filter((source) => source.id.startsWith("screen:")); + const screen = screens[request.displayIndex]; + if (!screen) { + throw new Error( + `Display index ${request.displayIndex} not found (${screens.length} screen(s) available)`, + ); + } + return screen; +} + +async function resolveMicDeviceId(deviceNameFilter: string | null): Promise<{ + deviceId: string | undefined; + deviceName: string | undefined; +}> { + if (!deviceNameFilter) return { deviceId: undefined, deviceName: undefined }; + + // Labels require an active permission grant; a short-lived stream unlocks them. + let probeStream: MediaStream | null = null; + try { + probeStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + // Enumeration below may still work with empty labels. + } + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const needle = deviceNameFilter.toLowerCase(); + const match = devices.find( + (device) => device.kind === "audioinput" && device.label.toLowerCase().includes(needle), + ); + if (!match) { + const labels = devices + .filter((d) => d.kind === "audioinput" && d.label) + .map((d) => ` - ${d.label}`) + .join("\n"); + throw new Error(`No microphone label contains "${deviceNameFilter}". Devices:\n${labels}`); + } + return { deviceId: match.deviceId, deviceName: match.label }; + } finally { + probeStream?.getTracks().forEach((track) => track.stop()); + } +} + +/** A minimal .openscreen project referencing the finished recording, with all + * editor settings at their defaults — ready for `openscreen export` or the GUI. */ +function buildDefaultProject(session: { + screenVideoPath: string; + webcamVideoPath?: string; + cursorCaptureMode?: string; +}) { + return { + version: PROJECT_VERSION, + media: { + screenVideoPath: session.screenVideoPath, + ...(session.webcamVideoPath ? { webcamVideoPath: session.webcamVideoPath } : {}), + ...(session.cursorCaptureMode ? { cursorCaptureMode: session.cursorCaptureMode } : {}), + }, + editor: normalizeProjectEditor({}), + }; +} + +export function CliRecordRunner() { + const recorder = useScreenRecorder(); + const startedRef = useRef(false); + const requestRef = useRef<CliRecordRequest | null>(null); + // Re-render trigger once the bootstrap has applied all recorder settings; + // the start effect below cannot rely on recorder state deps alone because + // default-valued settings (no mic, no system audio) never change. + const [requestReady, setRequestReady] = useState<CliRecordRequest | null>(null); + const phaseRef = useRef<Phase>("init"); + const recordingStartedAtRef = useRef<number | null>(null); + // A stop (SIGINT/stdin) can land while the capture helper is still starting; + // remember it and apply as soon as recording flips on. + const stopRequestedRef = useRef(false); + const [status, setStatus] = useState("Preparing recording…"); + + const { + recording, + saving, + startRecordingImmediately, + toggleRecording, + setMicrophoneEnabled, + setMicrophoneDeviceId, + setMicrophoneDeviceName, + setSystemAudioEnabled, + setCursorCaptureMode, + } = recorder; + + // Keep latest values in refs for the stop/finish effects. + const toggleRecordingRef = useRef(toggleRecording); + const recordingRef = useRef(recording); + useEffect(() => { + toggleRecordingRef.current = toggleRecording; + recordingRef.current = recording; + }); + + const fail = async (error: unknown) => { + phaseRef.current = "done"; + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + }; + + // Bootstrap: pick source, configure recorder, start. + // biome-ignore lint/correctness/useExhaustiveDependencies: intentional run-once bootstrap; startedRef guards re-entry + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + // The recorder hook surfaces some failures via blocking alert(); a + // hidden window must never show (or hang on) a modal. + window.alert = (message?: unknown) => { + window.electronAPI.cliLog("error", `Recorder: ${String(message)}`); + }; + + const request = (await window.electronAPI.cliGetRequest()) as CliRecordRequest; + if (request.kind !== "record") { + throw new Error(`cli-record window received a ${request.kind} request`); + } + requestRef.current = request; + + const source = await pickSource(request); + await window.electronAPI.selectSource(source); + window.electronAPI.cliLog("info", `Recording source: ${source.name}`); + + if (request.mic) { + const mic = await resolveMicDeviceId(request.micDevice); + setMicrophoneEnabled(true); + setMicrophoneDeviceId(mic.deviceId); + setMicrophoneDeviceName(mic.deviceName); + if (mic.deviceName) { + window.electronAPI.cliLog("info", `Microphone: ${mic.deviceName}`); + } + } + setSystemAudioEnabled(request.systemAudio); + setCursorCaptureMode(request.cursorMode); + setStatus("Starting recording…"); + setRequestReady(request); + } catch (error) { + await fail(error); + } + })(); + }, []); + + // The setters above land on the *next* render; start only once they have. + const configuredRef = useRef(false); + // biome-ignore lint/correctness/useExhaustiveDependencies: fires when recorder settings match the request; other referenced values are stable refs/callbacks + useEffect(() => { + const request = requestReady; + if (!request || configuredRef.current || phaseRef.current !== "init") return; + const micReady = !request.mic || recorder.microphoneEnabled; + const systemAudioReady = recorder.systemAudioEnabled === request.systemAudio; + const cursorReady = recorder.cursorCaptureMode === request.cursorMode; + if (!micReady || !systemAudioReady || !cursorReady) return; + + configuredRef.current = true; + phaseRef.current = "recording"; + void (async () => { + try { + await startRecordingImmediately(); + // The hook reports start failures via toast/console, not by + // rejecting — without a deadline a failed start would hang the + // CLI forever. + setTimeout(() => { + if (recordingStartedAtRef.current === null && phaseRef.current === "recording") { + void fail( + new Error( + "Recording did not start within 30s — see stderr for the underlying capture error", + ), + ); + } + }, 30_000); + } catch (error) { + await fail(error); + } + })(); + }, [ + requestReady, + recorder.microphoneEnabled, + recorder.systemAudioEnabled, + recorder.cursorCaptureMode, + ]); + + // Recording state transitions: report start and arm the duration timer. + useEffect(() => { + const request = requestRef.current; + if (recording && recordingStartedAtRef.current === null) { + recordingStartedAtRef.current = Date.now(); + setStatus("Recording…"); + window.electronAPI.cliLog("info", "Recording started"); + window.electronAPI.cliRecordingStarted(); + + if (stopRequestedRef.current) { + phaseRef.current = "stopping"; + setStatus("Stopping…"); + toggleRecordingRef.current(); + return; + } + + if (request?.durationMs) { + const timer = setTimeout(() => { + if (recordingRef.current && phaseRef.current === "recording") { + phaseRef.current = "stopping"; + window.electronAPI.cliLog("info", `Duration reached (${request.durationMs}ms)`); + toggleRecordingRef.current(); + } + }, request.durationMs); + return () => clearTimeout(timer); + } + } + }, [recording]); + + // External stop (SIGINT / stdin via main process). + useEffect(() => { + return window.electronAPI.onCliStopRecording(() => { + if (recordingRef.current && phaseRef.current === "recording") { + phaseRef.current = "stopping"; + setStatus("Stopping…"); + toggleRecordingRef.current(); + } else { + // Capture is still starting; stop as soon as it comes up. + stopRequestedRef.current = true; + } + }); + }, []); + + // Completion: recording flipped off and the session finished saving. + // biome-ignore lint/correctness/useExhaustiveDependencies: completion is keyed on recording/saving; fail is stable + useEffect(() => { + if (recordingStartedAtRef.current === null) return; + if (recording || saving) return; + if (phaseRef.current === "done") return; + phaseRef.current = "done"; + + void (async () => { + try { + const sessionResult = await window.electronAPI.getCurrentRecordingSession(); + const session = sessionResult?.session; + if (!session?.screenVideoPath) { + throw new Error("Recording finished but no session manifest was stored"); + } + const durationMs = Date.now() - (recordingStartedAtRef.current ?? Date.now()); + const request = requestRef.current; + await window.electronAPI.cliDone({ + success: true, + screenVideoPath: session.screenVideoPath, + webcamVideoPath: session.webcamVideoPath, + cursorDataPath: `${session.screenVideoPath}.cursor.json`, + durationMs, + ...(request?.projectOut ? { projectData: buildDefaultProject(session) } : {}), + }); + } catch (error) { + await fail(error); + } + })(); + }, [recording, saving]); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliRecordRunner; diff --git a/src/cli/CliSourcesRunner.tsx b/src/cli/CliSourcesRunner.tsx new file mode 100644 index 0000000000..a1e464e232 --- /dev/null +++ b/src/cli/CliSourcesRunner.tsx @@ -0,0 +1,87 @@ +// Hidden-window runner for `openscreen sources`: enumerates capturable +// displays/windows (via the same get-sources IPC the GUI picker uses) and +// microphone inputs, then hands the payload to the CLI controller to print. + +import { useEffect, useRef, useState } from "react"; +import type { CliSourcesResult } from "@/lib/cliContracts"; + +async function enumerateMicrophones(): Promise<{ + microphones: { label: string }[]; + microphoneLabelsUnavailable: boolean; +}> { + const listInputs = async () => + (await navigator.mediaDevices.enumerateDevices()).filter( + (device) => device.kind === "audioinput", + ); + + let inputs = await listInputs(); + + // Labels are blank until a getUserMedia grant exists; a short-lived probe + // stream unlocks them without leaving anything recording. + if (inputs.length > 0 && inputs.every((device) => !device.label)) { + let probeStream: MediaStream | null = null; + try { + probeStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + inputs = await listInputs(); + } catch { + // Permission denied — report devices without labels. + } finally { + probeStream?.getTracks().forEach((track) => track.stop()); + } + } + + const labeled = inputs.filter((device) => device.label); + return { + microphones: labeled.map((device) => ({ label: device.label })), + microphoneLabelsUnavailable: inputs.length > 0 && labeled.length === 0, + }; +} + +async function enumerateSources(): Promise<CliSourcesResult> { + const sources = await window.electronAPI.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 32, height: 18 }, + }); + + const displays = sources + .filter((source) => source.id.startsWith("screen:")) + .map((source, index) => ({ index, id: source.id, name: source.name })); + const windows = sources + .filter((source) => source.id.startsWith("window:")) + .map((source) => ({ id: source.id, name: source.name })); + + const { microphones, microphoneLabelsUnavailable } = await enumerateMicrophones(); + return { displays, windows, microphones, microphoneLabelsUnavailable }; +} + +export function CliSourcesRunner() { + const startedRef = useRef(false); + const [status] = useState("Enumerating sources…"); + + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + const request = await window.electronAPI.cliGetRequest(); + if (request.kind !== "sources") { + throw new Error(`cli-sources window received a ${request.kind} request`); + } + const sources = await enumerateSources(); + await window.electronAPI.cliDone({ success: true, sources }); + } catch (error) { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + } + })(); + }, []); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliSourcesRunner; diff --git a/src/cli/multiWindowCompositor.ts b/src/cli/multiWindowCompositor.ts new file mode 100644 index 0000000000..e808d6f008 --- /dev/null +++ b/src/cli/multiWindowCompositor.ts @@ -0,0 +1,251 @@ +// Offline compositor for multi-window captures: decodes every window's video +// in lockstep, draws the focused window (sliding between windows on focus +// hand-offs), and encodes one intermediate video that the normal export +// pipeline then treats as the "screen recording". + +import { VideoMuxer } from "@/lib/exporter/muxer"; +import { StreamingVideoDecoder } from "@/lib/exporter/streamingDecoder"; +import type { MultiWindowManifest } from "@/lib/windowSwitch/contracts"; +import { + buildWindowSwitchTimeline, + switchStateAt, + type WindowSwitchTimeline, +} from "@/lib/windowSwitch/switchTimeline"; + +const FRAME_RATE = 60; +const BITRATE = 18_000_000; +const CODEC = "avc1.640033"; +const KEYFRAME_INTERVAL = 150; +const BACKGROUND = "#0b0b0d"; + +/** Single-slot handoff between a decoder's push callback and the consumer. */ +class FrameChannel { + private frame: VideoFrame | null = null; + private waiting: (() => void) | null = null; + private consumed: (() => void) | null = null; + private done = false; + + async push(frame: VideoFrame): Promise<void> { + this.frame = frame; + this.waiting?.(); + this.waiting = null; + await new Promise<void>((resolve) => { + this.consumed = resolve; + }); + } + + finish(): void { + this.done = true; + this.waiting?.(); + this.waiting = null; + } + + /** Returns the next frame, or null once the stream has ended. */ + async next(): Promise<VideoFrame | null> { + if (this.frame === null && !this.done) { + await new Promise<void>((resolve) => { + this.waiting = resolve; + }); + } + const frame = this.frame; + this.frame = null; + this.consumed?.(); + this.consumed = null; + return frame; + } +} + +interface SourceStream { + channel: FrameChannel; + /** Most recent frame, kept alive until replaced (sources can end early). */ + lastFrame: VideoFrame | null; + width: number; + height: number; + durationSec: number; + decoder: StreamingVideoDecoder; + decodeError: Error | null; +} + +function drawSource( + ctx: CanvasRenderingContext2D, + source: SourceStream, + canvasWidth: number, + canvasHeight: number, + offsetX: number, +): void { + const frame = source.lastFrame; + if (!frame) return; + const scale = Math.min(canvasWidth / source.width, canvasHeight / source.height); + const drawWidth = source.width * scale; + const drawHeight = source.height * scale; + const x = offsetX + (canvasWidth - drawWidth) / 2; + const y = (canvasHeight - drawHeight) / 2; + ctx.drawImage(frame, x, y, drawWidth, drawHeight); +} + +export interface ComposeProgress { + percentage: number; + currentFrame: number; + totalFrames: number; +} + +export async function composeMultiWindowVideo( + manifest: MultiWindowManifest, + videoUrls: string[], + onProgress: (progress: ComposeProgress) => void, +): Promise<{ blob: Blob; durationMs: number; timeline: WindowSwitchTimeline }> { + if (videoUrls.length !== manifest.windows.length || videoUrls.length < 2) { + throw new Error("Multi-window manifest and video list are inconsistent"); + } + + // Load metadata for every source to size the canvas and bound the duration. + const sources: SourceStream[] = []; + for (const url of videoUrls) { + const decoder = new StreamingVideoDecoder(); + const info = await decoder.loadMetadata(url); + sources.push({ + channel: new FrameChannel(), + lastFrame: null, + width: info.width, + height: info.height, + durationSec: info.duration, + decoder, + decodeError: null, + }); + } + + const canvasWidth = Math.ceil(Math.max(...sources.map((source) => source.width)) / 2) * 2; + const canvasHeight = Math.ceil(Math.max(...sources.map((source) => source.height)) / 2) * 2; + const durationSec = Math.min( + manifest.durationMs / 1000, + Math.max(...sources.map((source) => source.durationSec)), + ); + const totalFrames = Math.max(1, Math.floor(durationSec * FRAME_RATE)); + const timeline = buildWindowSwitchTimeline(manifest, durationSec * 1000); + + // Start every decoder; frames flow through the channels with backpressure. + for (const source of sources) { + void source.decoder + .decodeAll(FRAME_RATE, undefined, undefined, async (frame) => { + await source.channel.push(frame); + }) + .catch((error: unknown) => { + source.decodeError = error instanceof Error ? error : new Error(String(error)); + }) + .finally(() => { + source.channel.finish(); + }); + } + + const canvas = document.createElement("canvas"); + canvas.width = canvasWidth; + canvas.height = canvasHeight; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Failed to create compositor canvas"); + + const muxer = new VideoMuxer( + { width: canvasWidth, height: canvasHeight, frameRate: FRAME_RATE, bitrate: BITRATE }, + false, + ); + await muxer.initialize(); + + let encodeError: Error | null = null; + const encoder = new VideoEncoder({ + output: (chunk, meta) => { + void muxer.addVideoChunk(chunk, meta).catch((error: unknown) => { + encodeError = error instanceof Error ? error : new Error(String(error)); + }); + }, + error: (error) => { + encodeError = error; + }, + }); + encoder.configure({ + codec: CODEC, + width: canvasWidth, + height: canvasHeight, + bitrate: BITRATE, + framerate: FRAME_RATE, + }); + + const frameDurationUs = 1_000_000 / FRAME_RATE; + try { + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) { + const timeMs = (frameIndex / FRAME_RATE) * 1000; + + // Advance every source that still has frames; hold the last frame of + // sources that ended early. + for (const source of sources) { + const frame = await source.channel.next(); + if (frame) { + source.lastFrame?.close(); + source.lastFrame = frame; + } + if (source.decodeError) throw source.decodeError; + } + + const state = switchStateAt(timeline, timeMs); + ctx.fillStyle = BACKGROUND; + ctx.fillRect(0, 0, canvasWidth, canvasHeight); + + if (state.transition) { + const { outgoingIndex, incomingIndex, progress, direction } = state.transition; + const sign = direction === "from-right" ? 1 : -1; + drawSource( + ctx, + sources[outgoingIndex], + canvasWidth, + canvasHeight, + -sign * progress * canvasWidth, + ); + drawSource( + ctx, + sources[incomingIndex], + canvasWidth, + canvasHeight, + sign * (1 - progress) * canvasWidth, + ); + } else { + drawSource(ctx, sources[state.activeIndex], canvasWidth, canvasHeight, 0); + } + + const composedFrame = new VideoFrame(canvas, { + timestamp: Math.round(frameIndex * frameDurationUs), + duration: Math.round(frameDurationUs), + }); + // Basic encoder backpressure. + while (encoder.encodeQueueSize > 60) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + encoder.encode(composedFrame, { keyFrame: frameIndex % KEYFRAME_INTERVAL === 0 }); + composedFrame.close(); + if (encodeError) throw encodeError; + + if (frameIndex % 30 === 0 || frameIndex === totalFrames - 1) { + onProgress({ + percentage: ((frameIndex + 1) / totalFrames) * 100, + currentFrame: frameIndex + 1, + totalFrames, + }); + } + } + + await encoder.flush(); + encoder.close(); + const blob = await muxer.finalize(); + return { blob, durationMs: (totalFrames / FRAME_RATE) * 1000, timeline }; + } finally { + for (const source of sources) { + source.lastFrame?.close(); + source.lastFrame = null; + // Drain so pending decodeAll calls can finish and release resources. + void (async () => { + let frame = await source.channel.next(); + while (frame) { + frame.close(); + frame = await source.channel.next(); + } + })(); + } + } +} diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index af95b4722a..42458f1e67 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -54,6 +54,8 @@ type UseScreenRecorderReturn = { saving: boolean; elapsedSeconds: number; toggleRecording: () => void; + /** Starts recording with no countdown overlay. Used by the headless CLI runner. */ + startRecordingImmediately: () => Promise<void>; togglePaused: () => void; canPauseRecording: boolean; restartRecording: () => void; @@ -1728,6 +1730,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { saving, elapsedSeconds, toggleRecording, + startRecordingImmediately: () => startRecording(), togglePaused, canPauseRecording, restartRecording, diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts new file mode 100644 index 0000000000..371b70ada9 --- /dev/null +++ b/src/lib/cliContracts.ts @@ -0,0 +1,140 @@ +// Shared request/response contracts between the CLI entry in the Electron main +// process (electron/cli/) and the hidden renderer runners (src/cli/). Keep this +// file dependency-free so both build targets can import it. + +import type { ExportQuality, GifFrameRate, GifSizePreset } from "./exporter/types"; + +export type CliCursorCaptureMode = "editable-overlay" | "system"; + +export interface CliExportRequest { + kind: "export"; + /** Absolute path to the .openscreen project file. */ + projectPath: string; + /** Absolute output path; null = derive from projectPath + format. */ + outPath: string | null; + /** null = use the format stored in the project. */ + format: "mp4" | "gif" | null; + /** null = use the quality stored in the project. */ + quality: ExportQuality | null; + gifFrameRate: GifFrameRate | null; + gifSizePreset: GifSizePreset | null; + /** + * Reference preview box used to scale annotation text and border radii the + * same way the editor's on-screen preview does. The composition is fitted + * into this box, mirroring the editor layout. Defaults to 1280x720. + */ + previewWidth: number | null; + previewHeight: number | null; + /** + * Add automatic zoom regions derived from cursor-dwell telemetry (same + * suggestion engine as the editor's magic wand) before rendering. Existing + * zoom regions are preserved; suggestions never overlap them. + */ + autoZoom: boolean; + /** + * Convert the recording's window-focus telemetry (`<video>.focus.json`, + * from `record --follow-windows`) into pan/zoom regions that follow the + * focused window. + */ + followWindows: boolean; + /** Absolute path to a voiceover audio file to mix into the export (MP4 only). */ + audioPath: string | null; + /** "mix" layers the voiceover over the recording's audio; "replace" drops the original. */ + audioMode: "mix" | "replace"; + /** Delay before the voiceover starts, in seconds. */ + audioOffsetSec: number; +} + +export interface CliRecordRequest { + kind: "record"; + /** Index into the available screen sources (0 = primary display). */ + displayIndex: number; + /** Case-insensitive substring match against window titles; overrides displayIndex. */ + windowTitle: string | null; + mic: boolean; + /** Microphone device label substring; null = system default. */ + micDevice: string | null; + systemAudio: boolean; + cursorMode: CliCursorCaptureMode; + /** Auto-stop after this many milliseconds; null = stop via signal/stdin. */ + durationMs: number | null; + /** When set, write a ready-to-export .openscreen project here after recording. */ + projectOut: string | null; + /** + * Record window-focus telemetry alongside the capture (macOS). The export + * step's --follow-windows turns it into automatic pan/zoom between the + * windows the user focused. Requires display capture (not --window). + */ + followWindows: boolean; + /** + * Multi-window capture (macOS): comma-separated window-title filters. + * Every matched window is recorded continuously in parallel; export + * switches between them following focus, with slide transitions. + */ + windows: string[] | null; +} + +export interface CliSourcesRequest { + kind: "sources"; +} + +export interface CliCaptionsRequest { + kind: "captions"; + /** Absolute path to the .openscreen project file (updated in place). */ + projectPath: string; + minWordsPerCaption: number; + maxWordsPerCaption: number; +} + +export type CliRequest = + | CliExportRequest + | CliRecordRequest + | CliSourcesRequest + | CliCaptionsRequest; + +export interface CliSourcesResult { + displays: { index: number; id: string; name: string }[]; + windows: { id: string; name: string }[]; + microphones: { label: string }[]; + /** True when microphone labels required a permission the user hasn't granted. */ + microphoneLabelsUnavailable: boolean; +} + +export interface CliProgressEvent { + percentage: number; + currentFrame?: number; + totalFrames?: number; + estimatedTimeRemaining?: number; + phase?: string; +} + +export interface CliDoneResult { + success: boolean; + error?: string; + warnings?: string[]; + /** Export: the written output file. */ + outputPath?: string; + format?: string; + width?: number; + height?: number; + /** Record: produced artifacts. */ + screenVideoPath?: string; + webcamVideoPath?: string; + cursorDataPath?: string; + /** Record --follow-windows: written window-focus telemetry sidecar. */ + focusDataPath?: string; + /** Record --windows: written multi-window manifest. */ + multiWindowManifestPath?: string; + videoPaths?: string[]; + projectPath?: string; + durationMs?: number; + /** + * Record: a ready-to-save .openscreen project object built by the runner. + * The main process writes it to the --project path (renderer has no fs). + */ + projectData?: unknown; + /** Sources: enumeration payload printed by the main process. */ + sources?: CliSourcesResult; + /** Captions: number of caption annotations generated. */ + captionCount?: number; +} diff --git a/src/lib/exporter/voiceoverMix.ts b/src/lib/exporter/voiceoverMix.ts new file mode 100644 index 0000000000..0c63056277 --- /dev/null +++ b/src/lib/exporter/voiceoverMix.ts @@ -0,0 +1,147 @@ +// Post-export voiceover mixing for the CLI (`openscreen export --audio`). +// +// Takes the finished MP4 blob, copies its video packets untouched (no +// re-encode), renders a new audio track with OfflineAudioContext — the +// original audio and the voiceover mixed, or the voiceover alone — and +// re-muxes both into a new MP4 via mediabunny. + +import { + ALL_FORMATS, + AudioBufferSource, + BlobSource, + BufferTarget, + EncodedPacketSink, + EncodedVideoPacketSource, + Input, + Mp4OutputFormat, + Output, +} from "mediabunny"; + +export type VoiceoverMixMode = "mix" | "replace"; + +export interface VoiceoverMixOptions { + /** Encoded audio file bytes (mp3/wav/m4a — anything decodeAudioData accepts). */ + voiceoverData: ArrayBuffer; + mode: VoiceoverMixMode; + /** Delay before the voiceover starts, in seconds. */ + offsetSec: number; + /** Gain applied to the original track in "mix" mode (0..1). */ + originalGain?: number; +} + +// Duck the original bed under the voiceover by default so the unity-gain sum +// of two loud sources doesn't hard-clip. +const DEFAULT_ORIGINAL_GAIN = 0.4; + +const OUTPUT_SAMPLE_RATE = 48_000; +const OUTPUT_CHANNELS = 2; +const VOICEOVER_AUDIO_BITRATE = 192_000; + +async function decodeToBuffer( + context: OfflineAudioContext, + data: ArrayBuffer, +): Promise<AudioBuffer> { + // decodeAudioData detaches the buffer, so hand it a copy. + return context.decodeAudioData(data.slice(0)); +} + +/** Renders the final audio track: original bed (optional) + offset voiceover. */ +async function renderMixedAudio( + videoData: ArrayBuffer | null, + durationSec: number, + options: VoiceoverMixOptions, +): Promise<AudioBuffer> { + const frameCount = Math.max(1, Math.ceil(durationSec * OUTPUT_SAMPLE_RATE)); + const context = new OfflineAudioContext(OUTPUT_CHANNELS, frameCount, OUTPUT_SAMPLE_RATE); + + const voiceover = await decodeToBuffer(context, options.voiceoverData); + const voiceoverNode = context.createBufferSource(); + voiceoverNode.buffer = voiceover; + voiceoverNode.connect(context.destination); + voiceoverNode.start(Math.max(0, options.offsetSec)); + + if (options.mode === "mix" && videoData) { + try { + const original = await decodeToBuffer(context, videoData); + const originalNode = context.createBufferSource(); + originalNode.buffer = original; + const gainNode = context.createGain(); + gainNode.gain.value = options.originalGain ?? DEFAULT_ORIGINAL_GAIN; + originalNode.connect(gainNode); + gainNode.connect(context.destination); + originalNode.start(0); + } catch { + // The exported video has no decodable audio track; the voiceover + // becomes the only audio, same as "replace". + } + } + + return context.startRendering(); +} + +/** + * Returns a new MP4 blob with the same video stream and the mixed audio track. + * The video packets are copied without re-encoding. + */ +export async function mixVoiceoverIntoVideo( + videoBlob: Blob, + options: VoiceoverMixOptions, +): Promise<Blob> { + const input = new Input({ source: new BlobSource(videoBlob), formats: ALL_FORMATS }); + try { + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("Exported file has no video track to remux"); + } + const codec = videoTrack.codec; + if (!codec) { + throw new Error("Exported file's video codec was not recognized"); + } + const decoderConfig = await videoTrack.getDecoderConfig(); + if (!decoderConfig) { + throw new Error("Exported file's video decoder config could not be read"); + } + const durationSec = await input.computeDuration(); + + // The full-file bytes are only needed to decode the original bed in + // "mix" mode; "replace" skips the copy entirely. + const videoData = options.mode === "mix" ? await videoBlob.arrayBuffer() : null; + const mixedAudio = await renderMixedAudio(videoData, durationSec, options); + + const target = new BufferTarget(); + const output = new Output({ + format: new Mp4OutputFormat({ fastStart: "in-memory" }), + target, + }); + try { + const videoSource = new EncodedVideoPacketSource(codec); + output.addVideoTrack(videoSource); + const audioSource = new AudioBufferSource({ + codec: "aac", + bitrate: VOICEOVER_AUDIO_BITRATE, + }); + output.addAudioTrack(audioSource); + await output.start(); + + const sink = new EncodedPacketSink(videoTrack); + let isFirstPacket = true; + for await (const packet of sink.packets()) { + await videoSource.add(packet, isFirstPacket ? { decoderConfig } : undefined); + isFirstPacket = false; + } + await audioSource.add(mixedAudio); + + await output.finalize(); + } catch (error) { + await output.cancel().catch(() => undefined); + throw error; + } + const buffer = target.buffer; + if (!buffer) { + throw new Error("Voiceover remux produced no output"); + } + return new Blob([buffer], { type: "video/mp4" }); + } finally { + input.dispose(); + } +} diff --git a/src/lib/windowFocus/contracts.ts b/src/lib/windowFocus/contracts.ts new file mode 100644 index 0000000000..2f9486c9d6 --- /dev/null +++ b/src/lib/windowFocus/contracts.ts @@ -0,0 +1,38 @@ +// Window-focus telemetry sidecar (`<video>.focus.json`), produced by +// `openscreen record --follow-windows` and consumed by +// `openscreen export --follow-windows`. Kept dependency-free so both the +// Electron main process and the renderer can import it. + +export interface FocusSample { + /** Milliseconds relative to the start of the recording. */ + timeMs: number; + appName: string; + windowTitle: string; + /** CGWindowID of the focused window (0 when unknown). */ + windowNumber: number; + /** Window bounds in screen points (macOS global display space). */ + x: number; + y: number; + width: number; + height: number; + /** CGDirectDisplayID of the display containing the window's center. */ + displayId: number; +} + +export interface FocusDisplayInfo { + /** Electron display id — on macOS this is the CGDirectDisplayID. */ + id: number; + bounds: { x: number; y: number; width: number; height: number }; + scaleFactor: number; + isPrimary: boolean; +} + +export interface FocusRecordingData { + version: 1; + /** Display that was recorded. */ + recordedDisplayId: number; + displays: FocusDisplayInfo[]; + samples: FocusSample[]; +} + +export const FOCUS_SIDECAR_SUFFIX = ".focus.json"; diff --git a/src/lib/windowFocus/focusToZoomRegions.test.ts b/src/lib/windowFocus/focusToZoomRegions.test.ts new file mode 100644 index 0000000000..b726d4fd69 --- /dev/null +++ b/src/lib/windowFocus/focusToZoomRegions.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; +import type { FocusRecordingData, FocusSample } from "./contracts"; +import { focusTelemetryToZoomRegions, MIN_FOCUS_DWELL_MS } from "./focusToZoomRegions"; + +const DISPLAY = { + id: 1, + bounds: { x: 0, y: 0, width: 1600, height: 1000 }, + scaleFactor: 2, + isPrimary: true, +}; + +function sample(overrides: Partial<FocusSample>): FocusSample { + return { + timeMs: 0, + appName: "App", + windowTitle: "Window", + x: 400, + y: 250, + width: 800, + height: 500, + displayId: 1, + ...overrides, + }; +} + +function data(samples: FocusSample[]): FocusRecordingData { + return { version: 1, recordedDisplayId: 1, displays: [DISPLAY], samples }; +} + +describe("focusTelemetryToZoomRegions", () => { + it("frames a focused window with a centered zoom region", () => { + const regions = focusTelemetryToZoomRegions(data([sample({ timeMs: 0 })]), { + totalMs: 10_000, + existingRegions: [], + }); + expect(regions).toHaveLength(1); + expect(regions[0].startMs).toBe(0); + expect(regions[0].endMs).toBe(10_000); + expect(regions[0].focus.cx).toBeCloseTo(0.5, 2); + expect(regions[0].focus.cy).toBeCloseTo(0.5, 2); + // 800/1600 wide with margin → scale ≈ 1/0.56 ≈ 1.79 + expect(regions[0].customScale).toBeGreaterThan(1.5); + expect(regions[0].customScale).toBeLessThan(2.1); + }); + + it("splits regions when focus moves to another window", () => { + const regions = focusTelemetryToZoomRegions( + data([ + sample({ timeMs: 0, appName: "Terminal", x: 100, y: 100, width: 700, height: 450 }), + sample({ timeMs: 4000, appName: "Browser", x: 700, y: 300, width: 800, height: 600 }), + ]), + { totalMs: 10_000, existingRegions: [] }, + ); + expect(regions).toHaveLength(2); + expect(regions[0].endMs).toBe(4000); + expect(regions[1].startMs).toBe(4000); + expect(regions[0].focus.cx).toBeLessThan(regions[1].focus.cx); + }); + + it("drops short dwells and merges brief interruptions", () => { + const regions = focusTelemetryToZoomRegions( + data([ + sample({ timeMs: 0, appName: "Main" }), + // 400 ms detour — below MIN_FOCUS_DWELL_MS, then back to Main + sample({ timeMs: 5000, appName: "Popup", x: 0, y: 0, width: 300, height: 200 }), + sample({ timeMs: 5400, appName: "Main" }), + ]), + { totalMs: 12_000, existingRegions: [] }, + ); + expect(regions).toHaveLength(1); + expect(regions[0].startMs).toBe(0); + expect(regions[0].endMs).toBe(12_000); + }); + + it("skips near-fullscreen windows entirely", () => { + const regions = focusTelemetryToZoomRegions( + data([sample({ timeMs: 0, x: 0, y: 0, width: 1580, height: 990 })]), + { totalMs: 8000, existingRegions: [] }, + ); + expect(regions).toHaveLength(0); + }); + + it("caps the zoom scale for tiny windows", () => { + const regions = focusTelemetryToZoomRegions( + data([sample({ timeMs: 0, width: 200, height: 120 })]), + { totalMs: 8000, existingRegions: [] }, + ); + expect(regions).toHaveLength(1); + expect(regions[0].customScale).toBeLessThanOrEqual(5); + }); + + it("never overlaps manually placed regions", () => { + const regions = focusTelemetryToZoomRegions(data([sample({ timeMs: 0 })]), { + totalMs: 10_000, + existingRegions: [{ startMs: 2000, endMs: 3000 }], + }); + expect(regions).toHaveLength(0); + }); + + it("ignores samples from other displays and short totals", () => { + const regions = focusTelemetryToZoomRegions(data([sample({ timeMs: 0, displayId: 99 })]), { + totalMs: 10_000, + existingRegions: [], + }); + expect(regions).toHaveLength(0); + + const shortRegions = focusTelemetryToZoomRegions(data([sample({ timeMs: 0 })]), { + totalMs: MIN_FOCUS_DWELL_MS - 100, + existingRegions: [], + }); + expect(shortRegions).toHaveLength(0); + }); +}); diff --git a/src/lib/windowFocus/focusToZoomRegions.ts b/src/lib/windowFocus/focusToZoomRegions.ts new file mode 100644 index 0000000000..6dc9dc6812 --- /dev/null +++ b/src/lib/windowFocus/focusToZoomRegions.ts @@ -0,0 +1,138 @@ +// Converts window-focus telemetry (`<video>.focus.json`) into zoom regions +// that frame whichever window the user had focused — the export renderer's +// zoom spring then pans/zooms between windows cinematically. Pure module so +// it can be unit-tested and reused by a future GUI toggle. + +import type { ZoomRegion } from "@/components/video-editor/types"; +import type { FocusRecordingData, FocusSample } from "./contracts"; + +/** Focus shorter than this is treated as "passing through" and ignored. */ +export const MIN_FOCUS_DWELL_MS = 1500; +/** Same-window intervals separated by less than this are merged. */ +export const MERGE_GAP_MS = 800; +/** Padding around the framed window, as a fraction of its size per side. */ +export const FRAME_MARGIN = 0.06; +/** Windows this close to filling the display are shown without zooming. */ +export const MIN_ZOOM_SCALE = 1.15; +/** Matches the editor's maximum zoom scale. */ +export const MAX_ZOOM_SCALE = 5; + +interface FocusInterval { + startMs: number; + endMs: number; + sample: FocusSample; +} + +function sampleKey(sample: FocusSample): string { + // Bucket bounds so window-move/resize animations don't fragment intervals. + const bucket = (value: number) => Math.round(value / 48); + return [ + sample.appName, + bucket(sample.x), + bucket(sample.y), + bucket(sample.width), + bucket(sample.height), + ].join("|"); +} + +function buildIntervals(samples: FocusSample[], totalMs: number): FocusInterval[] { + const intervals: FocusInterval[] = []; + for (const sample of samples) { + if (sample.timeMs >= totalMs) break; + const previous = intervals[intervals.length - 1]; + if (previous && sampleKey(previous.sample) === sampleKey(sample)) { + previous.endMs = totalMs; + previous.sample = sample; // keep the freshest bounds + continue; + } + if (previous) { + previous.endMs = sample.timeMs; + } + intervals.push({ startMs: sample.timeMs, endMs: totalMs, sample }); + } + + // Drop sub-dwell detours first so that "Main → brief popup → Main" leaves + // two adjacent Main intervals, then merge same-window neighbors. + const dwelled = intervals.filter( + (interval) => interval.endMs - interval.startMs >= MIN_FOCUS_DWELL_MS, + ); + const merged: FocusInterval[] = []; + for (const interval of dwelled) { + const previous = merged[merged.length - 1]; + if ( + previous && + sampleKey(previous.sample) === sampleKey(interval.sample) && + interval.startMs - previous.endMs < MERGE_GAP_MS + ) { + previous.endMs = interval.endMs; + previous.sample = interval.sample; + continue; + } + merged.push({ ...interval }); + } + return merged; +} + +export interface FocusZoomOptions { + totalMs: number; + existingRegions: { startMs: number; endMs: number }[]; +} + +/** + * Returns zoom regions framing each sufficiently-long window focus. Windows + * near display size produce no region (the camera pulls back to full frame). + * Regions overlapping existing ones are dropped so manual edits win. + */ +export function focusTelemetryToZoomRegions( + data: FocusRecordingData, + options: FocusZoomOptions, +): ZoomRegion[] { + const display = data.displays.find((candidate) => candidate.id === data.recordedDisplayId); + if (!display || display.bounds.width <= 0 || display.bounds.height <= 0) { + return []; + } + + const displaySamples = data.samples.filter( + (sample) => sample.displayId === data.recordedDisplayId, + ); + const intervals = buildIntervals(displaySamples, options.totalMs); + + const regions: ZoomRegion[] = []; + let nextId = 1; + for (const interval of intervals) { + const { sample } = interval; + const normWidth = (sample.width * (1 + 2 * FRAME_MARGIN)) / display.bounds.width; + const normHeight = (sample.height * (1 + 2 * FRAME_MARGIN)) / display.bounds.height; + if (normWidth <= 0 || normHeight <= 0) continue; + + const scale = Math.min(MAX_ZOOM_SCALE, 1 / Math.max(normWidth, normHeight)); + if (scale < MIN_ZOOM_SCALE) continue; // window ~fills the display + + const centerX = (sample.x + sample.width / 2 - display.bounds.x) / display.bounds.width; + const centerY = (sample.y + sample.height / 2 - display.bounds.y) / display.bounds.height; + + const startMs = Math.max(0, Math.round(interval.startMs)); + const endMs = Math.min(options.totalMs, Math.round(interval.endMs)); + if (endMs <= startMs) continue; + + const overlapsExisting = options.existingRegions.some( + (region) => endMs > region.startMs && startMs < region.endMs, + ); + if (overlapsExisting) continue; + + regions.push({ + id: `follow-window-${nextId++}`, + startMs, + endMs, + depth: 3, + customScale: Math.max(1, Math.min(MAX_ZOOM_SCALE, scale)), + focus: { + cx: Math.max(0, Math.min(1, centerX)), + cy: Math.max(0, Math.min(1, centerY)), + }, + focusMode: "manual", + source: "auto", + }); + } + return regions; +} diff --git a/src/lib/windowSwitch/contracts.ts b/src/lib/windowSwitch/contracts.ts new file mode 100644 index 0000000000..3abda3f409 --- /dev/null +++ b/src/lib/windowSwitch/contracts.ts @@ -0,0 +1,26 @@ +// Multi-window capture manifest (`<primary video>.multiwindow.json`), produced +// by `openscreen record --windows` and consumed by the export step's +// window-switch compositor. Every listed window was captured continuously for +// the whole recording; the focus timeline decides which one is on screen. + +import type { FocusRecordingData } from "../windowFocus/contracts"; + +export interface CapturedWindow { + /** CGWindowID — matches FocusSample.windowNumber. */ + windowId: number; + appName: string; + title: string; + videoPath: string; + /** Window bounds in screen points at recording start. */ + bounds: { x: number; y: number; width: number; height: number }; +} + +export interface MultiWindowManifest { + version: 1; + windows: CapturedWindow[]; + focus: FocusRecordingData; + /** Total capture duration reported by the recorder, in milliseconds. */ + durationMs: number; +} + +export const MULTIWINDOW_SIDECAR_SUFFIX = ".multiwindow.json"; diff --git a/src/lib/windowSwitch/switchTimeline.test.ts b/src/lib/windowSwitch/switchTimeline.test.ts new file mode 100644 index 0000000000..5de10da7f4 --- /dev/null +++ b/src/lib/windowSwitch/switchTimeline.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; +import type { MultiWindowManifest } from "./contracts"; +import { buildWindowSwitchTimeline, switchStateAt } from "./switchTimeline"; + +function manifest(samples: { timeMs: number; windowNumber: number }[]): MultiWindowManifest { + return { + version: 1, + windows: [ + { + windowId: 101, + appName: "Terminal", + title: "Terminal", + videoPath: "/r/w0.mp4", + bounds: { x: 100, y: 100, width: 800, height: 600 }, + }, + { + windowId: 202, + appName: "Browser", + title: "Browser", + videoPath: "/r/w1.mp4", + bounds: { x: 900, y: 100, width: 700, height: 600 }, + }, + ], + focus: { + version: 1, + recordedDisplayId: 1, + displays: [], + samples: samples.map((sample) => ({ + timeMs: sample.timeMs, + windowNumber: sample.windowNumber, + appName: "", + windowTitle: "", + x: 0, + y: 0, + width: 100, + height: 100, + displayId: 1, + })), + }, + durationMs: 20_000, + }; +} + +describe("buildWindowSwitchTimeline", () => { + it("switches at focus changes with geometry-based slide direction", () => { + const timeline = buildWindowSwitchTimeline( + manifest([ + { timeMs: 0, windowNumber: 101 }, + { timeMs: 8000, windowNumber: 202 }, + ]), + 20_000, + ); + expect(timeline.segments).toEqual([ + { windowIndex: 0, startMs: 0, endMs: 8000 }, + { windowIndex: 1, startMs: 8000, endMs: 20_000 }, + ]); + expect(timeline.transitions).toHaveLength(1); + // Browser sits to the right of Terminal → slides in from the right. + expect(timeline.transitions[0].direction).toBe("from-right"); + }); + + it("keeps the previous window when focus moves to an unrecorded window", () => { + const timeline = buildWindowSwitchTimeline( + manifest([ + { timeMs: 0, windowNumber: 101 }, + { timeMs: 5000, windowNumber: 999 }, + { timeMs: 9000, windowNumber: 202 }, + ]), + 20_000, + ); + expect(timeline.segments.map((segment) => segment.windowIndex)).toEqual([0, 1]); + expect(timeline.segments[1].startMs).toBe(9000); + }); + + it("ignores sub-dwell focus flickers", () => { + const timeline = buildWindowSwitchTimeline( + manifest([ + { timeMs: 0, windowNumber: 101 }, + { timeMs: 6000, windowNumber: 202 }, + { timeMs: 6400, windowNumber: 101 }, + ]), + 20_000, + ); + expect(timeline.segments).toHaveLength(1); + expect(timeline.segments[0].windowIndex).toBe(0); + }); + + it("defaults to the primary window without focus data", () => { + const timeline = buildWindowSwitchTimeline(manifest([]), 10_000); + expect(timeline.segments).toEqual([{ windowIndex: 0, startMs: 0, endMs: 10_000 }]); + expect(timeline.transitions).toHaveLength(0); + }); +}); + +describe("switchStateAt", () => { + const timeline = buildWindowSwitchTimeline( + manifest([ + { timeMs: 0, windowNumber: 101 }, + { timeMs: 8000, windowNumber: 202 }, + ]), + 20_000, + 400, + ); + + it("reports plain segments outside transitions", () => { + expect(switchStateAt(timeline, 1000)).toEqual({ activeIndex: 0 }); + expect(switchStateAt(timeline, 15_000)).toEqual({ activeIndex: 1 }); + }); + + it("reports eased progress inside the transition window", () => { + const mid = switchStateAt(timeline, 8000); + expect(mid.transition).toBeDefined(); + expect(mid.transition?.outgoingIndex).toBe(0); + expect(mid.transition?.incomingIndex).toBe(1); + expect(mid.transition?.progress).toBeCloseTo(0.5, 1); + const early = switchStateAt(timeline, 7800); + expect(early.transition?.progress ?? 1).toBeLessThan(0.5); + }); +}); diff --git a/src/lib/windowSwitch/switchTimeline.ts b/src/lib/windowSwitch/switchTimeline.ts new file mode 100644 index 0000000000..24ea6ed62a --- /dev/null +++ b/src/lib/windowSwitch/switchTimeline.ts @@ -0,0 +1,156 @@ +// Builds the window-switch timeline for multi-window captures: which recorded +// window is on screen when, and how each hand-off animates. Pure module, +// unit-tested, shared by the CLI compositor and any future GUI integration. + +import type { MultiWindowManifest } from "./contracts"; + +/** Focus shorter than this never takes over the screen. */ +export const MIN_SWITCH_DWELL_MS = 1200; +export const DEFAULT_TRANSITION_MS = 450; + +export type SlideDirection = "from-right" | "from-left"; + +export interface SwitchSegment { + windowIndex: number; + startMs: number; + endMs: number; +} + +export interface SwitchTransition { + /** Transition midpoint — the segment boundary. */ + atMs: number; + fromIndex: number; + toIndex: number; + direction: SlideDirection; + durationMs: number; +} + +export interface WindowSwitchTimeline { + segments: SwitchSegment[]; + transitions: SwitchTransition[]; +} + +/** + * Maps the focus timeline onto the recorded windows. Focus on windows that + * were not captured keeps the previous window on screen; sub-dwell focus + * flickers are ignored. The slide direction follows screen geometry: a window + * that sat to the right of the previous one slides in from the right. + */ +export function buildWindowSwitchTimeline( + manifest: MultiWindowManifest, + totalMs: number, + transitionMs: number = DEFAULT_TRANSITION_MS, +): WindowSwitchTimeline { + const windowIndexById = new Map<number, number>(); + manifest.windows.forEach((window, index) => { + windowIndexById.set(window.windowId, index); + }); + + // Focus samples → candidate switch points (recorded windows only). + const switchPoints: { timeMs: number; windowIndex: number }[] = []; + for (const sample of manifest.focus.samples) { + const windowIndex = windowIndexById.get(sample.windowNumber); + if (windowIndex === undefined) continue; + const previous = switchPoints[switchPoints.length - 1]; + if (previous && previous.windowIndex === windowIndex) continue; + switchPoints.push({ timeMs: Math.max(0, sample.timeMs), windowIndex }); + } + + // The primary window opens the video even if focus data starts late. + if (switchPoints.length === 0 || switchPoints[0].timeMs > 0) { + switchPoints.unshift({ timeMs: 0, windowIndex: switchPoints[0]?.windowIndex ?? 0 }); + } + + // Collapse flickers: a segment must hold the screen for MIN_SWITCH_DWELL_MS. + const held: { timeMs: number; windowIndex: number }[] = []; + for (const point of switchPoints) { + const previous = held[held.length - 1]; + if (!previous) { + held.push(point); + continue; + } + if (point.windowIndex === previous.windowIndex) continue; + if (point.timeMs - previous.timeMs < MIN_SWITCH_DWELL_MS) { + // Too soon after the previous switch: replace it if it was itself a + // flicker start, otherwise ignore this blip. + const beforePrevious = held[held.length - 2]; + if (beforePrevious && beforePrevious.windowIndex === point.windowIndex) { + held.pop(); + } + continue; + } + held.push(point); + } + + const segments: SwitchSegment[] = []; + for (let index = 0; index < held.length; index++) { + const startMs = index === 0 ? 0 : held[index].timeMs; + const endMs = index + 1 < held.length ? held[index + 1].timeMs : totalMs; + if (endMs <= startMs) continue; + segments.push({ windowIndex: held[index].windowIndex, startMs, endMs }); + } + if (segments.length === 0) { + segments.push({ windowIndex: 0, startMs: 0, endMs: totalMs }); + } + + const transitions: SwitchTransition[] = []; + for (let index = 1; index < segments.length; index++) { + const fromWindow = manifest.windows[segments[index - 1].windowIndex]; + const toWindow = manifest.windows[segments[index].windowIndex]; + const fromCenter = fromWindow.bounds.x + fromWindow.bounds.width / 2; + const toCenter = toWindow.bounds.x + toWindow.bounds.width / 2; + transitions.push({ + atMs: segments[index].startMs, + fromIndex: segments[index - 1].windowIndex, + toIndex: segments[index].windowIndex, + direction: toCenter >= fromCenter ? "from-right" : "from-left", + durationMs: transitionMs, + }); + } + + return { segments, transitions }; +} + +/** Per-frame render state: which windows to draw and at which x-offsets. */ +export interface SwitchFrameState { + activeIndex: number; + /** Present during a transition. */ + transition?: { + outgoingIndex: number; + incomingIndex: number; + /** 0..1 eased progress of the slide. */ + progress: number; + direction: SlideDirection; + }; +} + +function easeInOutCubic(t: number): number { + return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2; +} + +export function switchStateAt(timeline: WindowSwitchTimeline, timeMs: number): SwitchFrameState { + for (const transition of timeline.transitions) { + const start = transition.atMs - transition.durationMs / 2; + const end = transition.atMs + transition.durationMs / 2; + if (timeMs >= start && timeMs < end) { + const raw = (timeMs - start) / transition.durationMs; + return { + activeIndex: transition.toIndex, + transition: { + outgoingIndex: transition.fromIndex, + incomingIndex: transition.toIndex, + progress: easeInOutCubic(Math.max(0, Math.min(1, raw))), + direction: transition.direction, + }, + }; + } + } + + let activeIndex = timeline.segments[0]?.windowIndex ?? 0; + for (const segment of timeline.segments) { + if (timeMs >= segment.startMs) { + activeIndex = segment.windowIndex; + } + } + return { activeIndex }; +}