diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 2aa748b49..ba68df87f 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -660,22 +660,41 @@ interface Window { error?: string; }>; openAudioFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>; - openWhisperExecutablePicker: () => Promise<{ + openWhisperExecutablePicker: (options?: { + currentPath?: string | null; + selectionMode?: "file" | "directory"; + }) => Promise<{ success: boolean; path?: string; canceled?: boolean; error?: string; }>; - openWhisperModelPicker: () => Promise<{ + openWhisperModelPicker: (options?: { currentPath?: string | null }) => Promise<{ success: boolean; path?: string; canceled?: boolean; error?: string; }>; + getWhisperRuntimeStatus: (options?: { currentPath?: string | null }) => Promise<{ + success: boolean; + exists: boolean; + path?: string | null; + error?: string; + }>; + getCaptionFfmpegStatus: () => Promise<{ + success: boolean; + exists: boolean; + path?: string | null; + error?: string; + }>; + showCaptionPathInFolder: ( + path?: string | null, + ) => Promise<{ success: boolean; error?: string }>; getWhisperSmallModelStatus: () => Promise<{ success: boolean; exists: boolean; path?: string | null; + expectedPath?: string; error?: string; }>; downloadWhisperSmallModel: () => Promise<{ @@ -693,6 +712,12 @@ interface Window { error?: string; }) => void, ) => () => void; + onCaptionGenerationProgress: ( + callback: (state: { + stage: "preparing" | "extracting-audio" | "transcribing" | "finalizing"; + progress: number; + }) => void, + ) => () => void; generateAutoCaptions: (options: { videoPath: string; whisperExecutablePath?: string; diff --git a/electron/ipc/captions/generate.test.ts b/electron/ipc/captions/generate.test.ts new file mode 100644 index 000000000..d9e6b995f --- /dev/null +++ b/electron/ipc/captions/generate.test.ts @@ -0,0 +1,182 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +function getWhisperCliName() { + return process.platform === "win32" ? "whisper-cli.exe" : "whisper-cli"; +} + +async function writeExecutable(filePath: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, ""); + if (process.platform !== "win32") { + await fs.chmod(filePath, 0o755); + } +} + +describe("Whisper executable resolution", () => { + let tempRoot: string; + let appPath: string; + + beforeEach(async () => { + tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-whisper-paths-")); + appPath = path.join(tempRoot, "App"); + + vi.resetModules(); + vi.doMock("electron", () => ({ + app: { + isPackaged: false, + getAppPath: () => appPath, + getPath: () => tempRoot, + }, + })); + }); + + afterEach(async () => { + vi.resetModules(); + vi.doUnmock("electron"); + vi.doUnmock("node:child_process"); + vi.unstubAllEnvs(); + await fs.rm(tempRoot, { recursive: true, force: true }); + }); + + it("does not treat a directory as an executable", async () => { + const { isExecutableFile } = await import("./generate"); + + expect(await isExecutableFile(tempRoot)).toBe(false); + }); + + it("reports a labeled missing file clearly", async () => { + const { ensureReadableFile } = await import("./generate"); + const missingModelPath = path.join(tempRoot, "missing", "ggml-small.bin"); + + await expect( + ensureReadableFile(missingModelPath, { label: "Whisper model file" }), + ).rejects.toThrow(`Whisper model file was not found at ${missingModelPath}.`); + }); + + it("parses Whisper progress output", async () => { + const { parseWhisperProgressPercent } = await import("./generate"); + + expect( + parseWhisperProgressPercent("whisper_print_progress_callback: progress = 42%"), + ).toBe(42); + expect(parseWhisperProgressPercent("progress = 5%\nprogress = 100%")).toBe(100); + expect(parseWhisperProgressPercent("no progress here")).toBeNull(); + }); + + it("resolves WHISPER_CPP_PATH when it points to a runtime directory", async () => { + const runtimeDir = path.join(tempRoot, "whisper-runtime"); + const executablePath = path.join(runtimeDir, getWhisperCliName()); + await writeExecutable(executablePath); + vi.stubEnv("WHISPER_CPP_PATH", runtimeDir); + + const { resolveWhisperExecutablePath } = await import("./generate"); + + expect(await resolveWhisperExecutablePath()).toBe(executablePath); + }); + + it("resolves WHISPER_CPP_PATH when it points directly to an executable", async () => { + const executablePath = path.join(tempRoot, "direct", getWhisperCliName()); + await writeExecutable(executablePath); + vi.stubEnv("WHISPER_CPP_PATH", executablePath); + + const { resolveWhisperExecutablePath } = await import("./generate"); + + expect(await resolveWhisperExecutablePath()).toBe(executablePath); + }); + + it("resolves a selected runtime directory", async () => { + const runtimeDir = path.join(tempRoot, "selected-runtime"); + const executablePath = path.join(runtimeDir, getWhisperCliName()); + await writeExecutable(executablePath); + + const { resolveWhisperExecutablePath } = await import("./generate"); + + expect(await resolveWhisperExecutablePath(runtimeDir)).toBe(executablePath); + }); + + it("resolves a selected executable file", async () => { + const executablePath = path.join(tempRoot, "selected-file", getWhisperCliName()); + await writeExecutable(executablePath); + + const { resolveWhisperExecutablePath } = await import("./generate"); + + expect(await resolveWhisperExecutablePath(executablePath)).toBe(executablePath); + }); + + it("resolves WHISPER_CPP_PATH when it points to a whisper.cpp checkout", async () => { + const checkoutDir = path.join(tempRoot, "whisper.cpp"); + const executablePath = path.join( + checkoutDir, + "build", + "bin", + "Release", + getWhisperCliName(), + ); + await writeExecutable(executablePath); + vi.stubEnv("WHISPER_CPP_PATH", checkoutDir); + + const { resolveWhisperExecutablePath } = await import("./generate"); + + expect(await resolveWhisperExecutablePath()).toBe(executablePath); + }); + + it("adds companion audio sidecars as caption fallback candidates", async () => { + const videoPath = path.join(tempRoot, "recording-1.mp4"); + const micPath = path.join(tempRoot, "recording-1.mic.wav"); + const systemPath = path.join(tempRoot, "recording-1.system.wav"); + await fs.writeFile(micPath, "mic audio"); + await fs.writeFile(systemPath, "system audio"); + + const { resolveCaptionAudioCandidates } = await import("./generate"); + + expect(await resolveCaptionAudioCandidates(videoPath)).toEqual([ + { path: videoPath, label: "recording" }, + { path: systemPath, label: "source system audio" }, + { path: micPath, label: "source microphone audio" }, + ]); + }); + + it("falls back to companion audio when the recording audio cannot be extracted", async () => { + const videoPath = path.join(tempRoot, "recording-2.mp4"); + const systemPath = path.join(tempRoot, "recording-2.system.wav"); + const wavPath = path.join(tempRoot, "captions.wav"); + await fs.writeFile(videoPath, "video without extractable audio"); + await fs.writeFile(systemPath, "system audio"); + + const execFileMock = vi.fn( + ( + _: string, + args: string[], + __: unknown, + callback: (error: Error | null, stdout?: string, stderr?: string) => void, + ) => { + const inputPath = args[args.indexOf("-i") + 1]; + if (inputPath === videoPath) { + callback(new Error("Stream map '0:a:0' matches no streams.")); + return; + } + + callback(null, "", ""); + }, + ); + vi.doMock("node:child_process", () => ({ + execFile: execFileMock, + spawn: vi.fn(), + spawnSync: vi.fn(() => ({ status: 1, stdout: "" })), + })); + + const { extractCaptionAudioSource } = await import("./generate"); + + await expect( + extractCaptionAudioSource({ + videoPath, + ffmpegPath: "ffmpeg", + wavPath, + }), + ).resolves.toEqual({ path: systemPath, label: "source system audio" }); + expect(execFileMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 33c345848..62f9ac942 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -1,19 +1,154 @@ +import { execFile, spawn, spawnSync } from "node:child_process"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; -import { execFile, spawnSync } from "node:child_process"; import { promisify } from "node:util"; import { app } from "electron"; +import { COMPANION_AUDIO_LAYOUTS } from "../constants"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { getBundledWhisperExecutableCandidates } from "../paths/binaries"; -import { parseWhisperJsonCues, parseSrtCues, shouldRetryWhisperWithoutJson } from "./parser"; -import { normalizeVideoSourcePath } from "../utils"; import { resolveRecordingSession } from "../project/session"; +import { normalizeVideoSourcePath } from "../utils"; +import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser"; const execFileAsync = promisify(execFile); -export async function ensureReadableFile(filePath: string, options?: { executable?: boolean }) { - await fs.access(filePath, fsConstants.R_OK); +export type CaptionGenerationProgress = { + stage: "preparing" | "extracting-audio" | "transcribing" | "finalizing"; + progress: number; +}; + +function clampProgress(progress: number) { + return Math.min(100, Math.max(0, Math.round(progress))); +} + +function emitProgress( + onProgress: ((progress: CaptionGenerationProgress) => void) | undefined, + progress: CaptionGenerationProgress, +) { + onProgress?.({ ...progress, progress: clampProgress(progress.progress) }); +} + +export function parseWhisperProgressPercent(output: string) { + const matches = [...output.matchAll(/progress\s*=\s*(\d{1,3}(?:\.\d+)?)%/gi)]; + const match = matches[matches.length - 1]; + if (!match) { + return null; + } + + return clampProgress(Number(match[1])); +} + +function runWhisperCommand( + whisperExecutablePath: string, + args: string[], + onProgress?: (progress: CaptionGenerationProgress) => void, +) { + return new Promise((resolve, reject) => { + const child = spawn(whisperExecutablePath, args, { windowsHide: true }); + let recentOutput = ""; + const timeout = setTimeout( + () => { + child.kill(); + reject(new Error("Whisper timed out after 30 minutes.")); + }, + 30 * 60 * 1000, + ); + + const clearWhisperTimeout = () => clearTimeout(timeout); + + const handleOutput = (chunk: Buffer) => { + const output = chunk.toString("utf-8"); + recentOutput = `${recentOutput}${output}`.slice(-20_000); + const whisperProgress = parseWhisperProgressPercent(recentOutput); + if (whisperProgress !== null) { + emitProgress(onProgress, { + stage: "transcribing", + progress: 15 + whisperProgress * 0.8, + }); + } + }; + + child.stdout?.on("data", handleOutput); + child.stderr?.on("data", handleOutput); + child.on("error", (error) => { + clearWhisperTimeout(); + reject(error); + }); + child.on("close", (code) => { + clearWhisperTimeout(); + if (code === 0) { + resolve(); + return; + } + + reject(new Error(`Whisper exited with code ${code}. ${recentOutput.trim()}`.trim())); + }); + }); +} + +function getWhisperBinaryNames() { + return process.platform === "win32" + ? ["whisper-cli.exe", "whisper-cpp.exe", "whisper.exe", "main.exe"] + : ["whisper-cli", "whisper-cpp", "whisper", "main"]; +} + +export function getWhisperExecutableCandidatesFromPath(candidatePath?: string | null) { + const trimmedPath = candidatePath?.trim(); + if (!trimmedPath) { + return []; + } + + const resolvedPath = path.resolve(trimmedPath); + const directorySearchPaths = [ + [], + ["bin"], + ["bin", "Release"], + ["build", "bin"], + ["build", "bin", "Release"], + ]; + const candidates = [ + resolvedPath, + ...directorySearchPaths.flatMap((segments) => + getWhisperBinaryNames().map((binaryName) => + path.join(resolvedPath, ...segments, binaryName), + ), + ), + ]; + + return [...new Set(candidates)]; +} + +export async function ensureReadableFile( + filePath: string, + options?: { executable?: boolean; label?: string }, +) { + let stats: Awaited>; + try { + stats = await fs.stat(filePath); + } catch (error) { + if (!options?.label) { + throw error; + } + throw new Error(`${options.label} was not found at ${filePath}.`); + } + + if (!stats.isFile()) { + throw new Error( + options?.executable + ? "The selected Whisper executable is not a file." + : `${options?.label ?? "The selected file"} is not a file.`, + ); + } + + try { + await fs.access(filePath, fsConstants.R_OK); + } catch (error) { + if (!options?.label) { + throw error; + } + throw new Error(`${options.label} is not readable at ${filePath}.`); + } if (options?.executable) { try { await fs.access(filePath, fsConstants.X_OK); @@ -25,6 +160,10 @@ export async function ensureReadableFile(filePath: string, options?: { executabl export async function isExecutableFile(filePath: string) { try { + const stats = await fs.stat(filePath); + if (!stats.isFile()) { + return false; + } await fs.access(filePath, fsConstants.R_OK | fsConstants.X_OK); return true; } catch { @@ -34,9 +173,15 @@ export async function isExecutableFile(filePath: string) { export async function resolveWhisperExecutablePath(preferredPath?: string | null) { const candidatePaths = [ - preferredPath?.trim() || null, + ...getWhisperExecutableCandidatesFromPath(preferredPath), + ...getWhisperExecutableCandidatesFromPath(process.env["WHISPER_CPP_PATH"]), + ...getWhisperExecutableCandidatesFromPath( + process.platform === "win32" ? "C:\\Tools\\whisper" : null, + ), + ...getWhisperExecutableCandidatesFromPath( + process.platform === "win32" ? "C:\\whisper" : null, + ), ...getBundledWhisperExecutableCandidates(), - process.env["WHISPER_CPP_PATH"]?.trim() || null, process.platform === "darwin" ? "/opt/homebrew/bin/whisper-cli" : null, process.platform === "darwin" ? "/usr/local/bin/whisper-cli" : null, process.platform === "darwin" ? "/opt/homebrew/bin/whisper-cpp" : null, @@ -51,12 +196,8 @@ export async function resolveWhisperExecutablePath(preferredPath?: string | null } const pathCommand = process.platform === "win32" ? "where" : "which"; - const binaryNames = - process.platform === "win32" - ? ["whisper-cli.exe", "whisper.exe", "main.exe"] - : ["whisper-cli", "whisper-cpp", "whisper", "main"]; - for (const binaryName of binaryNames) { + for (const binaryName of getWhisperBinaryNames()) { const result = spawnSync(pathCommand, [binaryName], { encoding: "utf-8" }); if (result.status === 0) { const resolvedPath = result.stdout @@ -71,7 +212,7 @@ export async function resolveWhisperExecutablePath(preferredPath?: string | null } throw new Error( - "No Whisper runtime was found. Recordly looked for a bundled binary first, then checked common system install locations.", + "Whisper engine was not found. In Captions, choose the folder that contains whisper-cli, choose the whisper-cli executable, or set WHISPER_CPP_PATH to that location.", ); } @@ -94,6 +235,25 @@ export async function resolveCaptionAudioCandidates(videoPath: string) { const requestedRecordingSession = await resolveRecordingSession(videoPath); pushCandidate(requestedRecordingSession?.webcamPath, "linked webcam recording"); + const basePath = videoPath.replace(/\.[^.]+$/u, ""); + for (const layout of COMPANION_AUDIO_LAYOUTS) { + const companionPaths = [ + { path: `${basePath}${layout.systemSuffix}`, label: "source system audio" }, + { path: `${basePath}${layout.micSuffix}`, label: "source microphone audio" }, + ]; + + for (const companion of companionPaths) { + try { + const stats = await fs.stat(companion.path); + if (stats.isFile() && stats.size > 0) { + pushCandidate(companion.path, companion.label); + } + } catch { + // Companion audio is optional and only used as a fallback. + } + } + } + return candidates; } @@ -160,7 +320,9 @@ export async function generateAutoCaptionsFromVideo(options: { whisperExecutablePath?: string; whisperModelPath: string; language?: string; + onProgress?: (progress: CaptionGenerationProgress) => void; }) { + emitProgress(options.onProgress, { stage: "preparing", progress: 0 }); const ffmpegPath = getFfmpegBinaryPath(); const normalizedVideoPath = normalizeVideoSourcePath(options.videoPath); if (!normalizedVideoPath) { @@ -169,8 +331,11 @@ export async function generateAutoCaptionsFromVideo(options: { const whisperExecutablePath = await resolveWhisperExecutablePath(options.whisperExecutablePath); const whisperModelPath = path.resolve(options.whisperModelPath); - await ensureReadableFile(whisperExecutablePath, { executable: true }); - await ensureReadableFile(whisperModelPath); + await ensureReadableFile(whisperExecutablePath, { + executable: true, + label: "Whisper runtime", + }); + await ensureReadableFile(whisperModelPath, { label: "Whisper model file" }); const tempBase = path.join( app.getPath("temp"), @@ -182,11 +347,13 @@ export async function generateAutoCaptionsFromVideo(options: { const jsonPath = `${outputBase}.json`; try { + emitProgress(options.onProgress, { stage: "extracting-audio", progress: 5 }); const audioSource = await extractCaptionAudioSource({ videoPath: normalizedVideoPath, ffmpegPath, wavPath, }); + emitProgress(options.onProgress, { stage: "transcribing", progress: 15 }); const language = options.language && options.language.trim() ? options.language.trim() : "auto"; @@ -200,15 +367,16 @@ export async function generateAutoCaptionsFromVideo(options: { outputBase, "-l", language, - "-np", + "-pp", ]; let jsonEnabled = true; try { - await execFileAsync(whisperExecutablePath, [...whisperBaseArgs, "-ojf"], { - timeout: 30 * 60 * 1000, - maxBuffer: 20 * 1024 * 1024, - }); + await runWhisperCommand( + whisperExecutablePath, + [...whisperBaseArgs, "-ojf"], + options.onProgress, + ); } catch (error) { if (!shouldRetryWhisperWithoutJson(error)) { throw error; @@ -219,12 +387,11 @@ export async function generateAutoCaptionsFromVideo(options: { "[auto-captions] Whisper runtime does not support JSON full output, retrying with SRT only:", error, ); - await execFileAsync(whisperExecutablePath, whisperBaseArgs, { - timeout: 30 * 60 * 1000, - maxBuffer: 20 * 1024 * 1024, - }); + emitProgress(options.onProgress, { stage: "transcribing", progress: 15 }); + await runWhisperCommand(whisperExecutablePath, whisperBaseArgs, options.onProgress); } + emitProgress(options.onProgress, { stage: "finalizing", progress: 96 }); const timedCues = jsonEnabled ? parseWhisperJsonCues(await fs.readFile(jsonPath, "utf-8")) : []; @@ -234,6 +401,7 @@ export async function generateAutoCaptionsFromVideo(options: { throw new Error("Whisper completed, but no caption cues were produced."); } + emitProgress(options.onProgress, { stage: "finalizing", progress: 100 }); return { cues, audioSourceLabel: audioSource.label, diff --git a/electron/ipc/captions/whisper.ts b/electron/ipc/captions/whisper.ts index c8e774c62..f22f78497 100644 --- a/electron/ipc/captions/whisper.ts +++ b/electron/ipc/captions/whisper.ts @@ -1,9 +1,12 @@ -import { createWriteStream } from "node:fs"; -import { constants as fsConstants } from "node:fs"; +import { createWriteStream, constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import { get as httpsGet } from "node:https"; import type Electron from "electron"; -import { WHISPER_MODEL_DIR, WHISPER_MODEL_DOWNLOAD_URL, WHISPER_SMALL_MODEL_PATH } from "../constants"; +import { + WHISPER_MODEL_DIR, + WHISPER_MODEL_DOWNLOAD_URL, + WHISPER_SMALL_MODEL_PATH, +} from "../constants"; export function sendWhisperModelDownloadProgress( webContents: Electron.WebContents, @@ -24,12 +27,14 @@ export async function getWhisperSmallModelStatus() { success: true, exists: true, path: WHISPER_SMALL_MODEL_PATH, + expectedPath: WHISPER_SMALL_MODEL_PATH, }; } catch { return { success: true, exists: false, path: null, + expectedPath: WHISPER_SMALL_MODEL_PATH, }; } } @@ -106,7 +111,9 @@ export function downloadFileWithProgress( return request(url); } -export async function downloadWhisperSmallModel(webContents: Electron.WebContents): Promise { +export async function downloadWhisperSmallModel( + webContents: Electron.WebContents, +): Promise { await fs.mkdir(WHISPER_MODEL_DIR, { recursive: true }); const tempPath = `${WHISPER_SMALL_MODEL_PATH}.download`; diff --git a/electron/ipc/register/captions.ts b/electron/ipc/register/captions.ts index fe93afd70..be724f072 100644 --- a/electron/ipc/register/captions.ts +++ b/electron/ipc/register/captions.ts @@ -1,6 +1,11 @@ +import fs from "node:fs/promises"; import path from "node:path"; -import { dialog, ipcMain } from "electron"; -import { generateAutoCaptionsFromVideo } from "../captions/generate"; +import { dialog, ipcMain, shell } from "electron"; +import { + type CaptionGenerationProgress, + generateAutoCaptionsFromVideo, + resolveWhisperExecutablePath, +} from "../captions/generate"; import { deleteWhisperSmallModel, downloadWhisperSmallModel, @@ -8,6 +13,7 @@ import { sendWhisperModelDownloadProgress, } from "../captions/whisper"; import { LEGACY_PROJECT_FILE_EXTENSIONS, PROJECT_FILE_EXTENSION } from "../constants"; +import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { hasProjectFileExtension, loadProjectFromPath } from "../project/manager"; import { setCurrentProjectPath } from "../state"; import { approveUserPath, getRecordingsDir } from "../utils"; @@ -15,10 +21,80 @@ import { approveUserPath, getRecordingsDir } from "../utils"; const VIDEO_FILE_EXTENSIONS = ["webm", "mp4", "mov", "avi", "mkv"]; const PROJECT_FILE_EXTENSIONS = [PROJECT_FILE_EXTENSION, ...LEGACY_PROJECT_FILE_EXTENSIONS]; +function getErrorMessage(error: unknown) { + if (error instanceof Error) { + return error.message; + } + + if (typeof error === "string") { + return error.replace(/^Error:\s*/i, ""); + } + + return "Something went wrong"; +} + type OpenVideoFilePickerOptions = { includeProjects?: boolean; }; +type WhisperFilePickerOptions = { + currentPath?: string | null; + selectionMode?: "file" | "directory"; +}; + +async function resolveExistingDialogPath(candidatePath?: string | null) { + const trimmedPath = candidatePath?.trim(); + if (!trimmedPath) { + return null; + } + + try { + const stats = await fs.stat(trimmedPath); + return stats.isDirectory() ? trimmedPath : path.dirname(trimmedPath); + } catch { + const parentPath = path.dirname(trimmedPath); + try { + const stats = await fs.stat(parentPath); + return stats.isDirectory() ? parentPath : null; + } catch { + return null; + } + } +} + +async function resolveDialogDefaultPath(candidates: Array) { + for (const candidatePath of candidates) { + const dialogPath = await resolveExistingDialogPath(candidatePath); + if (dialogPath) { + return dialogPath; + } + } + + return undefined; +} + +function getWhisperRuntimeDefaultPathCandidates(currentPath?: string | null) { + return [ + currentPath, + process.env["WHISPER_CPP_PATH"], + process.platform === "win32" ? "C:\\Tools\\whisper" : null, + process.platform === "win32" ? "C:\\whisper" : null, + process.platform === "darwin" ? "/opt/homebrew/bin" : null, + process.platform === "darwin" ? "/usr/local/bin" : null, + ]; +} + +function sendCaptionGenerationProgress( + webContents: Electron.WebContents, + payload: CaptionGenerationProgress, +) { + if (webContents.isDestroyed()) { + return; + } + + webContents.send("caption-generation-progress", payload); +} + export function registerCaptionHandlers() { ipcMain.handle("open-video-file-picker", async (_, options?: OpenVideoFilePickerOptions) => { try { @@ -112,15 +188,65 @@ export function registerCaptionHandlers() { } }); - ipcMain.handle("open-whisper-executable-picker", async () => { + ipcMain.handle( + "open-whisper-executable-picker", + async (_, options?: WhisperFilePickerOptions) => { + try { + const selectionMode = options?.selectionMode ?? "directory"; + const defaultPath = await resolveDialogDefaultPath( + getWhisperRuntimeDefaultPathCandidates(options?.currentPath), + ); + const result = await dialog.showOpenDialog({ + title: + selectionMode === "file" + ? "Choose whisper-cli" + : "Choose Whisper Engine Folder", + defaultPath, + buttonLabel: + selectionMode === "file" ? "Use This Executable" : "Use This Folder", + filters: + selectionMode === "file" + ? [ + { + name: "Whisper Engine", + extensions: + process.platform === "win32" + ? ["exe", "cmd", "bat"] + : ["*"], + }, + { name: "All Files", extensions: ["*"] }, + ] + : undefined, + properties: [selectionMode === "file" ? "openFile" : "openDirectory"], + }); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + approveUserPath(result.filePaths[0]); + return { success: true, path: result.filePaths[0] }; + } catch (error) { + console.error("Failed to open Whisper executable picker:", error); + return { success: false, error: String(error) }; + } + }, + ); + + ipcMain.handle("open-whisper-model-picker", async (_, options?: WhisperFilePickerOptions) => { try { + const modelStatus = await getWhisperSmallModelStatus(); + const defaultPath = await resolveDialogDefaultPath([ + options?.currentPath, + modelStatus.path, + modelStatus.expectedPath, + ]); const result = await dialog.showOpenDialog({ - title: "Select Whisper Executable", + title: "Choose Whisper Model", + defaultPath, + buttonLabel: "Use This Model", filters: [ - { - name: "Executables", - extensions: process.platform === "win32" ? ["exe", "cmd", "bat"] : ["*"], - }, + { name: "Whisper Models", extensions: ["bin"] }, { name: "All Files", extensions: ["*"] }, ], properties: ["openFile"], @@ -133,31 +259,60 @@ export function registerCaptionHandlers() { approveUserPath(result.filePaths[0]); return { success: true, path: result.filePaths[0] }; } catch (error) { - console.error("Failed to open Whisper executable picker:", error); + console.error("Failed to open Whisper model picker:", error); return { success: false, error: String(error) }; } }); - ipcMain.handle("open-whisper-model-picker", async () => { + ipcMain.handle("get-whisper-runtime-status", async (_, options?: WhisperFilePickerOptions) => { try { - const result = await dialog.showOpenDialog({ - title: "Select Whisper Model", - filters: [ - { name: "Whisper Models", extensions: ["bin"] }, - { name: "All Files", extensions: ["*"] }, - ], - properties: ["openFile"], - }); + const runtimePath = await resolveWhisperExecutablePath(options?.currentPath); + return { success: true, exists: true, path: runtimePath }; + } catch (error) { + return { + success: true, + exists: false, + path: null, + error: getErrorMessage(error), + }; + } + }); - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true }; + ipcMain.handle("get-caption-ffmpeg-status", async () => { + try { + const ffmpegPath = getFfmpegBinaryPath(); + return { success: true, exists: true, path: ffmpegPath }; + } catch (error) { + return { + success: true, + exists: false, + path: null, + error: getErrorMessage(error), + }; + } + }); + + ipcMain.handle("show-caption-path-in-folder", async (_, targetPath?: string | null) => { + const trimmedPath = targetPath?.trim(); + if (!trimmedPath) { + return { success: false, error: "No path is selected." }; + } + + try { + const stats = await fs.stat(trimmedPath); + if (stats.isFile()) { + shell.showItemInFolder(trimmedPath); + return { success: true }; } - approveUserPath(result.filePaths[0]); - return { success: true, path: result.filePaths[0] }; + const openError = await shell.openPath(trimmedPath); + if (openError) { + return { success: false, error: openError }; + } + + return { success: true }; } catch (error) { - console.error("Failed to open Whisper model picker:", error); - return { success: false, error: String(error) }; + return { success: false, error: getErrorMessage(error) }; } }); @@ -224,7 +379,7 @@ export function registerCaptionHandlers() { ipcMain.handle( "generate-auto-captions", async ( - _, + event, options: { videoPath: string; whisperExecutablePath: string; @@ -233,7 +388,10 @@ export function registerCaptionHandlers() { }, ) => { try { - const result = await generateAutoCaptionsFromVideo(options); + const result = await generateAutoCaptionsFromVideo({ + ...options, + onProgress: (progress) => sendCaptionGenerationProgress(event.sender, progress), + }); return { success: true, cues: result.cues, @@ -244,10 +402,11 @@ export function registerCaptionHandlers() { }; } catch (error) { console.error("Failed to generate auto captions:", error); + const errorMessage = getErrorMessage(error); return { success: false, - error: String(error), - message: "Failed to generate auto captions", + error: errorMessage, + message: `Failed to generate auto captions: ${errorMessage}`, }; } }, diff --git a/electron/preload.ts b/electron/preload.ts index 9a15edb99..ad90f1ce0 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -677,11 +677,23 @@ contextBridge.exposeInMainWorld("electronAPI", { openAudioFilePicker: () => { return ipcRenderer.invoke("open-audio-file-picker"); }, - openWhisperExecutablePicker: () => { - return ipcRenderer.invoke("open-whisper-executable-picker"); + openWhisperExecutablePicker: (options?: { + currentPath?: string | null; + selectionMode?: "file" | "directory"; + }) => { + return ipcRenderer.invoke("open-whisper-executable-picker", options); + }, + openWhisperModelPicker: (options?: { currentPath?: string | null }) => { + return ipcRenderer.invoke("open-whisper-model-picker", options); + }, + getWhisperRuntimeStatus: (options?: { currentPath?: string | null }) => { + return ipcRenderer.invoke("get-whisper-runtime-status", options); + }, + getCaptionFfmpegStatus: () => { + return ipcRenderer.invoke("get-caption-ffmpeg-status"); }, - openWhisperModelPicker: () => { - return ipcRenderer.invoke("open-whisper-model-picker"); + showCaptionPathInFolder: (path?: string | null) => { + return ipcRenderer.invoke("show-caption-path-in-folder", path); }, getWhisperSmallModelStatus: () => { return ipcRenderer.invoke("get-whisper-small-model-status"); @@ -712,6 +724,22 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("whisper-small-model-download-progress", listener); return () => ipcRenderer.removeListener("whisper-small-model-download-progress", listener); }, + onCaptionGenerationProgress: ( + callback: (state: { + stage: "preparing" | "extracting-audio" | "transcribing" | "finalizing"; + progress: number; + }) => void, + ) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: { + stage: "preparing" | "extracting-audio" | "transcribing" | "finalizing"; + progress: number; + }, + ) => callback(payload); + ipcRenderer.on("caption-generation-progress", listener); + return () => ipcRenderer.removeListener("caption-generation-progress", listener); + }, generateAutoCaptions: (options: { videoPath: string; whisperExecutablePath?: string; diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index fbf390b4b..63e281299 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -45,6 +45,7 @@ import { useI18n, useScopedT } from "../../contexts/I18nContext"; import type { AppLocale } from "../../i18n/config"; import { SUPPORTED_LOCALES } from "../../i18n/config"; import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; +import { getCaptionSetupStatus } from "./captionSetupStatus"; import { CURSOR_MOTION_PRESETS, type CursorMotionPresetId, @@ -807,10 +808,15 @@ interface SettingsPanelProps { whisperModelPath?: string | null; whisperModelDownloadStatus?: "idle" | "downloading" | "downloaded" | "error"; whisperModelDownloadProgress?: number; + captionGenerationError?: string | null; + captionFfmpegPath?: string | null; + captionFfmpegError?: string | null; + captionGenerationProgress?: number | null; isGeneratingCaptions?: boolean; onAutoCaptionSettingsChange?: (settings: AutoCaptionSettings) => void; - onPickWhisperExecutable?: () => void; + onPickWhisperExecutable?: (selectionMode?: "file" | "directory") => void; onPickWhisperModel?: () => void; + onShowCaptionPathInFolder?: (path?: string | null) => void; onGenerateAutoCaptions?: () => void; onClearAutoCaptions?: () => void; onDownloadWhisperSmallModel?: () => void; @@ -819,6 +825,15 @@ interface SettingsPanelProps { onOpenNativeCaptureUnavailableModal?: () => void; } +function getPathDisplayName(filePath?: string | null) { + const trimmedPath = filePath?.trim(); + if (!trimmedPath) { + return ""; + } + + return trimmedPath.split(/[\\/]/).filter(Boolean).pop() ?? trimmedPath; +} + const ZOOM_DEPTH_OPTIONS: Array<{ depth: ZoomDepth; label: string }> = [ { depth: 1, label: "1.25×" }, { depth: 2, label: "1.5×" }, @@ -1239,12 +1254,19 @@ export function SettingsPanel({ onAnnotationDelete, autoCaptions = [], autoCaptionSettings = DEFAULT_AUTO_CAPTION_SETTINGS, + whisperExecutablePath, whisperModelPath, whisperModelDownloadStatus = "idle", whisperModelDownloadProgress = 0, + captionGenerationError = null, + captionFfmpegPath, + captionFfmpegError, + captionGenerationProgress = null, isGeneratingCaptions = false, onAutoCaptionSettingsChange, + onPickWhisperExecutable, onPickWhisperModel, + onShowCaptionPathInFolder, onGenerateAutoCaptions, onClearAutoCaptions, onDownloadWhisperSmallModel, @@ -1283,6 +1305,167 @@ export function SettingsPanel({ [extensionWallpapers], ); const captionCueCount = autoCaptions.length; + const roundedCaptionGenerationProgress = + typeof captionGenerationProgress === "number" + ? Math.min(100, Math.max(0, Math.round(captionGenerationProgress))) + : null; + const captionSetupStatus = getCaptionSetupStatus({ + captionCueCount, + captionsEnabled: autoCaptionSettings.enabled, + whisperModelPath, + whisperExecutablePath, + captionFfmpegPath, + captionFfmpegError, + captionGenerationError, + isGeneratingCaptions, + }); + const { + isCaptionFfmpegChecking, + isMissingAudioError: captionErrorIsMissingAudio, + isMissingEngineError: captionErrorIsMissingEngine, + isMissingFfmpegError: captionErrorIsMissingFfmpeg, + } = captionSetupStatus; + const whisperModelDisplayName = whisperModelPath + ? getPathDisplayName(whisperModelPath) + : tSettings("captions.noModelSelected", "No model selected"); + const whisperEngineDisplayName = whisperExecutablePath + ? getPathDisplayName(whisperExecutablePath) + : tSettings("captions.noEngineSelected", "No engine selected"); + const whisperModelHelpText = whisperModelPath + ? whisperModelPath + : tSettings("captions.modelHelp", "Download the small model or choose a .bin model file."); + const whisperEngineHelpText = whisperExecutablePath + ? whisperExecutablePath + : tSettings( + "captions.engineHelp", + "Choose the folder that contains whisper-cli, such as C:\\Tools\\whisper on Windows or /opt/homebrew/bin on macOS.", + ); + const captionFfmpegDisplayName = captionFfmpegPath + ? getPathDisplayName(captionFfmpegPath) + : isCaptionFfmpegChecking + ? tSettings("captions.ffmpegChecking", "Checking FFmpeg") + : tSettings("captions.ffmpegMissing", "FFmpeg not found"); + const captionFfmpegHelpText = captionFfmpegPath + ? captionFfmpegPath + : isCaptionFfmpegChecking + ? tSettings( + "captions.ffmpegCheckingHelp", + "Looking for Recordly's bundled FFmpeg or an ffmpeg command on PATH.", + ) + : captionFfmpegError || + tSettings( + "captions.ffmpegHelp", + "Recordly needs FFmpeg to extract audio before Whisper can transcribe it.", + ); + const captionStatusTitle = + captionSetupStatus.id === "generating" + ? tSettings("captions.statusGenerating", "Generating captions") + : captionSetupStatus.id === "missing-audio" + ? tSettings("captions.noAudioTitle", "No audio to transcribe") + : captionSetupStatus.id === "missing-model" + ? tSettings("captions.modelNeedsSetup", "Choose a caption model") + : captionSetupStatus.id === "missing-engine" + ? tSettings("captions.engineNeedsSetup", "Choose Whisper engine") + : captionSetupStatus.id === "checking-ffmpeg" + ? tSettings("captions.ffmpegChecking", "Checking FFmpeg") + : captionSetupStatus.id === "missing-ffmpeg" + ? tSettings("captions.ffmpegNeedsSetup", "FFmpeg is unavailable") + : captionSetupStatus.id === "captions-hidden" + ? tSettings( + "captions.statusHiddenWithCaptions", + "Captions are hidden", + ) + : captionSetupStatus.id === "ready-with-captions" + ? tSettings( + "captions.statusReadyWithCaptions", + "Captions are ready", + ) + : captionSetupStatus.id === "error" + ? tSettings( + "captions.needsSetup", + "Captions need setup", + ) + : tSettings( + "captions.statusReadyToGenerate", + "Ready to generate", + ); + const captionStatusText = + captionSetupStatus.id === "generating" + ? roundedCaptionGenerationProgress !== null + ? `${tSettings( + "captions.generatingStatus", + "Generating captions. This can take a moment.", + )} ${roundedCaptionGenerationProgress}%` + : tSettings( + "captions.generatingStatus", + "Generating captions. This can take a moment.", + ) + : captionSetupStatus.id === "missing-audio" + ? tSettings( + "captions.noAudioHelp", + "This video does not contain an audio track Recordly can transcribe. Open a video with audio, or record again with microphone or system audio enabled.", + ) + : captionSetupStatus.id === "missing-model" + ? tSettings( + "captions.guideModelDescription", + "Download the small model, or choose an existing .bin model file.", + ) + : captionSetupStatus.id === "missing-engine" + ? tSettings( + "captions.guideEngineDescription", + "Choose the folder that contains whisper-cli. On Windows, C:\\Tools\\whisper is a good place to keep it.", + ) + : captionSetupStatus.id === "checking-ffmpeg" || + captionSetupStatus.id === "missing-ffmpeg" + ? captionFfmpegHelpText + : captionSetupStatus.id === "captions-hidden" + ? tSettings( + "captions.guideHiddenDescription", + "Generated captions are currently hidden. Turn on Show to display them in the preview and export.", + ) + : captionSetupStatus.id === "ready-with-captions" + ? tSettings( + "captions.guideReadyDescription", + "Generated captions are visible in the preview. Press Play to review timing before export.", + ) + : captionSetupStatus.id === "error" + ? (captionGenerationError ?? "") + : tSettings( + "captions.guideGenerateDescription", + "Model, engine, and audio extraction are ready.", + ); + const captionStatusToneClassName = + captionSetupStatus.tone === "error" + ? "border-red-500/20 bg-red-500/5" + : captionSetupStatus.tone === "ready" + ? "border-[#2563EB]/15 bg-[#2563EB]/5" + : "border-amber-500/20 bg-amber-500/5"; + const captionStatusDotClassName = + captionSetupStatus.tone === "error" + ? "bg-red-500" + : captionSetupStatus.tone === "ready" + ? "bg-[#2563EB]" + : "bg-amber-500"; + const captionGenerationHelpTitle = captionErrorIsMissingEngine + ? tSettings("captions.engineNeedsSetup", "Whisper engine needs setup") + : captionErrorIsMissingAudio + ? tSettings("captions.noAudioTitle", "No audio to transcribe") + : captionErrorIsMissingFfmpeg + ? tSettings("captions.ffmpegNeedsSetup", "FFmpeg is unavailable") + : tSettings("captions.needsSetup", "Captions need setup"); + const captionGenerationHelpText = captionErrorIsMissingEngine + ? tSettings( + "captions.engineErrorHelp", + "Whisper engine was not found. Choose the folder that contains whisper-cli, such as C:\\Tools\\whisper on Windows or /opt/homebrew/bin on macOS.", + ) + : captionErrorIsMissingAudio + ? tSettings( + "captions.noAudioHelp", + "This video does not contain an audio track Recordly can transcribe. Open a video with audio, or record again with microphone or system audio enabled.", + ) + : captionErrorIsMissingFfmpeg + ? captionFfmpegHelpText + : captionGenerationError; const updateAutoCaptionSettings = (partial: Partial) => { onAutoCaptionSettingsChange?.({ ...autoCaptionSettings, @@ -2607,15 +2790,167 @@ export function SettingsPanel({
-
- +
+
+ +
+ {captionStatusTitle} +
+
+
+ {captionStatusText} +
+
+ +
+
+
+
+
+ {tSettings("captions.modelLabel", "Model")} +
+
+ {whisperModelDisplayName} +
+
+
+ {whisperModelPath ? ( + + ) : ( + + )} + +
+
+
+ {whisperModelHelpText} +
+
+ +
+
+
+
+ {tSettings("captions.engineLabel", "Whisper engine")} +
+
+ {whisperEngineDisplayName} +
+
+
+ {whisperExecutablePath ? ( + + ) : null} + + +
+
+
+ {whisperEngineHelpText} +
+
+ +
+
+
+
+ {tSettings("captions.audioExtractorLabel", "Audio extractor")} +
+
+ {captionFfmpegDisplayName} +
+
+ {captionFfmpegPath ? ( + + ) : null} +
+
+ {captionFfmpegHelpText} +
+
@@ -2639,16 +2974,7 @@ export function SettingsPanel({
- {whisperModelDownloadStatus === "downloading" ? ( - - ) : whisperModelPath ? ( + {whisperModelPath ? ( )}
+ {captionGenerationHelpText ? ( +
+
+ {captionGenerationHelpTitle} +
+
+ {captionGenerationHelpText} +
+ {captionErrorIsMissingEngine || + (!whisperModelPath && !captionErrorIsMissingAudio) ? ( +
+ {captionErrorIsMissingEngine ? ( + + ) : null} + {!whisperModelPath && !captionErrorIsMissingAudio ? ( + + ) : null} +
+ ) : null} +
+ ) : null}