diff --git a/.gitignore b/.gitignore index 4b0047e..79a7ae3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ AGENTS.md # Misc *.spec .signing/ +__pycache__/ +*.pyc diff --git a/Tome/Package.swift b/Tome/Package.swift index 7ba9b86..210315e 100644 --- a/Tome/Package.swift +++ b/Tome/Package.swift @@ -43,12 +43,20 @@ let package = Package( ], path: "Sources/VoiceprintAudit" ), + // JSONL manifest parse/emit shared by ASRBench (manifest mode) and its + // tests. Plain library, no ASR deps — kept separate so TomeTests can + // depend on it without pulling in FluidAudio/WhisperKit. + .target( + name: "BenchSupport", + path: "Sources/BenchSupport" + ), // ASR load-test harness comparing Parakeet vs Whisper latency (see // docs/superpowers/specs/2026-07-08-*.md §8). Not part of the app; // never run in CI (downloads GBs of models, needs ANE). .executableTarget( name: "ASRBench", dependencies: [ + "BenchSupport", .product(name: "FluidAudio", package: "FluidAudio"), .product(name: "WhisperKit", package: "argmax-oss-swift"), ], @@ -60,7 +68,7 @@ let package = Package( // Nothing here touches audio devices, permissions, or the ASR models. .testTarget( name: "TomeTests", - dependencies: ["Tome"], + dependencies: ["Tome", "BenchSupport"], path: "Tests/TomeTests" ), ] diff --git a/Tome/Sources/ASRBench/main.swift b/Tome/Sources/ASRBench/main.swift index 64e368b..4a46591 100644 --- a/Tome/Sources/ASRBench/main.swift +++ b/Tome/Sources/ASRBench/main.swift @@ -7,6 +7,7 @@ // Usage: swift run -c release ASRBench [--json out.json] import AVFoundation +import BenchSupport import Foundation import FluidAudio import WhisperKit @@ -25,6 +26,36 @@ let sampleRate = 16_000.0 let maxChunkSamples = 480_000 let minChunkSamples = 8_000 +// Manifest mode: ASRBench --manifest in.jsonl --backend parakeet|whisper --out hyp.jsonl +// Reuses the same hand-mirrored model config as the bench functions below (keep in sync). +if let mi = CommandLine.arguments.firstIndex(of: "--manifest") { + let args = CommandLine.arguments + guard args.count > mi + 1, + let bi = args.firstIndex(of: "--backend"), args.count > bi + 1, + let oi = args.firstIndex(of: "--out"), args.count > oi + 1 else { + FileHandle.standardError.write(Data("usage: ASRBench --manifest in.jsonl --backend parakeet|whisper --out hyp.jsonl\n".utf8)) + exit(2) + } + let entries = try BenchManifest.parse(String(contentsOfFile: args[mi + 1], encoding: .utf8)) + let backend = args[bi + 1] + // Extract the model-loading half of benchParakeet()/benchWhisper() into + // loadParakeet() / loadWhisper() helpers returning a `(String) async throws -> String` + // transcribe closure (WAV path in, text out), reusing the existing sample-loading + // code these bench functions already use for their own WAVs. + let transcribe: (String) async throws -> String = backend == "whisper" + ? try await loadWhisper() + : try await loadParakeet() + var hyps: [HypothesisEntry] = [] + for (i, e) in entries.enumerated() { + let text = (try? await transcribe(e.wav)) ?? "" + hyps.append(HypothesisEntry(id: e.id, text: text)) + if i % 50 == 0 { print("[\(backend)] \(i)/\(entries.count)") } + } + try BenchManifest.emit(hyps).write(toFile: args[oi + 1], atomically: true, encoding: .utf8) + print("[\(backend)] wrote \(hyps.count) hypotheses → \(args[oi + 1])") + exit(0) +} + var argv = Array(CommandLine.arguments.dropFirst()) var jsonOut: String? if let i = argv.firstIndex(of: "--json"), i + 1 < argv.count { @@ -80,7 +111,15 @@ func peakRSSMB() -> Double { func now() -> Double { CFAbsoluteTimeGetCurrent() } // --- Parakeet --- -func benchParakeet(chunks: [[Float]]) async throws -> BackendReport { + +// Model-loading half of benchParakeet(): downloads (if needed), does a cold +// load then a warm load (mirroring the bench's cold/warm timing dance), and +// hands back the warm-loaded manager plus the report fields benchParakeet() +// needs (download/load timings, cache dir). Also used directly by manifest +// mode via the `transcribe` closure it derives its own wrapper from. +func loadParakeetManager() async throws -> ( + asr: AsrManager, downloadSeconds: Double?, dir: URL, loadCold: Double, loadWarm: Double +) { let cached = AsrModels.modelsExist( at: AsrModels.defaultCacheDirectory(for: .v3), version: .v3) let tDownload = now() @@ -99,6 +138,26 @@ func benchParakeet(chunks: [[Float]]) async throws -> BackendReport { try await asr.loadModels(warmModels) let loadWarm = now() - tWarm + return (asr, downloadSeconds, dir, loadCold, loadWarm) +} + +// Manifest-mode helper: loads Parakeet and returns a (WAV path in, text out) +// transcribe closure, reusing loadParakeetManager()'s load path and the same +// AudioProcessor.loadAudioAsFloatArray sample-loading the top-level bench +// driver uses for its own files. +func loadParakeet() async throws -> (String) async throws -> String { + let (asr, _, _, _, _) = try await loadParakeetManager() + return { path in + let samples = try AudioProcessor.loadAudioAsFloatArray(fromPath: path) + var state = TdtDecoderState.make() + let result = try await asr.transcribe(samples, decoderState: &state, language: .english) + return result.text + } +} + +func benchParakeet(chunks: [[Float]]) async throws -> BackendReport { + let (asr, downloadSeconds, dir, loadCold, loadWarm) = try await loadParakeetManager() + func transcribe(_ samples: [Float]) async throws -> Double { var state = TdtDecoderState.make() let t = now() @@ -121,7 +180,14 @@ func benchParakeet(chunks: [[Float]]) async throws -> BackendReport { } // --- Whisper --- -func benchWhisper(chunks: [[Float]], variant: String, base: URL) async throws -> BackendReport { + +// Model-loading half of benchWhisper(): resolves/downloads the model folder, +// does a cold load then a warm load (mirroring the bench's cold/warm timing +// dance), and hands back the warm-loaded kit plus the report fields +// benchWhisper() needs (download timing, folder, variant). +func loadWhisperKit(variant: String, base: URL) async throws -> ( + kit: WhisperKit, downloadSeconds: Double?, folder: URL, loadCold: Double, loadWarm: Double +) { let expectedFolder = base.appendingPathComponent( "models/argmaxinc/whisperkit-coreml/\(variant)", isDirectory: true) let cached = FileManager.default.fileExists( @@ -151,6 +217,28 @@ func benchWhisper(chunks: [[Float]], variant: String, base: URL) async throws -> let kit = try await WhisperKit(config) let loadWarm = now() - tWarm + return (kit, downloadSeconds, folder, loadCold, loadWarm) +} + +// Manifest-mode helper: loads Whisper (default family/base, same as the top- +// level bench driver uses) and returns a (WAV path in, text out) transcribe +// closure, reusing loadWhisperKit()'s load path and the existing +// AudioProcessor.loadAudioAsFloatArray sample-loading code. +func loadWhisper() async throws -> (String) async throws -> String { + let (kit, _, _, _, _) = try await loadWhisperKit(variant: whisperVariant, base: whisperBase) + return { path in + let samples = try AudioProcessor.loadAudioAsFloatArray(fromPath: path) + let results = try await kit.transcribe( + audioArray: samples, + decodeOptions: DecodingOptions(task: .transcribe, language: "en")) + return results.map(\.text).joined(separator: " ") + .trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +func benchWhisper(chunks: [[Float]], variant: String, base: URL) async throws -> BackendReport { + let (kit, downloadSeconds, folder, loadCold, loadWarm) = try await loadWhisperKit(variant: variant, base: base) + func transcribe(_ samples: [Float]) async throws -> Double { let t = now() _ = try await kit.transcribe( diff --git a/Tome/Sources/BenchSupport/BenchManifest.swift b/Tome/Sources/BenchSupport/BenchManifest.swift new file mode 100644 index 0000000..d0b487a --- /dev/null +++ b/Tome/Sources/BenchSupport/BenchManifest.swift @@ -0,0 +1,27 @@ +import Foundation + +public struct ManifestEntry: Codable, Equatable, Sendable { + public let id: String + public let wav: String + public init(id: String, wav: String) { self.id = id; self.wav = wav } +} + +public struct HypothesisEntry: Codable, Equatable, Sendable { + public let id: String + public let text: String + public init(id: String, text: String) { self.id = id; self.text = text } +} + +public enum BenchManifest { + public static func parse(_ jsonl: String) throws -> [ManifestEntry] { + try jsonl.split(separator: "\n", omittingEmptySubsequences: true) + .filter { !$0.trimmingCharacters(in: .whitespaces).isEmpty } + .map { try JSONDecoder().decode(ManifestEntry.self, from: Data($0.utf8)) } + } + public static func emit(_ hyps: [HypothesisEntry]) -> String { + let enc = JSONEncoder() + enc.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + return hyps.map { String(data: try! enc.encode($0), encoding: .utf8)! } + .joined(separator: "\n") + (hyps.isEmpty ? "" : "\n") + } +} diff --git a/Tome/Sources/Tome/App/TomeApp.swift b/Tome/Sources/Tome/App/TomeApp.swift index 286f1ce..d830f8c 100644 --- a/Tome/Sources/Tome/App/TomeApp.swift +++ b/Tome/Sources/Tome/App/TomeApp.swift @@ -245,6 +245,11 @@ final class AppDelegate: NSObject, NSApplicationDelegate { let response = alert.runModal() if response == .alertSecondButtonReturn { + // A shadow-phase sidecar may still be running in the queue's + // in-flight job; nothing outside Transcription/ holds a reference + // to it, so this process-global kill is the only way to + // guarantee it doesn't outlive Tome. + SidecarRegistry.killAll() queue.shutdown() return .terminateNow } @@ -256,6 +261,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate { while queue.isAnyJobRunning && Date() < deadline { try? await Task.sleep(for: .milliseconds(250)) } + // Same backstop as the "Quit Anyway" branch above: the 60s cap + // may expire with a shadow sidecar still live in the job that's + // about to be torn down by shutdown(). + SidecarRegistry.killAll() queue.shutdown() NSApp.reply(toApplicationShouldTerminate: true) } diff --git a/Tome/Sources/Tome/Transcription/AudioWAVExport.swift b/Tome/Sources/Tome/Transcription/AudioWAVExport.swift new file mode 100644 index 0000000..6d25c6a --- /dev/null +++ b/Tome/Sources/Tome/Transcription/AudioWAVExport.swift @@ -0,0 +1,70 @@ +import AVFoundation + +/// Converts arbitrary PCM buffers to the 16 kHz mono PCM16 WAV bytes the +/// granite sidecar consumes (granite_request.md pins format: "wav"). +enum AudioWAVExport { + enum ExportError: Error { case formatUnavailable, conversionFailed } + + static func wav16kMonoPCM16(from buffer: AVAudioPCMBuffer) throws -> Data { + guard let outFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16000, + channels: 1, interleaved: true), + let converter = AVAudioConverter(from: buffer.format, to: outFmt) + else { throw ExportError.formatUnavailable } + let ratio = 16000.0 / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 4096 + guard let out = AVAudioPCMBuffer(pcmFormat: outFmt, frameCapacity: capacity) + else { throw ExportError.conversionFailed } + var fed = false + var totalFrames: AVAudioFrameCount = 0 + var pcmData = Data() + + // Drain loop: keep converting until converter exhausts input/output or + // reports error. Each iteration's produced bytes are appended to + // `pcmData` immediately, before `out` is reset for the next iteration — + // correct regardless of whether convert() ever splits its output across + // multiple .haveData chunks (a single iteration is the common case + // here, since `capacity` is sized to fit the whole conversion up + // front, but nothing below depends on that). The previous version + // copied bytes only once, after the loop, from whatever `out` held on + // the FINAL iteration — correct only by the single-iteration + // assumption; a real multi-chunk conversion would have discarded the + // earlier chunks' samples while still counting their frames. + while true { + var drainError: NSError? + let status = converter.convert(to: out, error: &drainError) { _, status in + if fed { status.pointee = .endOfStream; return nil } + fed = true; status.pointee = .haveData; return buffer + } + if let drainError { throw drainError } + if status == .error { throw ExportError.conversionFailed } + if out.frameLength > 0 { + pcmData.append(Data(bytes: out.int16ChannelData![0], count: Int(out.frameLength) * 2)) + totalFrames += out.frameLength + } + if status == .endOfStream || out.frameLength == 0 { break } + out.frameLength = 0 // Reset for next iteration + } + + // Sanity check: output length should be ~expected; silent truncation is an error + let expected = Double(buffer.frameLength) * 16000.0 / buffer.format.sampleRate + if Double(totalFrames) < expected - 4096 { + throw ExportError.conversionFailed + } + + var data = riffHeader(dataByteCount: pcmData.count) + data.append(pcmData) + return data + } + + static func riffHeader(dataByteCount: Int) -> Data { + var d = Data() + func le32(_ v: UInt32) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } } + func le16(_ v: UInt16) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } } + d.append(contentsOf: "RIFF".utf8); le32(UInt32(36 + dataByteCount)) + d.append(contentsOf: "WAVE".utf8) + d.append(contentsOf: "fmt ".utf8); le32(16); le16(1) /* PCM */; le16(1) /* mono */ + le32(16000); le32(16000 * 2) /* byte rate */; le16(2) /* block align */; le16(16) + d.append(contentsOf: "data".utf8); le32(UInt32(dataByteCount)) + return d + } +} diff --git a/Tome/Sources/Tome/Transcription/CurlModelFetcher.swift b/Tome/Sources/Tome/Transcription/CurlModelFetcher.swift new file mode 100644 index 0000000..d7cb5d4 --- /dev/null +++ b/Tome/Sources/Tome/Transcription/CurlModelFetcher.swift @@ -0,0 +1,309 @@ +import Foundation + +/// Curl-based fallback fetcher for HuggingFace model files. +/// +/// Why this exists: on some networks `URLSession` (and Python `urllib`) cannot +/// connect to the HF Xet CDN at any timeout, while `/usr/bin/curl` connects in +/// ~30ms — a client-stack issue, NOT the SDK's 10s timeout (documented in +/// docs/superpowers/plans/2026-07-08-benchmark-results.md "Known issue"). On +/// those networks `WhisperKit.download` always fails, so Whisper could never be +/// installed in-app. This shells out to curl instead. +/// +/// Shelling out is permitted because Tome is NOT sandboxed — `Tome.entitlements` +/// declares only `com.apple.security.device.audio-input` and +/// `com.apple.security.device.screen-capture`, with no +/// `com.apple.security.app-sandbox` key — so `Foundation.Process` may exec curl. +/// +/// The on-disk layout it produces mirrors HubApi (and `WhisperBackend`'s path +/// helpers) exactly: every file lands at `destRoot/models//`, so +/// `WhisperBackend.modelFolder(variant:)` / `.tokenizerJSON` resolve to the same +/// paths the SDK download would have written. +enum CurlModelFetcher { + + enum FetchError: Error, LocalizedError { + case listFailed(repo: String, path: String, detail: String) + case parseFailed(String) + case emptyListing(repo: String, path: String) + case downloadFailed(url: String, detail: String) + + var errorDescription: String? { + switch self { + case .listFailed(let repo, let path, let detail): + return "curl could not list \(repo)/\(path): \(detail)" + case .parseFailed(let detail): + return "could not parse HF tree listing: \(detail)" + case .emptyListing(let repo, let path): + return "HF tree listing for \(repo)/\(path) contained no files" + case .downloadFailed(let url, let detail): + return "curl failed to download \(url): \(detail)" + } + } + } + + /// One file entry from the HF tree listing: its repo-relative path and + /// (when the tree API reports it) its expected byte size. `size` is nil + /// for the explicit tokenizer file list (`fetchFiles`), which has no tree + /// listing to draw a size from. + struct RemoteFile: Equatable { + let path: String + let size: Int? + } + + // MARK: - Pure URL / path / progress helpers (unit-tested, no network) + + /// The HF tree-API URL that lists every file under a repo path. + static func treeURL(repo: String, path: String) -> URL { + URL(string: "https://huggingface.co/api/models/\(repo)/tree/main/\(path)?recursive=true")! + } + + /// The HF `resolve` URL that serves a single file's bytes. + static func resolveURL(repo: String, filePath: String) -> URL { + URL(string: "https://huggingface.co/\(repo)/resolve/main/\(filePath)")! + } + + /// Local destination for a downloaded file. Mirrors HubApi / + /// `WhisperBackend.modelFolder`: `destRoot/models//`. + static func destination(destRoot: URL, repo: String, filePath: String) -> URL { + destRoot.appendingPathComponent("models/\(repo)/\(filePath)") + } + + /// Parse the HF `tree` JSON (an array of `{type, path, size, ...}` objects) + /// into the list of files, keeping only `type == "file"` entries. Split out + /// so the parse is unit-testable from a fixture string with no network. + static func fileList(fromTreeJSON data: Data) throws -> [RemoteFile] { + let root: Any + do { + root = try JSONSerialization.jsonObject(with: data) + } catch { + throw FetchError.parseFailed(error.localizedDescription) + } + guard let entries = root as? [[String: Any]] else { + throw FetchError.parseFailed("expected a top-level array of objects") + } + return entries.compactMap { entry in + guard (entry["type"] as? String) == "file", + let path = entry["path"] as? String + else { return nil } + return RemoteFile(path: path, size: entry["size"] as? Int) + } + } + + /// Progress as completed/total, clamped to 0…1. Total == 0 reads as complete. + static func progressFraction(completed: Int, total: Int) -> Double { + guard total > 0 else { return 1 } + return min(max(Double(completed) / Double(total), 0), 1) + } + + /// Whether an already-present destination file can stand in for a fresh + /// download. True only when BOTH sizes are known and equal — an unknown + /// existing size (no file yet) or unknown expected size (no tree-listing + /// size, e.g. the explicit tokenizer fetch) always redownloads. Pure so a + /// fixture test can cover it with no filesystem or network access. + /// + /// This replaces `-C -` resume: `-C -` on a `--fail` request against an + /// already-complete file gets HTTP 416 from HF and the whole fetch fails. + /// Comparing sizes up front avoids the 416 entirely and gives resume at + /// file granularity (skip whole files that already match). + static func shouldSkipDownload(existingFileSize: Int?, expectedSize: Int?) -> Bool { + guard let existingFileSize, let expectedSize else { return false } + return existingFileSize == expectedSize + } + + // MARK: - Fetch entry points + + /// List every file under `repo`/`path` via the HF tree API (fetched WITH + /// curl too — the urllib-class clients that hang on the CDN also hang on + /// these API calls) and download each into `destRoot`. Progress is + /// completedFiles/totalFiles as 0…1. + static func fetchVariant( + repo: String, + path: String, + into destRoot: URL, + onProgress: @Sendable (Double) -> Void + ) async throws { + let treeData: Data + do { + treeData = try await runCurl( + args: dataArgs(url: treeURL(repo: repo, path: path)), + context: "list \(repo)/\(path)" + ) + } catch { + throw FetchError.listFailed(repo: repo, path: path, detail: error.localizedDescription) + } + let files = try fileList(fromTreeJSON: treeData) + guard !files.isEmpty else { throw FetchError.emptyListing(repo: repo, path: path) } + try await download(repo: repo, files: files, into: destRoot, onProgress: onProgress) + } + + /// Download an EXPLICIT set of files from a repo (used for the tokenizer, + /// where listing the whole repo would pull the multi-GB PyTorch weights we + /// don't need). No tree listing means no known size, so these always + /// redownload rather than skip — acceptable since the tokenizer files are + /// tiny. Progress is completedFiles/totalFiles as 0…1. + static func fetchFiles( + repo: String, + files: [String], + into destRoot: URL, + onProgress: @Sendable (Double) -> Void + ) async throws { + let remoteFiles = files.map { RemoteFile(path: $0, size: nil) } + try await download(repo: repo, files: remoteFiles, into: destRoot, onProgress: onProgress) + } + + // MARK: - Internals + + private static func download( + repo: String, + files: [RemoteFile], + into destRoot: URL, + onProgress: @Sendable (Double) -> Void + ) async throws { + onProgress(0) + var completed = 0 + for file in files { + // A cancelled provisioning cycle must not keep pulling gigabytes + // file-by-file just because no single curl invocation is in flight + // at the moment the cancellation lands. + try Task.checkCancellation() + + let dest = destination(destRoot: destRoot, repo: repo, filePath: file.path) + try FileManager.default.createDirectory( + at: dest.deletingLastPathComponent(), withIntermediateDirectories: true) + + if shouldSkipDownload(existingFileSize: fileSize(at: dest), expectedSize: file.size) { + completed += 1 + onProgress(progressFraction(completed: completed, total: files.count)) + continue + } + // Not a size match — clear any partial file from a prior + // interrupted run before downloading fresh (no `-C -`): resuming a + // partial that's actually already-complete-but-unequal, or corrupt, + // would otherwise mix stale and fresh bytes in one file. + try? FileManager.default.removeItem(at: dest) + + let url = resolveURL(repo: repo, filePath: file.path) + do { + _ = try await runCurl( + args: downloadArgs(url: url, dest: dest), + context: "download \(file.path)" + ) + } catch is CancellationError { + throw CancellationError() + } catch { + throw FetchError.downloadFailed(url: url.absoluteString, detail: error.localizedDescription) + } + completed += 1 + onProgress(progressFraction(completed: completed, total: files.count)) + } + } + + private static func fileSize(at url: URL) -> Int? { + guard let attrs = try? FileManager.default.attributesOfItem(atPath: url.path) else { return nil } + return attrs[.size] as? Int + } + + /// curl args to fetch a URL's bytes to stdout. + private static func dataArgs(url: URL) -> [String] { + ["-sSL", "--fail", "--retry", "3", url.absoluteString] + } + + /// curl args to download a URL to a file. No `-C -`: the size check above + /// already decides skip-vs-redownload at file granularity, and `-C -` + /// against an already-complete file gets `--fail`ed with HTTP 416. + private static func downloadArgs(url: URL, dest: URL) -> [String] { + ["-sSL", "--fail", "--retry", "3", "-o", dest.path, url.absoluteString] + } + + /// Bridges `Task` cancellation into the blocking curl child. The + /// `withTaskCancellationHandler` `onCancel` closure can fire immediately + /// (synchronously, before the background block even runs) or concurrently + /// from any thread once the child is running, so `process`/`cancelled` are + /// guarded behind a lock: `register` reports back if cancellation already + /// happened so the caller terminates the just-created process itself. + private final class CancellableProcessBox: @unchecked Sendable { + private let lock = NSLock() + private var process: Process? + private(set) var isCancelled = false + + /// Returns false if cancellation already arrived — the caller is then + /// responsible for terminating `process` itself, since `cancel()` has + /// nothing to terminate yet. + func register(_ process: Process) -> Bool { + lock.lock() + defer { lock.unlock() } + if isCancelled { return false } + self.process = process + return true + } + + /// `Process.terminate()` is documented safe to call from any thread. + func cancel() { + lock.lock() + isCancelled = true + let p = process + lock.unlock() + p?.terminate() + } + } + + /// Run `/usr/bin/curl` off the main actor and return its stdout. `Process` is + /// blocking, so it runs on a background queue and resumes a continuation; + /// throws if curl exits non-zero (stderr is surfaced in the message). + /// + /// Cancellation: wrapped in `withTaskCancellationHandler` so a cancelled + /// `Task` terminates the curl child instead of leaving it downloading with + /// the await pending forever. The continuation is resumed exactly once + /// either way — normal exit/non-zero-exit resumes inline after + /// `waitUntilExit()` returns; a cancel-triggered `terminate()` unblocks + /// that same `waitUntilExit()` (no separate `terminationHandler` needed, + /// so there's no second resume path to reason about) and the box's + /// `isCancelled` flag turns that same resume into `CancellationError`. + private static func runCurl(args: [String], context: String) async throws -> Data { + let box = CancellableProcessBox() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + DispatchQueue.global(qos: .utility).async { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/curl") + process.arguments = args + let outPipe = Pipe() + let errPipe = Pipe() + process.standardOutput = outPipe + process.standardError = errPipe + do { + try process.run() + } catch { + continuation.resume(throwing: error) + return + } + if !box.register(process) { + // onCancel already fired before we could register — + // terminate right away instead of downloading further. + process.terminate() + } + // Drain stdout before waiting to avoid a full-pipe deadlock on + // large bodies; with -sS, stderr stays tiny (errors only). + let outData = outPipe.fileHandleForReading.readDataToEndOfFile() + let errData = errPipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + if box.isCancelled { + continuation.resume(throwing: CancellationError()) + return + } + if process.terminationStatus != 0 { + let stderr = String(data: errData, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let detail = stderr.isEmpty + ? "\(context): curl exited \(process.terminationStatus)" + : "\(context): \(stderr)" + continuation.resume(throwing: FetchError.downloadFailed(url: context, detail: detail)) + return + } + continuation.resume(returning: outData) + } + } + } onCancel: { + box.cancel() + } + } +} diff --git a/Tome/Sources/Tome/Transcription/GraniteRequest.swift b/Tome/Sources/Tome/Transcription/GraniteRequest.swift new file mode 100644 index 0000000..4bc25c3 --- /dev/null +++ b/Tome/Sources/Tome/Transcription/GraniteRequest.swift @@ -0,0 +1,38 @@ +import Foundation + +/// Builds/parses granite llama-server requests. The contract is pinned in +/// scripts/asr-bench/granite_request.md — Phase 0 validated it; change both +/// together or not at all. +/// +/// Note: The client-side request timeout used by callers is 600 s. See the +/// granite sidecar implementation for the constant definition. +enum GraniteRequest { + static let prompt = "can you transcribe the speech into a written format?" + static let endpointPath = "/v1/chat/completions" + + enum ParseError: Error { case unexpectedShape } + + static func build(wavData: Data) -> Data { + let body: [String: Any] = [ + "messages": [[ + "role": "user", + "content": [ + ["type": "input_audio", + "input_audio": ["data": wavData.base64EncodedString(), "format": "wav"]], + ["type": "text", "text": prompt], + ], + ]], + "temperature": 0, "max_tokens": 2048, "stream": false, + ] + return try! JSONSerialization.data(withJSONObject: body) + } + + static func parseResponse(_ data: Data) throws -> String { + guard let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let choices = obj["choices"] as? [[String: Any]], + let message = choices.first?["message"] as? [String: Any], + let content = message["content"] as? String + else { throw ParseError.unexpectedShape } + return content.trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/Tome/Sources/Tome/Transcription/GraniteShadow.swift b/Tome/Sources/Tome/Transcription/GraniteShadow.swift new file mode 100644 index 0000000..71f69d0 --- /dev/null +++ b/Tome/Sources/Tome/Transcription/GraniteShadow.swift @@ -0,0 +1,265 @@ +import AVFoundation +import Foundation + +protocol SegmentTranscribing: Sendable { + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String +} + +/// Bridges the sidecar into the per-segment loop: buffer → 16 kHz WAV → HTTP. +struct GraniteSidecarTranscriber: SegmentTranscribing { + let sidecar: GraniteSidecar + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String { + try await sidecar.transcribe(wavData: try AudioWAVExport.wav16kMonoPCM16(from: buffer)) + } +} + +struct ShadowSegment: Codable, Sendable, Equatable { + let startTime: Float + let speaker: String + let durationSec: Double + let text: String? + let error: String? + let latencySec: Double +} + +struct ShadowRunOutput: Sendable { + let segments: [ShadowSegment] + let incomplete: Bool +} + +/// Runs granite over the SAME merged segments the primary path used +/// (SegmentAudio guarantees identical audio — spec §4). Unlike +/// SegmentReTranscriber, errors are recorded per segment, not placeholdered. +struct ShadowRunner: Sendable { + let transcriber: any SegmentTranscribing + + func run(fileURL: URL, diarSegments: [DiarizedSegment], speakerNumberBase: Int) async -> ShadowRunOutput { + let audioFile: AVAudioFile + do { audioFile = try AVAudioFile(forReading: fileURL) } catch { + diagLog("[SHADOW] cannot open \(fileURL.lastPathComponent): \(error)") + return ShadowRunOutput(segments: [], incomplete: true) + } + let sampleRate = audioFile.processingFormat.sampleRate + let totalFrames = AVAudioFramePosition(audioFile.length) + let merged = SegmentAudio.merge(diarSegments) + // merge() preserves first-occurrence order of distinct speaker IDs, so + // labels match SegmentReTranscriber's raw-array map. + let speakerMap = speakerLabels(from: merged.map(\.speakerId), startingAt: speakerNumberBase) + var results: [ShadowSegment] = [] + var incomplete = false + let clock = ContinuousClock() + for seg in merged { + if Task.isCancelled { incomplete = true; break } + let speaker = speakerMap[seg.speakerId] ?? "Speaker \(speakerNumberBase)" + let duration = Double(seg.endTime - seg.startTime) + guard let range = SegmentAudio.paddedFrameRange( + startTime: seg.startTime, endTime: seg.endTime, + sampleRate: sampleRate, totalFrames: totalFrames) + else { + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: nil, + error: "segment read failed", latencySec: 0)) + continue + } + // SegmentAudio.readSegment throws on file-read failure and returns + // nil on buffer-allocation failure — both are non-fatal here, and + // both continue to the next segment (correction vs the brief's + // single-nil check, since the signature now throws). + let buffer: AVAudioPCMBuffer + do { + guard let b = try SegmentAudio.readSegment(file: audioFile, start: range.start, count: range.count) else { + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: nil, + error: "segment allocation failed", latencySec: 0)) + continue + } + buffer = b + } catch { + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: nil, + error: "segment read failed: \(error)", latencySec: 0)) + continue + } + let t0 = clock.now + do { + let text = try await transcriber.transcribe(buffer: buffer) + .trimmingCharacters(in: .whitespacesAndNewlines) + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: text, error: nil, + latencySec: secondsSince(t0, clock: clock))) + } catch { + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: nil, + error: String(describing: error), + latencySec: secondsSince(t0, clock: clock))) + if let sidecarError = error as? GraniteSidecar.SidecarError { + // .notReady = sidecar dead/stopped; .requestFailed = + // relaunch budget exhausted, process torn down. Either way + // the sidecar is gone — stop burning segments. (Exhaustive + // switch so a future recoverable case must pick its policy + // here at compile time.) + switch sidecarError { + case .notReady, .requestFailed: + incomplete = true + } + break + } + } + } + return ShadowRunOutput(segments: results, incomplete: incomplete) + } + + /// Duration → seconds. `Double(truncating: duration / .seconds(1) as NSNumber)` + /// doesn't compile for `Duration`; decompose into seconds + attoseconds instead. + private func secondsSince(_ t0: ContinuousClock.Instant, clock: ContinuousClock) -> Double { + let d = clock.now - t0 + return Double(d.components.seconds) + Double(d.components.attoseconds) * 1e-18 + } +} + +struct ShadowSessionInfo: Codable, Sendable { + let sessionID: String + let transcriptPath: String + let sessionType: String + let primaryModel: String + let graniteModel: String +} + +struct ShadowComparisonSegment: Codable, Sendable { + let startTime: Float + let speaker: String + let durationSec: Double + let primaryText: String + let graniteText: String + let graniteError: String? + let graniteLatencySec: Double +} + +struct ShadowComparisonTotals: Codable, Sendable { + let segmentCount: Int + let erroredCount: Int + let audioSeconds: Double + let shadowWallClockSec: Double + let rtf: Double +} + +struct ShadowComparison: Codable, Sendable { + let session: ShadowSessionInfo + let incomplete: Bool + let segments: [ShadowComparisonSegment] + let totals: ShadowComparisonTotals +} + +enum ShadowArtifacts { + static func defaultDirectory() -> URL { + FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("Tome/GraniteShadow") + } + + static func write(session: ShadowSessionInfo, primary: [ReTranscribedSegment], + shadow: ShadowRunOutput, to dir: URL) throws -> (md: URL, json: URL) { + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + // Pair by merged-segment startTime + speaker (spec §4: primary skips + // empty-text segments, so array positions don't line up; "" marks a + // missing side). Both sides derive startTime from the same + // SegmentAudio.merge(...) output (Float, same source segments), but + // startTime ALONE isn't unique: overlapping-speaker diarization can + // produce two merged segments with an identical startTime for + // different speakers, and keying on startTime alone would collide + // (uniquingKeysWith silently drops one primary text). A same-speaker + // same-startTime pair can't survive merge(), so startTime+speaker is + // unique on both sides. + func pairKey(startTime: Float, speaker: String) -> String { "\(startTime)|\(speaker)" } + let primaryByKey = Dictionary(primary.map { (pairKey(startTime: $0.startTime, speaker: $0.speaker), $0.text) }, + uniquingKeysWith: { a, _ in a }) + let segments = shadow.segments.map { s in + ShadowComparisonSegment(startTime: s.startTime, speaker: s.speaker, + durationSec: s.durationSec, + primaryText: primaryByKey[pairKey(startTime: s.startTime, speaker: s.speaker)] ?? "", + graniteText: s.text ?? "", + graniteError: s.error, + graniteLatencySec: s.latencySec) + } + let audioSeconds = shadow.segments.reduce(0) { $0 + $1.durationSec } + let wall = shadow.segments.reduce(0) { $0 + $1.latencySec } + let comparison = ShadowComparison( + session: session, incomplete: shadow.incomplete, segments: segments, + totals: ShadowComparisonTotals(segmentCount: segments.count, + erroredCount: shadow.segments.filter { $0.error != nil }.count, + audioSeconds: audioSeconds, + shadowWallClockSec: wall, + rtf: audioSeconds > 0 ? wall / audioSeconds : 0)) + let jsonURL = dir.appendingPathComponent("\(session.sessionID).comparison.json") + let enc = JSONEncoder() + enc.outputFormatting = [.prettyPrinted, .sortedKeys] + try enc.encode(comparison).write(to: jsonURL, options: .atomic) + + var md = """ + # Granite shadow transcript — \(session.sessionID) + - Primary model: \(session.primaryModel) + - Shadow model: \(session.graniteModel) + - Segments: \(segments.count) (\(comparison.totals.erroredCount) errored)\(shadow.incomplete ? " — INCOMPLETE" : "") + - Shadow RTF: \(String(format: "%.3f", comparison.totals.rtf)) + + """ + for s in shadow.segments where s.text?.isEmpty == false { + md += "\(s.speaker): \(s.text!)\n\n" + } + let mdURL = dir.appendingPathComponent("\(session.sessionID).granite.md") + try md.write(to: mdURL, atomically: true, encoding: .utf8) + return (mdURL, jsonURL) + } +} + +/// Best-effort orchestration — the ONLY entry point PostProcessingJob calls. +/// Never throws; every failure is a diagLog + recorded artifact state. +enum GraniteShadowPhase { + static func shouldRun(config: ShadowConfig?, didRebuild: Bool, + primary: [ReTranscribedSegment]?) -> Bool { + // Flag off is the common case — stay silent (spec: only log skips + // when the flag is actually on). + guard config != nil else { return false } + guard didRebuild, let primary, !primary.isEmpty else { + diagLog("[SHADOW] flag on but skipping shadow phase this session (didRebuild=\(didRebuild), primarySegments=\(primary?.count ?? 0))") + return false + } + return true + } + + static func run(config: ShadowConfig, bufferURL: URL, diarSegments: [DiarizedSegment], + speakerNumberBase: Int, primary: [ReTranscribedSegment], + session: ShadowSessionInfo, + outputDir: URL = ShadowArtifacts.defaultDirectory(), + sidecar: GraniteSidecar? = nil) async { + guard FileManager.default.isExecutableFile(atPath: config.serverPath) else { + diagLog("[SHADOW] llama-server missing at \(config.serverPath) — skipping (run scripts/setup-granite-shadow.sh)") + return + } + guard config.filesPresent() else { + diagLog("[SHADOW] model files missing in \(config.modelDir.path) — skipping (run scripts/setup-granite-shadow.sh)") + return + } + // Spawn-per-phase: a fresh sidecar instance for this run. Never call + // start() again on an instance that has already been stop()ed. + let sc = sidecar ?? GraniteSidecar(config: config) + diagLog("[SHADOW] starting sidecar for \(session.sessionID) (\(diarSegments.count) diar segments)") + guard await sc.start() else { + diagLog("[SHADOW] sidecar failed to start — skipping session \(session.sessionID)") + return + } + // From here on the sidecar process is live: every exit path — normal + // completion, early return, or the run() call itself throwing (it + // doesn't, but artifact writing below can) — must stop it first so no + // llama-server outlives the phase. + let output = await ShadowRunner(transcriber: GraniteSidecarTranscriber(sidecar: sc)) + .run(fileURL: bufferURL, diarSegments: diarSegments, speakerNumberBase: speakerNumberBase) + await sc.stop() + do { + let (md, json) = try ShadowArtifacts.write(session: session, primary: primary, + shadow: output, to: outputDir) + diagLog("[SHADOW] wrote \(md.lastPathComponent) + \(json.lastPathComponent) (\(output.segments.count) segments, incomplete=\(output.incomplete))") + } catch { + diagLog("[SHADOW] artifact write failed (non-fatal): \(error)") + } + } +} diff --git a/Tome/Sources/Tome/Transcription/GraniteSidecar.swift b/Tome/Sources/Tome/Transcription/GraniteSidecar.swift new file mode 100644 index 0000000..58f6460 --- /dev/null +++ b/Tome/Sources/Tome/Transcription/GraniteSidecar.swift @@ -0,0 +1,339 @@ +import Foundation + +protocol SidecarProcess: Sendable { + var isRunning: Bool { get } + func terminate() + func forceKill() +} + +protocol SidecarProcessLauncher: Sendable { + func launch(executable: URL, arguments: [String]) throws -> any SidecarProcess +} + +protocol SidecarHTTP: Sendable { + func healthStatus(_ url: URL) async -> Int? + /// Returns the response body alongside its HTTP status so callers can + /// distinguish a 2xx-with-unparseable-body (ParseError, no relaunch) from + /// a 4xx/5xx (connection-class failure, relaunch path) — see + /// GraniteSidecar.transcribe. + func post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> (Data, Int) +} + +/// Owns one llama-server child process, spawn-per-job (spec §3): ~4 GB of +/// model RAM stays off the machine between jobs. One relaunch on connection +/// failure; a second failure fails the phase. +/// +/// Plain actor (not @MainActor) — Task 10's shadow phase drives this from +/// PostProcessingJob, which is @MainActor-bound; calls into this actor hop +/// off the main actor via the usual actor-to-actor await. +actor GraniteSidecar { + enum State: Equatable { case idle, ready, failed } + enum SidecarError: Error { case notReady, requestFailed } + + /// A non-2xx llama-server response. Thrown from transcribe()'s do-block + /// so the outer catch treats it exactly like a dropped connection (the + /// relaunch path) — the body here is opaque error text, not an + /// unparseable-but-2xx transcript shape, so it must NOT be confused with + /// GraniteRequest.ParseError. + private struct HTTPStatusError: Error, CustomStringConvertible { + let status: Int + var description: String { "HTTP \(status)" } + } + + private let config: ShadowConfig + private let launcher: any SidecarProcessLauncher + private let http: any SidecarHTTP + private let readyTimeout: TimeInterval + private let sleep: @Sendable (TimeInterval) async -> Void + /// App-quit gate, checked at BOTH spawn points (start(), and the relaunch + /// branch in transcribe()). SidecarRegistry.killAll() blocks the main + /// thread during quit, but this actor keeps running on its own executor — + /// without the gate, an in-flight transcribe() whose connection drops + /// while its server is being killed takes the relaunch branch (state + /// still .ready, Task not cancelled, didRelaunch false) and spawns + + /// registers a FRESH child after killAll's victim snapshot, reproducing + /// the very orphan the registry exists to prevent. Injectable so tests + /// stay deterministic and isolated from the permanent process-global flag + /// (which other tests in the same process may flip via killAll()). + private let isQuitting: @Sendable () -> Bool + private var process: (any SidecarProcess)? + private var didRelaunch = false + /// Monotonic teardown generation (pattern: ASRCoordinator.lastInstallToken). + /// stop() bumps it; an in-flight start() or relaunch that captured an older + /// value is stale and must tear down rather than surface a running process — + /// every await in those paths is an interleaving opportunity for stop(). + private var generation = 0 + private(set) var state: State = .idle + + init(config: ShadowConfig, + launcher: any SidecarProcessLauncher = DefaultProcessLauncher(), + http: any SidecarHTTP = URLSessionSidecarHTTP(), + readyTimeout: TimeInterval = 60, + sleep: @Sendable @escaping (TimeInterval) async -> Void = { try? await Task.sleep(for: .seconds($0)) }, + isQuitting: @Sendable @escaping () -> Bool = { SidecarRegistry.isQuitting }) { + self.config = config + self.launcher = launcher + self.http = http + self.readyTimeout = readyTimeout + self.sleep = sleep + self.isQuitting = isQuitting + } + + @discardableResult + func start() async -> Bool { + guard !isQuitting() else { + diagLog("[SHADOW] app quitting — refusing sidecar launch") + state = .failed + return false + } + let gen = generation + // Guard against a second start() while a process is already tracked + // (ready or mid-poll) — without this, the prior handle would be + // silently overwritten below and its llama-server orphaned. + if process != nil { + await endProcess() + // stop() may have landed during endProcess's grace suspension — + // honor it: don't launch a replacement the caller just tore down. + guard generation == gen else { + state = .idle + return false + } + } + let healthURL = config.baseURL.appendingPathComponent("health") + // Pre-launch probe: if something is already answering /health on our + // port before we've launched anything, it's a foreign/orphaned + // llama-server (e.g. leaked by a prior crash) — refuse to adopt it. + // Launching on top would either fail to bind or silently hand our + // requests to a process we don't control and can't clean up. + let probeStatus = await http.healthStatus(healthURL) + // The probe await is an interleaving opportunity for stop() — + // re-check the generation BEFORE acting on the probe result, in + // either direction: a stale 200 must not stomp .failed over the + // .idle a concurrent stop() just established. + guard generation == gen else { + state = .idle + return false + } + if probeStatus == 200 { + diagLog("[SHADOW] port \(config.port) already serving /health — refusing to adopt foreign llama-server (kill it or change graniteShadowPort)") + state = .failed + return false + } + do { + process = try launcher.launch( + executable: URL(fileURLWithPath: config.serverPath), + arguments: ["-m", config.modelGGUF.path, + "--mmproj", config.mmprojGGUF.path, + "--host", "127.0.0.1", + "--port", String(config.port), + // 16k context: granite supports it, and Q8 KV at + // this size is fine on 64 GB (verified in + // Phase 0). Note mtmd still internally chunks + // audio >30s into 30s windows regardless of + // context size — the resulting boundary-quality + // caveat is tracked in the shadow-week results + // doc, not addressed here. + "-c", "16384", + "--no-webui"]) + } catch { + diagLog("[SHADOW] sidecar launch failed: \(error)") + state = .failed + return false + } + let iterations = Int(readyTimeout / 0.5) + for _ in 0.. String { + guard state == .ready else { throw SidecarError.notReady } + let url = config.baseURL.appendingPathComponent( + GraniteRequest.endpointPath.trimmingCharacters(in: CharacterSet(charactersIn: "/"))) + let body = GraniteRequest.build(wavData: wavData) + do { + let (data, status) = try await http.post(url, body: body, timeout: 600) + guard (200..<300).contains(status) else { + // llama-server responded but with an error status — the body + // is opaque error text, not the expected transcript shape, so + // this is a connection-class failure (relaunch/give-up path + // below), not a ParseError. + throw HTTPStatusError(status: status) + } + return try GraniteRequest.parseResponse(data) + } catch let error as GraniteRequest.ParseError { + // A parse error means the server responded (200) but with an + // unexpected shape — retrying won't fix that, so it propagates + // as-is without relaunching or touching sidecar state. + throw error + } catch { + // stop() may have run while this call was suspended in http.post + // (the actor serializes these, but the await point is a + // interleaving opportunity). If so, honor the stop — don't + // resurrect a process the caller already asked to tear down. + guard state == .ready else { throw SidecarError.notReady } + if Task.isCancelled { + // Don't burn the one relaunch budget respawning a sidecar for + // a caller that's already gone — let the cancellation unwind. + throw error + } + if isQuitting() { + // App is exiting; the connection likely dropped BECAUSE + // SidecarRegistry.killAll() just terminated our server. Same + // treatment as cancellation: rethrow without relaunching — + // spawning a fresh child now would orphan it past killAll's + // victim snapshot. + throw error + } + // Any other thrown error from http.post (including the + // HTTPStatusError above) is treated as a connection-level failure + // and triggers the single relaunch path. + guard !didRelaunch else { + diagLog("[SHADOW] request failed after relaunch — failing sidecar: \(error)") + await endProcess() + state = .failed + throw SidecarError.requestFailed + } + diagLog("[SHADOW] request failed (\(error)) — relaunching sidecar once") + didRelaunch = true + // The endProcess/start awaits below are interleaving opportunities + // for stop(): capture the teardown generation and re-validate after + // each, so a stop() landing mid-relaunch is honored instead of the + // relaunch resurrecting a fresh process after stop() returned. + let gen = generation + await endProcess() + guard generation == gen else { + state = .idle + throw SidecarError.notReady + } + let started = await start() + guard generation == gen else { + await endProcess() + state = .idle + throw SidecarError.notReady + } + guard started else { throw SidecarError.requestFailed } + // Recurse so a failure on the retried attempt is handled by the + // same didRelaunch-guarded catch above (fails the sidecar and + // marks .failed instead of leaking a third http.post call). + return try await transcribe(wavData: wavData) + } + } + + func stop() async { + // Bump first: invalidates any in-flight start()/relaunch that captured + // an older generation. Rest state before the endProcess suspension so a + // reentrant transcribe() sees not-ready instead of posting to a + // process that is mid-teardown (and then relaunching it). + generation += 1 + state = .idle + await endProcess() + } + + /// Terminate then escalate to SIGKILL after a bounded grace period, using + /// the injected `sleep` so tests stay deterministic (fakes' `sleep` is a + /// no-op and FakeProcess drops `isRunning` immediately on terminate(), + /// so this returns after the first check in tests). + private func endProcess() async { + guard let p = process else { return } + p.terminate() + if p.isRunning { + for _ in 0..<5 { + await sleep(1) + if !p.isRunning { break } + } + } + if p.isRunning { p.forceKill() } + process = nil + } +} + +// MARK: - Real implementations + +struct DefaultProcessLauncher: SidecarProcessLauncher { + func launch(executable: URL, arguments: [String]) throws -> any SidecarProcess { + let p = Process() + p.executableURL = executable + p.arguments = arguments + p.standardOutput = FileHandle.nullDevice + p.standardError = FileHandle.nullDevice + try p.run() + // Registered process-globally (not just held by this GraniteSidecar + // instance) so the app's quit path can kill it even if the sidecar + // that spawned it has already gone out of scope — see SidecarRegistry. + SidecarRegistry.register(pid: p.processIdentifier) + return RealSidecarProcess(process: p) + } +} + +/// Wraps Process; forceKill sends SIGKILL. A leaked llama-server must not +/// outlive Tome: Process children die with the parent only if killed, so +/// terminationHandler is not enough — the shadow phase's defer + this +/// wrapper's deinit both call terminate. +final class RealSidecarProcess: SidecarProcess, @unchecked Sendable { + private let process: Process + init(process: Process) { self.process = process } + var isRunning: Bool { process.isRunning } + func terminate() { + if process.isRunning { process.terminate() } + SidecarRegistry.unregister(pid: process.processIdentifier) + } + func forceKill() { + if process.isRunning { kill(process.processIdentifier, SIGKILL) } + SidecarRegistry.unregister(pid: process.processIdentifier) + } + deinit { + if process.isRunning { process.terminate() } + SidecarRegistry.unregister(pid: process.processIdentifier) + } +} + +struct URLSessionSidecarHTTP: SidecarHTTP { + // localhost URLSession is fine here — the known URLSession/HF-CDN stall + // issue (see ModelProvisioner/downloads) is remote-CDN-specific; this + // talks only to 127.0.0.1, never leaves the loopback interface. + func healthStatus(_ url: URL) async -> Int? { + var req = URLRequest(url: url) + req.timeoutInterval = 2 + guard let (_, resp) = try? await URLSession.shared.data(for: req) else { return nil } + return (resp as? HTTPURLResponse)?.statusCode + } + func post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> (Data, Int) { + var req = URLRequest(url: url) + req.httpMethod = "POST" + req.httpBody = body + req.timeoutInterval = timeout + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + let (data, resp) = try await URLSession.shared.data(for: req) + return (data, (resp as? HTTPURLResponse)?.statusCode ?? 0) + } +} diff --git a/Tome/Sources/Tome/Transcription/PostProcessingJob.swift b/Tome/Sources/Tome/Transcription/PostProcessingJob.swift index 5849c72..54427a9 100644 --- a/Tome/Sources/Tome/Transcription/PostProcessingJob.swift +++ b/Tome/Sources/Tome/Transcription/PostProcessingJob.swift @@ -38,13 +38,19 @@ final class PostProcessingJob: Identifiable { /// the finalized transcript. Call captures only — needs a diarized system stream. let exportVoiceprints: Bool - init(handle: SessionHandle, clusterThreshold: Float, numberOfSpeakers: Int, retention: RecordingRetentionConfig? = nil, exportVoiceprints: Bool = false) { + /// Hidden-flag shadow-transcription config, read from UserDefaults at job + /// creation (the default-parameter expression evaluates when `init` runs, + /// which IS the spec's "read at job creation" semantics). Nil = flag off. + let shadowConfig: ShadowConfig? + + init(handle: SessionHandle, clusterThreshold: Float, numberOfSpeakers: Int, retention: RecordingRetentionConfig? = nil, exportVoiceprints: Bool = false, shadowConfig: ShadowConfig? = ShadowConfig.fromDefaults()) { self.id = handle.id self.handle = handle self.clusterThreshold = clusterThreshold self.numberOfSpeakers = numberOfSpeakers self.retention = retention self.exportVoiceprints = exportVoiceprints + self.shadowConfig = shadowConfig } /// Run the full pipeline. The main-actor boundary between steps is where @@ -90,6 +96,7 @@ final class PostProcessingJob: Identifiable { var diarOutput: DiarizationOutput? var didRebuildSpeakers = false + var primaryResults: [ReTranscribedSegment]? = nil if let bufferURL = diarBufferURL { let fileSize = (try? FileManager.default.attributesOfItem(atPath: bufferURL.path)[.size] as? Int) ?? -1 diagLog("[JOB \(id)] buffer file size=\(fileSize) bytes, exists=\(FileManager.default.fileExists(atPath: bufferURL.path))") @@ -134,6 +141,7 @@ final class PostProcessingJob: Identifiable { segments: segments, speakerNumberBase: speakerBase ) + primaryResults = results if Task.isCancelled { // As above: keep the capture files so the orphan scan can recover. @@ -221,6 +229,23 @@ final class PostProcessingJob: Identifiable { } } + // 2c. Granite shadow transcription (hidden flag; spec 2026-07-09). + // Best-effort and additive: runs while the capture WAVs still exist, + // never throws, never touches the primary transcript or cleanup. + if GraniteShadowPhase.shouldRun(config: shadowConfig, didRebuild: didRebuildSpeakers, + primary: primaryResults), + let bufferURL = diarBufferURL, let diar = diarOutput { + await GraniteShadowPhase.run( + config: shadowConfig!, bufferURL: bufferURL, diarSegments: diar.segments, + speakerNumberBase: speakerBase, primary: primaryResults!, + session: ShadowSessionInfo( + sessionID: id, + transcriptPath: savedPath.path, + sessionType: String(describing: handle.sessionType), + primaryModel: await asr.activeModel?.displayName ?? "unknown", + graniteModel: ShadowConfig.modelFilename)) + } + // 3. Retain the combined recording before deleting the source WAVs. The // transcript is already saved, so a retention failure doesn't fail the // job — but it MUST block cleanup: the user explicitly asked to keep this diff --git a/Tome/Sources/Tome/Transcription/SegmentAudio.swift b/Tome/Sources/Tome/Transcription/SegmentAudio.swift new file mode 100644 index 0000000..dc49c49 --- /dev/null +++ b/Tome/Sources/Tome/Transcription/SegmentAudio.swift @@ -0,0 +1,55 @@ +import AVFoundation + +/// Segment mechanics shared by the primary re-transcriber and the granite +/// shadow runner — both must see byte-identical audio (spec §4). +enum SegmentAudio { + /// Merge consecutive same-speaker segments separated by < gapThreshold seconds. + static func merge(_ segments: [DiarizedSegment], gapThreshold: Float = 0.5) -> [DiarizedSegment] { + var merged: [DiarizedSegment] = [] + for seg in segments { + if let last = merged.last, last.speakerId == seg.speakerId, + seg.startTime - last.endTime < gapThreshold { + merged[merged.count - 1] = DiarizedSegment( + speakerId: last.speakerId, startTime: last.startTime, endTime: seg.endTime) + } else { + merged.append(seg) + } + } + return merged + } + + /// Frame range for a segment, padded to minSeconds (Parakeet's floor — + /// applied to all backends deliberately; see spec §4) and clamped to the file. + static func paddedFrameRange( + startTime: Float, endTime: Float, sampleRate: Double, + totalFrames: AVAudioFramePosition, minSeconds: Double = 1.5 + ) -> (start: AVAudioFramePosition, count: AVAudioFrameCount)? { + var startFrame = AVAudioFramePosition(Double(startTime) * sampleRate) + var endFrame = min(AVAudioFramePosition(Double(endTime) * sampleRate), totalFrames) + var frameCount = Int(endFrame - startFrame) + let minSamples = Int(sampleRate * minSeconds) + if frameCount < minSamples && frameCount > 0 { + let deficit = minSamples - frameCount + let padBefore = min(AVAudioFramePosition(deficit / 2), startFrame) + let padAfter = min(deficit - Int(padBefore), Int(totalFrames - endFrame)) + startFrame -= padBefore + endFrame += AVAudioFramePosition(padAfter) + frameCount = Int(endFrame - startFrame) + } + guard frameCount > 0 else { return nil } + return (startFrame, AVAudioFrameCount(frameCount)) + } + + /// Read one segment's PCM out of an open file. + /// nil = allocation failure, caller skips; throws = read failure, caller decides + /// visibility (the primary re-transcriber surfaces a "[transcription failed]" + /// placeholder; a shadow runner may handle it differently). + static func readSegment(file: AVAudioFile, start: AVAudioFramePosition, + count: AVAudioFrameCount) throws -> AVAudioPCMBuffer? { + file.framePosition = start + guard let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: count) + else { return nil } + try file.read(into: buffer, frameCount: count) + return buffer + } +} diff --git a/Tome/Sources/Tome/Transcription/SegmentReTranscriber.swift b/Tome/Sources/Tome/Transcription/SegmentReTranscriber.swift index f24ce76..edc49cc 100644 --- a/Tome/Sources/Tome/Transcription/SegmentReTranscriber.swift +++ b/Tome/Sources/Tome/Transcription/SegmentReTranscriber.swift @@ -17,51 +17,25 @@ struct SegmentReTranscriber: Sendable { do { let audioFile = try AVAudioFile(forReading: fileURL) let sampleRate = audioFile.processingFormat.sampleRate - let totalFrames = AVAudioFrameCount(audioFile.length) + let totalFrames = AVAudioFramePosition(audioFile.length) let speakerMap = speakerLabels(from: segments.map(\.speakerId), startingAt: speakerNumberBase) // Merge consecutive segments from the same speaker (< 0.5s gap) - var merged: [DiarizedSegment] = [] - for seg in segments { - if let last = merged.last, last.speakerId == seg.speakerId, - seg.startTime - last.endTime < 0.5 { - merged[merged.count - 1] = DiarizedSegment( - speakerId: last.speakerId, - startTime: last.startTime, - endTime: seg.endTime - ) - } else { - merged.append(seg) - } - } + let merged = SegmentAudio.merge(segments) var output: [ReTranscribedSegment] = [] - let minSamples = Int(sampleRate * 1.5) // 1.5s to clear Parakeet's 1s minimum after resampling - for seg in merged { - var startFrame = AVAudioFramePosition(Double(seg.startTime) * sampleRate) - var endFrame = min(AVAudioFramePosition(Double(seg.endTime) * sampleRate), AVAudioFramePosition(totalFrames)) - var frameCount = Int(endFrame - startFrame) - - // Pad short segments to meet Parakeet's minimum - if frameCount < minSamples && frameCount > 0 { - let deficit = minSamples - frameCount - let padBefore = min(AVAudioFramePosition(deficit / 2), startFrame) - let padAfter = min(deficit - Int(padBefore), Int(AVAudioFramePosition(totalFrames) - endFrame)) - startFrame -= padBefore - endFrame += AVAudioFramePosition(padAfter) - frameCount = Int(endFrame - startFrame) - } - - guard frameCount > 0 else { continue } - let avFrameCount = AVAudioFrameCount(frameCount) + guard let range = SegmentAudio.paddedFrameRange( + startTime: seg.startTime, endTime: seg.endTime, + sampleRate: sampleRate, totalFrames: totalFrames + ) else { continue } - audioFile.framePosition = startFrame - guard let buffer = AVAudioPCMBuffer(pcmFormat: audioFile.processingFormat, frameCapacity: avFrameCount) else { continue } do { - try audioFile.read(into: buffer, frameCount: avFrameCount) + // nil = buffer allocation failure → silent skip (as before); a read + // failure throws into the catch below → "[transcription failed]". + guard let buffer = try SegmentAudio.readSegment(file: audioFile, start: range.start, count: range.count) else { continue } let result = try await asrCoordinator.transcribe(buffer: buffer, source: .system) let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines) guard !text.isEmpty else { continue } diff --git a/Tome/Sources/Tome/Transcription/ShadowConfig.swift b/Tome/Sources/Tome/Transcription/ShadowConfig.swift new file mode 100644 index 0000000..f42f8e1 --- /dev/null +++ b/Tome/Sources/Tome/Transcription/ShadowConfig.swift @@ -0,0 +1,36 @@ +import Foundation + +/// Hidden-flag configuration for granite shadow transcription. Read at job +/// creation (not app launch) so toggling applies from the next session. +/// Spec: docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md +struct ShadowConfig: Sendable, Equatable { + let serverPath: String + let modelDir: URL + let port: Int + + // Keep in sync with scripts/setup-granite-shadow.sh + static let modelFilename = "granite-speech-4.1-2b-Q8_0.gguf" + static let mmprojFilename = "mmproj-model-f16.gguf" + + var modelGGUF: URL { modelDir.appendingPathComponent(Self.modelFilename) } + var mmprojGGUF: URL { modelDir.appendingPathComponent(Self.mmprojFilename) } + var baseURL: URL { URL(string: "http://127.0.0.1:\(port)")! } + + func filesPresent(fileManager: FileManager = .default) -> Bool { + fileManager.fileExists(atPath: modelGGUF.path) && fileManager.fileExists(atPath: mmprojGGUF.path) + } + + static func fromDefaults(_ defaults: UserDefaults = .standard) -> ShadowConfig? { + guard defaults.bool(forKey: "graniteShadowEnabled") else { return nil } + let server = defaults.string(forKey: "graniteShadowServerPath") ?? "/opt/homebrew/bin/llama-server" + let dir: URL + if let override = defaults.string(forKey: "graniteShadowModelDir") { + dir = URL(fileURLWithPath: (override as NSString).expandingTildeInPath) + } else { + dir = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("Tome/Granite") + } + let port = (defaults.object(forKey: "graniteShadowPort") as? Int) ?? 8873 + return ShadowConfig(serverPath: server, modelDir: dir, port: port) + } +} diff --git a/Tome/Sources/Tome/Transcription/SidecarRegistry.swift b/Tome/Sources/Tome/Transcription/SidecarRegistry.swift new file mode 100644 index 0000000..60e9edf --- /dev/null +++ b/Tome/Sources/Tome/Transcription/SidecarRegistry.swift @@ -0,0 +1,85 @@ +import Foundation +import os + +/// Process-global record of every live sidecar (llama-server) child pid. +/// +/// A `GraniteSidecar` is spawned fresh per shadow-phase run (spec §3) and may +/// already be out of scope by the time the app is asked to quit — nothing +/// outside Transcription/ knows a sidecar exists, so there's no instance to +/// ask to stop(). This registry is the process-wide backstop: the app's quit +/// path calls `killAll()` unconditionally so no llama-server ever outlives +/// Tome, independent of whatever GraniteSidecar instance (if any) is still +/// holding a reference to the process. +/// +/// `enum` (no instances, process-wide) protected by an os_unfair_lock +/// (`OSAllocatedUnfairLock`, the same primitive MicCapture/SystemAudioCapture +/// already use for cross-thread state) rather than an actor, because +/// `killAll()` must be callable synchronously from +/// `applicationShouldTerminate`, which is not async. +enum SidecarRegistry { + /// Injectable so tests can record signals instead of sending real ones. + /// Defaults to the real libc `kill`. Used both to deliver SIGTERM/SIGKILL + /// and, via `signaler(pid, 0)`, to poll liveness during the grace period + /// (the standard "kill(pid, 0) == 0 means still alive" idiom). + typealias Signaler = @Sendable (Int32, Int32) -> Int32 + + /// pids + the quitting gate share one lock so killAll's victim snapshot + /// and the gate flip are a single atomic step — no window where a + /// concurrent spawn could observe "not quitting" after the snapshot was + /// already taken. + private struct Registry { + var pids: Set = [] + var quitting = false + } + + private static let state = OSAllocatedUnfairLock(uncheckedState: Registry()) + + static func register(pid: Int32) { + state.withLock { r in _ = r.pids.insert(pid) } + } + + static func unregister(pid: Int32) { + state.withLock { r in _ = r.pids.remove(pid) } + } + + /// Test/inspection only. + static var registeredPids: Set { state.withLock { $0.pids } } + + /// True once killAll() has run — the app is exiting. GraniteSidecar + /// checks this at both of its spawn points (start(), and transcribe()'s + /// relaunch branch) so an in-flight job whose connection drops DURING + /// the kill can't respawn a fresh llama-server after killAll's victim + /// snapshot was taken: killAll blocks the caller's thread (main, at + /// quit), but the sidecar actor keeps running on its own executor. + /// Never reset — there is no un-quit. + static var isQuitting: Bool { state.withLock { $0.quitting } } + + /// SIGTERM every registered pid, poll for up to `graceSeconds` (50ms + /// steps via `usleep`) for each to exit, then SIGKILL any stragglers. + /// Synchronous and safe to call from any thread — callers include + /// `applicationShouldTerminate`, which is not async. Idempotent: the + /// registry is drained atomically up front, so a second call (or a + /// concurrent one) has nothing left to act on. Also flips the permanent + /// `isQuitting` gate in the same lock acquisition as the snapshot, so no + /// new sidecar can be spawned after the victims are chosen. + @discardableResult + static func killAll(graceSeconds: TimeInterval = 2.0, signaler: Signaler = kill) -> Int { + let victims = state.withLock { r -> Set in + r.quitting = true + defer { r.pids.removeAll() } + return r.pids + } + guard !victims.isEmpty else { return 0 } + for pid in victims { _ = signaler(pid, SIGTERM) } + + let pollIntervalUsec: UInt32 = 50_000 // 50ms + let deadline = Date().addingTimeInterval(graceSeconds) + var alive = victims + while !alive.isEmpty && Date() < deadline { + usleep(pollIntervalUsec) + alive = alive.filter { signaler($0, 0) == 0 } + } + for pid in alive { _ = signaler(pid, SIGKILL) } + return victims.count + } +} diff --git a/Tome/Sources/Tome/Transcription/TranscriptionEngine.swift b/Tome/Sources/Tome/Transcription/TranscriptionEngine.swift index 3037e37..2326f19 100644 --- a/Tome/Sources/Tome/Transcription/TranscriptionEngine.swift +++ b/Tome/Sources/Tome/Transcription/TranscriptionEngine.swift @@ -58,8 +58,24 @@ final class TranscriptionEngine { /// `PostProcessingQueue` — live streaming and batch re-transcription must route /// through one actor for safe interleaving. let asrCoordinator: ASRCoordinator + + /// The shared VAD manager, loaded once and reused for the lifetime of the + /// engine. Loading a `VadManager` takes ~1.7–2.1s on an M2 Max; doing it on + /// every `start()` (the old behavior) delayed the mic-tap install by that + /// much, trimming the opening ~2s off every recording. It is deliberately + /// NOT nilled in `stop()` — the CoreML model is immutable and streaming + /// state is per-run (see `loadVADManager`), so one instance serves every + /// session and both the mic + system transcribers concurrently. private var vadManager: VadManager? + /// Single-flight handle for an in-progress VAD load. `preloadVAD()` (fired at + /// launch) and an early `start()` can race; both run on `@MainActor`, so they + /// only interleave at awaits. Sharing one load `Task` (rather than each + /// constructing a `VadManager`) guarantees the two callers await the SAME + /// load instead of double-loading. Matches ModelProvisioner's task-tracking + /// style. Nil when no load is in flight. + private var vadLoadTask: Task? + /// The WAV buffer path for the currently-capturing session. The engine owns this URL /// between start and stop; post-processing methods use it explicitly rather than /// reaching into `SystemAudioCapture`. @@ -92,6 +108,53 @@ final class TranscriptionEngine { self.asrCoordinator = asrCoordinator } + /// Load (or reuse) the shared VAD manager, single-flighted. + /// + /// Reuse across sessions and across the concurrent mic/system transcribers is + /// safe: `FluidAudio.VadManager` is an actor whose only stored state is the + /// immutable CoreML model + config + serialized ANE buffer pool. All + /// per-run streaming state lives OUTSIDE the manager — `makeStreamState()` + /// returns a fresh `VadStreamState` and `processStreamingChunk(state:)` + /// threads it in and out — so one manager instance can serve every recording + /// (verified against .build/checkouts/FluidAudio VadManager.swift + + /// VadManager+Streaming.swift; the two live transcribers already share one). + /// + /// Single-flight: if a load is already in flight (preload racing start()), + /// both callers await the same `Task` rather than each building a manager. + /// The `vadManager == nil` fast path and the task handle are only read/written + /// on `@MainActor`, so the guard/assignment can only interleave at the await + /// on `task.value` — which is exactly what the shared handle covers. + private func loadVADManager() async throws -> VadManager { + if let vadManager { return vadManager } + if let vadLoadTask { return try await vadLoadTask.value } + let task = Task { try await VadManager() } + vadLoadTask = task + do { + let manager = try await task.value + vadManager = manager + vadLoadTask = nil + return manager + } catch { + // Clear the failed handle so a later start()/preload can retry. + vadLoadTask = nil + throw error + } + } + + /// Warm the VAD model at app launch so the first recording doesn't lose ~2s + /// to an on-demand load between `start()` and the mic-tap install. Fire this + /// and forget from ContentView's boot task; it single-flights with `start()` + /// via `loadVADManager`, so an early record while preloading shares one load. + func preloadVAD() async { + do { + _ = try await loadVADManager() + diagLog("[ENGINE-VAD-PRELOAD] VAD model preloaded") + } catch { + // Non-fatal: start() will load it on demand (and surface any error there). + diagLog("[ENGINE-VAD-PRELOAD-FAIL] \(error.localizedDescription)") + } + } + func start( locale: Locale, inputDeviceID: AudioDeviceID = 0, @@ -120,10 +183,18 @@ final class TranscriptionEngine { guard await asrCoordinator.isReady else { throw ASRCoordinatorError.notInitialized } - assetStatus = "Loading VAD model..." - diagLog("[ENGINE-1b] loading VAD model...") - let vad = try await VadManager() - self.vadManager = vad + if vadManager == nil { + assetStatus = "Loading VAD model..." + diagLog("[ENGINE-1b] loading VAD model...") + } + // Load once and reuse (single-flight — see loadVADManager). Preloaded + // at launch, so this is normally an instant no-op and the mic tap + // installs without the ~2s VAD load that used to eat the opening of + // every recording. loadVADManager() already assigns `self.vadManager` + // on its success path (mirrors preloadVAD()'s `_ = try await`), so + // capturing the return here would just be a redundant second write — + // discard it and read the property below instead. + _ = try await loadVADManager() assetStatus = "Models ready" diagLog("[ENGINE-2] models ready") diff --git a/Tome/Sources/Tome/Transcription/WhisperBackend.swift b/Tome/Sources/Tome/Transcription/WhisperBackend.swift index 8465772..2dbcd5a 100644 --- a/Tome/Sources/Tome/Transcription/WhisperBackend.swift +++ b/Tome/Sources/Tome/Transcription/WhisperBackend.swift @@ -56,6 +56,18 @@ final actor WhisperBackend: ASRBackend { return hasCore && fm.fileExists(atPath: tokenizerJSON.path) } + /// Both the SDK download and the curl fallback failed. Carries both messages + /// so the Settings failure line explains what was actually tried. + enum FetchError: Error, LocalizedError { + case bothFailed(sdk: String, fallback: String) + var errorDescription: String? { + switch self { + case .bothFailed(let sdk, let fallback): + return "Model download failed. SDK: \(sdk) — curl fallback: \(fallback)" + } + } + } + func prepare(onEvent: @Sendable @escaping (PrepareEvent) -> Void) async throws { guard whisperKit == nil else { return } let variant = Self.resolveVariant() @@ -65,13 +77,47 @@ final actor WhisperBackend: ASRBackend { onEvent(.loading) } else { onEvent(.downloading(progress: 0)) - folder = try await WhisperKit.download( - variant: variant, - downloadBase: Self.downloadBase, - progressCallback: { progress in - onEvent(.downloading(progress: progress.fractionCompleted)) + // Test hook: force the curl path on a healthy network so the fallback + // can be live-exercised without breaking URLSession's connectivity. + if ProcessInfo.processInfo.environment["TOME_FORCE_CURL_MODEL_FETCH"] == "1" { + diagLog("[WHISPER-FETCH] TOME_FORCE_CURL_MODEL_FETCH=1 — using curl fetcher directly") + // Outcome must reach the unified log: a throw here only lands in + // the Settings lastFailure line, which headless test runs and + // post-hoc log forensics can't see. + do { + try await Self.curlFallbackFetch(variant: variant, onEvent: onEvent) + diagLog("[WHISPER-FETCH] curl fetcher COMPLETED for \(variant)") + } catch { + diagLog("[WHISPER-FETCH] curl fetcher FAILED: \(error.localizedDescription)") + throw error + } + } else { + do { + _ = try await WhisperKit.download( + variant: variant, + downloadBase: Self.downloadBase, + progressCallback: { progress in + onEvent(.downloading(progress: progress.fractionCompleted)) + } + ) + } catch { + // Some networks can't reach the HF Xet CDN via URLSession at + // any timeout while curl connects fine (see CurlModelFetcher). + // Fall back to curl; if THAT also fails, throw the original + // SDK error annotated with both so Settings is informative. + diagLog("[WHISPER-FETCH] SDK download failed (\(error.localizedDescription)) — falling back to curl fetcher") + do { + try await Self.curlFallbackFetch(variant: variant, onEvent: onEvent) + diagLog("[WHISPER-FETCH] curl fetcher COMPLETED for \(variant)") + } catch let fallbackError { + diagLog("[WHISPER-FETCH] curl fetcher FAILED: \(fallbackError.localizedDescription)") + throw FetchError.bothFailed( + sdk: error.localizedDescription, + fallback: fallbackError.localizedDescription) + } } - ) + } + folder = Self.modelFolder(variant: variant) onEvent(.loading) } let config = WhisperKitConfig( @@ -84,6 +130,30 @@ final actor WhisperBackend: ASRBackend { whisperKit = try await WhisperKit(config) } + /// Fetch the model variant + tokenizer via curl into the same on-disk layout + /// the SDK download produces, so the subsequent `WhisperKitConfig` load + /// (folder = `modelFolder(variant:)`) is identical. The variant is the bulk, + /// so it drives 0…0.9 of the reported progress; the tokenizer drives 0.9…1.0. + private static func curlFallbackFetch( + variant: String, + onEvent: @Sendable @escaping (PrepareEvent) -> Void + ) async throws { + try await CurlModelFetcher.fetchVariant( + repo: "argmaxinc/whisperkit-coreml", + path: variant, + into: downloadBase, + onProgress: { p in onEvent(.downloading(progress: p * 0.9)) } + ) + // Tokenizer lives in a DIFFERENT repo (see tokenizerJSON); fetch just the + // files an offline load needs rather than the whole multi-GB repo. + try await CurlModelFetcher.fetchFiles( + repo: "openai/whisper-large-v3", + files: ["tokenizer.json", "tokenizer_config.json", "config.json"], + into: downloadBase, + onProgress: { p in onEvent(.downloading(progress: 0.9 + p * 0.1)) } + ) + } + func transcribe(samples: [Float], language: Language) async throws -> ASRResult { guard let whisperKit else { throw ASRCoordinatorError.notInitialized } let start = ContinuousClock.now diff --git a/Tome/Sources/Tome/Views/ContentView.swift b/Tome/Sources/Tome/Views/ContentView.swift index 359f8e7..2e7d0ce 100644 --- a/Tome/Sources/Tome/Views/ContentView.swift +++ b/Tome/Sources/Tome/Views/ContentView.swift @@ -163,6 +163,12 @@ struct ContentView: View { // which awaits the provisioner settling). services.modelProvisioner.provision(settings.transcriberModel) + // Warm the VAD model now so the first recording's mic tap installs + // immediately instead of stalling ~2s on an on-demand VAD load + // (which trimmed the opening of every recording). Fire-and-forget; + // it single-flights with the load inside start(). + Task { await engine.preloadVAD() } + // Sanitize the persisted mic selection: AudioDeviceIDs are transient, // so a device chosen last session (AirPods) may be absent — or worse, // its numeric id reassigned — at this launch. An absent selection left diff --git a/Tome/Tests/TomeTests/AudioWAVExportTests.swift b/Tome/Tests/TomeTests/AudioWAVExportTests.swift new file mode 100644 index 0000000..d706988 --- /dev/null +++ b/Tome/Tests/TomeTests/AudioWAVExportTests.swift @@ -0,0 +1,34 @@ +import AVFoundation +import Testing +@testable import Tome + +@Suite struct AudioWAVExportTests { + @Test func riffHeaderFields() { + let h = AudioWAVExport.riffHeader(dataByteCount: 32000) + #expect(h.count == 44) + #expect(String(data: h[0..<4], encoding: .ascii) == "RIFF") + #expect(String(data: h[8..<12], encoding: .ascii) == "WAVE") + // chunk size = 36 + data + #expect(h[4..<8].withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) } == 32036) + // sample rate 16000 @ offset 24, channels 1 @ 22, bits 16 @ 34 + #expect(h[24..<28].withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) } == 16000) + #expect(h[22..<24].withUnsafeBytes { $0.loadUnaligned(as: UInt16.self) } == 1) + #expect(h[34..<36].withUnsafeBytes { $0.loadUnaligned(as: UInt16.self) } == 16) + } + @Test func convertsStereo48kToMono16k() throws { + let fmt = AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 2)! + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: 48000)! + buf.frameLength = 48000 // 1 second of silence + let data = try AudioWAVExport.wav16kMonoPCM16(from: buf) + let samples = (data.count - 44) / 2 + #expect(abs(samples - 16000) < 64) // ~1 s at 16 kHz (converter may prime ±) + } + @Test func convertsMono8kTo16k() throws { + let fmt = AVAudioFormat(standardFormatWithSampleRate: 8000, channels: 1)! + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: 8000)! + buf.frameLength = 8000 // 1 second of silence at 8 kHz + let data = try AudioWAVExport.wav16kMonoPCM16(from: buf) + let samples = (data.count - 44) / 2 + #expect(abs(samples - 16000) < 64) // ~1 s at 16 kHz (converter may prime ±) + } +} diff --git a/Tome/Tests/TomeTests/BenchManifestTests.swift b/Tome/Tests/TomeTests/BenchManifestTests.swift new file mode 100644 index 0000000..d7262cd --- /dev/null +++ b/Tome/Tests/TomeTests/BenchManifestTests.swift @@ -0,0 +1,23 @@ +import Testing +@testable import BenchSupport + +@Suite struct BenchManifestTests { + @Test func parsesJSONLAndSkipsBlankLines() throws { + let jsonl = """ + {"id": "ami-0001", "wav": "/tmp/a.wav"} + + {"id": "ami-0002", "wav": "/tmp/b.wav"} + """ + let entries = try BenchManifest.parse(jsonl) + #expect(entries == [ManifestEntry(id: "ami-0001", wav: "/tmp/a.wav"), + ManifestEntry(id: "ami-0002", wav: "/tmp/b.wav")]) + } + @Test func emitRoundTrips() throws { + let hyps = [HypothesisEntry(id: "x", text: "hello there")] + let out = BenchManifest.emit(hyps) + #expect(out == #"{"id":"x","text":"hello there"}"# + "\n") + } + @Test func parseRejectsMalformedLine() { + #expect(throws: (any Error).self) { try BenchManifest.parse("not json") } + } +} diff --git a/Tome/Tests/TomeTests/CurlModelFetcherTests.swift b/Tome/Tests/TomeTests/CurlModelFetcherTests.swift new file mode 100644 index 0000000..a341be9 --- /dev/null +++ b/Tome/Tests/TomeTests/CurlModelFetcherTests.swift @@ -0,0 +1,140 @@ +import Foundation +import Testing +@testable import Tome + +/// CI-safe: these exercise the pure URL / path / parse / progress helpers only. +/// No curl is ever spawned and no network is touched. +@Suite struct CurlModelFetcherTests { + + // MARK: - Tree JSON → file-list parsing + + @Test func fileListKeepsOnlyFileEntries() throws { + // Directory entries (type == "directory") must be dropped; only files kept. + let json = """ + [ + {"type": "directory", "path": "openai_whisper-large-v3-v20240930/AudioEncoder.mlmodelc"}, + {"type": "file", "path": "openai_whisper-large-v3-v20240930/AudioEncoder.mlmodelc/coremldata.bin", "size": 123}, + {"type": "file", "path": "openai_whisper-large-v3-v20240930/config.json", "size": 45}, + {"type": "directory", "path": "openai_whisper-large-v3-v20240930/TextDecoder.mlmodelc"} + ] + """.data(using: .utf8)! + let files = try CurlModelFetcher.fileList(fromTreeJSON: json) + #expect(files == [ + CurlModelFetcher.RemoteFile( + path: "openai_whisper-large-v3-v20240930/AudioEncoder.mlmodelc/coremldata.bin", size: 123), + CurlModelFetcher.RemoteFile(path: "openai_whisper-large-v3-v20240930/config.json", size: 45), + ]) + } + + @Test func fileListToleratesMissingSize() throws { + // Some tree entries may omit `size`; that must parse as nil, not throw. + let json = """ + [{"type": "file", "path": "config.json"}] + """.data(using: .utf8)! + let files = try CurlModelFetcher.fileList(fromTreeJSON: json) + #expect(files == [CurlModelFetcher.RemoteFile(path: "config.json", size: nil)]) + } + + @Test func fileListEmptyForNoFiles() throws { + let json = "[{\"type\": \"directory\", \"path\": \"foo\"}]".data(using: .utf8)! + #expect(try CurlModelFetcher.fileList(fromTreeJSON: json).isEmpty) + } + + @Test func fileListThrowsOnNonArrayJSON() { + let json = "{\"error\": \"not found\"}".data(using: .utf8)! + #expect(throws: CurlModelFetcher.FetchError.self) { + _ = try CurlModelFetcher.fileList(fromTreeJSON: json) + } + } + + // MARK: - URL derivation + + @Test func treeURLIsRecursiveAgainstTheModelsAPI() { + let url = CurlModelFetcher.treeURL(repo: "argmaxinc/whisperkit-coreml", path: "openai_whisper-large-v3-v20240930") + #expect(url.absoluteString == + "https://huggingface.co/api/models/argmaxinc/whisperkit-coreml/tree/main/openai_whisper-large-v3-v20240930?recursive=true") + } + + @Test func resolveURLForNestedFile() { + let url = CurlModelFetcher.resolveURL( + repo: "argmaxinc/whisperkit-coreml", + filePath: "openai_whisper-large-v3-v20240930/AudioEncoder.mlmodelc/coremldata.bin") + #expect(url.absoluteString == + "https://huggingface.co/argmaxinc/whisperkit-coreml/resolve/main/openai_whisper-large-v3-v20240930/AudioEncoder.mlmodelc/coremldata.bin") + } + + // MARK: - Destination path derivation (must match HubApi / WhisperBackend layout) + + @Test func destinationForNestedFileMatchesModelFolderLayout() { + let root = URL(fileURLWithPath: "/tmp/base") + let variant = "openai_whisper-large-v3-v20240930" + let filePath = "\(variant)/AudioEncoder.mlmodelc/coremldata.bin" + let dest = CurlModelFetcher.destination( + destRoot: root, repo: "argmaxinc/whisperkit-coreml", filePath: filePath) + #expect(dest.path == + "/tmp/base/models/argmaxinc/whisperkit-coreml/openai_whisper-large-v3-v20240930/AudioEncoder.mlmodelc/coremldata.bin") + + // The curl layout must land inside exactly the folder WhisperBackend loads + // from — otherwise the offline load after a fallback would miss the files. + let modelFolder = WhisperBackend.modelFolder(variant: variant) + let expected = modelFolder.appendingPathComponent("AudioEncoder.mlmodelc/coremldata.bin") + let viaFetcher = CurlModelFetcher.destination( + destRoot: WhisperBackend.downloadBase, repo: "argmaxinc/whisperkit-coreml", filePath: filePath) + #expect(viaFetcher.path == expected.path) + } + + @Test func destinationForTokenizerMatchesTokenizerJSONLayout() { + let dest = CurlModelFetcher.destination( + destRoot: WhisperBackend.downloadBase, + repo: "openai/whisper-large-v3", + filePath: "tokenizer.json") + #expect(dest.path == WhisperBackend.tokenizerJSON.path) + } + + // MARK: - Progress fraction math + + @Test func progressFractionIsCompletedOverTotal() { + #expect(CurlModelFetcher.progressFraction(completed: 0, total: 4) == 0) + #expect(CurlModelFetcher.progressFraction(completed: 1, total: 4) == 0.25) + #expect(CurlModelFetcher.progressFraction(completed: 4, total: 4) == 1) + } + + @Test func progressFractionTreatsZeroTotalAsComplete() { + #expect(CurlModelFetcher.progressFraction(completed: 0, total: 0) == 1) + } + + @Test func progressFractionClampsToUnitInterval() { + #expect(CurlModelFetcher.progressFraction(completed: 5, total: 4) == 1) + #expect(CurlModelFetcher.progressFraction(completed: -1, total: 4) == 0) + } + + // MARK: - Skip-vs-redownload decision (replaces `-C -` resume; avoids HTTP 416) + + @Test func shouldSkipWhenExistingSizeMatchesExpected() { + #expect(CurlModelFetcher.shouldSkipDownload(existingFileSize: 4096, expectedSize: 4096)) + } + + @Test func shouldNotSkipWhenExistingSizeIsSmallerThanExpected() { + // The common interrupted-download case: a partial file present. + #expect(!CurlModelFetcher.shouldSkipDownload(existingFileSize: 1024, expectedSize: 4096)) + } + + @Test func shouldNotSkipWhenExistingSizeIsLargerThanExpected() { + // A mismatch either way must redownload, not just a short partial. + #expect(!CurlModelFetcher.shouldSkipDownload(existingFileSize: 8192, expectedSize: 4096)) + } + + @Test func shouldNotSkipWhenNoFileExistsYet() { + #expect(!CurlModelFetcher.shouldSkipDownload(existingFileSize: nil, expectedSize: 4096)) + } + + @Test func shouldNotSkipWhenExpectedSizeIsUnknown() { + // `fetchFiles` (tokenizer) has no tree-listing size to compare against — + // must always redownload rather than trust a same-named local file. + #expect(!CurlModelFetcher.shouldSkipDownload(existingFileSize: 4096, expectedSize: nil)) + } + + @Test func shouldNotSkipWhenNeitherSizeIsKnown() { + #expect(!CurlModelFetcher.shouldSkipDownload(existingFileSize: nil, expectedSize: nil)) + } +} diff --git a/Tome/Tests/TomeTests/GraniteRequestTests.swift b/Tome/Tests/TomeTests/GraniteRequestTests.swift new file mode 100644 index 0000000..5b603cf --- /dev/null +++ b/Tome/Tests/TomeTests/GraniteRequestTests.swift @@ -0,0 +1,31 @@ +import Foundation +import Testing +@testable import Tome + +@Suite struct GraniteRequestTests { + @Test func buildMatchesPinnedTemplate() throws { + // Golden contract: scripts/asr-bench/granite_request.md + let wav = Data([0x52, 0x49, 0x46, 0x46]) // "RIFF" + let body = try JSONSerialization.jsonObject(with: GraniteRequest.build(wavData: wav)) as! [String: Any] + #expect(body["temperature"] as? Double == 0) + #expect(body["max_tokens"] as? Int == 2048) + #expect(body["stream"] as? Bool == false) + let msgs = body["messages"] as! [[String: Any]] + #expect(msgs.count == 1 && msgs[0]["role"] as? String == "user") + let content = msgs[0]["content"] as! [[String: Any]] + let audio = content[0]["input_audio"] as! [String: Any] + #expect(audio["format"] as? String == "wav") + #expect(audio["data"] as? String == wav.base64EncodedString()) + #expect(content[1]["text"] as? String == GraniteRequest.prompt) + #expect(GraniteRequest.prompt == "can you transcribe the speech into a written format?") + } + @Test func parseExtractsContent() throws { + let json = #"{"choices":[{"message":{"role":"assistant","content":" hello world \n"}}]}"# + #expect(try GraniteRequest.parseResponse(Data(json.utf8)) == "hello world") + } + @Test func parseThrowsOnMalformed() { + #expect(throws: (any Error).self) { + try GraniteRequest.parseResponse(Data(#"{"error":"boom"}"#.utf8)) + } + } +} diff --git a/Tome/Tests/TomeTests/GraniteShadowTests.swift b/Tome/Tests/TomeTests/GraniteShadowTests.swift new file mode 100644 index 0000000..42e5a8b --- /dev/null +++ b/Tome/Tests/TomeTests/GraniteShadowTests.swift @@ -0,0 +1,178 @@ +import AVFoundation +import Foundation +import Testing +@testable import Tome + +final class FakeSegmentTranscriber: SegmentTranscribing, @unchecked Sendable { + var results: [Result] + init(_ results: [Result]) { self.results = results } + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String { + try results.removeFirst().get() + } +} +private struct Boom: Error {} + +@Suite struct GraniteShadowTests { + // -- policy -- + @Test func shouldRunRequiresConfigRebuildAndResults() { + let cfg = ShadowConfig(serverPath: "/x", modelDir: URL(fileURLWithPath: "/x"), port: 1) + let seg = [ReTranscribedSegment(speaker: "Speaker 2", text: "hi", startTime: 0)] + #expect(GraniteShadowPhase.shouldRun(config: cfg, didRebuild: true, primary: seg)) + #expect(!GraniteShadowPhase.shouldRun(config: nil, didRebuild: true, primary: seg)) + #expect(!GraniteShadowPhase.shouldRun(config: cfg, didRebuild: false, primary: seg)) + #expect(!GraniteShadowPhase.shouldRun(config: cfg, didRebuild: true, primary: nil)) + #expect(!GraniteShadowPhase.shouldRun(config: cfg, didRebuild: true, primary: [])) + } + // -- runner: uses a real tiny WAV fixture so SegmentAudio paths execute -- + private func fixtureWAV() throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("shadow-\(UUID().uuidString).wav") + let fmt = AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1)! + let file = try AVAudioFile(forWriting: url, settings: fmt.settings) + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: 16000 * 10)! + buf.frameLength = 16000 * 10 // 10 s silence + try file.write(from: buf) + return url + } + @Test func runnerProducesResultPerMergedSegmentIncludingErrors() async throws { + let wav = try fixtureWAV() + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 2.0), + DiarizedSegment(speakerId: "S1", startTime: 3.0, endTime: 5.0)] + let runner = ShadowRunner(transcriber: FakeSegmentTranscriber([.success("hello"), .failure(Boom())])) + let out = await runner.run(fileURL: wav, diarSegments: segs, speakerNumberBase: 2) + #expect(out.segments.count == 2) + #expect(out.segments[0].text == "hello" && out.segments[0].error == nil) + #expect(out.segments[1].text == nil && out.segments[1].error != nil) + #expect(!out.incomplete) + } + // -- pairing + artifacts -- + @Test func artifactsPairByStartTimeAndHandleMissingPrimary() throws { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let session = ShadowSessionInfo(sessionID: "s1", transcriptPath: "/t.md", + sessionType: "callCapture", + primaryModel: "Parakeet-TDT v3", + graniteModel: "granite-speech-4.1-2b-Q8_0") + // primary skipped the 3.0 segment (empty text) — granite has it + let primary = [ReTranscribedSegment(speaker: "Speaker 2", text: "hi there", startTime: 0.0)] + let shadow = ShadowRunOutput(segments: [ + ShadowSegment(startTime: 0.0, speaker: "Speaker 2", durationSec: 2, text: "hi there friend", error: nil, latencySec: 0.5), + ShadowSegment(startTime: 3.0, speaker: "Speaker 3", durationSec: 2, text: "quarterly numbers", error: nil, latencySec: 0.4), + ], incomplete: false) + let (md, json) = try ShadowArtifacts.write(session: session, primary: primary, shadow: shadow, to: dir) + let comparison = try JSONDecoder().decode(ShadowComparison.self, from: Data(contentsOf: json)) + #expect(comparison.segments.count == 2) + #expect(comparison.segments[0].primaryText == "hi there") + #expect(comparison.segments[1].primaryText == "") // "" for missing side (spec §4) + #expect(comparison.segments[1].graniteText == "quarterly numbers") + #expect(comparison.totals.segmentCount == 2 && comparison.totals.erroredCount == 0) + let mdText = try String(contentsOf: md, encoding: .utf8) + #expect(mdText.contains("Speaker 3: quarterly numbers")) + #expect(mdText.contains("granite-speech-4.1-2b-Q8_0")) + } + + // Overlapping-speaker diarization can produce two merged segments that + // share a startTime but belong to different speakers. Keying the pairing + // dict on startTime alone collides and drops one primary text (FIX 5) — + // this pins the startTime+speaker key so both pair correctly. + @Test func artifactsPairBySameStartTimeDifferentSpeakers() throws { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let session = ShadowSessionInfo(sessionID: "s2", transcriptPath: "/t.md", + sessionType: "callCapture", + primaryModel: "Parakeet-TDT v3", + graniteModel: "granite-speech-4.1-2b-Q8_0") + let primary = [ + ReTranscribedSegment(speaker: "Speaker 2", text: "primary A", startTime: 5.0), + ReTranscribedSegment(speaker: "Speaker 3", text: "primary B", startTime: 5.0), + ] + let shadow = ShadowRunOutput(segments: [ + ShadowSegment(startTime: 5.0, speaker: "Speaker 2", durationSec: 1, text: "shadow A", error: nil, latencySec: 0.1), + ShadowSegment(startTime: 5.0, speaker: "Speaker 3", durationSec: 1, text: "shadow B", error: nil, latencySec: 0.1), + ], incomplete: false) + let (_, json) = try ShadowArtifacts.write(session: session, primary: primary, shadow: shadow, to: dir) + let comparison = try JSONDecoder().decode(ShadowComparison.self, from: Data(contentsOf: json)) + let bySpeaker = Dictionary(uniqueKeysWithValues: comparison.segments.map { ($0.speaker, $0) }) + #expect(bySpeaker["Speaker 2"]?.primaryText == "primary A") + #expect(bySpeaker["Speaker 2"]?.graniteText == "shadow A") + #expect(bySpeaker["Speaker 3"]?.primaryText == "primary B") + #expect(bySpeaker["Speaker 3"]?.graniteText == "shadow B") + } + + // -- corrections coverage: readSegment throws (allocation-nil vs read-error) -- + @Test func runnerRecordsAllocationFailureDistinctFromReadFailure() async throws { + let wav = try fixtureWAV() + // A single segment whose padded range is well within the 10s fixture, + // so SegmentAudio.readSegment succeeds; this test exercises the success + // path plumbing through the corrected throwing signature (nil vs throw + // are exercised indirectly since we can't force AVAudioFile to fail + // without corrupting the fixture — covered by reading a nonexistent file + // via the file-open failure path below instead). + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 1.0)] + let runner = ShadowRunner(transcriber: FakeSegmentTranscriber([.success("ok")])) + let out = await runner.run(fileURL: wav, diarSegments: segs, speakerNumberBase: 2) + #expect(out.segments.count == 1) + #expect(out.segments[0].text == "ok") + } + + @Test func runnerReturnsIncompleteWhenFileCannotBeOpened() async { + let missing = FileManager.default.temporaryDirectory.appendingPathComponent("no-such-\(UUID().uuidString).wav") + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 1.0)] + let runner = ShadowRunner(transcriber: FakeSegmentTranscriber([.success("unused")])) + let out = await runner.run(fileURL: missing, diarSegments: segs, speakerNumberBase: 2) + #expect(out.segments.isEmpty) + #expect(out.incomplete) + } + + // -- sidecar notReady mid-run marks incomplete and stops burning segments -- + @Test func runnerStopsAndMarksIncompleteOnSidecarNotReady() async throws { + let wav = try fixtureWAV() + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 2.0), + DiarizedSegment(speakerId: "S1", startTime: 3.0, endTime: 5.0)] + let runner = ShadowRunner(transcriber: FakeSegmentTranscriber([.failure(GraniteSidecar.SidecarError.notReady)])) + let out = await runner.run(fileURL: wav, diarSegments: segs, speakerNumberBase: 2) + #expect(out.segments.count == 1) // stopped after the first (notReady) segment + #expect(out.incomplete) + } + + // -- requestFailed means the sidecar is equally dead (relaunch budget + // exhausted, process torn down by GraniteSidecar) — same early-break -- + @Test func runnerStopsAndMarksIncompleteOnSidecarRequestFailed() async throws { + let wav = try fixtureWAV() + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 2.0), + DiarizedSegment(speakerId: "S1", startTime: 3.0, endTime: 5.0)] + let runner = ShadowRunner(transcriber: FakeSegmentTranscriber([.failure(GraniteSidecar.SidecarError.requestFailed)])) + let out = await runner.run(fileURL: wav, diarSegments: segs, speakerNumberBase: 2) + #expect(out.segments.count == 1) // exactly one attempt — no extra segment burned + #expect(out.incomplete) + } + + // -- cancellation mid-run: partial results preserved, marked incomplete -- + /// A transcriber that signals a continuation after its first call so the + /// test can cancel the enclosing Task from outside, simulating cooperative + /// cancellation arriving between segments. Proves ShadowRunner's + /// `Task.isCancelled` check (not just the sidecar-notReady check) stops + /// the loop and reports incomplete, while keeping whatever partial results + /// were already collected — those still flow into GraniteShadowPhase.run's + /// unconditional artifact write. + final class SelfCancelingTranscriber: SegmentTranscribing, @unchecked Sendable { + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String { + withUnsafeCurrentTask { $0?.cancel() } + return "first" + } + } + @Test func runnerStopsAtNextSegmentAfterCancellationAndKeepsPartialResults() async throws { + let wav = try fixtureWAV() + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 2.0), + DiarizedSegment(speakerId: "S1", startTime: 3.0, endTime: 5.0), + DiarizedSegment(speakerId: "S2", startTime: 6.0, endTime: 8.0)] + // Run on a child Task so self-cancellation inside the transcriber + // doesn't propagate up and cancel the enclosing @Test's own task. + let child = Task { + let runner = ShadowRunner(transcriber: SelfCancelingTranscriber()) + return await runner.run(fileURL: wav, diarSegments: segs, speakerNumberBase: 2) + } + let out = await child.value + #expect(out.segments.count == 1) // first segment completed before cancellation observed + #expect(out.segments[0].text == "first") + #expect(out.incomplete) + } +} diff --git a/Tome/Tests/TomeTests/GraniteSidecarTests.swift b/Tome/Tests/TomeTests/GraniteSidecarTests.swift new file mode 100644 index 0000000..ab5f090 --- /dev/null +++ b/Tome/Tests/TomeTests/GraniteSidecarTests.swift @@ -0,0 +1,341 @@ +import Foundation +import Testing +@testable import Tome + +final class FakeProcess: SidecarProcess, @unchecked Sendable { + var running = true + var terminated = false, killed = false + /// A stubborn process ignores SIGTERM (terminate leaves it running) and + /// only dies to forceKill — exercises endProcess's escalation branch. + var stubborn = false + var isRunning: Bool { running } + func terminate() { terminated = true; if !stubborn { running = false } } + func forceKill() { killed = true; running = false } +} + +final class FakeLauncher: SidecarProcessLauncher, @unchecked Sendable { + var launched: [(URL, [String])] = [] + var processes: [FakeProcess] = [] + var launchError: (any Error)? + /// When true, vended processes are stubborn (see FakeProcess.stubborn). + var makeStubborn = false + func launch(executable: URL, arguments: [String]) throws -> any SidecarProcess { + if let launchError { throw launchError } + launched.append((executable, arguments)) + let p = FakeProcess(); p.stubborn = makeStubborn; processes.append(p); return p + } +} + +/// Continuation-gate parking follows FakeBackend.swift's style (see +/// ASRCoordinatorTests.installRevalidatesTokenAcrossUnloadSuspension): a test +/// arms hangNext*, waits for the *Parked flag, interleaves, then releases. +final class FakeHTTP: SidecarHTTP, @unchecked Sendable { + /// Every start() call now probes /health twice: once pre-launch (must + /// NOT be 200, or start() refuses to adopt a "foreign" server) and once + /// in the post-launch poll loop (200 == ready). The default models the + /// realistic case — nothing listening yet, then healthy right after + /// launch — so tests that just need "start() succeeds once" don't have + /// to seed this explicitly; tests with multiple start() calls (initial + + /// relaunch) must seed enough entries to cover every probe. + var healthResults: [Int?] = [nil, 200] + var postResults: [Result<(Data, Int), any Error>] = [] + /// When true, the next healthStatus call parks until releaseHealth(). + var hangNextHealth = false + /// When set, healthStatus parks on its Nth call (1-indexed) regardless + /// of hangNextHealth — lets a test target the post-launch poll + /// specifically without racing to flip hangNextHealth between the + /// pre-launch probe and the first loop iteration. + var hangHealthAtCallNumber: Int? + private var healthCallCount = 0 + private(set) var healthParked = false + private var healthGate: CheckedContinuation? + /// When true, the next post call parks until releasePost(). + var hangNextPost = false + private(set) var postParked = false + private var postGate: CheckedContinuation? + + func releaseHealth() { healthGate?.resume(); healthGate = nil } + func releasePost() { postGate?.resume(); postGate = nil } + + func healthStatus(_ url: URL) async -> Int? { + healthCallCount += 1 + if hangNextHealth || healthCallCount == hangHealthAtCallNumber { + hangNextHealth = false + healthParked = true + await withCheckedContinuation { healthGate = $0 } + healthParked = false + } + return healthResults.isEmpty ? 200 : healthResults.removeFirst() + } + func post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> (Data, Int) { + if hangNextPost { + hangNextPost = false + postParked = true + await withCheckedContinuation { postGate = $0 } + postParked = false + } + return try postResults.removeFirst().get() + } +} + +/// Bounded poll until `condition` — the park-wait pattern from +/// ASRCoordinatorTests (100 × 10 ms; the caller asserts afterwards). +private func waitFor(_ condition: @autoclosure @escaping () -> Bool) async throws { + for _ in 0..<100 { + if condition() { return } + try await Task.sleep(for: .milliseconds(10)) + } +} + +/// `isQuitting` defaults to `{ false }` here (NOT the production default of +/// `SidecarRegistry.isQuitting`): the registry's quitting gate is permanent +/// process-global state that SidecarRegistryTests flip via killAll() in this +/// same test process, and these tests must not depend on suite ordering. +private func makeSidecar(launcher: FakeLauncher = FakeLauncher(), http: FakeHTTP = FakeHTTP(), + readyTimeout: TimeInterval = 1, + sleep: @escaping @Sendable (TimeInterval) async -> Void = { _ in }, + isQuitting: @escaping @Sendable () -> Bool = { false }) + -> (GraniteSidecar, FakeLauncher, FakeHTTP) { + let config = ShadowConfig(serverPath: "/fake/llama-server", + modelDir: URL(fileURLWithPath: "/fake/models"), port: 9999) + let s = GraniteSidecar(config: config, launcher: launcher, http: http, + readyTimeout: readyTimeout, sleep: sleep, isQuitting: isQuitting) + return (s, launcher, http) +} + +private let ok = Data(#"{"choices":[{"message":{"content":"hi"}}]}"#.utf8) +private struct ConnErr: Error {} + +@Suite struct GraniteSidecarTests { + @Test func startLaunchesWithConfigArgsAndPollsHealth() async { + let (s, launcher, http) = makeSidecar() + http.healthResults = [503, 200] // pre-launch probe (not foreign), then ready + #expect(await s.start()) + let (exe, args) = launcher.launched[0] + #expect(exe.path == "/fake/llama-server") + #expect(args.contains("--port") && args.contains("9999") && args.contains("127.0.0.1")) + #expect(args.contains("/fake/models/\(ShadowConfig.modelFilename)")) + #expect(args.contains("-c") && args.contains("16384")) + #expect(args.contains("--no-webui")) + } + @Test func startFailsAfterTimeoutAndKills() async { + let (s, launcher, http) = makeSidecar() + http.healthResults = Array(repeating: 503 as Int?, count: 500) + #expect(await s.start() == false) + #expect(launcher.processes[0].terminated || launcher.processes[0].killed) + } + @Test func transcribeSendsRequestAndParses() async throws { + let (s, _, http) = makeSidecar() + http.postResults = [.success((ok, 200))] + _ = await s.start() + #expect(try await s.transcribe(wavData: Data([1])) == "hi") + } + @Test func connectionFailureRelaunchesOnceThenFails() async { + let (s, launcher, http) = makeSidecar() + http.healthResults = [nil, 200, nil, 200] // initial start + one relaunch start + http.postResults = [.failure(ConnErr()), .failure(ConnErr())] + _ = await s.start() + await #expect(throws: (any Error).self) { try await s.transcribe(wavData: Data([1])) } + #expect(launcher.launched.count == 2) // original + one relaunch + // subsequent calls fail fast without further launches + await #expect(throws: (any Error).self) { try await s.transcribe(wavData: Data([1])) } + #expect(launcher.launched.count == 2) + } + @Test func stopTerminatesProcess() async { + let (s, launcher, _) = makeSidecar() + _ = await s.start() + await s.stop() + #expect(launcher.processes[0].terminated) + } + + // MARK: - foreign-server / dead-child guards (FIX 2) + + @Test func startRefusesToAdoptForeignServerAlreadyOnPort() async { + let (s, launcher, http) = makeSidecar() + http.healthResults = [200] // something already answering /health before any launch + #expect(await s.start() == false) + #expect(launcher.launched.count == 0) + #expect(await s.state == .failed) + } + + @Test func stopDuringPrelaunchProbeYieldsIdleNotFailedEvenOn200() async throws { + // A stop() interleaving during the pre-launch probe suspension must + // win over the probe's outcome: releasing the probe with a 200 + // (foreign server present) must NOT stomp state = .failed over the + // .idle that stop() just established — and must not launch anything. + let (s, launcher, http) = makeSidecar() + http.healthResults = [200] // probe would report a foreign server + http.hangHealthAtCallNumber = 1 // park the probe itself + let job = Task { await s.start() } + try await waitFor(http.healthParked) + #expect(http.healthParked) + + await s.stop() + http.releaseHealth() + + #expect(await job.value == false) + #expect(await s.state == .idle) // stop()'s .idle survives, not .failed + #expect(launcher.launched.count == 0) + } + + // MARK: - app-quit gate (kill-vs-relaunch race) + + @Test func startRefusesToLaunchWhenAppIsQuitting() async { + let (s, launcher, _) = makeSidecar(isQuitting: { true }) + #expect(await s.start() == false) + #expect(launcher.launched.count == 0) + #expect(await s.state == .failed) + } + + @Test func postFailureWhileQuittingRethrowsWithoutRelaunch() async { + // The kill-vs-relaunch race: SidecarRegistry.killAll() terminates the + // server from the main thread while this actor's transcribe() is + // suspended in http.post. The dropped connection must NOT take the + // relaunch branch (state still .ready, Task not cancelled, relaunch + // budget unspent) — that would spawn and register a fresh child AFTER + // killAll's victim snapshot, orphaning it. + final class Gate: @unchecked Sendable { var quitting = false } + let gate = Gate() + let (s, launcher, http) = makeSidecar(isQuitting: { gate.quitting }) + http.postResults = [.failure(ConnErr()), .success((ok, 200))] // trailing sentinel + _ = await s.start() + gate.quitting = true // killAll() has run; connection then drops + await #expect(throws: ConnErr.self) { try await s.transcribe(wavData: Data([1])) } + #expect(launcher.launched.count == 1) // no relaunch spawned past the kill snapshot + #expect(http.postResults.count == 1) // sentinel untouched — no retry post either + } + + @Test func startFailsFastWhenChildDiesBeforeHealthy() async throws { + let launcher = FakeLauncher() + let http = FakeHTTP() + final class Counter: @unchecked Sendable { var sleeps = 0 } + let counter = Counter() + // A large readyTimeout (many iterations) so a slow/looping failure + // mode would be obvious in the sleep count; the dead-child guard + // should short-circuit long before that. + let config = ShadowConfig(serverPath: "/fake/llama-server", + modelDir: URL(fileURLWithPath: "/fake/models"), port: 9999) + let s = GraniteSidecar(config: config, launcher: launcher, http: http, + readyTimeout: 250, sleep: { _ in counter.sleeps += 1 }) + http.healthResults = Array(repeating: 503 as Int?, count: 1000) // never healthy + http.hangHealthAtCallNumber = 2 // the loop's first poll (after launch) + let job = Task { await s.start() } + try await waitFor(http.healthParked) + #expect(http.healthParked) + launcher.processes[0].running = false // simulate a bind failure right after launch + http.releaseHealth() + #expect(await job.value == false) + #expect(launcher.launched.count == 1) // no relaunch attempted at start() level + #expect(counter.sleeps <= 1) // failed fast, not through ~500 timeout iterations + } + + // MARK: - stop() reentrancy (generation guard) + + @Test func stopDuringRelaunchHealthPollAbortsAndLeavesNoProcess() async throws { + let (s, launcher, http) = makeSidecar() + http.healthResults = [nil, 200] // initial start succeeds + _ = await s.start() + // First post fails -> relaunch; the relaunch's start() clears its + // pre-launch probe, then parks in its post-launch health poll. The + // trailing .success is a sentinel: it must NOT be consumed (no post + // may go out after stop()). + http.postResults = [.failure(ConnErr()), .success((ok, 200))] + http.healthResults = [nil] // relaunch's pre-launch probe: proceed + http.hangHealthAtCallNumber = 4 // relaunch's first loop poll parks + let job = Task { try await s.transcribe(wavData: Data([1])) } + try await waitFor(http.healthParked) + #expect(http.healthParked) + + await s.stop() + http.releaseHealth() + + await #expect(throws: (any Error).self) { try await job.value } + #expect(launcher.launched.count == 2) // original + relaunch, none after stop + #expect(launcher.processes.allSatisfy { !$0.running }) + #expect(await s.state == .idle) + #expect(http.postResults.count == 1) // sentinel untouched + } + + @Test func stopDuringInitialStartHealthPollAbortsStart() async throws { + let (s, launcher, http) = makeSidecar() + http.healthResults = [nil] // pre-launch probe: nothing listening yet -> proceed + http.hangHealthAtCallNumber = 2 // loop's first poll (after launch) parks + let job = Task { await s.start() } + try await waitFor(http.healthParked) + #expect(http.healthParked) + + await s.stop() + http.releaseHealth() + + #expect(await job.value == false) + #expect(await s.state == .idle) + #expect(launcher.launched.count == 1) + #expect(launcher.processes.allSatisfy { !$0.running }) + } + + @Test func stopWhilePostInFlightFailsFastWithoutRelaunch() async throws { + let (s, launcher, http) = makeSidecar() + _ = await s.start() + http.postResults = [.failure(ConnErr()), .success((ok, 200))] // trailing sentinel + http.hangNextPost = true + let job = Task { try await s.transcribe(wavData: Data([1])) } + try await waitFor(http.postParked) + #expect(http.postParked) + + await s.stop() + http.releasePost() + + await #expect(throws: (any Error).self) { try await job.value } + #expect(launcher.launched.count == 1) // no relaunch after stop + #expect(launcher.processes.allSatisfy { !$0.running }) + #expect(await s.state == .idle) + #expect(http.postResults.count == 1) // sentinel untouched + } + + // MARK: - error taxonomy & escalation + + @Test func parseErrorPropagatesWithoutRelaunch() async throws { + let (s, launcher, http) = makeSidecar() + http.postResults = [.success((Data("not json".utf8), 200))] + _ = await s.start() + await #expect(throws: GraniteRequest.ParseError.self) { + try await s.transcribe(wavData: Data([1])) + } + #expect(launcher.launched.count == 1) // relaunch budget not burned + #expect(await s.state == .ready) // sidecar state untouched + } + + @Test func httpErrorStatusRelaunchesOnceThenFails() async { + // A 4xx/5xx llama-server response must be treated the same as a + // dropped connection (FIX 3) — not surfaced as a ParseError, and not + // silently ignored. + let (s, launcher, http) = makeSidecar() + http.healthResults = [nil, 200, nil, 200] // initial start + one relaunch start + http.postResults = [.success((Data("server error".utf8), 500)), + .success((Data("server error".utf8), 500))] + _ = await s.start() + await #expect(throws: (any Error).self) { try await s.transcribe(wavData: Data([1])) } + #expect(launcher.launched.count == 2) // original + one relaunch, same as connection failure + } + + @Test func cancelledTaskSkipsRelaunchAndRethrows() async { + let (s, launcher, http) = makeSidecar() + http.postResults = [.failure(ConnErr())] + _ = await s.start() + let task = Task { + try await s.transcribe(wavData: Data([1])) + } + task.cancel() + await #expect(throws: (any Error).self) { try await task.value } + #expect(launcher.launched.count == 1) // no relaunch attempted once cancelled + } + + @Test func stubbornProcessEscalatesToForceKill() async { + let (s, launcher, _) = makeSidecar() + launcher.makeStubborn = true + _ = await s.start() + await s.stop() + let p = launcher.processes[0] + #expect(p.terminated && p.killed && !p.running) + } +} diff --git a/Tome/Tests/TomeTests/SegmentAudioTests.swift b/Tome/Tests/TomeTests/SegmentAudioTests.swift new file mode 100644 index 0000000..f860dc8 --- /dev/null +++ b/Tome/Tests/TomeTests/SegmentAudioTests.swift @@ -0,0 +1,34 @@ +import AVFoundation +import Testing +@testable import Tome + +@Suite struct SegmentAudioTests { + @Test func mergesSameSpeakerWithinHalfSecond() { + let segs = [ + DiarizedSegment(speakerId: "A", startTime: 0.0, endTime: 1.0), + DiarizedSegment(speakerId: "A", startTime: 1.3, endTime: 2.0), // gap 0.3 < 0.5 → merge + DiarizedSegment(speakerId: "A", startTime: 2.6, endTime: 3.0), // gap 0.6 ≥ 0.5 → new + DiarizedSegment(speakerId: "B", startTime: 3.1, endTime: 4.0), // speaker change → new + ] + let merged = SegmentAudio.merge(segs) + #expect(merged.count == 3) + #expect(merged[0].startTime == 0.0 && merged[0].endTime == 2.0) + #expect(merged[1].startTime == 2.6 && merged[2].speakerId == "B") + } + @Test func padsShortSegmentCentered() { + // 0.5 s segment at 16 kHz in a long file: deficit = 24000-8000 = 16000 → 8000 both sides + let r = SegmentAudio.paddedFrameRange(startTime: 10, endTime: 10.5, sampleRate: 16000, + totalFrames: 10_000_000) + #expect(r! == (start: 152_000, count: 24_000)) + } + @Test func padClampsAtFileStart() { + // Segment at t=0: no room before, pad goes after + let r = SegmentAudio.paddedFrameRange(startTime: 0, endTime: 0.5, sampleRate: 16000, + totalFrames: 10_000_000) + #expect(r! == (start: 0, count: 24_000)) + } + @Test func zeroLengthSegmentIsNil() { + #expect(SegmentAudio.paddedFrameRange(startTime: 5, endTime: 5, sampleRate: 16000, + totalFrames: 80_000) == nil) + } +} diff --git a/Tome/Tests/TomeTests/ShadowConfigTests.swift b/Tome/Tests/TomeTests/ShadowConfigTests.swift new file mode 100644 index 0000000..fb0b68c --- /dev/null +++ b/Tome/Tests/TomeTests/ShadowConfigTests.swift @@ -0,0 +1,42 @@ +import Foundation +import Testing +@testable import Tome + +@Suite struct ShadowConfigTests { + private func makeDefaults() -> UserDefaults { + let name = "ShadowConfigTests-\(UUID().uuidString)" + let d = UserDefaults(suiteName: name)! + d.removePersistentDomain(forName: name) + return d + } + @Test func disabledByDefault() { + #expect(ShadowConfig.fromDefaults(makeDefaults()) == nil) + } + @Test func enabledUsesDefaults() { + let d = makeDefaults(); d.set(true, forKey: "graniteShadowEnabled") + let c = try! #require(ShadowConfig.fromDefaults(d)) + #expect(c.serverPath == "/opt/homebrew/bin/llama-server") + #expect(c.port == 8873) + #expect(c.modelDir.path.hasSuffix("Tome/Granite")) + #expect(c.modelGGUF.lastPathComponent == "granite-speech-4.1-2b-Q8_0.gguf") + #expect(c.mmprojGGUF.lastPathComponent == "mmproj-model-f16.gguf") + } + @Test func overridesRespectedAndTildeExpanded() { + let d = makeDefaults() + d.set(true, forKey: "graniteShadowEnabled") + d.set("/usr/local/bin/llama-server", forKey: "graniteShadowServerPath") + d.set("~/granite-models", forKey: "graniteShadowModelDir") + d.set(9001, forKey: "graniteShadowPort") + let c = try! #require(ShadowConfig.fromDefaults(d)) + #expect(c.serverPath == "/usr/local/bin/llama-server") + #expect(c.port == 9001) + #expect(!c.modelDir.path.contains("~")) + #expect(c.modelDir.path.hasSuffix("/granite-models")) + } + @Test func filesPresentFalseOnEmptyDir() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let c = ShadowConfig(serverPath: "/x", modelDir: tmp, port: 1) + #expect(!c.filesPresent()) + } +} diff --git a/Tome/Tests/TomeTests/SidecarRegistryTests.swift b/Tome/Tests/TomeTests/SidecarRegistryTests.swift new file mode 100644 index 0000000..7683c66 --- /dev/null +++ b/Tome/Tests/TomeTests/SidecarRegistryTests.swift @@ -0,0 +1,110 @@ +import Foundation +import Testing +@testable import Tome + +/// SidecarRegistry is process-global mutable state, so these tests run +/// serialized (not concurrently with each other) and never invoke the real +/// `kill` — every killAll() call below injects a recording signaler instead. +/// Fake pids are large/UUID-derived to make accidental collisions between +/// tests vanishingly unlikely even so. +@Suite(.serialized) struct SidecarRegistryTests { + private final class Recorder: @unchecked Sendable { + var calls: [(pid: Int32, sig: Int32)] = [] + } + + /// Large pid drawn from a UUID so concurrent test runs (or leftover state + /// from another test) can't collide with it. + private func fakePid() -> Int32 { + Int32.random(in: 100_000...2_000_000_000) + } + + @Test func registerAndUnregisterUpdateMembership() { + let pid = fakePid() + SidecarRegistry.register(pid: pid) + #expect(SidecarRegistry.registeredPids.contains(pid)) + SidecarRegistry.unregister(pid: pid) + #expect(!SidecarRegistry.registeredPids.contains(pid)) + } + + @Test func unregisterOfUnknownPidIsNoOpAndLeavesOthersIntact() { + let known = fakePid() + let unknown = fakePid() + SidecarRegistry.register(pid: known) + SidecarRegistry.unregister(pid: unknown) // never registered + #expect(SidecarRegistry.registeredPids.contains(known)) + SidecarRegistry.unregister(pid: known) + } + + @Test func killAllSendsSIGTERMToEveryRegisteredPidAndDrainsRegistry() { + let a = fakePid(), b = fakePid() + SidecarRegistry.register(pid: a) + SidecarRegistry.register(pid: b) + let recorder = Recorder() + let count = SidecarRegistry.killAll { pid, sig in + recorder.calls.append((pid, sig)) + // kill(pid, 0) semantics: 0 == still alive, -1 == no such + // process. Reporting "gone" on the liveness poll here so the + // grace loop exits immediately instead of waiting out the full + // grace period. + return sig == 0 ? -1 : 0 + } + #expect(count == 2) + let termed = Set(recorder.calls.filter { $0.sig == SIGTERM }.map(\.pid)) + #expect(termed == [a, b]) + #expect(!SidecarRegistry.registeredPids.contains(a)) + #expect(!SidecarRegistry.registeredPids.contains(b)) + } + + @Test func killAllEscalatesToSIGKILLWhenLivenessPollReportsStillAlive() { + let pid = fakePid() + SidecarRegistry.register(pid: pid) + let recorder = Recorder() + // graceSeconds tiny so the escalation path doesn't slow the suite — + // production still defaults to the spec's 2s grace. + SidecarRegistry.killAll(graceSeconds: 0.05) { p, sig in + recorder.calls.append((p, sig)) + return sig == 0 ? 0 : 0 // kill(pid, 0) == 0 means "still alive" + } + let sigkills = recorder.calls.filter { $0.sig == SIGKILL } + #expect(!sigkills.isEmpty) + #expect(sigkills.allSatisfy { $0.pid == pid }) + #expect(!SidecarRegistry.registeredPids.contains(pid)) + } + + @Test func killAllWithNothingRegisteredIsIdempotentNoOp() { + // Drain any leftover registrations from other tests deterministically + // first, so this assertion doesn't depend on suite ordering. + _ = SidecarRegistry.killAll { _, _ in 0 } + let count = SidecarRegistry.killAll { _, _ in + Issue.record("signaler must not be called when the registry is empty") + return 0 + } + #expect(count == 0) + } + + @Test func killAllFlipsThePermanentQuittingGate() { + // The gate is never reset (the app is exiting), so this can only + // assert the forward direction: after killAll, isQuitting is true — + // including on an empty registry (the flip must not be skipped by the + // nothing-to-kill early return). GraniteSidecarTests inject their own + // gate closure precisely because this flip is permanent process-global + // state shared across suites. + _ = SidecarRegistry.killAll { _, _ in -1 } + #expect(SidecarRegistry.isQuitting) + _ = SidecarRegistry.killAll { _, _ in -1 } // idempotent: still true + #expect(SidecarRegistry.isQuitting) + } + + @Test func killAllIsSynchronousAndSafeToCallFromANonMainThread() async { + let pid = fakePid() + SidecarRegistry.register(pid: pid) + let recorder = Recorder() + await Task.detached { + SidecarRegistry.killAll { p, sig in + recorder.calls.append((p, sig)) + return sig == 0 ? -1 : 0 // reports gone -> no escalation wait + } + }.value + #expect(recorder.calls.contains { $0.pid == pid && $0.sig == SIGTERM }) + } +} diff --git a/docs/superpowers/plans/2026-07-09-granite-phase0-results.md b/docs/superpowers/plans/2026-07-09-granite-phase0-results.md new file mode 100644 index 0000000..d7b162b --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-granite-phase0-results.md @@ -0,0 +1,192 @@ +# Granite Phase 0 Benchmark Results (2026-07-09/10, Nic's M2 Max 64GB) + +Harness: `scripts/asr-bench/bench.py` (spec §0). Sets: first ~3 h of each ESB +test set (AMI 1278 utts, Earnings-22 781, TED-LIUM 1155 — 314 ignore-marker +rows filtered). Backends: granite-speech-4.1-2b Q8_0 via `llama-server` +(single-stream, spawn-per-run), Parakeet-TDT v3 + Whisper large-v3-turbo via +`ASRBench --manifest` (Tome's real FluidAudio/WhisperKit code paths). Scoring: +Whisper tokenizer normalizer on refs and hyps, `jiwer` WER — same recipe as the +Open ASR Leaderboard. + +## WER (%) + +| Set | granite-4.1-2b | Parakeet v3 | Whisper turbo | Published granite (raw) | +|---|---|---|---|---| +| AMI (meetings) | **5.98** | 7.47 | 13.85 | 7.72 | +| Earnings-22 (accents/compression) | **8.01** | 10.39 | 10.74 | 8.23 | +| TED-LIUM (held-out for granite) | **3.12** | 3.29 | 3.68 | — | + +Relative error reduction, granite vs: +- Parakeet v3 (current default): AMI **−20.0%**, E22 **−22.9%**, TED −5.2% +- Whisper turbo (current accuracy option): AMI **−56.8%**, E22 −25.4%, TED −15.2% + +## Speed (single-stream RTF, M2 Max, Q8_0) + +AMI 0.048 · Earnings-22 0.047 · TED-LIUM 0.053 — ~0.05 overall, i.e. a +60-minute meeting shadow-transcribes in ~3 minutes. Zero request errors across +3,214 scored utterances (granite stage log: 0 ERR lines). + +## Gate verdicts (spec Success Criteria, Phase 0) + +1. **Fidelity — PASS (with disclosed caveat).** Our-pipeline granite landed at + AMI 5.98 vs published 7.72 (−1.74, outside the ±1.5 letter of the gate on + the *favorable* side) and E22 8.01 vs 8.23 (−0.22, within gate). The AMI + deviation is subset bias, not pipeline effect: the ESB test-only dataset is + *sorted*, and our 3 h slice is systematically easier for **all** backends + (Parakeet −3.1 and Whisper −1.3 vs their published numbers, same + direction). The gate exists to catch pipeline bugs that would make granite + look artificially *bad*; there is no such signature — granite's relative + advantage matches or exceeds the leaderboard's. llama.cpp front-end + fidelity confirmed. +2. **Speed — PASS.** RTF ≈ 0.05 ≤ 0.25, 5× headroom. +3. **Accuracy — PASS.** Granite beats BOTH current backends on AMI and + Earnings-22, and *also wins* (not merely matches) the held-out TED-LIUM — + so the advantage is not an in-domain-training artifact. + +## Decision + +**GO for Phase 1: enable `graniteShadowEnabled` after install.** Shadow-week +data collection on real meetings proceeds; review EOD Wednesday 2026-07-15. + +## Notes / caveats for the Wednesday review + +- Subset = easier first-3h slice of each sorted test set; absolute WERs + under-state full-set numbers for every backend; cross-model deltas are the + meaningful signal. +- TED-LIUM substituted for CORAAL as the held-out set (CORAAL is long-form, + needs its own chunking; TED-LIUM is ungated, short-form, and absent from + granite's training list). CORAAL remains a stretch follow-up. +- GraniteRequest caps `max_tokens` at 2048 ≈ 8–9 min of speech per segment; + merged shadow segments longer than that would truncate. Not observed in + Phase 0 (all ESB utterances are short); watch for it in shadow data from + monologue-heavy meetings. +- Shadow placement (before retention) means a job's completion and the next + queued job are delayed by the shadow runtime (~3 min per meeting-hour) — + inherent to keeping the capture WAVs alive for the phase. + +## Live smoke (Task 13) + +Run: 2026-07-10, ~00:00–00:08 local, unattended (Nic asleep), on his live +machine. Build SHA `f8796209fb7f1aebf562891089d8bb0129e48a5d` (HEAD at the +time — this doc's own "GO" commit; no code changes since). `swift test`: 129 +tests passed. Built via `scripts/build_swift_app.sh` (auto-install step +temporarily disabled for a controlled backup/quit/ditto install, then the +script's edit was reverted — `git status` clean throughout and after). + +**Install.** Backed up the running v1.4.4 app to `/tmp/tome-backup/Tome.app.bak` +(codesign/byte-identical verified). Confirmed no active session (`/health`: +`isRecording:false`; `/status`: `idle`) before quitting. Quit via +`osascript … quit app "Tome"` (clean exit, ~2 s). `ditto`'d the new build +(same `Tome Self-Signed` identity, so Screen Recording/Mic TCC grants carried +over — confirmed empirically, see below) over `/Applications/Tome.app`. +`defaults write com.dloomis.tome graniteShadowEnabled -bool YES`. Relaunched; +`/health` answered within ~1 s. + +**Deviation — orphan-WAV relocation.** 12 pre-existing crashed-recording WAVs +sat in `~/Library/Application Support/Tome/sessions/` from Nic's own past +sessions (unrelated to this work). `ContentView.checkForOrphanedSessionsOnce()` +runs a blocking `NSAlert.runModal()` on relaunch when orphans are found, which +starves the `@MainActor`-bound API (`/health` hangs) with no way to dismiss it +headlessly. To keep the relaunch unattended-safe, the 12 WAVs were `mv`'d to a +scratch holding dir immediately before quitting Tome and `mv`'d back right +after `/health` confirmed the new process was up (past the once-per-launch +scan). MD5 of all 12 files verified identical before move, after move, and +again at end-of-run — zero data loss, nothing recovered/discarded. This is a +real, pre-existing product gap (a launch-time modal can starve the local API) +worth a follow-up but out of scope here. + +**Live smoke — silent-tap result: needed a display wake, not a volume bump.** +First attempt at output volume 0 failed as a genuine capture start failure, +not muted audio: `[ENGINE-5-FAIL] Failed to start system audio: … CaptureError +error 0` (= `.noDisplay`) immediately on start. Root-caused via unified log +(`log show --predicate 'subsystem == "com.apple.TCC"'`): Screen Recording TCC +was fine (`AUTHREQ_RESULT authValue=2` = allowed, confirmed for +`com.dloomis.tome` against `kTCCServiceScreenCapture`) — the actual cause was +`system_profiler SPDisplaysDataType` showing `Display Asleep: Yes`. +ScreenCaptureKit's `SCShareableContent` returns zero displays while the +built-in display is idle-asleep, which the call-capture system-audio tap +depends on even for audio-only capture. This is expected during real meetings +(display is always awake then) but not at midnight with nobody at the +keyboard — an artifact of unattended testing, not a shadow-transcription bug. +A second attempt at volume 0 (no display change) failed identically, +confirming it wasn't a launch-warm-up race. One retry (per brief) was spent +addressing the diagnosed cause instead of the brief's volume-15 fallback +(which would not have fixed a zero-display condition): woke the display for +~35 s via `caffeinate -u -d -t 35`, re-ran at output volume 0/muted (still +fully silent), then let the timer expire naturally — display returned to +`Display Asleep: Yes` on its own, matching the state found at task start; no +brightness/settings changed. + +- Session `session_2026-07-10_00-06-59` (subject "Task 13 Granite Shadow + Smoke Test Retry2"), call capture, 34 s, volume held at 0 throughout capture + and speech. +- Primary transcript (Whisper large-v3-turbo): 3 real utterances across + "Speaker 2/3/4" (diarization split the second voice in two), text matches + the two spoken passages verbatim modulo casing/punctuation. +- `session_2026-07-10_00-06-59.granite.md` and `.comparison.json` both + appeared in `~/Library/Application Support/Tome/GraniteShadow/`, paired + correctly by session ID. Granite text contains all the distinguishing + content words from both passages ("quarterly planning", "granite shadow + transcription rollout", "budget allocations", "customer onboarding + metrics", "follow-up meeting for next tuesday"). +- Shadow totals from the comparison JSON: 3 segments, 0 errored, 24.36 s + audio, 1.156 s shadow wall-clock → **RTF 0.0474** — matches the Phase 0 + benchmark RTF (~0.05) closely. +- `pgrep llama-server` empty after the job completed — spawn-per-job + lifecycle confirmed on a real session, not just in Phase 0's harness. +- `python3 scripts/granite-shadow-report.py "~/Library/Application + Support/Tome/GraniteShadow" -o /tmp/shadow-smoke-report.html` rendered (3 + segments); report HTML contains the same passage phrases. + +**Verdict: PASS.** Silent (volume-0) system-audio tap capture works and the +full granite shadow pipeline (spawn sidecar → transcribe → compare → write +artifacts → stop sidecar) ran correctly end-to-end on a real installed build, +with the one caveat above (needs an awake display — true of real meetings, +not of this unattended test window). + +**End state confirmed:** Tome running (new build, `graniteShadowEnabled=YES`), +`isRecording:false`, output volume restored to 50/unmuted (pre-task baseline), +display back to idle-asleep (pre-task state), no `llama-server` process, no +stray `say` processes, the 12 pre-existing orphan WAVs untouched (MD5-verified), +two failed-attempt test artifacts (empty transcripts/recordings from the +display-asleep failures) deleted from Nic's vault, the one successful smoke +session's transcript/recording/voiceprints left in place as evidence, +`/tmp/tome-backup/Tome.app.bak` left in place as a rollback point, `git +status` clean except this doc. + +Shadow is live for Friday's meetings; review lands EOD Wednesday 2026-07-15 +via `scripts/granite-shadow-report.py`. + +## Reinstall (final-review fixes, fc8ab15) + +2026-07-10 ~01:10–01:14 local, unattended. The whole-branch final review +landed five fix commits after the live-smoke install (sidecar +orphan-on-quit registry, foreign-server adoption refusal, HTTP-status +handling in post(), `-c 16384 --no-webui` launch args, pairing-key +collision, quitting-gate race) — the running app predated them, so the same +install discipline was repeated on build +`fc8ab1535277e7d216e22dff2bc9caea81134465` (144/144 tests green). + +- **Sidecar-args sanity (pre-install):** manually launched llama-server with + the NEW args (`-m … --mmproj … --host 127.0.0.1 --port 8873 -c 16384 + --no-webui`); /health 200 in ~1 s; `granite_client.py /tmp/probe.wav` → + "the quick brown fox jumps over the lazy dog" in 0.2 s; killed, port 8873 + confirmed free. Note for the Wednesday review: llama.cpp warns + `n_ctx_seq (16384) > n_ctx_train (4096)` and allocates 4 slots of + `n_ctx_slot = 4096` — the args are accepted and functional, but the + effective per-slot context is 4096, not 16384 (and `--no-webui` is + deprecated spelling for `--no-ui`; still honored). +- **Install:** same procedure as the live smoke — verified idle via API, + MD5-relocated the 12 orphan WAVs around the relaunch (restored, + re-verified identical), graceful quit (~1 s), `ditto` install, `/health` + up ~2 s after launch. `graniteShadowEnabled` still 1 (untouched). + Installed CDHash `30c668f6…`, CFBundleVersion `1.4.4-63-gfc8ab15-dirty` + (`-dirty` cosmetic: temporary build-script edit during the build, reverted). +- **No audio smoke this time** (per controller: pipeline shape unchanged and + unit-verified; args covered by the manual sanity above). Volume and display + never touched — volume read-verified at baseline 50/unmuted throughout. +- **Backups rotated:** `/tmp/tome-backup/Tome.app.bak-orig` = original + v1.4.4 (`1.4.4-33-g573341d`), `/tmp/tome-backup/Tome.app.bak` = outgoing + shadow-v1 build (`1.4.4-57-gf879620-dirty`, CDHash `91c07752…`). +- **End state:** Tome running (fc8ab15 build, flag ON), not recording, no + llama-server, orphan WAVs byte-identical, git clean except this note. diff --git a/docs/superpowers/plans/2026-07-09-granite-shadow-transcription.md b/docs/superpowers/plans/2026-07-09-granite-shadow-transcription.md new file mode 100644 index 0000000..672ac75 --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-granite-shadow-transcription.md @@ -0,0 +1,1519 @@ +# Granite Shadow Transcription Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Phase 0 benchmark (granite-speech-4.1-2b via llama.cpp vs Tome's backends on public test sets with references) + hidden-flag shadow transcription of real meetings, per `docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md`. + +**Architecture:** A `llama-server` sidecar (spawn-per-job) serves granite; a Python harness (`scripts/asr-bench/`) measures WER on the leaderboard's ESB test sets against Tome's real backends (via a new ASRBench manifest mode); in Tome, a best-effort shadow phase inside `PostProcessingJob` re-transcribes the same merged diarized segments through granite and writes comparison artifacts. Nothing touches ASRCoordinator/ModelProvisioner/TranscriberModel. + +**Tech Stack:** Swift 6 (SwiftPM, actors, Swift Testing suite as in existing tests), llama.cpp (`llama-server`, mtmd audio), Python 3.11+ via `uv` (datasets/jiwer/transformers for Phase 0; stdlib-only for the report script), bash + curl for setup. + +**DEADLINE:** shadow must be live in Nic's installed Tome by **Friday 2026-07-10 morning** (meetings that day; review EOD Wednesday 2026-07-15). Phase 0 gates block **enabling** the flag, not building. Task 4's compute can run while Tasks 5–12 proceed. + +## Global Constraints + +- Branch: `granite-shadow-transcription`. Commit at the end of every task (`Co-Authored-By: Claude Fable 5 `). +- `swift test` and `swift build` require `DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer`. +- NEVER modify `Tome/Sources/Tome/API/**` (frozen pending Nic/Dan discussion). +- This feature adds NO `TranscriberModel` enum case, no Settings UI, no ModelProvisioner/ASRCoordinator changes. +- UserDefaults domain `com.dloomis.tome`; keys exactly: `graniteShadowEnabled` (Bool), `graniteShadowServerPath` (String), `graniteShadowModelDir` (String), `graniteShadowPort` (Int). +- Model files: IBM official GGUFs from `ibm-granite/granite-speech-4.1-2b-GGUF` (Q8_0 + `mmproj-model-f16.gguf`), stored in `~/Library/Application Support/Tome/Granite/`. llama.cpp ≥ **b9045**. Downloads use **curl** (never URLSession — known HF-CDN failure). +- Request template source of truth: `scripts/asr-bench/granite_request.md` (Task 2). Swift `GraniteRequest` must match it (golden test). +- Shadow artifacts dir: `~/Library/Application Support/Tome/GraniteShadow/`. +- Shadow phase NEVER throws out of `PostProcessingJob`; primary transcript/lifecycle behavior byte-identical when flag off AND when shadow fails. +- Existing 89 tests must stay green after every task. +- `scripts/granite-shadow-report.py` is Python-stdlib-only. `scripts/asr-bench/*` may use uv-managed deps. + +--- + +### Task 1: Setup script + real model download + +**Files:** +- Create: `scripts/setup-granite-shadow.sh` + +**Interfaces:** +- Produces: `llama-server` verified ≥ b9045; `~/Library/Application Support/Tome/Granite/{granite-speech-4.1-2b-Q8_0.gguf, mmproj-model-f16.gguf}` on disk; a proven `llama-server` launch command. + +- [ ] **Step 1: Confirm exact GGUF filenames** (they are constants consumed by Task 5's `ShadowConfig` — if they differ from the names below, update BOTH this script and the `ShadowConfig` filename constants in Task 5 before proceeding): + +Run: `curl -s https://huggingface.co/api/models/ibm-granite/granite-speech-4.1-2b-GGUF | python3 -c "import json,sys; [print(s['rfilename']) for s in json.load(sys.stdin)['siblings']]"` +Expected: a `*Q8_0.gguf` file and `mmproj-model-f16.gguf`. + +- [ ] **Step 2: Write the script** + +```bash +#!/usr/bin/env bash +# Setup for granite shadow transcription (spec: docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md) +set -euo pipefail + +MODEL_DIR="$HOME/Library/Application Support/Tome/Granite" +REPO="https://huggingface.co/ibm-granite/granite-speech-4.1-2b-GGUF/resolve/main" +MODEL="granite-speech-4.1-2b-Q8_0.gguf" # keep in sync with ShadowConfig.modelFilename +MMPROJ="mmproj-model-f16.gguf" # keep in sync with ShadowConfig.mmprojFilename +SERVER="${LLAMA_SERVER:-/opt/homebrew/bin/llama-server}" +PORT="${GRANITE_PORT:-8873}" + +if [[ ! -x "$SERVER" ]]; then + echo "llama-server not found at $SERVER — run: brew install llama.cpp" >&2 + exit 1 +fi +# b9045+ required for granite-speech mtmd support +BUILD=$("$SERVER" --version 2>&1 | grep -oE 'b[0-9]+' | head -1 | tr -d 'b') +if [[ -z "$BUILD" || "$BUILD" -lt 9045 ]]; then + echo "llama.cpp build b${BUILD:-unknown} < b9045 — run: brew upgrade llama.cpp" >&2 + exit 1 +fi + +mkdir -p "$MODEL_DIR" +for f in "$MODEL" "$MMPROJ"; do + echo "Downloading $f (resumable)…" + curl -L -C - --fail -o "$MODEL_DIR/$f" "$REPO/$f" +done +ls -lh "$MODEL_DIR" + +cat < dict`, `transcribe(base_url: str, wav_path: str) -> tuple[str, float]` (text, latency-seconds). + +- [ ] **Step 1: Discover the working request shape** against the live server from Task 1. Try the OpenAI-compatible endpoint first: + +```bash +python3 - <<'EOF' +import base64, json, urllib.request +wav = open("/System/Library/Sounds/Submarine.aiff", "rb") # placeholder — use a real 16 kHz mono WAV, see below +EOF +``` +Use a real speech WAV: record 10 s (`say -o /tmp/probe.aiff "the quick brown fox jumps over the lazy dog" && afconvert -f WAVE -d LEI16@16000 -c 1 /tmp/probe.aiff /tmp/probe.wav`). POST to `http://127.0.0.1:8873/v1/chat/completions`: + +```json +{"messages": [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + {"type": "text", "text": "can you transcribe the speech into a written format?"}]}], + "temperature": 0, "max_tokens": 2048, "stream": false} +``` +Expected: `choices[0].message.content` ≈ "the quick brown fox jumps over the lazy dog" (case/punct may vary). If `input_audio` is rejected, consult `llama-server --help` / llama.cpp `docs/multimodal.md` for the accepted audio content type and record what works. If the server path cannot transcribe at all, STOP: fall back to `llama-mtmd-cli` per the spec's risk section and record that decision in `granite_request.md` (the sidecar then shells out instead of HTTP — adjust Task 9 accordingly). + +- [ ] **Step 2: Write `granite_request.md`** documenting exactly: endpoint path, full JSON body (with prompt string verbatim), required server launch flags, response extraction path (`choices[0].message.content`), and the probe transcript observed. This file is the single source of truth; both Python and Swift cite it. + +- [ ] **Step 3: Write `granite_client.py`** + +```python +"""Granite llama-server client. Request contract: see granite_request.md (source of truth).""" +import base64, json, time, urllib.request + +PROMPT = "can you transcribe the speech into a written format?" # granite_request.md + +def build_request(wav_bytes: bytes, prompt: str = PROMPT) -> dict: + return { + "messages": [{"role": "user", "content": [ + {"type": "input_audio", + "input_audio": {"data": base64.b64encode(wav_bytes).decode(), "format": "wav"}}, + {"type": "text", "text": prompt}, + ]}], + "temperature": 0, "max_tokens": 2048, "stream": False, + } + +def transcribe(base_url: str, wav_path: str) -> tuple[str, float]: + body = json.dumps(build_request(open(wav_path, "rb").read())).encode() + req = urllib.request.Request(f"{base_url}/v1/chat/completions", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.monotonic() + with urllib.request.urlopen(req, timeout=600) as resp: + out = json.load(resp) + return out["choices"][0]["message"]["content"].strip(), time.monotonic() - t0 + +if __name__ == "__main__": + import sys + text, dt = transcribe(sys.argv[1] if len(sys.argv) > 2 else "http://127.0.0.1:8873", sys.argv[-1]) + print(f"[{dt:.1f}s] {text}") +``` +(Adjust `build_request` to whatever Step 1 actually pinned.) + +- [ ] **Step 4: Verify** — `python3 scripts/asr-bench/granite_client.py /tmp/probe.wav` prints the fox sentence. Note the latency: first M2 Max RTF datapoint. +- [ ] **Step 5: Commit** — `git add scripts/asr-bench && git commit -m "feat: pin granite llama-server request template + python client"` + +### Task 3: BenchSupport manifest library + ASRBench manifest mode + +**Files:** +- Create: `Tome/Sources/BenchSupport/BenchManifest.swift` +- Modify: `Tome/Package.swift` (add `BenchSupport` library target; `ASRBench` and `TomeTests` depend on it) +- Modify: `Tome/Sources/ASRBench/main.swift` (manifest mode) +- Test: `Tome/Tests/TomeTests/BenchManifestTests.swift` + +**Interfaces:** +- Produces: `BenchManifest.parse(_ jsonl: String) throws -> [ManifestEntry]` where `ManifestEntry(id: String, wav: String)`; `BenchManifest.emit(_ hyps: [HypothesisEntry]) -> String` where `HypothesisEntry(id: String, text: String)`. CLI: `ASRBench --manifest in.jsonl --backend parakeet|whisper --out hyp.jsonl` (Task 4 consumes). + +- [ ] **Step 1: Failing test** (`BenchManifestTests.swift`, match the existing suite's Swift Testing style): + +```swift +import Testing +@testable import BenchSupport + +@Suite struct BenchManifestTests { + @Test func parsesJSONLAndSkipsBlankLines() throws { + let jsonl = """ + {"id": "ami-0001", "wav": "/tmp/a.wav"} + + {"id": "ami-0002", "wav": "/tmp/b.wav"} + """ + let entries = try BenchManifest.parse(jsonl) + #expect(entries == [ManifestEntry(id: "ami-0001", wav: "/tmp/a.wav"), + ManifestEntry(id: "ami-0002", wav: "/tmp/b.wav")]) + } + @Test func emitRoundTrips() throws { + let hyps = [HypothesisEntry(id: "x", text: "hello there")] + let out = BenchManifest.emit(hyps) + #expect(out == #"{"id":"x","text":"hello there"}"# + "\n") + } + @Test func parseRejectsMalformedLine() { + #expect(throws: (any Error).self) { try BenchManifest.parse("not json") } + } +} +``` + +- [ ] **Step 2: Run** — `cd Tome && DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer swift test --filter BenchManifestTests`. Expected: FAIL (module missing). +- [ ] **Step 3: Implement.** Package.swift: add `.target(name: "BenchSupport")`; add `"BenchSupport"` to the `ASRBench` executable target's and `TomeTests`' dependencies. + +```swift +// Tome/Sources/BenchSupport/BenchManifest.swift +import Foundation + +public struct ManifestEntry: Codable, Equatable, Sendable { + public let id: String + public let wav: String + public init(id: String, wav: String) { self.id = id; self.wav = wav } +} + +public struct HypothesisEntry: Codable, Equatable, Sendable { + public let id: String + public let text: String + public init(id: String, text: String) { self.id = id; self.text = text } +} + +public enum BenchManifest { + public static func parse(_ jsonl: String) throws -> [ManifestEntry] { + try jsonl.split(separator: "\n", omittingEmptySubsequences: true) + .filter { !$0.trimmingCharacters(in: .whitespaces).isEmpty } + .map { try JSONDecoder().decode(ManifestEntry.self, from: Data($0.utf8)) } + } + public static func emit(_ hyps: [HypothesisEntry]) -> String { + let enc = JSONEncoder() + enc.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + return hyps.map { String(data: try! enc.encode($0), encoding: .utf8)! } + .joined(separator: "\n") + (hyps.isEmpty ? "" : "\n") + } +} +``` + +- [ ] **Step 4: Run tests** — same command. Expected: PASS. Then full suite: `swift test` — 89 + 3 green. +- [ ] **Step 5: Manifest mode in `main.swift`.** At the top of the existing top-level code, before the bench runs: + +```swift +// Manifest mode: ASRBench --manifest in.jsonl --backend parakeet|whisper --out hyp.jsonl +// Reuses the same hand-mirrored model config as the bench functions below (keep in sync). +if let mi = CommandLine.arguments.firstIndex(of: "--manifest") { + let args = CommandLine.arguments + guard args.count > mi + 1, + let bi = args.firstIndex(of: "--backend"), args.count > bi + 1, + let oi = args.firstIndex(of: "--out"), args.count > oi + 1 else { + FileHandle.standardError.write(Data("usage: ASRBench --manifest in.jsonl --backend parakeet|whisper --out hyp.jsonl\n".utf8)) + exit(2) + } + let entries = try BenchManifest.parse(String(contentsOfFile: args[mi + 1], encoding: .utf8)) + let backend = args[bi + 1] + // Extract the model-loading half of benchParakeet()/benchWhisper() into + // loadParakeet() / loadWhisper() helpers returning a `(String) async throws -> String` + // transcribe closure (WAV path in, text out), reusing the existing sample-loading + // code these bench functions already use for their own WAVs. + let transcribe: (String) async throws -> String = backend == "whisper" + ? try await loadWhisper() + : try await loadParakeet() + var hyps: [HypothesisEntry] = [] + for (i, e) in entries.enumerated() { + let text = (try? await transcribe(e.wav)) ?? "" + hyps.append(HypothesisEntry(id: e.id, text: text)) + if i % 50 == 0 { print("[\(backend)] \(i)/\(entries.count)") } + } + try BenchManifest.emit(hyps).write(toFile: args[oi + 1], atomically: true, encoding: .utf8) + print("[\(backend)] wrote \(hyps.count) hypotheses → \(args[oi + 1])") + exit(0) +} +``` +Import `BenchSupport` at the top of main.swift. The `loadParakeet`/`loadWhisper` extraction must not change bench behavior — the existing `benchParakeet()`/`benchWhisper()` call the new helpers. + +- [ ] **Step 6: Build + spot-check** — `swift build -c release`; run manifest mode on 2 WAVs (the Task 2 probe file twice) for each backend; verify hyp.jsonl content is sane. +- [ ] **Step 7: Commit** — `git commit -am "feat: ASRBench manifest mode + BenchSupport library"` + +### Task 4: Phase 0 benchmark harness — build AND run + +**Files:** +- Create: `scripts/asr-bench/bench.py` +- Create: `docs/superpowers/plans/2026-07-09-granite-phase0-results.md` (results, produced by running) + +**Interfaces:** +- Consumes: `granite_client.transcribe`, `ASRBench --manifest`. +- Produces: the Phase 0 results doc with the WER table + RTF + go/no-go against the spec's three Phase 0 gates. + +- [ ] **Step 1: Write `bench.py`** (uv inline-deps script; stages so granite/ASRBench runs are restartable): + +```python +# /// script +# requires-python = ">=3.11" +# dependencies = ["datasets[audio]>=3", "soundfile", "jiwer", "transformers", "torch", "numpy"] +# /// +"""Phase 0 ASR benchmark. Stages: + uv run bench.py export --work /tmp/asrbench --sets ami,earnings22,tedlium --max-hours 3 + (then run ASRBench manifest mode for parakeet + whisper — command is printed) + uv run bench.py granite --work /tmp/asrbench --url http://127.0.0.1:8873 + uv run bench.py score --work /tmp/asrbench +Reference/normalizer per Open ASR Leaderboard: WhisperTokenizer._normalize.""" +import argparse, json, pathlib, sys, time + +SETS = {"ami": "ami", "earnings22": "earnings22", "tedlium": "tedlium"} +ESB = "hf-audio/esb-datasets-test-only-sorted" + +def export(work, sets, max_hours): + import soundfile as sf + from datasets import load_dataset, Audio + for s in sets: + d = work / s; (d / "wav").mkdir(parents=True, exist_ok=True) + ds = load_dataset(ESB, SETS[s], split="test", streaming=True) + ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + refcol = next(c for c in ("text", "norm_transcript", "transcription", "sentence") + if c in ds.column_names) + total, manifest, refs = 0.0, [], {} + for i, row in enumerate(ds): + audio = row["audio"]; dur = len(audio["array"]) / audio["sampling_rate"] + if total + dur > max_hours * 3600: break + total += dur + rid = f"{s}-{i:05d}"; wav = d / "wav" / f"{rid}.wav" + sf.write(wav, audio["array"], 16000, subtype="PCM_16") + manifest.append({"id": rid, "wav": str(wav)}); refs[rid] = {"ref": row[refcol], "dur": dur} + (d / "manifest.jsonl").write_text("".join(json.dumps(m) + "\n" for m in manifest)) + (d / "refs.json").write_text(json.dumps(refs)) + print(f"[{s}] {len(manifest)} utts, {total/3600:.2f} h (ref column: {refcol})") + print("\nNow produce Tome-backend hypotheses (from Tome/):") + for s in sets: + for b in ("parakeet", "whisper"): + print(f" DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer swift run -c release ASRBench " + f"--manifest {work}/{s}/manifest.jsonl --backend {b} --out {work}/{s}/hyp_{b}.jsonl") + +def granite(work, url): + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + from granite_client import transcribe + for d in sorted(p for p in work.iterdir() if (p / "manifest.jsonl").exists()): + out, wall, audio_s = [], 0.0, 0.0 + refs = json.loads((d / "refs.json").read_text()) + for line in (d / "manifest.jsonl").read_text().splitlines(): + m = json.loads(line) + try: + text, dt = transcribe(url, m["wav"]) + except Exception as e: # noqa: BLE001 — record and continue + text, dt = "", 0.0; print(f" ERR {m['id']}: {e}") + out.append({"id": m["id"], "text": text}); wall += dt; audio_s += refs[m["id"]]["dur"] + if len(out) % 50 == 0: print(f"[{d.name}] {len(out)} done, RTF so far {wall/max(audio_s,1):.3f}") + (d / "hyp_granite.jsonl").write_text("".join(json.dumps(o) + "\n" for o in out)) + print(f"[{d.name}] granite RTF (single-stream M2 Max): {wall/max(audio_s,1):.3f}") + +def score(work): + import jiwer + from transformers import WhisperTokenizer + tok = WhisperTokenizer.from_pretrained("openai/whisper-tiny") + rows = [] + for d in sorted(p for p in work.iterdir() if (p / "refs.json").exists()): + refs = json.loads((d / "refs.json").read_text()) + for hyp_file in sorted(d.glob("hyp_*.jsonl")): + hyps = {json.loads(l)["id"]: json.loads(l)["text"] for l in hyp_file.read_text().splitlines()} + pairs = [(tok._normalize(refs[i]["ref"]), tok._normalize(hyps.get(i, ""))) + for i in refs if tok._normalize(refs[i]["ref"]).strip()] + wer = jiwer.wer([r for r, _ in pairs], [h for _, h in pairs]) * 100 + rows.append((d.name, hyp_file.stem.removeprefix("hyp_"), wer, len(pairs))) + print(f"{'set':<12}{'backend':<12}{'WER%':>8}{'utts':>7}") + for s, b, w, n in rows: print(f"{s:<12}{b:<12}{w:>8.2f}{n:>7}") + +if __name__ == "__main__": + ap = argparse.ArgumentParser(); ap.add_argument("stage", choices=["export", "granite", "score"]) + ap.add_argument("--work", type=pathlib.Path, required=True) + ap.add_argument("--sets", default="ami,earnings22,tedlium"); ap.add_argument("--max-hours", type=float, default=3) + ap.add_argument("--url", default="http://127.0.0.1:8873") + a = ap.parse_args(); a.work.mkdir(parents=True, exist_ok=True) + {"export": lambda: export(a.work, a.sets.split(","), a.max_hours), + "granite": lambda: granite(a.work, a.url), + "score": lambda: score(a.work)}[a.stage]() +``` +Held-out set note: **TED-LIUM stands in for CORAAL** (short-form, ungated, absent from granite's training list; CORAAL is long-form and needs its own chunking — stretch goal only if time allows). Record this substitution in the results doc. + +- [ ] **Step 2: Run export** (`uv run scripts/asr-bench/bench.py export --work /tmp/asrbench`). Expected: 3 sets × ~3 h exported. If a dataset config name or ref column errors, check `open_asr_leaderboard`'s normalizer/dataset usage on GitHub and fix the constant. +- [ ] **Step 3: Run the two ASRBench manifest commands per set** (printed by export). Parakeet is minutes; Whisper tens of minutes. +- [ ] **Step 4: Run granite stage** (server from Task 1 running). Record per-set RTF lines. +- [ ] **Step 5: Score + write results doc** `docs/superpowers/plans/2026-07-09-granite-phase0-results.md`: the WER table, published leaderboard raw numbers alongside (granite AMI 7.72 / E22 8.23; parakeet-v3 AMI 10.58 / E22 10.77; whisper-turbo AMI 15.16 / E22 11.07), M2 Max RTF, and explicit pass/fail on the spec's three Phase 0 gates (fidelity ±1.5 pts on AMI+E22; RTF ≤ 0.25; granite beats both backends on AMI+E22 and ≥ matches parakeet on TED-LIUM). End with go/no-go for enabling shadow. +- [ ] **Step 6: Commit** — `git add scripts/asr-bench docs/superpowers/plans/2026-07-09-granite-phase0-results.md && git commit -m "feat: phase 0 ASR benchmark harness + results"` + +### Task 5: ShadowConfig + +**Files:** +- Create: `Tome/Sources/Tome/Transcription/ShadowConfig.swift` +- Test: `Tome/Tests/TomeTests/ShadowConfigTests.swift` + +**Interfaces:** +- Produces: `ShadowConfig` (`serverPath: String`, `modelDir: URL`, `port: Int`; `modelGGUF/mmprojGGUF: URL`; `filesPresent() -> Bool`; `static func fromDefaults(_ defaults: UserDefaults) -> ShadowConfig?`). Consumed by Tasks 9–11. + +- [ ] **Step 1: Failing tests** + +```swift +import Foundation +import Testing +@testable import Tome + +@Suite struct ShadowConfigTests { + private func makeDefaults() -> UserDefaults { + let d = UserDefaults(suiteName: "ShadowConfigTests-\(UUID().uuidString)")! + d.removePersistentDomain(forName: d.description) + return d + } + @Test func disabledByDefault() { + #expect(ShadowConfig.fromDefaults(makeDefaults()) == nil) + } + @Test func enabledUsesDefaults() { + let d = makeDefaults(); d.set(true, forKey: "graniteShadowEnabled") + let c = try! #require(ShadowConfig.fromDefaults(d)) + #expect(c.serverPath == "/opt/homebrew/bin/llama-server") + #expect(c.port == 8873) + #expect(c.modelDir.path.hasSuffix("Tome/Granite")) + #expect(c.modelGGUF.lastPathComponent == "granite-speech-4.1-2b-Q8_0.gguf") + #expect(c.mmprojGGUF.lastPathComponent == "mmproj-model-f16.gguf") + } + @Test func overridesRespectedAndTildeExpanded() { + let d = makeDefaults() + d.set(true, forKey: "graniteShadowEnabled") + d.set("/usr/local/bin/llama-server", forKey: "graniteShadowServerPath") + d.set("~/granite-models", forKey: "graniteShadowModelDir") + d.set(9001, forKey: "graniteShadowPort") + let c = try! #require(ShadowConfig.fromDefaults(d)) + #expect(c.serverPath == "/usr/local/bin/llama-server") + #expect(c.port == 9001) + #expect(!c.modelDir.path.contains("~")) + #expect(c.modelDir.path.hasSuffix("/granite-models")) + } + @Test func filesPresentFalseOnEmptyDir() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let c = ShadowConfig(serverPath: "/x", modelDir: tmp, port: 1) + #expect(!c.filesPresent()) + } +} +``` + +- [ ] **Step 2: Run → FAIL.** `swift test --filter ShadowConfigTests` +- [ ] **Step 3: Implement** + +```swift +import Foundation + +/// Hidden-flag configuration for granite shadow transcription. Read at job +/// creation (not app launch) so toggling applies from the next session. +/// Spec: docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md +struct ShadowConfig: Sendable, Equatable { + let serverPath: String + let modelDir: URL + let port: Int + + // Keep in sync with scripts/setup-granite-shadow.sh + static let modelFilename = "granite-speech-4.1-2b-Q8_0.gguf" + static let mmprojFilename = "mmproj-model-f16.gguf" + + var modelGGUF: URL { modelDir.appendingPathComponent(Self.modelFilename) } + var mmprojGGUF: URL { modelDir.appendingPathComponent(Self.mmprojFilename) } + var baseURL: URL { URL(string: "http://127.0.0.1:\(port)")! } + + func filesPresent(fileManager: FileManager = .default) -> Bool { + fileManager.fileExists(atPath: modelGGUF.path) && fileManager.fileExists(atPath: mmprojGGUF.path) + } + + static func fromDefaults(_ defaults: UserDefaults = .standard) -> ShadowConfig? { + guard defaults.bool(forKey: "graniteShadowEnabled") else { return nil } + let server = defaults.string(forKey: "graniteShadowServerPath") ?? "/opt/homebrew/bin/llama-server" + let dir: URL + if let override = defaults.string(forKey: "graniteShadowModelDir") { + dir = URL(fileURLWithPath: (override as NSString).expandingTildeInPath) + } else { + dir = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("Tome/Granite") + } + let port = (defaults.object(forKey: "graniteShadowPort") as? Int) ?? 8873 + return ShadowConfig(serverPath: server, modelDir: dir, port: port) + } +} +``` + +- [ ] **Step 4: Run → PASS**, then full suite green. +- [ ] **Step 5: Commit** — `git commit -am "feat: ShadowConfig (hidden granite shadow flag)"` + +### Task 6: SegmentAudio extraction (merge/pad/read) + SegmentReTranscriber refactor + +**Files:** +- Create: `Tome/Sources/Tome/Transcription/SegmentAudio.swift` +- Modify: `Tome/Sources/Tome/Transcription/SegmentReTranscriber.swift` (use the extracted functions; behavior byte-identical) +- Test: `Tome/Tests/TomeTests/SegmentAudioTests.swift` + +**Interfaces:** +- Produces: `SegmentAudio.merge(_ segments: [DiarizedSegment], gapThreshold: Float = 0.5) -> [DiarizedSegment]`; `SegmentAudio.paddedFrameRange(startTime: Float, endTime: Float, sampleRate: Double, totalFrames: AVAudioFramePosition, minSeconds: Double = 1.5) -> (start: AVAudioFramePosition, count: AVAudioFrameCount)?`; `SegmentAudio.readSegment(file: AVAudioFile, start: AVAudioFramePosition, count: AVAudioFrameCount) -> AVAudioPCMBuffer?`. Both the primary path and Task 10's shadow runner use these — identical audio is the spec's apples-to-apples guarantee. + +- [ ] **Step 1: Failing tests** pinning the EXACT current behavior (transcribed from SegmentReTranscriber.swift:25-56): + +```swift +import AVFoundation +import Testing +@testable import Tome + +@Suite struct SegmentAudioTests { + @Test func mergesSameSpeakerWithinHalfSecond() { + let segs = [ + DiarizedSegment(speakerId: "A", startTime: 0.0, endTime: 1.0), + DiarizedSegment(speakerId: "A", startTime: 1.3, endTime: 2.0), // gap 0.3 < 0.5 → merge + DiarizedSegment(speakerId: "A", startTime: 2.6, endTime: 3.0), // gap 0.6 ≥ 0.5 → new + DiarizedSegment(speakerId: "B", startTime: 3.1, endTime: 4.0), // speaker change → new + ] + let merged = SegmentAudio.merge(segs) + #expect(merged.count == 3) + #expect(merged[0].startTime == 0.0 && merged[0].endTime == 2.0) + #expect(merged[1].startTime == 2.6 && merged[2].speakerId == "B") + } + @Test func padsShortSegmentCentered() { + // 0.5 s segment at 16 kHz in a long file: deficit = 24000-8000 = 16000 → 8000 both sides + let r = SegmentAudio.paddedFrameRange(startTime: 10, endTime: 10.5, sampleRate: 16000, + totalFrames: 10_000_000) + #expect(r! == (start: 152_000, count: 24_000)) + } + @Test func padClampsAtFileStart() { + // Segment at t=0: no room before, pad goes after + let r = SegmentAudio.paddedFrameRange(startTime: 0, endTime: 0.5, sampleRate: 16000, + totalFrames: 10_000_000) + #expect(r! == (start: 0, count: 24_000)) + } + @Test func zeroLengthSegmentIsNil() { + #expect(SegmentAudio.paddedFrameRange(startTime: 5, endTime: 5, sampleRate: 16000, + totalFrames: 80_000) == nil) + } +} +``` +IMPORTANT: before finalizing expected values, re-derive them from the current code (SegmentReTranscriber.swift:44-58) — the tests must encode what the code DOES today (e.g. clamp of `endFrame` to totalFrames, `frameCount > 0` guard returning nil). + +- [ ] **Step 2: Run → FAIL** (`SegmentAudio` undefined). +- [ ] **Step 3: Implement** by MOVING the logic (not copying with edits): + +```swift +import AVFoundation + +/// Segment mechanics shared by the primary re-transcriber and the granite +/// shadow runner — both must see byte-identical audio (spec §4). +enum SegmentAudio { + /// Merge consecutive same-speaker segments separated by < gapThreshold seconds. + static func merge(_ segments: [DiarizedSegment], gapThreshold: Float = 0.5) -> [DiarizedSegment] { + var merged: [DiarizedSegment] = [] + for seg in segments { + if let last = merged.last, last.speakerId == seg.speakerId, + seg.startTime - last.endTime < gapThreshold { + merged[merged.count - 1] = DiarizedSegment( + speakerId: last.speakerId, startTime: last.startTime, endTime: seg.endTime) + } else { + merged.append(seg) + } + } + return merged + } + + /// Frame range for a segment, padded to minSeconds (Parakeet's floor — + /// applied to all backends deliberately; see spec §4) and clamped to the file. + static func paddedFrameRange( + startTime: Float, endTime: Float, sampleRate: Double, + totalFrames: AVAudioFramePosition, minSeconds: Double = 1.5 + ) -> (start: AVAudioFramePosition, count: AVAudioFrameCount)? { + var startFrame = AVAudioFramePosition(Double(startTime) * sampleRate) + var endFrame = min(AVAudioFramePosition(Double(endTime) * sampleRate), totalFrames) + var frameCount = Int(endFrame - startFrame) + let minSamples = Int(sampleRate * minSeconds) + if frameCount < minSamples && frameCount > 0 { + let deficit = minSamples - frameCount + let padBefore = min(AVAudioFramePosition(deficit / 2), startFrame) + let padAfter = min(deficit - Int(padBefore), Int(totalFrames - endFrame)) + startFrame -= padBefore + endFrame += AVAudioFramePosition(padAfter) + frameCount = Int(endFrame - startFrame) + } + guard frameCount > 0 else { return nil } + return (startFrame, AVAudioFrameCount(frameCount)) + } + + /// Read one segment's PCM out of an open file. Nil on allocation/read failure. + static func readSegment(file: AVAudioFile, start: AVAudioFramePosition, + count: AVAudioFrameCount) -> AVAudioPCMBuffer? { + file.framePosition = start + guard let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: count) + else { return nil } + do { try file.read(into: buffer, frameCount: count) } catch { return nil } + return buffer + } +} +``` +Then rewrite `SegmentReTranscriber.run()`'s merge loop and frame math to call these three functions (delete the inlined versions). The `guard !text.isEmpty else continue` and `"[transcription failed]"` conventions stay exactly where they are. + +- [ ] **Step 4: Run → PASS**; full suite green (any existing test touching SegmentReTranscriber must be untouched and green). +- [ ] **Step 5: Commit** — `git commit -am "refactor: extract SegmentAudio merge/pad/read (shared with granite shadow)"` + +### Task 7: AudioWAVExport (buffer → 16 kHz mono PCM16 WAV bytes) + +**Files:** +- Create: `Tome/Sources/Tome/Transcription/AudioWAVExport.swift` +- Test: `Tome/Tests/TomeTests/AudioWAVExportTests.swift` + +**Interfaces:** +- Produces: `AudioWAVExport.wav16kMonoPCM16(from buffer: AVAudioPCMBuffer) throws -> Data`; `AudioWAVExport.riffHeader(dataByteCount: Int) -> Data`. Consumed by Task 10. + +- [ ] **Step 1: Failing tests** + +```swift +import AVFoundation +import Testing +@testable import Tome + +@Suite struct AudioWAVExportTests { + @Test func riffHeaderFields() { + let h = AudioWAVExport.riffHeader(dataByteCount: 32000) + #expect(h.count == 44) + #expect(String(data: h[0..<4], encoding: .ascii) == "RIFF") + #expect(String(data: h[8..<12], encoding: .ascii) == "WAVE") + // chunk size = 36 + data + #expect(h[4..<8].withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) } == 32036) + // sample rate 16000 @ offset 24, channels 1 @ 22, bits 16 @ 34 + #expect(h[24..<28].withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) } == 16000) + #expect(h[22..<24].withUnsafeBytes { $0.loadUnaligned(as: UInt16.self) } == 1) + #expect(h[34..<36].withUnsafeBytes { $0.loadUnaligned(as: UInt16.self) } == 16) + } + @Test func convertsStereo48kToMono16k() throws { + let fmt = AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 2)! + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: 48000)! + buf.frameLength = 48000 // 1 second of silence + let data = try AudioWAVExport.wav16kMonoPCM16(from: buf) + let samples = (data.count - 44) / 2 + #expect(abs(samples - 16000) < 64) // ~1 s at 16 kHz (converter may prime ±) + } +} +``` + +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** + +```swift +import AVFoundation + +/// Converts arbitrary PCM buffers to the 16 kHz mono PCM16 WAV bytes the +/// granite sidecar consumes (granite_request.md pins format: "wav"). +enum AudioWAVExport { + enum ExportError: Error { case formatUnavailable, conversionFailed } + + static func wav16kMonoPCM16(from buffer: AVAudioPCMBuffer) throws -> Data { + guard let outFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16000, + channels: 1, interleaved: true), + let converter = AVAudioConverter(from: buffer.format, to: outFmt) + else { throw ExportError.formatUnavailable } + let ratio = 16000.0 / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 4096 + guard let out = AVAudioPCMBuffer(pcmFormat: outFmt, frameCapacity: capacity) + else { throw ExportError.conversionFailed } + var fed = false + var error: NSError? + converter.convert(to: out, error: &error) { _, status in + if fed { status.pointee = .endOfStream; return nil } + fed = true; status.pointee = .haveData; return buffer + } + if let error { throw error } + let byteCount = Int(out.frameLength) * 2 + var data = riffHeader(dataByteCount: byteCount) + data.append(Data(bytes: out.int16ChannelData![0], count: byteCount)) + return data + } + + static func riffHeader(dataByteCount: Int) -> Data { + var d = Data() + func le32(_ v: UInt32) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } } + func le16(_ v: UInt16) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } } + d.append(contentsOf: "RIFF".utf8); le32(UInt32(36 + dataByteCount)) + d.append(contentsOf: "WAVE".utf8) + d.append(contentsOf: "fmt ".utf8); le32(16); le16(1) /* PCM */; le16(1) /* mono */ + le32(16000); le32(16000 * 2) /* byte rate */; le16(2) /* block align */; le16(16) + d.append(contentsOf: "data".utf8); le32(UInt32(dataByteCount)) + return d + } +} +``` + +- [ ] **Step 4: Run → PASS**; full suite green. +- [ ] **Step 5: Commit** — `git commit -am "feat: AudioWAVExport (16 kHz mono PCM16 WAV for granite sidecar)"` + +### Task 8: GraniteRequest (build/parse, golden-matched to granite_request.md) + +**Files:** +- Create: `Tome/Sources/Tome/Transcription/GraniteRequest.swift` +- Test: `Tome/Tests/TomeTests/GraniteRequestTests.swift` + +**Interfaces:** +- Consumes: the pinned template in `scripts/asr-bench/granite_request.md` (Task 2). +- Produces: `GraniteRequest.prompt: String`; `GraniteRequest.build(wavData: Data) -> Data` (JSON body); `GraniteRequest.parseResponse(_ data: Data) throws -> String`. Consumed by Task 9. + +- [ ] **Step 1: Failing tests.** The golden test decodes the built body and asserts every field the template pins (adapt to what Task 2 actually recorded — the values below assume the OpenAI-compatible shape): + +```swift +import Foundation +import Testing +@testable import Tome + +@Suite struct GraniteRequestTests { + @Test func buildMatchesPinnedTemplate() throws { + // Golden contract: scripts/asr-bench/granite_request.md + let wav = Data([0x52, 0x49, 0x46, 0x46]) // "RIFF" + let body = try JSONSerialization.jsonObject(with: GraniteRequest.build(wavData: wav)) as! [String: Any] + #expect(body["temperature"] as? Double == 0) + #expect(body["max_tokens"] as? Int == 2048) + #expect(body["stream"] as? Bool == false) + let msgs = body["messages"] as! [[String: Any]] + #expect(msgs.count == 1 && msgs[0]["role"] as? String == "user") + let content = msgs[0]["content"] as! [[String: Any]] + let audio = content[0]["input_audio"] as! [String: Any] + #expect(audio["format"] as? String == "wav") + #expect(audio["data"] as? String == wav.base64EncodedString()) + #expect(content[1]["text"] as? String == GraniteRequest.prompt) + #expect(GraniteRequest.prompt == "can you transcribe the speech into a written format?") + } + @Test func parseExtractsContent() throws { + let json = #"{"choices":[{"message":{"role":"assistant","content":" hello world \n"}}]}"# + #expect(try GraniteRequest.parseResponse(Data(json.utf8)) == "hello world") + } + @Test func parseThrowsOnMalformed() { + #expect(throws: (any Error).self) { + try GraniteRequest.parseResponse(Data(#"{"error":"boom"}"#.utf8)) + } + } +} +``` + +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** + +```swift +import Foundation + +/// Builds/parses granite llama-server requests. The contract is pinned in +/// scripts/asr-bench/granite_request.md — Phase 0 validated it; change both +/// together or not at all. +enum GraniteRequest { + static let prompt = "can you transcribe the speech into a written format?" + static let endpointPath = "/v1/chat/completions" + + enum ParseError: Error { case unexpectedShape } + + static func build(wavData: Data) -> Data { + let body: [String: Any] = [ + "messages": [[ + "role": "user", + "content": [ + ["type": "input_audio", + "input_audio": ["data": wavData.base64EncodedString(), "format": "wav"]], + ["type": "text", "text": prompt], + ], + ]], + "temperature": 0, "max_tokens": 2048, "stream": false, + ] + return try! JSONSerialization.data(withJSONObject: body) + } + + static func parseResponse(_ data: Data) throws -> String { + guard let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let choices = obj["choices"] as? [[String: Any]], + let message = choices.first?["message"] as? [String: Any], + let content = message["content"] as? String + else { throw ParseError.unexpectedShape } + return content.trimmingCharacters(in: .whitespacesAndNewlines) + } +} +``` + +- [ ] **Step 4: Run → PASS**; full suite green. +- [ ] **Step 5: Commit** — `git commit -am "feat: GraniteRequest pinned to granite_request.md template"` + +### Task 9: GraniteSidecar actor (process lifecycle state machine) + +**Files:** +- Create: `Tome/Sources/Tome/Transcription/GraniteSidecar.swift` +- Test: `Tome/Tests/TomeTests/GraniteSidecarTests.swift` (+ fakes in the test file) + +**Interfaces:** +- Consumes: `ShadowConfig` (Task 5), `GraniteRequest` (Task 8). +- Produces: `actor GraniteSidecar` — `init(config: ShadowConfig, launcher: any SidecarProcessLauncher = DefaultProcessLauncher(), http: any SidecarHTTP = URLSessionSidecarHTTP(), readyTimeout: TimeInterval = 60, sleep: @Sendable (TimeInterval) async -> Void = { try? await Task.sleep(for: .seconds($0)) })`; `func start() async -> Bool`; `func transcribe(wavData: Data) async throws -> String`; `func stop() async`. Protocols: `SidecarProcess` (`isRunning: Bool`, `terminate()`, `forceKill()`), `SidecarProcessLauncher` (`launch(executable: URL, arguments: [String]) throws -> any SidecarProcess`), `SidecarHTTP` (`healthStatus(_ url: URL) async -> Int?`, `post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> Data`). Consumed by Task 10. + +- [ ] **Step 1: Failing tests** (fakes included; follow FakeBackend's style): + +```swift +import Foundation +import Testing +@testable import Tome + +final class FakeProcess: SidecarProcess, @unchecked Sendable { + var running = true + var terminated = false, killed = false + var isRunning: Bool { running } + func terminate() { terminated = true; running = false } + func forceKill() { killed = true; running = false } +} + +final class FakeLauncher: SidecarProcessLauncher, @unchecked Sendable { + var launched: [(URL, [String])] = [] + var processes: [FakeProcess] = [] + var launchError: (any Error)? + func launch(executable: URL, arguments: [String]) throws -> any SidecarProcess { + if let launchError { throw launchError } + launched.append((executable, arguments)) + let p = FakeProcess(); processes.append(p); return p + } +} + +final class FakeHTTP: SidecarHTTP, @unchecked Sendable { + var healthResults: [Int?] = [200] + var postResults: [Result] = [] + func healthStatus(_ url: URL) async -> Int? { + healthResults.isEmpty ? 200 : healthResults.removeFirst() + } + func post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> Data { + try postResults.removeFirst().get() + } +} + +private func makeSidecar(launcher: FakeLauncher = FakeLauncher(), http: FakeHTTP = FakeHTTP()) + -> (GraniteSidecar, FakeLauncher, FakeHTTP) { + let config = ShadowConfig(serverPath: "/fake/llama-server", + modelDir: URL(fileURLWithPath: "/fake/models"), port: 9999) + let s = GraniteSidecar(config: config, launcher: launcher, http: http, + readyTimeout: 1, sleep: { _ in }) + return (s, launcher, http) +} + +private let ok = Data(#"{"choices":[{"message":{"content":"hi"}}]}"#.utf8) +private struct ConnErr: Error {} + +@Suite struct GraniteSidecarTests { + @Test func startLaunchesWithConfigArgsAndPollsHealth() async { + let (s, launcher, http) = makeSidecar() + http.healthResults = [503, 200] + #expect(await s.start()) + let (exe, args) = launcher.launched[0] + #expect(exe.path == "/fake/llama-server") + #expect(args.contains("--port") && args.contains("9999") && args.contains("127.0.0.1")) + #expect(args.contains("/fake/models/\(ShadowConfig.modelFilename)")) + } + @Test func startFailsAfterTimeoutAndKills() async { + let (s, launcher, http) = makeSidecar() + http.healthResults = Array(repeating: 503 as Int?, count: 500) + #expect(await s.start() == false) + #expect(launcher.processes[0].terminated || launcher.processes[0].killed) + } + @Test func transcribeSendsRequestAndParses() async throws { + let (s, _, http) = makeSidecar() + http.postResults = [.success(ok)] + _ = await s.start() + #expect(try await s.transcribe(wavData: Data([1])) == "hi") + } + @Test func connectionFailureRelaunchesOnceThenFails() async { + let (s, launcher, http) = makeSidecar() + http.postResults = [.failure(ConnErr()), .failure(ConnErr())] + _ = await s.start() + await #expect(throws: (any Error).self) { try await s.transcribe(wavData: Data([1])) } + #expect(launcher.launched.count == 2) // original + one relaunch + // subsequent calls fail fast without further launches + await #expect(throws: (any Error).self) { try await s.transcribe(wavData: Data([1])) } + #expect(launcher.launched.count == 2) + } + @Test func stopTerminatesProcess() async { + let (s, launcher, _) = makeSidecar() + _ = await s.start() + await s.stop() + #expect(launcher.processes[0].terminated) + } +} +``` + +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** + +```swift +import Foundation + +protocol SidecarProcess: Sendable { + var isRunning: Bool { get } + func terminate() + func forceKill() +} + +protocol SidecarProcessLauncher: Sendable { + func launch(executable: URL, arguments: [String]) throws -> any SidecarProcess +} + +protocol SidecarHTTP: Sendable { + func healthStatus(_ url: URL) async -> Int? + func post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> Data +} + +/// Owns one llama-server child process, spawn-per-job (spec §3): ~4 GB of +/// model RAM stays off the machine between jobs. One relaunch on connection +/// failure; a second failure fails the phase. +actor GraniteSidecar { + enum State: Equatable { case idle, ready, failed } + enum SidecarError: Error { case notReady, requestFailed } + + private let config: ShadowConfig + private let launcher: any SidecarProcessLauncher + private let http: any SidecarHTTP + private let readyTimeout: TimeInterval + private let sleep: @Sendable (TimeInterval) async -> Void + private var process: (any SidecarProcess)? + private var didRelaunch = false + private(set) var state: State = .idle + + init(config: ShadowConfig, + launcher: any SidecarProcessLauncher = DefaultProcessLauncher(), + http: any SidecarHTTP = URLSessionSidecarHTTP(), + readyTimeout: TimeInterval = 60, + sleep: @Sendable @escaping (TimeInterval) async -> Void = { try? await Task.sleep(for: .seconds($0)) }) { + self.config = config + self.launcher = launcher + self.http = http + self.readyTimeout = readyTimeout + self.sleep = sleep + } + + @discardableResult + func start() async -> Bool { + do { + process = try launcher.launch( + executable: URL(fileURLWithPath: config.serverPath), + arguments: ["-m", config.modelGGUF.path, + "--mmproj", config.mmprojGGUF.path, + "--host", "127.0.0.1", + "--port", String(config.port)]) + } catch { + diagLog("[SHADOW] sidecar launch failed: \(error)") + state = .failed + return false + } + let deadline = readyTimeout / 0.5 + for _ in 0.. String { + guard state == .ready else { throw SidecarError.notReady } + let url = config.baseURL.appendingPathComponent( + GraniteRequest.endpointPath.trimmingCharacters(in: CharacterSet(charactersIn: "/"))) + let body = GraniteRequest.build(wavData: wavData) + do { + return try GraniteRequest.parseResponse(try await http.post(url, body: body, timeout: 600)) + } catch { + guard !didRelaunch else { + diagLog("[SHADOW] request failed after relaunch — failing sidecar: \(error)") + endProcess() + state = .failed + throw SidecarError.requestFailed + } + diagLog("[SHADOW] request failed (\(error)) — relaunching sidecar once") + didRelaunch = true + endProcess() + guard await start() else { throw SidecarError.requestFailed } + return try GraniteRequest.parseResponse(try await http.post(url, body: body, timeout: 600)) + } + } + + func stop() async { + endProcess() + state = .idle + } + + private func endProcess() { + guard let p = process else { return } + p.terminate() + // Escalation handled synchronously in DefaultProcessLauncher's process + // wrapper (terminate → 5 s grace in a detached task → forceKill). + if p.isRunning { p.forceKill() } + process = nil + } +} + +// MARK: - Real implementations + +struct DefaultProcessLauncher: SidecarProcessLauncher { + func launch(executable: URL, arguments: [String]) throws -> any SidecarProcess { + let p = Process() + p.executableURL = executable + p.arguments = arguments + p.standardOutput = FileHandle.nullDevice + p.standardError = FileHandle.nullDevice + try p.run() + return RealSidecarProcess(process: p) + } +} + +/// Wraps Process; forceKill sends SIGKILL. A leaked llama-server must not +/// outlive Tome: Process children die with the parent only if killed, so +/// terminationHandler is not enough — the shadow phase's defer + this +/// wrapper's deinit both call terminate. +final class RealSidecarProcess: SidecarProcess, @unchecked Sendable { + private let process: Process + init(process: Process) { self.process = process } + var isRunning: Bool { process.isRunning } + func terminate() { if process.isRunning { process.terminate() } } + func forceKill() { if process.isRunning { kill(process.processIdentifier, SIGKILL) } } + deinit { if process.isRunning { process.terminate() } } +} + +struct URLSessionSidecarHTTP: SidecarHTTP { + func healthStatus(_ url: URL) async -> Int? { + var req = URLRequest(url: url) + req.timeoutInterval = 2 + guard let (_, resp) = try? await URLSession.shared.data(for: req) else { return nil } + return (resp as? HTTPURLResponse)?.statusCode + } + func post(_ url: URL, body: Data, timeout: TimeInterval) async throws -> Data { + var req = URLRequest(url: url) + req.httpMethod = "POST" + req.httpBody = body + req.timeoutInterval = timeout + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + let (data, _) = try await URLSession.shared.data(for: req) + return data + } +} +``` +Note: localhost URLSession is fine — the known HF-CDN URLSession issue is remote-CDN-specific; downloads still use curl. + +- [ ] **Step 4: Run → PASS**; full suite green. +- [ ] **Step 5: Commit** — `git commit -am "feat: GraniteSidecar actor (spawn-per-job llama-server lifecycle)"` + +### Task 10: Shadow runner + artifacts + +**Files:** +- Create: `Tome/Sources/Tome/Transcription/GraniteShadow.swift` (SegmentTranscribing, ShadowRunner, artifact builders, GraniteShadowPhase) +- Test: `Tome/Tests/TomeTests/GraniteShadowTests.swift` + +**Interfaces:** +- Consumes: `SegmentAudio` (Task 6), `AudioWAVExport` (Task 7), `GraniteSidecar` (Task 9), `ShadowConfig` (Task 5), `DiarizedSegment`/`ReTranscribedSegment` (existing). +- Produces: `protocol SegmentTranscribing: Sendable { func transcribe(buffer: AVAudioPCMBuffer) async throws -> String }`; `struct GraniteSidecarTranscriber: SegmentTranscribing` (wraps sidecar via AudioWAVExport); `ShadowRunner.run(fileURL: URL, diarSegments: [DiarizedSegment], speakerNumberBase: Int) async -> ShadowRunOutput` where `ShadowRunOutput(segments: [ShadowSegment], incomplete: Bool)` and `ShadowSegment(startTime: Float, speaker: String, durationSec: Double, text: String?, error: String?, latencySec: Double)`; `ShadowArtifacts.write(session: ShadowSessionInfo, primary: [ReTranscribedSegment], shadow: ShadowRunOutput, to dir: URL) throws -> (md: URL, json: URL)` with `ShadowSessionInfo(sessionID: String, transcriptPath: String, sessionType: String, primaryModel: String, graniteModel: String)`; `GraniteShadowPhase.run(config:bufferURL:diarSegments:speakerNumberBase:primary:session:) async` (never throws). `GraniteShadowPhase.shouldRun(config: ShadowConfig?, didRebuild: Bool, primary: [ReTranscribedSegment]?) -> Bool` (pure policy). Task 11 consumes `GraniteShadowPhase`. + +- [ ] **Step 1: Failing tests** — policy, runner (fake transcriber), pairing, artifacts: + +```swift +import AVFoundation +import Foundation +import Testing +@testable import Tome + +final class FakeSegmentTranscriber: SegmentTranscribing, @unchecked Sendable { + var results: [Result] + init(_ results: [Result]) { self.results = results } + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String { + try results.removeFirst().get() + } +} +private struct Boom: Error {} + +@Suite struct GraniteShadowTests { + // -- policy -- + @Test func shouldRunRequiresConfigRebuildAndResults() { + let cfg = ShadowConfig(serverPath: "/x", modelDir: URL(fileURLWithPath: "/x"), port: 1) + let seg = [ReTranscribedSegment(speaker: "Speaker 2", text: "hi", startTime: 0)] + #expect(GraniteShadowPhase.shouldRun(config: cfg, didRebuild: true, primary: seg)) + #expect(!GraniteShadowPhase.shouldRun(config: nil, didRebuild: true, primary: seg)) + #expect(!GraniteShadowPhase.shouldRun(config: cfg, didRebuild: false, primary: seg)) + #expect(!GraniteShadowPhase.shouldRun(config: cfg, didRebuild: true, primary: nil)) + #expect(!GraniteShadowPhase.shouldRun(config: cfg, didRebuild: true, primary: [])) + } + // -- runner: uses a real tiny WAV fixture so SegmentAudio paths execute -- + private func fixtureWAV() throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("shadow-\(UUID().uuidString).wav") + let fmt = AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1)! + let file = try AVAudioFile(forWriting: url, settings: fmt.settings) + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: 16000 * 10)! + buf.frameLength = 16000 * 10 // 10 s silence + try file.write(from: buf) + return url + } + @Test func runnerProducesResultPerMergedSegmentIncludingErrors() async throws { + let wav = try fixtureWAV() + let segs = [DiarizedSegment(speakerId: "S0", startTime: 0.0, endTime: 2.0), + DiarizedSegment(speakerId: "S1", startTime: 3.0, endTime: 5.0)] + let runner = ShadowRunner(transcriber: FakeSegmentTranscriber([.success("hello"), .failure(Boom())])) + let out = await runner.run(fileURL: wav, diarSegments: segs, speakerNumberBase: 2) + #expect(out.segments.count == 2) + #expect(out.segments[0].text == "hello" && out.segments[0].error == nil) + #expect(out.segments[1].text == nil && out.segments[1].error != nil) + #expect(!out.incomplete) + } + // -- pairing + artifacts -- + @Test func artifactsPairByStartTimeAndHandleMissingPrimary() throws { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let session = ShadowSessionInfo(sessionID: "s1", transcriptPath: "/t.md", + sessionType: "callCapture", + primaryModel: "Parakeet-TDT v3", + graniteModel: "granite-speech-4.1-2b-Q8_0") + // primary skipped the 3.0 segment (empty text) — granite has it + let primary = [ReTranscribedSegment(speaker: "Speaker 2", text: "hi there", startTime: 0.0)] + let shadow = ShadowRunOutput(segments: [ + ShadowSegment(startTime: 0.0, speaker: "Speaker 2", durationSec: 2, text: "hi there friend", error: nil, latencySec: 0.5), + ShadowSegment(startTime: 3.0, speaker: "Speaker 3", durationSec: 2, text: "quarterly numbers", error: nil, latencySec: 0.4), + ], incomplete: false) + let (md, json) = try ShadowArtifacts.write(session: session, primary: primary, shadow: shadow, to: dir) + let comparison = try JSONDecoder().decode(ShadowComparison.self, from: Data(contentsOf: json)) + #expect(comparison.segments.count == 2) + #expect(comparison.segments[0].primaryText == "hi there") + #expect(comparison.segments[1].primaryText == "") // "" for missing side (spec §4) + #expect(comparison.segments[1].graniteText == "quarterly numbers") + #expect(comparison.totals.segmentCount == 2 && comparison.totals.erroredCount == 0) + let mdText = try String(contentsOf: md, encoding: .utf8) + #expect(mdText.contains("Speaker 3: quarterly numbers")) + #expect(mdText.contains("granite-speech-4.1-2b-Q8_0")) + } +} +``` + +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** in `GraniteShadow.swift`: + +```swift +import AVFoundation +import Foundation + +protocol SegmentTranscribing: Sendable { + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String +} + +/// Bridges the sidecar into the per-segment loop: buffer → 16 kHz WAV → HTTP. +struct GraniteSidecarTranscriber: SegmentTranscribing { + let sidecar: GraniteSidecar + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String { + try await sidecar.transcribe(wavData: try AudioWAVExport.wav16kMonoPCM16(from: buffer)) + } +} + +struct ShadowSegment: Codable, Sendable, Equatable { + let startTime: Float + let speaker: String + let durationSec: Double + let text: String? + let error: String? + let latencySec: Double +} + +struct ShadowRunOutput: Sendable { + let segments: [ShadowSegment] + let incomplete: Bool +} + +/// Runs granite over the SAME merged segments the primary path used +/// (SegmentAudio guarantees identical audio — spec §4). Unlike +/// SegmentReTranscriber, errors are recorded per segment, not placeholdered. +struct ShadowRunner: Sendable { + let transcriber: any SegmentTranscribing + + func run(fileURL: URL, diarSegments: [DiarizedSegment], speakerNumberBase: Int) async -> ShadowRunOutput { + let audioFile: AVAudioFile + do { audioFile = try AVAudioFile(forReading: fileURL) } catch { + diagLog("[SHADOW] cannot open \(fileURL.lastPathComponent): \(error)") + return ShadowRunOutput(segments: [], incomplete: true) + } + let sampleRate = audioFile.processingFormat.sampleRate + let totalFrames = AVAudioFramePosition(audioFile.length) + let merged = SegmentAudio.merge(diarSegments) + let speakerMap = speakerLabels(from: merged.map(\.speakerId), startingAt: speakerNumberBase) + var results: [ShadowSegment] = [] + var incomplete = false + let clock = ContinuousClock() + for seg in merged { + if Task.isCancelled { incomplete = true; break } + let speaker = speakerMap[seg.speakerId] ?? "Speaker \(speakerNumberBase)" + let duration = Double(seg.endTime - seg.startTime) + guard let range = SegmentAudio.paddedFrameRange( + startTime: seg.startTime, endTime: seg.endTime, + sampleRate: sampleRate, totalFrames: totalFrames), + let buffer = SegmentAudio.readSegment(file: audioFile, start: range.start, count: range.count) + else { + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: nil, + error: "segment read failed", latencySec: 0)) + continue + } + let t0 = clock.now + do { + let text = try await transcriber.transcribe(buffer: buffer) + .trimmingCharacters(in: .whitespacesAndNewlines) + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: text, error: nil, + latencySec: Double(truncating: (clock.now - t0) / .seconds(1) as NSNumber))) + } catch { + results.append(ShadowSegment(startTime: seg.startTime, speaker: speaker, + durationSec: duration, text: nil, + error: String(describing: error), + latencySec: Double(truncating: (clock.now - t0) / .seconds(1) as NSNumber))) + if error is GraniteSidecar.SidecarError, case GraniteSidecar.SidecarError.notReady = error { + incomplete = true; break // sidecar dead — stop burning segments + } + } + } + return ShadowRunOutput(segments: results, incomplete: incomplete) + } +} +``` +(If `speakerLabels(from:startingAt:)` is private to SegmentReTranscriber's file, make it internal — it's already Tome-module-internal logic. `Duration`→seconds: use `Double(components.seconds) + Double(components.attoseconds) * 1e-18` if the NSNumber cast doesn't compile.) + +```swift +struct ShadowSessionInfo: Codable, Sendable { + let sessionID: String + let transcriptPath: String + let sessionType: String + let primaryModel: String + let graniteModel: String +} + +struct ShadowComparisonSegment: Codable, Sendable { + let startTime: Float + let speaker: String + let durationSec: Double + let primaryText: String + let graniteText: String + let graniteError: String? + let graniteLatencySec: Double +} + +struct ShadowComparisonTotals: Codable, Sendable { + let segmentCount: Int + let erroredCount: Int + let audioSeconds: Double + let shadowWallClockSec: Double + let rtf: Double +} + +struct ShadowComparison: Codable, Sendable { + let session: ShadowSessionInfo + let incomplete: Bool + let segments: [ShadowComparisonSegment] + let totals: ShadowComparisonTotals +} + +enum ShadowArtifacts { + static func defaultDirectory() -> URL { + FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("Tome/GraniteShadow") + } + + static func write(session: ShadowSessionInfo, primary: [ReTranscribedSegment], + shadow: ShadowRunOutput, to dir: URL) throws -> (md: URL, json: URL) { + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + // Pair by merged-segment startTime (spec §4: primary skips empty-text + // segments, so array positions don't line up; "" marks a missing side). + let primaryByStart = Dictionary(primary.map { ($0.startTime, $0.text) }, + uniquingKeysWith: { a, _ in a }) + let segments = shadow.segments.map { s in + ShadowComparisonSegment(startTime: s.startTime, speaker: s.speaker, + durationSec: s.durationSec, + primaryText: primaryByStart[s.startTime] ?? "", + graniteText: s.text ?? "", + graniteError: s.error, + graniteLatencySec: s.latencySec) + } + let audioSeconds = shadow.segments.reduce(0) { $0 + $1.durationSec } + let wall = shadow.segments.reduce(0) { $0 + $1.latencySec } + let comparison = ShadowComparison( + session: session, incomplete: shadow.incomplete, segments: segments, + totals: ShadowComparisonTotals(segmentCount: segments.count, + erroredCount: shadow.segments.filter { $0.error != nil }.count, + audioSeconds: audioSeconds, + shadowWallClockSec: wall, + rtf: audioSeconds > 0 ? wall / audioSeconds : 0)) + let jsonURL = dir.appendingPathComponent("\(session.sessionID).comparison.json") + let enc = JSONEncoder() + enc.outputFormatting = [.prettyPrinted, .sortedKeys] + try enc.encode(comparison).write(to: jsonURL, options: .atomic) + + var md = """ + # Granite shadow transcript — \(session.sessionID) + - Primary model: \(session.primaryModel) + - Shadow model: \(session.graniteModel) + - Segments: \(segments.count) (\(comparison.totals.erroredCount) errored)\(shadow.incomplete ? " — INCOMPLETE" : "") + - Shadow RTF: \(String(format: "%.3f", comparison.totals.rtf)) + + """ + for s in shadow.segments where s.text?.isEmpty == false { + md += "\(s.speaker): \(s.text!)\n\n" + } + let mdURL = dir.appendingPathComponent("\(session.sessionID).granite.md") + try md.write(to: mdURL, atomically: true, encoding: .utf8) + return (mdURL, jsonURL) + } +} + +/// Best-effort orchestration — the ONLY entry point PostProcessingJob calls. +/// Never throws; every failure is a diagLog + recorded artifact state. +enum GraniteShadowPhase { + static func shouldRun(config: ShadowConfig?, didRebuild: Bool, + primary: [ReTranscribedSegment]?) -> Bool { + guard let config, didRebuild, let primary, !primary.isEmpty else { return false } + _ = config + return true + } + + static func run(config: ShadowConfig, bufferURL: URL, diarSegments: [DiarizedSegment], + speakerNumberBase: Int, primary: [ReTranscribedSegment], + session: ShadowSessionInfo, + outputDir: URL = ShadowArtifacts.defaultDirectory(), + sidecar: GraniteSidecar? = nil) async { + guard FileManager.default.isExecutableFile(atPath: config.serverPath) else { + diagLog("[SHADOW] llama-server missing at \(config.serverPath) — skipping (run scripts/setup-granite-shadow.sh)") + return + } + guard config.filesPresent() else { + diagLog("[SHADOW] model files missing in \(config.modelDir.path) — skipping (run scripts/setup-granite-shadow.sh)") + return + } + let sc = sidecar ?? GraniteSidecar(config: config) + diagLog("[SHADOW] starting sidecar for \(session.sessionID) (\(diarSegments.count) diar segments)") + guard await sc.start() else { + diagLog("[SHADOW] sidecar failed to start — skipping session \(session.sessionID)") + return + } + let output = await ShadowRunner(transcriber: GraniteSidecarTranscriber(sidecar: sc)) + .run(fileURL: bufferURL, diarSegments: diarSegments, speakerNumberBase: speakerNumberBase) + await sc.stop() + do { + let (md, json) = try ShadowArtifacts.write(session: session, primary: primary, + shadow: output, to: outputDir) + diagLog("[SHADOW] wrote \(md.lastPathComponent) + \(json.lastPathComponent) (\(output.segments.count) segments, incomplete=\(output.incomplete))") + } catch { + diagLog("[SHADOW] artifact write failed (non-fatal): \(error)") + } + } +} +``` + +- [ ] **Step 4: Run → PASS**; full suite green. +- [ ] **Step 5: Commit** — `git commit -am "feat: granite shadow runner, artifacts, phase orchestration"` + +### Task 11: PostProcessingJob wiring + +**Files:** +- Modify: `Tome/Sources/Tome/Transcription/PostProcessingJob.swift` +- Test: extend `Tome/Tests/TomeTests/GraniteShadowTests.swift` (policy coverage already there; this task's safety is the no-behavior-change property of the full suite) + +**Interfaces:** +- Consumes: `GraniteShadowPhase` (Task 10), `ShadowConfig` (Task 5). +- Produces: shadow runs on real sessions when flag on. NO signature changes visible to callers (default parameter). + +- [ ] **Step 1: Modify `PostProcessingJob`:** + 1. Add stored property + init parameter with default (evaluated at job creation — exactly the spec's "read at job creation" semantics, zero call-site changes): + ```swift + let shadowConfig: ShadowConfig? + init(handle: SessionHandle, clusterThreshold: Float, numberOfSpeakers: Int, + retention: RecordingRetentionConfig? = nil, exportVoiceprints: Bool = false, + shadowConfig: ShadowConfig? = ShadowConfig.fromDefaults()) { + ...existing assignments... + self.shadowConfig = shadowConfig + } + ``` + 2. In `run(using:)`, hoist the re-transcription results so the shadow phase can see them: before the `if let bufferURL = diarBufferURL {` block add `var primaryResults: [ReTranscribedSegment]? = nil`; inside, where `let results = await TranscriptionEngine.reTranscribe(...)` is assigned, add `primaryResults = results`. Keep `didRebuildSpeakers` as the `didRebuild` signal. + 3. Insert the shadow phase AFTER the voiceprint block (after the `if exportVoiceprints { ... }` closing brace) and BEFORE the retention comment `// 3. Retain the combined recording…`: + ```swift + // 2c. Granite shadow transcription (hidden flag; spec 2026-07-09). + // Best-effort and additive: runs while the capture WAVs still exist, + // never throws, never touches the primary transcript or cleanup. + if GraniteShadowPhase.shouldRun(config: shadowConfig, didRebuild: didRebuildSpeakers, + primary: primaryResults), + let bufferURL = diarBufferURL, let diar = diarOutput { + let speakerBase = handle.sessionType == .callCapture ? 2 : 1 + await GraniteShadowPhase.run( + config: shadowConfig!, bufferURL: bufferURL, diarSegments: diar.segments, + speakerNumberBase: speakerBase, primary: primaryResults!, + session: ShadowSessionInfo( + sessionID: id, + transcriptPath: savedPath.path, + sessionType: String(describing: handle.sessionType), + primaryModel: await asr.activeModel?.displayName ?? "unknown", + graniteModel: ShadowConfig.modelFilename)) + } + ``` + Note `speakerBase` re-derivation must match the switch at the top of `run` (callCapture → 2, voiceMemo → 1) — or better, hoist the existing `speakerBase` local so it's in scope here (it already is: it's declared before the diarization block — verify and reuse it instead of re-deriving). +- [ ] **Step 2: Full suite** — `swift test`: everything green (no existing test constructs PostProcessingJob; the default parameter keeps call sites source-compatible — verify with `swift build`). +- [ ] **Step 3: Flag-off no-op check** — `grep -n "GraniteShadowPhase\|shadowConfig" Tome/Sources/Tome/Transcription/PostProcessingJob.swift`: the ONLY behavioral entry is guarded by `shouldRun`, which requires a non-nil config, which requires `graniteShadowEnabled=true`. +- [ ] **Step 4: Commit** — `git commit -am "feat: wire granite shadow phase into PostProcessingJob (hidden flag)"` + +### Task 12: Shadow comparison report script + +**Files:** +- Create: `scripts/granite-shadow-report.py` (stdlib only) +- Create: `scripts/tests/test_shadow_report.py` (stdlib unittest + fixture inline) + +**Interfaces:** +- Consumes: `*.comparison.json` files (Task 10's `ShadowComparison` schema). +- Produces: `report.html` — aggregate stats + per-session side-by-side with word-diff highlighting, highest-disagreement first. + +- [ ] **Step 1: Failing test** + +```python +import json, pathlib, subprocess, sys, tempfile, unittest + +SAMPLE = { + "session": {"sessionID": "s1", "transcriptPath": "/t.md", "sessionType": "callCapture", + "primaryModel": "Parakeet-TDT v3", "graniteModel": "granite-q8"}, + "incomplete": False, + "segments": [ + {"startTime": 0.0, "speaker": "Speaker 2", "durationSec": 2.0, + "primaryText": "the quarterly numbers look grim", "graniteText": "the quarterly numbers look green", + "graniteError": None, "graniteLatencySec": 0.4}, + {"startTime": 3.0, "speaker": "Speaker 2", "durationSec": 1.5, + "primaryText": "same words", "graniteText": "same words", + "graniteError": None, "graniteLatencySec": 0.2}, + ], + "totals": {"segmentCount": 2, "erroredCount": 0, "audioSeconds": 3.5, + "shadowWallClockSec": 0.6, "rtf": 0.171}, +} + +class ReportTest(unittest.TestCase): + def test_report(self): + with tempfile.TemporaryDirectory() as d: + d = pathlib.Path(d) + (d / "s1.comparison.json").write_text(json.dumps(SAMPLE)) + out = d / "report.html" + script = pathlib.Path(__file__).parent.parent / "granite-shadow-report.py" + subprocess.run([sys.executable, str(script), str(d), "-o", str(out)], check=True) + html = out.read_text() + self.assertIn("grim", html) # disagreement segment present + self.assertIn("green", html) + self.assertIn("0.171", html) # RTF surfaced + # disagreeing segment sorted before identical one + self.assertLess(html.index("grim"), html.index("same words")) + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run → FAIL** (`python3 scripts/tests/test_shadow_report.py`). +- [ ] **Step 3: Implement** (stdlib: `json`, `pathlib`, `difflib`, `html`, `argparse`): + +```python +#!/usr/bin/env python3 +"""Render granite shadow comparison JSONs into one side-by-side HTML report. +Usage: python3 granite-shadow-report.py "~/Library/Application Support/Tome/GraniteShadow" [-o report.html] +Stdlib only (spec §7).""" +import argparse, difflib, html, json, pathlib + +def word_diff(a: str, b: str) -> tuple[float, str, str]: + aw, bw = a.split(), b.split() + sm = difflib.SequenceMatcher(a=aw, b=bw) + left, right = [], [] + for op, i1, i2, j1, j2 in sm.get_opcodes(): + at, bt = " ".join(aw[i1:i2]), " ".join(bw[j1:j2]) + if op == "equal": + left.append(html.escape(at)); right.append(html.escape(bt)) + else: + if at: left.append(f"{html.escape(at)}") + if bt: right.append(f"{html.escape(bt)}") + return 1 - sm.ratio(), " ".join(left), " ".join(right) + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("dir", type=pathlib.Path) + ap.add_argument("-o", "--out", type=pathlib.Path, default=pathlib.Path("report.html")) + args = ap.parse_args() + sessions = [json.loads(p.read_text()) + for p in sorted(args.dir.expanduser().glob("*.comparison.json"))] + rows, agg = [], {"sessions": len(sessions), "segments": 0, "errored": 0, + "audio": 0.0, "wall": 0.0, "disagree": 0} + for s in sessions: + agg["segments"] += s["totals"]["segmentCount"]; agg["errored"] += s["totals"]["erroredCount"] + agg["audio"] += s["totals"]["audioSeconds"]; agg["wall"] += s["totals"]["shadowWallClockSec"] + for seg in s["segments"]: + score, lh, rh = word_diff(seg["primaryText"], seg["graniteText"]) + if score > 0.05: agg["disagree"] += 1 + rows.append((score, s["session"]["sessionID"], s["session"]["primaryModel"], + s["totals"]["rtf"], seg, lh, rh)) + rows.sort(key=lambda r: -r[0]) + rtf = agg["wall"] / agg["audio"] if agg["audio"] else 0 + body = [f"

Granite shadow report

", + f"

{agg['sessions']} sessions · {agg['segments']} segments · " + f"{agg['disagree']} disagreeing (>5% word diff) · {agg['errored']} errored · " + f"aggregate shadow RTF {rtf:.3f}

", + "", + ""] + for score, sid, pmodel, srtf, seg, lh, rh in rows: + body.append(f"" + f"") + body.append("
diffsessiontprimarygranite
{score:.2f}{html.escape(sid)}
{html.escape(pmodel)}" + f" · RTF {srtf:.3f}		{seg['startTime']:.0f}s{lh}{rh}
") + args.out.write_text("\n".join(body)) + print(f"wrote {args.out} ({agg['segments']} segments)") + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 4: Run → PASS.** +- [ ] **Step 5: Commit** — `git commit -am "feat: granite shadow HTML comparison report (stdlib)"` + +### Task 13: Live smoke + install + enable (GATED on Task 4 go) + +**Files:** +- None created (operational task); update `docs/superpowers/plans/2026-07-09-granite-phase0-results.md` with the live-smoke note. + +- [ ] **Step 1: GATE** — read the Phase 0 results doc. Proceed to enabling ONLY on "go" (all three Phase 0 gates). On "no-go": still complete Steps 2–3 with the flag OFF (code is inert), report to Nic, stop. +- [ ] **Step 2: Full verification** — `DEVELOPER_DIR=… swift test` (all green) and `swift build -c release`. +- [ ] **Step 3: Build + install the app the same way the whisper-v3-turbo work did its smoke restoration** (check `scripts/` for the packaging script used then; reuse it exactly). Install to the location Nic's running copy lives. +- [ ] **Step 4: Enable + live smoke** — `defaults write com.dloomis.tome graniteShadowEnabled -bool YES`; launch Tome; record a ~60 s voice memo with 2 speakers (or play a meeting clip); stop; wait for post-processing; verify: + - primary transcript identical in structure to a flag-off run (spot-check), + - `~/Library/Application Support/Tome/GraniteShadow/.granite.md` + `.comparison.json` exist and pair correctly, + - `python3 scripts/granite-shadow-report.py "~/Library/Application Support/Tome/GraniteShadow" -o /tmp/report.html` renders, + - no llama-server process survives app quit (`pgrep llama-server`). +- [ ] **Step 5: Record results** in the Phase 0 results doc (live-smoke section: RTF observed, artifacts OK); commit — `git commit -am "test: granite shadow live smoke on real session"`. +- [ ] **Step 6: Notify Nic**: shadow is live for Friday's meetings; review lands EOD Wednesday 2026-07-15 via the report script. + +--- + +## Self-Review Notes + +- **Spec coverage:** §0→Task 4 (+2 for template, +3 for manifest mode, TED-LIUM substitution documented in-task); §1→Task 5; §2→Task 1; §3→Task 9; §4→Tasks 6+10 (pairing-by-startTime in ShadowArtifacts); §5→Task 11 (placement after voiceprints, before retention; cancellation via Task.isCancelled in ShadowRunner; `.finalizing` retained — no new Phase case); §6→Task 10; §7→Task 12; §8 testing→each task's test steps. Spec's "WAVs still present when the phase runs" ordering test is downgraded to a live-smoke check (Task 13) — a job-level test harness would need SessionHandle fixtures that don't exist; documented deviation. +- **Deviation from spec §4 wording:** SegmentReTranscriber keeps its `ASRCoordinator` directly; the `SegmentTranscribing` seam lives on the shadow side, and identical audio is guaranteed by the shared `SegmentAudio` functions instead. Same intent (identical inputs + testable seam), less churn on the primary path, and error conventions stay cleanly separated (placeholders for primary, recorded errors for shadow). +- **Type consistency check:** `ShadowConfig` field names match between Tasks 5/9/10/11; `ShadowSegment/ShadowRunOutput/ShadowComparison*` defined once in Task 10 and consumed in 11/12 (report reads the JSON keys `primaryText/graniteText/graniteLatencySec` — matches the Codable names); `BenchManifest` names match between Tasks 3/4. +- **Known adaptation points for the executor** (not placeholders — decision recorded where discovered): Task 1 GGUF filenames (API check), Task 2 request shape (probe), Task 4 ESB ref-column fallback chain, Task 10 `speakerLabels` visibility + Duration→seconds conversion. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/README.md b/docs/superpowers/research/2026-07-09-asr-model-roi/README.md new file mode 100644 index 0000000..dc55ba9 --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/README.md @@ -0,0 +1,25 @@ +# ASR model ROI research — 2026-07-09 + +Six research tracks (five web, one repo review), each report followed by its +adversarial verification verdicts (48 total; refuted claims carry the +correction in the verdict note). Produced for the granite shadow +transcription decision — see +[the design spec](../../specs/2026-07-09-granite-shadow-transcription-design.md). + +**Recommendation: granite-speech-4.1-2b** (AR base variant), llama.cpp +sidecar on IBM's official GGUFs, validated first via a week of hidden-flag +shadow transcription against the primary model on real meetings. + +| File | Track | Net result | +|---|---|---| +| [research-granite.md](research-granite.md) | ibm-granite/granite-speech 4.x deep-dive | Winner. No 4.2 exists; NAR is a datacenter-batch artifact; `-plus` adds speaker tags but drops punctuation. Official GGUFs + merged llama.cpp support. | +| [research-accuracy.md](research-accuracy.md) | Leaderboard deep-dive, meeting-centric deltas | Granite ≈ halves whisper-large-v3-turbo's AMI WER (−49%), −26% vs parakeet-v3. Screenshot's average had AMI toggled off. "28%" figure reconstructed (avg vs whisper-large-v3 at launch, not an accents metric). Caveats: in-domain training splits, no long-form track entry. | +| [research-runtime.md](research-runtime.md) | Apple Silicon runtime landscape | llama.cpp mtmd (official IBM GGUFs) = most credible path; mlx-audio-swift lists Granite but only demonstrates the 1B checkpoint; FluidAudio/WhisperKit will not deliver LLM-decoder models. Leaderboard RTFx = batched H200/A100 throughput; ÷15–25 for 2B MLX-class on M2 Max. | +| [research-higgs.md](research-higgs.md) | bosonai/higgs-audio-v3-8b-stt-v2 | Rejected: rank is a clean-speech artifact; loses to granite on AMI/Earnings22; 17.8 GB; no viable Mac path; repetition-loop mitigations shipped by vendor. | +| [research-canary-qwen.md](research-canary-qwen.md) | nvidia/canary-qwen-2.5b | Rejected: worst meeting profile of the top set despite AMI oversampled to 15% of training; NeMo/CUDA-only; no port. | +| [research-repo.md](research-repo.md) | Tome model-setup scalability review | Model N+1 ≈ 1 day (compiler-enforced switches); post-processing-only model not expressible today (single-slot everywhere, no supportsLive); dual-slot is a coordinator/provisioner change, not a pipeline rewrite. NOTE: one claim refuted — backend actors do NOT head-of-line block (they're reentrant, suspending at SDK calls); the dual-slot case rests on capability/contention/UX grounds instead. | + +Re-check candidates if the shadow week disappoints: granite-speech-4.1-2b-plus +(speaker attribution), bosonai/higgs-audio-v3-stt (2.68B sibling — best +overall average, AMI 7.19, fringe ggml ports), and whatever mlx-audio-swift +demonstrates for 4.1-2b by then. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/research-accuracy.md b/docs/superpowers/research/2026-07-09-asr-model-roi/research-accuracy.md new file mode 100644 index 0000000..62bf50d --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/research-accuracy.md @@ -0,0 +1,108 @@ +# Accuracy ROI for Tome meeting transcription — Open ASR Leaderboard deep-dive (data pulled 2026-07-09) + +## 0. Data provenance & the screenshot discrepancy + +The leaderboard space (https://huggingface.co/spaces/hf-audio/open_asr_leaderboard) loads its English short-form table from `english_short_latest.csv` in the dataset repo **hf-audio/open-asr-leaderboard-results** (long-form from `Steveeeeeeen/leaderboard_longform`; confirmed by reading the space's `init.py`). I downloaded both CSVs today. + +Two things about the user's screenshot: +- **The screenshot's "Average WER" is a 4-dataset average, not the leaderboard default.** Mean of its visible columns reproduces it exactly: granite-nar (8.44+1.28+2.77+3.33)/4 = 3.955 → "3.95"; granite-4.1-2b → 3.995 → "3.99"; higgs → 4.0225 → "4.02". AMI, Gigaspeech and VoxPopuli were toggled off, which is why AMI (the column Tome cares most about) was missing and the averages look lower than the canonical ones. +- The screenshot's per-dataset values differ slightly from the current CSV (e.g., nar Earnings22 8.44 vs 8.15 in CSV; RTFx values differ ~2x) — the results file was re-generated recently: per the leaderboard GitHub README, recent English short-form evals migrated to HF Jobs on **H200** GPUs (https://github.com/huggingface/open_asr_leaderboard). Rankings are essentially unchanged; I use the canonical CSV below. **RTFx is datacenter-GPU (H200/A100-class), not Apple Silicon.** + +The CSV has both raw and "Cleaned" reference columns for AMI/Gigaspeech/VoxPopuli (cleaned = re-processed references; neither the app code nor constants.py documents them, so I report both). + +## 1. Full extracted table (english_short_latest.csv, sorted by default cleaned average) + +| Model | Avg (cleaned) | Avg (orig) | RTFx | AMI-Cleaned | AMI (raw) | Earnings22 | VoxPop-Cleaned | VoxPop (raw) | LS Clean | LS Other | SPGI | Params (B) | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| bosonai/higgs-audio-v3-stt | 4.62 | 5.04 | 110 | 7.19 | 7.86 | 8.26 | 3.73 | 5.93 | 1.07 | 2.60 | 2.07 | 2.68 | +| bosonai/higgs-audio-v3-8b-stt-v2 | 4.73 | 5.25 | 139 | 8.37 | 9.35 | 8.45 | 2.94 | 5.62 | 0.95 | 2.05 | 3.23 | 8.91 | +| ibm-granite/granite-speech-4.1-2b | 4.90 | 5.18 | 547 | 7.06 | 7.72 | 8.23 | 4.18 | 5.40 | 1.02 | 2.17 | 3.47 | 2 | +| ibm-granite/granite-speech-4.1-2b-nar | 4.95 | 5.25 | 2079 | **6.96** | **7.64** | **8.15** | 4.25 | 5.63 | 1.04 | 2.40 | 3.23 | 2 | +| Qwen/Qwen3-ASR-1.7B | 5.02 | 5.59 | 394 | 8.31 | 9.26 | 9.88 | 3.01 | 5.99 | 1.24 | 2.92 | 2.58 | 2.04 | +| nvidia/canary-qwen-2.5b | 5.06 | 5.41 | 861 | 7.91 | 9.05 | 10.04 | 4.14 | 5.38 | 1.23 | 2.62 | 1.70 | 2.5 | +| ibm-granite/granite-4.0-1b-speech | 5.10 | 5.37 | 661 | 7.37 | 8.12 | 8.33 | 4.41 | 5.51 | 1.10 | 2.49 | 3.55 | 2 | +| CohereLabs/cohere-transcribe-03-2026 | 5.20 | 5.35 | 916 | 7.01 | 7.80 | 10.38 | 5.38 | 5.58 | 0.96 | 2.04 | 2.74 | 2 | +| ibm-granite/granite-speech-3.3-8b | 5.29 | 5.54 | 264 | 7.70 | 8.43 | 9.07 | 4.54 | 5.44 | 1.11 | 2.52 | 3.54 | 9 | +| **nvidia/parakeet-tdt-0.6b-v2** (Tome default option) | 5.39 | 5.86 | 6038 | 9.10 | 10.40 | 10.78 | 3.78 | 5.68 | 1.27 | 2.73 | 1.94 | 0.6 | +| **nvidia/parakeet-tdt-0.6b-v3** (FluidAudio v3) | 5.66 | 6.22 | 6098 | 9.41 | 10.58 | 10.77 | 3.19 | 5.89 | 1.51 | 3.12 | 3.63 | 0.6 | +| nyrahealth/CrisperWhisper | 5.76 | 6.56 | 33 | 7.10 | 8.04 | 12.43 | 4.27 | 8.60 | 1.99 | 3.95 | 1.94 | 2 | +| distil-whisper/distil-large-v3.5 | 6.10 | 7.03 | 874 | 12.09 | 13.36 | 10.83 | 2.52 | 7.69 | 1.93 | 4.50 | 2.62 | 0.8 | +| **openai/whisper-large-v3** | 6.55 | 7.33 | 462 | 13.63 | 14.86 | 11.59 | 4.53 | 8.70 | 1.55 | 3.52 | 2.71 | 2 | +| **openai/whisper-large-v3-turbo** (Tome accuracy option) | 7.01 | 7.80 | 783 | 13.87 | 15.16 | 11.07 | 7.02 | 11.22 | 2.13 | 3.70 | 2.79 | 0.8 | + +Source: https://huggingface.co/datasets/hf-audio/open-asr-leaderboard-results (file `english_short_latest.csv`). Note: whisper-large-v3-turbo is actually *worse* than plain whisper-large-v3 across the board (avg 7.01 vs 6.55; VoxPopuli raw 11.22 vs 8.70) — Tome's "accuracy" option is the weakest large-Whisper variant on this board. Also note **CrisperWhisper** (a verbatim-focused whisper-large-v3 fine-tune) hits AMI 8.04 — evidence that most of the AMI gap is conversational fine-tuning, not architecture. + +## 2. Deltas that matter for Tome (relative WER reduction, cleaned refs; raw AMI in parens) + +**vs whisper-large-v3-turbo (current accuracy option):** + +| Candidate | Avg | AMI | Earnings22 | +|---|---|---|---| +| granite-speech-4.1-2b | 4.90 (**−30%**) | 7.06 (**−49%**; raw −49%) | 8.23 (**−26%**) | +| granite-speech-4.1-2b-nar | 4.95 (−29%) | 6.96 (**−50%**; raw −50%) | 8.15 (−26%) | +| higgs-audio-v3-stt (2.7B) | 4.62 (−34%) | 7.19 (−48%) | 8.26 (−25%) | +| higgs-audio-v3-8b-stt-v2 | 4.73 (−33%) | 8.37 (−40%) | 8.45 (−24%) | +| canary-qwen-2.5b | 5.06 (−28%) | 7.91 (−43%) | 10.04 (−9%) | +| granite-4.0-1b-speech | 5.10 (−27%) | 7.37 (−47%) | 8.33 (−25%) | + +**vs parakeet-tdt-0.6b-v3 (current default):** + +| Candidate | Avg | AMI | Earnings22 | +|---|---|---|---| +| granite-speech-4.1-2b | −13% | −25% (raw −27%) | −24% | +| granite-speech-4.1-2b-nar | −13% | −26% (raw −28%) | −24% | +| higgs-audio-v3-stt | −18% | −24% | −23% | +| canary-qwen-2.5b | −11% | −16% | −7% | + +Headline: on AMI (the meeting corpus), granite-4.1 roughly **halves** whisper-large-v3-turbo's errors (15.16→7.64 raw) and cuts parakeet-v3's by ~27% (10.58→7.64). Against parakeet, granite's biggest wins are exactly on the meeting-like sets (AMI, Earnings22); on VoxPopuli parakeet-v3 is actually better (3.19 vs 4.25 cleaned) — the gains are concentrated where Tome needs them. + +## 3. What the delta means in practice + +At meeting-like WER (8–15%), per 1,000 spoken words (~6–7 min of meeting at 150 wpm): +- whisper-large-v3-turbo on AMI-style audio: ~152 errors/1000 words ≈ 22 errors/minute-of-speech. granite-4.1: ~77/1000 ≈ 11/min. **~680 fewer errors per hour-long meeting.** +- parakeet-v3 → granite-4.1 on AMI: ~106 → ~77/1000, ~29 fewer per 1000 (~260/hour). +- A 2–4 point absolute drop in the 8–12% band = 17–40% relative = going from roughly one error every 1.5 sentences to one every 2–3 sentences. +- Distribution matters more than the count: leaderboard WER is computed after Whisper-style normalization (punctuation/case/numeral/filler forgiveness — confirmed in the space's constants.py), so remaining errors concentrate on **content words, proper nouns, and domain terms** — exactly what poisons meeting summaries and action items. Two mitigations specific to the granite 4.x family: **keyword-list biasing** (prime the model with attendee/project names; announced for granite-4.0-1b-speech, https://huggingface.co/blog/ibm-granite/granite-4-speech) and **granite-speech-4.1-2b-plus**, which adds speaker-attributed transcripts + word timings (https://huggingface.co/ibm-granite/granite-speech-4.1-2b-plus). +- Qualitative accented/noisy comparisons: (a) an independent (small: 58 clips, diverse accents) edge benchmark measured granite-speech-3.3-8b at **8.18% clean / 15.72% noisy** vs Whisper **19.96% clean / 29.80% noisy** (https://www.ionio.ai/blog/2025-edge-speech-to-text-model-benchmark-whisper-vs-competitors); (b) IBM cites a Royal Flying Doctor Service field test where granite was "far better at handling the background noise than any other commercial models available" (https://research.ibm.com/blog/granite-4-1-ai-foundation-models); (c) on the long-form board's CORAAL (African-American vernacular English — an accent-robustness proxy), parakeet-v3 15.57 beats whisper-large-v3 18.89 and canary-qwen 19.05; granite is absent. The leaderboard also now runs private accented/held-out test sets from Appen and DataOcean (repos `hf-audio/appen_shortform_results` / `dataocean_shortform_results` — access-gated, I couldn't pull them), built specifically to catch public-benchmark overfitting. + +## 4. The "28% on English for poor audio/accents" quote — hunt result + +**No verbatim published "28%" claim was found** in IBM marketing, IBM Research blogs, or press coverage after several targeted searches. What exists: +- IBM Research, 29 Apr 2026 (Granite 4.1 announcement): "Granite Speech 4.1 2B achieves a 5.33% word-error rate (WER), placing it among the top models" — no relative % (https://research.ibm.com/blog/granite-4-1-ai-foundation-models). +- **The arithmetic that reproduces 28%:** 5.33 (granite-4.1-2b leaderboard avg at launch) vs whisper-large-v3's then-displayed ~7.4 avg → (7.4−5.33)/7.4 = **28.0% relative WER reduction**. Third-party coverage did the same math for the 1B model and headlined "beating Whisper Large V3 by 25% on word error rate" (5.52 vs ~7.4) (https://awesomeagents.ai/news/ibm-granite-4-speech-edge-asr/). So the remembered "28%" is almost certainly **relative WER reduction vs whisper-large-v3 on the Open ASR Leaderboard average** — a mixed benchmark, *not* a specific accents/poor-audio measurement. IBM's accent/noise language is qualitative ("industry-leading transcription accuracy across accents, domains and noisy environments", https://www.ibm.com/granite). Using today's CSV, the equivalent numbers are −25% vs whisper-large-v3 and −30% vs whisper-large-v3-turbo — so the figure is real in spirit, and on AMI specifically it *understates* the gap (−49%). +- Also verified: **there is no granite-speech-4.2 / "4.2.1-2b"**. The ibm-granite collection tops out at the 4.1 family: granite-speech-4.1-2b, -2b-plus, -2b-nar (all Apache-2.0), plus granite-4.0-1b-speech (https://huggingface.co/collections/ibm-granite/granite-speech). The user's "4.2.1-2b" is a misremembering of "4.1-2b". + +## 5. Leaderboard caveats + +1. **In-domain training (biggest caveat):** granite-speech-4.1-2b's model card lists **AMI (100h) and Earnings-22 (105h)** — plus VoxPopuli, CommonVoice, LibriSpeech, Fisher, Switchboard — in its training data (https://huggingface.co/ibm-granite/granite-speech-4.1-2b). These are train partitions, not the test files, so it's not literal contamination, but granite's AMI/E22 edge is partly "trained on the same corpora's training splits" — expect the real-world gap on Zoom/Teams meeting audio to be smaller than the leaderboard gap (though training on meeting speech is arguably exactly the specialization Tome wants). CrisperWhisper's AMI 8.04 from a whisper fine-tune supports this read. +2. **Normalization:** WER computed after Whisper-style normalization (case/punct/numerals/fillers removed) — verbatim fidelity differences are hidden; punctuation quality of the raw output still differs between models. +3. **Short-form vs long-form:** the main table is short-segment (<~30s) evaluation. On the separate **long-form track** (full recordings: earnings21/22, TED-LIUM, CORAAL; https://huggingface.co/datasets/Steveeeeeeen/leaderboard_longform): **parakeet-tdt-0.6b-v3 is the best open model (avg 10.72, RTFx 1003), beating whisper-large-v3-turbo (11.01, RTFx 148), whisper-large-v3 (11.23) and canary-qwen-2.5b (11.20, RTFx 16 — LLM-decoder long-form is ~60x slower)**. **Granite-speech and Higgs are absent from the long-form track entirely.** Whisper has native sequential long-form decoding; granite 4.1 is built around short segments — the -plus variant explicitly supports chunked long-form via "incremental decoding with prefix passing" to keep speaker numbering consistent across chunk seams (https://www.mindstudio.ai/blog/ibm-granite-speech-41-vs-whisper-x-transcription-pipeline); higgs (frozen Whisper-Large-v3 encoder + Qwen3-8B decoder, 8.91B, Apache-2.0, https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2) inherits Whisper's 30s window and would need a VAD/chunking pipeline. Since Tome already segments audio for streaming, chunked inference is a solved problem architecturally, but leaderboard AMI numbers are on pre-segmented audio — chunk-boundary errors are extra. +4. **RTFx hardware:** recent English short-form evals run on HF Jobs with **H200** GPUs (https://github.com/huggingface/open_asr_leaderboard); earlier numbers were A100-class. All RTFx figures are meaningless for Apple Silicon except as relative ordering within an architecture class (e.g., nar's 2079 vs 4.1-2b's 547 ≈ 3.8x speedup from non-autoregressive decoding should roughly carry over). +5. **Leaderboard churn:** per-dataset values shifted a few tenths between the user's screenshot and today's CSV (eval re-runs); rankings stable. The "Cleaned" AMI/Gigaspeech/VoxPopuli columns (re-processed references) are undocumented in the app code; I report both — conclusions are identical either way. + +## 6. Net read for Tome + +For a meeting recorder that can spend minutes on post-processing, **granite-speech-4.1-2b (or -nar for ~4x decode speed at equal accuracy, or -plus for built-in speaker attribution)** is the standout accuracy target: ~50% fewer errors than whisper-large-v3-turbo and ~27% fewer than parakeet-v3 on the meeting corpus, 2B params (laptop-friendly), Apache-2.0. Higgs-audio-v3-stt (2.7B) edges it on the overall average but not on AMI, and the 8B v2 is worse on AMI than its own 2.7B sibling. Canary-qwen-2.5b is dominated by granite on every meeting-relevant axis. The main open risks are (a) in-domain-training inflation of the AMI delta, and (b) no CoreML/Swift runtime today — granite/higgs ports to Apple frameworks are still open requests (e.g., MLX: https://github.com/Blaizzy/mlx-audio/issues/737) — which is the adjacent workstream's question. Keeping parakeet-v3 for live streaming remains well-supported: it's the best open model on the long-form board and top-tier RTFx. + +## BOTTOM LINE +On the Open ASR Leaderboard's AMI meeting corpus, granite-speech-4.1-2b(-nar) roughly halves whisper-large-v3-turbo's WER (15.16→7.64 raw, −50%) and cuts parakeet-tdt-0.6b-v3's by ~27% (10.58→7.64) — ~680 fewer errors per hour-long meeting vs turbo — making it the best accuracy target for Tome's post-processing slot; higgs-audio-v3 wins the overall average but not AMI. The user's "28%" figure is not a published accents/poor-audio metric: it's the relative reduction of granite-4.1-2b's launch average WER (5.33) vs whisper-large-v3's ~7.4 on the leaderboard, and no granite-speech-4.2 exists (4.1 family is latest). Two caveats temper the AMI delta: granite trains on AMI/Earnings-22 training splits (in-domain advantage), and granite is absent from the long-form track, where parakeet-tdt-0.6b-v3 is the best open model (10.72 avg) — so a fast-live-parakeet + granite-post-processing split, with chunking for long audio, is well supported by the data. + +## VERIFICATION VERDICTS + +- [CONFIRMED] english_short_latest.csv: granite-speech-4.1-2b-nar 7.64 AMI (6.96 cleaned) vs whisper-large-v3-turbo 15.16 (13.87) and parakeet-tdt-0.6b-v3 10.58 (9.41); ~50% and ~27% relative error reduction + NOTE: Fetched the raw CSV from hf-audio/open-asr-leaderboard-results (english_short_latest.csv). Digit-by-digit match: granite-speech-4.1-2b-nar AMI WER 7.64 / AMI-Cleaned 6.96; openai/whisper-large-v3-turbo 15.16 / 13.87; nvidia/parakeet-tdt-0.6b-v3 10.58 / 9.41. Relative reductions compute to 49.6% (raw) / 49.8% (cleaned) vs turbo and 27.8% (raw) / 26.0% (cleaned) vs parakeet — '~50%' and '~27%' are fair. + +- [CONFIRMED] No granite-speech-4.2 exists as of 2026-07-09; newest IBM speech models are granite-speech-4.1-2b, -2b-plus, -2b-nar plus granite-4.0-1b-speech, all Apache-2.0 + NOTE: HF API model search for 'granite-speech-4.2' returns 0 results; ibm-granite org listing shows granite-speech-4.1-2b (created 2026-04-16), -2b-plus (2026-04-16), -2b-nar (2026-03-10), granite-4.0-1b-speech (2026-02-27), plus older 3.x models and GGUF conversions — nothing newer. All four carry license:apache-2.0 tags. Minor: the cited collection URL is a stub (real HF collection URLs need a slug-hash), but I verified via the org's model API directly; substance holds. + +- [CONFIRMED] granite-speech-4.1-2b model card lists AMI (100h) and Earnings-22 (105h) in training data, so AMI/Earnings22 leaderboard scores are partly in-domain + NOTE: Raw README.md of ibm-granite/granite-speech-4.1-2b contains a Training Data table with rows 'AMI English | ASR | 100 | edinburghcstr/ami' and 'Earnings-22 English | ASR | 105 | esb/datasets'. Training uses the train splits while the leaderboard tests on test splits, so 'in-domain rather than zero-shot' is the accurate characterization — confirmed. + +- [CONFIRMED] IBM Granite 4.1 announcement (29 Apr 2026) states 5.33% average WER on Open ASR Leaderboard; no published IBM '28% better for accents/poor audio' claim; 28% matches relative reduction of 5.33 vs whisper-large-v3's ~7.4 average + NOTE: Fetched research.ibm.com/blog/granite-4-1-ai-foundation-models: published 29 Apr 2026, states 'Granite Speech 4.1 2B achieves a 5.33% word-error rate (WER), placing it among the top models on the OpenASR Leaderboard.' The blog contains no '28%' or accent/noise percentage claim, and a web search found no IBM-published 28% figure either (only third-party MindStudio posts, none citing 28%). Arithmetic: (7.4-5.33)/7.4 = 27.97% ≈ 28%. Caveat: whisper-large-v3's current leaderboard original average is 7.33, so '~7.4 then-listed' is plausible but the exact historical snapshot value was not independently verifiable; the derivation is a reasonable inference, not an IBM statement. + +- [CONFIRMED] Long-form track (earnings21, earnings22, TED-LIUM, CORAAL): parakeet-tdt-0.6b-v3 avg 10.72, beating whisper-large-v3-turbo 11.01, whisper-large-v3 11.23, canary-qwen-2.5b 11.20 (RTFx ~16); granite and higgs absent + NOTE: Fetched longform_latest.csv from Steveeeeeeen/leaderboard_longform. Exact values: parakeet-tdt-0.6b-v3 Average 10.72 (RTFx 1002.91); whisper-large-v3-turbo 11.01; whisper-large-v3 11.2275 (rounds to 11.23); canary-qwen-2.5b 11.2025 (rounds to 11.20) with RTFx 16.05. Columns are earnings21, earnings22, tedlium, coraal_avg. No ibm-granite or bosonai/higgs rows exist in the file. Note the claim only says parakeet beats those named open models, which is true; several proprietary entries (e.g. elevenlabs/scribe_v2 7.32, assembly/universal-3-pro 8.34) score lower overall, but the claim does not assert parakeet is #1. + +- [CONFIRMED] higgs-audio-v3-8b-stt-v2 is Apache-2.0, 8.91B params, frozen Whisper-Large-v3 encoder + Qwen3-8B decoder, model card reports 10.14% AMI WER (leaderboard 9.35 raw / 8.37 cleaned), worse on AMI than 2.7B sibling higgs-audio-v3-stt + NOTE: Model card README states: license apache-2.0; 'Encoder: Whisper-Large-v3 (frozen)'; 'Decoder: Qwen3-8B (LoRA fine-tuned, merged)'; 'Total parameters: 8.91B' (safetensors API: 8,905,965,568 ≈ 8.91B); performance table lists AMI 10.14%. Leaderboard CSV confirms 9.35 AMI raw / 8.37 cleaned. Sibling bosonai/higgs-audio-v3-stt is 2,675,546,112 params (2.68B ≈ '2.7B') with AMI 7.86 raw / 7.19 cleaned — better than the 8B on AMI, as claimed. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/research-canary-qwen.md b/docs/superpowers/research/2026-07-09-asr-model-roi/research-canary-qwen.md new file mode 100644 index 0000000..d77db83 --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/research-canary-qwen.md @@ -0,0 +1,87 @@ +# Deep-dive: nvidia/canary-qwen-2.5b for Tome + +## 1. Model card facts (architecture, size, license, languages) + +Source: [HF model card](https://huggingface.co/nvidia/canary-qwen-2.5b) (released to HF 2025-07-17). + +- **Architecture**: SALM (Speech-Augmented Language Model) — a **FastConformer encoder + Qwen3-1.7B LLM decoder**, joined by a linear projection (1024→2048), with **LoRA applied to the LLM**. Total **2.5B parameters**. +- **License**: **CC-BY-4.0** — explicitly "ready for commercial use". No license blocker for Tome. +- **Languages**: **English only**. Encoder was pretrained on De/Fr/Es speech but the card says it is "unlikely to be reliable as a multilingual model." (Tome's current Parakeet-TDT v3 covers 25 EU languages; adopting canary-qwen would be an English-only regression for the post-processing path.) +- **Training data**: 234.5k hours across 26 English datasets; majority from Granary (YouTube-Commons 109.5k h, YODAS2 77k h, LibriLight 13.6k h). **AMI is in the training set and was oversampled to ~15% of total training data. Earnings22 and SPGISpeech are NOT in training** ([README](https://huggingface.co/nvidia/canary-qwen-2.5b/raw/main/README.md)). +- **Context limits**: max training audio duration **40 s**, max sequence 1024 tokens. Longer inputs "may technically" work but with degraded accuracy — so meeting-length audio requires VAD/chunking + stitching in the runtime layer (Tome would own that). +- **Punctuation/capitalization**: yes, trained with PnC transcripts. +- **Noise robustness** (from card): WER 9.83% at SNR 0 dB, 30.60% at SNR −5 dB — degrades steeply in noise, which matters for "imperfect audio" meetings. + +## 2. Framework dependency: NeMo without CUDA + +- Official inference path is **NeMo ≥ 2.5.0** via `nemo.collections.speechlm2.models.SALM` (`SALM.from_pretrained(...)`, `model.generate(...)`) — note this is the **speechlm2** collection, *not* the classic `asr` collection. +- The model card lists only NVIDIA GPU architectures (Ampere/Hopper/Blackwell, tested on A6000/A100/RTX 5090) and Linux/Windows as the runtime environment ([model card](https://huggingface.co/nvidia/canary-qwen-2.5b)). +- NeMo itself has *partial* macOS support: the install docs guarantee **only the ASR collection on MacBook**, and MPS inference requires `PYTORCH_ENABLE_MPS_FALLBACK=1` plus `allow_mps=true` because not all ops are implemented on MPS ([NeMo repo/docs](https://github.com/NVIDIA-NeMo/NeMo)). The speechlm2/SALM collection has **no documented Mac/MPS path**. +- Real-world evidence: in [HF discussion #11](https://huggingface.co/nvidia/canary-qwen-2.5b/discussions/11), a user on Ubuntu could not even get CPU-only inference working (dtype/device errors); NVIDIA staff (Piotr Żelasko) responded only with CUDA-based solutions. **No reports anywhere of canary-qwen running on Apple Silicon via NeMo.** +- Practically moot for Tome anyway: Tome is a Swift/SwiftPM app; embedding a Python NeMo stack is a non-starter. Any integration must go through a native port. + +## 3. Ports & Apple Silicon precedent + +**Mainstream Apple Silicon ASR runtimes do NOT ship canary-qwen:** + +- **FluidAudio** (Tome's current Parakeet vendor): ships parakeet-tdt-0.6b v2/v3 CoreML, parakeet-ctc zh, and (per docs) Qwen3-ASR — **no canary of any kind**; model requests are funneled through [issue #49](https://github.com/FluidInference/FluidAudio/issues/49) ([repo](https://github.com/FluidInference/FluidAudio)). +- **Argmax (WhisperKit Pro / Argmax SDK)**: ships NVIDIA Parakeet v2/v3 on ANE and references **canary-1b-v2** (the encoder-decoder Canary, not canary-qwen) in its pro/commercial SDK, with accuracy testing so far only on Parakeet ([Argmax blog](https://www.argmaxinc.com/blog/nvidia-frontier-speech-models-on-argmax-sdk)). +- **onnx-asr** (CPU/CoreML-EP ONNX runtime pkg): supports "Parakeet v2/v3, Canary v1/v2" — **not canary-qwen** ([PyPI](https://pypi.org/project/onnx-asr/)). +- **sherpa-onnx** and **mlx-audio**: support Canary-1b-family and Qwen3-ASR, **not canary-qwen** ([sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx), [mlx-audio](https://github.com/Blaizzy/mlx-audio)). + +**Fringe canary-qwen-specific ports DO exist (verified directly):** + +- **CoreML**: [phequals/canary-qwen-2.5b-coreml-fp16](https://huggingface.co/phequals/canary-qwen-2.5b-coreml-fp16) (+int8/static variants) — a full 4-stage pipeline (encoder.mlpackage, projection.mlpackage, stateful decoder with KV cache targeting macOS 15 CoreML state, LM head). **Artifacts only**: "long-audio chunking, prompt formatting, and transcript stitching live in the runtime layer and are not included." ~35 downloads/month, unofficial, no published accuracy/perf validation. Using it means Tome writes the entire Swift runtime (audio frontend, chunking, autoregressive decode loop across 4 CoreML models, stitching). +- **GGML/GGUF**: [CrispASR](https://github.com/CrispStrobe/CrispASR) (whisper.cpp fork, C++ with C-ABI, `-DGGML_METAL=ON` for Apple Silicon) explicitly lists a `canary-qwen` backend (FastConformer + Qwen3-1.7B SALM) with GGUF weights at [cstr/canary-qwen-2.5b-GGUF](https://huggingface.co/cstr/canary-qwen-2.5b-GGUF). This is the most practical Mac path today — C++ is embeddable from Swift — but it's a hobby-scale v0.8.x project; its own docs cite ~0.3× realtime on M1+Metal for a comparable LLM-decoder ASR model (quant dequant bottleneck). On an M2 Max/M4 Ultra, a 1-hour meeting would plausibly take minutes-to-tens-of-minutes — within Tome's stated tolerance, but unproven. +- **ONNX**: [onnx-community/canary-qwen-2.5b-ONNX](https://huggingface.co/onnx-community/canary-qwen-2.5b-ONNX) exists (13.7 GB) but has no model card, 13 downloads, and the visible file listing suggests LLM/tokenizer files without a clear audio-encoder pipeline — not a usable path. + +**Precedent summary**: Canary *encoder-decoder* models (canary-1b-v2) have real Apple Silicon support (Argmax Pro, mlx-audio, onnx-asr, sherpa-onnx). **canary-qwen specifically has zero mainstream support** — only two low-adoption community effort tracks. The SALM structure (CoreML/ggml encoder + autoregressive 1.7B LLM decode per chunk) is inherently harder to port and slower on ANE/Metal than parakeet-style transducers. + +## 4. Accuracy profile for meetings + +Leaderboard RTFx context: all Open ASR Leaderboard RTFx numbers (canary-qwen 418) are measured on an **NVIDIA A100-SXM4-80GB, CUDA 12.6, batch size up to 64** ([Open ASR Leaderboard paper](https://arxiv.org/html/2510.06961)) — they do not transfer to Apple Silicon. + +Per-dataset WER ([model card README](https://huggingface.co/nvidia/canary-qwen-2.5b/raw/main/README.md)): + +| Dataset | canary-qwen-2.5b | granite-speech-4.1-2b | Notes | +|---|---|---|---| +| LS Clean | 1.60 | 1.33 | clean read speech | +| LS Other | 3.10 | 2.50 | | +| TEDLIUM | 2.72 | — | prepared talks | +| SPGISpeech | **1.90** | 3.78 | pro-quality earnings-call segments, **not in canary-qwen training** | +| GigaSpeech | 9.41 | — | web/podcast audio | +| **AMI (meetings)** | **10.18** | **8.09** | AMI in BOTH models' training (canary-qwen oversampled it to ~15%) | +| **Earnings22** | **10.42** | **8.37** | accented, telephone/webcast-quality; in granite's training (105 h), NOT in canary-qwen's | + +Characterization: canary-qwen **wins on clean, well-mic'd, professionally produced audio** (its SPGISpeech 1.90 is genuinely impressive since SPGI was out-of-training) and **loses precisely on Tome's target conditions** — accented speakers, far-field mics, degraded channels (AMI, Earnings22, GigaSpeech), plus steep noise degradation (9.8% WER at SNR 0 dB, 30.6% at −5 dB). Caveat: granite's Earnings22/AMI edge is partly train-set exposure ([granite-speech-4.1-2b trained on Earnings-22 105 h and AMI 100 h](https://huggingface.co/ibm-granite/granite-speech-4.1-2b)), so the ~2-point gap overstates granite's generalization advantage somewhat — but canary-qwen had AMI oversampled to 15% of its diet and *still* posts 10.18 on AMI, which is a genuinely weak meeting-domain result for a 2.5B model. For meeting transcription, canary-qwen's headline 4.26 avg WER is carried by clean-audio test sets that don't resemble Tome's input. + +## 5. ASR+LLM hybrid mode — relevant or gimmick? + +The card documents two modes: ASR mode (transcribe) and **LLM mode**, activated by `model.llm.disable_adapter()` — this turns off the ASR LoRA and exposes the **original text-only Qwen3-1.7B**, which "can be used to post-process the transcript, e.g. summarize it or answer questions about it" ([model card](https://huggingface.co/nvidia/canary-qwen-2.5b)). Key facts: LLM mode takes **text in, text out — it cannot attend to the audio**; it is literally a stock 1.7B Qwen bundled in the same checkpoint. For Tome this is a **gimmick**: any summarization Tome wants can be done better by a separate, larger local LLM on Nic's 64 GB M2 Max (or Claude), with zero coupling to the ASR runtime. The hybrid mode's real value is NVIDIA's research story (one deployment does both), not accuracy or capability. + +## Net assessment for Tome + +- License and quality-on-clean-audio are fine, but canary-qwen is the **wrong accuracy profile** (weakest exactly on meeting-like/accented/noisy audio among the leaderboard's top models), **English-only**, has a **40 s window** requiring Tome-owned chunking, and has **no production-grade Apple Silicon runtime** — only an artifacts-only CoreML conversion and a hobby ggml fork. Integration cost is very high; expected accuracy payoff on meetings vs. current Parakeet-TDT v3 is modest and vs. granite-class models is negative. + +## BOTTOM LINE +canary-qwen-2.5b is commercially licensed (CC-BY-4.0) and strong on clean audio, but it is English-only, capped at ~40 s per inference window, and — critically for Tome — its WER is worst exactly on meeting-like conditions (AMI 10.18, Earnings22 10.42 vs granite-4.1-2b's 8.09/8.37), despite AMI making up ~15% of its training data. There is no supported non-CUDA path: NeMo's speechlm2/SALM collection has no Mac/MPS story, and no mainstream Apple Silicon runtime (FluidAudio, Argmax, mlx-audio, sherpa-onnx, onnx-asr) ships canary-qwen — only an artifacts-only community CoreML conversion (~35 downloads/mo) and a hobby ggml/Metal fork (CrispASR) exist. Its LLM mode is just the bundled text-only Qwen3-1.7B post-processing the transcript — a gimmick Tome can beat with any separate local LLM. Recommend against integrating canary-qwen; models with better accented/noisy-meeting accuracy and viable Apple Silicon ports are stronger candidates. + +## VERIFICATION VERDICTS + +- [CONFIRMED] nvidia/canary-qwen-2.5b is a 2.5B-parameter SALM combining a FastConformer encoder with a Qwen3-1.7B decoder (linear projection + LoRA), released under CC-BY-4.0 with commercial use permitted, English-only, with max training audio duration of 40 seconds. + NOTE: Raw README (huggingface.co/nvidia/canary-qwen-2.5b/raw/main/README.md) verified directly: '2.5 billion parameters', SALM architecture badge and description 'Speech-Augmented Language Model (SALM) with FastConformer Encoder and Transformer Decoder... built using nvidia/canary-1b-flash and Qwen/Qwen3-1.7B, a linear projection, and low-rank adaptation (LoRA) applied to the LLM'; 'license: cc-by-4.0' and 'This model is ready for commercial use'; 'English-only language support'; 'The maximum audio duration in training was 40s'. Every element matches. + +- [CONFIRMED] canary-qwen-2.5b scores 10.18 WER on AMI and 10.42 on Earnings22 (vs 1.60 LS Clean, 1.90 SPGISpeech), even though AMI was oversampled to about 15% of its 234.5k-hour training data while Earnings22 and SPGISpeech were not in training. + NOTE: README leaderboard table row matches digit-for-digit: AMI 10.18, Earnings22 10.42, LS Clean 1.60, SPGISpeech 1.90 (row: 418 | 5.63 | 10.18 | 9.41 | 1.60 | 3.10 | 10.42 | 1.90 | 2.72 | 5.66). Training section: 'English (234.5k hours)' and 'AMI was oversampled during model training to constitute about 15% of the total data observed'. Earnings22 and SPGISpeech are absent from the training dataset list (18 training datasets; both appear only under evaluation). Minor caveat, not material: the card's YAML model-index widget carries slightly different values (AMI 10.19, Earnings 10.45, LS clean 1.61) than the README table the claim cites; the claim's numbers match the cited table exactly. + +- [CONFIRMED] ibm-granite/granite-speech-4.1-2b scores 8.09 WER on AMI and included both AMI (100 h) and Earnings-22 (105 h) in its training data, under an Apache 2.0 license. + NOTE: Raw README verified: 'license: apache-2.0'; training data table rows 'AMI English | ASR | 100' and 'Earnings-22 English | ASR | 105' (hours). The 8.09 AMI WER is not in the README text (per-dataset WERs are chart images), but the rendered model page's Evaluation results widget contains exactly "task_id":"ami_wer","value":8.09 (verified in page HTML), and the card's own WER bar chart shows granite-speech-4.1-2b at ~8.1 on AMI_IHM, consistent. Mean WER 5.33 / RTFx 231.29 also on the page, corroborated by the card's Open ASR Leaderboard scatter plot. + +- [CONFIRMED] The onnx-asr package supports NVIDIA Parakeet v2/v3 and Canary v1/v2 on CPU/CoreML across macOS and Arm, but does not support canary-qwen. + NOTE: PyPI page (and the identical upstream README at github.com/istupakov/onnx-asr) states: 'Supports Parakeet v2 (En) / v3 (Multilingual), Canary v1/v2 (Multilingual) and GigaAM v2/v3 (Ru) models!' and 'Works on Windows, Linux, and macOS on x86 and Arm CPUs, with support for CUDA, TensorRT, CoreML, DirectML, ROCm, and WebGPU'. Zero occurrences of 'qwen' anywhere in the README — canary-qwen is not in the supported model list, supporting the negative claim. + +- [CONFIRMED] A community CoreML conversion of canary-qwen-2.5b exists (phequals/canary-qwen-2.5b-coreml-fp16) containing encoder, projection, stateful decoder, and LM head mlpackages, but it ships model artifacts only — chunking, prompt formatting, and transcript stitching are explicitly not included. + NOTE: Repo README verified via raw fetch: lists encoder.mlpackage, projection.mlpackage, canary_decoder_stateful.mlpackage ('FP16 stateful autoregressive decoder with KV cache'), and canary_lm_head.mlpackage; states verbatim 'This repo contains model artifacts only.' and 'Long-audio chunking, prompt formatting, and transcript stitching live in the runtime layer and are not included here.' Card also self-describes as a community (not official NVIDIA) CoreML FP16 conversion. + +- [CONFIRMED] Open ASR Leaderboard RTFx figures (canary-qwen: 418) are measured on an NVIDIA A100-SXM4-80GB GPU with CUDA 12.6 and batch sizes up to 64, not on Apple Silicon. + NOTE: arXiv 2510.06961 (Open ASR Leaderboard paper) states: 'The evaluation scripts for each model were run on an NVIDIA A100-SXM4-80GB GPU (driver 560.28.03, CUDA 12.6)' using 'a batch size of 64 whenever memory allowed, and reduced adaptively (48, 32, 16, ...)'. Table 3 lists NVIDIA Canary Qwen 2.5B at RTFx 418 (avg WER 5.63), matching the model card's own 418 RTFx figure. 'Batch sizes up to 64' is an accurate rendering, and the hardware is an NVIDIA datacenter GPU, not Apple Silicon. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/research-granite.md b/docs/superpowers/research/2026-07-09-asr-model-roi/research-granite.md new file mode 100644 index 0000000..7e3325c --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/research-granite.md @@ -0,0 +1,105 @@ +# IBM granite-speech 4.x — Deep Dive for Tome Integration + +## 1. Model enumeration (verified on HF org page, 2026-07-09) + +Current speech models under [ibm-granite](https://huggingface.co/ibm-granite) ([Granite Speech collection](https://huggingface.co/collections/ibm-granite/granite-speech)): + +| Model ID | Released/updated | Notes | +|---|---|---| +| `ibm-granite/granite-speech-4.1-2b` | Apr 29–30, 2026 (updated ~Jun 12) | AR flagship: ASR + bidirectional AST, 6 languages, keyword biasing, punctuation/caps. 465k downloads | +| `ibm-granite/granite-speech-4.1-2b-nar` | Apr 30, 2026 | Non-autoregressive, ASR-only, 5 languages | +| `ibm-granite/granite-speech-4.1-2b-plus` | ~Jun 16, 2026 (newest) | Adds speaker-attributed ASR + word-level timestamps | +| `ibm-granite/granite-speech-4.1-2b-GGUF` and `granite-speech-4.1-2b-plus-GGUF` | 2026 | **Official IBM GGUF releases for llama.cpp** | +| `ibm-granite/granite-4.0-1b-speech` | Mar 6, 2026 | Predecessor (naming anomaly: "4.0-1b-speech" not "speech-4.0") | +| `granite-speech-3.3-8b`, `-3.3-2b`, `-3.2-8b` | 2025 | Legacy | + +**"granite-speech-4.2.1-2b" does not exist.** No 4.2 of any kind is on the org page or findable via search; the latest is 4.1. Almost certainly a mangled recollection of **4.1-2b** (possibly crossed with the `-plus` variant or the odd `granite-4.0-1b-speech` naming). + +## 2. The NAR variant explained + +Source: [NAR model card](https://huggingface.co/ibm-granite/granite-speech-4.1-2b-nar). + +- **Architecture**: Instead of autoregressive token-by-token decoding, the CTC conformer encoder emits an initial hypothesis; that hypothesis is interleaved with insertion slots, concatenated with projected audio embeddings, and a **bidirectional LLM edits it (copy/insert/delete/replace) in a single forward pass**, exploiting the "identity mapping bias" of Transformers. One pass replaces hundreds of sequential decode steps — that's why leaderboard RTFx is ~3.8× higher (879 vs 231 on A100); IBM reports ~1820 RTFx on a single H100 at batch size 128 ("one hour of audio in under two seconds", per [MarkTechPost](https://www.marktechpost.com/2026/04/30/ibm-releases-two-granite-speech-4-1-2b-models-autoregressive-asr-with-translation-and-non-autoregressive-editing-for-fast-inference/)). +- **Documented trade-offs**: drops Japanese, drops speech translation, drops keyword biasing; and — directly relevant to meetings — the card states a **conservative editing bias: "prefers deletions over insertions, which reduces hallucination risk but may occasionally drop words in noisy conditions."** On the leaderboard the NAR is actually marginally better on average (3.95 vs 3.99) but slightly worse on Earnings22 (8.44 vs 8.37). +- **Apple Silicon reality check**: NAR **requires `flash_attention_2`** in transformers (CUDA-only → no MPS path); no llama.cpp GGUF exists for it; mlx-audio's Python release notes claim NAR support ([releases](https://github.com/Blaizzy/mlx-audio/releases), v0.4.1) but it's unvalidated. +- **Verdict for batch post-processing on a Mac**: the NAR's entire advantage is *datacenter batch throughput*, which is irrelevant for single-stream on-device post-processing where minutes are acceptable. **Tome wants the AR model (`4.1-2b`) or the `-plus` variant**, which also have far better Apple Silicon runtime coverage. + +## 3. Architecture and footprint + +Source: [granite-speech-4.1-2b model card](https://huggingface.co/ibm-granite/granite-speech-4.1-2b). + +- **Encoder**: 16 conformer blocks (1024 hidden, 8 heads), trained with CTC using a novel **dual-head CTC** (credited for the 4.1 accuracy gain). **Projector**: 2-layer window-query transformer (qformer), 10× temporal downsampling. **Decoder**: LLM based on `granite-4.0-1b-base` (128k context), with an **audio-specific LoRA adapter** activated only for audio inputs. Total ~2B params, bf16. +- **Disk (official GGUF repo,** [granite-speech-4.1-2b-GGUF](https://huggingface.co/ibm-granite/granite-speech-4.1-2b-GGUF)**)**: Q4_K_M 1.14 GB, Q5_K_M 1.32 GB, Q6_K 1.51 GB, Q8_0 1.96 GB, bf16 3.68 GB, plus `mmproj-model-f16.gguf` (audio encoder+projector) 1.16 GB. Safetensors bf16 ≈ 4–5 GB. +- **RAM at inference**: Q8_0 + f16 mmproj ≈ 3.5–4 GB working set; bf16 ≈ 6 GB. Trivial on a 64 GB M2 Max or a Mac Studio. +- **MLX**: mlx-community published 4-bit and bf16 conversions of 4.1-2b and 4.1-2b-plus ([mlx-community](https://huggingface.co/mlx-community), [mlx-audio issue #737](https://github.com/Blaizzy/mlx-audio/issues/737)). + +## 4. Meeting-audio accuracy evidence + +- **Leaderboard (A100)**: 4.1-2b Earnings22 **8.37** WER vs Parakeet-TDT-0.6b-v2's 11.15 (~25% relative better) and Whisper-class models ~11+. Model card reports **AMI 8.09** WER for 4.1-2b. The `-plus` card reports AMI 8.63 / Earnings22 8.68 (plus omits punctuation/caps). +- **The "28% gain for poor audio/accents" claim could NOT be located in any primary source.** I checked the raw model cards for 4.1-2b and 4.0-1b-speech (the string "28" does not appear), the [IBM Research 4.1 announcement blog](https://research.ibm.com/blog/granite-4-1-ai-foundation-models), the [IBM Research leaderboard blog](https://research.ibm.com/blog/granite-speech-recognition-hugging-face-chart), the [HF granite-4-speech blog](https://huggingface.co/blog/ibm-granite/granite-4-speech), and press coverage. Closest real claims: (a) 4.0-1b-speech "provides higher transcription accuracy for English ASR" vs granite-speech-3.3 (qualitative, no number); (b) 4.1 has "higher transcription accuracy for multilingual ASR due to a novel dual-head CTC encoder"; (c) IBM's Royal Flying Doctor Service anecdote — Granite Speech "proved in testing to be far better at handling the background noise than any other commercial models" (no percentage); (d) the `-plus` card's speaker-attribution claim WDER **0.9% vs 2.8%** for Microsoft VibeVoice on FISHER — note "2.8" here is a plausible origin of a misremembered "28%". Treat the 28% figure as unverified folklore; the checkable numbers above (Earnings22/AMI) are strong on their own. + +## 5. Apple Silicon inference paths (the critical section) + +**(a) llama.cpp — most mature, IBM-official.** llama.cpp's mtmd multimodal stack natively supports the Granite-Speech architecture (conformer encoder + qformer projector via mmproj, with automatic audio-LoRA toggling). IBM ships **official GGUFs** and documents macOS usage in the model card: `brew install llama.cpp` then `llama-cli -st -hf ibm-granite/granite-speech-4.1-2b-GGUF:Q8_0 --audio audio.wav -p "transcribe the speech..."`; requires **build b9045+**. `llama-server` works too (multimodal audio input supported). Caveats: support is recent (a 2026 bugfix for infinite-asterisk output shows active churn); audio is 16 kHz mono WAV/MP3. For Tome (Swift): either **sidecar `llama-server`** (simplest, HTTP, robust) or in-process via a llama.cpp xcframework binding (heavier lift). No published Apple Silicon RTF numbers exist; estimate for M2 Max: ~1.5B-effective LLM at Q8 decodes ≳80–150 tok/s on Metal, ASR output is only ~3–4 tokens per second of speech, so expect **RTF roughly 0.03–0.15 single-stream** (estimate, unverified) — comfortably "minutes for an hour-long meeting." + +**(b) MLX — the native-Swift path.** IBM's own model card documents `mlx-audio` (Python, ≥0.4.1): `python -m mlx_audio.stt.generate --model ibm-granite/granite-speech-4.1-2b`. Granite Speech 4.0 + 4.1 (incl. NAR) support landed in [mlx-audio v0.4.1](https://github.com/Blaizzy/mlx-audio/releases); mlx-community 4-bit/bf16 conversions exist. Crucially for Tome, **[mlx-audio-swift](https://github.com/Blaizzy/mlx-audio-swift) (SwiftPM package `MLXAudioSTT`, v0.1.3 July 2026, ~700 stars, macOS 14+) explicitly lists Granite Speech among supported STT models** — in-process Swift, async/await, auto HF download. Maturity is the risk: v0.1.x, granite support weeks old, no published benchmarks; [issue #737](https://github.com/Blaizzy/mlx-audio/issues/737) shows 4.1-family validation was still being firmed up in May 2026. Needs a hands-on smoke test before committing. + +**(c) transformers on MPS — not recommended.** Granite-speech is natively in transformers (PR [#36801](https://github.com/huggingface/transformers/pull/36801)), but MPS is undocumented/unvalidated for it, granite multimodal siblings have known MPS flakiness reports, the NAR variant requires CUDA-only flash_attention_2, and it would force a Python sidecar into a Swift app anyway. + +**(d) ONNX / CoreML — nothing.** No official or community ONNX/CoreML conversions surfaced in any search. There is no FluidAudio/WhisperKit-style CoreML package for granite-speech; anyone wanting CoreML would be doing the conversion themselves (conformer + qformer + LoRA-modulated LLM = nontrivial). + +## 6. License, languages, streaming, length limits + +- **License**: Apache 2.0, all variants. +- **Languages**: 4.1-2b: EN/FR/DE/ES/PT/JA (ASR + bidirectional speech translation). NAR and plus: EN/FR/DE/ES/PT, ASR only. +- **Streaming**: **None.** All variants are chunk/batch — audio in, text out; no streaming decoder is documented anywhere. This fits Tome's "split" plan: keep Parakeet-TDT for live captions, granite for accurate post-processing. +- **Max audio length**: `-plus` card is explicit: **up to 9 minutes per call for ASR and speaker attribution, up to 3.5 minutes for word timestamps** ([plus card](https://huggingface.co/ibm-granite/granite-speech-4.1-2b-plus)). Base 4.1-2b card states no limit but shares the architecture, so a 1-hour meeting needs chunked calls (~5–9 min segments) with stitching — same pattern Tome already uses for Whisper-style batch. + +## Leaderboard RTFx caveat (confirmed) + +Open ASR Leaderboard RTFx is measured on an **NVIDIA A100-SXM4-80GB (CUDA 12.6), batch size 64** where memory allows ([Open ASR Leaderboard paper, arXiv:2510.06961](https://arxiv.org/abs/2510.06961)). Absolute RTFx does not transfer to Apple Silicon; only relative comparisons are meaningful, and the NAR's batch-throughput edge specifically evaporates at batch size 1 on a Mac. + +## Net recommendation for Tome + +`granite-speech-4.1-2b-plus` is the most interesting candidate for the post-processing slot: best-in-class meeting-domain WER at 2B scale, **plus free speaker attribution and word timestamps** (features Tome would otherwise need a diarization pipeline for), Apache 2.0, ~2–4 GB RAM. Two viable ship paths: llama.cpp sidecar (official IBM GGUFs, most proven today) or mlx-audio-swift in-process (cleanest Swift integration, youngest code). Both need an empirical WER + RTF bake-off on Nic's M2 Max against WhisperKit large-v3-turbo before adoption; no Apple Silicon throughput numbers exist publicly. + +## BOTTOM LINE +No granite-speech 4.2/4.2.1 exists — the latest is the 4.1-2b family (AR, NAR, and the June-2026 "plus" with speaker attribution + word timestamps), all Apache 2.0, batch-only (no streaming), ~1.1–3.7 GB GGUF on disk. The NAR variant's 4x RTFx edge is a datacenter batch-throughput artifact (single-pass edit of a CTC hypothesis, measured on A100/H100) and it may drop words in noisy audio — for Tome's single-stream Mac post-processing the AR or plus variant is the right pick. Apple Silicon support is real and IBM-documented via two paths: official GGUFs on llama.cpp's mtmd stack, and MLX (mlx-audio ≥0.4.1, plus the SwiftPM package mlx-audio-swift which lists Granite Speech as a supported STT model) — but both are weeks-to-months old and no public Apple Silicon RTF numbers exist, so a local bake-off is required. The user's recalled "28% accuracy gain" claim could not be found in any primary IBM source; the concrete evidence is Earnings22 8.37 / AMI 8.09 WER, roughly 25% relatively better than Parakeet-TDT on Earnings22. + +## VERIFICATION VERDICTS + +- [CONFIRMED] As of 2026-07-09, the newest IBM speech models on Hugging Face are granite-speech-4.1-2b, granite-speech-4.1-2b-nar, and granite-speech-4.1-2b-plus; no granite-speech 4.2 or 4.2.1 model exists. + NOTE: HF API listing of ibm-granite speech models confirms the 4.1 trio is the newest model generation (granite-speech-4.1-2b and -plus created 2026-04-16, -nar created 2026-03-10, modified through 2026-06-18) and no 4.2/4.2.1 exists. Minor completeness caveat, not an error: there are also official GGUF companion repos (granite-speech-4.1-2b-GGUF, and granite-speech-4.1-2b-plus-GGUF created 2026-06-30, technically the most recently created speech repo) plus an older granite-4.0-1b-speech — but these don't contradict the claim about the newest model generation. + +- [CONFIRMED] granite-speech-4.1-2b-nar is non-autoregressive, edits a CTC hypothesis in a single forward pass using a bidirectional LLM, supports only EN/FR/DE/ES/PT (no Japanese, no translation), and the card says it prefers deletions over insertions and may occasionally drop words in noisy conditions. + NOTE: Model card matches verbatim: 'edits a CTC hypothesis in a single forward pass using a bidirectional LLM'; languages are exactly English, French, German, Spanish, Portuguese; card explicitly redirects Japanese users to the autoregressive granite-speech-4.1-2b; no translation capability mentioned. Exact quote found: 'it prefers deletions over insertions, which reduces hallucination risk but may occasionally drop words in noisy conditions.' + +- [CONFIRMED] IBM publishes ibm-granite/granite-speech-4.1-2b-GGUF with quantizations from Q4_K_M (1.14 GB) to bf16 (3.68 GB) plus a 1.16 GB f16 mmproj audio-encoder file, runnable with llama.cpp build b9045+ including on macOS via brew. + NOTE: Repo file tree verifies digit-by-digit: Q4_K_M = 1,139,247,200 B (1.14 GB), bf16 = 3,678,444,864 B (3.68 GB), mmproj-model-f16.gguf = 1,159,354,752 B (1.16 GB); README specifies 'llama.cpp build: b9045'. One sourcing nit: the README's install instructions use a curl install script, not brew — the 'via brew' route is not in the cited source (though llama.cpp is separately available in Homebrew). Not material to the claim's substance. + +- [CONFIRMED] granite-speech-4.1-2b-plus adds prompt-invoked speaker-attributed ASR and word-level timestamps, handles up to 9 minutes per call for ASR/speaker-attribution (3.5 minutes for timestamps), and is Apache 2.0. + NOTE: Model card confirms all three: SAA ('[Speaker 1]:' tags) and word-level timestamps ('[T:N]' centisecond tags) are 'controlled by different prompts'; card states 'works well with audio segments up to 9 minutes long for ASR and SAA, and up to 3.5 minutes for timestamps'; license is Apache 2.0. Worth knowing (omitted by researcher, not contradictory): plus variant drops punctuation/capitalization, has slightly worse WER than base (5.71 vs 5.33 avg), and timestamps roll over modulo 1000 every 10 s. + +- [CONFIRMED] Blaizzy/mlx-audio-swift (MLXAudioSTT product, macOS 14+) explicitly lists Granite Speech among supported STT models, enabling in-process MLX inference in a Swift Mac app. + NOTE: GitHub README confirms: repo is 'a modular Swift SDK for audio processing with MLX on Apple Silicon', built on MLX Swift; MLXAudioSTT is a named component; requirements state macOS 14+; Granite Speech appears explicitly in the supported STT model list (alongside Whisper, Parakeet, Canary, Qwen3-ASR, etc.). + +- [CONFIRMED] Open ASR Leaderboard RTFx values are measured on an NVIDIA A100-SXM4-80GB GPU with batch size up to 64, so absolute RTFx numbers do not transfer to Apple Silicon. + NOTE: Not in the abstract, but the full paper (arXiv 2510.06961, Results section) states evaluations 'were conducted on an NVIDIA A100-SXM4-80GB GPU (driver 560.28.03, CUDA 12.6)' with 'a batch size of 64 whenever memory allowed, and reduced adaptively (48, 32, 16, …)' — matching 'up to 64' exactly. The non-transferability to Apple Silicon is the researcher's (sound) inference, not a paper statement. + +- [CONFIRMED] As of 2026-07-09, the newest IBM speech models on Hugging Face are granite-speech-4.1-2b, granite-speech-4.1-2b-nar, and granite-speech-4.1-2b-plus; no granite-speech 4.2 or 4.2.1 model exists. + NOTE: HF API listing for ibm-granite shows exactly these three as the newest speech models (created 2026-03-10 to 2026-04-16); the only later speech uploads are GGUF conversions of the same models (granite-speech-4.1-2b-GGUF 2026-05-11, granite-speech-4.1-2b-plus-GGUF 2026-06-30). Older models: granite-4.0-1b-speech, granite-speech-3.2/3.3. Targeted search for granite-speech-4.2/4.2.1 returns nothing; IBM's own Granite 4.1 blog describes the 4.1 trio as the current release. + +- [CONFIRMED] granite-speech-4.1-2b-nar is a non-autoregressive model that edits a CTC hypothesis in a single forward pass using a bidirectional LLM, supports only EN/FR/DE/ES/PT (no Japanese, no translation), and its model card states it prefers deletions over insertions and may occasionally drop words in noisy conditions. + NOTE: Model card verified point-for-point: it edits a CTC hypothesis in a single forward pass using a bidirectional LLM; languages are English, French, German, Spanish, Portuguese — the card explicitly redirects Japanese users to the autoregressive granite-speech-4.1-2b, and AST (translation) is only in the AR variant. Card quote: 'it prefers deletions over insertions, which reduces hallucination risk but may occasionally drop words in noisy conditions.' Apache 2.0. + +- [CONFIRMED] IBM publishes an official GGUF repo (ibm-granite/granite-speech-4.1-2b-GGUF) with quantizations from Q4_K_M (1.14 GB) to bf16 (3.68 GB) plus a 1.16 GB f16 mmproj audio-encoder file, runnable with llama.cpp (build b9045+) including on macOS via brew. + NOTE: Repo file tree matches exactly: Q4_K_M 1.14 GB, Q5_K_M 1.32 GB, Q6_K 1.51 GB, Q8_0 1.96 GB, bf16 3.68 GB, and mmproj-model-f16.gguf at 1.16 GB. Card states 'llama.cpp build: b9045' as the requirement. One nuance: the card's macOS/Linux install instructions use a curl script and do not mention brew — but the Homebrew llama.cpp formula is currently at b9910 (>= b9045) with Apple Silicon macOS support, so the brew route is independently valid. Not materially wrong. + +- [CONFIRMED] granite-speech-4.1-2b-plus adds prompt-invoked speaker-attributed ASR and word-level timestamps, handles audio up to 9 minutes per call for ASR/speaker-attribution (3.5 minutes for timestamps), and is Apache 2.0 licensed. + NOTE: Model card confirms prompt-controlled speaker attribution ('[Speaker 1]:' turn labels) and word-level timestamps ('[T:N]' in centiseconds mod 1000). Exact card quote: 'This model works well with audio segments up to 9 minutes long for ASR and SAA, and up to 3.5 minutes for timestamps.' License: Apache 2.0. Card also notes the trade-off: Plus drops punctuation/capitalization relative to the base model. + +- [CONFIRMED] The Swift package mlx-audio-swift (Blaizzy/mlx-audio-swift, MLXAudioSTT product, macOS 14+) explicitly lists Granite Speech among its supported speech-to-text models, enabling in-process MLX inference in a Swift Mac app. + NOTE: All literal elements verified: Granite Speech is in the README's supported STT table; the package exposes an MLXAudioSTT product; requirement is macOS 14+/iOS 17+; project is active (v0.1.3 released 2026-07-09, 307 commits). It is real code, not aspiration: Sources/MLXAudioSTT/Models/GraniteSpeech/{GraniteSpeech.swift, GraniteSpeechConfig.swift} plus test coverage. IMPORTANT variant caveat: the GraniteSpeech module README lists only mlx-community/granite-4.0-1b-speech-5bit (~1B) as the available checkpoint — no MLX conversion of granite-speech-4.1-2b/-nar/-plus is listed, so running the 4.1-2b variants through this package is not demonstrated. The claim as written does not assert 4.1 support, so it stands, but do not extrapolate it to the 4.1 models. + +- [CONFIRMED] Open ASR Leaderboard RTFx values are measured on an NVIDIA A100-SXM4-80GB GPU with batch size up to 64, so absolute RTFx numbers do not transfer to Apple Silicon. + NOTE: arXiv 2510.06961 (the Open ASR Leaderboard paper) full text states measurements were 'conducted on an NVIDIA A100-SXM4-80GB GPU (driver 560.28.03, CUDA 12.6)' 'using a batch size of 64 whenever memory allowed, and reduced adaptively (48, 32, 16, ...)'. The inference that absolute RTFx does not transfer to Apple Silicon (different hardware, no 64-way batching in a local single-stream app) is sound. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/research-higgs.md b/docs/superpowers/research/2026-07-09-asr-model-roi/research-higgs.md new file mode 100644 index 0000000..2621c96 --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/research-higgs.md @@ -0,0 +1,88 @@ +# Deep-dive: bosonai/higgs-audio-v3-8b-stt-v2 for Tome + +## 1. What it is + +Per the [HF model card](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2), this is **not** a general audio LLM in the higgs-audio TTS lineage — it is a dedicated English-only ASR model built as a **Whisper-Large-v3 encoder + Qwen3-8B decoder, 8.91B total parameters**, with LoRA fine-tuning of the decoder across ASR benchmarks. The [config.json](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2/raw/main/config.json) confirms: `HiggsAudio3Model` / `model_type: higgs_audio_3`; text decoder is Qwen3-8B-shaped (hidden 4096, 36 layers, 32 heads, vocab 151936, 32k context, rope_theta 1e6); audio encoder is Whisper-large-v3-shaped (d_model 1280, 32 layers, 20 heads, 128 mel bins). Base model field: `bosonai/higgs-audio-v3-8b` — which returns **HTTP 401** (private/inaccessible) on HF, so base provenance is opaque. + +**STT invocation is prompting, not a dedicated head.** The bundled [transcribe.py](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2/raw/main/transcribe.py) (loaded via `trust_remote_code=True`) builds a ChatML prompt: `<|im_start|>user\n` + instruction + `<|audio_bos|><|AUDIO|><|audio_eos|>` + `<|im_end|>\n<|im_start|>assistant\n`. Default instruction: *"Transcribe the speech. Output only the spoken words in lowercase with no punctuation."* — i.e., out of the box it produces leaderboard-normalized lowercase/no-punctuation text, not readable meeting transcripts; changing the prompt for formatted output departs from the benchmarked configuration. Generation is greedy (`do_sample=False`), `max_new_tokens=1024`, stop on `<|im_end|>`. There is an optional **chain-of-thought "thinking" mode** (`enable_thinking`, default True; disabling injects an empty `\n\n` block, Qwen3-style). A sibling smaller variant exists: [bosonai/higgs-audio-v3-stt](https://huggingface.co/bosonai/higgs-audio-v3-stt) (2.68B, Whisper-Large-v3 + Qwen3-1.7B, also Apache-2.0). + +Note: the user-suspected "granite-speech-4.2.1-2b" is out of scope here, but for the record no such higgs analogue matters; this repo's latest STT is v2 of the 8B (last modified 2026-05-22 per the [HF API](https://huggingface.co/api/models/bosonai/higgs-audio-v3-8b-stt-v2)). + +## 2. License + +- The repo's YAML frontmatter and HF API both report **`license: apache-2.0`**, and the API confirms **`gated: false`** ([model page](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2), [API metadata](https://huggingface.co/api/models/bosonai/higgs-audio-v3-8b-stt-v2)). Commercial use in a shipped Mac app is permitted under Apache-2.0. +- **Caveats:** (a) there is **no standalone LICENSE file** in the [repo tree](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2/tree/main) — the grant rests on the metadata tag alone; (b) Boson's **TTS** v3 models ship under a "Boson Higgs Audio v3 Research and Non-Commercial License" with commercial use requiring a separate license (e.g., [higgs-audio-v3-tts-4b](https://huggingface.co/bosonai/higgs-audio-v3-tts-4b)) — the STT repos are the exception, tagged Apache; (c) the base model `bosonai/higgs-audio-v3-8b` is private, so you cannot inspect its terms. Net: usable commercially per the published tag, with low-moderate residual ambiguity. Ecosystem is tiny: ~1,740 downloads, 13 likes, one discussion thread. + +## 3. Why RTFx is only 75.66, and M2 Max wall-clock estimate + +**Why slow:** It's an 8B **autoregressive decoder** that must generate every transcript token serially — vs. Parakeet-TDT (0.6B non-autoregressive-ish TDT, RTFx 3386) or granite-4.1-2b-nar (non-autoregressive, RTFx 879). Three compounding factors, all verifiable in the eval setup: +1. 8B decoder, BF16, HF `transformers` with custom code — README/eval use `attn_implementation="eager"` (no FlashAttention), with a repo-supplied `cuda_graph_runner.py` as the only speed optimization (CUDA-only). +2. The [leaderboard harness](https://github.com/huggingface/open_asr_leaderboard) (`higgs_audio/run_eval_higgs_audio.py` + `run_higgs_audio.sh`) runs `bosonai/higgs-audio-v3-8b-stt-v2` at **batch size 64** (32 for VoxPopuli) and never passes `enable_thinking` — so `transcribe_batch` uses its **default thinking=True**; the harness's own `--max_new_tokens` help text says it "includes the chain-of-thought block". CoT tokens inflate generation length. +3. Leaderboard hardware is **1x NVIDIA H200 (141 GB)** per the [open_asr_leaderboard README](https://github.com/huggingface/open_asr_leaderboard/blob/main/README.md) (not A100 as historically). RTFx = total audio seconds / total transcription seconds, so 75.66 at batch 64 means only ~1.2x real-time **per concurrent stream** on a ~990 TFLOPS / 4.8 TB/s datacenter GPU. + +**M2 Max estimate for a 60-min meeting** (M2 Max GPU: ~400 GB/s bandwidth, roughly 13-27 TFLOPS FP16 — i.e., ~12x less bandwidth, ~35-70x less compute than H200): +- Token math (batch-1, memory-bandwidth-bound decode): ~150 wpm × 60 min ≈ 9k words ≈ 11-13k output tokens, plus thinking overhead if enabled; plus 120× 30-s chunk Whisper-encoder passes and prefills. 8B BF16 decode ceiling on M2 Max ≈ 400 GB/s ÷ 16.4 GB ≈ 24 tok/s theoretical; realistic MLX-class ≈ 15-20 tok/s; `transformers`-on-MPS realistically 5-10 tok/s. +- **Unported path (transformers on MPS, if it can be made to run at all): ~30-60+ minutes per hour of audio (RTF ≈ 0.5-1.0), worse with thinking enabled — potentially slower than real time.** +- Hypothetical native MLX FP16 port: ~10-15 min/hr (RTF ≈ 0.17-0.25) — roughly matching, not beating, Tome's already-working WhisperKit large-v3-turbo (measured p95 RTF 0.130 ≈ 8 min/hr). A 4-bit port (~5 GB decoder) could reach ~5-8 min/hr, but no such port exists and quantized WER is uncharacterized. +- Cross-check by compute scaling: H200 processes 60 min in 3600/75.66 ≈ 48 s batched; ×35-70 compute deficit → 28-56 min on M2 Max, consistent with the token math. + +## 4. Apple Silicon paths — none exist + +- **MLX:** No port. [mlx-audio](https://github.com/Blaizzy/mlx-audio) supports Higgs Audio **TTS** (v2/v3) and STT for Whisper, Parakeet, Voxtral Realtime, Qwen3-ASR, VibeVoice — **not** Higgs STT. Nothing in [mlx-community](https://huggingface.co/mlx-community). +- **GGUF/llama.cpp:** No GGUFs on HF; custom `higgs_audio_3` architecture with a Whisper-style audio encoder has no llama.cpp support. +- **vLLM:** Not mentioned on the card; the only published inference path is HF `transformers` + `trust_remote_code` with CUDA-specific helpers (`cuda_graph_runner.py`, examples hard-coding `device_map="cuda:0"`). +- **MPS reports:** None found — no community attempts surfaced in the repo's single discussion thread, Reddit, or HN. Integration into Tome (Swift) would mean hand-porting ~230 KB of custom Python modeling code to MLX/CoreML yourself. + +## 5. Accuracy on meeting-like audio — the profile is wrong for Tome + +Full ESB table from the [model card](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2): LS Clean **1.25**, LS Other 2.38, TED-LIUM 3.09, SPGISpeech 3.60, VoxPopuli 5.92, GigaSpeech 8.47, Earnings22 **8.73**, **AMI 10.14**; 8-set average 5.449 (the leaderboard's 4.02 uses a different composite; its independently reproduced Earnings22 8.79 matches the card's 8.73). The rank-3 average is driven by clean read speech. On the two meeting-like sets it **loses to granite**: AMI 10.14 vs [granite-speech-4.1-2b](https://huggingface.co/ibm-granite/granite-speech-4.1-2b)'s **8.09** (a 2B Apache-2.0 model), and Earnings22 8.73-8.79 vs granite's 8.37-8.44. English-only — no multilingual support; accented-English performance is proxied by Earnings22, where it is mediocre. + +## 6. Practical verdict ingredients + +- **Download:** 4 safetensors shards, **~17.8 GB** BF16 ([repo tree](https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2/tree/main)). **RAM:** ~20-24 GB peak BF16 (weights + KV + encoder) — fits the 64 GB M2 Max but is 10-20x Tome's current models. 4-bit would be ~5-6 GB but doesn't exist. +- **Chunking:** fixed 30-second chunks via the collator (`chunk_size_seconds=30`), **no overlap, no VAD** — cross-chunk word-truncation risk on hour-long meetings; no documented long-form pipeline. +- **Hallucination/repetition:** the vendor ships mitigations in-repo — `_fix_repetitions()` caps consecutive identical words at 3, plus Whisper EnglishTextNormalizer post-processing; the sibling v3-stt card notes a June 2026 update "adding deterministic post-processing for repetition reduction." That's vendor-acknowledged repetition-loop tendency typical of LLM decoders. +- **Output format trap:** benchmarked WERs assume the lowercase/no-punctuation prompt + normalizer; readable transcripts require an unbenchmarked prompt. + +**Verdict for Tome: not viable.** No Apple Silicon runtime exists (porting cost is very high), estimated 30-60+ min per meeting-hour on M2 Max even if ported naively, ~18 GB download, English-only, fixed 30s chunking, and — decisively — it is *worse* on meeting audio (AMI, Earnings22) than granite-speech-4.1-2b, a 2B Apache-2.0 model 9x smaller. Its leaderboard rank is a clean-speech artifact irrelevant to Tome's accented/imperfect-meeting-audio goal. + +## BOTTOM LINE +higgs-audio-v3-8b-stt-v2 is an 8.91B Whisper-Large-v3-encoder + Qwen3-8B-decoder ASR model, Apache-2.0 tagged and ungated, invoked by ChatML prompting with optional chain-of-thought — which, combined with eager-attention HF-transformers inference, explains RTFx 75.66 at batch 64 on the leaderboard's H200. It has zero Apple Silicon path (no MLX, GGUF, or MPS reports; CUDA-specific custom code), ~17.8 GB BF16 weights, and would take an estimated 30-60+ minutes to transcribe a 60-minute meeting on M2 Max even if ported. Critically, its rank-3 average is driven by clean read speech: on meeting-like audio it is worse than granite-speech-4.1-2b (AMI 10.14 vs 8.09; Earnings22 8.73 vs 8.37). Not recommended for Tome — granite or the existing Whisper large-v3-turbo path dominate it on every axis that matters. + +## VERIFICATION VERDICTS + +- [CONFIRMED] bosonai/higgs-audio-v3-8b-stt-v2 is an English-only ASR model combining a Whisper-Large-v3 encoder with a Qwen3-8B decoder (LoRA fine-tuned), 8.91B total parameters, invoked via ChatML prompting with trust_remote_code and an optional 'thinking' (chain-of-thought) mode. + NOTE: Model card (https://huggingface.co/bosonai/higgs-audio-v3-8b-stt-v2) states English-only, encoder 'Whisper-Large-v3 (frozen)', decoder 'Qwen3-8B (LoRA fine-tuned, merged)', '8.91B total parameters' (API safetensors count 8,905,965,568 BF16 = 8.906B, rounds to 8.91B). Usage requires trust_remote_code=True; bundled transcribe.py builds ChatML prompts (<|im_start|>user ... <|im_start|>assistant) and has an enable_thinking flag that appends a block. Every element checks out. + +- [CONFIRMED] The bosonai/higgs-audio-v3-8b-stt-v2 HF repo is tagged license: apache-2.0 and is not gated (gated: false), though it contains no standalone LICENSE file; Boson's v3 TTS models by contrast use a research/non-commercial license. + NOTE: API JSON (https://huggingface.co/api/models/bosonai/higgs-audio-v3-8b-stt-v2) shows tag 'license:apache-2.0' and gated: false; siblings list has 4 safetensors shards, transcribe.py, configs — no LICENSE file. Boson's v3 TTS model bosonai/higgs-tts-3-4b (a.k.a. higgs-audio-v3-tts-4b) is 'Released for research and non-commercial use under the Boson Higgs TTS 3 Research and Non-Commercial License' (license tag 'other'). Contrast is accurate. + +- [CONFIRMED] The model card reports WER of 10.14 on AMI and 8.73 on Earnings22 (vs 1.25 on LibriSpeech Clean), while ibm-granite/granite-speech-4.1-2b reports AMI WER 8.09 — i.e., higgs is worse on meeting-like audio despite its better leaderboard average. + NOTE: Higgs card table matches digit-for-digit: AMI 10.14%, Earnings22 8.73%, LibriSpeech Clean 1.25% (card average 5.449%). Granite-speech-4.1-2b HF page shows AMI 8.09 (mean 5.33). The comparative thesis also holds on the live leaderboard data (hf-audio/open-asr-leaderboard-results english_short_latest.csv): higgs avg WER 4.73 beats granite 4.90, yet higgs is worse on AMI (8.37 vs 7.06) and Earnings22 (8.45 vs 8.23). Caveat: comparing self-reported card averages alone (5.449 vs 5.33) would reverse the average ranking; the claim is correct specifically for the leaderboard-run averages, which is what it asserts. + +- [CONFIRMED] The Open ASR Leaderboard measures RTFx on a single NVIDIA H200 (141 GB) GPU, and its higgs_audio harness evaluates bosonai/higgs-audio-v3-8b-stt-v2 at batch size 64 via HF transformers transcribe_batch, with max_new_tokens documented as including the chain-of-thought block. + NOTE: README of github.com/huggingface/open_asr_leaderboard specifies '1x H200 (141 GB)' (23 vCPU, 256 GB RAM) with Dockerized runs. higgs_audio/run_higgs_audio.sh sets MODEL=bosonai/higgs-audio-v3-8b-stt-v2 with default batch size 64; run_eval_higgs_audio.py loads via AutoModel.from_pretrained(..., trust_remote_code=True) and calls transcribe_batch(..., max_new_tokens=args.max_new_tokens), whose help text reads 'Maximum number of tokens to generate (includes the chain-of-thought block).' Minor caveat: voxpopuli is overridden to batch size 32; all other datasets use 64. + +- [REFUTED] No Apple Silicon port of higgs-audio STT exists: mlx-audio supports Higgs Audio TTS plus Whisper/Parakeet/Voxtral/Qwen3-ASR for STT, but not Higgs STT, and no GGUF/llama.cpp or MPS ports are published. + NOTE: The mlx-audio half is accurate (README lists Higgs Audio v2/v3 under TTS only; STT table has Whisper, Distil-Whisper, Qwen3-ASR, Parakeet, Voxtral/Voxtral Realtime, Canary, etc., no Higgs STT), and no GGUF weights or llama.cpp port were found. But the headline assertion 'No Apple Silicon port of higgs-audio STT exists' is stale: 0xShug0/audio.cpp — a pure C++ ggml inference engine with a Metal backend enabled by default on Apple platforms — released Higgs Audio v3 STT support on 2026-07-08 (repo table: 'higgs_audio_stt | ASR | en | Higgs Audio v3 STT | released'; it loads original HF weights, GGUF loading 'planned, but not supported yet'). So a ggml/Metal Apple Silicon port does exist as of yesterday, though it is brand-new and I could not confirm whether it targets the 8b-stt-v2 checkpoint specifically or the smaller higgs-audio-v3-stt. Sources: https://github.com/Blaizzy/mlx-audio, https://github.com/0xShug0/audio.cpp + +- [CONFIRMED] The model weights are 4 BF16 safetensors shards totaling approximately 17.8 GB, and the bundled transcribe.py chunks audio into fixed 30-second segments with a repetition-capping post-processor (_fix_repetitions limiting consecutive identical words to 3). + NOTE: API siblings list model-00001-of-00004.safetensors through model-00004-of-00004.safetensors; safetensors dtype breakdown is 100% BF16 (8,905,965,568 params × 2 bytes = 17.81 GB, matching ~17.8 GB). transcribe.py (raw/main/transcribe.py) uses chunk_size_seconds=getattr(config, 'chunk_size_seconds', 30) and defines _fix_repetitions with max_repeat=3 (docstring: '"he he he he he" with max_repeat=3 becomes "he he he"'); transcribe_batch(model, tokenizer, audios, sample_rates=16000, max_new_tokens=1024) is the public API. + +- [CONFIRMED] bosonai/higgs-audio-v3-8b-stt-v2 is an English-only ASR model combining a Whisper-Large-v3 encoder with a Qwen3-8B decoder (LoRA fine-tuned), 8.91B total parameters, invoked via ChatML prompting with trust_remote_code and an optional 'thinking' (chain-of-thought) mode. + NOTE: All elements verified from the model card and bundled code. Card states Whisper-Large-v3 (frozen) encoder + Qwen3-8B decoder 'LoRA fine-tuned, merged', 8.91B params, language: en only, 'Supports: Thinking mode for improved accuracy'. Usage requires AutoModel.from_pretrained(..., trust_remote_code=True). The bundled transcribe.py builds ChatML messages (<|im_start|>user ... <|im_end|>, audio via <|audio_bos|><|AUDIO|><|audio_eos|>) and exposes enable_thinking (default True), injecting an empty \n\n block to suppress CoT when disabled. One nuance: the ChatML/thinking mechanics live in the bundled script, not the card prose. + +- [CONFIRMED] The bosonai/higgs-audio-v3-8b-stt-v2 HF repo is tagged license: apache-2.0 and is not gated (gated: false), though it contains no standalone LICENSE file; Boson's v3 TTS models by contrast use a research/non-commercial license. + NOTE: HF API shows "license:apache-2.0" tag and "gated":false. Full file tree (25 files) contains no LICENSE file — license exists only as YAML front-matter metadata. Contrast verified: bosonai/higgs-tts-3-4b (Boson's v3 TTS) is tagged license:other and its card specifies the 'Boson Higgs TTS 3 Research and Non-Commercial License' (commercial/hosted use requires a separate license; includes a Creator Use Grant). All bosonai STT repos are apache-2.0; all their TTS repos are license:other. + +- [CONFIRMED] The model card reports WER of 10.14 on AMI and 8.73 on Earnings22 (vs 1.25 on LibriSpeech Clean), while ibm-granite/granite-speech-4.1-2b reports AMI WER 8.09 — i.e., higgs is worse on meeting-like audio despite its better leaderboard average. + NOTE: Higgs card: AMI 10.14, Earnings22 8.73, LS-Clean 1.25, 8-benchmark average 5.449 — all verified. Granite AMI 8.09 verified from the repo's primary eval file (.eval_results/open_asr_leaderboard.yaml, dated 2026-04-23). The live leaderboard's own re-normalized data agrees directionally: higgs-8b-v2 AMI 8.37 vs granite 7.06, while higgs holds the better 'avg cleaned' (4.73 vs 4.90) — supporting 'worse on meetings despite better leaderboard average'. Caveat: by self-reported card numbers granite's simple 8-set mean (5.33) actually beats higgs's 5.449, and on the leaderboard's 'avg original' metric granite also edges ahead (5.18 vs 5.25); the 'better average' holds only on the leaderboard's headline cleaned metric. The core comparative fact (higgs worse on meeting audio) holds in every source. + +- [CONFIRMED] The Open ASR Leaderboard measures RTFx on a single NVIDIA H200 (141 GB) GPU, and its higgs_audio harness evaluates bosonai/higgs-audio-v3-8b-stt-v2 at batch size 64 via HF transformers transcribe_batch, with max_new_tokens documented as including the chain-of-thought block. + NOTE: Repo README lists exactly one hardware flavor for eval jobs: 'h200 ... 1x H200 (141 GB)'; no other GPU mentioned. higgs_audio/run_higgs_audio.sh sets MODEL_ID=bosonai/higgs-audio-v3-8b-stt-v2 and BATCH_SIZE=64 (minor nuance: VoxPopuli is overridden to 32). run_eval_higgs_audio.py loads via transformers AutoModel(trust_remote_code=True), obtains transcribe_batch from the model's remote code and calls transcribe_batch(model, tokenizer, audios, ...); its --max_new_tokens (default 1024) help text reads 'Maximum number of tokens to generate (includes the chain-of-thought block).' All verified from the repo files. + +- [REFUTED] No Apple Silicon port of higgs-audio STT exists: mlx-audio supports Higgs Audio TTS plus Whisper/Parakeet/Voxtral/Qwen3-ASR for STT, but not Higgs STT, and no GGUF/llama.cpp or MPS ports are published. + NOTE: The mlx-audio half is accurate (Higgs Audio v2/v3 appear only in the TTS list; STT list is Whisper/Parakeet/Voxtral/Voxtral-Realtime/Qwen3-ASR, no Higgs STT). But 'no GGUF/llama.cpp ports are published' is false: (1) cstr/higgs-audio-v3-stt-GGUF (created 2026-06-29, updated 2026-07-03) publishes f16/q8_0/q4_k/q3_k+imatrix GGUFs of bosonai/higgs-audio-v3-stt (the Whisper-large-v3 + Qwen3-1.7B variant) with working inference via CrispASR — a ggml C++ runtime (whisper.cpp fork) that lists higgs-stt as a supported backend and builds on Apple Silicon with -DGGML_METAL=ON, i.e. an Apple Silicon path for Higgs STT does exist, ~485 downloads/month. (2) nopesadly/higgs-audio-v3-8b-stt-v2-Q4_K_M-GGUF (2026-05-28) publishes a GGUF of this exact 8B variant, though it's a GGUF-my-repo auto-conversion (single 4.68 GB q4_k_m, no audio-encoder/mmproj) and almost certainly cannot do ASR in stock llama.cpp. Correct statement: no Apple Silicon port of the 8B v2 variant is demonstrated, but Higgs STT GGUF ports are published and the 1.7B variant runs on Apple Silicon via CrispASR. + +- [CONFIRMED] The model weights are 4 BF16 safetensors shards totaling approximately 17.8 GB, and the bundled transcribe.py chunks audio into fixed 30-second segments with a repetition-capping post-processor (_fix_repetitions limiting consecutive identical words to 3). + NOTE: Tree API: exactly 4 shards (model-00001..00004-of-00004.safetensors) totaling 17,811,032,664 bytes = 17.81 GB. BF16 consistent with card/eval usage (torch_dtype=bfloat16). transcribe.py verified: chunk_size_seconds defaults to 30 (passed to HiggsAudioSampleCollator), and _fix_repetitions exists with docstring 'Cap consecutive word repetitions at max_repeat. E.g. "he he he he he" with max_repeat=3 becomes "he he he"' — i.e., consecutive identical words capped at 3, guarding against runaway greedy loops. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/research-repo.md b/docs/superpowers/research/2026-07-09-asr-model-roi/research-repo.md new file mode 100644 index 0000000..11d9645 --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/research-repo.md @@ -0,0 +1,153 @@ +# Tome ASR Model Setup — Scalability Review (model N+1 and live/post split) + +Reviewed at commit `573341d` on `main`. All paths relative to `/Users/nic/programming/tome`. + +## 1. Cost of model N+1 — inventory of every code site + +The whisper-v3-turbo work (spec `docs/superpowers/specs/2026-07-08-whisper-v3-turbo-model-option-design.md`) left a clean seam: everything downstream of the enum is model-agnostic. Adding a third **live-capable, in-process** model touches: + +### Production code (7 sites, 5 of them compiler-enforced exhaustive switches) + +| # | Site | Change | +|---|------|--------| +| 1 | `Tome/Sources/Tome/Transcription/TranscriberModel.swift:3-5` | New enum case + stable raw value (persisted format — pick carefully, it's forever) | +| 2 | `TranscriberModel.swift:7-12` | `displayName` switch arm | +| 3 | `TranscriberModel.swift:15-20` | `pickerSubtitle` switch arm | +| 4 | `TranscriberModel.swift:32-37` | `isInstalled` switch arm → `NewBackend.isInstalled()` | +| 5 | `TranscriberModel.swift:41-47` | `approxDownloadSize` switch arm | +| 6 | `Tome/Sources/Tome/App/AppServices.swift:69-74` | `makeBackend` factory switch arm → `NewBackend()` | +| 7 | **New file** `Tome/Sources/Tome/Transcription/NewBackend.swift` | The real cost: an actor conforming to `ASRBackend` (Parakeet is 63 lines, Whisper 150 — the delta is download-location plumbing, `isInstalled` completeness, and result mapping) | + +Sites 1–6 are exhaustive switches with no `default:` — the compiler produces the checklist. That's a deliberate, good property at N=3–5. + +### Sites that need NO change (verified) + +- `Tome/Sources/Tome/Views/SettingsView.swift:152` — picker iterates `TranscriberModel.allCases`; row subtitles (`:238-243`) derive from `model.isInstalled`/`approxDownloadSize`. Scales automatically. +- `ModelProvisioner.swift` — fully model-agnostic (selection/factory injected as closures). +- `ASRCoordinator.swift`, `StreamingTranscriber.swift`, `SegmentReTranscriber.swift`, `PostProcessingJob/Queue.swift`, `Recovery.swift:114`, `ContentView.swift:142-164` (onChange + boot kick), `TranscriptionEngine.swift:309` (status string uses `activeModel?.displayName`), APIServer `/health`. + +### Tests (additive only) + +- `Tome/Tests/TomeTests/TranscriberModelTests.swift:6-21` — add raw-value pin, displayName, and `from(persisted:)` lines for the new case. +- A new `NewBackendTests.swift` in the pattern of `WhisperBackendTests.swift` (variant/path resolution, pure functions only). +- `ModelProvisionerTests`/`ASRCoordinatorTests`/`FakeBackend` — untouched; they use the two existing cases as arbitrary distinct labels. + +### The one genuinely non-scaling site: ASRBench + +`Tome/Sources/ASRBench/main.swift` hand-mirrors backend config because "SwiftPM forbids importing the app executable from here" (`main.swift:14-15`, explicit "keep in sync" comment at `:16-18`). Each backend has its own ~50-line bench function (`benchParakeet` :83-121, `benchWhisper` :124-174), and the top level hardcodes exactly two runs, two reports, and a two-element JSON array (`main.swift:222-235`). Model N+1 = another hand-mirrored config block + bench function + edits at 3 places, with silent-drift risk against the real backend. Fixable by extracting a `TomeASR` library target both the app and ASRBench import (~half a day, mechanical). + +**Net cost of a live-capable model N+1: ~1 day of plumbing + the backend itself + ASRBench mirror + manual smoke/bench runs.** The plumbing is not the bottleneck; validating the backend is. + +## 2. Is `ASRBackend` adequate for a 2B-LLM-class backend? + +The protocol (`Tome/Sources/Tome/Transcription/ASRBackend.swift:20-33`): + +```swift +protocol ASRBackend: AnyObject, Sendable { + var model: TranscriberModel { get } + static func isInstalled() -> Bool + func prepare(onEvent:) async throws + func transcribe(samples:language:) async throws -> ASRResult + func transcribe(buffer:language:) async throws -> ASRResult + func unload() async +} +``` + +### (a) Out-of-process sidecar: mostly adequate + +A proxy actor speaking XPC/stdio to a helper process conforms fine: `prepare` = spawn + load (both `PrepareEvent` phases map), `unload` = terminate, `AnyObject` identity gives the coordinator's `ObjectIdentifier`-keyed in-flight/retired tracking (`ASRCoordinator.swift:24-26`) a stable key. Gaps: + +- **No health/restart notion.** A crashed sidecar surfaces only as thrown `transcribe` errors. Live: `StreamingTranscriber.swift:124,147` tolerates 10 consecutive errors, then kills the leg with "restart session". Batch: `SegmentReTranscriber.swift:71-78` writes `"[transcription failed]"` per segment and *keeps going* — a dead sidecar produces a transcript of failure placeholders rather than aborting the job. Nothing re-`prepare`s a backend except a user-driven provision/retry cycle or the flip-back re-assert (`ModelProvisioner.swift:130-144`). +- **`static func isInstalled()` is per-type**, so one Swift class per model. An LLM *family* (e.g. 2B vs 8B quantizations of the same runner) forces class-per-variant or breaking the static. (Whisper dodges this with device-resolved variants inside its statics, `WhisperBackend.swift:20-26` — workable once, not a pattern.) +- Protocol couples to FluidAudio's `Language` and `ASRResult` types (`ASRBackend.swift:28-29`); every non-FluidAudio backend hand-constructs `ASRResult` as WhisperBackend already does (`WhisperBackend.swift:104-110`). Acceptable, but the adapter drags FluidAudio into sidecar code. + +### (b) Minutes-long batch latency: this is where it actually breaks + +- **Backends are actors** ("Conformances are actors: they own mutable SDK handles… that must be serialized" — `ASRBackend.swift:13-14`; `ParakeetBackend.swift:15`, `WhisperBackend.swift:10`). The coordinator itself does NOT serialize (it suspends at the backend await — `ASRCoordinator.swift:9-13`), but the backend actor does. With one shared model, a minutes-long batch segment transcription blocks every live VAD chunk queued behind it on the same actor. Live becomes unusable during post-processing. **A slow accuracy model requires the dual-model split; it cannot ride the current single slot.** +- **No cancellation or timeout inside the batch loop**: `SegmentReTranscriber.run()` (`:43-80`) has zero `Task.isCancelled` checks; `PostProcessingJob` checks only between phases (`:106,138`). Cancellation of a minutes-per-segment job would take up to a full segment to land, and only if the SDK cooperates. +- **No progress**: `PostProcessingJob.progress` (`PostProcessingJob.swift:24`) is declared and never written anywhere. Invisible for a 30-second job; unacceptable for a 20-minute one. +- **Boot coupling**: orphan recovery awaits `modelProvisioner.awaitSettled()` (`ContentView.swift:891,1061`; poll loop `ModelProvisioner.swift:109-113`) — a multi-GB model download at launch blocks recovery for its duration. +- **Picker lock duration**: `SettingsView.swift:229-234` locks model changes while any job runs — correct policy, but with minutes-long jobs the lock window becomes very long, and it's one combined lock (can't change the live model while an accuracy job runs). + +### (c) No live/streaming support: not expressible at all + +- No capability flag exists on `ASRBackend` or `TranscriberModel`. The picker (`SettingsView.swift:151-165`) is one radio group whose selection drives **both** paths — the spec says so explicitly ("The selected model serves both live streaming transcription and post-processing re-transcription", spec Goals; per-task split is an explicit Non-Goal at spec line 38-39). +- `canStartRecording` (`ModelProvisioner.swift:48-50`) is `activity == .none && servingModel == selection()` — a post-processing-only model, once serving, would **enable** recording and route live audio through it. +- The F2 failure fallback (`ModelProvisioner.swift:210-215`) falls back to `lastGoodModel` with no notion of live-capability. + +### How the transcribers acquire the backend (traced) + +Neither ever holds a backend — this is the architecture's best asset for a split: + +- `StreamingTranscriber` holds `ASRCoordinator` (`StreamingTranscriber.swift:7`), calls `asrCoordinator.transcribe(samples:source:)` per VAD segment (`:173`). +- `SegmentReTranscriber` holds `ASRCoordinator` (`SegmentReTranscriber.swift:8`), calls `transcribe(buffer:source:)` per diarized segment (`:65`), `source` hardcoded `.system`. +- The coordinator resolves `activeBackend` fresh per call (`ASRCoordinator.swift:109-112`) with per-backend in-flight counting. + +So a dual-slot coordinator changes **one line at each call site** (name a role). But note: `ASRCoordinator.transcribe` accepts `source: AudioSource` and **never reads it** (`:83-107` — the parameter is dead), and it's the wrong routing axis anyway: batch re-transcription of call captures also passes `.system`, so live-system-audio and batch calls are indistinguishable at the coordinator today. The routing key a split needs is *purpose* (live vs accuracy), not source. + +### Dual-model coordinator — full touch surface + +- `ASRCoordinator.swift`: `activeBackend`(:15) → `[ASRRole: any ASRBackend]`; `lastInstallToken`(:22) per role; `install(backend:role:token:)`; `transcribe(…, role:)`; `isReady`/`activeModel`(:32-33) per role. `inFlight`/`retired`(:24-26) are already per-backend-identity and generalize — but a new hazard appears when both slots hold the *same* instance ("accuracy = same as live"): retiring it from one slot must not unload it while the other slot still serves it. Genuinely new state; needs its own tests. +- `ModelProvisioner.swift`: everything is single-slot — `servingModel`/`servingBackend`(:37-41), one `generation`(:70), one `lastGoodKey`(:60), the whole F1/F2/F3 ladder(:204-220). Cleanest path: **two provisioner instances** with distinct defaults keys + a "same as live" sentinel for the accuracy slot, rather than one dual-slot machine. +- `AppSettings.swift:30` + new persisted key; `ContentView.swift:142-147,164` onChange/boot kick ×2; `SettingsView` second picker with **per-slot** lock conditions (live model locked while recording; accuracy model locked while jobs/recovery run). +- `TranscriptionEngine.swift:120` (`isReady` gate) and `:309` (status string) → live slot; `Recovery.swift:114` → accuracy slot; APIServer `/health` → live slot. +- Memory: two models steady-state resident — the spec's stated invariant is one (spec Risks: "steady state is one model") and a 2B LLM + Parakeet is the point where that matters. + +## 3. Refactor options + +### Option 1 — ModelDescriptor registry (~0.5–1 day) +Keep `TranscriberModel` as the persistence-identity enum (raw values are pinned by `TranscriberModelTests.swift:6-10` — do not touch), move `displayName`/`pickerSubtitle`/`isInstalled`/`approxDownloadSize` + the `AppServices` factory into a descriptor struct in a static table. Kills the layering inversion (identity enum importing concrete backends, `TranscriberModel.swift:34-35`) and un-statics `isInstalled` (enables variant families). N+1 becomes: one enum case + one table row + backend file. +**Tests pinned today:** `TranscriberModelTests` (4 tests: raw stability, displayName, persisted fallback, didSet persistence) pass unchanged if the computed properties become lookups. `ModelProvisionerTests` (15 tests), `ASRCoordinatorTests` (7 tests) untouched. +**Verdict:** cheap, marginal payoff at N=3 by itself, but the prerequisite for option 2. + +### Option 2 — Capability flags (`supportsLive`, latency class) (~1–2 days, on top of 1) +Descriptor gains `supportsLive: Bool` (+ optional caution copy — spec §8 already anticipated "may lag during live transcription"). Changes: picker annotates/filters (`SettingsView.swift:152`); `canStartRecording` adds `servingModel.supportsLive` (`ModelProvisioner.swift:48-50`); F2 fallback (`:210-215`) skips non-live-capable models; ControlBar/API error copy. +**Test impact:** `TranscriberModelTests` pins the flags; `ModelProvisionerTests` +~3 cases (post-only model selected → recording gated with correct message; fallback skips post-only; retry); `FakeBackend` (`Tome/Tests/TomeTests/FakeBackend.swift:8`) gains a capabilities knob. +**Verdict:** the minimum to *safely* add a post-only model — but in the single-slot world it means "recording disabled while the accuracy model is selected", which is poor UX. It's a guard rail, not the feature. + +### Option 3 — Dual-slot coordinator + second provisioner (~3–5 days + state-machine re-audit) +As traced in §2. This is what a 2B accuracy model actually requires (the actor-serialization problem in §2b makes options 1–2 insufficient). +**Test impact:** `ASRCoordinatorTests` — all 7 tests gain a role parameter (mechanical) + new cross-slot tests (install into accuracy slot doesn't disturb live backend; shared-instance retire must not unload a backend the other slot still serves; per-role token ordering). `ModelProvisionerTests` — the existing 15 run unchanged against the live-slot instance if provisioners are per-slot; +~5 interplay tests. The spec's §9 "state-audit agent pass" needs re-running — the F-1/I-1 token races were hand-audited for one slot. + +**Recommended sequencing:** Option 1 now (do it as part of model N+1 — same files), Option 2 when the first post-only model is real, Option 3 only with ASRBench-class evidence that a live/accuracy split pays for its complexity. + +## 4. Existing tech debt that makes model N+1 riskier + +1. **Dead `source` parameter** — `ASRCoordinator.swift:83,96`: accepted, never read, and the wrong axis for the routing a split needs (batch passes `.system` at `SegmentReTranscriber.swift:65`, identical to live system audio). A future author may reasonably assume it routes. Repurpose to a `purpose`/role or delete. +2. **ASRBench hand-mirroring** — `Sources/ASRBench/main.swift:14-18` ("keep in sync") and the hardcoded two-model run/report/JSON at `:222-235`. Every new model doubles down on drift risk in the very tool used to accept it. +3. **Model-specific constants in shared pipeline code** — `StreamingTranscriber.swift:129` drops sub-8000-sample segments with the rationale "Parakeet emits garbage below this threshold", applied to *all* backends; `SegmentReTranscriber.swift:41` pads to 1.5s "to clear Parakeet's 1s minimum", ditto; `StreamingTranscriber.swift:47-48` flush interval tuned "for Parakeet-TDT". A new model inherits Parakeet's tuning silently. These belong on the backend/descriptor. +4. **SDK calls in UI-render paths** — `TranscriberModel.approxDownloadSize` (`TranscriberModel.swift:41-47`) calls `WhisperBackend.resolveVariant()` → `WhisperKit.recommendedModels()`, and `isInstalled` (`:32-37`) stats the filesystem; both run per row per Settings render (`SettingsView.swift:238-243`). Fine at N=2; a pattern that accretes cost per model. +5. **`PostProcessingJob.progress` never written** (`PostProcessingJob.swift:24`) — latent now, blocking for any slow model. +6. **Blocking `awaitSettled` at boot recovery** (`ModelProvisioner.swift:109-113` polled from `ContentView.swift:891,1061`) — orphan recovery waits behind the full model download; the wait scales with model size. +7. **Layering inversion** — `TranscriberModel.swift:34-35` (identity enum → concrete backends) while backends reference the enum back (`ParakeetBackend.swift:16`, `WhisperBackend.swift:11`). Every N+1 touches the cycle; option 1 dissolves it. +8. **Error policy is model-blind** — the consecutive-error threshold of 10 (`StreamingTranscriber.swift:124,147`) and the per-segment `"[transcription failed]"` convention (`SegmentReTranscriber.swift:71-78`) were tuned for fast in-process backends; a sidecar that crashes or a slow model that times out hits them with very different user impact and no differentiated handling. + +## What the TDD suite pins (summary) + +- **`ASRCoordinatorTests.swift` (7 tests):** not-initialized throw; install/route; immediate unload on idle swap; deferred unload under in-flight call; token-ordered stale-install refusal; higher-token re-assert no-op; token re-validation across unload suspension. These pin exactly the invariants a dual-slot refactor must preserve per slot — the highest-value regression net for option 3. +- **`ModelProvisionerTests.swift` (15 tests):** full F1/F2/F3 ladder, generation guard (late failure/success inert), flip-back cancel + re-assert re-prepare, lastGood semantics, awaitSettled-spans-F2. Untouched by options 1–2; run-as-is against the live slot in option 3. +- **`TranscriberModelTests.swift` (4 tests):** raw-value stability (the on-disk contract), display names, unknown-persisted fallback (rollback compatibility: a future model's raw value reads as Parakeet in an older build), didSet persistence. +- **`WhisperBackendTests.swift` (4 tests):** variant resolution incl. the misnamed-variant trap, HubApi path layout — the template for any new backend's pure-function tests. + +## BOTTOM LINE +Adding a third live-capable, in-process model is cheap and well-guarded: 5 compiler-enforced switch arms + one factory arm + a new backend file + additive tests (~1 day of plumbing; ASRBench's hand-mirrored config is the only non-scaling site). But the architecture is single-slot by design at every layer — one activeBackend, one install token, one servingModel/lastGood, one picker driving both live and batch — and backends are actors, so a minutes-long batch call would serialize ahead of live chunks on the same instance. A post-processing-only 2B-class model is therefore not expressible today: it needs capability flags at minimum (to stop canStartRecording from enabling live on a batch-only model) and realistically the dual-slot coordinator (~3–5 days + re-audit). The good news is the seam is right where it needs to be: neither StreamingTranscriber nor SegmentReTranscriber ever holds a backend — both route every call through ASRCoordinator — so the split is a coordinator/provisioner change, not a pipeline rewrite, and the existing 22 coordinator+provisioner tests pin exactly the invariants the refactor must preserve. + +## VERIFICATION VERDICTS + +- [CONFIRMED] Adding model N+1 requires exactly six production edits (five in TranscriberModel.swift, one makeBackend switch in AppServices.swift) plus one new backend file; the Settings picker iterates allCases and needs no change. + NOTE: All six sites verified: case declarations at Tome/Sources/Tome/Transcription/TranscriberModel.swift:4-5, displayName switch :8-11, pickerSubtitle :16-19, isInstalled :33-36, approxDownloadSize :42-46, and the makeBackend factory switch at Tome/Sources/Tome/App/AppServices.swift:70-73. These are the only exhaustive switches over TranscriberModel in Sources (grep-verified); from(persisted:) at TranscriberModel.swift:24-26 has a nil-coalescing default and needs no edit; ModelProvisioner and AppSettings are fully generic. SettingsView.swift:152 uses ForEach(TranscriberModel.allCases) with generic rowSubtitle(for:) at :238-243 — no change needed. Trivial imprecision only: 'cases :3-5' is really lines 4-5 (line 3 is the enum declaration), and one of the 'five switch arms' is the case declaration itself, not a switch. Tests and ASRBench would also want updates, but neither breaks compilation and the claim scoped itself to production edits. + +- [CONFIRMED] ASRCoordinator.transcribe accepts source: AudioSource but never reads it in either overload; SegmentReTranscriber passes .system for all batch calls, so live system-audio and post-processing calls are indistinguishable at the coordinator. + NOTE: Both overloads at Tome/Sources/Tome/Transcription/ASRCoordinator.swift:83-94 and :96-107 declare `source: AudioSource` and never reference it in their bodies. SegmentReTranscriber.swift:65 passes `source: .system`. Live system-audio streaming also arrives as .system (TranscriptionEngine.swift:281 constructs its StreamingTranscriber with audioSource: .system; StreamingTranscriber.swift:173 forwards it), so even if the coordinator read the parameter, live-system and batch would collide on the same value. Minor nuance not affecting the verdict: the two call classes are incidentally distinguishable by overload today — live paths use the samples overload, batch uses the buffer overload — but that is an accident of plumbing, not a semantic routing key, and WhisperBackend.swift:113-114 immediately collapses the buffer overload into the samples one. + +- [CONFIRMED] Every layer is single-slot (one activeBackend/lastInstallToken in ASRCoordinator, one servingModel/servingBackend and one lastGood key in ModelProvisioner), and canStartRecording checks only activity/selection, so a post-processing-only model once serving would enable live recording — no supportsLive capability exists. + NOTE: All cites verified: ASRCoordinator.swift:15 (single activeBackend), :22 (single lastInstallToken); ModelProvisioner.swift:37 (servingModel), :41 (servingBackend), :60 (static lastGoodKey). canStartRecording at ModelProvisioner.swift:48-50 is actually `activity == .none && servingModel != nil && servingModel == selection()` — the claim omits the nil check, but it is logically redundant (Optional == non-Optional is never true for nil), so not material. Grep for supportsLive/capability across Sources finds nothing; ASRBackend (ASRBackend.swift:20-33) exposes only model/isInstalled/prepare/transcribe/unload. All recording gates — ControlBar.swift:149,174 record buttons, ContentView.swift:546 start guard, APIServer.swift:355,425 — consume canStartRecording with no per-model capability check, so any serving model enables live recording. + +- [REFUTED] Backend conformances are actors that serialize their own transcribe calls, so a minutes-long batch segment on a shared model blocks all queued live VAD chunks; the coordinator itself does not serialize, meaning a slow accuracy model requires a dual-slot split rather than tuning. + NOTE: The cited facts exist (final actor at ParakeetBackend.swift:15 and WhisperBackend.swift:10; 'must be serialized' comment at ASRBackend.swift:13-14; coordinator suspends at the backend await per ASRCoordinator.swift:9-13) but the mechanism is wrong: Swift actors are reentrant, and the very same doc comment says so (ASRBackend.swift:18-19: 'Swift actors are reentrant — a swap can land while a transcribe is suspended mid-call'). Both backends suspend at their SDK call (ParakeetBackend.swift:50 `await asrManager.transcribe`, WhisperBackend.swift:91 `await whisperKit.transcribe`), so a queued live chunk enters the backend actor and proceeds — it is not blocked behind a long batch call at the actor. WhisperKit is an `open class` (.build/checkouts/argmax-oss-swift/Sources/WhisperKit/Core/WhisperKit.swift:11), so nothing in app code serializes concurrent Whisper inferences at all; AsrManager is an actor (.build/checkouts/FluidAudio/.../AsrManager.swift:6) but also reentrant with internal awaits (e.g. per-window decodeWithTimings at :284,300,327), so calls interleave rather than queue whole-call FIFO. The actor comment refers to serializing access to the mutable SDK handle, not whole transcribe calls. The directionally-right residue — coordinator doesn't serialize, single shared slot, live latency degrades under batch load via compute contention (and possibly unsafe concurrent WhisperKit use) — does not rescue the stated head-of-line-blocking mechanism, which is the load-bearing evidence for 'requires a dual-slot split rather than tuning'. + +- [CONFIRMED] ASRBench duplicates WhisperBackend's variant/path config by hand with an explicit 'keep in sync' comment because SwiftPM can't import the app executable (main.swift:14-18), and hardcodes a two-model run, report, and JSON array (main.swift:222-235) — the only code site that scales linearly-with-drift-risk per added model. + NOTE: Tome/Sources/ASRBench/main.swift:14-15 has the exact comment ('Mirrors WhisperBackend ... keep in sync; SwiftPM forbids importing the app executable from here'); the duplicated variant config is at :16-18 and the duplicated download-base path at :19-20 (path config sits two lines past the cited range — immaterial). The two-model run is hardcoded at :223-226 (benchParakeet/benchWhisper), the Whisper-specific acceptance print at :228-230, and the literal `[parakeet, whisper]` JSON array at :235 (write spans :232-236). It also duplicates StreamingTranscriber's 480k/8k caps at :24-26. 'Only code site with drift risk' holds for Sources: the other per-model sites (TranscriberModel switches, makeBackend) are compiler-enforced exhaustive switches, so they can't silently drift; ASRBench's string/path copies can. Tests hardcode the variant strings too (WhisperBackendTests.swift:7-24) but exercise the app's own code, so they don't drift silently either. + +- [CONFIRMED] Model-specific tuning is baked into shared pipeline code (StreamingTranscriber's sub-8000-sample drop citing Parakeet at :129, SegmentReTranscriber's 1.5s pad for Parakeet's 1s minimum at :41) that every new backend silently inherits; and PostProcessingJob.progress is declared but never written (:24), leaving long batch jobs with zero visible progress. + NOTE: StreamingTranscriber.swift:129 logs exactly '(<8000 ≈ 0.5s, Parakeet emits garbage below this threshold)' for the drop decided at :118 (and again at :161-164 for end-of-stream remnants); SegmentReTranscriber.swift:41 is `let minSamples = Int(sampleRate * 1.5) // 1.5s to clear Parakeet's 1s minimum after resampling` with padding applied at :49-56. Both run upstream of the backend-agnostic ASRCoordinator, so any backend inherits them — e.g. Whisper (no such minimums) still gets Parakeet-tuned drops/padding. PostProcessingJob is at Tome/Sources/Tome/Transcription/PostProcessingJob.swift (not a PostProcessing/ directory); `private(set) var progress: Double = 0` at :24 is never assigned anywhere in the codebase — in fact it is never read either (fully dead). During a batch job the UI shows only the static string 'Finalizing…' (ContentView.swift:414-415), so 'zero visible progress' for a slow model is accurate; the phase enum (:12-20) transitions but no view renders per-phase or percentage detail. diff --git a/docs/superpowers/research/2026-07-09-asr-model-roi/research-runtime.md b/docs/superpowers/research/2026-07-09-asr-model-roi/research-runtime.md new file mode 100644 index 0000000..6d5099e --- /dev/null +++ b/docs/superpowers/research/2026-07-09-asr-model-roi/research-runtime.md @@ -0,0 +1,99 @@ +# Apple Silicon on-device ASR runtime landscape — 2026-07-09 + +Viewpoint: Tome (Swift/SwiftPM macOS app) currently shipping FluidAudio (CoreML Parakeet) + WhisperKit (CoreML Whisper). Goal: max accuracy post-processing; minutes-long batch latency acceptable. + +## 1. FluidAudio (FluidInference) + +- Current ASR roster (README + v0.15.x releases, latest **v0.15.5, 2026-07-07**): **Parakeet-TDT v3 0.6B** (25 EU languages + JA, default), **Parakeet-TDT v2 0.6B** (EN-only, "highest recall"), **Parakeet Unified 0.6B** (chunked-attention streaming + offline batch in one model), **Parakeet EOU 120M**, **Nemotron 3.5 ASR** (streaming multilingual 0.6B, 40 locales, CoreML/ANE), **SenseVoice Small** (non-autoregressive multilingual) and **Paraformer-large** (Mandarin). Sources: https://github.com/FluidInference/FluidAudio and https://github.com/FluidInference/FluidAudio/releases +- v0.15.x line also added custom-vocabulary boosting with per-term CTC thresholds, resumable model downloads, per-token timings — i.e., active, production-oriented development. +- **No granite-speech, canary, or any LLM-decoder ASR**, and no public roadmap item for them. Everything FluidAudio ships is a CoreML-converted CTC/TDT-style encoder model; LLM-decoder models (granite/canary-qwen/higgs) do not fit its CoreML/ANE conversion pipeline, so waiting for FluidAudio to deliver a leaderboard-top model is not a plan. +- Implication for Tome: FluidAudio remains the best **live-streaming** leg (Parakeet v3 or the new Parakeet Unified / Nemotron streaming), not the accuracy leg. + +## 2. WhisperKit / Argmax + +- Open-source WhisperKit hit **v1.0.0 (2026-05-01)** and remains **Whisper-only** (large-v3-turbo etc.), MIT. Source: https://github.com/argmaxinc/WhisperKit +- **NVIDIA Parakeet v2/v3 on Apple Silicon is a paid Argmax Pro SDK feature** ("ParakeetKit", HF repo `argmaxinc/parakeetkit-pro`), announced 2025-06-19: re-implemented for the Neural Engine, >100x real-time even on M1 Air, real-time streaming Parakeet v3. Sources: https://www.argmaxinc.com/blog/nvidia-frontier-speech-models-on-argmax-sdk , https://huggingface.co/argmaxinc/parakeetkit-pro , https://www.argmaxinc.com/blog/argmax-sdk-2 +- Argmax Pro SDK 2 adds speaker diarization, custom vocabulary, and a local server for non-native apps — but still nothing LLM-decoder-based (no granite/canary/higgs). For Tome, Argmax Pro duplicates what FluidAudio already gives you for free; it does not unlock leaderboard-top accuracy. + +## 3. MLX route (the big mover) + +**Python — Blaizzy/mlx-audio** (https://github.com/Blaizzy/mlx-audio): now the broadest ASR zoo on Apple Silicon; **v0.4.5 released today (2026-07-09)**. STT model directories include: `granite_speech`, **`granite_speech_nar`** (added v0.4.4 with "granite-speech-4.1-2b-nar (non-autoregressive ASR)"), `canary` (**Canary-1B-v2**, added v0.4.1 — *not* canary-qwen), **`higgs_audio_3`** ("Higgs audio 3 stt support", added v0.4.5, i.e., days old), `qwen3_asr`, `voxtral` + `voxtral_realtime`, `vibevoice_asr` (Microsoft 9B ASR w/ diarization), `cohere_asr`, `nemotron_asr`, `parakeet`, `whisper`, and ~10 more. Sources: https://github.com/Blaizzy/mlx-audio/releases , https://github.com/Blaizzy/mlx-audio/tree/main/mlx_audio/stt/models + +**Swift — Blaizzy/mlx-audio-swift** (https://github.com/Blaizzy/mlx-audio-swift): first native Swift SDK for MLX audio, **v0.1.0 announced ~June 2026, v0.1.3 on 2026-07-09**; SwiftPM, macOS 14+/iOS 17+, ~700 stars. STT list includes **Parakeet, Qwen3-ASR, Voxtral Realtime, Granite Speech (4.1-2B-NAR per announcement), Canary, Cohere Transcribe, Nemotron ASR, GLM-ASR, SenseVoice, Whisper**. This is the first in-process Swift path to an LLM-decoder leaderboard model — but it is weeks old, v0.1.x, no published M-series benchmarks. Alternative smaller Swift package: DePasqualeOrg/mlx-swift-audio (Whisper, Fun-ASR). The official ml-explore/mlx-swift-examples still has no speech-LLM support — the community package is the only game in town. + +**Leaderboard-model MLX coverage today**: granite-speech-4.1-2b + -nar ✅ (Python + Swift); higgs-audio-v3 STT ✅ Python-only, merged this week; canary-qwen-2.5b ❌ (only Canary-1B-v2); qwen3-asr ✅ (Python + Swift + llama.cpp); voxtral ✅ (Mini/Realtime). No credible published RTF numbers yet for granite/higgs on M-series; nearest datapoint: vllm-mlx measured whisper-large-v3-turbo at 14.3x RT on M1 Max and 55x on M4 Max, Parakeet ~54x on M1 Max (https://github.com/waybarrios/vllm-mlx/blob/main/docs/benchmarks/audio.md). + +## 4. llama.cpp route (mtmd) + +- Officially documented audio-input models (https://github.com/ggml-org/llama.cpp/blob/master/docs/multimodal.md): **Ultravox 0.5 (1B/8B), Voxtral-Mini-3B-2507, Qwen3-ASR-0.6B and -1.7B** (ggml-org GGUFs), plus mixed-modality **Qwen2.5-Omni / Qwen3-Omni**; Qwen2-Audio exists but is flagged as giving "very poor result" (no pre-quantized GGUF). Voxtral-Small-24B and Phi-4-multimodal are **not** in the supported list. +- **granite-speech support was merged into mtmd** (PR ~#22101 per search results), and — decisively — **IBM now publishes official GGUFs**: `ibm-granite/granite-speech-4.1-2b-GGUF`, `granite-speech-4.1-2b-plus-GGUF` (Q4_K_M 1.02 GB → BF16 3.27 GB), `granite-4.0-1b-speech-GGUF`, with documented `llama-server -hf ...` usage. Sources: https://huggingface.co/ibm-granite/granite-speech-4.1-2b-plus-GGUF , https://github.com/IBM/gguf +- Long-audio caveat: mtmd audio models are chat-style ("transcribe this clip") — meeting-length audio must be VAD-chunked into ≤30–40s segments by the host app; no built-in long-form pipeline. Quality of llama.cpp audio front-ends has historically lagged reference implementations (cf. Qwen2-Audio warning), so WER should be re-validated vs. the Python reference before trusting it. +- Embedding in a Swift app: two proven patterns — (a) **`llama-server` sidecar** (ships as plain binaries; OpenAI-compatible HTTP; simplest, process-isolated, easy to update) or (b) **XCFramework**: llama.cpp publishes an official `llama.xcframework` build target / release artifact linkable from Swift (used by many iOS/macOS apps). For a batch post-processing lane, the sidecar is lower-risk; note also that IBM's GGUF pages document Docker/LM Studio/Jan as runners, confirming the server path is the mainstream one. +- Bonus: **granite-speech-4.1-2b-plus** (new, updated ~2 weeks ago) exists on HF alongside the leaderboard-listed 4.1-2b; and **no granite-speech 4.2 / "4.2.1" exists** — 4.1 is the latest line (https://huggingface.co/ibm-granite/models). + +## 5. Python-sidecar route + +- **Strong precedent: LM Studio** ships `mlx-engine` — a **Python 3.11 runtime bundled inside the Mac desktop app** (mlx-lm + mlx-vlm, MIT, https://github.com/lmstudio-ai/mlx-engine, https://lmstudio.ai/blog/lmstudio-v0.3.4). ComfyUI Desktop similarly bundles Python via uv. So "notarized Mac app + embedded Python MLX inference" is a shipped, mainstream pattern — and it would give Tome day-one access to everything in mlx-audio (granite-nar, higgs-STT, canary-1b-v2, vibevoice) without waiting for Swift ports. +- Known pitfalls (well documented for PyInstaller-style bundling): sign **inside-out per-binary, never `--deep`**; hardened runtime requires `com.apple.security.cs.allow-unsigned-executable-memory` for Python; `base_library.zip` placed in `Contents/MacOS` breaks notarization (must relocate to Resources); use `notarytool` (altool dead); .so files from wheels must each be signed; inconsistent Gatekeeper failures across macOS versions. Sources: https://haim.dev/posts/2020-08-08-python-macos-app , https://github.com/pyinstaller/pyinstaller/issues/7937 , https://github.com/pyinstaller/pyinstaller/issues/5112 +- Practical costs: +300–600 MB app size (CPython + mlx + numpy etc.) before model weights; slower cold start; a second update channel. A uv-managed venv downloaded on first run (LM Studio-style "runtime" downloads) sidesteps notarizing the wheels inside the app bundle. + +## 6. Leaderboard RTFx baseline and scaling to M2 Max + +- **Hardware**: The Open ASR Leaderboard companion paper (arXiv 2510.06961) states all measurements ran on an **NVIDIA A100-SXM4-80GB, batch size 64** (reduced adaptively when OOM). The leaderboard repo README (updated June 2026) says English short-form evals now run via HF Jobs on a **1x NVIDIA H200 (141 GB)** flavor. Either way: datacenter GPU, **batched** throughput — RTFx is NOT single-stream latency. Sources: https://github.com/huggingface/open_asr_leaderboard , https://arxiv.org/html/2510.06961 +- Empirical anchors: parakeet-tdt-0.6b-v2 = 3386 RTFx on leaderboard vs ~77 RTFx measured in Tome (p95 RTF 0.013, CoreML, M2 Max) → **÷44**; whisper-large-v3-turbo ≈ 216 leaderboard vs 7.7 in Tome (WhisperKit) → **÷28** (vllm-mlx gets 14x on M1 Max, so MLX can do better than WhisperKit here). +- Rules of thumb (leaderboard RTFx → single-stream M2 Max 64GB): + - (a) **CoreML-optimized encoder models (parakeet-class): ÷30–50** → still 50–100x real-time locally. Batch-parallel encoders benefit most from datacenter batching, hence the big divisor. + - (b) **2B-class LLM-decoder ASR under MLX 4-bit (granite-4.1-2b): ÷15–25** → est. RTFx ~8–15 (60-min meeting in ~4–8 min). Decode is memory-bandwidth-bound (H200 4.8 TB/s vs M2 Max 0.4 TB/s ≈ 12x) plus loss of batching. The **NAR variant should beat this** (non-autoregressive decode removes the token-rate bottleneck — likely why it exists and why it's the variant everyone ported). + - (c) **8B-class under MLX 4-bit (higgs-8b): ÷15–25 → est. RTFx ~3–6** (60-min meeting in ~10–20 min; "thinking mode" tokens push toward the slow end). Feasible within the stated "minutes are fine" budget on M2 Max 64GB; comfortable on the coworker's Ultra (~2x bandwidth). + - All (b)/(c) numbers are estimates — no published granite/higgs M-series benchmarks exist yet; treat as hypothesis to verify with a 10-min meeting sample. + +## Verdicts per target model + +| Model | Most credible path for Tome | Maturity | +|---|---|---| +| **granite-speech-4.1-2b-nar / -2b** | **MLX**: `mlx-audio-swift` in-process (SwiftPM, has Granite Speech incl. NAR) — with **llama.cpp sidecar on IBM's official GGUFs** as the more battle-tested fallback (mtmd support merged; Q4_K_M is 1 GB) | **Experimental → near-shippable.** Two independent runtimes + official vendor GGUFs; but Swift package is v0.1.x (weeks old) and app must own VAD/chunking for meeting-length audio. Best accuracy-per-effort of the three. Also note: **no granite-speech 4.2/"4.2.1" exists — 4.1 is latest**, and there's a newer `-plus` variant worth WER-testing. | +| **higgs-audio-v3-8b-stt-v2** | **Python sidecar running mlx-audio** (support merged v0.4.5, released 2026-07-09 — literally days old); LM Studio's bundled-Python mlx-engine proves the packaging pattern. No GGUF/llama.cpp, no Swift port, upstream is CUDA/transformers + `trust_remote_code`. | **Experimental.** Runnable this week by a developer; not shippable to end users yet (unproven port, ~5 GB at 4-bit, est. RTFx 3–6, thinking-mode semantics untested on MLX). Re-evaluate in 1–2 months. | +| **canary-qwen-2.5b** | None credible. NeMo-trunk + PyTorch only (FSDP2, CUDA-centric); **no MLX, GGUF, or CoreML port exists** (mlx-audio's "canary" is Canary-1B-v2); trained on ≤40s clips with 1024-token ceiling, no long-form recipe. Source: https://huggingface.co/nvidia/canary-qwen-2.5b | **No path** (short of Tome doing its own MLX port). If the FastConformer+LLM shape is appealing, granite-4.1-2b delivers better leaderboard WER anyway (3.99 vs 4.26). | + +**Cross-cutting note**: two leaderboard-adjacent models have unusually easy paths and are worth including in any bake-off: **Qwen3-ASR-1.7B** (4.53 WER; official ggml-org GGUFs in llama.cpp mtmd AND mlx-audio(-swift) support) and **Voxtral-Mini-3B** (llama.cpp + MLX). And the split-architecture idea (FluidAudio Parakeet live + LLM-decoder model for post-processing) is exactly what the ecosystem now supports. + +## BOTTOM LINE +The accuracy-leaderboard models are reachable on Apple Silicon today, but through MLX and llama.cpp, not through Tome's current runtimes: FluidAudio (v0.15.5) and open-source WhisperKit (v1.0.0) remain encoder/Whisper-only, with Parakeet-on-ANE locked behind Argmax's paid Pro SDK. The strongest single move is granite-speech-4.1-2b(-nar) — no 4.2 exists — which has both a native SwiftPM path (mlx-audio-swift, v0.1.3, weeks old) and a battle-tested llama.cpp path on IBM's official GGUFs (grade: experimental→near-shippable); higgs-audio-v3-8b-stt is Python-mlx-audio-only as of a release cut today (experimental); canary-qwen-2.5b is NeMo/CUDA-only with no port (no path). Leaderboard RTFx is batched datacenter-GPU throughput (A100-80GB batch-64 per the paper, now H200 via HF Jobs), so divide by roughly 30–50x for CoreML encoders and 15–25x for MLX LLM-decoders to estimate single-stream M2 Max speed — putting granite-2b post-processing of a 60-minute meeting at roughly 4–8 minutes. + +## VERIFICATION VERDICTS + +- [CONFIRMED] Open ASR Leaderboard RTFx numbers are batched datacenter-GPU throughput: paper arXiv 2510.06961 used NVIDIA A100-SXM4-80GB with batch size 64; leaderboard repo English short-form evals run on HF Jobs 1x NVIDIA H200 (141 GB) + NOTE: Paper full text (arxiv.org/html/2510.06961) states verbatim: evaluation scripts ran on an 'NVIDIA A100-SXM4-80GB GPU (driver 560.28.03, CUDA 12.6), using a batch size of 64 whenever memory allowed, and reduced adaptively (48, 32, 16, ...)'. Repo README confirms 'English short-form evaluations use Hugging Face Jobs' on '1x H200 (141 GB)'. Only nuance: batch size is 64 *when memory allows*, adaptively reduced for larger models — the claim's flat 'batch size 64' slightly overstates uniformity but is not materially wrong. + +- [CONFIRMED] No granite-speech 4.2 or 4.2.1 exists as of 2026-07-09; newest IBM speech models are the granite-speech-4.1-2b line (incl. -nar and -plus), and IBM publishes official GGUF builds (granite-speech-4.1-2b-plus-GGUF, Q4_K_M ~1.02 GB) documented to run with llama.cpp + NOTE: HF ibm-granite org search for 'speech' lists exactly: granite-speech-4.1-2b-plus-GGUF (updated ~9 days ago), granite-speech-4.1-2b-nar, granite-speech-4.1-2b-plus, granite-speech-4.1-2b, plus older 3.3-2b/3.3-8b/3.2-8b. No 4.2 or 4.2.1 anywhere. The GGUF repo exists under ibm-granite, lists Q4_K_M at exactly 1.02 GB (also Q5_K_M 1.18 GB, Q6_K 1.34 GB, Q8_0 1.74 GB, BF16 3.27 GB), with extensive llama.cpp run instructions. + +- [CONFIRMED] llama.cpp mtmd officially supports audio for Ultravox 0.5, Voxtral-Mini-3B-2507, Qwen3-ASR 0.6B/1.7B via ggml-org GGUFs, plus Qwen2.5/Qwen3-Omni; granite-speech support merged; Voxtral-Small-24B and Phi-4-multimodal not in the supported audio list + NOTE: docs/multimodal.md audio section lists exactly ggml-org/ultravox-v0_5 (1b & 8b), ggml-org/Voxtral-Mini-3B-2507-GGUF, ggml-org/Qwen3-ASR-0.6B-GGUF and -1.7B-GGUF; Qwen2.5-Omni and Qwen3-Omni appear under mixed modalities. Voxtral-Small-24B and Phi-4-multimodal are absent. Granite-speech is not named in multimodal.md, but merged support is independently evidenced: merged regression-fix PR ggml-org/llama.cpp#24357 (2026-06-09) restores 'granite speech inference' broken by #23545, third-party GGUFs (e.g. staghado/granite-speech-4.0-1b-GGUF) cite llama.cpp audio-multimodal support via PR #22101, and CUDA eval issue #23015 concerns running Granite Speech in llama.cpp. + +- [CONFIRMED] Blaizzy/mlx-audio-swift is a native SwiftPM package (v0.1.3, 2026-07-09, macOS 14+) with STT roster incl. Parakeet, Qwen3-ASR, Voxtral Realtime, Granite Speech, Canary, Cohere Transcribe, Nemotron ASR, Whisper; Python mlx-audio v0.4.5 (2026-07-09) added Higgs Audio 3 STT; its 'canary' is Canary-1B-v2, not canary-qwen + NOTE: GitHub API: mlx-audio-swift v0.1.3 published_at 2026-07-09T16:58:18Z; repo is a SwiftPM package requiring macOS 14+/iOS 17+; STT list includes all eight named models (plus others: MOSS-Transcribe-Diarize, GLMASR, FireRedASR2, SenseVoice, Moonshine, Wav2Vec2 CTC, MMS). Python mlx-audio v0.4.5 published_at 2026-07-09T16:32:10Z with release note 'Higgs audio 3 stt support'. Canary was added in v0.4.1 with note 'add canary stt model (nvidia canary-1b-v2)' — Canary-1B-v2, not canary-qwen. All details check out. + +- [CONFIRMED] nvidia/canary-qwen-2.5b runs only via NVIDIA NeMo with PyTorch 2.6+ (no transformers/vLLM/MLX/GGUF/CoreML ports), trained on clips up to 40s with a 1024-token sequence ceiling, no documented long-form chunking recipe + NOTE: HF model card states verbatim: 'To train, fine-tune or transcribe with Canary-Qwen-2.5B, you will need to install NVIDIA NeMo' with 'PyTorch 2.6+ for FSDP2 support', and 'The maximum audio duration in training was 40s, and the maximum token sequence length was 1024 tokens (including prompt, audio, and response)' — digits match exactly. Card documents no chunking recipe, only that longer sequences 'may technically' work with degraded accuracy. Searches found no transformers/vLLM/MLX/GGUF/CoreML ports; Replicate/Cog listings wrap NeMo. mlx-audio's Canary port is Canary-1B-v2, not canary-qwen. Minor caveat: I confirmed 'NeMo required', not the specific word 'trunk' (git main) — the card just says install NVIDIA NeMo. + +- [CONFIRMED] Parakeet v2/v3 on Apple Neural Engine via paid Argmax Pro SDK (announced 2025-06-19, >100x real-time on M1 MacBook Air); open-source WhisperKit v1.0.0 (May 2026) still supports only Whisper models + NOTE: Argmax blog post is dated June 19, 2025, announces Parakeet v2 (v3 multilingual added later) on the Apple Neural Engine, says 'ship it with your app using Argmax Pro SDK today!' (paid tier), and states 'even the least capable Apple Silicon Mac from 5 years ago, M1 Macbook Air with 8 GB RAM is able to surpass a speed factor of 100'. GitHub API: WhisperKit v1.0.0 published_at 2026-05-01T21:42:14Z (May 2026). v1.0.0 notes describe graduation into the 'Argmax Open-Source SDK' bundling WhisperKit + SpeakerKit (diarization) + TTSKit — but for ASR/STT it remains Whisper-only; Parakeet is not in the open-source package. Claim accurate as stated for ASR models. + +- [CONFIRMED] The Open ASR Leaderboard RTFx numbers are batched datacenter-GPU throughput: the companion paper (arXiv 2510.06961) ran all measurements on an NVIDIA A100-SXM4-80GB with batch size 64, and the leaderboard repo's English short-form evals now run on HF Jobs 1x NVIDIA H200 (141 GB). + NOTE: Paper full text (arxiv.org/html/2510.06961v1) states verbatim: 'conducted on an NVIDIA A100-SXM4-80GB GPU (driver 560.28.03, CUDA 12.6), using a batch size of 64 whenever memory allowed, and reduced adaptively (48, 32, 16, ...) when necessary.' Repo README confirms English short-form evals run on HF Jobs with hardware flavor 'h200 ... 1x H200 (141 GB)' and cites arXiv 2510.06961 as the companion paper. Minor nuance only: batch 64 is the default, adaptively reduced for large models — not materially different from the claim. + +- [CONFIRMED] No granite-speech 4.2 or 4.2.1 exists as of 2026-07-09; the newest IBM speech models are the granite-speech-4.1-2b line (including -nar and a recent -plus variant), and IBM publishes official GGUF builds (e.g. granite-speech-4.1-2b-plus-GGUF, Q4_K_M ~1.02 GB) documented to run with llama.cpp. + NOTE: HF model search for 'granite-speech' shows no 4.2/4.2.1; newest official models are granite-speech-4.1-2b (updated ~27 days ago), granite-speech-4.1-2b-plus (~23 days), granite-speech-4.1-2b-nar (~22 days), all newer than granite-4.0-1b-speech and the 3.x line. huggingface.co/ibm-granite/granite-speech-4.1-2b-plus-GGUF exists under the official ibm-granite org, lists Q4_K_M at exactly 1.02 GB, and documents llama.cpp run instructions for macOS/Linux/Windows. + +- [CONFIRMED] llama.cpp's multimodal (mtmd) stack officially supports audio input for Ultravox 0.5, Voxtral-Mini-3B-2507, and Qwen3-ASR 0.6B/1.7B via ggml-org GGUFs, plus Qwen2.5/Qwen3-Omni, and granite-speech support has been merged; Voxtral-Small-24B and Phi-4-multimodal are not in the supported audio list. + NOTE: docs/multimodal.md (raw, master) audio section lists exactly ggml-org/ultravox-v0_5-* GGUFs, ggml-org/Voxtral-Mini-3B-2507-GGUF, and ggml-org/Qwen3-ASR-0.6B/1.7B-GGUF; Qwen2.5-Omni (3B/7B) and Qwen3-Omni (30B) appear in the mixed audio+vision section. Voxtral-Small-24B and Phi-4-multimodal appear nowhere in the document. Granite-speech merge verified independently via GitHub API: PR #22101 'mtmd: add granite-speech support (ibm-granite/granite-4.0-1b-speech)' merged 2026-05-06, PR #24818 'model: Granite Speech Plus' merged 2026-06-23, and inference fix PR #24357 merged 2026-06-09. Caveat: granite-speech is merged but not (yet) listed in multimodal.md itself — the claim's wording ('support has been merged') is accurate. + +- [CONFIRMED] Blaizzy/mlx-audio-swift is a native SwiftPM package (v0.1.3, 2026-07-09, macOS 14+) whose STT roster includes Parakeet, Qwen3-ASR, Voxtral Realtime, Granite Speech, Canary, Cohere Transcribe, Nemotron ASR and Whisper; the Python mlx-audio v0.4.5 (2026-07-09) additionally added Higgs Audio 3 STT support, and its 'canary' implementation is Canary-1B-v2, not canary-qwen. + NOTE: Repo confirms a SwiftPM-distributed Swift SDK for MLX on Apple Silicon, macOS 14+/iOS 17+, latest release v0.1.3 dated July 9, 2026. README STT table includes all eight named models (plus GLMASR, FireRedASR2, SenseVoice, Moonshine, Wav2Vec2 CTC, MMS). Python Blaizzy/mlx-audio latest release v0.4.5 (July 9) notes 'Higgs audio 3 stt support' (PR #811); Canary was added in v0.4.1 as 'add canary stt model (nvidia canary-1b-v2)' (PR #550) — no canary-qwen implementation exists there. + +- [CONFIRMED] nvidia/canary-qwen-2.5b runs only via NVIDIA NeMo trunk with PyTorch 2.6+ (no transformers/vLLM/MLX/GGUF/CoreML ports), was trained on audio clips up to 40 seconds with a 1024-token sequence ceiling, and has no documented long-form chunking recipe. + NOTE: Model card states: install NVIDIA NeMo to transcribe, 'Currently requires installing the latest trunk version of NeMo, and PyTorch 2.6+ for FSDP2 support'; 'The maximum audio duration in training was 40s, and the maximum token sequence length was 1024 tokens (including prompt, audio, and response)' with accuracy degradation beyond; no chunking recipe is documented. Adversarial search for MLX/GGUF/vLLM/transformers ports found none — only a Replicate cog wrapper that itself uses NeMo, and mlx-audio's Canary support is Canary-1B-v2, not canary-qwen, corroborating the no-ports claim. + +- [CONFIRMED] NVIDIA Parakeet v2/v3 on the Apple Neural Engine is available through the paid Argmax Pro SDK (announced 2025-06-19, >100x real-time on an M1 MacBook Air), while open-source WhisperKit v1.0.0 (May 2026) still supports only Whisper models. + NOTE: Argmax blog is dated June 19, 2025 and states Parakeet was reimplemented for Apple Silicon 'achieving near-peak utilization of 10+ TFLops on the Apple Neural Engine', production use requires the paid Argmax Pro SDK, and 'M1 Macbook Air with 8 GB RAM is able to surpass a speed factor of 100' (Parakeet v2 at announcement, multilingual v3 added later). GitHub API on the repo (now renamed argmaxinc/argmax-oss-swift) confirms v1.0.0 published 2026-05-01 — note an initial WebFetch misread this as 2025; the API date is authoritative and matches the claim. README's STT support table lists only Whisper variants; no Parakeet in open source (Pro SDK advertises 'additional models'). One nuance: v1.0.0 graduated WhisperKit into a multi-kit SDK that also ships non-ASR models (Qwen3-TTS, Pyannote diarization), but for speech-to-text it remains Whisper-only, which is what the claim asserts. diff --git a/docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md b/docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md new file mode 100644 index 0000000..ff7a3e5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md @@ -0,0 +1,365 @@ +# Granite Shadow Transcription — Design + +**Date:** 2026-07-09 +**Status:** Approved by Nic (pending spec review) +**Author:** Claude + Nic + +## Why (evidence summary) + +Goal: highest transcription accuracy for meetings (accented speakers, imperfect +audio); speed secondary; Tome never ships publicly (internal tool for Nic + Dan, +so licenses and sidecar processes are not gates). + +Research (2026-07-09, adversarially verified against primary sources): + +- **ibm-granite/granite-speech-4.1-2b** is the best meeting-audio model with a + credible Apple Silicon path. On the Open ASR Leaderboard's AMI (meeting + corpus): WER 7.06 (cleaned refs) vs whisper-large-v3-turbo 13.87 (−49%) and + parakeet-tdt-0.6b-v3 9.41 (−26%). Earnings22: 8.23 vs 11.07 / 10.77. + ~2B params, Apache-2.0, punctuation + capitalization, keyword biasing. + (There is **no granite-speech 4.2 / “4.2.1”** — 4.1 is the latest family.) +- **Runtime:** IBM ships official GGUFs + (`ibm-granite/granite-speech-4.1-2b-GGUF`: Q8_0 1.96 GB + f16 mmproj + 1.16 GB) with documented macOS `llama.cpp` usage; granite-speech support is + merged in llama.cpp's multimodal (mtmd) stack. `llama-server` accepts audio. + No published Apple Silicon RTF numbers exist; estimate RTF ≈ 0.03–0.15 + single-stream on M2 Max (unverified — measured by this feature). +- **Rejected:** `-nar` variant (batch-throughput artifact, word-drop risk in + noise, no GGUF); `-plus` variant for now (speaker attribution + timestamps + but drops punctuation/caps and slightly worse WER — Tome's per-segment + pipeline gets speakers from SpeakerKit upstream anyway; revisit later); + bosonai/higgs-audio-v3-8b-stt-v2 (worse than granite on AMI/Earnings22, + 17.8 GB, no Mac runtime); nvidia/canary-qwen-2.5b (worst meeting profile of + the top set, NeMo/CUDA-only, no port). +- **Honest discounts:** granite trained on AMI + Earnings22 *train* splits + (in-domain inflation — expect a smaller real-world gap), and llama.cpp's + audio front-end WER is unvalidated vs the Python reference. **We do not + know granite beats the current models on Nic's real meetings.** Hence: + shadow mode, not a committed migration. + +## Goals + +- **Phase 0 (objective, first):** a benchmark harness measures real WER for + granite-via-llama.cpp vs Tome's current backends on public test corpora + with reference transcripts (AMI, Earnings-22, CORAAL sample), scored with + the leaderboard's normalizer — validating llama.cpp pipeline fidelity + against granite's published numbers before any Tome integration is trusted. +- **Phase 1 (domain confirmation):** behind a hidden flag (no UI), every + post-processed session is *additionally* transcribed by + granite-speech-4.1-2b via a local `llama-server` sidecar, producing a + parallel transcript and a per-segment comparison artifact — a few days of + real meetings, confirming Phase 0 on Nic's actual audio domain (his mic + chain, meeting codecs, colleagues' accents, transcript readability). +- Both models see **byte-identical segment audio** in shadow mode (same + diarization, same merge/pad logic) so the comparison is apples-to-apples. +- Shadow is strictly best-effort: no shadow failure may fail, delay-block, or + alter the primary transcript or the session lifecycle guarantees + (WAV-preservation-on-failure, orphan recovery, retention). +- A report script renders all comparison artifacts into one side-by-side + HTML so Nic can judge the disagreements. +- TDD throughout; existing 89-test suite stays green and untouched. + +## Non-Goals (deferred until shadow results are in) + +- Dual-slot coordinator / second provisioner ("live model" + "accuracy model" + pickers) — the ~1–1.5 week full design. The shadow week decides if it's paid for. +- `supportsLive` capability flags, ModelDescriptor registry refactor, ASRBench + library extraction (debt noted in the scalability review, not needed here). +- Any `TranscriberModel` enum case, ModelProvisioner integration, or Settings + UI for granite — the shadow model is not user-selectable. +- `-plus` speaker-attribution vs SpeakerKit comparison (own experiment, later). +- WER scoring of the *shadow* data (Nic's meetings have no ground truth; + human judgment on disagreements is the shadow metric — objective WER lives + in Phase 0, where references exist). + +## Decisions already made (with Nic) + +| Decision | Choice | +|---|---| +| Model | `granite-speech-4.1-2b` (AR base; not -nar, not -plus) | +| Runtime | `llama-server` sidecar on IBM's official GGUFs (Q8_0 + f16 mmproj) | +| Scope | Phase 0 public-corpus benchmark first (objective WER, pipeline-fidelity gate), then hidden-flag shadow comparison for a few days of real meetings, then decide | +| Speaker tagging | Stays SpeakerKit's job (per-segment pipeline); granite sees single-speaker segments | +| Setup | Manual script (brew llama.cpp + curl model download) — **curl, not URLSession** (known Tome issue: URLSession can't reach the HF CDN on some networks) | +| Bake-off | Superseded by Phase 0: the benchmark harness subsumes the smoke test and adds WER-with-references (Nic, 2026-07-09: "benchmark first + short shadow") | + +## Current architecture facts this design builds on + +- `PostProcessingJob.run(using:)` + ([PostProcessingJob.swift](../../../Tome/Sources/Tome/Transcription/PostProcessingJob.swift)) + is the whole post-session pipeline: diarize → re-transcribe → rebuild → + `finalizeFrontmatter` (savedPath exists after :179) → voiceprints → + retention → `cleanupCaptureFiles()` (:242-244) → `.complete`. + **The capture WAVs are deleted on the success path** — any shadow work that + reads session audio MUST run inside the job, before cleanup. +- Re-transcription is per-diarized-segment: + `TranscriptionEngine.reTranscribe` → `SegmentReTranscriber` + ([SegmentReTranscriber.swift](../../../Tome/Sources/Tome/Transcription/SegmentReTranscriber.swift)), + which merges same-speaker segments < 0.5 s apart (:25-37), pads to ≥ 1.5 s + (:41-56, Parakeet minimum), reads each segment's `AVAudioPCMBuffer` from the + WAV, and calls `asrCoordinator.transcribe(buffer:source:)` (:65). Failures + produce the `"[transcription failed]"` placeholder per segment (:71-78). + Its output `[ReTranscribedSegment]` (speaker label, text, startTime) is + exactly the per-segment primary text the comparison needs. +- Solo voice memos (≤ 1 detected speaker) skip re-transcription entirely and + keep the live transcript (PostProcessingJob.swift:119-126). +- `ASRCoordinator` is the user-selected-model machinery (install tokens, + provisioner ladder). The shadow path deliberately does NOT touch it. +- Jobs run serially on `PostProcessingQueue`; SettingsView locks the model + picker while jobs run — shadow lengthens jobs, so it lengthens that lock + window (accepted for the experiment; called out in Risks). +- `swift test` needs `DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer`. + +## Design + +### 0. Phase 0 — public-corpus benchmark (runs before any Tome integration) + +A standalone harness under `scripts/asr-bench/` (Python via `uv`, internal +tool — dependency freedom, unlike the stdlib-only report script): + +- **Datasets:** the Open ASR Leaderboard's prepared ESB test sets on HF + (pre-segmented audio + references — the *same inputs* behind the published + numbers; exact dataset ids pinned at implementation from the + `open_asr_leaderboard` harness). Sets: **AMI test** (meetings; granite + in-domain), **Earnings-22 test** (accents/compression; granite in-domain), + and a **CORAAL sample** (held-out accent check — granite absent from the + long-form board that uses it). ~3–5 h per set — thousands of words, stable + deltas, sane runtime (granite est. RTF 0.03–0.15 → tens of minutes per set). +- **Backends:** + - granite: the harness drives `llama-server` directly with the pinned + request template (see below) — no Swift required for Phase 0; + - Parakeet v3 + Whisper turbo: `ASRBench` gains a **manifest mode** + (`--manifest in.jsonl --out hyp.jsonl`: per-line WAV path in, hypothesis + out) so hypotheses come from Tome's *actual* backends, not a + reimplementation. No other ASRBench changes (the extraction refactor + stays deferred). +- **Request template:** the llama-server request shape/prompt/params for + granite is pinned by this harness's smoke test and recorded in ONE place + (`scripts/asr-bench/granite_request.md`); the later Swift + `GraniteRequest.build(...)` implements the same template with a golden + test against it, so Phase 0 fidelity transfers to shadow mode. +- **Scoring:** Whisper `EnglishTextNormalizer` on refs and hyps (same as the + leaderboard), WER via `jiwer`. Output: one table — per-set WER per backend, + with the leaderboard's published numbers alongside. +- **Fidelity gate:** granite-on-our-stack must land within ~1.5 points + absolute of its published raw WER on AMI and Earnings-22. A miss means a + pipeline bug (resampling, prompt, chunking) — fix or fall back to + `llama-mtmd-cli` BEFORE building any Swift integration. This gate replaces + the old "one known-content clip" smoke test. + +Phase 0's deliverable is a committed results file +(`docs/superpowers/plans/2026-07-09-granite-phase0-results.md`) with the +table, RTF measurements on the M2 Max, and a go/no-go call for Phase 1. + +### 1. Flag and configuration + +A `ShadowConfig` value type, read from UserDefaults **at job creation** (so +toggling applies from the next session, no restart): + +- `graniteShadowEnabled` (Bool, default false) — master switch. +- `graniteShadowServerPath` (String, default `/opt/homebrew/bin/llama-server`). +- `graniteShadowModelDir` (String, default + `~/Library/Application Support/Tome/Granite`) — must contain the Q8_0 model + GGUF and the f16 mmproj file under the exact names the setup script + downloads (the script is the source of truth for filenames). +- `graniteShadowPort` (Int, default 8873; server binds 127.0.0.1 only). + +Enable: `defaults write com.dloomis.tome graniteShadowEnabled -bool YES`. +If the flag is on but the binary or model files are missing, the shadow phase +logs one clear diagnostic per job and skips — never errors. + +### 2. Setup script + +`scripts/setup-granite-shadow.sh`: +1. Checks `llama-server` exists (advises `brew install llama.cpp` if not; + requires llama.cpp ≥ b9045 — the script checks `llama-server --version`). +2. Downloads the two GGUF files from + `https://huggingface.co/ibm-granite/granite-speech-4.1-2b-GGUF` into the + model dir **via curl** (resumable, `-C -`), verifying file sizes. +3. Prints the `defaults write` commands and a one-shot smoke-test command + (transcribe a bundled 10 s WAV) so setup ends with proof-of-life. + +### 3. `GraniteSidecar` (actor) — process lifecycle + +Owns exactly one `llama-server` child process. Spawn-per-job (not resident): +launched at shadow-phase start, terminated at phase end — keeps ~4 GB of +model RAM off the machine between jobs; GGUF mmap reload costs seconds, +negligible against job length. + +States: `idle → launching → ready → terminating → idle`, plus `failed`. +- `start()`: spawns `llama-server -m --mmproj --host 127.0.0.1 + --port ` via `Process`, then polls `GET /health` until ready (timeout + 60 s → kill + `failed`). +- `transcribe(wavData:) async throws -> String`: POST to the server's + OpenAI-compatible chat completion endpoint with the audio as a base64 + `input_audio` content part and the granite ASR prompt, greedy decoding + (temperature 0). **The exact request shape/prompt was already pinned by the + Phase 0 harness** (`scripts/asr-bench/granite_request.md`); + `GraniteRequest.build(...)` is a pure function implementing that template, + with a golden test against it. +- `stop()`: SIGTERM, escalate to SIGKILL after 5 s. Also invoked from app + termination and `deinit` defensively — a leaked llama-server must not + outlive Tome. +- One mid-phase relaunch: if a request fails with a connection-level error + while `ready`, the sidecar relaunches once; a second failure marks the + phase `failed` and remaining segments are recorded as errored. + +The actor is built behind a `SidecarProcessRunning` protocol seam (spawn, +poll, request, terminate) so the state machine is fully unit-testable with a +fake — same discipline as `FakeBackend` in the existing suite. + +### 4. Shared segment mechanics (small refactor, tested) + +`SegmentReTranscriber` gains a `SegmentTranscribing` seam: + +```swift +protocol SegmentTranscribing: Sendable { + func transcribe(buffer: AVAudioPCMBuffer) async throws -> String +} +``` + +- Default conformance wraps `ASRCoordinator` (existing behavior, byte-for-byte). +- `GraniteSegmentTranscriber` wraps `GraniteSidecar` (buffer → 16 kHz mono + WAV data → HTTP). +- The merge (< 0.5 s gap) and pad (≥ 1.5 s) logic is extracted into pure + static functions with unit tests, used identically by both runs. The + Parakeet-motivated padding intentionally applies to granite too — identical + input audio is the point of the comparison. + +The shadow run therefore produces `[ReTranscribedSegment]` with the same +speaker labels and start times as the primary run. Pairing keys on the merged +segment's `startTime` (both runs iterate the identical merged list, so the +key is exact), NOT on array position: the existing primary path *skips* +segments whose transcription came back empty (`guard !text.isEmpty else +continue`, SegmentReTranscriber.swift:67), so output arrays can differ in +length. A merged segment with no entry on one side is recorded as `""` for +that side in the comparison JSON. + +### 5. Shadow phase placement in `PostProcessingJob` + +New optional step after the voiceprint step (§2b in the job — savedPath +exists, primary transcript durable, sidecar path refreshed) and immediately +before the retention step (§3) — i.e. while the capture WAVs are still on +disk: + +1. Runs only when: config present ∧ the primary path actually re-transcribed + (`shouldRebuild` && diarized segments non-empty) ∧ the primary + `[ReTranscribedSegment]` results were captured. Solo memos and + relabel-fallback sessions skip with a logged reason ("no re-transcribed + segments — nothing to compare"). +2. Start sidecar → transcribe each merged segment → stop sidecar. +3. Honors cancellation cooperatively: checks `Task.isCancelled` between + segments; on cancel, stops the sidecar, writes artifacts marked + `"incomplete": true`, and **returns normally** (the job's own + cancellation semantics after finalize are unchanged — shadow never throws). +4. Phase reporting: the job stays in `.finalizing` (no new `Phase` case — the + enum is observed by UI/tests; shadow is invisible by design). Progress and + timing go to `diagLog` with a `[SHADOW]` prefix. +5. Nothing in the phase mutates `handle.transcript`, savedPath content, + retention behavior, or cleanup decisions. + +Failure policy inside the phase: per-segment errors record +`{"error": "..."}` for that segment and continue; sidecar-level failure +(launch timeout, double connection failure, missing binary/models) abandons +remaining segments, records the reason in the JSON, logs, and returns. + +### 6. Artifacts + +Written to `~/Library/Application Support/Tome/GraniteShadow/` (NOT next to +the transcript — keeps the notes vault clean; the HTML report is the review +surface. Flip to vault-adjacent later if in-Obsidian reading proves wanted): + +- `.granite.md` — human-readable shadow transcript: header + (session title, date, primary model, granite model+quant, total shadow time, + realtime factor) + `Speaker N: text` lines. +- `.comparison.json` — machine-readable: + session id, transcript path, session type, primary model raw value, granite + model id + quant + server build, incomplete flag, per-segment records + `{startTime, speaker, durationSec, primaryText, graniteText | error, + graniteLatencySec}`, and totals (segments, errored, audio seconds, shadow + wall-clock, RTF). + +### 7. Comparison report + +`scripts/granite-shadow-report.py` (Python 3 stdlib only): +reads all `*.comparison.json`, emits `report.html` with: +- Aggregate table: sessions, segments, disagreement rate (word-level + Levenshtein on normalized text), granite RTF distribution, error counts. +- Per-session side-by-side view, word-level diff highlighting, sorted so the + highest-disagreement segments float to the top (those are the ones worth + human judgment). +No WER claims — no ground truth. The report structures Nic's eyeball pass. + +### 8. Testing (all without real models, network, or processes) + +- **Merge/pad pure functions:** existing behavior pinned (gap merge, padding + arithmetic, boundary clamps) — these tests also protect the primary path + through the refactor. +- **`GraniteSidecar` state machine** (fake `SidecarProcessRunning`): launch → + ready; launch timeout → failed + kill; connection error → one relaunch; + second error → failed; stop always terminates; termination escalation. +- **`GraniteRequest`/response parsing:** pure request-build + response-extract + functions, golden-file JSON. +- **Shadow phase policy** (fake `SegmentTranscribing` + temp dirs): flag off → + no-op; missing binary/models → skip + log; happy path → both artifacts + written, counts/pairing correct; per-segment error → recorded, phase + completes; sidecar failure mid-run → incomplete artifacts, job still + `.complete`; cancellation mid-run → incomplete artifacts, job semantics + unchanged; solo-memo session → skip. Also: WAVs still present when the + phase runs (ordering regression test). +- **`ShadowConfig`:** defaults parsing, path expansion, disabled default. +- **ASRBench manifest mode:** manifest parse/emit as pure functions with unit + tests; the transcribe loop reuses the existing bench plumbing. +- Report script: golden-input test run in CI via `python3` if available, + else exercised manually (script is stdlib-only, deterministic). +- Phase 0 harness: validated by its own fidelity gate (reproducing published + numbers IS the test); no CI coverage — it's a run-once-per-decision tool. + +## Risks / open questions + +- **llama-server audio API details** (exact endpoint shape/prompt for mtmd + audio) are pinned by the Phase 0 harness before any Swift HTTP code is + written. If the server path proves broken for granite audio, fallback is + shelling out to `llama-mtmd-cli` per segment (same artifacts, worse + latency) — decided in Phase 0, not later. +- **llama.cpp front-end WER fidelity** is measured directly by Phase 0's + fidelity gate (reproduce published AMI/Earnings-22 numbers within ~1.5 + points) — a gross fidelity bug can no longer masquerade as "granite is + bad" in the shadow data. +- **Phase 0 flatters granite on AMI/Earnings-22** (their train splits are in + granite's training data). The CORAAL held-out check and the shadow pass on + Nic's real meetings are the counterweights; a granite win that appears + ONLY on the in-domain sets is a yellow flag, not a green one. +- **Job duration grows** by the shadow time (est. 2–10 min per meeting hour); + the Settings model-picker lock window grows with it. Accepted for the + experiment; the report's RTF numbers feed the eventual dual-slot design. +- **Memory spike** during shadow phase: ~4 GB (Q8_0 + mmproj + KV) on top of + the resident live model. Fine on both target machines (64 GB / Studio). +- Segment-level transcription forfeits cross-segment context granite could + use (it's a 128k-context LLM); the comparison measures granite *in Tome's + pipeline shape*, not granite's ceiling. Noted so a mediocre result prompts + "try longer windows" before "reject model". + +## Success criteria + +**Phase 0 gates (before *enabling* shadow mode — building may proceed in +parallel; the gates protect the shadow data's trustworthiness, and the +schedule needs shadow capturing meetings from Friday 2026-07-10):** +1. Fidelity: granite via our llama-server pipeline reproduces its published + AMI + Earnings-22 raw WER within ~1.5 points absolute. +2. Speed: RTF on M2 Max ≤ ~0.25 (hour meeting in ≤ 15 min). +3. Accuracy: granite beats BOTH current backends on AMI and Earnings-22, and + at least matches Parakeet v3 on the held-out CORAAL sample (a win only on + in-domain sets is a yellow flag → discuss before proceeding). + +**Phase 1 confirmation (a few days of real-meeting shadow data):** +4. Nic's judgment on the top-disagreement segments favors granite clearly + more often than the primary (names, accents, cross-talk are the cases to + watch), and +5. No systemic pathologies (hallucinated segments, dropped words in noise, + repetition loops) beyond what the primary shows. + +All five → promote granite to the full dual-slot design. Otherwise: keep the +artifacts, write up findings, revisit when runtimes/models move (the research +memo lists granite-4.1-2b-plus and higgs-2.7B as the next candidates to +re-check). diff --git a/scripts/asr-bench/bench.py b/scripts/asr-bench/bench.py new file mode 100644 index 0000000..ac243c6 --- /dev/null +++ b/scripts/asr-bench/bench.py @@ -0,0 +1,126 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["datasets[audio]>=3", "soundfile", "jiwer", "transformers", "numpy"] +# /// +"""Phase 0 ASR benchmark. Stages: + uv run bench.py export --work /tmp/asrbench --sets ami,earnings22,tedlium --max-hours 3 + (then run ASRBench manifest mode for parakeet + whisper — command is printed) + uv run bench.py granite --work /tmp/asrbench --url http://127.0.0.1:8873 + uv run bench.py score --work /tmp/asrbench +Reference/normalizer per Open ASR Leaderboard: WhisperTokenizer.normalize (public +method on transformers 5.x; the brief's `_normalize` was the private name on an +older transformers release and no longer exists — verified live against +transformers 5.13.0, see task-4-report.md). + +Dataset sourcing (verified against github.com/huggingface/open_asr_leaderboard and +live HF Hub probes — see docs/superpowers/plans task-4-report.md for the full trail): + - ami, earnings22: hf-audio/open-asr-leaderboard (parquet, config name == set name, + split "test", ref column "text"). This is the successor repo behind the + hf-audio/esb-datasets-test-only-sorted alias (which 307-redirects to it) — + used directly here to avoid the redirect. NOTE: this repo's own README lists a + "tedlium" config, but the tedlium/ directory does not actually exist there + (confirmed via the Hub tree API) — a genuine gap in that repo, not a config + typo on our part. + - tedlium: distil-whisper/tedlium, config "default", split "test", ref column + "text", pinned to revision "refs/convert/parquet" (the Hub's auto-generated + parquet mirror of this legacy loading-script dataset — `datasets>=3` refuses + to execute loading scripts at all, so the un-pinned repo is unusable; the + parquet-convert branch collapses the script's "release3" config name down to + "default"). Raw text retains TED-LIUM STM artifacts (`ignore_time_segment_in_scoring`, + and similar bracket/gap tokens) that upstream's own loader would normally drop + at generation time — export() filters those explicitly (see SKIP_MARKERS) + and reports a skipped count. +""" +import argparse, json, pathlib, sys, time + +# (dataset repo id, config name, revision-or-None) per set — see module docstring. +DATASETS = { + "ami": ("hf-audio/open-asr-leaderboard", "ami", None), + "earnings22": ("hf-audio/open-asr-leaderboard", "earnings22", None), + "tedlium": ("distil-whisper/tedlium", "default", "refs/convert/parquet"), +} +# TED-LIUM STM scoring-gap / non-speech markers (mirrors hf-audio/esb-datasets-test-only's +# TED-LIUM `ignore_segments` set from datasets-test-only.py, which upstream's now-unusable +# loading script applied at generation time). Matched against the *raw* (pre-normalization) reference text. +# Source: https://huggingface.co/datasets/hf-audio/esb-datasets-test-only/raw/main/datasets-test-only.py +SKIP_MARKERS = {"ignore_time_segment_in_scoring", "", "", "", "[noise]", + "[laughter]", "[silence]", "[vocalized-noise]", "", "", + "", "", ""} + + +def export(work, sets, max_hours): + import soundfile as sf + from datasets import load_dataset, Audio + for s in sets: + repo, config, revision = DATASETS[s] + d = work / s; (d / "wav").mkdir(parents=True, exist_ok=True) + kwargs = {"revision": revision} if revision else {} + ds = load_dataset(repo, config, split="test", streaming=True, **kwargs) + ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + refcol = next(c for c in ("text", "norm_transcript", "transcription", "sentence") + if c in ds.column_names) + total, manifest, refs, skipped = 0.0, [], {}, 0 + for i, row in enumerate(ds): + ref_raw = row[refcol] + if ref_raw is None or ref_raw.strip() in SKIP_MARKERS: + skipped += 1; continue + audio = row["audio"]; dur = len(audio["array"]) / audio["sampling_rate"] + if total + dur > max_hours * 3600: break + total += dur + rid = f"{s}-{i:05d}"; wav = d / "wav" / f"{rid}.wav" + sf.write(wav, audio["array"], 16000, subtype="PCM_16") + manifest.append({"id": rid, "wav": str(wav)}); refs[rid] = {"ref": ref_raw, "dur": dur} + (d / "manifest.jsonl").write_text("".join(json.dumps(m) + "\n" for m in manifest)) + (d / "refs.json").write_text(json.dumps(refs)) + print(f"[{s}] {len(manifest)} utts, {total/3600:.2f} h, {skipped} skipped (repo: {repo}, ref column: {refcol})") + print("\nNow produce Tome-backend hypotheses (from Tome/, using the prebuilt release binary):") + for s in sets: + for b in ("parakeet", "whisper"): + print(f" DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer Tome/.build/release/ASRBench " + f"--manifest {work}/{s}/manifest.jsonl --backend {b} --out {work}/{s}/hyp_{b}.jsonl") + + +def granite(work, url): + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + from granite_client import transcribe + for d in sorted(p for p in work.iterdir() if (p / "manifest.jsonl").exists()): + out, wall, audio_s = [], 0.0, 0.0 + refs = json.loads((d / "refs.json").read_text()) + for line in (d / "manifest.jsonl").read_text().splitlines(): + m = json.loads(line) + try: + text, dt = transcribe(url, m["wav"]) + except Exception as e: # noqa: BLE001 — record and continue + text, dt = "", 0.0; print(f" ERR {m['id']}: {e}") + out.append({"id": m["id"], "text": text}); wall += dt; audio_s += refs[m["id"]]["dur"] + if len(out) % 50 == 0: print(f"[{d.name}] {len(out)} done, RTF so far {wall/max(audio_s,1):.3f}") + (d / "hyp_granite.jsonl").write_text("".join(json.dumps(o) + "\n" for o in out)) + print(f"[{d.name}] granite RTF (single-stream M2 Max): {wall/max(audio_s,1):.3f}") + + +def score(work): + import jiwer + from transformers import WhisperTokenizer + tok = WhisperTokenizer.from_pretrained("openai/whisper-tiny") + rows = [] + for d in sorted(p for p in work.iterdir() if (p / "refs.json").exists()): + refs = json.loads((d / "refs.json").read_text()) + for hyp_file in sorted(d.glob("hyp_*.jsonl")): + hyps = {json.loads(l)["id"]: json.loads(l)["text"] for l in hyp_file.read_text().splitlines()} + pairs = [(tok.normalize(refs[i]["ref"]), tok.normalize(hyps.get(i, ""))) + for i in refs if tok.normalize(refs[i]["ref"]).strip()] + wer = jiwer.wer([r for r, _ in pairs], [h for _, h in pairs]) * 100 + rows.append((d.name, hyp_file.stem.removeprefix("hyp_"), wer, len(pairs))) + print(f"{'set':<12}{'backend':<12}{'WER%':>8}{'utts':>7}") + for s, b, w, n in rows: print(f"{s:<12}{b:<12}{w:>8.2f}{n:>7}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(); ap.add_argument("stage", choices=["export", "granite", "score"]) + ap.add_argument("--work", type=pathlib.Path, required=True) + ap.add_argument("--sets", default="ami,earnings22,tedlium"); ap.add_argument("--max-hours", type=float, default=3) + ap.add_argument("--url", default="http://127.0.0.1:8873") + a = ap.parse_args(); a.work.mkdir(parents=True, exist_ok=True) + {"export": lambda: export(a.work, a.sets.split(","), a.max_hours), + "granite": lambda: granite(a.work, a.url), + "score": lambda: score(a.work)}[a.stage]() diff --git a/scripts/asr-bench/granite_client.py b/scripts/asr-bench/granite_client.py new file mode 100644 index 0000000..1b8b24e --- /dev/null +++ b/scripts/asr-bench/granite_client.py @@ -0,0 +1,31 @@ +"""Granite llama-server client. Request contract: see granite_request.md (source of truth).""" +import base64, json, time, urllib.request + +PROMPT = "can you transcribe the speech into a written format?" # granite_request.md + + +def build_request(wav_bytes: bytes, prompt: str = PROMPT) -> dict: + return { + "messages": [{"role": "user", "content": [ + {"type": "input_audio", + "input_audio": {"data": base64.b64encode(wav_bytes).decode(), "format": "wav"}}, + {"type": "text", "text": prompt}, + ]}], + "temperature": 0, "max_tokens": 2048, "stream": False, + } + + +def transcribe(base_url: str, wav_path: str) -> tuple[str, float]: + body = json.dumps(build_request(open(wav_path, "rb").read())).encode() + req = urllib.request.Request(f"{base_url}/v1/chat/completions", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.monotonic() + with urllib.request.urlopen(req, timeout=600) as resp: + out = json.load(resp) + return out["choices"][0]["message"]["content"].strip(), time.monotonic() - t0 + + +if __name__ == "__main__": + import sys + text, dt = transcribe(sys.argv[1] if len(sys.argv) > 2 else "http://127.0.0.1:8873", sys.argv[-1]) + print(f"[{dt:.1f}s] {text}") diff --git a/scripts/asr-bench/granite_request.md b/scripts/asr-bench/granite_request.md new file mode 100644 index 0000000..6a8fb27 --- /dev/null +++ b/scripts/asr-bench/granite_request.md @@ -0,0 +1,200 @@ +# Granite llama-server request contract + +**Status: PINNED.** This is the single source of truth for the request shape +used to transcribe audio via the `llama-server` sidecar running the +IBM granite-speech-4.1-2b GGUF. Task 8's Swift `GraniteRequest` and the +Phase 0 Python benchmark both implement this verbatim. Do not change this +file casually — anything that depends on the request shape (Swift and +Python) must be updated together with it. + +## Result: the brief's proposed shape worked on the first try + +The OpenAI-compatible `input_audio` content part, exactly as drafted in the +Task 2 brief, was accepted by `llama-server` b9910 with no modification. +No fallback (alternate content-type, `llama-mtmd-cli`) was needed. + +## Server launch (verbatim, matches `scripts/setup-granite-shadow.sh` and +the spec's `GraniteSidecar.start()`) + +```bash +/opt/homebrew/bin/llama-server \ + -m "$HOME/Library/Application Support/Tome/Granite/granite-speech-4.1-2b-Q8_0.gguf" \ + --mmproj "$HOME/Library/Application Support/Tome/Granite/mmproj-model-f16.gguf" \ + --host 127.0.0.1 --port 8873 +``` + +- Binds `127.0.0.1` only (never `0.0.0.0`). +- Model load + mmproj load took ~1.2 s combined in this probe run (llama.cpp + b9910, M2 Max); server log line `llama_server: model loaded` confirms + readiness alongside `GET /health`. +- Readiness check: poll `GET http://127.0.0.1:8873/health` until it returns + HTTP 200 with body `{"status":"ok"}`. In this probe run it was already + `ok` on the very first poll (~0.1 s after process spawn), well under the + 60 s timeout the spec's `GraniteSidecar.start()` uses. +- Server log emits one experimental-feature warning on load, expected and + harmless: + `W init_audio: audio input is in experimental stage and may have reduced quality` + (https://github.com/ggml-org/llama.cpp/discussions/13759). + +## Endpoint + +``` +POST http://127.0.0.1:8873/v1/chat/completions +Content-Type: application/json +``` + +## Request body (exact JSON shape, prompt string verbatim) + +```json +{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { "data": "", "format": "wav" } + }, + { + "type": "text", + "text": "can you transcribe the speech into a written format?" + } + ] + } + ], + "temperature": 0, + "max_tokens": 2048, + "stream": false +} +``` + +- `input_audio.data`: base64 of the raw WAV file bytes (16 kHz mono PCM16, + see below — the model/mmproj combo expects this format; other sample + rates/channel counts were not tested here and are out of scope for this + pin). +- `input_audio.format`: literal string `"wav"`. +- Prompt text is verbatim: `can you transcribe the speech into a written format?` + Do not reword — it has not been varied/tested against alternatives in this + task, and the fidelity-gate work in Phase 0 assumes this exact string. +- `temperature: 0` — greedy decoding, deterministic output, required per spec. +- `max_tokens: 2048` — generous ceiling; the fox-sentence probe used only 10 + completion tokens. +- `stream: false` — the sidecar and Python client both use blocking requests. + +## Response extraction path + +```python +response_json["choices"][0]["message"]["content"] +``` + +The content is a plain string (not itself an array/content-parts structure) +containing the transcript. Strip leading/trailing whitespace before use. + +Example full response body observed (probe run, see below for audio): + +```json +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "the quick brown fox jumps over the lazy dog" + } + } + ], + "created": 1783653774, + "model": "/Users/nic/Library/Application Support/Tome/Granite/granite-speech-4.1-2b-Q8_0.gguf", + "system_fingerprint": "b9910-f5525f7e7", + "object": "chat.completion", + "usage": { + "completion_tokens": 10, + "prompt_tokens": 46, + "total_tokens": 56, + "prompt_tokens_details": { "cached_tokens": 0 } + }, + "id": "chatcmpl-MncjOJMwl0vLLOyI6PLzAhJJDAlvzrhc", + "timings": { + "cache_n": 0, + "prompt_n": 46, + "prompt_ms": 651.714, + "prompt_per_token_ms": 14.167695652173915, + "prompt_per_second": 70.58310854147678, + "predicted_n": 10, + "predicted_ms": 73.517, + "predicted_per_token_ms": 7.351699999999999, + "predicted_per_second": 136.02296067576208 + } +} +``` + +## Probe audio + +Generated per the brief: + +```bash +say -o /tmp/probe.aiff "the quick brown fox jumps over the lazy dog" +afconvert -f WAVE -d LEI16@16000 -c 1 /tmp/probe.aiff /tmp/probe.wav +``` + +Verified with `afinfo /tmp/probe.wav`: + +``` +Data format: 1 ch, 16000 Hz, Int16 +estimated duration: 2.533250 sec +``` + +16 kHz, mono, Int16 (PCM16) WAV — confirmed. Note: `say`'s rendering of the +fox sentence is ~2.53 s long, not the "10 s" placeholder mentioned in the +brief's Step 1 preamble (that referred to the general idea of recording ~10 +s of speech, not a hard requirement — the brief's own repro command is the +`say`/`afconvert` one-liner above, which is what was actually used). + +## Probe transcript: observed vs expected + +- Expected (per brief): `"the quick brown fox jumps over the lazy dog"` + (case/punctuation may vary) +- Observed: `"the quick brown fox jumps over the lazy dog"` +- Exact match, no case or punctuation drift. + +## Latency / RTF (first M2 Max datapoint) + +Four requests were sent to the same warm server (model resident, KV cache +cold each time since each request is a fresh conversation): + +| run | latency (s) | +|-----|-------------| +| 1 (server just became ready) | 0.75 | +| 2 | 0.109 | +| 3 | 0.091 | +| 4 | 0.092 | + +Run 1 (0.75 s) includes one-time warmup costs (first-token / graph +compilation, mmproj activation) not present in steady-state calls; runs 2-4 +(~0.09-0.11 s) are the representative steady-state figure. + +Audio duration: 2.533 s. + +- Steady-state RTF (using run 2, 0.109 s / 2.533 s): **~0.043** (i.e. ~23x + faster than real time for this short 2.5 s clip on an M2 Max, warm + server). +- First-call RTF (using run 1, 0.75 s / 2.533 s): **~0.30** (~3.3x + real-time), representative of the very first request after server + readiness. + +Caveat: this is a single short (2.5 s) utterance, not the 10 s clip +originally envisioned, and is not representative of RTF on the longer +AMI/Earnings-22 Phase 0 benchmark clips — those will produce the +authoritative RTF numbers. This datapoint only confirms the request/response +plumbing and gives a rough order-of-magnitude sanity check. + +## What was NOT needed + +The brief's contingency path (`llama-server --help` / `docs/multimodal.md` +consultation for an alternate content type, or the `llama-mtmd-cli` HTTP +fallback) was not exercised — the OpenAI-compatible `input_audio` shape +worked immediately. This section is left here for completeness in case a +future llama.cpp upgrade breaks the shape and someone needs to know it was +deliberately not investigated further (not because alternatives don't +exist, but because the primary path succeeded). diff --git a/scripts/granite-shadow-report.py b/scripts/granite-shadow-report.py new file mode 100755 index 0000000..1115572 --- /dev/null +++ b/scripts/granite-shadow-report.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Render granite shadow comparison JSONs into one side-by-side HTML report. +Usage: python3 granite-shadow-report.py "~/Library/Application Support/Tome/GraniteShadow" [-o report.html] +Stdlib only (spec §7).""" +import argparse, difflib, html, json, pathlib, sys + + +def word_diff(a: str, b: str) -> tuple[float, str, str]: + aw, bw = a.split(), b.split() + sm = difflib.SequenceMatcher(a=aw, b=bw) + left, right = [], [] + for op, i1, i2, j1, j2 in sm.get_opcodes(): + at, bt = " ".join(aw[i1:i2]), " ".join(bw[j1:j2]) + if op == "equal": + left.append(html.escape(at)); right.append(html.escape(bt)) + else: + if at: left.append(f"{html.escape(at)}") + if bt: right.append(f"{html.escape(bt)}") + # Two empty strings (e.g. both sides blank) are trivially "equal" per + # SequenceMatcher (ratio 1.0) — but an errored segment has empty + # graniteText vs non-empty primaryText, which correctly scores as 100% + # diff and sorts to the top. + return 1 - sm.ratio(), " ".join(left), " ".join(right) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("dir", type=pathlib.Path) + ap.add_argument("-o", "--out", type=pathlib.Path, default=pathlib.Path("report.html")) + args = ap.parse_args() + sessions = [] + unreadable = 0 + for p in sorted(args.dir.expanduser().glob("*.comparison.json")): + try: + sessions.append(json.loads(p.read_text())) + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: + print(f"warning: skipping malformed {p.name}: {e}", file=sys.stderr) + unreadable += 1 + rows, agg = [], {"sessions": len(sessions), "segments": 0, "errored": 0, + "audio": 0.0, "wall": 0.0, "disagree": 0} + for s in sessions: + agg["segments"] += s["totals"]["segmentCount"]; agg["errored"] += s["totals"]["erroredCount"] + agg["audio"] += s["totals"]["audioSeconds"]; agg["wall"] += s["totals"]["shadowWallClockSec"] + for seg in s["segments"]: + score, lh, rh = word_diff(seg["primaryText"], seg["graniteText"]) + if score > 0.05: agg["disagree"] += 1 + rows.append((score, s["session"]["sessionID"], s["session"]["primaryModel"], + s["totals"]["rtf"], s["incomplete"], seg, lh, rh)) + rows.sort(key=lambda r: -r[0]) + rtf = agg["wall"] / agg["audio"] if agg["audio"] else 0 + body = ["

Granite shadow report

", + f"

{agg['sessions']} sessions · {agg['segments']} segments · " + f"{agg['disagree']} disagreeing (>5% word diff) · {agg['errored']} errored · " + f"{unreadable} unreadable · " + f"aggregate shadow RTF {rtf:.3f}

", + "", + "", + ""] + for score, sid, pmodel, srtf, incomplete, seg, lh, rh in rows: + incomplete_marker = ' INCOMPLETE' if incomplete else "" + error_badge = (f'ERROR: {html.escape(seg["graniteError"])}
' + if seg.get("graniteError") else "") + body.append(f"" + f"") + body.append("
diffsessiontprimarygranite
{score:.2f}{html.escape(sid)}{incomplete_marker}
{html.escape(pmodel)}" + f" · RTF {srtf:.3f}		{seg['startTime']:.0f}s{lh}{error_badge}{rh}
") + args.out.write_text("\n".join(body)) + print(f"wrote {args.out} ({agg['segments']} segments)") + + +if __name__ == "__main__": + main() diff --git a/scripts/setup-granite-shadow.sh b/scripts/setup-granite-shadow.sh new file mode 100755 index 0000000..59d4f64 --- /dev/null +++ b/scripts/setup-granite-shadow.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Setup for granite shadow transcription (spec: docs/superpowers/specs/2026-07-09-granite-shadow-transcription-design.md) +set -euo pipefail + +MODEL_DIR="$HOME/Library/Application Support/Tome/Granite" +REPO="https://huggingface.co/ibm-granite/granite-speech-4.1-2b-GGUF/resolve/main" +MODEL="granite-speech-4.1-2b-Q8_0.gguf" # keep in sync with ShadowConfig.modelFilename +MMPROJ="mmproj-model-f16.gguf" # keep in sync with ShadowConfig.mmprojFilename +SERVER="${LLAMA_SERVER:-/opt/homebrew/bin/llama-server}" +PORT="${GRANITE_PORT:-8873}" + +if [[ ! -x "$SERVER" ]]; then + echo "llama-server not found at $SERVER — run: brew install llama.cpp" >&2 + exit 1 +fi +# b9045+ required for granite-speech mtmd support +# --version prints either "bNNNNN" (older builds) or "version: NNNNN (hash)" +# (current homebrew llama.cpp) — match either form. `|| true` on each grep +# guards against `set -e` killing the script when a pattern doesn't match. +VERSION_OUTPUT=$("$SERVER" --version 2>&1 || true) +BUILD=$(printf '%s' "$VERSION_OUTPUT" | grep -oE 'b[0-9]+' | head -1 | tr -d 'b' || true) +if [[ -z "$BUILD" ]]; then + BUILD=$(printf '%s' "$VERSION_OUTPUT" | grep -oE 'version: [0-9]+' | head -1 | grep -oE '[0-9]+' || true) +fi +if [[ -z "$BUILD" || "$BUILD" -lt 9045 ]]; then + echo "llama.cpp build b${BUILD:-unknown} < b9045 — run: brew upgrade llama.cpp" >&2 + exit 1 +fi + +mkdir -p "$MODEL_DIR" +for f in "$MODEL" "$MMPROJ"; do + echo "Downloading $f (resumable)…" + curl -L -C - --fail -o "$MODEL_DIR/$f" "$REPO/$f" +done +ls -lh "$MODEL_DIR" + +cat <alert(1) now", + "graniteText": "click now", + "graniteError": None, "graniteLatencySec": 0.1}, + ] + sample["totals"]["segmentCount"] = 1 + with tempfile.TemporaryDirectory() as d: + d = pathlib.Path(d) + (d / "s4.comparison.json").write_text(json.dumps(sample)) + out = d / "report.html" + run_report(d, out) + html_text = out.read_text() + self.assertNotIn("", html_text) + self.assertIn("<script>", html_text) + + +if __name__ == "__main__": + unittest.main()