From 90bc98d62c65df5b7fcffc431d444430df7e1935 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Mon, 27 Jul 2026 21:41:12 +0200 Subject: [PATCH 01/55] feat(compositor): fail legibly when D3D11 init fails, instead of falling back to WARP PR #162 scoped a WARP retry for the compositor's D3D11 device. Measured, WARP cannot serve this pipeline at all -- and not for the expected reason (speed): - WARP + D3D11_CREATE_DEVICE_VIDEO_SUPPORT does not create. It returns DXGI_ERROR_UNSUPPORTED (0x887A0004). - Drop the flag and WARP creates at FL 11_1, but QueryInterface for ID3D11VideoDevice returns E_NOINTERFACE -- zero decoder profiles. pipeline.rs hands Gpu's device to ffmpeg as the AVD3D11VADeviceContext, so preview and export decode every frame on it. A WARP device would produce none: the retry would only convert a clear startup failure into an obscure ffmpeg one. crates/compositor/tests/warp_device_cannot_decode.rs pins both measurements and fails if Windows ever changes them. So there is no fallback, and the answer to "WARP for export or preview only?" is neither. What the compositor does instead is fail in a way the user can act on: - Gpu::create re-probes on the failure path (same call, minus VIDEO_SUPPORT) to tell "this adapter has no video decoder" -- the RDP / VM case -- apart from "no FL 11_1 adapter at all", and says which, plus what to do. Export gets this for free: ExportDialog already renders the native message. - That message now reaches the preview too. create_view returns an id long before the render thread can die, so the failure existed only as an eprintln! and the user just saw a black canvas. The thread stores its fatal error in live::Shared, read_frame relays it as an Err on the next pull (~33 ms), and NativeCompositorOverlay renders it in place of the canvas. The WGC capture helper stays hardware-only on purpose: it requests no VIDEO_SUPPORT so a WARP device would be creatable there, but recording on a host whose compositor cannot start only produces footage the user can neither edit nor export. --- crates/compositor-view-napi/src/lib.rs | 13 +- crates/compositor/src/d3d.rs | 95 ++++++++++---- crates/compositor/src/live.rs | 19 ++- .../tests/warp_device_cannot_decode.rs | 77 ++++++++++++ .../ai-edition/NativeCompositorOverlay.tsx | 35 +++++- src/i18n/locales/ar/editor.json | 3 +- src/i18n/locales/en/editor.json | 3 +- src/i18n/locales/es/editor.json | 3 +- src/i18n/locales/fr/editor.json | 3 +- src/i18n/locales/it/editor.json | 3 +- src/i18n/locales/ja-JP/editor.json | 3 +- src/i18n/locales/ko-KR/editor.json | 3 +- src/i18n/locales/pt-BR/editor.json | 3 +- src/i18n/locales/ru/editor.json | 3 +- src/i18n/locales/tr/editor.json | 3 +- src/i18n/locales/vi/editor.json | 3 +- src/i18n/locales/zh-CN/editor.json | 3 +- src/i18n/locales/zh-TW/editor.json | 3 +- .../hooks/useNativeCompositorView.test.ts | 118 ++++++++++++++++++ src/native/hooks/useNativeCompositorView.ts | 33 ++++- .../architecture/native-compositor.md | 47 +++++-- 21 files changed, 425 insertions(+), 51 deletions(-) create mode 100644 crates/compositor/tests/warp_device_cannot_decode.rs create mode 100644 src/native/hooks/useNativeCompositorView.test.ts diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 9b2e428810..d51bfb6d3d 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -138,7 +138,18 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { // bloquerait tout autre appel napi (`set_rect`, `destroy_view`, ...). let slot = match registry().lock().unwrap().get(&id) { None => return Ok(None), - Some(v) => v.latest_frame_since(since_gen.max(0.0) as u64), + Some(v) => { + // Le thread de rendu est mort (device D3D11 indisponible, décodeur en échec…) : + // il ne publiera plus jamais de frame. Sans ce relais, `create_view` a déjà + // répondu Ok et l'échec ne se voyait que dans un `eprintln!` — l'utilisateur + // restait devant un canvas noir sans explication (PR #162). La boucle de pull + // du renderer appelle ceci ~30×/s, donc l'erreur remonte tout de suite, et par + // le chemin d'erreur que `read_frame` a déjà (`Result`), sans changer le contrat. + if let Some(fatal) = v.fatal_error() { + return Err(Error::from_reason(fatal)); + } + v.latest_frame_since(since_gen.max(0.0) as u64) + } }; Ok(slot.map(|(gen, w, h, pixels)| { debug_assert_eq!(pixels.len(), (w as usize) * (h as usize) * 4); diff --git a/crates/compositor/src/d3d.rs b/crates/compositor/src/d3d.rs index a5fb74b9f0..b5a0c05e4b 100644 --- a/crates/compositor/src/d3d.rs +++ b/crates/compositor/src/d3d.rs @@ -7,11 +7,11 @@ use anyhow::{bail, Result}; use windows::core::Interface; use windows::Win32::Foundation::HMODULE; use windows::Win32::Graphics::Direct3D::{ - D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_1, + D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_1, }; use windows::Win32::Graphics::Direct3D11::{ D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Multithread, - D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_DEBUG, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_DEBUG, D3D11_CREATE_DEVICE_FLAG, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, }; @@ -21,6 +21,72 @@ pub struct Gpu { pub feature_level: D3D_FEATURE_LEVEL, } +/// Une tentative `D3D11CreateDevice` à FL 11_1. Extraite pour que le chemin d'échec +/// puisse re-sonder avec d'autres flags/driver et dire POURQUOI la vraie tentative +/// a échoué (voir `diagnose`), au lieu de remonter un HRESULT nu. +fn try_create( + driver: D3D_DRIVER_TYPE, + flags: D3D11_CREATE_DEVICE_FLAG, +) -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext, D3D_FEATURE_LEVEL)> { + let levels = [D3D_FEATURE_LEVEL_11_1]; + let mut device: Option = None; + let mut context: Option = None; + let mut got = D3D_FEATURE_LEVEL::default(); + unsafe { + D3D11CreateDevice( + None, + driver, + HMODULE::default(), + flags, + Some(&levels), + D3D11_SDK_VERSION, + Some(&mut device), + Some(&mut got), + Some(&mut context), + )?; + } + // Le SDK garantit les deux sorties quand l'appel réussit ; `E_UNEXPECTED` plutôt + // qu'un `unwrap` pour que l'impossible reste une erreur, pas un panic. + match (device, context) { + (Some(device), Some(context)) => Ok((device, context, got)), + _ => Err(windows::core::Error::from(windows::Win32::Foundation::E_UNEXPECTED)), + } +} + +/// Message d'échec ACTIONNABLE : re-sonde pour distinguer les deux causes réelles. +/// +/// PR #162 proposait de retomber sur `D3D_DRIVER_TYPE_WARP`. Mesuré sur cette machine +/// (`tests/warp_device_cannot_decode.rs`, qui échouera si Windows change d'avis) : +/// WARP + `VIDEO_SUPPORT` ne se crée même pas (`DXGI_ERROR_UNSUPPORTED`), et WARP sans +/// ce flag n'expose aucun `ID3D11VideoDevice` (`E_NOINTERFACE`, 0 profil décodeur). +/// Or `pipeline.rs` passe CE device à ffmpeg comme `AVD3D11VADeviceContext` : preview +/// comme export décodent chaque frame en D3D11VA. Un device WARP produirait donc zéro +/// frame et transformerait cet échec net en panne ffmpeg obscure. D'où : pas de repli, +/// mais un échec qui se lit. +fn diagnose(err: &windows::core::Error) -> String { + // Le décodeur est le point de rupture le plus probable (RDP, VM sans passthrough, + // Microsoft Basic Render Driver) : si l'appel passe SANS VIDEO_SUPPORT, l'adaptateur + // est là, c'est son décodeur qui manque. La sonde ne garde que BGRA — surtout pas + // `flags` moins VIDEO_SUPPORT, qui traînerait `DEBUG` avec lui : sans les Graphics + // Tools de Windows la couche debug fait échouer la sonde aussi, et on accuserait + // l'adaptateur à tort. Ce cas-là se lit déjà dans `{err}` + // (`DXGI_ERROR_SDK_COMPONENT_MISSING`), il n'a pas besoin de sa propre branche. + if try_create(D3D_DRIVER_TYPE_HARDWARE, D3D11_CREATE_DEVICE_BGRA_SUPPORT).is_ok() { + return format!( + "this display adapter has no D3D11 video decoder ({err}). OpenScreen decodes \ + every preview and export frame with D3D11VA on the same device it composites \ + with, so the decoder is not optional and there is no CPU path behind it. \ + Remote Desktop sessions and VMs without GPU passthrough land here: run on the \ + physical machine, or update the display driver." + ); + } + format!( + "no Direct3D 11 feature level 11_1 display adapter ({err}). OpenScreen's compositor \ + requires one for both preview and export. Update the display driver, or run on a \ + machine with a GPU that reaches feature level 11_1." + ) +} + impl Gpu { /// Crée le device conforme au §2. `debug=false` impératif dans tout run mesuré /// (§10 : la couche debug valide et sérialise chaque appel — facteur, pas %). @@ -32,27 +98,10 @@ impl Gpu { flags |= D3D11_CREATE_DEVICE_DEBUG; } - let levels = [D3D_FEATURE_LEVEL_11_1]; - let mut device: Option = None; - let mut context: Option = None; - let mut got = D3D_FEATURE_LEVEL::default(); - - unsafe { - D3D11CreateDevice( - None, - D3D_DRIVER_TYPE_HARDWARE, - HMODULE::default(), - flags, - Some(&levels), - D3D11_SDK_VERSION, - Some(&mut device), - Some(&mut got), - Some(&mut context), - )?; - } - - let device = device.ok_or_else(|| anyhow::anyhow!("D3D11CreateDevice: pas de device"))?; - let context = context.ok_or_else(|| anyhow::anyhow!("D3D11CreateDevice: pas de contexte"))?; + let (device, context, got) = match try_create(D3D_DRIVER_TYPE_HARDWARE, flags) { + Ok(gpu) => gpu, + Err(err) => bail!("{}", diagnose(&err)), + }; if got != D3D_FEATURE_LEVEL_11_1 { bail!("feature level obtenu {:?} != 11_1", got); diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 0ba8e9e884..1fb1f1ce70 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -510,6 +510,13 @@ struct Shared { /// vide la ferait repartir à 1 — donc rejouer des générations déjà peintes. Monotone, /// jamais remise à zéro. frame_gen: AtomicU64, + /// Erreur fatale du thread de rendu (device D3D11 introuvable, décodeur qui refuse + /// le fichier…). Le thread meurt sur la première erreur ; sans ce champ, elle + /// finissait dans un `eprintln!` que personne ne lit et l'utilisateur n'avait + /// qu'un canvas noir — exactement le « on dirait que l'app rame » de la PR #162. + /// `read_frame` la relaie en `Err` au prochain tour de la boucle de pull (~33 ms), + /// donc elle remonte jusqu'à l'UI par le chemin d'erreur qui existe déjà. + fatal: Mutex>, } /// Handle d'une vue live. `Drop` arrête le rendu. @@ -556,12 +563,16 @@ impl LiveView { stop: AtomicBool::new(false), latest_frame: Mutex::new(None), frame_gen: AtomicU64::new(0), + fatal: Mutex::new(None), }); let sh = shared.clone(); let (s, wc, cj) = (screen.to_string(), webcam.to_string(), cursor_json.to_string()); let thread = std::thread::spawn(move || { - if let Err(e) = unsafe { render_thread(sh, &s, &wc, &cj) } { + if let Err(e) = unsafe { render_thread(sh.clone(), &s, &wc, &cj) } { eprintln!("[live] render thread error: {e:#}"); + if let Ok(mut fatal) = sh.fatal.lock() { + *fatal = Some(format!("{e:#}")); + } } }); @@ -581,6 +592,12 @@ impl LiveView { } } + /// Message de l'erreur qui a tué le thread de rendu, `None` tant qu'il tourne. + /// Définitif : le thread ne redémarre pas. + pub fn fatal_error(&self) -> Option { + self.shared.fatal.lock().ok().and_then(|guard| guard.clone()) + } + /// Récupère la dernière frame readback (gen + taille + RGBA8 tightly-packed). /// `None` si rien n'a encore été composé (jamais écrit). **Coût : O(w·h)** /// (copie du `Vec` — nécessaire pour traverser la frontière thread + le diff --git a/crates/compositor/tests/warp_device_cannot_decode.rs b/crates/compositor/tests/warp_device_cannot_decode.rs new file mode 100644 index 0000000000..382b2e3ab6 --- /dev/null +++ b/crates/compositor/tests/warp_device_cannot_decode.rs @@ -0,0 +1,77 @@ +//! Pourquoi il n'y a PAS de repli WARP (PR #162). +//! +//! `d3d::Gpu::create` demande `D3D_DRIVER_TYPE_HARDWARE` et `VIDEO_SUPPORT`, sans +//! second choix. La proposition évidente — retenter en `D3D_DRIVER_TYPE_WARP`, le +//! rastériseur CPU de Microsoft — ne marche pas, et pas pour une raison de vitesse : +//! WARP n'a tout simplement pas de décodeur vidéo. Or `pipeline.rs` passe le device +//! de `Gpu` à ffmpeg comme `AVD3D11VADeviceContext` (`(*d3dctx).device = ...`), donc +//! preview ET export décodent chaque frame en D3D11VA sur CE device. Un device WARP +//! se créerait puis ne produirait aucune frame. +//! +//! Ce test verrouille la mesure. S'il ÉCHOUE, c'est que WARP a gagné une capacité +//! vidéo sur cette machine/version de Windows — et alors le repli redevient une +//! question ouverte : relire `diagnose()` dans `src/d3d.rs`. + +#![cfg(windows)] + +use windows::core::Interface; +use windows::Win32::Foundation::HMODULE; +use windows::Win32::Graphics::Direct3D::{ + D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_WARP, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_1, +}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11VideoDevice, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, +}; + +fn create( + driver: D3D_DRIVER_TYPE, + flags: D3D11_CREATE_DEVICE_FLAG, +) -> windows::core::Result { + let levels = [D3D_FEATURE_LEVEL_11_1]; + let mut device: Option = None; + let mut context: Option = None; + let mut got = D3D_FEATURE_LEVEL::default(); + unsafe { + D3D11CreateDevice( + None, + driver, + HMODULE::default(), + flags, + Some(&levels), + D3D11_SDK_VERSION, + Some(&mut device), + Some(&mut got), + Some(&mut context), + )?; + } + device.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_UNEXPECTED)) +} + +/// Exactement les flags de `Gpu::create`. Mesuré : `DXGI_ERROR_UNSUPPORTED` (0x887A0004) +/// — le device WARP ne se crée même pas avec `VIDEO_SUPPORT`. +#[test] +fn warp_rejects_the_video_support_flag_gpu_create_requires() { + let err = create( + D3D_DRIVER_TYPE_WARP, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT, + ) + .expect_err("WARP a accepté VIDEO_SUPPORT — le repli WARP redevient envisageable"); + assert_eq!(err.code().0 as u32, 0x887A_0004, "attendu DXGI_ERROR_UNSUPPORTED, eu {err}"); +} + +/// Et laisser tomber le flag ne sauve rien : le device se crée bien à FL 11_1, mais +/// il n'expose aucun `ID3D11VideoDevice`, donc zéro profil décodeur pour D3D11VA. +#[test] +fn warp_without_the_flag_still_exposes_no_video_device() { + let device = + create(D3D_DRIVER_TYPE_WARP, D3D11_CREATE_DEVICE_BGRA_SUPPORT).expect("WARP FL 11_1"); + let profiles = device + .cast::() + .map(|video| unsafe { video.GetVideoDecoderProfileCount() }); + assert!( + matches!(profiles, Err(_) | Ok(0)), + "WARP expose {profiles:?} profils décodeur — le repli WARP redevient envisageable" + ); +} diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index c232e61652..1c0073781e 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; +import { useScopedT } from "@/contexts/I18nContext"; import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap"; @@ -98,9 +99,10 @@ export function NativeCompositorOverlay() { }, [document]); const ready = sources !== null; - const { viewId } = useNativeCompositorView(canvasRef, { + const { viewId, error } = useNativeCompositorView(canvasRef, { sources: sources ?? undefined, }); + const t = useScopedT("editor"); // publie l'id de la vue active dans le store → l'inspector peut pousser des params // via setNativeParam sans connaître cet overlay. @@ -210,6 +212,37 @@ export function NativeCompositorOverlay() { return null; } + // The render thread died — no D3D11 device on this host, a decoder that refused the + // recording, etc. There is no second renderer to fall back to (the CPU and web preview + // paths were removed on purpose), and WARP is not one either: it has no video decoder, + // so it could not produce a single frame (see `crates/compositor/src/d3d.rs`). All we + // can do is say so. The canvas would otherwise just stay black, which reads as "the app + // is slow today" — the native message names the real cause and is worth surfacing raw, + // since it is what a bug report needs. + if (error) { + return ( +
+ {t("errors.previewCompositorUnavailable")} + {error} +
+ ); + } + // The canvas's CSS box (width: 100%; height: 100%) is what drives the // geometry; the hook manages the DRAWING BUFFER (canvas.width/height DOM // attrs) to match the offscreen render-target resolution, and paints each diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index bf035d34fa..ee74d85c77 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "فشل التصدير: تعذر تحميل صورة الخلفية ({{url}})", "failedToSaveExport": "فشل حفظ التصدير", "failedToSaveExportedVideo": "فشل حفظ الفيديو المُصدَّر", - "failedToRevealInFolder": "خطأ في الكشف في المجلد: {{error}}" + "failedToRevealInFolder": "خطأ في الكشف في المجلد: {{error}}", + "previewCompositorUnavailable": "المعاينة غير متوفرة على هذا الجهاز" }, "export": { "canceled": "تم إلغاء التصدير", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 90a5f02f98..8262a019d4 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "Export failed: could not load background image ({{url}})", "failedToSaveExport": "Failed to save export", "failedToSaveExportedVideo": "Failed to save exported video", - "failedToRevealInFolder": "Error revealing in folder: {{error}}" + "failedToRevealInFolder": "Error revealing in folder: {{error}}", + "previewCompositorUnavailable": "Preview unavailable on this machine" }, "export": { "canceled": "Export canceled", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 09219b0c60..f560a0ac80 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -11,7 +11,8 @@ "exportBackgroundLoadFailed": "La exportación falló: no se pudo cargar la imagen de fondo ({{url}})", "failedToSaveExport": "Error al guardar la exportación", "failedToSaveExportedVideo": "Error al guardar el video exportado", - "failedToRevealInFolder": "Error al mostrar en la carpeta: {{error}}" + "failedToRevealInFolder": "Error al mostrar en la carpeta: {{error}}", + "previewCompositorUnavailable": "Vista previa no disponible en este equipo" }, "export": { "canceled": "Exportación cancelada", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 8e463b6692..26df2114fa 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -17,7 +17,8 @@ "exportBackgroundLoadFailed": "L'export a échoué : impossible de charger l'image d'arrière-plan ({{url}})", "failedToSaveExport": "Échec de l'enregistrement de l'export", "failedToSaveExportedVideo": "Échec de l'enregistrement de la vidéo exportée", - "failedToRevealInFolder": "Erreur lors de l'affichage dans le dossier : {{error}}" + "failedToRevealInFolder": "Erreur lors de l'affichage dans le dossier : {{error}}", + "previewCompositorUnavailable": "Aperçu indisponible sur cette machine" }, "export": { "canceled": "Export annulé", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 0f7fbe7496..945afe3583 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "Esportazione fallita: impossibile caricare l'immagine di sfondo ({{url}})", "failedToSaveExport": "Impossibile salvare l'esportazione", "failedToSaveExportedVideo": "Impossibile salvare il video esportato", - "failedToRevealInFolder": "Errore durante la visualizzazione nella cartella: {{error}}" + "failedToRevealInFolder": "Errore durante la visualizzazione nella cartella: {{error}}", + "previewCompositorUnavailable": "Anteprima non disponibile su questo computer" }, "export": { "canceled": "Esportazione annullata", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 0b50559052..ca90aea7e6 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -19,7 +19,8 @@ "failedToSaveExport": "エクスポートの保存に失敗しました", "failedToSaveExportedVideo": "エクスポートした動画の保存に失敗しました", "failedToRevealInFolder": "フォルダの表示に失敗しました: {{error}}", - "exportBackgroundLoadFailed": "エクスポートに失敗しました: 背景画像を読み込めませんでした ({{url}})" + "exportBackgroundLoadFailed": "エクスポートに失敗しました: 背景画像を読み込めませんでした ({{url}})", + "previewCompositorUnavailable": "このマシンではプレビューを表示できません" }, "export": { "canceled": "エクスポートがキャンセルされました", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 4e97f0736c..4032d3abba 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "내보내기 실패: 배경 이미지를 불러올 수 없습니다 ({{url}})", "failedToSaveExport": "내보낸 파일 저장에 실패했습니다", "failedToSaveExportedVideo": "내보낸 비디오 저장에 실패했습니다", - "failedToRevealInFolder": "폴더에서 파일 표시 오류: {{error}}" + "failedToRevealInFolder": "폴더에서 파일 표시 오류: {{error}}", + "previewCompositorUnavailable": "이 컴퓨터에서는 미리보기를 사용할 수 없습니다" }, "export": { "canceled": "내보내기가 취소되었습니다", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 63e56581fd..571ada2b32 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "Falha na exportação: não foi possível carregar a imagem de fundo ({{url}})", "failedToSaveExport": "Falha ao salvar exportação", "failedToSaveExportedVideo": "Falha ao salvar vídeo exportado", - "failedToRevealInFolder": "Erro ao mostrar na pasta: {{error}}" + "failedToRevealInFolder": "Erro ao mostrar na pasta: {{error}}", + "previewCompositorUnavailable": "Pré-visualização indisponível neste computador" }, "export": { "canceled": "Exportação cancelada", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 6722076484..303c22fd73 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "Экспорт не удался: не удалось загрузить фоновое изображение ({{url}})", "failedToSaveExport": "Не удалось сохранить экспорт", "failedToSaveExportedVideo": "Не удалось сохранить экспортированное видео", - "failedToRevealInFolder": "Ошибка при показе в папке: {{error}}" + "failedToRevealInFolder": "Ошибка при показе в папке: {{error}}", + "previewCompositorUnavailable": "Предпросмотр недоступен на этом компьютере" }, "export": { "canceled": "Экспорт отменён", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index c683d9273a..96de7405b6 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -11,7 +11,8 @@ "exportBackgroundLoadFailed": "Dışa aktarım başarısız: arka plan görüntüsü yüklenemedi ({{url}})", "failedToSaveExport": "Dışa aktarım kaydedilemedi", "failedToSaveExportedVideo": "Dışa aktarılan video kaydedilemedi", - "failedToRevealInFolder": "Klasörde gösterme hatası: {{error}}" + "failedToRevealInFolder": "Klasörde gösterme hatası: {{error}}", + "previewCompositorUnavailable": "Bu makinede önizleme kullanılamıyor" }, "export": { "canceled": "Dışa aktarım iptal edildi", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 34e86cc22f..bc05026c1d 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "Xuất thất bại: không thể tải hình nền ({{url}})", "failedToSaveExport": "Không thể lưu bản xuất", "failedToSaveExportedVideo": "Không thể lưu video đã xuất", - "failedToRevealInFolder": "Lỗi khi hiển thị trong thư mục: {{error}}" + "failedToRevealInFolder": "Lỗi khi hiển thị trong thư mục: {{error}}", + "previewCompositorUnavailable": "Không thể xem trước trên máy này" }, "export": { "canceled": "Đã hủy xuất", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 8afd7be95d..037fced821 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "导出失败：无法加载背景图片（{{url}}）", "failedToSaveExport": "保存导出文件失败", "failedToSaveExportedVideo": "保存导出的视频失败", - "failedToRevealInFolder": "在文件夹中显示时出错：{{error}}" + "failedToRevealInFolder": "在文件夹中显示时出错：{{error}}", + "previewCompositorUnavailable": "此设备无法使用预览" }, "export": { "canceled": "导出已取消", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index 7d4db14ffe..b152666ebc 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -19,7 +19,8 @@ "exportBackgroundLoadFailed": "匯出失敗：無法載入背景圖片（{{url}}）", "failedToSaveExport": "儲存匯出檔案失敗", "failedToSaveExportedVideo": "儲存匯出的影片失敗", - "failedToRevealInFolder": "在資料夾中顯示時出錯：{{error}}" + "failedToRevealInFolder": "在資料夾中顯示時出錯：{{error}}", + "previewCompositorUnavailable": "此裝置無法使用預覽" }, "export": { "canceled": "匯出已取消", diff --git a/src/native/hooks/useNativeCompositorView.test.ts b/src/native/hooks/useNativeCompositorView.test.ts new file mode 100644 index 0000000000..8ef722c501 --- /dev/null +++ b/src/native/hooks/useNativeCompositorView.test.ts @@ -0,0 +1,118 @@ +/** + * The fatal-error channel (PR #162). + * + * `createView` returns an id long before the native render thread can fail, so a host + * that cannot create a D3D11 device used to leave the user with a black canvas and an + * `eprintln!` nobody reads. The addon now reports the dead thread through `readFrame`, + * and this hook turns that into `error`. + * + * The half worth guarding is the negative one: `readFrame` also rejects when there is no + * Electron bridge at all (pure web `npm run dev`, jsdom), and the addon being absent is a + * normal no-op, not a failure. Neither may raise the banner. + */ + +import { renderHook, waitFor } from "@testing-library/react"; +import type { RefObject } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + createCompositorView: vi.fn(), + readCompositorFrame: vi.fn(), + destroyCompositorView: vi.fn(), +})); + +vi.mock("../compositorViewClient", () => ({ + createCompositorView: mocks.createCompositorView, + readCompositorFrame: mocks.readCompositorFrame, + destroyCompositorView: mocks.destroyCompositorView, + setCompositorParam: vi.fn(), + setCompositorPlaying: vi.fn(), + setCompositorRect: vi.fn(), +})); + +import { useNativeCompositorView } from "./useNativeCompositorView"; + +// jsdom ships no ResizeObserver; the hook constructs one to track the canvas box. +// Nothing here observes anything — these tests only exercise the pull loop. +globalThis.ResizeObserver = class { + observe() { + // inert on purpose: the canvas box never changes in these tests + } + unobserve() { + // see observe() + } + disconnect() { + // see observe() + } +} as unknown as typeof ResizeObserver; + +/** A canvas with a stubbed 2D context — jsdom has none, and the pull loop bails without it. */ +function stubCanvasRef(): RefObject { + const canvas = document.createElement("canvas"); + canvas.getContext = vi.fn(() => ({})) as unknown as HTMLCanvasElement["getContext"]; + return { current: canvas }; +} + +const DEVICE_FAILURE = + "this display adapter has no D3D11 video decoder (0x887A0004). OpenScreen decodes every preview and export frame with D3D11VA"; + +describe("useNativeCompositorView", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("surfaces the native message when the render thread dies", async () => { + mocks.createCompositorView.mockResolvedValue({ id: 7 }); + mocks.readCompositorFrame.mockRejectedValue(new Error(DEVICE_FAILURE)); + + const ref = stubCanvasRef(); + const { result } = renderHook(() => + useNativeCompositorView(ref, { sources: { screenPath: "rec.mp4" } }), + ); + + await waitFor(() => expect(result.current.error).toBe(DEVICE_FAILURE)); + }); + + it("stops polling once the error is terminal — the thread never restarts", async () => { + mocks.createCompositorView.mockResolvedValue({ id: 7 }); + mocks.readCompositorFrame.mockRejectedValue(new Error(DEVICE_FAILURE)); + + const ref = stubCanvasRef(); + const { result } = renderHook(() => + useNativeCompositorView(ref, { sources: { screenPath: "rec.mp4" } }), + ); + + await waitFor(() => expect(result.current.error).toBe(DEVICE_FAILURE)); + const callsAtFailure = mocks.readCompositorFrame.mock.calls.length; + await new Promise((resolve) => setTimeout(resolve, 120)); + expect(mocks.readCompositorFrame).toHaveBeenCalledTimes(callsAtFailure); + }); + + it("stays quiet when the addon is absent (synthetic id, no frames, no error)", async () => { + mocks.createCompositorView.mockResolvedValue({ id: -1 }); + mocks.readCompositorFrame.mockResolvedValue(null); + + const ref = stubCanvasRef(); + const { result } = renderHook(() => + useNativeCompositorView(ref, { sources: { screenPath: "rec.mp4" } }), + ); + + await waitFor(() => expect(mocks.readCompositorFrame).toHaveBeenCalled()); + expect(result.current.error).toBeNull(); + }); + + it("stays quiet without an Electron bridge — no view id, so nothing is ever polled", async () => { + mocks.createCompositorView.mockRejectedValue(new Error("Native bridge unavailable.")); + mocks.readCompositorFrame.mockResolvedValue(null); + + const ref = stubCanvasRef(); + const { result } = renderHook(() => + useNativeCompositorView(ref, { sources: { screenPath: "rec.mp4" } }), + ); + + await waitFor(() => expect(mocks.createCompositorView).toHaveBeenCalled()); + await new Promise((resolve) => setTimeout(resolve, 120)); + expect(mocks.readCompositorFrame).not.toHaveBeenCalled(); + expect(result.current.error).toBeNull(); + }); +}); diff --git a/src/native/hooks/useNativeCompositorView.ts b/src/native/hooks/useNativeCompositorView.ts index 1b129b069c..b26c9ed410 100644 --- a/src/native/hooks/useNativeCompositorView.ts +++ b/src/native/hooks/useNativeCompositorView.ts @@ -51,6 +51,18 @@ export interface UseNativeCompositorViewResult { viewId: number | null; setParam: (key: string, value: CompositorParamValue) => void; setPlaying: (playing: boolean) => void; + /** + * Native message from a render thread that died — no D3D11 device on this host, + * a decoder that refused the recording, etc. `null` while things work AND in every + * environment without the native path (pure web `npm run dev`, jsdom): the pull + * loop that produces this only runs once a real `viewId` exists, so an absent + * bridge or addon reads as "no frames", never as an error. + * + * Terminal: the render thread does not restart, so the pull loop stops with it. + * Callers show it instead of the canvas — it is the only thing that distinguishes + * "this machine cannot run the compositor" from a preview that is merely black. + */ + error: string | null; } /** swallow+warn wrapper for native-bridge calls. The renderer can run without @@ -79,6 +91,7 @@ export function useNativeCompositorView( // project) so it never keeps showing a stale clip. const screenPath = opts.sources?.screenPath; const [viewId, setViewId] = useState(null); + const [error, setError] = useState(null); // Mirror into a ref so async callbacks always see the freshest id without // re-subscribing the main effect. const viewIdRef = useRef(null); @@ -98,6 +111,9 @@ export function useNativeCompositorView( let pullTick = 0; let lastRect: CompositorViewRect | null = null; let disposed = false; + // Fresh view (source or enablement changed) → the previous view's fatal error + // says nothing about this one. + setError(null); /** Resize the canvas's DRAWING BUFFER to match the offscreen render * target's pixel dimensions. Setting `canvas.width` / `canvas.height` @@ -233,9 +249,20 @@ export function useNativeCompositorView( // plutôt que moyennés avec des périodes de repos. noteUiProbePreviewFrame(); }) - .catch((error: unknown) => { + .catch((cause: unknown) => { inFlight = false; - console.warn("[compositor-view] readFrame failed:", error); + console.warn("[compositor-view] readFrame failed:", cause); + // Native reports the render thread's fatal error here (see the addon's + // `read_frame`), and it is the ONLY place it can surface: `createView` + // has long since returned an id by the time the thread dies. Stop + // polling — a dead thread never publishes another frame — and hand the + // message up so the user gets it instead of a silent black canvas. + if (disposed) { + return; + } + cancelAnimationFrame(pullRafHandle); + pullRafHandle = 0; + setError(cause instanceof Error ? cause.message : String(cause)); }); }; @@ -311,5 +338,5 @@ export function useNativeCompositorView( safelyCall("setPlaying", () => setCompositorPlaying(id, playing)); }, []); - return { viewId, setParam, setPlaying }; + return { viewId, setParam, setPlaying, error }; } diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index 83802f37da..b8e4047135 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -250,16 +250,45 @@ licence. D3D11VA + AMF survive the LGPL-shared build (verified: ## Known gaps -- **No software/CPU fallback.** `d3d::Gpu::create` +- **No software/CPU fallback, and WARP cannot be one.** `d3d::Gpu::create` ([`crates/compositor/src/d3d.rs`](../../crates/compositor/src/d3d.rs)) requests - `D3D_DRIVER_TYPE_HARDWARE` (no `WARP` / no `REFERENCE`) and pins - `D3D_FEATURE_LEVEL_11_1` — any other feature level fails with `bail!`, - and `VIDEO_SUPPORT` is mandatory for `D3D11VA`. A machine without a - GPU that exposes FL 11_1 with video support will hard-fail at startup; - there is no path that decodes on CPU or falls back to a reference - rasteriser. The compositor is therefore unusable on virtualised - environments that do not pass through a compatible adapter, and there - is no second renderer behind it. + `D3D_DRIVER_TYPE_HARDWARE` and pins `D3D_FEATURE_LEVEL_11_1`, with + `VIDEO_SUPPORT` mandatory for `D3D11VA`. A machine without a GPU that + exposes FL 11_1 with video support hard-fails at startup; there is no + path that decodes on CPU, so the compositor is unusable on virtualised + environments that do not pass through a compatible adapter. + + Retrying in `D3D_DRIVER_TYPE_WARP` — the obvious repair, and what PR #162 + originally scoped — does not work, and the reason is capability, not + speed. Measured (`crates/compositor/tests/warp_device_cannot_decode.rs`, + which fails if this ever stops being true): WARP **rejects the + `VIDEO_SUPPORT` flag outright** (`DXGI_ERROR_UNSUPPORTED`, `0x887A0004`), + and dropping the flag yields a FL 11_1 device that exposes no + `ID3D11VideoDevice` at all (`E_NOINTERFACE`, zero decoder profiles). + Since `pipeline.rs` hands this very device to ffmpeg as the + `AVD3D11VADeviceContext`, preview *and* export decode every frame on it — + a WARP device would produce none. Shipping the retry would only convert a + clear startup failure into an obscure ffmpeg one. So the answer to "is + WARP acceptable for export, or preview only?" is neither. + + What the compositor does instead is **fail legibly**: `Gpu::create` + re-probes on the failure path (same call, minus `VIDEO_SUPPORT`) to tell + "this adapter has no video decoder" — the Remote Desktop / VM case — apart + from "no FL 11_1 adapter at all", and says which, plus what to do about it. + That message reaches the user rather than a log: the render thread stores + its fatal error in `live::Shared`, the addon's `read_frame` returns it as + an `Err` on the next pull (~33 ms), and `NativeCompositorOverlay` renders + it in place of the canvas. Before this, `create_view` had already returned + `Ok` by the time the thread died, so the failure existed only as an + `eprintln!` and the user just saw a black preview. + + The WGC capture helper + ([`electron/native/wgc-capture/src/wgc_session.cpp`](../../electron/native/wgc-capture/src/wgc_session.cpp)) + is deliberately left hardware-only too. It asks for no `VIDEO_SUPPORT`, so + a WARP device *would* be creatable there — but recording on a host whose + compositor cannot start would only produce footage the user can neither + edit nor export. Capture degrading past the editor is worse than capture + failing with it. - **Software VP9 encoding is not supported.** A software VP9 encoder was implemented, measured too slow without a hardware VP9 path on the target GPU, and removed. The export pipeline now offers H.264 (AMF) and From 4775a7ce67b2af6e54545fd3a0b9edb88211fee1 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Mon, 27 Jul 2026 21:50:08 +0200 Subject: [PATCH 02/55] docs(compositor): correct the WGC rationale -- capture is not mandatory The previous paragraph justified leaving the WGC helper hardware-only with "recording on a host whose compositor cannot start only produces footage the user can neither edit nor export". That premise is false: Windows recording already falls through to getDisplayMedia + MediaRecorder when startNativeWindowsRecordingIfAvailable returns false, and the compositor is built to ingest that output -- allow_d3d11va_h264_baseline exists because Chrome's MediaRecorder emits plain H.264 Baseline. The conclusion survives, with a better reason: capture already has a fallback that needs no D3D device at all, so a WARP device there would add a slow path alongside a working one. But the correction exposes a real gap, now documented rather than asserted away: that fallback is only reachable through the pre-flight probe, and is-native-windows-capture-available only checks the Windows build and whether the helper binary is on disk -- it never touches D3D. A host whose helper fails createD3DDevice is told available:true, commits to native, and then startNativeWindowsRecordingIfAvailable rethrows instead of returning false, so the browser path is never reached. Recording fails next to a route that works. Not fixed here: by then the renderer has released the webcam preview stream for the helper's exclusive session, so a bare `return false` would silently record screen-only. A correct fix re-acquires that stream on the fallback route, or makes the probe truthful. --- .../architecture/native-compositor.md | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index b8e4047135..82f91fa663 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -284,11 +284,38 @@ licence. D3D11VA + AMF survive the LGPL-shared build (verified: The WGC capture helper ([`electron/native/wgc-capture/src/wgc_session.cpp`](../../electron/native/wgc-capture/src/wgc_session.cpp)) - is deliberately left hardware-only too. It asks for no `VIDEO_SUPPORT`, so - a WARP device *would* be creatable there — but recording on a host whose - compositor cannot start would only produce footage the user can neither - edit nor export. Capture degrading past the editor is worse than capture - failing with it. + is left hardware-only too, but for a different reason than the compositor: + **WGC capture is not mandatory.** Windows recording already has a non-D3D + path — `startNativeWindowsRecordingIfAvailable` returning `false` falls + through to `getDisplayMedia` + `MediaRecorder` + ([`src/hooks/useScreenRecorder.ts`](../../src/hooks/useScreenRecorder.ts)) — + and the compositor is built to ingest its output: `allow_d3d11va_h264_baseline` + in `pipeline.rs` exists precisely because Chrome's `MediaRecorder` emits plain + H.264 Baseline. So capture has a better fallback available to it than a CPU + rasteriser: one that needs no D3D device at all. Giving the helper a WARP + device would add a slow path nobody needs alongside a working one. + +### Known gap: the capture fallback is unreachable on a host that fails D3D + + That fallback is only reachable through the pre-flight probe, and the probe + does not ask the question that matters. `is-native-windows-capture-available` + ([`electron/ipc/handlers.ts`](../../electron/ipc/handlers.ts)) checks two + things — Windows build ≥ 19041, and the helper binary being on disk. It never + touches D3D. So on a host where the helper's own `createD3DDevice` fails, the + probe answers `available: true`, the renderer commits to the native path, the + helper dies, and `startNativeWindowsRecordingIfAvailable` **rethrows** rather + than returning `false` — so the browser path two calls down its own call site + is never reached. The recording fails next to a route that would have worked. + + Repairing it is not a one-line `return false`. By the time the helper's + failure is known, the renderer has already called `stopWebcamPreviewStream()` + — deliberately, because the helper needs exclusive ownership of the webcam + device before it opens it. Falling through at that point lands in the browser + path's `if (!webcamStream.current)` branch, which disables the camera and + records screen-only: a silent downgrade rather than a failure. A correct fix + either re-acquires the preview stream on the fallback route, or makes the + availability probe truthful by having the helper report its D3D capability + before the renderer commits. Neither belongs in this PR's diff. - **Software VP9 encoding is not supported.** A software VP9 encoder was implemented, measured too slow without a hardware VP9 path on the target GPU, and removed. The export pipeline now offers H.264 (AMF) and From 9ace24e832882758ac2a3d1a3f04388704c81d92 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Mon, 27 Jul 2026 23:10:42 +0200 Subject: [PATCH 03/55] feat(compositor): add a CPU backend -- WARP render + software decode, iso output Rendering and decoding are two independent axes, and no platform has a software rasteriser that also decodes video (WARP here, lavapipe on Linux, nothing at all on macOS). So the CPU fallback needed both halves, not just a driver-type swap: - d3d::Backend::{Hardware, Cpu}. Cpu = D3D_DRIVER_TYPE_WARP without VIDEO_SUPPORT, which WARP rejects outright. - cpu_frames.rs: libavcodec software decode -> swscale to NV12 -> upload into one owned D3D11 texture, presented as an AVFrame whose data[0]/data[1] are the texture and its slice. That last part is the whole design. Everything the compositor knows about a decoded frame is compositor::nv12_srvs and compositor::tex_dims, which read four fields. Fill those and compositor.rs, every HLSL shader and the scene contract stay untouched -- a Metal or Vulkan port replaces d3d.rs and cpu_frames.rs and nothing else. Measured, C1..C8 on the frozen fixture: - iso render: 93-95% of channels bit-identical to the GPU path, max deviation 3/255, matching mean levels (neither frame is blank). Every effect layer survives the swap; the residual is rasteriser/FP difference. - two shaders own the entire speed gap, both multi-tap sampling loops: background blur +4.53 ms on hardware vs +75.43 ms on WARP (17x), motion blur +2.30 vs +52.65 (23x). Everything else is within ~2.2x of the GPU. - so C1-C3 hold ~30 fps (a usable preview) and C4+ sit at 6-9 fps. Benchmarking it needed a preview-shaped harness: the CPU backend cannot reach the export path, since h264_amf requires the real GPU. --preview measures decode -> compose -> readback with no encoder, --backend picks the device, and each run writes a PPM so a backend that composes black cannot post a flattering fps. C0 is excluded from preview mode -- "decode + encode, no composite" has no meaning without an encoder. swscale was already linked but never bound; wrapper.h and the bindgen allowlist now cover it. Preferred over a hand-written UV interleave because it also handles 10-bit and 4:2:2, which a hand-rolled loop would corrupt silently. Nothing selects this backend automatically -- Gpu::create still returns the hardware device. A silent switch to a 6-9 fps preview is the same "the app is slow today" failure the rest of this branch removes. The effect policy C4+ needs is likewise not wired. --- crates/.gitignore | 6 +- crates/compositor/build.rs | 3 + crates/compositor/src/cpu_frames.rs | 241 ++++++++++++++++++ crates/compositor/src/d3d.rs | 85 +++++- crates/compositor/src/lib.rs | 1 + crates/compositor/src/live.rs | 2 + crates/compositor/src/pipeline.rs | 135 +++++++++- .../tests/warp_device_cannot_decode.rs | 25 +- crates/compositor/wrapper.h | 6 + crates/poc-d3d/src/bench.rs | 59 ++++- .../architecture/native-compositor.md | 16 +- .../engineering/rendering-performance.md | 53 ++++ 12 files changed, 586 insertions(+), 46 deletions(-) create mode 100644 crates/compositor/src/cpu_frames.rs diff --git a/crates/.gitignore b/crates/.gitignore index f7e4e976d7..3ecbf6ad43 100644 --- a/crates/.gitignore +++ b/crates/.gitignore @@ -4,8 +4,10 @@ # dépendance téléchargée : build ffmpeg LGPL-shared BtbN (~160 Mo, voir README) /thirdparty -# sorties générées (vidéos C*.mp4, PNG, GIF, dumps) -/out +# sorties générées (vidéos C*.mp4, PNG, PPM, GIF, dumps). Le glob couvre les `--out` +# nommés par backend (out-hw/, out-cpu/) : un PPM de preuve pèse 6 Mo et il en sort un +# par cfg et par backend. +/out* *.raw *.nv12 diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index dd34657962..85b2225c31 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -29,8 +29,11 @@ fn main() { .allowlist_function("avformat_.*") .allowlist_function("avio_.*") .allowlist_function("swr_.*") + .allowlist_function("sws_.*") .allowlist_type("AV.*") .allowlist_type("SwrContext") + .allowlist_type("SwsContext") + .allowlist_var("SWS_.*") .allowlist_var("AV_.*") .allowlist_var("AVERROR.*") .allowlist_var("FF_.*") diff --git a/crates/compositor/src/cpu_frames.rs b/crates/compositor/src/cpu_frames.rs new file mode 100644 index 0000000000..c7a319308f --- /dev/null +++ b/crates/compositor/src/cpu_frames.rs @@ -0,0 +1,241 @@ +//! L'axe DÉCODAGE du backend CPU : une frame libavcodec en mémoire système devient une +//! texture NV12 D3D11, présentée exactement comme si D3D11VA l'avait produite. +//! +//! Pourquoi ce fichier existe séparément : le rendu et le décodage sont deux axes +//! indépendants (voir `d3d::Backend`). WARP couvre le premier et *rien* du second — aucun +//! rastériseur logiciel, sur aucune plateforme, ne décode de la vidéo. Le repli logiciel +//! demandait donc cette pièce-ci en plus, et c'est elle (avec `d3d.rs`) qu'un portage +//! Metal/Vulkan réécrit. `compositor.rs`, les shaders HLSL et le contrat de scène ne +//! bougent pas d'un octet. +//! +//! Le contrat tenu ici est minuscule et c'est ce qui rend le tout iso. Tout ce que le +//! compositeur lit d'une frame, c'est (`compositor::nv12_srvs` / `compositor::tex_dims`) : +//! - `data[0]` : un `ID3D11Texture2D*` NV12, +//! - `data[1]` : l'index de tranche d'array, +//! - `width`/`height` : les dimensions VISIBLES dans cette texture. +//! On remplit ces quatre champs et rien d'autre change. + +use crate::ffi::*; +use anyhow::{bail, Result}; +use std::ptr; +use windows::core::Interface; +use windows::Win32::Graphics::Direct3D11 as d3d11; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +/// Le flag d'algorithme de swscale. Bindgen ne génère pas les `SWS_*` d'algorithme (des +/// macros), et leurs valeurs sont figées par l'ABI de libswscale. `POINT` (plus proche +/// voisin) est le choix honnête : la conversion se fait à dimensions ÉGALES, donc aucun +/// rééchantillonnage n'a lieu — seul le convertisseur de format travaille, et le filtre +/// choisi n'a aucun effet sur la sortie. +const SWS_POINT: i32 = 0x10; + +/// Source de frames du backend CPU, attachée à un `Decoder` quand `Backend::Cpu`. +pub(crate) struct CpuFrames { + dev: d3d11::ID3D11Device, + ctx: d3d11::ID3D11DeviceContext, + sws: *mut SwsContext, + /// `(w, h, format source)` du contexte swscale courant. Un flux qui change de + /// résolution en cours de route (rare mais légal) le reconstruit au lieu de + /// convertir de travers. + sws_key: (i32, i32, i32), + /// NV12 en mémoire système : la cible de swscale, la source de l'upload. + nv12: *mut AVFrame, + /// La texture NV12 échantillonnée par les shaders. UNE seule, réécrite à chaque + /// frame — le `srv_cache` du compositeur (clé `(ptr, slice)`) n'a donc qu'une entrée + /// et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. + // ponytail: une seule texture = le CPU peut attendre que le GPU ait fini de lire la + // frame précédente. Sur WARP tout est CPU et le pilote sérialise déjà ; si un backend + // GPU réutilise ce chemin un jour et que le Map bloque, double-bufferiser ici. + tex: Option, + tex_dims: (u32, u32), + /// La frame remise au compositeur. Ne possède aucun pixel : ses `data[0]`/`data[1]` + /// pointent la texture ci-dessus, exactement comme une frame `AV_PIX_FMT_D3D11`. + present: *mut AVFrame, +} + +impl CpuFrames { + pub(crate) fn new(gpu: &crate::d3d::Gpu) -> Result { + let present = unsafe { av_frame_alloc() }; + let nv12 = unsafe { av_frame_alloc() }; + if present.is_null() || nv12.is_null() { + bail!("av_frame_alloc (backend CPU)"); + } + Ok(CpuFrames { + dev: gpu.device.clone(), + ctx: gpu.context.clone(), + sws: ptr::null_mut(), + sws_key: (0, 0, -1), + nv12, + tex: None, + tex_dims: (0, 0), + present, + }) + } + + /// Convertit `src` (sortie décodeur, mémoire système) en NV12, l'uploade, et rend la + /// frame de présentation. Le pointeur reste valide jusqu'au prochain appel — même + /// contrat que `Decoder::next` côté matériel. + pub(crate) unsafe fn present(&mut self, src: *mut AVFrame) -> Result<*mut AVFrame> { + let (w, h) = ((*src).width, (*src).height); + if w <= 0 || h <= 0 { + bail!("frame décodée sans dimensions ({w}x{h})"); + } + self.ensure_sws(w, h, (*src).format)?; + self.ensure_nv12(w, h)?; + + // Les plans NV12 de destination sont ceux de `self.nv12` : swscale écrit + // directement au bon format, on n'entrelace rien à la main (10 bits, 4:2:2 et + // consorts passent donc aussi, là où une boucle écrite ici casserait en silence). + let converted = sws_scale( + self.sws, + (*src).data.as_ptr() as *const *const u8, + (*src).linesize.as_ptr(), + 0, + h, + (*self.nv12).data.as_mut_ptr(), + (*self.nv12).linesize.as_ptr(), + ); + if converted <= 0 { + bail!("sws_scale a converti {converted} lignes"); + } + + self.upload(w, h)?; + Ok(self.present) + } + + unsafe fn ensure_sws(&mut self, w: i32, h: i32, src_fmt: i32) -> Result<()> { + let key = (w, h, src_fmt); + if self.sws_key == key && !self.sws.is_null() { + return Ok(()); + } + if !self.sws.is_null() { + sws_freeContext(self.sws); + } + self.sws = sws_getContext( + w, + h, + src_fmt as AVPixelFormat::Type, + w, + h, + AVPixelFormat::AV_PIX_FMT_NV12, + SWS_POINT, + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + ); + if self.sws.is_null() { + bail!("sws_getContext {w}x{h} fmt {src_fmt} → NV12"); + } + self.sws_key = key; + Ok(()) + } + + unsafe fn ensure_nv12(&mut self, w: i32, h: i32) -> Result<()> { + if (*self.nv12).width == w + && (*self.nv12).height == h + && (*self.nv12).format == AVPixelFormat::AV_PIX_FMT_NV12 as i32 + { + return Ok(()); + } + av_frame_unref(self.nv12); + (*self.nv12).width = w; + (*self.nv12).height = h; + (*self.nv12).format = AVPixelFormat::AV_PIX_FMT_NV12 as i32; + if av_frame_get_buffer(self.nv12, 32) < 0 { + bail!("av_frame_get_buffer NV12 {w}x{h}"); + } + Ok(()) + } + + /// (Re)crée la texture NV12 si les dimensions ont changé. NV12 impose des dimensions + /// paires : on arrondit AU-DESSUS pour la texture et on laisse `present.width/height` + /// aux dimensions visibles — c'est le même écart texture/visible que produit + /// l'alignement macrobloc de D3D11VA (1080 → 1088), et le compositeur le gère déjà. + unsafe fn ensure_tex(&mut self, w: i32, h: i32) -> Result<()> { + let dims = ((w as u32 + 1) & !1, (h as u32 + 1) & !1); + if self.tex.is_some() && self.tex_dims == dims { + return Ok(()); + } + let desc = d3d11::D3D11_TEXTURE2D_DESC { + Width: dims.0, + Height: dims.1, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: d3d11::D3D11_USAGE_DYNAMIC, + BindFlags: d3d11::D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: d3d11::D3D11_CPU_ACCESS_WRITE.0 as u32, + MiscFlags: 0, + }; + let mut tex: Option = None; + self.dev.CreateTexture2D(&desc, None, Some(&mut tex))?; + self.tex = Some(tex.ok_or_else(|| anyhow::anyhow!("CreateTexture2D NV12 sans texture"))?); + self.tex_dims = dims; + Ok(()) + } + + /// Copie le NV12 système dans la texture. `Map(WRITE_DISCARD)` rend UN pointeur pour + /// les deux plans : Y sur `tex_h` lignes de `RowPitch`, puis UV sur `tex_h/2` lignes + /// au même pitch — c'est la disposition NV12 mappée que documente D3D11. + unsafe fn upload(&mut self, w: i32, h: i32) -> Result<()> { + self.ensure_tex(w, h)?; + let tex = self.tex.clone().expect("texture créée juste au-dessus"); + let resource: d3d11::ID3D11Resource = tex.cast()?; + + let mut mapped = d3d11::D3D11_MAPPED_SUBRESOURCE::default(); + self.ctx.Map(&resource, 0, d3d11::D3D11_MAP_WRITE_DISCARD, 0, Some(&mut mapped))?; + + let dst = mapped.pData as *mut u8; + let pitch = mapped.RowPitch as usize; + let (tex_w, tex_h) = (self.tex_dims.0 as usize, self.tex_dims.1 as usize); + let src_y = (*self.nv12).data[0]; + let src_uv = (*self.nv12).data[1]; + let sp_y = (*self.nv12).linesize[0] as usize; + let sp_uv = (*self.nv12).linesize[1] as usize; + // Ne copier que ce qui existe des deux côtés : la texture est arrondie au pair et + // les lignes de swscale sont paddées à leur propre alignement SIMD. + let row = tex_w.min(sp_y).min(pitch); + for y in 0..tex_h.min(h as usize) { + ptr::copy_nonoverlapping(src_y.add(y * sp_y), dst.add(y * pitch), row); + } + let uv_base = dst.add(pitch * tex_h); + let uv_row = tex_w.min(sp_uv).min(pitch); + for y in 0..(tex_h / 2).min((h as usize).div_ceil(2)) { + ptr::copy_nonoverlapping(src_uv.add(y * sp_uv), uv_base.add(y * pitch), uv_row); + } + + self.ctx.Unmap(&resource, 0); + + // Le contrat que lit le compositeur, et rien de plus : texture, tranche, visible. + // `data` n'est adossé à aucun `buf[]`, donc `av_frame_free` ne libérera jamais la + // texture — c'est nous qui la possédons, via `self.tex`. + (*self.present).data[0] = tex.as_raw() as *mut u8; + (*self.present).data[1] = ptr::null_mut(); // tranche 0 : notre texture n'est pas un array + (*self.present).width = w; + (*self.present).height = h; + (*self.present).format = AVPixelFormat::AV_PIX_FMT_D3D11 as i32; + Ok(()) + } + + /// La frame de présentation courante (jamais nulle) — `Decoder::cur_frame` en backend CPU. + pub(crate) fn current(&self) -> *mut AVFrame { + self.present + } +} + +impl Drop for CpuFrames { + fn drop(&mut self) { + unsafe { + // `present` n'a que des pointeurs empruntés : les remettre à zéro avant de + // libérer, pour qu'aucun code ffmpeg ne croie posséder notre texture. + (*self.present).data[0] = ptr::null_mut(); + (*self.present).data[1] = ptr::null_mut(); + av_frame_free(&mut self.present); + av_frame_free(&mut self.nv12); + if !self.sws.is_null() { + sws_freeContext(self.sws); + } + } + } +} diff --git a/crates/compositor/src/d3d.rs b/crates/compositor/src/d3d.rs index b5a0c05e4b..09c4a3b1a6 100644 --- a/crates/compositor/src/d3d.rs +++ b/crates/compositor/src/d3d.rs @@ -7,7 +7,8 @@ use anyhow::{bail, Result}; use windows::core::Interface; use windows::Win32::Foundation::HMODULE; use windows::Win32::Graphics::Direct3D::{ - D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_1, + D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP, D3D_FEATURE_LEVEL, + D3D_FEATURE_LEVEL_11_1, }; use windows::Win32::Graphics::Direct3D11::{ D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Multithread, @@ -15,10 +16,52 @@ use windows::Win32::Graphics::Direct3D11::{ D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, }; +/// Qui exécute le pipeline. Le rendu et le décodage sont DEUX axes distincts, et aucune +/// plateforme n'a de rastériseur logiciel qui décode aussi la vidéo (WARP ici, lavapipe +/// sous Linux, rien du tout sous macOS) — un backend fixe donc les deux ensemble. +/// +/// Le contrat de scène, les shaders HLSL et tout `compositor.rs` sont identiques d'un +/// backend à l'autre : c'est tout l'intérêt. Un portage Metal/Vulkan remplace ce que fait +/// ce fichier et `Decoder`, pas le moteur. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Backend { + /// GPU : rastérisation matérielle + décodage D3D11VA sur le même device (zéro copie). + /// Le seul backend qui puisse exporter — l'encodeur AMF exige lui aussi le vrai GPU. + Hardware, + /// CPU : rastérisation WARP + décodage logiciel libavcodec, uploadé en NV12. + /// Pour les hôtes sans GPU D3D11 utilisable (VM, RDP, Basic Render Driver). + Cpu, +} + +impl Backend { + fn driver(self) -> D3D_DRIVER_TYPE { + match self { + Backend::Hardware => D3D_DRIVER_TYPE_HARDWARE, + Backend::Cpu => D3D_DRIVER_TYPE_WARP, + } + } + + /// WARP REFUSE `VIDEO_SUPPORT` (`DXGI_ERROR_UNSUPPORTED`, mesuré dans + /// `tests/warp_device_cannot_decode.rs`) : ce flag n'a de sens que sur le device + /// matériel, où il conditionne D3D11VA. Le backend CPU ne décode pas sur le GPU, + /// il n'en a donc pas besoin. + fn base_flags(self) -> D3D11_CREATE_DEVICE_FLAG { + match self { + Backend::Hardware => { + D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT + } + Backend::Cpu => D3D11_CREATE_DEVICE_BGRA_SUPPORT, + } + } +} + pub struct Gpu { pub device: ID3D11Device, pub context: ID3D11DeviceContext, pub feature_level: D3D_FEATURE_LEVEL, + /// Lu par `Decoder::open` pour choisir D3D11VA ou le décodage logiciel. Porté par le + /// `Gpu` plutôt que passé partout : tout ce qui tient un device sait déjà qui il est. + pub backend: Backend, } /// Une tentative `D3D11CreateDevice` à FL 11_1. Extraite pour que le chemin d'échec @@ -55,14 +98,17 @@ fn try_create( /// Message d'échec ACTIONNABLE : re-sonde pour distinguer les deux causes réelles. /// -/// PR #162 proposait de retomber sur `D3D_DRIVER_TYPE_WARP`. Mesuré sur cette machine -/// (`tests/warp_device_cannot_decode.rs`, qui échouera si Windows change d'avis) : -/// WARP + `VIDEO_SUPPORT` ne se crée même pas (`DXGI_ERROR_UNSUPPORTED`), et WARP sans -/// ce flag n'expose aucun `ID3D11VideoDevice` (`E_NOINTERFACE`, 0 profil décodeur). -/// Or `pipeline.rs` passe CE device à ffmpeg comme `AVD3D11VADeviceContext` : preview -/// comme export décodent chaque frame en D3D11VA. Un device WARP produirait donc zéro -/// frame et transformerait cet échec net en panne ffmpeg obscure. D'où : pas de repli, -/// mais un échec qui se lit. +/// Ce message reste utile MÊME avec `Backend::Cpu` disponible : rien ne bascule tout +/// seul (voir `create`), donc quelqu'un doit savoir pourquoi le matériel a refusé et si +/// le repli logiciel est la bonne réponse à son cas. +/// +/// PR #162 proposait de retomber sur `D3D_DRIVER_TYPE_WARP` en gardant tout le reste. +/// Mesuré (`tests/warp_device_cannot_decode.rs`) : WARP + `VIDEO_SUPPORT` ne se crée même +/// pas (`DXGI_ERROR_UNSUPPORTED`), et sans ce flag il n'expose aucun `ID3D11VideoDevice` +/// (`E_NOINTERFACE`, 0 profil décodeur). Comme `pipeline.rs` passe CE device à ffmpeg +/// comme `AVD3D11VADeviceContext`, un simple changement de driver type aurait produit zéro +/// frame. C'est ce qui a donné à `Backend::Cpu` sa forme : WARP pour le rendu PLUS un +/// décodage logiciel (`cpu_frames.rs`) — le rendu et le décodage sont deux axes. fn diagnose(err: &windows::core::Error) -> String { // Le décodeur est le point de rupture le plus probable (RDP, VM sans passthrough, // Microsoft Basic Render Driver) : si l'appel passe SANS VIDEO_SUPPORT, l'adaptateur @@ -90,16 +136,27 @@ fn diagnose(err: &windows::core::Error) -> String { impl Gpu { /// Crée le device conforme au §2. `debug=false` impératif dans tout run mesuré /// (§10 : la couche debug valide et sérialise chaque appel — facteur, pas %). + /// + /// Reste le device MATÉRIEL : le backend CPU ne s'obtient qu'en le demandant + /// (`create_backend`). Rien ne bascule tout seul — un repli silencieux vers un + /// rendu logiciel serait précisément le « l'app rame aujourd'hui » qu'on veut éviter. pub fn create(debug: bool) -> Result { - // VIDEO_SUPPORT : requis pour que D3D11VA décode sur CE device. - // BGRA_SUPPORT : utile (interop D2D éventuelle) et sans coût. - let mut flags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT; + Gpu::create_backend(Backend::Hardware, debug) + } + + /// Le device du backend demandé. `Backend::Cpu` ne diagnostique pas : si WARP + /// lui-même échoue, il n'y a plus rien derrière à proposer. + pub fn create_backend(backend: Backend, debug: bool) -> Result { + let mut flags = backend.base_flags(); if debug { flags |= D3D11_CREATE_DEVICE_DEBUG; } - let (device, context, got) = match try_create(D3D_DRIVER_TYPE_HARDWARE, flags) { + let (device, context, got) = match try_create(backend.driver(), flags) { Ok(gpu) => gpu, + Err(err) if backend == Backend::Cpu => { + bail!("WARP (rastériseur logiciel) indisponible sur cet hôte : {err}") + } Err(err) => bail!("{}", diagnose(&err)), }; @@ -117,6 +174,6 @@ impl Gpu { } } - Ok(Gpu { device, context, feature_level: got }) + Ok(Gpu { device, context, feature_level: got, backend }) } } diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index c1d01bb342..b509f6dcda 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio; pub mod compositor; pub mod config; +mod cpu_frames; pub mod cursor; pub mod d3d; pub mod ffi; diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 1fb1f1ce70..76323ce2e0 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -126,6 +126,7 @@ impl Player { device: gpu.device.clone(), context: gpu.context.clone(), feature_level: gpu.feature_level, + backend: gpu.backend, }, webcam_offset_sec: 0.0, has_current_frame: false, @@ -871,6 +872,7 @@ unsafe fn maybe_start_prefetch( device: gpu.device.clone(), context: gpu.context.clone(), feature_level: gpu.feature_level, + backend: gpu.backend, }; let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { diff --git a/crates/compositor/src/pipeline.rs b/crates/compositor/src/pipeline.rs index f2361da4a2..0359cd2a04 100644 --- a/crates/compositor/src/pipeline.rs +++ b/crates/compositor/src/pipeline.rs @@ -8,8 +8,9 @@ use crate::audio::{ }; use crate::compositor::{Compositor, OUT_H, OUT_W}; use crate::config::Cfg; +use crate::cpu_frames::CpuFrames; use crate::cursor::CursorTrack; -use crate::d3d::Gpu; +use crate::d3d::{Backend, Gpu}; use crate::ffi::*; use crate::regions::{speed_segments_for_window, SpeedSegment}; use crate::scene::Scene; @@ -389,6 +390,77 @@ unsafe fn run_c0_inner(screen: &str, out: &str, gpu: &Gpu) -> Result { Ok(Stats { frames, wall_s, fps, video_duration_s: frames as f64 / 60.0 }) } +/// Boucle de PREVIEW mesurée : décode → compose → readback, sans encodeur. +/// +/// Pourquoi pas `run_composited` : celui-ci encode en h264_amf, qui exige le vrai GPU. +/// Le backend CPU ne peut donc pas le traverser, et les deux backends ne seraient pas +/// comparables. L'encodage est de toute façon un TROISIÈME axe (comme le rendu et le +/// décodage) et il n'a pas de repli logiciel ici — ce qui fait de la preview la seule +/// surface que le backend CPU vise réellement. C'est exactement ce que cette boucle mesure, +/// et c'est la même séquence que le thread de rendu de `live.rs`. +/// +/// `frames` = nombre de frames composées ; la source boucle (`seek_to(0)`) si elle est +/// plus courte, pour que les deux backends voient exactement la même charge. +/// +/// Rend aussi le DERNIER readback (`w`, `h`, RGBA8) : un backend qui compose du noir +/// serait rapide et parfaitement inutile, donc le chiffre ne veut rien dire sans l'image +/// qui va avec. C'est ce qui permet de comparer pixel à pixel les deux backends. +pub fn run_preview_bench( + screen: &str, + webcam: &str, + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, + frames: u64, +) -> Result<(Stats, (u32, u32, Vec))> { + unsafe { + let mut sdec = Decoder::open(screen, gpu)?; + let mut wdec = Decoder::open(webcam, gpu)?; + + // Hors mesure : première frame de chaque source. Le premier décodage porte + // l'allocation du pool (matériel) ou de la texture NV12 + du contexte swscale + // (CPU) ; le compter fausserait surtout les runs courts. + let mut sf = sdec.next()?; + let mut wf = wdec.next()?; + if sf.is_null() || wf.is_null() { + bail!("source vide (screen ou webcam ne rend aucune frame)"); + } + comp.compose_frame(sf, wf, 0.0, cfg)?; + let _ = comp.readback_direct()?; + + let mut last = (0u32, 0u32, Vec::new()); + let t0 = Instant::now(); + for i in 0..frames { + sf = sdec.next()?; + if sf.is_null() { + sf = sdec.seek_to(0.0)?; + } + wf = wdec.next()?; + if wf.is_null() { + wf = wdec.seek_to(0.0)?; + } + if sf.is_null() || wf.is_null() { + bail!("source épuisée après rembobinage à la frame {i}"); + } + comp.compose_frame(sf, wf, i as f32, cfg)?; + // Le readback fait partie de la mesure : c'est ce que la preview paie + // réellement pour afficher une frame (GPU→CPU puis canvas). + last = comp.readback_direct()?; + } + let wall_s = t0.elapsed().as_secs_f64(); + + Ok(( + Stats { + frames, + wall_s, + fps: frames as f64 / wall_s, + video_duration_s: frames as f64 / 60.0, + }, + last, + )) + } +} + /// Décodeur qui rend une frame à la fois (pour composer 2 sources en lockstep). /// `pub(crate)` : réutilisé par la preview/playback (voir `app.rs`). pub(crate) struct Decoder { @@ -404,6 +476,10 @@ pub(crate) struct Decoder { /// impossible de savoir si `frame` contient quoi que ce soit d'exploitable — un /// `AVFrame` fraîchement alloué a un `best_effort_timestamp` indéterminé. cur_pts: Option, + /// Backend CPU uniquement : convertit la frame système en texture NV12 et la présente + /// sous le même contrat que D3D11VA (voir `cpu_frames`). `None` en matériel — le + /// décodeur rend alors directement la texture du pool D3D11VA, sans copie. + cpu: Option, } // SAFETY: `Decoder` only owns FFI pointers into FFmpeg's own heap-allocated state, which @@ -433,15 +509,34 @@ impl Decoder { averr(avcodec_parameters_to_context(dctx, codecpar), "params_to_ctx")?; allow_d3d11va_h264_baseline(dctx); - let hwdev = av_hwdevice_ctx_alloc(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA); - let hwdc = (*hwdev).data as *mut AVHWDeviceContext; - let d3dctx = (*hwdc).hwctx as *mut AVD3D11VADeviceContext; - let dev_clone = gpu.device.clone(); - (*d3dctx).device = dev_clone.as_raw() as *mut ID3D11Device; - std::mem::forget(dev_clone); - averr(av_hwdevice_ctx_init(hwdev), "hwdevice_ctx_init")?; - (*dctx).hw_device_ctx = av_buffer_ref(hwdev); - (*dctx).get_format = Some(get_hw_format); + // Backend CPU : on n'attache AUCUN hw_device_ctx et on ne force pas `get_format`, + // donc libavcodec choisit son décodeur logiciel et sort en mémoire système. Passer + // le device WARP à D3D11VA ne marcherait pas de toute façon — WARP n'expose pas + // d'`ID3D11VideoDevice` (`tests/warp_device_cannot_decode.rs`). + let cpu = if gpu.backend == Backend::Cpu { + // `threads = 0` : libavcodec prend le nombre de cœurs. C'est le seul réglage + // qui compte vraiment ici — sans lui le décodage logiciel est mono-thread et + // le benchmark mesurerait surtout ça. + (*dctx).thread_count = 0; + Some(CpuFrames::new(gpu)?) + } else { + None + }; + + let hwdev = if cpu.is_some() { + ptr::null_mut() + } else { + let hwdev = av_hwdevice_ctx_alloc(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA); + let hwdc = (*hwdev).data as *mut AVHWDeviceContext; + let d3dctx = (*hwdc).hwctx as *mut AVD3D11VADeviceContext; + let dev_clone = gpu.device.clone(); + (*d3dctx).device = dev_clone.as_raw() as *mut ID3D11Device; + std::mem::forget(dev_clone); + averr(av_hwdevice_ctx_init(hwdev), "hwdevice_ctx_init")?; + (*dctx).hw_device_ctx = av_buffer_ref(hwdev); + (*dctx).get_format = Some(get_hw_format); + hwdev + }; averr(avcodec_open2(dctx, dec, ptr::null_mut()), "avcodec_open2")?; Ok(Decoder { @@ -453,13 +548,22 @@ impl Decoder { frame: av_frame_alloc(), sent_eof: false, cur_pts: None, + cpu, }) } /// Dernière frame décodée (valide jusqu'au prochain `next`) — pour recomposer /// la frame courante après un changement de config, sans réavancer (preview). + /// + /// En backend CPU c'est la frame de PRÉSENTATION (la texture NV12 uploadée), pas la + /// frame système du décodeur : `cur_frame` alimente `compose_frame` au même titre que + /// `next`, donc les deux doivent rendre la même chose. Le temps (`cur_time_sec`), lui, + /// continue de se lire sur la vraie frame décodée. pub(crate) fn cur_frame(&self) -> *mut AVFrame { - self.frame + match &self.cpu { + Some(cpu) => cpu.current(), + None => self.frame, + } } /// Repositionne le flux à la première keyframe (t=0) et vide le codec — pour boucler @@ -500,8 +604,10 @@ impl Decoder { let cur = pts as f64 * tb_sec; let frame_dur = 1.0 / self.fps().max(1.0); // 1) La frame courante EST celle demandée : rien à décoder du tout. + // `cur_frame()`, pas `self.frame` : en backend CPU la frame exploitable + // est la texture NV12 déjà présentée, pas la frame système du décodeur. if (cur - seconds).abs() < frame_dur * 0.5 { - return Ok(self.frame); + return Ok(self.cur_frame()); } // 2) La cible est DEVANT et à portée : dérouler depuis ici. Au-delà du seuil, // repartir d'une image clé redevient moins cher — un seek coûte en moyenne @@ -593,7 +699,10 @@ impl Decoder { if r == 0 { let pts = (*self.frame).best_effort_timestamp; self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; - return Ok(self.frame); + return match &mut self.cpu { + Some(cpu) => cpu.present(self.frame), + None => Ok(self.frame), + }; } if r == AVERROR_EOF { return Ok(ptr::null_mut()); diff --git a/crates/compositor/tests/warp_device_cannot_decode.rs b/crates/compositor/tests/warp_device_cannot_decode.rs index 382b2e3ab6..d9cbad14cc 100644 --- a/crates/compositor/tests/warp_device_cannot_decode.rs +++ b/crates/compositor/tests/warp_device_cannot_decode.rs @@ -1,16 +1,19 @@ -//! Pourquoi il n'y a PAS de repli WARP (PR #162). +//! Pourquoi le backend CPU décode en LOGICIEL et pas sur le device WARP (PR #162). //! -//! `d3d::Gpu::create` demande `D3D_DRIVER_TYPE_HARDWARE` et `VIDEO_SUPPORT`, sans -//! second choix. La proposition évidente — retenter en `D3D_DRIVER_TYPE_WARP`, le -//! rastériseur CPU de Microsoft — ne marche pas, et pas pour une raison de vitesse : -//! WARP n'a tout simplement pas de décodeur vidéo. Or `pipeline.rs` passe le device -//! de `Gpu` à ffmpeg comme `AVD3D11VADeviceContext` (`(*d3dctx).device = ...`), donc -//! preview ET export décodent chaque frame en D3D11VA sur CE device. Un device WARP -//! se créerait puis ne produirait aucune frame. +//! La proposition initiale était de simplement retenter `D3D11CreateDevice` en +//! `D3D_DRIVER_TYPE_WARP` quand le matériel échoue. Ça ne suffit pas, et pas pour une +//! raison de vitesse : WARP n'a pas de décodeur vidéo du tout. Or le chemin matériel +//! passe le device de `Gpu` à ffmpeg comme `AVD3D11VADeviceContext` +//! (`pipeline.rs`, `(*d3dctx).device = ...`) — un device WARP branché là se créerait +//! puis ne produirait aucune frame. //! -//! Ce test verrouille la mesure. S'il ÉCHOUE, c'est que WARP a gagné une capacité -//! vidéo sur cette machine/version de Windows — et alors le repli redevient une -//! question ouverte : relire `diagnose()` dans `src/d3d.rs`. +//! D'où la forme qu'a prise `Backend::Cpu` : WARP pour le RENDU, libavcodec en mémoire +//! système pour le DÉCODAGE, uploadé en NV12 par `cpu_frames.rs`. Deux axes, deux +//! solutions — c'est ce test qui dit pourquoi le second existe. +//! +//! S'il ÉCHOUE, c'est que WARP a gagné une capacité vidéo sur cette machine/version de +//! Windows : `Backend::Cpu` pourrait alors décoder directement sur son device et se +//! passer de tout `cpu_frames.rs`. #![cfg(windows)] diff --git a/crates/compositor/wrapper.h b/crates/compositor/wrapper.h index 88fb4be894..86612f96ae 100644 --- a/crates/compositor/wrapper.h +++ b/crates/compositor/wrapper.h @@ -6,3 +6,9 @@ #include #include #include +/* Software decode path (CPU backend) : les décodeurs logiciels sortent du YUV420P, + la chaîne D3D échantillonne du NV12. swscale était déjà LIÉ (build.rs) sans être + bindé — c'est la conversion la mieux optimisée qu'on ait déjà sous la main, et + elle couvre les formats exotiques (10 bits, 4:2:2) qu'un interleave écrit à la + main casserait silencieusement. */ +#include diff --git a/crates/poc-d3d/src/bench.rs b/crates/poc-d3d/src/bench.rs index c29d5d6c8f..2c5fc9c9ab 100644 --- a/crates/poc-d3d/src/bench.rs +++ b/crates/poc-d3d/src/bench.rs @@ -67,7 +67,7 @@ fn run_bench(args: &[String]) -> Result<()> { // sélection des cfg let all = config::all(); - let cfgs: Vec = if cfg_arg.contains("..") { + let mut cfgs: Vec = if cfg_arg.contains("..") { all } else { cfg_arg @@ -76,8 +76,36 @@ fn run_bench(args: &[String]) -> Result<()> { .collect() }; - let gpu = d3d::Gpu::create(false)?; - println!("d3d11 device ok (feature_level 0x{:X})", gpu.feature_level.0 as u32); + // `--backend cpu` : rastérisation WARP + décodage logiciel (voir d3d::Backend). + // Il n'encode pas (AMF exige le GPU), donc il n'est mesurable qu'en mode preview — + // `--preview` est imposé plus bas plutôt que de laisser le run échouer sur l'encodeur. + let backend = match get("--backend", "hardware").as_str() { + "cpu" | "warp" => d3d::Backend::Cpu, + "hardware" | "gpu" => d3d::Backend::Hardware, + other => anyhow::bail!("--backend {other} inconnu (hardware|cpu)"), + }; + let preview_only = args.iter().any(|a| a == "--preview") || backend == d3d::Backend::Cpu; + // Frames composées par run en mode preview. Assez pour noyer le bruit, assez court + // pour qu'un backend lent reste mesurable en une poignée de minutes. + let preview_frames: u64 = get("--frames", "300").parse().unwrap_or(300); + + // C0 = « décode + encode, aucun composite ». Sans encodeur, il n'a pas d'équivalent : + // le mesurer en preview reviendrait à composer quand même et à publier un C0 qui est + // en fait un C1. On le retire plutôt que d'imprimer une ligne trompeuse. + if preview_only { + cfgs.retain(|c| c.composite); + if cfgs.is_empty() { + anyhow::bail!("aucune cfg composite à mesurer (C0 n'a pas de sens sans encodeur)"); + } + } + + let gpu = d3d::Gpu::create_backend(backend, false)?; + println!( + "d3d11 device ok — backend {:?}, feature_level 0x{:X}{}", + backend, + gpu.feature_level.0 as u32, + if preview_only { ", mode preview (décode+compose+readback, sans encodeur)" } else { "" } + ); let mut comp = Compositor::new(&gpu)?; let track = cursor::CursorTrack::load(&format!("{fixture}/screen.cursor.json"), 100_000.0, 6.0)?; comp.set_cursor(track); @@ -102,14 +130,26 @@ fn run_bench(args: &[String]) -> Result<()> { let mut frames = 0u64; for r in 0..repeat { let path = format!("{out}/{}.mp4", cfg.name); - let s = if cfg.composite { + let s = if preview_only { + let (stats, (fw, fh, rgba)) = + pipeline::run_preview_bench(&screen, &webcam, &gpu, &comp, cfg, preview_frames)?; + // Preuve visuelle, et surtout comparable : un backend qui compose du noir + // afficherait un fps flatteur. Le PPM est nommé par backend pour qu'un + // diff hardware/cpu soit direct. + if r == 0 { + let name = format!("{out}/{}_{:?}.ppm", cfg.name, backend).to_lowercase(); + write_ppm(&name, fw, fh, &rgba)?; + } + stats + } else if cfg.composite { pipeline::run_composited(&screen, &webcam, &path, &gpu, &comp, cfg, &mut |_| {})? } else { pipeline::run_c0(&screen, &path, &gpu)? }; frames = s.frames; fps_runs.push(s.fps); - if r == 0 { + // Pas de MP4 produit en mode preview (aucun encodeur) — rien à extraire. + if r == 0 && !preview_only { // extraction PNG f60/f180/f300 sur le 1er run (§11) extract_pngs(&path, &out, cfg.name); } @@ -273,6 +313,15 @@ fn run_gif_bench( out_path = out_path.display(), ); println!("\nreport-gif.json + out/gif.gif écrits dans {out}/"); +/// Écrit un readback RGBA8 en PPM binaire (P6, RGB) — format le plus bête qui se lise +/// partout, et qui se compare octet à octet entre deux backends sans passer par un codec. +fn write_ppm(path: &str, w: u32, h: u32, rgba: &[u8]) -> Result<()> { + let mut buf = format!("P6\n{w} {h}\n255\n").into_bytes(); + buf.reserve(rgba.len() / 4 * 3); + for px in rgba.chunks_exact(4) { + buf.extend_from_slice(&px[..3]); + } + std::fs::write(path, buf).with_context(|| format!("écriture {path}"))?; Ok(()) } diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index 82f91fa663..f3619a660d 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -250,7 +250,21 @@ licence. D3D11VA + AMF survive the LGPL-shared build (verified: ## Known gaps -- **No software/CPU fallback, and WARP cannot be one.** `d3d::Gpu::create` +- **The CPU backend exists, but nothing selects it automatically yet.** + `d3d::Backend::Cpu` runs the same pipeline on a WARP device with libavcodec + software decode ([`cpu_frames.rs`](../../crates/compositor/src/cpu_frames.rs)), + and renders **iso** with the GPU path (max deviation 3/255 across C1–C8 — + see [rendering-performance.md](../engineering/rendering-performance.md#the-cpu-backend-warp--software-decode--2026-07-27)). + `Gpu::create` still returns the hardware device and never falls back on its + own: a silent switch to a 6–9 fps preview is exactly the "the app is slow + today" failure this whole thread set out to remove. Selecting it, and the + effect policy it needs (background blur and motion blur cost 17× and 23× on + WARP and are what make C4+ unusable), is not wired. + **Export cannot use it at all**: `h264_amf` requires the real GPU, and encode + is a third axis with no software path — though the vendored LGPL build does + ship `libopenh264` and `h264_mf`, so that is a wiring gap rather than a wall. + +- **The hardware path has no automatic fallback, and WARP alone could not be one.** `d3d::Gpu::create` ([`crates/compositor/src/d3d.rs`](../../crates/compositor/src/d3d.rs)) requests `D3D_DRIVER_TYPE_HARDWARE` and pins `D3D_FEATURE_LEVEL_11_1`, with `VIDEO_SUPPORT` mandatory for `D3D11VA`. A machine without a GPU that diff --git a/technical-documentation/engineering/rendering-performance.md b/technical-documentation/engineering/rendering-performance.md index 37d4443258..a361e27084 100644 --- a/technical-documentation/engineering/rendering-performance.md +++ b/technical-documentation/engineering/rendering-performance.md @@ -83,6 +83,59 @@ C8's 104.0 fps sits under the ~126 headline above. Different session and thermal > A first attempt the same day was **VOID** and is not reported: five of nine configs blew the spread gate (up to 42.5 %) while ~40 browser and Electron processes were live, and C3 came out **+15.8 fps faster than C2** — adding a layer. Cumulative configs cannot speed up; that is the tell that noise had swamped the signal. It is recorded here only because it is a clean example of why the gate exists. +### The CPU backend (WARP + software decode) — 2026-07-27 + +`d3d::Backend::Cpu` runs the *same* pipeline on a WARP device with libavcodec software +decode ([`cpu_frames.rs`](../../crates/compositor/src/cpu_frames.rs)), for hosts with no +usable D3D11 GPU. Rendering and decoding are two independent axes — WARP covers the +first and *nothing* of the second, on any platform — so the fallback needed both halves. + +Measured with a preview-shaped workload, because the CPU backend cannot reach the export +path at all: `h264_amf` requires the real GPU, and encode is a **third** axis with no +software fallback wired today. So the harness grew `--preview` (decode → compose → +readback, no encoder) and `--backend`, which is what makes the two comparable: + +```bash +x.bat run --release -- --cfg C1..C8 --backend cpu --preview --frames 300 --repeat 3 +``` + +C0 is excluded: it is "decode + encode, no composite", which has no meaning without an +encoder. + +| cfg | HW fps | HW ms/f | CPU fps | CPU ms/f | Δ ms/f (CPU) | gap | +|---|---:|---:|---:|---:|---:|---:| +| C1 | 65.3 | 15.30 | 30.7 | 32.60 | — | 2.1× | +| C2 | 65.2 | 15.34 | 30.1 | 33.25 | +0.65 | 2.2× | +| C3 | 63.1 | 15.84 | 28.5 | 35.06 | +1.81 | 2.2× | +| C4 | 49.1 | 20.37 | 9.1 | 110.49 | **+75.43** | 5.4× | +| C5 | 51.2 | 19.54 | 7.9 | 126.14 | +15.65 | 6.5× | +| C6 | 54.3 | 18.42 | 9.5 | 105.78 | −20.36 | 5.7× | +| C7 | 53.9 | 18.55 | 9.2 | 108.53 | +2.75 | 5.9× | +| C8 | 48.0 | 20.85 | 6.2 | 161.18 | **+52.65** | 7.7× | + +**Two shaders account for the whole gap, and both are multi-tap sampling loops.** +Background blur costs +4.53 ms on hardware and **+75.43 ms** on WARP (17×); motion blur +costs +2.30 ms and **+52.65 ms** (23×). Everything else — compositing, SDF rounded +corners, drop shadows, zoom, layout animation, cursor — runs within ~2.2× of the GPU. +So WARP is not uniformly slow: it is fine at single-pass geometry and collapses on +per-pixel sampling loops. C1–C3 at ~30 fps is a usable editing preview; C4 onward, at +6–9 fps, is not. + +**The render is iso**, which is the property that makes a backend swap worth having at +all. Comparing the same fixture frame per config, full-image over all 6 220 800 channels: +93–95 % of channels bit-identical, **max deviation 3/255**, nothing above 2 outside a +handful of pixels, and matching mean levels (217.77 vs 217.81 — neither frame is blank). +Every effect layer survives the swap. The residual is rasteriser/FP difference, so any +pixel-golden test that spans backends needs a tolerance rather than an exact baseline. + +> **Not an admissible run under [§ spread thresholds](#spread-thresholds).** Four of +> sixteen rows blow the 15 % gate (CPU C1 21.3 %, C3 37.7 %, C8 25.5 %; HW C7 18.8 %) — +> ~14 browser processes were live. The **layer attribution** is what this run claims and +> it is robust: both cliffs are 3–5× drops bracketed by rows at 3.7 %/10.5 % and +> 4.9 %/6.5 % spread. The **absolutes are not quotable** until a re-run on a quiet +> machine. C6's negative delta is the same C5–C7 plateau noise the +> [admissible hardware run](#one-admissible-run--2026-07-27) documents. + ## How we got here — the WebCodecs trail > **This section is history.** It records the measurements that killed the browser-based export pipeline and motivated the native one. The code it describes is **gone**: `src/lib/exporter/videoExporter.ts`, `src/bench/runBench.ts` and the `npm run bench:export` script were deleted with the web MP4 pipeline. It is kept because it is the evidence for [why the compositor, not the encoder, was the wall](#the-wall-is-the-compositor) — which is the entire reason `crates/compositor/` exists — and because the [measurement hazards](#measurement-hazards) it uncovered still apply to any new benchmark here. From 0554ebb440477025c6b253f4d3d1ebeaec3b98d5 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Mon, 27 Jul 2026 23:55:48 +0200 Subject: [PATCH 04/55] feat(compositor): select the CPU backend automatically, and tell the user Everything for the fallback existed; nothing chose it. Gpu::create_auto now prefers the hardware device and falls back to Backend::Cpu, and the production paths (live render thread, napi export) use it. Gpu::create stays hardware-strict for tests and goldens, where measuring the GPU path on a software rasteriser would be meaningless. The switch is never silent, which was the whole objection to an automatic fallback: - create_auto logs why the hardware device was refused, via diagnose() -- "this adapter has no video decoder" (RDP, VMs, Basic Render Driver) vs "no FL 11_1 adapter at all". That distinction is what tells a user whether a driver update fixes their machine. - probeBackend() answers "hardware" | "cpu" | "none" without allocating a view, because the export dialog needs it before any preview exists. Cached both sides: it is a property of the machine, and creating a device is not free. - The preview carries a small persistent notice, and the export dialog warns before the export starts. Persistent rather than a toast in the preview: the question it answers ("why is this choppy?") recurs every time the user looks at it. NO effect is disabled on the CPU path. The earlier plan was to force background blur and motion blur off, since they cost 17x and 23x on WARP and are what pull C4+ down to ~8 fps. Dropped: 8 fps is what the 1.7.0 preview delivered, so users have already lived with it, and disabling effects would trade away the iso-render property (max deviation 3/255) that makes a backend swap worth having. Slow and correct beats fast and wrong, in both preview and export. "none" is not a warning. It means there is no native compositor at all -- pure-web dev, jsdom, an addon that failed to load -- which is the normal state in development; warning there would put "no compatible GPU" in front of every developer on every run. Only "cpu" is a degraded machine, and the probe resolves to null first so a machine that turns out to have a GPU never flashes the notice. Both cases are tested. Verified through the real addon, not just the mocks: probeBackend() returns "hardware" on this machine and stays cached on the second call. 1144 vitest + 78 Rust tests pass. The known-gaps entry in native-compositor.md said the compositor was "unusable on virtualised environments" and that nothing fell back -- both were true when written and are now the opposite. Rewritten rather than appended to. --- crates/compositor-view-napi/src/lib.rs | 24 ++++- crates/compositor/src/d3d.rs | 53 +++++++++-- crates/compositor/src/live.rs | 3 +- electron/ipc/nativeBridge.ts | 5 + .../services/compositorViewService.ts | 21 +++++ electron/native/compositor-view/addon.d.ts | 10 ++ src/components/ai-edition/ExportDialog.tsx | 24 ++++- .../ai-edition/NativeCompositorOverlay.tsx | 44 ++++++++- src/i18n/locales/ar/editor.json | 4 + src/i18n/locales/en/editor.json | 4 + src/i18n/locales/es/editor.json | 4 + src/i18n/locales/fr/editor.json | 4 + src/i18n/locales/it/editor.json | 4 + src/i18n/locales/ja-JP/editor.json | 4 + src/i18n/locales/ko-KR/editor.json | 4 + src/i18n/locales/pt-BR/editor.json | 4 + src/i18n/locales/ru/editor.json | 4 + src/i18n/locales/tr/editor.json | 4 + src/i18n/locales/vi/editor.json | 4 + src/i18n/locales/zh-CN/editor.json | 4 + src/i18n/locales/zh-TW/editor.json | 4 + src/native/compositorViewClient.ts | 19 ++++ src/native/contracts.ts | 20 ++++ src/native/hooks/useCompositorBackend.test.ts | 67 ++++++++++++++ src/native/hooks/useCompositorBackend.ts | 57 ++++++++++++ src/native/index.ts | 1 + .../architecture/native-compositor.md | 92 +++++++++---------- 27 files changed, 430 insertions(+), 62 deletions(-) create mode 100644 src/native/hooks/useCompositorBackend.test.ts create mode 100644 src/native/hooks/useCompositorBackend.ts diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index d51bfb6d3d..15a9970459 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -10,7 +10,7 @@ use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFun use napi::{Env, JsFunction, Task}; use napi_derive::napi; use openscreen_compositor::compositor::{live_params_from_scene, Compositor}; -use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::d3d::{Backend, Gpu}; use openscreen_compositor::gif_export::{GifExportParams, GifStats}; use openscreen_compositor::live::{LiveView, PausedPreviews}; use openscreen_compositor::scene::Scene; @@ -52,6 +52,24 @@ fn registry() -> &'static Mutex> { /// /// `rect` ne sert plus que pour `width`/`height` (résolution cible du preview) ; /// `x`/`y` sont ignorés (compat structurelle — la position est gérée par CSS). +/// Quel backend cette machine utilisera : `"hardware"`, `"cpu"`, ou `"none"` si aucun +/// device D3D11 ne se crée (la vue échouera alors avec son propre message, plus précis). +/// +/// Sert à PRÉVENIR : sur `"cpu"`, le rendu passe par WARP + décodage logiciel — la +/// preview tombe à ~8 fps avec tous les effets et l'export met des minutes au lieu de +/// secondes. L'utilisateur doit le savoir AVANT de lancer un export, pas après. D'où une +/// question posée au système et non à une vue : la modale d'export la pose sans qu'aucune +/// preview n'existe. Réponse mise en cache côté Rust — c'est une propriété de la machine. +#[napi] +pub fn probe_backend() -> String { + match Gpu::probe() { + Some(Backend::Hardware) => "hardware", + Some(Backend::Cpu) => "cpu", + None => "none", + } + .to_string() +} + #[napi] pub fn create_view( rect: CompositorViewRect, @@ -368,7 +386,9 @@ impl Task for ExportMultiTask { // Previews paused for the whole render (GPU 3D engine freed) and restored // exactly as found when this guard drops, including on the error paths. let _previews = PreviewPause::begin(); - let gpu = Gpu::create(false).map_err(|e| Error::from_reason(format!("{e:#}")))?; + // Même sélection que la preview : l'export d'un hôte sans GPU passe par + // libopenh264 au lieu d'AMF, plutôt que d'échouer. + let gpu = Gpu::create_auto(false).map_err(|e| Error::from_reason(format!("{e:#}")))?; let mut cfg = config::all().pop().expect("au moins une config"); // C8 cfg.zoom = false; cfg.layout_anim = false; diff --git a/crates/compositor/src/d3d.rs b/crates/compositor/src/d3d.rs index 09c4a3b1a6..9733b86e02 100644 --- a/crates/compositor/src/d3d.rs +++ b/crates/compositor/src/d3d.rs @@ -4,6 +4,7 @@ //! ffmpeg et notre boucle de rendu toucheront le device depuis des threads distincts. use anyhow::{bail, Result}; +use std::sync::OnceLock; use windows::core::Interface; use windows::Win32::Foundation::HMODULE; use windows::Win32::Graphics::Direct3D::{ @@ -98,9 +99,11 @@ fn try_create( /// Message d'échec ACTIONNABLE : re-sonde pour distinguer les deux causes réelles. /// -/// Ce message reste utile MÊME avec `Backend::Cpu` disponible : rien ne bascule tout -/// seul (voir `create`), donc quelqu'un doit savoir pourquoi le matériel a refusé et si -/// le repli logiciel est la bonne réponse à son cas. +/// Ce message reste utile MÊME maintenant que `create_auto` replie sur le backend CPU : +/// il part dans les logs à chaque repli, et c'est lui qui dit si l'utilisateur subit un +/// pilote à mettre à jour (réparable en cinq minutes) ou une VM sans GPU (structurel). +/// Sans lui, un utilisateur au rendu logiciel ne saurait jamais qu'il lui manque un +/// pilote. Et si WARP échoue aussi, c'est ce message-ci que `create_auto` remonte. /// /// PR #162 proposait de retomber sur `D3D_DRIVER_TYPE_WARP` en gardant tout le reste. /// Mesuré (`tests/warp_device_cannot_decode.rs`) : WARP + `VIDEO_SUPPORT` ne se crée même @@ -137,13 +140,51 @@ impl Gpu { /// Crée le device conforme au §2. `debug=false` impératif dans tout run mesuré /// (§10 : la couche debug valide et sérialise chaque appel — facteur, pas %). /// - /// Reste le device MATÉRIEL : le backend CPU ne s'obtient qu'en le demandant - /// (`create_backend`). Rien ne bascule tout seul — un repli silencieux vers un - /// rendu logiciel serait précisément le « l'app rame aujourd'hui » qu'on veut éviter. + /// MATÉRIEL STRICT, sans repli : échoue plutôt que de rendre un device WARP. C'est ce + /// que veulent les tests et les goldens (mesurer ou comparer le chemin GPU n'a aucun + /// sens sur un rastériseur logiciel). Le chemin de production, lui, prend `create_auto`. pub fn create(debug: bool) -> Result { Gpu::create_backend(Backend::Hardware, debug) } + /// Le device de PRODUCTION : matériel si possible, backend CPU sinon. + /// + /// C'est ici que le repli devient automatique, et il ne l'est qu'accompagné : l'app + /// demande `probe()` et prévient l'utilisateur. Un basculement muet vers un rendu à + /// ~8 fps serait exactement le « l'app rame aujourd'hui » que cette branche corrige. + /// + /// Si les DEUX échouent, c'est le diagnostic MATÉRIEL qu'on remonte en tête : c'est + /// lui qui est actionnable (« pas de décodeur vidéo sur cet adaptateur »), pas + /// « WARP indisponible », qui ne dit rien à personne. + pub fn create_auto(debug: bool) -> Result { + let hw_err = match Gpu::create_backend(Backend::Hardware, debug) { + Ok(gpu) => return Ok(gpu), + Err(err) => err, + }; + eprintln!("[d3d] backend matériel indisponible ({hw_err:#}) — repli sur le backend CPU"); + Gpu::create_backend(Backend::Cpu, debug).map_err(|cpu_err| { + anyhow::anyhow!("{hw_err:#} (le repli logiciel a échoué aussi : {cpu_err:#})") + }) + } + + /// Le backend que cette machine obtiendra, sans créer de vue ni d'export. + /// + /// Mis en cache : créer un device coûte quelques dizaines de ms et la réponse ne + /// change pas en cours de session (un pilote qui tombe en marche est un redémarrage, + /// pas un rafraîchissement). `None` = ni matériel ni WARP — la vue échouera, et c'est + /// son message d'erreur, plus précis, qui doit parler. + pub fn probe() -> Option { + static PROBED: OnceLock> = OnceLock::new(); + *PROBED.get_or_init(|| { + for backend in [Backend::Hardware, Backend::Cpu] { + if Gpu::create_backend(backend, false).is_ok() { + return Some(backend); + } + } + None + }) + } + /// Le device du backend demandé. `Backend::Cpu` ne diagnostique pas : si WARP /// lui-même échoue, il n'y a plus rien derrière à proposer. pub fn create_backend(backend: Backend, debug: bool) -> Result { diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 76323ce2e0..26e7f00ac6 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -1038,7 +1038,8 @@ unsafe fn render_thread( webcam: &str, cursor_json: &str, ) -> Result<()> { - let gpu = Gpu::create(false)?; + // Chemin de PRODUCTION : matériel si possible, backend CPU sinon (voir `create_auto`). + let gpu = Gpu::create_auto(false)?; let mut comp = Compositor::new(&gpu)?; // Vue live = le VRAI enregistrement, pas la fenêtre fixture (100s@6s, taillée pour l'ancien // fixture POC). On charge toute la piste depuis t=0 ; 24h couvre large toute recording réelle. diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index b411ae2e98..c99126b64e 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -346,6 +346,11 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { }); return createSuccessResponse(requestId, { id }); } + case "probeBackend": + // No view needed: the export dialog asks before any preview exists. + return createSuccessResponse(requestId, { + backend: compositorViewService.probeBackend(), + }); case "setRect": compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 2e835d4b1d..fd30d95d40 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -6,6 +6,7 @@ import { app } from "electron"; import { resolveCursorSprites } from "../../../src/lib/cursor/cursorThemes"; import type { ClipInput, + CompositorBackend, CompositorParamValue, CompositorViewAddon, CompositorViewRect, @@ -364,6 +365,26 @@ export class CompositorViewService { return this.ensureAddon() !== null; } + /** Which backend the compositor will use on this machine. + * + * `"none"` when the addon is absent — no native path at all, so there is nothing to + * warn about; that is the pure-web/dev case, not a degraded GPU. Callers must not + * read `"none"` as "slow", only `"cpu"` means that. */ + probeBackend(): CompositorBackend { + const addon = this.ensureAddon(); + if (!addon) { + return "none"; + } + try { + return addon.probeBackend(); + } catch (err) { + // An older `.node` predates probeBackend. Treat as unknown rather than + // crashing the bridge: a stale addon should not take the editor down. + console.warn("[compositor-view] probeBackend unavailable:", err); + return "none"; + } + } + /** Allocates an offscreen compositor view sized to `rect.width`x`rect.height`. * `rect.x` / `rect.y` are vestigial (ignored native-side) — the renderer * keeps them on the wire so the existing `CompositorViewRect` shape stays diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 67e617515f..da0172c314 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -89,7 +89,17 @@ export interface ClipInput { webcamOffsetSec: number; } +/** Which backend the compositor will run on. `"cpu"` = WARP rasterisation + software + * decode/encode, used when no usable D3D11 GPU is present: correct output, but roughly + * 8 fps preview with all effects and minutes-long exports. `"none"` = no D3D11 device at + * all, so the view will fail with its own, more specific message. */ +export type CompositorBackend = "hardware" | "cpu" | "none"; + export interface CompositorViewAddon { + /** What this machine offers, asked without allocating a view — the export dialog + * needs the answer before any preview exists. Cached native-side. */ + probeBackend(): CompositorBackend; + /** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the * target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side). * No HWND/native-window-handle is passed: there's no OS window to parent to. The diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index 0d09dca373..63dc382e97 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -32,7 +32,8 @@ import { type GifSizePreset, } from "@/lib/exporter"; import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; -import { exportGifNative, exportMultiNative } from "@/native"; +import { exportGifNative, exportMultiNative, useIsCpuCompositor } from "@/native"; +import { nativeBridgeClient } from "@/native/client"; import type { CompositorClipInput } from "@/native/contracts"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { ModalShell } from "./Modals"; @@ -112,6 +113,9 @@ interface ExportDialogProps { export function ExportDialog({ open, onClose, document }: ExportDialogProps) { const t = useScopedT("editor"); const ts = useScopedT("settings"); + // No usable GPU: the export still applies every effect (output is identical), it + // just runs on the software encoder and takes minutes instead of seconds. + const cpuCompositor = useIsCpuCompositor(); const [format, setFormat] = useState("mp4"); const [quality, setQuality] = useState("good"); const [fps, setFps] = useState<24 | 30 | 60>(60); @@ -608,6 +612,24 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { savedPath={savedPath} /> + {cpuCompositor && phase !== "done" && ( + // Placed next to the export button, not in a toast: it has to land while + // the user is still deciding. A CPU export renders every effect correctly + // but takes minutes rather than seconds, and an unexplained ten-minute + // wait reads as a hang. +

+ {t("cpuCompositor.exportWarning")} +

+ )} +