diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..154f7c37f6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# The compositor is developed across Windows, macOS and Linux. Without this, a +# checkout on Windows can reintroduce CRLF and the shader sources drift from the +# `end_of_line = lf` / `charset = utf-8` that .editorconfig already mandates — +# which is how blur.wgsl, layer.wgsl and linux_decode.rs ended up with CRLF and +# double-encoded French comments in the first place. +*.wgsl text eol=lf +*.rs text eol=lf +*.ts text eol=lf +*.tsx text eol=lf diff --git a/.github/workflows/build-whisper-stt.yml b/.github/workflows/build-whisper-stt.yml index abc07fb5b7..0d50a40067 100644 --- a/.github/workflows/build-whisper-stt.yml +++ b/.github/workflows/build-whisper-stt.yml @@ -172,7 +172,25 @@ jobs: mkdir -p "$BAG" # Copy the whole per-platform directory: the helper executable plus # every ggml backend sidecar/library it needs at runtime. - cp -v "electron/native/bin/${{ matrix.tag }}"/* "$BAG/" + # + # -a, because build-whisper-stt.sh stages a symlink farm on Linux and + # macOS (libggml-vulkan.so -> .so.0 -> .so.0.15.1) and plain cp follows + # every link named on the command line, expanding each into a full copy + # of the payload. libggml-vulkan.so.0.15.1 alone is 60 MB, so the bag + # carried it three times: measured 188 MB instead of 63 MB, and the + # uploaded tarball 59 MB instead of 20 MB (gzip's 32 KiB window cannot + # dedupe copies that far apart). `dir/.` rather than `dir/*` so the copy + # does not depend on the shell's glob skipping dotfiles. + # + # tar preserves the links from here on (no -h below), so this shrinks + # the artifact and the CI download. It does NOT change the installers: + # scripts/stage-whisper-stt.sh:65 is a second plain `cp` that re-expands + # the farm before electron-builder ever sees it. Making that one -a too + # would cut ~131 MB from every package (electron-builder recreates + # symlinks, and squashfs/AppImage stores them natively — both verified), + # but deb/pacman go through fpm, whose behaviour here is unverified, so + # that step needs a real package inspection rather than a guess. + cp -av "electron/native/bin/${{ matrix.tag }}/." "$BAG/" tar -czf "${BAG}.tar.gz" "$BAG" echo "Staged ${BAG}.tar.gz" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18b1582ad1..45e3d95085 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Typecheck tests against a baseline shell: bash env: - BASELINE: 71 + BASELINE: 66 run: | set -uo pipefail COUNT=$(npx tsc -p tsconfig.test.json --noEmit 2>&1 | grep -c 'error TS' || true) diff --git a/.gitignore b/.gitignore index 82bb66e0d1..80075ef838 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ dist-ssr /electron/native/screencapturekit/.build/ /electron/native/screencapturekit/.swiftpm/ /electron/native/bin/ +/electron/native/pipewire-capture/build/ +/electron/native/pipewire-capture/target/ # Native macOS generated files DerivedData/ diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index d96a2ad9b3..38ce24d92d 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -49,11 +49,28 @@ distributed by their own registries, not redistributed inside our binaries. - The speech model (`ggml-*.bin`) is **not** bundled — it is downloaded into the user's data directory on first use by `electron/stt/modelManager.ts`. +## PipeWire — headers (Linux only) + +- **Components**: header sources under + `electron/native/pipewire-capture/vendor/pipewire-1.0.5/include/`, compiled + into `openscreen-pipewire-helper` (the Linux cursor/capture helper) under + `resources/electron/native/bin/linux-*/`. +- **License**: **MIT** — . + Every vendored file keeps its upstream `SPDX-License-Identifier: MIT` header, + and the project's licence text is copied alongside them as `COPYING`. +- **Upstream**: PipeWire release 1.0.5. Only the header subset the helper + includes was vendored; `vendor/README.md` records exactly what was copied and + how to reproduce the selection. +- **No PipeWire binary is redistributed.** The helper resolves + `libpipewire-0.3.so.0` with `dlopen` at runtime, from the user's own system, + so nothing of PipeWire's ships inside our installers beyond the compiled + result of its headers (inline functions and struct layouts). + ## OpenScreen native helpers -`wgc-capture` (Windows Graphics Capture), the ScreenCaptureKit helper (macOS) -and the D3D11 compositor addon are part of this repository and are covered by -[LICENSE](LICENSE). +`wgc-capture` (Windows Graphics Capture), the ScreenCaptureKit helper (macOS), +the PipeWire helper (Linux) and the compositor addon are part of this repository +and are covered by [LICENSE](LICENSE). --- diff --git a/crates/.gitignore b/crates/.gitignore index 3ecbf6ad43..dcc6a0f143 100644 --- a/crates/.gitignore +++ b/crates/.gitignore @@ -1,5 +1,8 @@ # build /target +# `cargo test` lancé DEPUIS crates/compositor/ y crée son propre target/, et les tests +# compose_linux y écrivent leurs PPM de preuve. Même nature que /target ci-dessus. +/compositor/target # dépendance téléchargée : build ffmpeg LGPL-shared BtbN (~160 Mo, voir README) /thirdparty diff --git a/crates/Cargo.lock b/crates/Cargo.lock index df0c787e9f..dbe2f253a7 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -17,12 +17,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -44,11 +68,26 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex 1.3.0", "syn 2.0.119", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -60,6 +99,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "block" @@ -67,11 +109,31 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytemuck" version = "1.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "byteorder-lite" @@ -104,6 +166,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "clang-sys" version = "1.8.1" @@ -115,6 +183,16 @@ dependencies = [ "libloading", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "compositor-view-napi" version = "0.0.0" @@ -163,6 +241,39 @@ dependencies = [ "libc", ] +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + +[[package]] +name = "cosmic-text" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be17b688510d934ce13f48a2beba700e11583e281e0fda99c22bb256a14eda73" +dependencies = [ + "bitflags 2.13.1", + "fontdb", + "harfrust", + "linebender_resource_handle", + "log", + "rangemap", + "rustc-hash 2.1.3", + "self_cell", + "skrifa 0.40.0", + "smol_str", + "swash", + "sys-locale", + "unicode-bidi", + "unicode-linebreak", + "unicode-script", + "unicode-segmentation", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -182,12 +293,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "either" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "fdeflate" version = "0.3.7" @@ -213,6 +339,53 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "font-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7299a780854a6d391be2ae1c8521c9368471b559dbfd6a8dbd9f407eaff100" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "fontconfig-parser" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc773e24e02d4ddd8395fd30dc147524273a83e54e0f312d986ea30de5f5646" +dependencies = [ + "roxmltree", +] + +[[package]] +name = "fontdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457e789b3d1202543297a350643cf459f836cade38934e7a4cf6a39e7cde2905" +dependencies = [ + "fontconfig-parser", + "log", + "memmap2", + "slotmap", + "tinyvec", + "ttf-parser", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -240,12 +413,159 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + [[package]] name = "glob" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "glow" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e5ea60d70410161c8bf5da3fdfeaa1c72ed2c15f8bbb9d19fe3a4fad085f08" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gpu-alloc" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45cf04b2726f02df5508c6de726acdc90cdf97ac771a9a0ffd8ba10a6e696bf9" +dependencies = [ + "bitflags 2.13.1", + "gpu-alloc-types", +] + +[[package]] +name = "gpu-alloc-types" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bbed164dd10ed526c2e4fe3e721ca4a71c61730e5aafac6844b417b3227058" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "gpu-allocator" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c151a2a5ef800297b4e79efa4f4bec035c5f51d5ae587287c9b952bdf734cacd" +dependencies = [ + "log", + "presser", + "thiserror 1.0.69", + "windows", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags 2.13.1", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "harfrust" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da2e5ae821f6e96664977bf974d6d6a2d6682f9ccee23e62ec1d134246845f9" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "core_maths", + "read-fonts 0.37.0", + "smallvec", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "image" version = "0.25.10" @@ -261,6 +581,16 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "itertools" version = "0.13.0" @@ -276,6 +606,62 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + [[package]] name = "libc" version = "0.2.186" @@ -292,6 +678,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -313,6 +726,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "metal" version = "0.29.0" @@ -328,6 +750,21 @@ dependencies = [ "paste", ] +[[package]] +name = "metal" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" +dependencies = [ + "bitflags 2.13.1", + "block", + "core-graphics-types", + "foreign-types", + "log", + "objc", + "paste", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -354,6 +791,28 @@ dependencies = [ "pxfm", ] +[[package]] +name = "naga" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e380993072e52eef724eddfcde0ed013b0c023c3f0417336ed041aa9f076994e" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags 2.13.1", + "cfg_aliases", + "codespan-reporting", + "hexf-parse", + "indexmap", + "log", + "rustc-hash 1.1.0", + "spirv", + "strum", + "termcolor", + "thiserror 2.0.19", + "unicode-xid", +] + [[package]] name = "napi" version = "2.16.17" @@ -411,6 +870,15 @@ dependencies = [ "libloading", ] +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "nom" version = "7.1.3" @@ -454,20 +922,67 @@ dependencies = [ "block", "cc", "core-foundation", + "cosmic-text", "image", - "metal", + "metal 0.29.0", "objc", + "pollster", "serde", "serde_json", + "wgpu", "windows", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "png" version = "0.18.1" @@ -490,6 +1005,18 @@ dependencies = [ "windows", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + [[package]] name = "prettyplease" version = "0.2.37" @@ -509,6 +1036,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" + [[package]] name = "pxfm" version = "0.1.30" @@ -524,6 +1057,55 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "read-fonts" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b634fabf032fab15307ffd272149b622260f55974d9fad689292a5d33df02e5" +dependencies = [ + "bytemuck", + "core_maths", + "font-types 0.11.3", +] + +[[package]] +name = "read-fonts" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" +dependencies = [ + "bytemuck", + "font-types 0.12.2", + "once_cell", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "regex" version = "1.13.1" @@ -553,12 +1135,48 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rustc-hash" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "semver" version = "1.0.28" @@ -626,6 +1244,101 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "skrifa" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdfe3d2475fbd7ddd1f3e5cf8288a30eb3e5f95832829570cd88115a7434ac" +dependencies = [ + "bytemuck", + "read-fonts 0.37.0", +] + +[[package]] +name = "skrifa" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ab7d62b1d3e72d9d9dea5650bac30424f9111364bb94928dbf5ecad1baa68" +dependencies = [ + "bytemuck", + "read-fonts 0.41.0", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" + +[[package]] +name = "spirv" +version = "0.3.0+sdk-1.3.268.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda41003dc44290527a59b13432d4a0379379fa074b70174882adfbdfd917844" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "swash" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2499c2d826531388872b2268718aed907a39bd785ab0dcfe57fab26283f92e" +dependencies = [ + "skrifa 0.44.0", + "yazi", + "zeno", +] + [[package]] name = "syn" version = "2.0.119" @@ -648,18 +1361,319 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "libc", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl 2.0.19", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +dependencies = [ + "core_maths", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-script" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee" + [[package]] name = "unicode-segmentation" version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wgpu" +version = "24.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0b3436f0729f6cdf2e6e9201f3d39dc95813fad61d826c1ed07918b4539353" +dependencies = [ + "arrayvec", + "bitflags 2.13.1", + "cfg_aliases", + "document-features", + "js-sys", + "log", + "naga", + "parking_lot", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "24.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0aa306497a238d169b9dc70659105b4a096859a34894544ca81719242e1499" +dependencies = [ + "arrayvec", + "bit-vec", + "bitflags 2.13.1", + "cfg_aliases", + "document-features", + "indexmap", + "log", + "naga", + "once_cell", + "parking_lot", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.19", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-hal" +version = "24.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f112f464674ca69f3533248508ee30cb84c67cf06c25ff6800685f5e0294e259" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.13.1", + "block", + "bytemuck", + "cfg_aliases", + "core-graphics-types", + "glow", + "glutin_wgl_sys", + "gpu-alloc", + "gpu-allocator", + "gpu-descriptor", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "metal 0.31.0", + "naga", + "ndk-sys", + "objc", + "once_cell", + "ordered-float", + "parking_lot", + "profiling", + "range-alloc", + "raw-window-handle", + "renderdoc-sys", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.19", + "wasm-bindgen", + "web-sys", + "wgpu-types", + "windows", + "windows-core", +] + +[[package]] +name = "wgpu-types" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50ac044c0e76c03a0378e7786ac505d010a873665e2d51383dcff8dd227dc69c" +dependencies = [ + "bitflags 2.13.1", + "js-sys", + "log", + "web-sys", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows" version = "0.58.0" @@ -730,6 +1744,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -794,6 +1817,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "yazi" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01738255b5a16e78bbb83e7fbba0a1e7dd506905cfc53f4622d89015a03fbb5" + +[[package]] +name = "zeno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df3dc4292935e51816d896edcd52aa30bc297907c26167fec31e2b0c6a32524" + [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/Cargo.toml b/crates/Cargo.toml index ecb97d3c96..208f376cc5 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -29,6 +29,11 @@ serde_json = "1" # Décodage des wallpapers image (jpg/png) pour le fond natif. default-features off → # on ne tire que les décodeurs nécessaires (build plus léger). image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } +# Port Linux du compositor (PR #183) : wgpu (Vulkan) + pollster (block_on) + +# cosmic-text (rastérisation texte, remplace DirectWrite/CoreText). +wgpu = { version = "24", features = ["wgsl"] } +pollster = "0.4" +cosmic-text = "0.19" [workspace.dependencies.windows] version = "0.58" diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 3d56359701..a68b0fe7aa 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -656,3 +656,56 @@ pub fn export_gif( on_progress: make_progress_tsfn(on_progress)?, })) } + +/// Bilan d'un remux, tel que le voit la glue TS. +#[napi(object)] +pub struct RemuxStats { + /// Paquets recopiés, toutes pistes confondues. `f64` car napi n'expose pas + /// `u64` — sans risque, un enregistrement plausible reste très loin de 2^53. + pub packets: f64, + pub streams: u32, + pub wall_s: f64, +} + +pub struct RemuxTask { + input_path: String, + output_path: String, +} + +impl Task for RemuxTask { + type Output = openscreen_compositor::remux::RemuxStats; + type JsValue = RemuxStats; + + fn compute(&mut self) -> Result { + openscreen_compositor::remux::remux_to_seekable_matroska(&self.input_path, &self.output_path) + .map_err(|e| Error::from_reason(format!("{e:#}"))) + } + + fn resolve(&mut self, _env: Env, out: Self::Output) -> Result { + Ok(RemuxStats { + packets: out.packets as f64, + streams: out.streams, + wall_s: out.wall_s, + }) + } +} + +/// Recopie `input_path` vers `output_path` par le muxer matroska (aucun +/// ré-encodage) pour doter le fichier des `Cues`/`SeekHead` que `MediaRecorder` +/// n'écrit pas. Voir `openscreen_compositor::remux` pour le détail. +/// +/// `AsyncTask` et pas une fonction synchrone : le remux lit et réécrit tout le +/// fichier, ce qui se compte en secondes sur un long enregistrement. Le faire +/// sur le thread principal de Node gèlerait la fenêtre pendant la sauvegarde, +/// exactement au moment où l'UI affiche « enregistrement en cours ». +/// +/// `output_path` doit être un chemin TEMPORAIRE : c'est au caller TS de +/// renommer par-dessus l'original une fois la promesse résolue, pour qu'un échec +/// laisse l'enregistrement intact. +#[napi] +pub fn remux_seekable(input_path: String, output_path: String) -> AsyncTask { + AsyncTask::new(RemuxTask { + input_path, + output_path, + }) +} diff --git a/crates/compositor/Cargo.toml b/crates/compositor/Cargo.toml index 1a4692bfdc..f968b54970 100644 --- a/crates/compositor/Cargo.toml +++ b/crates/compositor/Cargo.toml @@ -44,3 +44,10 @@ metal = "0.29" objc = "0.2" block = "0.1" core-foundation = "0.9" + +# Linux : wgpu (Vulkan) pour le rendu, pollster pour block_on les ops wgpu, +# cosmic-text pour la rastérisation du texte (remplace DirectWrite/CoreText). +[target.'cfg(target_os = "linux")'.dependencies] +wgpu.workspace = true +pollster.workspace = true +cosmic-text.workspace = true diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index 7c4e9d4780..33db6e6c78 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -76,9 +76,13 @@ fn main() { // Le wrapper.h à binder dépend de la plateforme cible : // - Windows : D3D11VA (ID3D11VA*), - // - macOS : VideoToolbox (AVVideotoolboxContext). + // - macOS : VideoToolbox (AVVideotoolboxContext), + // - Linux : software/VAAPI, aucun hwcontext propriétaire (le d3d11.h / + // les headers VT n'existent pas → wrapper dédié). let wrapper = if target_is_macos { "wrapper_macos.h" + } else if target_os == "linux" { + "wrapper_linux.h" } else { "wrapper_windows.h" }; @@ -146,9 +150,64 @@ fn main() { .expect("bindgen a échoué sur les headers ffmpeg"); let out = PathBuf::from(env::var("OUT_DIR").unwrap()); - bindings - .write_to_file(out.join("ffi.rs")) - .expect("écriture ffi.rs"); + let ffi_path = out.join("ffi.rs"); + bindings.write_to_file(&ffi_path).expect("écriture ffi.rs"); + + // Sur Linux, les symboles ffmpeg des .so vendorisés sont renommés avec un préfixe + // avant l'édition de liens (cf. scripts/build-linux-compositor-addon.mjs). Il faut + // donc que CES déclarations importent les noms préfixés — sinon l'éditeur de liens + // ne trouve rien, ou pire, l'addon se rattache au ffmpeg de Chromium au runtime. + if let Ok(prefix) = env::var("OPENSCREEN_FFMPEG_SYMBOL_PREFIX") { + prefix_ffmpeg_symbols(&ffi_path, &prefix); + } +} + +/// Ajoute `#[link_name = ""]` devant chaque `pub fn` ffmpeg de `ffi.rs`. +/// +/// Seul le SYMBOLE importé change ; l'identifiant Rust reste `avformat_open_input`, donc +/// aucun site d'appel du crate ne bouge. bindgen ne génère ici que des fonctions (aucun +/// `pub static`), ce qui rend la réécriture sûre et purement textuelle. +/// +/// Pourquoi ce détour plutôt qu'un `#[link(name = ...)]` : le problème n'est pas de +/// désigner une bibliothèque mais d'éviter une COLLISION DE NOMS. Electron charge +/// `libffmpeg.so` (le ffmpeg de Chromium) comme dépendance directe, donc ses symboles +/// occupent la portée globale avant tout addon ; sous ELF, nos imports s'y rattachent +/// quoi que dise notre RUNPATH. Renommer supprime la collision à la racine, là où jouer +/// sur la portée de résolution (RTLD_DEEPBIND) casse l'interposition de l'allocateur de +/// Chromium et fait planter le process. +fn prefix_ffmpeg_symbols(ffi_path: &Path, prefix: &str) { + let source = std::fs::read_to_string(ffi_path).expect("relecture ffi.rs"); + let mut out = String::with_capacity(source.len() + 64 * 1024); + let mut renamed = 0usize; + + for line in source.lines() { + if let Some(name) = line + .strip_prefix(" pub fn ") + .and_then(|rest| rest.split('(').next()) + .filter(|n| is_ffmpeg_symbol(n)) + { + out.push_str(&format!(" #[link_name = \"{prefix}{name}\"]\n")); + renamed += 1; + } + out.push_str(line); + out.push('\n'); + } + + assert!( + renamed > 0, + "OPENSCREEN_FFMPEG_SYMBOL_PREFIX est posé mais aucune fonction ffmpeg n'a été \ + trouvée dans ffi.rs — le format de sortie de bindgen a changé, et l'addon se \ + lierait silencieusement au ffmpeg de Chromium." + ); + std::fs::write(ffi_path, out).expect("réécriture ffi.rs"); + println!("cargo:warning=ffmpeg: {renamed} symboles préfixés en `{prefix}`"); +} + +/// Les préfixes que ffmpeg expose : `av*` (avutil/avcodec/avformat + `avpriv_`), +/// `sws_*` (swscale) et `swr_*` (swresample). Aligné sur le filtre du script de build, +/// qui dérive la table de renommage des mêmes bibliothèques. +fn is_ffmpeg_symbol(name: &str) -> bool { + name.starts_with("av") || name.starts_with("sws_") || name.starts_with("swr_") } /// Pose `LIBCLANG_PATH` sur la toolchain Xcode/CommandLineTools active, pour bindgen. @@ -192,4 +251,4 @@ fn point_libclang_at_the_xcode_toolchain() { // découverte par défaut de clang-sys, qui sait aussi chercher toute seule. None => env::remove_var("LIBCLANG_PATH"), } -} \ No newline at end of file +} diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs new file mode 100644 index 0000000000..4fa4dd4756 --- /dev/null +++ b/crates/compositor/src/compositor_linux.rs @@ -0,0 +1,2154 @@ +//! Moteur de composition Linux -- wgpu / WGSL. +//! +//! Equivalent Linux de `compositor_windows.rs` / `compositor_macos.rs` : meme +//! surface publique (`Compositor::{new, new_sized, normalize_render_size, +//! render_size, set_scene, set_live_params, set_cursor, set_cursor_time, +//! set_timeline_time, clear_cursor, scene_snapshot, clear_srv_cache, +//! compose_frame, readback_direct}`) pour que `live.rs` et `compositor-view-napi` +//! (cfg-re-exportes via `crate::compositor`) l'utilisent sans connaitre la +//! plateforme. S'y ajoutent, specifiques a ce backend, les trois entrees de la +//! ring de staging (`set_readback_depth`, `readback_submit`, `readback_take`) : +//! seul l'export Linux les utilise, cf. `ReadbackRing`. +//! +//! **Iso-render.** La GEOMETRIE (placement de chaque calque) vient de +//! `frame_geometry::plan_frame` -- la MEME fonction que Windows/macOS, au pixel +//! pres. Ce module ne fait que RENDRE le `FrameGeometry` via wgpu/WGSL +//! (`vk_shaders/layer.wgsl`), la ou macOS le rend via Metal/MSL. +//! +//! **Portee actuelle.** `compose_frame` rend le coeur : fond uni + calque ecran +//! cover-fit (coins arrondis). Les calques riches (webcam PiP, curseur, +//! annotations texte mode 11, blur de fond, motion blur) sont dessines +//! par les memes primitives (`draw_layer`) et arrivent par iterations, comme le +//! port Metal les a ajoutes -- chacun reutilise `layer.wgsl` (modes deja portes) +//! ou une passe dediee (`blur.wgsl`). + +use std::cell::RefCell; + +use anyhow::Result; +use wgpu::util::DeviceExt; + +use crate::config::Cfg; +use crate::d3d::Gpu; +use crate::ffi::AVFrame; +// Re-exports que le code partage (live.rs, compositor-view-napi) consomme via +// `crate::compositor::…`, a l'identique de `compositor_macos`. +pub use crate::frame_geometry::{ + live_params_from_scene, webcam_shape_code, FIXTURE_FRAMES, LayerCB, LiveParams, OUT_H, OUT_W, +}; +use crate::frame_geometry::{ + cursor_sprite_dst, parse_hex, plan_cursor, plan_frame, CursorPlacement, CursorPlanInput, + FrameGeometryInput, +}; +use crate::scene::{Scene, SceneBackground}; + +const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); +const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); + +/// `&LayerCB` -> `&[u8; 128]`. `LayerCB` est `#[repr(C, align(16))]`, son layout +/// EST le buffer uniforme WGSL (16 vec4 + 1 vec2 + 2 f32 = 128 octets). +fn layer_bytes(cb: &LayerCB) -> &[u8] { + unsafe { std::slice::from_raw_parts(cb as *const LayerCB as *const u8, 128) } +} + +/// Une copie RT -> staging DEJA SOUMISE, dont le mapping est arme mais pas +/// encore recolte. On garde `idx` (l'index de soumission rendu par +/// `Queue::submit`) pour n'attendre QUE cette soumission-la, et les dimensions +/// telles qu'elles etaient au moment de la copie -- ce sont elles qui decrivent +/// le contenu du buffer, pas celles du compositeur au moment de la recolte. +struct PendingCopy { + buf: wgpu::Buffer, + idx: wgpu::SubmissionIndex, + rx: std::sync::mpsc::Receiver>, + w: u32, + h: u32, + bpr: u32, +} + +/// Ring de staging de la relecture. +/// +/// AVANT : `readback_direct` enregistrait la copie, la soumettait, puis bloquait +/// dans `device.poll(Maintain::Wait)`. Cette attente n'absorbait pas la copie +/// (8,3 Mo = ~0,33 ms de DMA) mais TOUTE la file GPU en cours -- la chaine +/// Kawase et chaque draw de calque, que `compose_frame` avait soumis sans +/// attendre juste avant. Mesure 1080p : 3,8 ms (scene simple) a 6,2 ms (scene +/// chargee) par frame, pendant que `sws_scale` + `avcodec_send_frame` (12,6 ms +/// de CPU pur) attendaient leur tour. Le GPU et le CPU ne se recouvraient +/// jamais. +/// +/// MAINTENANT : `readback_submit` soumet la copie de la frame N vers un buffer +/// libre, arme son `map_async` et rend la main ; il ne recolte que la frame la +/// plus ANCIENNE encore en vol. Avec `depth = 2`, c'est la frame N-1, dont la +/// copie a ete soumise avant l'encodage de N-1 et le decodage/composition de N : +/// le GPU a eu ~19 ms de CPU pour finir 6 ms de travail, l'attente tombe a zero. +/// +/// PROFONDEUR. 2 est le minimum utile et suffit ici : le seul travail a +/// recouvrir est ce que le CPU fait entre deux relectures (sws + encode, +/// 12,6 ms mesures) et il depasse deja largement la chaine GPU (3,8 a 6,2 ms). +/// Une 3e frame n'ajouterait que 8 Mo de memoire mappable et une frame de +/// latence de plus. La profondeur reste parametrable parce que la POLITIQUE +/// differe par chemin (cf. `set_readback_depth`), pas pour empiler les buffers. +/// +/// UN SEUL RT. Le RT n'est pas double-bufferise : la copie de la frame N est +/// soumise AVANT les commandes de composition de la frame N+1, sur la meme +/// queue, et wgpu insere la barriere qui va bien. Le GPU lit donc le RT avant +/// de le reecrire, sans que le CPU ait a l'attendre. +struct ReadbackRing { + depth: usize, + /// Buffers disponibles (aucune copie en vol, aucun mapping arme). + free: Vec, + /// Copies soumises, dans l'ordre de soumission (FIFO strict : les frames + /// sortent dans l'ordre ou elles ont ete composees). + pending: std::collections::VecDeque, +} + +pub struct Compositor { + gpu: Gpu, + render_w: u32, + render_h: u32, + + // Pipeline de calque (VS + FS `layer.wgsl`), sampler lineaire, bind group + // layout (uniform + 2 textures + sampler). Immuables apres `new_sized`. + pipeline: wgpu::RenderPipeline, + /// Meme shader et meme layout que `pipeline`, blend ADDITIF pondere par la + /// constante de blend. Sert a sommer les copies de la trainee du curseur + /// dans `accum` ; cf. `blend_add` cote Windows. + pipeline_add: wgpu::RenderPipeline, + /// Copie plein ecran d'`accum` vers le RT en « over » premultiplie + /// (`blur.wgsl` : `vs_fullscreen` + `fs_copy`). Utilise le layout du blur. + pipeline_copy: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + + // Chaine de blur Kawase du fond (`blur.wgsl`) : layout dedie (uniform + 1 + // tex + sampler), 2 pipelines (down/up), 3 textures de pyramide (1/2, 1/4, + // 1/8 de la sortie). Les `TextureView` gardent leurs textures en vie. + blur_bgl: wgpu::BindGroupLayout, + blur_down: wgpu::RenderPipeline, + blur_up: wgpu::RenderPipeline, + blur_half: wgpu::TextureView, + blur_qtr: wgpu::TextureView, + blur_oct: wgpu::TextureView, + + // Render target offscreen + ring de staging de la relecture (recrees au resize). + rt: wgpu::Texture, + rt_view: wgpu::TextureView, + /// Cible ISOLEE d'accumulation, meme taille et meme format que le RT. + /// `_accum` garde la texture en vie ; seule la vue est utilisee. + _accum: wgpu::Texture, + accum_view: wgpu::TextureView, + /// `bytes_per_row` padde a 256 (contrainte wgpu de copy_texture_to_buffer). + readback_bpr: u32, + /// Ring de buffers de staging (cf. `ReadbackRing`). `RefCell` : les methodes + /// publiques du compositeur sont `&self`, comme tout le reste de l'etat. + readback: RefCell, + + // Etat pilote par live.rs (interior mutability : les methodes sont `&self`). + live_params: RefCell, + scene: RefCell>, + cursor: RefCell>, + cursor_time: RefCell>, + timeline_time: RefCell>, + + /// Rasterizer de texte (annotations mode 11). `None` si l'init cosmic-text + /// echoue -- le rendu continue sans texte plutot que de tout casser. + #[allow(dead_code)] + text_raster: Option, + + /// Cache des sprites curseur (PNG RGBA -> texture wgpu), par chemin. Meme + /// role que `img_cache` cote macOS : un sprite chargé une fois par session. + img_cache: RefCell>, + + /// Copie mipmappee de la frame composee, lue par les annotations « flou » + /// (mode 10). `ann_copy` garde la texture en vie, `ann_copy_view` porte tous + /// les niveaux (echantillonnage), `ann_copy_mips` un niveau chacune (cibles + /// de la generation). Cf. `make_ann_copy`. + ann_copy: wgpu::Texture, + ann_copy_view: wgpu::TextureView, + ann_copy_mips: Vec, + + /// Images d'annotation, indexees par ID d'annotation -- PAS par chemin comme + /// `img_cache`. Une annotation image porte souvent une data-URI de plusieurs + /// mega-octets ; s'en servir comme cle de HashMap la ferait hacher a chaque + /// frame. La longueur de la source sert de temoin de changement, comme cote + /// macOS. + ann_img_cache: RefCell>, +} + +impl Compositor { + pub fn new(gpu: &Gpu) -> Result { + Self::new_sized(gpu, OUT_W, OUT_H) + } + + pub fn new_sized(gpu: &Gpu, w: u32, h: u32) -> Result { + let (w, h) = Self::normalize_render_size(w, h); + let gpu = Gpu { + device: gpu.device.clone(), + context: gpu.context.clone(), + backend: gpu.backend, + feature_level: gpu.feature_level, + }; + + let module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("layer.wgsl"), + source: wgpu::ShaderSource::Wgsl(LAYER_WGSL.into()), + }); + let sampler = gpu.device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("layer"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + // Trilineaire pour le LOD fractionnaire du mode 10 : `log2(rayon)` + // tombe entre deux niveaux, et en `Nearest` le flou avancerait par + // paliers visibles quand le rayon varie. Sans effet sur tout le + // reste -- aucune autre texture liee ici n'a plus d'un niveau. + mipmap_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + let bind_group_layout = + gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("layer"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(128), + }, + count: None, + }, + tex_entry(1), + tex_entry(2), + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let pipeline_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("layer"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + // Deux pipelines pour le MEME shader de calque : seul le blend change. + let mk_layer = |label: &str, blend: wgpu::BlendState| { + gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(blend), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) + }; + let pipeline = mk_layer("layer", wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + // SOMME pondere : `src * constante + dst`. La constante (posee par pass + // via `set_blend_constant`) vaut 1/taps, donc N copies d'un curseur + // parfaitement immobile redonnent exactement ce curseur. Transcription + // du `blend_add` D3D11 (BLEND_FACTOR / ONE / OP_ADD sur couleur ET + // alpha) ; l'alpha doit suivre la couleur, sinon la somme n'est plus + // premultipliee et la composition finale delave la trainee. + let add = wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::Constant, + dst_factor: wgpu::BlendFactor::One, + operation: wgpu::BlendOperation::Add, + }; + let pipeline_add = mk_layer( + "layer-add", + wgpu::BlendState { color: add, alpha: add }, + ); + + // --- Chaine de blur Kawase du fond (`blur.wgsl`) --- + let blur_module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("blur.wgsl"), + source: wgpu::ShaderSource::Wgsl(BLUR_WGSL.into()), + }); + // Layout blur : 0 = uniform, 1 = texture, 2 = sampler (blur.wgsl). + let blur_bgl = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("blur"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(128), + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let blur_pl = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("blur"), + bind_group_layouts: &[&blur_bgl], + push_constant_ranges: &[], + }); + let mk_blur = |entry: &str| { + gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(entry), + layout: Some(&blur_pl), + vertex: wgpu::VertexState { + module: &blur_module, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &blur_module, + entry_point: Some(entry), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) + }; + let blur_down = mk_blur("fs_kawase_down"); + let blur_up = mk_blur("fs_kawase_up"); + // Composition d'`accum` sur le RT : meme layout que le blur (uniform + + // 1 texture + sampler) et blend « over » premultiplie. Son VS est + // `vs_fullscreen` et non le `vs_main` du Kawase -- une passe UNIQUE ne + // pardonne pas une erreur d'orientation, cf. le commentaire la-bas. + let pipeline_copy = gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("accum-copy"), + layout: Some(&blur_pl), + vertex: wgpu::VertexState { + module: &blur_module, + entry_point: Some("vs_fullscreen"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &blur_module, + entry_point: Some("fs_copy"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + let mk_pyr = |dw: u32, dh: u32, label: &str| { + gpu.device + .create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: dw.max(1), + height: dh.max(1), + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + .create_view(&wgpu::TextureViewDescriptor::default()) + }; + let blur_half = mk_pyr(w / 2, h / 2, "blur-half"); + let blur_qtr = mk_pyr(w / 4, h / 4, "blur-qtr"); + let blur_oct = mk_pyr(w / 8, h / 8, "blur-oct"); + + let (rt, rt_view, accum, accum_view, readback_bpr) = Self::make_targets(&gpu, w, h); + let (ann_copy, ann_copy_view, ann_copy_mips) = Self::make_ann_copy(&gpu, w, h); + // Profondeur 1 par defaut = chemin synchrone historique, a l'octet et a + // la latence pres. C'est l'export qui demande explicitement 2 (cf. + // `set_readback_depth`) ; tout autre appelant garde l'ancien contrat. + let readback = RefCell::new(ReadbackRing { + depth: 1, + free: vec![Self::make_staging(&gpu, readback_bpr, h)], + pending: std::collections::VecDeque::new(), + }); + + Ok(Compositor { + gpu, + render_w: w, + render_h: h, + pipeline, + pipeline_add, + pipeline_copy, + bind_group_layout, + sampler, + blur_bgl, + blur_down, + blur_up, + blur_half, + blur_qtr, + blur_oct, + rt, + rt_view, + _accum: accum, + accum_view, + readback_bpr, + readback, + live_params: RefCell::new(LiveParams::default()), + scene: RefCell::new(None), + cursor: RefCell::new(None), + cursor_time: RefCell::new(None), + timeline_time: RefCell::new(None), + text_raster: crate::text::TextRasterizer::new().ok(), + img_cache: RefCell::new(std::collections::HashMap::new()), + ann_copy, + ann_copy_view, + ann_copy_mips, + ann_img_cache: RefCell::new(std::collections::HashMap::new()), + }) + } + + /// RT RGBA8, cible d'accumulation de meme geometrie, et `bytes_per_row` de + /// la relecture (padde a 256). + /// + /// `accum` est alloue ICI et pas ailleurs pour qu'il soit impossible de le + /// laisser a l'ancienne taille apres un changement de resolution : c'est le + /// meme appel qui produit les deux, et un accum plus petit que le RT ferait + /// une passe de composition tronquee. + fn make_targets( + gpu: &Gpu, + w: u32, + h: u32, + ) -> (wgpu::Texture, wgpu::TextureView, wgpu::Texture, wgpu::TextureView, u32) { + let mk = |label: &str, extra: wgpu::TextureUsages| { + gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | extra, + view_formats: &[], + }) + }; + let rt = mk("rt", wgpu::TextureUsages::COPY_SRC); + let accum = mk("accum", wgpu::TextureUsages::empty()); + let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); + let accum_view = accum.create_view(&wgpu::TextureViewDescriptor::default()); + let bpr = (w * 4).div_ceil(256) * 256; + (rt, rt_view, accum, accum_view, bpr) + } + + /// Copie du RT avec chaine de mips COMPLETE, source des annotations « flou ». + /// + /// Le mode 10 lit un niveau de mip choisi par `log2(rayon)` : c'est la + /// pyramide qui FAIT le flou, pas un noyau de taps (cf. le commentaire du + /// shader). Il lui faut donc tous les niveaux jusqu'a 1x1, sinon un grand + /// rayon demande un LOD qui n'existe pas et le sampler retombe sur le dernier + /// disponible -- le flou plafonne en silence. + /// + /// Retourne aussi une vue PAR NIVEAU : `generate_ann_mips` rend le niveau i + /// depuis le niveau i-1, et une vue de render target ne peut porter qu'un + /// seul niveau. + fn make_ann_copy( + gpu: &Gpu, + w: u32, + h: u32, + ) -> (wgpu::Texture, wgpu::TextureView, Vec) { + // floor(log2(max)) + 1 : le dernier niveau mesure 1x1. + let levels = 32 - w.max(h).max(1).leading_zeros(); + let tex = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("ann-copy"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: levels, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let mips = (0..levels) + .map(|level| { + tex.create_view(&wgpu::TextureViewDescriptor { + label: Some("ann-copy-mip"), + base_mip_level: level, + mip_level_count: Some(1), + ..Default::default() + }) + }) + .collect(); + (tex, view, mips) + } + + /// Fige la frame composee dans `ann_copy` et remplit sa pyramide. + /// + /// UNE seule fois par frame, AVANT toute annotation : les flous doivent voir + /// l'image composee SANS les flous eux-memes, sinon deux zones qui se + /// recouvrent s'echantillonnent l'une l'autre selon l'ordre de dessin. Meme + /// contrat que le `blit` + `generate_mipmaps` de `compositor_macos`. + /// + /// wgpu n'a pas de `generate_mipmaps` : chaque niveau est une passe de rendu + /// plein ecran qui echantillonne le precedent. Le filtre lineaire sur une + /// source exactement deux fois plus grande EST la moyenne 2x2, donc cette + /// boucle produit la meme pyramide que le blit Metal. + fn generate_ann_mips(&self, encoder: &mut wgpu::CommandEncoder) { + encoder.copy_texture_to_texture( + self.rt.as_image_copy(), + self.ann_copy.as_image_copy(), + wgpu::Extent3d { + width: self.render_w, + height: self.render_h, + depth_or_array_layers: 1, + }, + ); + // Les bind groups doivent survivre a leur passe : on les garde tous ici. + let mut keep: Vec<(wgpu::Buffer, wgpu::BindGroup)> = Vec::new(); + for level in 1..self.ann_copy_mips.len() { + let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("ann-mip-uniform"), + // `fs_copy` ne lit pas l'uniforme, mais le layout du blur l'exige. + contents: layer_bytes(&LayerCB::default()), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("ann-mip"), + layout: &self.blur_bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView( + &self.ann_copy_mips[level - 1], + ), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + keep.push((uniform, bind)); + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("ann-mip-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.ann_copy_mips[level], + resolve_target: None, + ops: wgpu::Operations { + // Clear plutot que Load : le niveau n'a jamais ete ecrit, + // et `pipeline_copy` blende « over ». Sur une cible vidée + // le « over » rend la source telle quelle -- l'ecrasement + // qu'on veut ici. + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, &keep[keep.len() - 1].1, &[]); + rpass.draw(0..3, 0..1); + } + } + + /// Un buffer de staging de la ring. La taille depend de `bpr` (donc de la + /// largeur de rendu) et de la hauteur : changer la geometrie de rendu impose + /// de les reallouer -- ce que fait `new_sized`, puisque la preview + /// RECONSTRUIT le compositeur au resize (`live.rs`) au lieu de le + /// redimensionner a chaud. Aucune copie ne peut donc etre en vol au moment + /// ou la taille change : l'ancien compositeur (et sa ring) est detruit + /// entier, wgpu gardant ses buffers vivants jusqu'a la fin des soumissions + /// qui les referencent. + fn make_staging(gpu: &Gpu, bpr: u32, h: u32) -> wgpu::Buffer { + gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback"), + size: u64::from(bpr) * u64::from(h), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }) + } + + /// Une passe Kawase : lit `src`, ecrit `dst` (fullscreen triangle, 3 + /// vertices). `src_px` = dimensions de la source (pour le pas de texel). + fn blur_pass( + &self, + encoder: &mut wgpu::CommandEncoder, + pipeline: &wgpu::RenderPipeline, + src: &wgpu::TextureView, + dst: &wgpu::TextureView, + src_px: [f32; 2], + ) { + let cb = LayerCB { + quad_px: src_px, + mode: -1.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: [2.0, 0.0, 0.0, 0.0], // texel offset Kawase + ..Default::default() + }; + let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("blur-uniform"), + contents: layer_bytes(&cb), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("blur"), + layout: &self.blur_bgl, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(src), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("blur-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: dst, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(pipeline); + rpass.set_bind_group(0, &bind, &[]); + rpass.draw(0..3, 0..1); + } + + /// Floute le RT (le fond deja dessine) : dual-Kawase 3 down (RT -> 1/2 -> + /// 1/4 -> 1/8) + 3 up (1/8 -> 1/4 -> 1/2 -> RT). ~gaussien a cout constant. + fn blur_bg(&self, encoder: &mut wgpu::CommandEncoder) { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let (hw, hh) = (rw * 0.5, rh * 0.5); + let (qw, qh) = (rw * 0.25, rh * 0.25); + let (ow, oh) = (rw * 0.125, rh * 0.125); + self.blur_pass(encoder, &self.blur_down, &self.rt_view, &self.blur_half, [rw, rh]); + self.blur_pass(encoder, &self.blur_down, &self.blur_half, &self.blur_qtr, [hw, hh]); + self.blur_pass(encoder, &self.blur_down, &self.blur_qtr, &self.blur_oct, [qw, qh]); + self.blur_pass(encoder, &self.blur_up, &self.blur_oct, &self.blur_qtr, [ow, oh]); + self.blur_pass(encoder, &self.blur_up, &self.blur_qtr, &self.blur_half, [qw, qh]); + self.blur_pass(encoder, &self.blur_up, &self.blur_half, &self.rt_view, [hw, hh]); + } + + /// Dimensions paires (NV12 4:2:0), min 2x2. Symetrie avec les autres backends. + pub fn normalize_render_size(w: u32, h: u32) -> (u32, u32) { + ((w.max(2) + 1) & !1, (h.max(2) + 1) & !1) + } + + pub fn render_size(&self) -> (u32, u32) { + (self.render_w, self.render_h) + } + + pub fn set_live_params(&self, p: LiveParams) { + *self.live_params.borrow_mut() = p; + } + + pub fn set_scene(&self, s: Option) { + *self.scene.borrow_mut() = s; + } + + pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { + *self.cursor.borrow_mut() = Some(track); + } + + pub fn set_cursor_time(&self, t: Option) { + *self.cursor_time.borrow_mut() = t; + } + + pub fn set_timeline_time(&self, t: Option) { + *self.timeline_time.borrow_mut() = t; + } + + pub fn clear_cursor(&self) { + *self.cursor.borrow_mut() = None; + } + + pub fn scene_snapshot(&self) -> Option { + self.scene.borrow().clone() + } + + /// Pas de cache de SRV cote wgpu (les `TextureView`s sont recreees a chaque + /// draw depuis le carrier) -- no-op conserve pour la symetrie d'API. + pub fn clear_srv_cache(&self) {} + + // -- seam frame (lit le carrier `data[0]`) -- + + fn pixel_buffer_of(frame: *const AVFrame) -> Option<()> { + if frame.is_null() || unsafe { (*frame).data[0] }.is_null() { + None + } else { + Some(()) + } + } + + unsafe fn nv12_srvs( + &self, + frame: *const AVFrame, + ) -> Result<(wgpu::TextureView, wgpu::TextureView)> { + crate::linux_frames::nv12_planes(frame) + } + + unsafe fn tex_dims(&self, frame: *const AVFrame) -> (u32, u32) { + if frame.is_null() || (*frame).data[0].is_null() { + return (1, 1); + } + crate::linux_frames::carrier_dims(frame) + } + + // -- rendu -- + + /// Prepare un draw de calque : buffer uniforme init a `cb` + bind group + /// (uniform + deux textures + sampler). Cree AVANT la render pass pour que + /// les ressources vivent pendant tout le pass. Un buffer PAR draw : + /// `write_buffer` entre draws d'une meme pass ne s'entrelace pas. + /// `LayerCB` d'une ombre portee (mode 2), identique a `draw_shadow` cote + /// macOS et au bloc equivalent cote Windows. + /// + /// Le quad est ELARGI de `spread` de chaque cote et decale de `offset_px` ; + /// le shader y trace un SDF de rect arrondi dont l'alpha decroit sur la + /// largeur du spread. C'est pour ca que `fx.x` porte le spread : le + /// fragment en a besoin pour normaliser sa penombre. + fn shadow_cb( + &self, + dst: [f32; 4], + size_px: [f32; 2], + radius: f32, + spread: f32, + offset_px: [f32; 2], + opacity: f32, + ) -> LayerCB { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let (sx, sy) = (spread / rw, spread / rh); + let (ox, oy) = (offset_px[0] / rw, offset_px[1] / rh); + LayerCB { + dst: [dst[0] - sx + ox, dst[1] - sy + oy, dst[2] + 2.0 * sx, dst[3] + 2.0 * sy], + quad_px: [size_px[0] + 2.0 * spread, size_px[1] + 2.0 * spread], + radius_px: radius, + mode: 2.0, + color: [0.0, 0.0, 0.0, opacity], + fx: [spread, 0.0, 0.0, 0.0], + mb: [0.0, 1.0, 1.0, 0.0], + ..Default::default() + } + } + + /// `LayerCB` de l'ombre d'un ecran INCLINE (mode 12) : la penombre suit le + /// quadrilatere projete, pas son rect englobant. Port de + /// `compositor_macos::draw_quad_shadow`. + fn quad_shadow_cb( + &self, + corners: &[(f32, f32); 4], + center_px: [f32; 2], + radius: f32, + spread: f32, + offset_px: [f32; 2], + opacity: f32, + ) -> LayerCB { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let (min_x, max_x) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(x, _)| (mn.min(x), mx.max(x))); + let (min_y, max_y) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(_, y)| (mn.min(y), mx.max(y))); + // La boite doit contenir la penombre entiere, sinon elle se coupe net. + let box_w = (max_x - min_x) + 2.0 * spread; + let box_h = (max_y - min_y) + 2.0 * spread; + let local = |(x, y): (f32, f32)| -> [f32; 2] { [x - min_x + spread, y - min_y + spread] }; + let [tl0, tl1] = local(corners[0]); + let [tr0, tr1] = local(corners[1]); + let [br0, br1] = local(corners[2]); + let [bl0, bl1] = local(corners[3]); + LayerCB { + dst: [ + (center_px[0] + min_x - spread + offset_px[0]) / rw, + (center_px[1] + min_y - spread + offset_px[1]) / rh, + box_w / rw, + box_h / rh, + ], + quad_px: [box_w, box_h], + radius_px: radius, + mode: 12.0, + color: [0.0, 0.0, 0.0, opacity], + fx: [tl0, tl1, tr0, tr1], + src_prev: [br0, br1, bl0, bl1], + // Le spread vit ici et NON dans `fx.x` : `fx` porte deja les coins. + mb: [0.0, spread, 1.0, 0.0], + ..Default::default() + } + } + + /// `LayerCB` de l'ecran incline (mode 8) : le quad projete est dessine dans sa + /// BBOX et le fragment remonte au (s,t) du plan par warp bilineaire inverse. + /// Port de `compositor_macos::draw_tilted_screen`. Pas de motion blur sur ce + /// chemin -- le tilt est bref, la simplification ne se voit pas. + fn tilted_screen_cb( + &self, + quad: &crate::regions::TiltedQuad, + s_px: [f32; 2], + center_px: [f32; 2], + cut: [f32; 4], + radius: f32, + ) -> LayerCB { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let corners = quad.corners; + // Taille du plan dans son propre repere, AVANT projection : c'est la que vit + // le rayon, pour qu'il reste constant le long du bord au lieu de s'etirer + // avec la perspective. + let plane_px = [s_px[0] * quad.scale, s_px[1] * quad.scale]; + let (min_x, max_x) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(x, _)| (mn.min(x), mx.max(x))); + let (min_y, max_y) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(_, y)| (mn.min(y), mx.max(y))); + let bbox_w = (max_x - min_x).max(1.0); + let bbox_h = (max_y - min_y).max(1.0); + // Coins en px LOCAUX a la bbox, pour matcher `i.local` du shader. + let local = |(x, y): (f32, f32)| -> [f32; 2] { [x - min_x, y - min_y] }; + let [tl0, tl1] = local(corners[0]); + let [tr0, tr1] = local(corners[1]); + let [br0, br1] = local(corners[2]); + let [bl0, bl1] = local(corners[3]); + LayerCB { + dst: [ + (center_px[0] + min_x) / rw, + (center_px[1] + min_y) / rh, + bbox_w / rw, + bbox_h / rh, + ], + src: cut, + quad_px: [bbox_w, bbox_h], + radius_px: radius * quad.scale, + mode: 8.0, + fx: [tl0, tl1, tr0, tr1], + src_prev: [br0, br1, bl0, bl1], + dst_prev: [plane_px[0], plane_px[1], 0.0, 0.0], + ..Default::default() + } + } + + fn make_bind( + &self, + cb: &LayerCB, + planes: Option<(&wgpu::TextureView, &wgpu::TextureView)>, + dummy: &wgpu::TextureView, + ) -> (wgpu::Buffer, wgpu::BindGroup) { + let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("layer-uniform"), + contents: layer_bytes(cb), + usage: wgpu::BufferUsages::UNIFORM, + }); + let (y, uv) = planes.unwrap_or((dummy, dummy)); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("layer"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(y), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(uv), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + (uniform, bind) + } + + /// Charge un PNG/JPEG (chemin fichier ou data URI) en texture RGBA8. Port + /// wgpu du `load_image_texture` macOS, memes chemins (`decode_data_uri` + /// partage, crate `image`). Sert aux sprites de curseur (mode 7). + fn load_image_texture(&self, path: &str) -> Result<(wgpu::Texture, u32, u32)> { + let img = if let Some(bytes) = crate::frame_geometry::decode_data_uri(path) { + image::load_from_memory(&bytes) + .map_err(|e| anyhow::anyhow!("data URI image ({} octets) : {e}", bytes.len()))? + .to_rgba8() + } else { + image::open(path) + .map_err(|e| anyhow::anyhow!("sprite {path} : {e}"))? + .to_rgba8() + }; + let (w, h) = (img.width(), img.height()); + let pixels = img.into_raw(); + let tex = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("sprite"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.gpu.context.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &pixels, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(w * 4), + rows_per_image: Some(h), + }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + Ok((tex, w, h)) + } + + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des + /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : + /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. + pub unsafe fn compose_frame( + &self, + screen: *const AVFrame, + webcam: *const AVFrame, + frame: f32, + cfg: &Cfg, + ) -> Result<()> { + if Self::pixel_buffer_of(screen).is_none() { + return self.clear_rt(); + } + let (sy, suv) = self.nv12_srvs(screen)?; + let (stw, sth) = self.tex_dims(screen); + let (wtw, wth) = self.tex_dims(webcam); + let (scw, sch) = ((*screen).width as f32, (*screen).height as f32); + let (wcw, wch) = if webcam.is_null() { + (1.0, 1.0) + } else { + ((*webcam).width as f32, (*webcam).height as f32) + }; + let u_max = scw / (stw.max(1)) as f32; + let v_max = sch / (sth.max(1)) as f32; + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + + let scene_ref = self.scene.borrow(); + let cursor_ref = self.cursor.borrow(); + let lp = *self.live_params.borrow(); + let g = plan_frame(&FrameGeometryInput { + render_px: [rw, rh], + screen_tex_px: [stw as f32, sth as f32], + screen_visible_px: [scw, sch], + webcam_visible_px: [wcw, wch], + u_max, + v_max, + frame, + cfg, + live: lp, + scene: scene_ref.as_ref(), + cursor: cursor_ref.as_ref(), + timeline_t_override: *self.timeline_time.borrow(), + }); + // (`wtw`/`wth` sont les dims de la TEXTURE webcam, consommees par le + // cover-crop du calque PiP plus bas.) + + // Fond : Color -> clear a la couleur ; Gradient -> mode 5 ; Image -> + // mode 6 wallpaper cover-fit (via load_image_texture). Le blur (si + // cfg.bg_blur) floute ensuite ce fond, avant l'ecran. + enum BgLayer { + Gradient(LayerCB), + Image(String), + } + let (bg_clear, bg_layer) = match scene_ref.as_ref().map(|s| s.background.clone()) { + Some(SceneBackground::Color { color }) => { + (parse_hex(&color).unwrap_or(lp.bg_color), None) + } + Some(SceneBackground::Gradient { angle_deg, stops }) => { + let c0 = stops.first().and_then(|s| parse_hex(s)).unwrap_or(lp.bg_color); + let c1 = stops.last().and_then(|s| parse_hex(s)).unwrap_or(c0); + let a = angle_deg.to_radians(); + let cb = LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src: [c1[0], c1[1], c1[2], c1[3]], + quad_px: [rw, rh], + mode: 5.0, + color: c0, + fx: [a.sin(), -a.cos(), 0.0, 0.0], + ..Default::default() + }; + ([0.0, 0.0, 0.0, 1.0], Some(BgLayer::Gradient(cb))) + } + Some(SceneBackground::Image { path }) => { + ([0.0, 0.0, 0.0, 1.0], Some(BgLayer::Image(path))) + } + None => (lp.bg_color, None), + }; + + // ROTATION 3D (presets iso/left/right d'une zoom region). La geometrie du + // tilt est calculee UNE fois : l'ombre et l'ecran doivent porter exactement + // le meme quadrilatere, sinon l'ombre se decolle des que l'un des deux + // change. `regions` fait toute la trigo (partagee avec macOS/Windows) ; ici + // on ne fait que l'empaqueter. + let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; + let tilt = (!crate::regions::is_identity_rotation(g.zoom_rotation)) + .then(|| crate::regions::rotated_quad_corners_px(s_px[0], s_px[1], g.zoom_rotation)); + let quad_center_px = [ + (g.s_dst[0] + g.s_dst[2] * 0.5) * rw, + (g.s_dst[1] + g.s_dst[3] * 0.5) * rh, + ]; + + // Calque ecran : mode 0 (rect droit, NV12 -> RGB) quand la rotation est + // neutre, mode 8 (warp bilineaire inverse dans la bbox du quad projete) + // sinon. Place par plan_frame (cover-fit + coins arrondis) ; + // `src = g.cut` (crop utilisateur + zoom en UV texture) dans les deux cas. + // + // FLOU DE VELOCITE, ET POURQUOI SEULEMENT SUR LE MODE 0. `src_prev`/ + // `dst_prev` decrivent le MEME calque a la frame precedente ; le shader + // remappe chaque pixel de sortie par ce couple pour retrouver l'UV qu'il + // occupait alors, et floute le long du segment. `src_prev = g.cut` et non + // un `cut` d'avant : la coupe est identique aux deux frames (`plan_frame` + // ne fait varier que le rect de DESTINATION entre `s_dst` et + // `s_dst_prev`), ce que Windows documente aussi. Le mouvement vient donc + // entierement de `dst_prev`. + // + // Le mode 8 n'en recoit PAS, et ce n'est pas un oubli : ces deux champs y + // portent deja les coins projetes du quad (BR/BL dans `src_prev`, + // `plane_px` dans `dst_prev`). Les deux sens ne peuvent pas cohabiter dans + // un meme draw. macOS et Windows sautent egalement le flou sur le chemin + // incline, pour la meme raison. + let screen_layer = match tilt.as_ref() { + None => LayerCB { + dst: g.s_dst, + src: g.cut, + quad_px: s_px, + radius_px: g.s_radius, + mode: 0.0, + color: [1.0, 1.0, 1.0, 1.0], + src_prev: g.cut, + dst_prev: g.s_dst_prev, + mb: [g.mb_taps, 1.0, 1.0, 0.0], + ..Default::default() + }, + Some(quad) => self.tilted_screen_cb(quad, s_px, quad_center_px, g.cut, g.s_radius), + }; + // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; + // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). + let dummy = self.dummy_view(); + let (_screen_uniform, screen_bind) = + self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + + // OMBRE PORTEE de l'ecran, dessinee JUSTE AVANT le calque ecran. Le shader + // la connait depuis le debut ; ce qui manquait etait uniquement le draw + // cote Rust, si bien que le curseur « Ombre » de l'UI ne faisait rien sur + // Linux. + // + // Les fractions de reglage viennent de `frame_geometry`, partagees avec + // macOS/Windows : l'ombre a la meme taille relative sur les trois + // plateformes quelle que soit la resolution de sortie. + // + // L'ombre suit la silhouette REELLEMENT affichee : rect arrondi (mode 2) + // quand l'ecran est droit, quadrilatere projete (mode 12) quand il penche. + let screen_shadow = cfg.shadow.then(|| { + let spread = crate::frame_geometry::SCREEN_SHADOW_SPREAD_FRAC * g.frame_min_px; + let offset = [0.0, crate::frame_geometry::SCREEN_SHADOW_OFFSET_FRAC * g.frame_min_px]; + let opacity = 0.45 * lp.shadow_scale; + let cb = match tilt.as_ref() { + None => self.shadow_cb(g.s_dst, s_px, g.s_radius, spread, offset, opacity), + Some(quad) => self.quad_shadow_cb( + &quad.corners, + quad_center_px, + g.s_radius * quad.scale, + spread, + offset, + opacity, + ), + }; + self.make_bind(&cb, None, &dummy) + }); + + // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. + // `_tex`/`_view` gardent l'image en vie pendant le pass. + struct BgDraw { + _buf: wgpu::Buffer, + _tex: Option, + _view: Option, + bind: wgpu::BindGroup, + } + let bg_draw = bg_layer.and_then(|bl| match bl { + BgLayer::Gradient(cb) => { + let (buf, bind) = self.make_bind(&cb, None, &dummy); + Some(BgDraw { _buf: buf, _tex: None, _view: None, bind }) + } + BgLayer::Image(path) => { + // Charge (ou recupere du cache) le wallpaper. Emprunt isole AVANT + // le borrow_mut (piege du double emprunt 1re frame, cf. macOS). + let cached = self.img_cache.borrow().get(path.as_str()).cloned(); + let (tex, iw, ih) = match cached { + Some(v) => v, + None => match self.load_image_texture(&path) { + Ok(v) => { + self.img_cache.borrow_mut().insert(path.clone(), v.clone()); + v + } + Err(e) => { + eprintln!("[fond image] \"{path}\" : {e:#}"); + return None; + } + }, + }; + // Cover-fit : l'image remplit tout le cadre, on rogne l'axe long. + let ai = iw as f32 / ih.max(1) as f32; + let ao = rw / rh; + let src = if ai > ao { + let vis = ao / ai; + [(1.0 - vis) * 0.5, 0.0, 1.0 - (1.0 - vis) * 0.5, 1.0] + } else { + let vis = ai / ao; + [0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5] + }; + let cb = LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src, + mode: 6.0, + ..Default::default() + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + Some(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + } + }); + + // Webcam PiP (mode 0) -- placee par plan_frame (`g.w_dst`, coins + // `g.w_radius`), gardee par `g.shape_fade > 0` (webcam visible). + // `webcam_planes` garde les vues en vie pendant le pass. + // `lp.has_webcam` is the gate Windows (`compositor_windows.rs`) and macOS + // (`compositor_macos.rs`) both apply and this backend did not. It is false + // when the clip has no camera — and in that case the "webcam" decoder holds + // the SCREEN video, because `open_and_seek_clip` falls back to it rather + // than leave the pair half-open. Without this check a recording with no + // camera drew its own screen picture inside the PiP box. + let webcam_planes = if lp.has_webcam && g.shape_fade > 0.0 && !webcam.is_null() { + self.nv12_srvs(webcam).ok() + } else { + None + }; + let webcam_draw = webcam_planes.as_ref().map(|(wy, wuv)| { + // COVER-CROP. `src` etait cable a [0,0,1,1], donc la texture entiere + // etait etiree sur la boite quelle que soit sa forme : le facteur de + // deformation valait exactement `box_ar / cam_ar`. Invisible en PiP + // rectangulaire (`compositeLayout.ts` y preserve deja le ratio), + // spectaculaire des que le masque est un cercle ou un carre, ou la + // boite est forcee carree et une camera 16:9 s'ecrase de 1,78x. + // + // `cover_crop_uv` est la primitive partagee que macOS et Windows + // utilisent ; elle rend le rect inchange quand il a deja le bon + // ratio, donc aucun placement correct ne bouge. + let (cu0, cv0, cu1, cv1) = crate::frame_geometry::cover_crop_uv( + [wcw, wch], + [wtw as f32, wth as f32], + g.w_px[0] / g.w_px[1].max(0.0001), + ); + // MIROIR : on inverse l'intervalle u. Le VS interpole `src` + // lineairement et `fs_main` ne re-clampe pas `i.uv`, donc un + // intervalle a l'envers suffit -- aucune retouche du WGSL. Apres le + // cover-crop les deux bornes sont strictement a l'interieur de la + // texture, donc le sampler ClampToEdge ne bave pas sur les bords. + let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; + // `src_prev` doit valoir EXACTEMENT le `src` de ce draw, miroir + // compris : le shader s'en sert pour reconstruire l'UV de la frame + // precedente, et un rect source qui ne correspond pas au calque + // dessine ferait diverger la trainee vers une zone de la texture qui + // n'a jamais ete affichee. Seul `dst_prev` porte le mouvement. + let cb = LayerCB { + dst: g.w_dst, + src: [u0, cv0, u1, cv1], + quad_px: g.w_px, + radius_px: g.w_radius, + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + src_prev: [u0, cv0, u1, cv1], + dst_prev: g.w_dst_prev, + mb: [g.mb_taps, 1.0, 1.0, 0.0], + ..Default::default() + }; + self.make_bind(&cb, Some((wy, wuv)), &dummy) + }); + + // OMBRE de la camera. Pas dans les presets « bloc » (dual-frame, + // vertical-stack) : la camera y est collee a l'ecran comme une tuile, + // et une ombre entre les deux dessinerait une couture. Meme condition + // que macOS. + let webcam_shadow = (cfg.shadow + && g.shape_fade > 0.0 + && webcam_draw.is_some() + && !matches!( + g.scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack") + )) + .then(|| { + let cb = self.shadow_cb( + g.w_dst, + g.w_px, + g.w_radius, + crate::frame_geometry::WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, + [0.0, crate::frame_geometry::WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], + crate::frame_geometry::WEBCAM_SHADOW_OPACITY * g.shape_fade, + ); + self.make_bind(&cb, None, &dummy) + }); + + // ANNOTATIONS -- calque le plus haut, place relativement au rect ecran + // `g.s_dst` (les coords x/y/w/h de l'annotation sont des fractions de ce + // rect, cf. `scene.rs`). Port de `compositor_macos::draw_annotations` : + // memes modes, memes replis, meme ordre. Seul le texte diverge, tinte + // cote shader (atlas R8) au lieu d'une couleur bakee dans la texture. + struct AnnDraw { + _buf: wgpu::Buffer, + /// Gardent l'atlas / la texture image en vie jusqu'au submit. `None` + /// pour les quads qui n'echantillonnent rien (plaque de fond, fleche). + _glyphs: Option, + _tex: Option, + bind: wgpu::BindGroup, + } + impl AnnDraw { + fn plain(buf: wgpu::Buffer, bind: wgpu::BindGroup) -> AnnDraw { + AnnDraw { _buf: buf, _glyphs: None, _tex: None, bind } + } + } + // FENETRE TEMPORELLE. Sans ce test, TOUTES les annotations du projet sont + // peintes sur TOUTES les frames : cinq sous-titres s'empilent les uns sur + // les autres du debut a la fin de l'export. C'est le defaut qui se lit + // comme « le texte s'affiche bizarrement » avant meme de regarder les + // glyphes. Mirroir de `visible()` dans compositor_macos.rs. + let visible = |a: &crate::scene::SceneAnnotation| { + g.source_t >= a.start_sec as f32 && g.source_t < a.end_sec as f32 + }; + // Un flou lit la frame composee ; il faut donc la figer AVANT de dessiner + // la moindre annotation. On ne le fait que si un flou est reellement + // visible : la pyramide coute une passe par niveau. + let needs_ann_copy = scene_ref + .as_ref() + .is_some_and(|s| s.annotations.iter().any(|a| a.kind == "blur" && visible(a))); + let mut ann_draws: Vec = Vec::new(); + if let Some(scene) = scene_ref.as_ref() { + // La liste arrive deja triee par zIndex cote app : l'ordre d'iteration + // EST l'ordre de peinture. + for a in &scene.annotations { + if !visible(a) { + continue; + } + let dst = [ + g.s_dst[0] + a.x * g.s_dst[2], + g.s_dst[1] + a.y * g.s_dst[3], + a.w * g.s_dst[2], + a.h * g.s_dst[3], + ]; + let quad_px = [dst[2] * rw, dst[3] * rh]; + // Une boite degeneree ferait un atlas 0x0 et un draw invisible ; + // macOS l'ecarte de la meme facon. + if quad_px[0] <= 0.0 || quad_px[1] <= 0.0 { + continue; + } + match a.kind.as_str() { + "figure" => { + let Some(figure) = a.figure.as_ref() else { continue }; + let (segments, half_stroke) = crate::regions::arrow_local_geometry( + &figure.direction, + figure.stroke_width, + quad_px, + ); + let cb = LayerCB { + dst, + quad_px, + mode: 9.0, + color: parse_hex(&figure.color).unwrap_or([1.0, 1.0, 1.0, 1.0]), + fx: segments[0], + src_prev: segments[1], + dst_prev: segments[2], + mb: [1.0, half_stroke, 0.0, 0.0], + ..Default::default() + }; + let (buf, bind) = self.make_bind(&cb, None, &dummy); + ann_draws.push(AnnDraw::plain(buf, bind)); + } + "blur" => { + let Some(blur) = a.blur.as_ref() else { continue }; + // Le masque en trace libre demanderait une liste de points + // cote GPU : on masque la BOITE ENGLOBANTE. Choix + // deliberement asymetrique -- ne rien dessiner laisserait + // passer en clair ce que l'utilisateur a designe comme a + // cacher, et un masque qui ne masque pas donne confiance a + // tort. + let freehand = blur.shape == "freehand"; + let is_blur = if blur.style == "blur" { 1.0 } else { 0.0 }; + let amount = + if is_blur > 0.5 { blur.intensity } else { blur.block_size }; + // Le repli passe par le rectangle, pas l'ovale : un ovale + // inscrit retirerait les coins, donc une partie de ce qui + // est couvert. + let is_oval = if blur.shape == "oval" && !freehand { 1.0 } else { 0.0 }; + // La teinte n'a de sens qu'en mosaique : un flou teinte ne + // ressemble plus a un flou. + let tinted = if is_blur > 0.5 { 0.0 } else { 1.0 }; + let tint = if blur.color == "black" { + [0.0, 0.0, 0.0, 1.0] + } else { + [1.0, 1.0, 1.0, 1.0] + }; + let cb = LayerCB { + dst, + quad_px, + mode: 10.0, + color: tint, + fx: [is_blur, amount.max(1.0), is_oval, tinted], + ..Default::default() + }; + // La copie mipmappee au binding 1 (texY), la ou le mode 10 + // la lit. + let (buf, bind) = self.make_bind( + &cb, + Some((&self.ann_copy_view, &self.ann_copy_view)), + &dummy, + ); + ann_draws.push(AnnDraw::plain(buf, bind)); + } + "image" => { + let Some(src) = a.image_path.as_ref().filter(|s| !s.is_empty()) else { + continue; + }; + let cached = { + let c = self.ann_img_cache.borrow(); + c.get(&a.id).filter(|(_, _, _, len)| *len == src.len()).cloned() + }; + let Some((tex, iw, ih, _)) = cached.or_else(|| { + match self.load_image_texture(src) { + Ok((tex, w, h)) => { + let e = (tex, w, h, src.len()); + self.ann_img_cache + .borrow_mut() + .insert(a.id.clone(), e.clone()); + Some(e) + } + Err(e) => { + eprintln!("[annotation image] {}: {e:#}", a.id); + None + } + } + }) else { + continue; + }; + if iw == 0 || ih == 0 { + continue; + } + // CONTAIN, pas cover : l'image tient entiere dans la boite + // et se centre. Etirer au rect deformerait une capture ou + // un logo, ce que le rendu web ne fait pas non plus. + let box_aspect = quad_px[0] / quad_px[1]; + let img_aspect = iw as f32 / ih as f32; + let (fit_w, fit_h) = if img_aspect > box_aspect { + (dst[2], dst[3] * (box_aspect / img_aspect)) + } else { + (dst[2] * (img_aspect / box_aspect), dst[3]) + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let cb = LayerCB { + dst: [ + dst[0] + (dst[2] - fit_w) * 0.5, + dst[1] + (dst[3] - fit_h) * 0.5, + fit_w, + fit_h, + ], + src: [0.0, 0.0, 1.0, 1.0], + quad_px: [fit_w * rw, fit_h * rh], + mode: 7.0, + color: [1.0, 1.0, 1.0, 1.0], + // Mode 7 clippe sur `fx` : un rect qui couvre tout le + // cadre = pas de clip. + fx: [0.0, 0.0, 1.0, 1.0], + ..Default::default() + }; + let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + ann_draws.push(AnnDraw { + _buf: buf, + _glyphs: None, + _tex: Some(tex), + bind, + }); + } + "text" => { + let Some(raster) = self.text_raster.as_ref() else { continue }; + let Some(text) = a.text.as_ref() else { continue }; + if text.content.trim().is_empty() { + continue; + } + let color = parse_hex(&text.color).unwrap_or([1.0, 1.0, 1.0, 1.0]); + let background = + parse_hex(&text.background_color).unwrap_or([0.0, 0.0, 0.0, 0.0]); + let spec = crate::text::TextSpec { + content: text.content.clone(), + color, + background, + font_size_px: text.font_size_rel * (g.s_dst[3] * rh), + font_family: text.font_family.clone(), + bold: text.font_weight == "bold", + italic: text.font_style == "italic", + underline: text.text_decoration == "underline", + align: text.text_align.clone(), + box_px: [ + quad_px[0].round().max(1.0) as u32, + quad_px[1].round().max(1.0) as u32, + ], + }; + let glyphs = match raster.rasterize(&self.gpu, &spec) { + Ok(gl) => gl, + Err(e) => { + eprintln!("[annotation texte] {}: {e:#}", a.id); + continue; + } + }; + + // ANIMATION D'APPARITION (`text_anim`, partage avec macOS + // et Windows). Les decalages sont exprimes en px A 1080p + // et remis a l'echelle de la sortie, comme la taille de + // police : en px absolus la meme animation sauterait deux + // fois plus haut dans un rendu 4K que dans l'apercu. + let anim = crate::text_anim::text_animation_state( + text.animation.as_deref(), + (g.source_t - a.start_sec as f32) * 1000.0, + ); + let anim_px = rh / crate::text_anim::ANIMATION_REFERENCE_HEIGHT; + let (mut ax, mut ay, mut aw, mut ah) = ( + dst[0] + anim.translate_x * anim_px / rw, + dst[1] + anim.translate_y * anim_px / rh, + dst[2], + dst[3], + ); + if (anim.scale - 1.0).abs() > 1e-4 { + let (cx, cy) = (ax + aw * 0.5, ay + ah * 0.5); + aw *= anim.scale; + ah *= anim.scale; + ax = cx - aw * 0.5; + ay = cy - ah * 0.5; + } + // Machine a ecrire : le quad ET son UV sont coupes a la + // meme fraction, donc la texture n'est pas etiree -- elle + // est revelee. + let reveal = anim.reveal.clamp(0.0, 1.0); + if reveal <= 0.0 { + continue; + } + let anim_dst = [ax, ay, aw * reveal, ah]; + + // PLAQUE DE FOND, dessinee AVANT les glyphes. + // + // macOS et Windows la peignent dans la texture de texte + // elle-meme ; ici c'est impossible : l'atlas est en R8, il + // ne porte qu'une couverture alpha et aucune couleur. + // Plutot que de convertir tout l'atlas en RGBA pour un + // aplat, on emet un quad mode 1 (couleur pleine + SDF de + // rect arrondi, cf. layer.wgsl) sous le quad de texte. + // Meme rect, meme rayon que le rendu web + // (`annotationRenderer.ts`). + // + // Sans ca le fond n'existait tout simplement pas : + // `spec.background` arrivait jusqu'au rasteriseur et + // mourait dans `cache_key()`. + if background[3] > 0.0 { + let plate = LayerCB { + dst: anim_dst, + src: [0.0, 0.0, 1.0, 1.0], + quad_px: [anim_dst[2] * rw, anim_dst[3] * rh], + mode: 1.0, + // La plaque suit l'opacite du texte : sinon un + // fondu ferait apparaitre un aplat plein d'un coup + // puis le texte dessus. + color: [ + background[0], + background[1], + background[2], + background[3] * anim.opacity, + ], + radius_px: 4.0 * (rh / 1080.0).max(0.5), + ..Default::default() + }; + let (pbuf, pbind) = self.make_bind(&plate, None, &dummy); + ann_draws.push(AnnDraw::plain(pbuf, pbind)); + } + + let cb = LayerCB { + dst: anim_dst, + src: [0.0, 0.0, reveal, 1.0], + quad_px: [anim_dst[2] * rw, anim_dst[3] * rh], + mode: 11.0, + color: [color[0], color[1], color[2], color[3] * anim.opacity], + ..Default::default() + }; + // Atlas R8 au binding 1 (texY) que le mode 11 echantillonne. + let (buf, bind) = + self.make_bind(&cb, Some((&glyphs.view, &glyphs.view)), &dummy); + ann_draws.push(AnnDraw { + _buf: buf, + _glyphs: Some(glyphs), + _tex: None, + bind, + }); + } + _ => {} + } + } + } + + // Curseur thematise : sprite RGBA droit (mode 7) ou pose sur le plan + // incline (mode 13) selon ce que `plan_cursor` a resolu. + // `_tex`/`_view`/`_bufs` gardent le sprite et les uniformes en vie + // pendant le pass. Miroir de la branche curseur de `compositor_macos`. + // + // TRAINEE (`plan.taps > 1`) : `binds` porte une copie par echantillon, + // interpolee entre `prev_placement` et le placement courant. Elles ne + // sont PAS dessinees sur le RT mais dans `accum`, puis compositees en une + // fois -- cf. le commentaire au point de dessin. + struct CursorDraw { + _bufs: Vec, + _tex: wgpu::Texture, + _view: wgpu::TextureView, + binds: Vec, + } + let cursor_draw: Option = (|| { + let track = cursor_ref.as_ref()?; + let plan = plan_cursor( + &g, + &CursorPlanInput { + render_px: [rw, rh], + u_max, + v_max, + cfg, + live: lp, + scene: scene_ref.as_ref(), + track, + t: self + .cursor_time + .borrow() + .unwrap_or(frame / crate::frame_geometry::FPS), + }, + )?; + + let sprites = scene_ref + .as_ref() + .map(|s| s.cursor.cursor_sprites.clone()) + .unwrap_or_default(); + let sprite = plan + .cursor_type + .as_deref() + .and_then(|t| sprites.get(t)) + .or_else(|| sprites.get("arrow"))?; + // Charge (ou recupere du cache) le sprite. Emprunt isole AVANT le + // borrow_mut, comme cote macOS (piege du double emprunt 1re frame). + let cached = self.img_cache.borrow().get(sprite.path.as_str()).cloned(); + let (tex, iw, ih) = match cached { + Some(v) => v, + None => match self.load_image_texture(&sprite.path) { + Ok(v) => { + self.img_cache.borrow_mut().insert(sprite.path.clone(), v.clone()); + v + } + Err(e) => { + eprintln!("[curseur] sprite \"{}\" : {e:#}", sprite.path); + return None; + } + }, + }; + // Ratio preserve : le sprite tient dans un carre de `size_px` de cote. + let ar = iw as f32 / ih.max(1) as f32; + let (pw, ph) = if ar >= 1.0 { + (plan.size_px, plan.size_px / ar) + } else { + (plan.size_px * ar, plan.size_px) + }; + let hotspot = [sprite.hotspot_x, sprite.hotspot_y]; + // `taps == 1` : un seul placement, celui de l'instant rendu -- le + // chemin net d'avant, inchange. Au-dela, on echelonne les copies + // regulierement de `prev_placement` (inclus) au placement courant + // (inclus) : c'est ce que font les deux autres backends, et inclure + // les deux bornes est ce qui fait que la trainee touche a la fois + // l'endroit d'ou le curseur vient et celui ou il est. + // + // On interpole des PLACEMENTS et non des centres : `lerp` sait + // traiter le cas incline, si bien qu'une trainee sous zoom incline + // reste dans le plan au lieu de repasser par un centre 2D qui + // l'aplatirait. + let placements: Vec = if plan.taps <= 1 { + vec![plan.placement] + } else { + (0..plan.taps) + .map(|k| { + let f = k as f32 / (plan.taps - 1) as f32; + plan.prev_placement.lerp(plan.placement, f) + }) + .collect() + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let (mut bufs, mut binds) = (Vec::new(), Vec::new()); + for placement in placements { + let cb = match placement { + CursorPlacement::Upright { center } => LayerCB { + dst: cursor_sprite_dst(center, pw / rw, ph / rh, hotspot), + src: [0.0, 0.0, 1.0, 1.0], + mode: 7.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: plan.clip, + ..Default::default() + }, + CursorPlacement::Tilted { plane_pt, quad, center_px, screen_px, .. } => { + // Le sprite est pose DANS le plan : sa taille devient une fraction + // du plan et ses quatre coins traversent la meme projection que la + // video. La reduction due au tilt vient donc de la projection -- + // rien a multiplier a la main. + let (wf, hf) = (pw / screen_px[0], ph / screen_px[1]); + let x0 = plane_pt[0] - hotspot[0] * wf; + let y0 = plane_pt[1] - hotspot[1] * hf; + let corners = [(x0, y0), (x0 + wf, y0), (x0 + wf, y0 + hf), (x0, y0 + hf)] + .map(|(fx, fy)| { + let (px, py) = quad.point_px(fx, fy); + (center_px[0] + px, center_px[1] + py) + }); + let (min_x, max_x) = corners + .iter() + .fold((f32::MAX, f32::MIN), |(mn, mx), &(x, _)| (mn.min(x), mx.max(x))); + let (min_y, max_y) = corners + .iter() + .fold((f32::MAX, f32::MIN), |(mn, mx), &(_, y)| (mn.min(y), mx.max(y))); + // Le quad projete d'un sprite peut etre tres fin de biais : une bbox + // d'un pixel de large ferait diverger le warp inverse, d'ou le + // plancher a 1 px. + let (bw, bh) = ((max_x - min_x).max(1.0), (max_y - min_y).max(1.0)); + let local = |(x, y): (f32, f32)| [x - min_x, y - min_y]; + let [tl0, tl1] = local(corners[0]); + let [tr0, tr1] = local(corners[1]); + let [br0, br1] = local(corners[2]); + let [bl0, bl1] = local(corners[3]); + LayerCB { + dst: [min_x / rw, min_y / rh, bw / rw, bh / rh], + quad_px: [bw, bh], + mode: 13.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: [tl0, tl1, tr0, tr1], + src_prev: [br0, br1, bl0, bl1], + // Le clip vit ici et NON dans `fx` (mode 7) : `fx` porte les coins. + dst_prev: plan.clip, + ..Default::default() + } + } + }; + // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. + let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + bufs.push(buf); + binds.push(bind); + } + Some(CursorDraw { _bufs: bufs, _tex: tex, _view: view, binds }) + })(); + // Bind group de la passe de composition d'`accum` (layout du blur : + // uniform + texture + sampler). Construit hors de la pass, comme les + // autres. L'uniforme n'est pas lu par `fs_copy` mais le layout l'exige. + let accum_bind = cursor_draw + .as_ref() + .filter(|c| c.binds.len() > 1) + .map(|_| { + let cb = LayerCB::default(); + let uniform = + self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("accum-copy-uniform"), + contents: layer_bytes(&cb), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("accum-copy"), + layout: &self.blur_bgl, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&self.accum_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + (uniform, bind) + }); + + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("compose"), + }); + // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel). + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("bg-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg_clear[0] as f64, + g: bg_clear[1] as f64, + b: bg_clear[2] as f64, + a: bg_clear[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + if let Some(bg) = &bg_draw { + rpass.set_pipeline(&self.pipeline); + rpass.set_bind_group(0, &bg.bind, &[]); + rpass.draw(0..4, 0..1); + } + } + // Blur du fond (avant l'ecran), si active par la scene/l'inspector. + if cfg.bg_blur { + self.blur_bg(&mut encoder); + } + // Passe 2 : avant-plan (ecran + webcam), compose par-dessus le fond + // (eventuellement floute) avec `LoadOp::Load`. Les annotations sont dans + // une passe a part, cf. plus bas. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("fg-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + // Chaque ombre est dessinee JUSTE AVANT le calque qu'elle porte : + // elle doit passer sous lui mais au-dessus du fond (et, pour la + // camera, au-dessus de l'ecran). + if let Some((_buf, bind)) = &screen_shadow { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } + rpass.set_bind_group(0, &screen_bind, &[]); + rpass.draw(0..4, 0..1); + if let Some((_buf, bind)) = &webcam_shadow { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } + if let Some((_buf, bind)) = &webcam_draw { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } + } + // Fige la frame composee pour les annotations « flou ». ICI et nulle part + // ailleurs : apres l'ecran et la camera (sinon un flou masquerait du vide) + // et avant la premiere annotation (sinon deux flous qui se recouvrent + // s'echantillonnent l'un l'autre). Une passe de rendu ne peut pas lire sa + // propre cible, d'ou la copie -- et d'ou le fait que les annotations + // doivent avoir leur propre passe. + if needs_ann_copy { + self.generate_ann_mips(&mut encoder); + } + // Passe 3 : annotations puis curseur net, par-dessus tout le reste. Elle + // existe meme sans flou : deux passes consecutives sur la MEME cible avec + // `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU + // desktop, et un seul chemin de code vaut mieux qu'un branchement qui ne + // serait exerce que dans un projet sur dix. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("ann-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + for a in &ann_draws { + rpass.set_bind_group(0, &a.bind, &[]); + rpass.draw(0..4, 0..1); + } + // Curseur en dernier : au-dessus de l'ecran et des annotations. + // Une seule copie = curseur net, il tient dans cette pass. La + // trainee, elle, a besoin de sa propre cible (voir plus bas). + if let Some(c) = cursor_draw.as_ref().filter(|c| c.binds.len() == 1) { + rpass.set_bind_group(0, &c.binds[0], &[]); + rpass.draw(0..4, 0..1); + } + } + // TRAINEE DU CURSEUR : flou REEL, pas des copies discretes. + // + // Les N echantillons s'accumulent dans une cible ISOLEE partie de zero, + // puis sont compositees « over » sur la scene. Les additionner + // directement sur le RT reviendrait a AJOUTER la couleur du curseur + // (souvent du blanc) a ce qui est deja dessous : sur un fond clair, deja + // proche du blanc, ajouter du blanc*(1/taps) ne change presque rien -- + // curseur quasi invisible. Dans une cible a part la somme reste + // correctement normalisee (alpha ~1 la ou les copies se recouvrent), et + // la composition finale est un « over » ordinaire, correct quel que soit + // le fond. Meme raisonnement, mot pour mot, cote macOS et Windows. + if let (Some(c), Some((_ubuf, abind))) = ( + cursor_draw.as_ref().filter(|c| c.binds.len() > 1), + accum_bind.as_ref(), + ) { + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-accum-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.accum_view, + resolve_target: None, + ops: wgpu::Operations { + // Le clear EST la raison d'etre de cette cible : elle + // doit partir vide a chaque frame, pas cumuler. + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline_add); + let w = 1.0 / c.binds.len() as f64; + rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); + for bind in &c.binds { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } + } + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-accum-composite"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, abind, &[]); + rpass.draw(0..3, 0..1); + } + self.gpu.context.submit(std::iter::once(encoder.finish())); + Ok(()) + } + + /// Clear le RT a la couleur de fond (ecran absent). + fn clear_rt(&self) -> Result<()> { + let bg = self.live_params.borrow().bg_color; + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("clear"), + }); + encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("clear-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg[0] as f64, + g: bg[1] as f64, + b: bg[2] as f64, + a: bg[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + self.gpu.context.submit(std::iter::once(encoder.finish())); + Ok(()) + } + + fn dummy_view(&self) -> wgpu::TextureView { + let t = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("dummy"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + t.create_view(&wgpu::TextureViewDescriptor::default()) + } + + /// Regle la profondeur de la ring de staging. A appeler AVANT la premiere + /// relecture (elle vide la ring, donc toute frame encore en vol serait + /// perdue -- d'ou le drain explicite plutot qu'un silence). + /// + /// POLITIQUE PAR CHEMIN, et c'est volontaire : + /// + /// - **Export** (`pipeline_linux::run_composited_multi`) : profondeur 2. Il + /// ne veut que du DEBIT, la latence d'une frame ne se voit nulle part + /// puisque la sortie est un fichier. Il draine la ring a la fin, donc + /// aucune frame ne manque au montage. + /// - **Preview live** (`live.rs`) : profondeur 1, inchangee. Une frame de + /// retard y est perceptible -- le canvas afficherait l'avant-derniere + /// frame composee, et surtout la boucle ne relit QUE quand elle a avance + /// (`stepped`) : au repos (fin d'un scrub, pause) la derniere frame + /// resterait coincee dans la ring et le canvas figerait sur la + /// precedente jusqu'au prochain evenement. Le pipeline demanderait donc + /// un drain sur inactivite pour n'etre que neutre visuellement, pour un + /// gain qui n'est pas le goulot mesure ici. On ne l'impose pas. + /// + /// A profondeur 1 le chemin est exactement l'ancien : soumettre, attendre, + /// mapper, depadder. + pub fn set_readback_depth(&self, depth: usize) -> Result<()> { + let depth = depth.max(1); + // Draine d'abord : les frames en vol appartiennent a l'appelant + // precedent, les jeter en silence serait une perte de donnees muette. + while unsafe { self.readback_take()? }.is_some() {} + let mut ring = self.readback.borrow_mut(); + ring.depth = depth; + while ring.free.len() > depth { + ring.free.pop(); + } + while ring.free.len() < depth { + let buf = Self::make_staging(&self.gpu, self.readback_bpr, self.render_h); + ring.free.push(buf); + } + Ok(()) + } + + /// Soumet la copie RT -> staging de la frame COURANTE sans l'attendre, puis + /// rend la frame la plus ancienne encore en vol des que la ring est pleine. + /// + /// PREMIERES FRAMES. Tant que moins de `depth` copies sont en vol, il n'y a + /// rien a rendre et la reponse est `Ok(None)` : c'est l'amorcage du + /// pipeline, et il coute exactement `depth - 1` frames de decalage (0 a + /// profondeur 1). L'appelant ne doit donc PAS supposer une frame par appel, + /// mais drainer a la fin (`readback_take`) -- sinon les `depth - 1` + /// dernieres frames composees ne sortiraient jamais. + pub unsafe fn readback_submit(&self) -> Result)>> { + let (w, h) = (self.render_w, self.render_h); + let bpr = self.readback_bpr; + // Invariant : cette fonction recolte toujours des que `pending` atteint + // `depth`, donc un buffer est libre a chaque entree. Un echec ici + // signalerait une ring desynchronisee -- on le dit plutot que d'allouer + // 8 Mo de plus en silence a chaque frame. + let buf = self + .readback + .borrow_mut() + .free + .pop() + .ok_or_else(|| anyhow::anyhow!("staging ring saturee (aucun buffer libre)"))?; + + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("readback"), + }); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.rt, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &buf, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bpr), + rows_per_image: Some(h), + }, + }, + wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + ); + // `submit` rend l'index de soumission : c'est LUI qui permet plus tard + // de n'attendre que cette copie-ci, au lieu de `Maintain::Wait` qui + // draine toute la file (donc la composition qui suit). + let idx = self.gpu.context.submit(std::iter::once(encoder.finish())); + // `map_async` juste apres la soumission : wgpu differe le mapping + // jusqu'a la fin de la soumission qui ecrit le buffer, le callback + // n'est tire que par un `poll`. + let (tx, rx) = std::sync::mpsc::channel(); + buf.slice(..).map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + { + let mut ring = self.readback.borrow_mut(); + ring.pending.push_back(PendingCopy { buf, idx, rx, w, h, bpr }); + if ring.pending.len() < ring.depth { + return Ok(None); // amorcage + } + } + self.readback_take() + } + + /// Recolte la frame la plus ancienne en vol (`None` si la ring est vide). + /// C'est le drain de fin de session : l'appeler en boucle apres la derniere + /// `readback_submit` rend les `depth - 1` frames encore en vol. + pub unsafe fn readback_take(&self) -> Result)>> { + let Some(p) = self.readback.borrow_mut().pending.pop_front() else { + return Ok(None); + }; + // N'attend QUE la soumission de cette copie. A profondeur >= 2 elle est + // terminee depuis longtemps (l'encodage de la frame precedente lui a + // laisse ~19 ms de CPU) et l'appel rend la main immediatement. + self.gpu.device.poll(wgpu::Maintain::WaitForSubmissionIndex(p.idx)); + p.rx + .recv() + .map_err(|_| anyhow::anyhow!("map_async channel"))? + .map_err(|e| anyhow::anyhow!("map_async: {e:?}"))?; + let slice = p.buf.slice(..); + let mapped = slice.get_mapped_range(); + + let (w, h) = (p.w, p.h); + let row = (w * 4) as usize; + let bpr = p.bpr as usize; + let total = row * h as usize; + + // `Vec::with_capacity` + `extend_from_slice`, PAS `vec![0u8; total]` : ce dernier + // memset 8 Mo (en 1080p) qu'on écrase intégralement ligne suivante. Mesuré : la + // relecture pèse 82 % de la frame de preview, et ce zero-fill en est une part + // gratuite à rendre. + let mut out = Vec::with_capacity(total); + if bpr == row { + // Cas courant, et il n'a rien d'exotique : wgpu aligne `bytes_per_row` sur 256 + // et une largeur RGBA multiple de 64 px l'est déjà (1280 et 1920 le sont). + // Il n'y a alors AUCUN padding à retirer, et la boucle ligne à ligne recopiait + // un tampon identique à l'octet près en `h` memcpy au lieu d'un seul. + out.extend_from_slice(&mapped[..total]); + } else { + for y in 0..h as usize { + out.extend_from_slice(&mapped[y * bpr..y * bpr + row]); + } + } + drop(mapped); + p.buf.unmap(); + // Buffer demappe -> reutilisable au prochain `readback_submit`. + self.readback.borrow_mut().free.push(p.buf); + Ok(Some((w, h, out))) + } + + /// Lit le RT en RGBA8 tightly-packed `(render_w * render_h * 4)`. Depadde le + /// `bytes_per_row` aligne a 256 exige par wgpu. + /// + /// Contrat SYNCHRONE : rend la frame que le RT contient MAINTENANT. A la + /// profondeur par defaut (1) c'est litteralement soumettre-attendre-mapper, + /// donc le chemin d'avant la ring. A profondeur > 1 elle vide le pipeline + /// pour honorer ce contrat -- a n'utiliser que la ou la frame courante est + /// exigee (preview, GIF, tests), pas dans une boucle d'export. + pub unsafe fn readback_direct(&self) -> Result<(u32, u32, Vec)> { + let mut last = self.readback_submit()?; + while let Some(next) = self.readback_take()? { + last = Some(next); + } + last.ok_or_else(|| anyhow::anyhow!("readback_direct: aucune frame recoltee")) + } +} diff --git a/crates/compositor/src/d3d_linux.rs b/crates/compositor/src/d3d_linux.rs new file mode 100644 index 0000000000..67a701c18e --- /dev/null +++ b/crates/compositor/src/d3d_linux.rs @@ -0,0 +1,163 @@ +//! Backend GPU Linux -- wgpu (Vulkan). +//! +//! Equivalent Linux de `d3d_windows.rs` / `d3d_macos.rs` : meme surface publique +//! (`Backend`, `Gpu`, `create`, `create_backend`, `create_auto`, `probe`, +//! `diagnose`) pour que `pipeline`, `live.rs` et `compositor-view-napi` +//! l'utilisent sans connaitre la plateforme (cf. `lib.rs`, qui re-exporte +//! `crate::d3d` vers `d3d_linux` sous `cfg(target_os = "linux")`). +//! +//! # `Backend::Cpu` sur Linux +//! +//! Contrairement a macOS (ou Metal n'a pas de rasteriseur logiciel), Linux EN A +//! un : Mesa **lavapipe** (`llvmpipe`), le pendant Vulkan de WARP. `probe()` le +//! classe donc en `Backend::Cpu` (le meme repli que WARP cote Windows : notice +//! dans la preview, warning a l'export), et un vrai GPU (RADV, dzn, NVK...) en +//! `Backend::Hardware`. + +use anyhow::{Context, Result}; +use std::sync::OnceLock; + +/// Qui execute le pipeline (symetrie d'API avec `d3d_windows::Backend`). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Backend { + /// Vrai GPU (RADV / dzn / NVK / ...) via Vulkan. + Hardware, + /// Mesa lavapipe (`llvmpipe`), rasteriseur logiciel Vulkan. + Cpu, +} + +/// Handle GPU Linux : `wgpu::Device` + `wgpu::Queue` (Arc internes cote wgpu, +/// `.clone()` bon marche). Les champs `device`/`context`/`backend`/ +/// `feature_level` sont alignes sur `d3d_windows::Gpu` / `d3d_macos::Gpu` pour +/// que `live.rs::Player` copie la struct champ par champ sans cfg-fendre le +/// constructeur. +pub struct Gpu { + pub device: wgpu::Device, + /// Pendant de `ID3D11DeviceContext` (D3D11) / `MTLCommandQueue` (Metal) : + /// la file de soumission wgpu. + pub context: wgpu::Queue, + pub backend: Backend, + /// Pas d'equivalent `D3D_FEATURE_LEVEL` en wgpu ; conserve a 0 pour la + /// symetrie d'API (les diagnostics futurs pourront le renseigner). + pub feature_level: u64, +} + +/// `probe()` -- propriete de la machine, mise en cache (la preview et la modale +/// d'export en ont besoin toutes les deux). `None` si aucun adaptateur wgpu +/// (headless sans lavapipe, kernel sans DRM ni ICD logiciel). +static PROBE: OnceLock> = OnceLock::new(); + +pub fn probe() -> Option { + *PROBE.get_or_init(|| create_backend(Backend::Hardware).ok().map(|g| g.backend)) +} + +/// Cree un device wgpu (Vulkan). `_backend` est indicatif : on prend le meilleur +/// adaptateur disponible (HighPerformance) et on reporte son type REEL via +/// `classify` (lavapipe -> `Cpu`, sinon `Hardware`) -- pas de chemin de rendu +/// distinct entre les deux cote Linux, seul le libelle change. +pub fn create_backend(_backend: Backend) -> Result { + pollster::block_on(create_async()) +} + +async fn create_async() -> Result { + let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor { + backends: wgpu::Backends::all(), + ..Default::default() + }); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::HighPerformance, + ..Default::default() + }) + .await + .context("aucun adaptateur graphique compatible")?; + let info = adapter.get_info(); + let backend = classify(&info); + let (device, queue) = adapter + .request_device( + &wgpu::DeviceDescriptor { + label: Some("openscreen-linux"), + required_features: wgpu::Features::empty(), + required_limits: wgpu::Limits::default(), + memory_hints: wgpu::MemoryHints::default(), + }, + None, + ) + .await + .context("request_device a echoue")?; + Ok(Gpu { + device, + context: queue, + backend, + feature_level: 0, + }) +} + +/// lavapipe expose "llvmpipe" dans le nom d'adaptateur -- c'est l'equivalent +/// Vulkan de WARP, a ranger sous `Cpu`. +fn classify(info: &wgpu::AdapterInfo) -> Backend { + let n = info.name.to_ascii_lowercase(); + if n.contains("llvmpipe") || n.contains("lavapipe") { + Backend::Cpu + } else { + Backend::Hardware + } +} + +impl Gpu { + /// Chemin de production. Symetrie d'API avec `d3d_windows::Gpu::create_auto` ; + /// `_debug` est le pendant de la couche de debug D3D11 (rien a faire ici, + /// wgpu a `WGPU_VALIDATION` en variable d'env). + pub fn create_auto(_debug: bool) -> Result { + create_backend(Backend::Hardware) + } + + /// Creation hardware-strict (tests et goldens). + pub fn create(_debug: bool) -> Result { + create_backend(Backend::Hardware) + } + + /// Le backend de cette machine, mis en cache. Expose comme fonction ASSOCIEE + /// (`Gpu::probe()`) parce que `compositor-view-napi` l'appelle ainsi. + pub fn probe() -> Option { + probe() + } +} + +/// Message d'echec actionnable (symetrie d'API avec `d3d_windows::diagnose`). +pub fn diagnose(err: &anyhow::Error) -> String { + format!("{err:#}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_cpu_pour_lavapipe() { + let info = wgpu::AdapterInfo { + name: "llvmpipe (LLVM 21.1.8, 256 bits)".into(), + vendor: 0x10005, + device: 0, + device_type: wgpu::DeviceType::Cpu, + driver: "llvmpipe".into(), + driver_info: String::new(), + backend: wgpu::Backend::Vulkan, + }; + assert_eq!(classify(&info), Backend::Cpu); + } + + #[test] + fn classify_hardware_pour_gpu_reel() { + let info = wgpu::AdapterInfo { + name: "Microsoft Direct3D12 (AMD Radeon(TM) Graphics)".into(), + vendor: 0x1002, + device: 0, + device_type: wgpu::DeviceType::IntegratedGpu, + driver: "Dozen".into(), + driver_info: String::new(), + backend: wgpu::Backend::Vulkan, + }; + assert_eq!(classify(&info), Backend::Hardware); + } +} diff --git a/crates/compositor/src/ffi.rs b/crates/compositor/src/ffi.rs index 6e2b530adf..0df8963a4a 100644 --- a/crates/compositor/src/ffi.rs +++ b/crates/compositor/src/ffi.rs @@ -31,6 +31,9 @@ pub const AVERROR_EAGAIN: i32 = -35; pub const AVERROR_EAGAIN: i32 = -11; /// `AVERROR_EOF` = `-MKTAG('E','O','F',' ')`. pub const AVERROR_EOF: i32 = -541478725; +/// `AVERROR_INVALIDDATA` = `-MKTAG('I','N','D','A')`. Contrairement à `AVERROR_EAGAIN`, +/// c'est un FFERRTAG et pas un errno, donc la valeur est la même sur toutes les cibles. +pub const AVERROR_INVALIDDATA: i32 = -1094995529; /// `AVSEEK_FLAG_BACKWARD` — chercher la keyframe <= ts. pub const AVSEEK_FLAG_BACKWARD: i32 = 1; diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index 79ea40c73f..d8972e7d1c 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -34,6 +34,11 @@ pub mod ffi; pub mod frame_geometry; pub mod gif_export; pub mod regions; +// Multiplateforme à dessein : n'utilise que libavformat (liée sur les trois +// cibles) et le shim C. Seul Linux l'appelle aujourd'hui, parce que c'est la +// seule plateforme dont la capture passe par `MediaRecorder`, mais rien dedans +// n'est spécifique à Linux. +pub mod remux; pub mod scene; pub mod text_anim; pub mod text_plate; @@ -52,6 +57,11 @@ pub mod d3d_macos; #[cfg(target_os = "macos")] pub use d3d_macos as d3d; +#[cfg(target_os = "linux")] +pub mod d3d_linux; +#[cfg(target_os = "linux")] +pub use d3d_linux as d3d; + // Source de frames du backend « CPU-like » : Windows → cpu_frames_windows (WARP + swscale), // macOS → mac_frames (logiciel → CVPixelBuffer). Ré-exporté sous le nom `cpu_frames` (privé). #[cfg(windows)] @@ -64,6 +74,13 @@ mod mac_frames; #[cfg(target_os = "macos")] use mac_frames as cpu_frames; +#[cfg(target_os = "linux")] +mod linux_frames; +#[cfg(target_os = "linux")] +use linux_frames as cpu_frames; +#[cfg(target_os = "linux")] +mod linux_decode; + // Moteur de composition + rastériseur de texte + pipeline : un fichier par plateforme. // Le pipeline est gardé séparé (pas de fusion comme live) parce que la ffmpeg-side // diffère entre D3D11VA et VideoToolbox : les types `AVD3D11VADeviceContext` vs @@ -84,6 +101,13 @@ pub mod pipeline_macos; #[cfg(target_os = "macos")] pub mod text_macos; +#[cfg(target_os = "linux")] +pub mod text_linux; +#[cfg(target_os = "linux")] +pub mod compositor_linux; +#[cfg(target_os = "linux")] +pub mod pipeline_linux; + #[cfg(windows)] pub use compositor_windows as compositor; #[cfg(windows)] @@ -98,6 +122,13 @@ pub use pipeline_macos as pipeline; #[cfg(target_os = "macos")] pub use text_macos as text; +#[cfg(target_os = "linux")] +pub use text_linux as text; +#[cfg(target_os = "linux")] +pub use compositor_linux as compositor; +#[cfg(target_os = "linux")] +pub use pipeline_linux as pipeline; + // `live.rs` est resté un fichier unique parce que sa machinerie principale (Player, // LiveView, render_thread) est entièrement cross-platform : elle ne touche qu'au // Compositor (cfg-ré-exporté) et au ffmpeg `Decoder` (portable). Seules les diff --git a/crates/compositor/src/linux_decode.rs b/crates/compositor/src/linux_decode.rs new file mode 100644 index 0000000000..e8bb81bc52 --- /dev/null +++ b/crates/compositor/src/linux_decode.rs @@ -0,0 +1,459 @@ +//! Décodeur logiciel ffmpeg — utilisé par la tranche verticale `vk_render` +//! pour ouvrir un MP4 fixture et en extraire la `n`-ième frame en mémoire +//! système, sans aucune dépendance à `D3D11VA`. Côté production, ce sera +//! `pipeline::Decoder::open` côté Windows (qui route par D3D11VA quand FL 11_1 +//! + vidéo disponible, par `vk_frames::VkFrames` sinon) ; ici on isole le +//! chemin « software decode + `vk_frames::present` » pour le démontrer sans +//! toucher `pipeline.rs` (cf. spec §3.4 — `pipeline.rs` est dans WP6). +//! +//! **Sécurité lifetime.** L'`AVFrame` retourné est alloué par `av_frame_alloc` +//! et libéré par `av_frame_free` — le caller doit soit appeler `free_frame()` +//! soit (mieux) laisser `vk_frames::VkFrames::present` la consommer puis +//! réécrire la prochaine. Garder une frame au-delà du prochain `decode_n` +//! libère l'ancienne, exactement comme `cpu_frames::present` côté #162. + +use anyhow::{bail, Context, Result}; +use std::ffi::CString; +use std::ptr; + +use crate::ffi::{ + av_frame_alloc, av_frame_free, av_frame_move_ref, av_frame_unref, av_packet_alloc, + av_packet_free, av_packet_unref, av_read_frame, av_seek_frame, avcodec_alloc_context3, + avcodec_find_decoder, avcodec_flush_buffers, avcodec_free_context, avcodec_open2, + avcodec_parameters_to_context, avcodec_receive_frame, avcodec_send_packet, + avformat_close_input, avformat_find_stream_info, avformat_open_input, AVCodecContext, + AVFormatContext, AVFrame, AVMediaType, AVPacket, AVStream, AVERROR_EAGAIN, AVERROR_EOF, + AVERROR_INVALIDDATA, AVSEEK_FLAG_BACKWARD, +}; + +/// `sn_fmt_stream` est défini dans `crates/compositor/shim.c` — bindgen ne le voit pas +/// (shim.c est compilé séparément par `cc::Build`). On le déclare ici en `extern "C"` +/// comme `pipeline.rs` le fait. La même convention apparaît à plusieurs endroits du +/// crate pour tous les accesseurs du shim. +extern "C" { + fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; +} + +/// `SEEK_SET` constant — la position de seek `av_seek_frame` interprète +/// `timestamp` comme un timestamp absolu (AV_TIME_BASE = microsecondes). +const SEEK_SET: i32 = 0; + +/// Cherche la vidéo du fichier, ouvre le décodeur, et rend un état prêt à +/// décoder. La struct expose `decode_at(frame_idx)` qui seek + décode jusqu'à +/// la frame `frame_idx` (0-indexée depuis le début du flux). +/// +/// Pub (pas `pub(crate)`) parce que `crates/compositor/tests/vk_cross_golden.rs` +/// est un crate externe vis-à-vis de la lib ; le test pilote la tranche. +pub struct SwDecoder { + fmt: *mut AVFormatContext, + dec: *mut AVCodecContext, + stream_idx: i32, + /// Timebase du flux vidéo (en secondes par tick). Permet de convertir un + /// `frame_idx` en timestamp de seek. + stream_timebase: f64, + /// Cadence reelle du flux (avg_frame_rate), PAS 1/time_base. + fps: f64, + /// Packet/frame persistants du pompage SEQUENTIEL (`next_frame`). + pkt: *mut AVPacket, + frame: *mut AVFrame, + sent_eof: bool, + cur_pts: Option, +} + +/// Libère toutes les ressources ffmpeg. `Drop` ne peut pas faillir ; on +/// panique sur une erreur double-free improbable (les handles sont nullifiés +/// après libération, un deuxième `Drop` les trouve à null et n'agit pas). +impl Drop for SwDecoder { + fn drop(&mut self) { + unsafe { + if !self.dec.is_null() { + avcodec_free_context(&mut self.dec); + } + if !self.fmt.is_null() { + avformat_close_input(&mut self.fmt); + } + if !self.frame.is_null() { + av_frame_free(&mut self.frame); + } + if !self.pkt.is_null() { + av_packet_free(&mut self.pkt); + } + } + } +} + +impl SwDecoder { + pub fn open(path: &str) -> Result { + unsafe { Self::open_inner(path) } + } + + unsafe fn open_inner(path: &str) -> Result { + let path_c = CString::new(path).context("chemin NUL inattendu")?; + let mut fmt: *mut AVFormatContext = ptr::null_mut(); + let r = avformat_open_input(&mut fmt, path_c.as_ptr(), ptr::null(), ptr::null_mut()); + if r < 0 { + bail!("avformat_open_input({path}) a échoué: {r}"); + } + if fmt.is_null() { + bail!("avformat_open_input({path}) a rendu un fmt null"); + } + let r = avformat_find_stream_info(fmt, ptr::null_mut()); + if r < 0 { + avformat_close_input(&mut fmt); + bail!("avformat_find_stream_info({path}) a échoué: {r}"); + } + // Trouver le premier flux vidéo. `av_find_best_stream` fait ça 1.0. + let stream_idx = crate::ffi::av_find_best_stream( + fmt, + AVMediaType::AVMEDIA_TYPE_VIDEO, + -1, + -1, + ptr::null_mut(), + 0, + ); + if stream_idx < 0 { + avformat_close_input(&mut fmt); + bail!("av_find_best_stream n'a pas trouvé de flux vidéo dans {path}: {stream_idx}"); + } + // Codec params → context → open. AVFormatContext est opaque : `sn_fmt_stream` + // (du `shim.c`) extrait `streams[i]` ; AVStream ne l'est pas, on lit son + // `codecpar` directement. Cf. `pipeline.rs` pour la convention. + let stream = sn_fmt_stream(fmt, stream_idx); + let mut dec = avcodec_alloc_context3(ptr::null()); + if dec.is_null() { + avformat_close_input(&mut fmt); + bail!("avcodec_alloc_context3 a échoué"); + } + let par = (*stream).codecpar; + let r = avcodec_parameters_to_context(dec, par); + if r < 0 { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!("avcodec_parameters_to_context: {r}"); + } + let codec = avcodec_find_decoder((*par).codec_id); + if codec.is_null() { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!( + "avcodec_find_decoder n'a pas trouvé de décodeur pour codec_id {}", + (*par).codec_id + ); + } + // Threads de decodage : 0 = « autant que de coeurs », exactement ce que + // `pipeline_windows.rs:526` et `pipeline_macos.rs:178` posent sur leur + // decodeur. Sans ca ffmpeg reste a 1 thread. + (*dec).thread_count = 0; + let r = avcodec_open2(dec, codec, ptr::null_mut()); + if r < 0 { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!("avcodec_open2: {r}"); + } + // Timebase du flux vidéo — `AVRational { num, den }`. ffmpeg utilise `num` ticks + // par `den` secondes. Le wrapper bindgen expose les deux champs en i32. + let stream_timebase = { + let num = (*stream).time_base.num as f64; + let den = (*stream).time_base.den as f64; + if den == 0.0 { + 1.0 / 60.0 // fallback : suppose 60 fps + } else { + num / den + } + }; + // fps reel du flux : avg_frame_rate d'abord, r_frame_rate en secours, + // 60 en dernier recours. PAS 1/time_base (le time_base est le timescale + // du conteneur, souvent 15360, pas la cadence). + let fps = { + let a = (*stream).avg_frame_rate; + let r = (*stream).r_frame_rate; + if a.num > 0 && a.den > 0 { + a.num as f64 / a.den as f64 + } else if r.num > 0 && r.den > 0 { + r.num as f64 / r.den as f64 + } else { + 60.0 + } + }; + let pkt = av_packet_alloc(); + let frame = av_frame_alloc(); + if pkt.is_null() || frame.is_null() { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!("av_packet_alloc/av_frame_alloc (pompage sequentiel)"); + } + Ok(SwDecoder { + fmt, + dec, + stream_idx, + stream_timebase, + fps, + pkt, + frame, + sent_eof: false, + cur_pts: None, + }) + } + + /// Rend la frame SUIVANTE du flux, valide jusqu'au prochain appel, ou null a + /// EOF. C'est le pompage `receive_frame`/`read_frame`/`send_packet` classique, + /// identique a `pipeline_windows::Decoder::next` et `pipeline_macos`. Il ne + /// seek PAS : le decodeur garde son etat, donc une lecture sequentielle coute + /// UN packet par frame au lieu d'un re-parcours de demi-GOP. + pub unsafe fn next_frame(&mut self) -> Result<*mut AVFrame> { + loop { + let r = avcodec_receive_frame(self.dec, self.frame); + if r == 0 { + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + return Ok(self.frame); + } + if r == AVERROR_EOF { + return Ok(ptr::null_mut()); + } + if r != AVERROR_EAGAIN { + bail!("avcodec_receive_frame: {r}"); + } + if self.sent_eof { + return Ok(ptr::null_mut()); + } + let rr = av_read_frame(self.fmt, self.pkt); + if rr < 0 { + // EOF (ou erreur de lecture) : on draine l'encodeur interne. + avcodec_send_packet(self.dec, ptr::null_mut()); + self.sent_eof = true; + } else { + if (*self.pkt).stream_index == self.stream_idx { + let sr = avcodec_send_packet(self.dec, self.pkt); + // AVERROR_INVALIDDATA : packet mal aligne apres un seek, on saute. + // La valeur etait ecrite en dur a -0x2A2A2A2A, soit le tag `****`, + // qui ne designe aucune erreur ffmpeg : le garde ne matchait donc + // jamais et une vraie donnee invalide avortait tout le decodage -- + // exactement le cas du scrub, qui seeke en permanence. + if sr < 0 && sr != AVERROR_INVALIDDATA && sr != AVERROR_EAGAIN { + av_packet_unref(self.pkt); + bail!("avcodec_send_packet: {sr}"); + } + } + av_packet_unref(self.pkt); + } + } + } + + /// Temps source (secondes) de la derniere frame rendue par `next_frame` / + /// `decode_at`, tire du pts REEL et non d'un compteur d'index. + pub fn cur_time_sec(&self) -> Option { + self.cur_pts.map(|pts| pts as f64 * self.stream_timebase) + } + + /// Seek vers la keyframe la plus proche AVANT `frame_idx`, puis décode + /// jusqu'à atteindre la frame demandée. Le seek est résolu par + /// `av_seek_frame` avec `SEEK_SET | BACKWARD` (cherche le keyframe + /// précédent le timestamp demandé). Renvoie une `AVFrame` allouée par + /// `av_frame_alloc` que le caller doit libérer via `free_frame` — + /// ou laisser `vk_frames::VkFrames::present` consommer (qui réécrit + /// `present` avec son carrier, l'ancienne frame devient inaccessible). + /// + /// **Robustesse.** Pour la tranche verticale (`crates/fixture/screen.mp4` + /// qui est un `-c copy` d'un fragment de recording), `av_seek_frame` peut + /// renvoyer un packet dont la première lecture NAL est mal alignée (le + /// moov de la source est en queue, le parser fait de son mieux mais le + /// premier packet après un BACKWARD seek contient parfois un NAL + /// fragmenté). On skippe ces packets avec `send_packet` qui renvoie + /// `AVERROR_INVALIDDATA` plutôt que de paniquer : la prochaine itération + /// lira le packet complet suivant. + pub unsafe fn decode_at(&mut self, frame_idx: u32) -> Result<*mut AVFrame> { + let fps = self.fps; + let target_ts = (frame_idx as f64 / fps) * 1_000_000.0; // AV_TIME_BASE = µs + // `AVSEEK_FLAG_BACKWARD` vaut 1, pas 4 — 4 est `AVSEEK_FLAG_ANY`. La constante + // était écrite en dur à 4 avec un commentaire affirmant le contraire, et c'est + // le seul seek du crate à ne pas passer par `ffi::AVSEEK_FLAG_BACKWARD` (cf. + // pipeline_windows.rs, pipeline_macos.rs, audio.rs). + // + // Sans BACKWARD, ffmpeg se cale sur la première position indexée AU NIVEAU OU + // APRÈS la cible, au lieu de la keyframe qui la précède. La boucle d'avance + // ci-dessous s'arrête dès que `pts >= target`, condition alors satisfaite par la + // toute première frame décodée : elle ne fait plus rien et `decode_at` rend la + // keyframe SUIVANTE. L'erreur est d'un GOP entier. + // + // D'où le symptôme asymétrique signalé : l'écran porte une keyframe toutes les + // ~1,78 s, la webcam toutes les ~6,73 s, donc l'écart y est ~4x plus grand. Et + // comme `live::Player::step` rattrape la webcam par une boucle monotone vers + // l'avant, une fois garée dans le futur elle ne revient jamais — elle fige. + let seek_flags = SEEK_SET | AVSEEK_FLAG_BACKWARD; + let r = av_seek_frame(self.fmt, -1, target_ts as i64, seek_flags); + if r < 0 { + // Repli : rembobiner au début et balayer en avant. + // + // Un WebM de `MediaRecorder` n'a NI Cues NI SeekHead — il est écrit en flux + // et personne ne revient poser l'index —, donc tout seek vers un timestamp + // arbitraire échoue. C'est le cas de tout enregistrement Linux tant qu'il + // n'existe pas de helper de capture natif : la capture y passe par + // getDisplayMedia/MediaRecorder, là où Windows et macOS ont des helpers qui + // écrivent des fichiers indexés. + // + // Rembobiner à 0 reste possible sans index (c'est le début du fichier), et + // la boucle ci-dessous sait déjà avancer jusqu'à `target_ts`. Le coût est + // linéaire, ce qui n'est acceptable que depuis le pompage séquentiel : le + // décodage mesure ~0,07 ms/frame, donc rejoindre la seconde 14 d'un + // enregistrement coûte quelques dizaines de ms au lieu d'échouer. + let rewound = av_seek_frame(self.fmt, -1, 0, seek_flags); + if rewound < 0 { + bail!( + "av_seek_frame(ts={target_ts:.0} µs) a échoué: {r}, et le rembobinage \ + aussi: {rewound}" + ); + } + } + // Flush le décodeur — sans ça, le seek laisse l'état interne avec les + // frames de l'ancien GOP, et la première `receive_frame` peut être + // une frame d'avant le seek. + avcodec_flush_buffers(self.dec); + // Le seek rouvre le flux : le drapeau EOF du pompage sequentiel retombe. + self.sent_eof = false; + + let mut pkt: *mut crate::ffi::AVPacket = ptr::null_mut(); + let mut frame: *mut AVFrame = ptr::null_mut(); + let mut found: *mut AVFrame = ptr::null_mut(); + + let target_ts_seconds = target_ts / 1_000_000.0; + let mut invalid_skips = 0u32; + 'outer: loop { + pkt = av_packet_alloc(); + if pkt.is_null() { + bail!("av_packet_alloc en boucle"); + } + let r = av_read_frame(self.fmt, pkt); + if r < 0 { + // EOF ou erreur : on a épuisé le fichier sans atteindre la cible. + av_packet_free(&mut pkt); + break 'outer; + } + if (*pkt).stream_index != self.stream_idx { + // Pas un packet vidéo — on le jette et on continue. + av_packet_free(&mut pkt); + continue; + } + let send_r = avcodec_send_packet(self.dec, pkt); + av_packet_free(&mut pkt); + if send_r == -0x2A2A2A2A { + // AVERROR_INVALIDDATA — packet mal aligné après un seek. On le + // saute et on continue ; le decodeur attendra un packet propre. + // Valeur ffmpeg = -1094995529 (0xBEEBBEEB), ici écrite comme + // un nombre négatif littéral pour éviter la dépendance `ffi::`. + invalid_skips += 1; + if invalid_skips > 8 { + bail!("plus de 8 packets invalides après seek — fichier ou codec cassé"); + } + continue; + } + if send_r < 0 && send_r != -11 { + bail!("avcodec_send_packet: {send_r}"); + } + frame = av_frame_alloc(); + if frame.is_null() { + bail!("av_frame_alloc en boucle"); + } + loop { + let recv_r = avcodec_receive_frame(self.dec, frame); + if recv_r == 0 { + if found.is_null() { + found = av_frame_alloc(); + if found.is_null() { + bail!("av_frame_alloc pour resultat"); + } + } + // FUITE MÉMOIRE si on l'oublie. `av_frame_move_ref` écrase + // `found` SANS déréférencer ce qu'il contenait — c'est écrit + // noir sur blanc dans libavutil/frame.h : « dst is not + // unreferenced, but directly overwritten without reading or + // deallocating its contents. Call av_frame_unref(dst) + // manually before calling this function to ensure that no + // memory is leaked. » + // + // Cette boucle décode en avant depuis la keyframe jusqu'à la + // cible, donc elle passe ici une fois par frame du GOP. Sans + // ce unref, chaque frame intermédiaire abandonnait ses + // buffers : ~3,1 Mo en 1080p YUV420P, plusieurs dizaines de + // fois par scrub. Symptôme observé : le scrubbing ralentit + // progressivement, puis l'app gèle et meurt. + av_frame_unref(found); + av_frame_move_ref(found, frame); + av_frame_unref(frame); + if (*found).best_effort_timestamp as f64 * self.stream_timebase >= target_ts_seconds { + break 'outer; + } + } else if recv_r == -11 { + break; + } else if recv_r == -541478725 { + // AVERROR_EOF + break 'outer; + } else if recv_r < 0 { + bail!("avcodec_receive_frame: {recv_r}"); + } else { + break; + } + } + av_frame_free(&mut frame); + } + if !frame.is_null() { + av_frame_free(&mut frame); + } + if !pkt.is_null() { + av_packet_free(&mut pkt); + } + if found.is_null() { + bail!("decode_at(frame_idx={frame_idx}) : aucune frame reçue"); + } + let pts = (*found).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + Ok(found) + } + + /// Cadence reelle du flux (images/s). Sert a convertir un temps en secondes + /// vers un index de frame pour le seek du preview Linux. + pub fn fps(&self) -> f64 { + self.fps + } + + /// Duree du flux video en secondes (`stream.duration * time_base`), lue via + /// le shim `sn_fmt_stream` (AVFormatContext opaque en bindgen). `None` si + /// indisponible. Pendant Linux de `pipeline_macos::Decoder::available_duration_sec`. + pub fn duration_sec(&self) -> Option { + unsafe { + let stream = sn_fmt_stream(self.fmt, self.stream_idx); + if stream.is_null() { + return None; + } + let duration = (*stream).duration; + if duration > 0 && self.stream_timebase > 0.0 { + let s = duration as f64 * self.stream_timebase; + if s.is_finite() && s > 0.0 { + return Some(s); + } + } + None + } + } + + /// Libère une frame renvoyée par `decode_at`. + pub unsafe fn free_frame(mut frame: *mut AVFrame) { + av_frame_free(&mut frame); + } +} + +// ---------- tests ---------- + +#[cfg(test)] +mod tests { + use super::*; + + /// Le décodeur ne paniquera pas si le fichier n'existe pas — il renvoie + /// `Err`. C'est ce que le test d'intégration attend pour skipper proprement + /// quand `crates/fixture/screen.mp4` est absent. + #[test] + fn open_sur_chemin_inexistant_renvoie_err() { + let r = SwDecoder::open("Z:/does/not/exist.mp4"); + assert!(r.is_err()); + } +} diff --git a/crates/compositor/src/linux_frames.rs b/crates/compositor/src/linux_frames.rs new file mode 100644 index 0000000000..dcb99bedb0 --- /dev/null +++ b/crates/compositor/src/linux_frames.rs @@ -0,0 +1,350 @@ +//! L'axe DECODAGE du backend « CPU-like » Linux : une frame libavcodec en +//! memoire systeme devient deux textures wgpu NV12-split (Y `R8Unorm`, UV +//! entrelacee `Rg8Unorm`), presentees exactement comme si un decodeur materiel +//! les avait produites. +//! +//! Equivalent Linux de `cpu_frames_windows.rs` / `mac_frames.rs`. Meme contrat +//! de « frame seam » (cf. `mac_frames.rs:12-19`) : `compositor::nv12_srvs()` +//! lit quatre champs de l'AVFrame presentee — +//! - `data[0]` : un carrier `Box` (les deux textures wgpu), +//! opaque cote Rust, relu par `compositor_linux::nv12_srvs`, +//! - `data[1]` : 0 (pas d'array), +//! - `width`/`height` : dimensions visibles. +//! `format` est pose a `AV_PIX_FMT_D3D11` comme sur Windows/macOS : un sentinel +//! « buffer GPU natif dans data[0] », jamais inspecte par ffmpeg dans ce pipeline. +//! +//! # Difference avec D3D11/Metal +//! +//! D3D11 a un format NV12 natif (une texture, deux sous-ressources) ; macOS a +//! le CVPixelBuffer IOSurface (zero-copy via CVMetalTextureCache). Vulkan/wgpu +//! n'a pas de format NV12 portable, donc on le decompose en DEUX textures +//! (Y + UV) uploadees par `write_texture`. Le shader WGSL echantillonne les deux +//! et fait le YUV->RGB (cf. `vk_shaders/layer.wgsl`). + +use anyhow::{bail, Result}; +use std::ptr; + +use crate::d3d::Gpu; +use crate::ffi::{ + av_frame_alloc, av_frame_free, av_frame_get_buffer, av_frame_unref, sws_freeContext, + sws_getContext, sws_scale, AVFrame, AVPixelFormat, SwsContext, +}; + +/// `SWS_POINT` (plus proche voisin) : la conversion se fait a dimensions EGALES, +/// aucun reechantillonnage. Valeur figee par l'ABI de libswscale (bindgen ne +/// genere pas les `SWS_*`, ce sont des macros). +const SWS_POINT: i32 = 0x10; + +/// Une frame decodee presentee au compositor sous forme de deux textures wgpu : +/// plane Y (`R8Unorm`, `w x h`) et plane UV entrelacee (`Rg8Unorm`, +/// `(w/2) x (h/2)`). Equivalent NV12-split de la `ID3D11Texture2D` NV12 (D3D11) +/// / du CVPixelBuffer (macOS). +pub(crate) struct VkFrameTex { + pub y: wgpu::Texture, + pub uv: wgpu::Texture, + pub width: u32, + pub height: u32, +} + +#[inline] +fn pack_carrier(tex: Box) -> *mut u8 { + Box::into_raw(tex) as *mut u8 +} + +#[inline] +unsafe fn unpack_carrier<'a>(p: *const u8) -> &'a VkFrameTex { + debug_assert!(!p.is_null()); + &*(p as *const VkFrameTex) +} + +/// Source de frames du backend « CPU-like » Linux. Meme surface que +/// `mac_frames::CpuFrames` (`new` / `present` / `current`) : `pipeline` garde la +/// meme mecanique. Allocation unique de textures reecrites a chaque frame. +pub(crate) struct CpuFrames { + device: wgpu::Device, + queue: wgpu::Queue, + sws: *mut SwsContext, + /// `(w, h, format source)` du contexte swscale courant. + sws_key: (i32, i32, i32), + /// NV12 en memoire systeme : cible de swscale, source de l'upload. + nv12: *mut AVFrame, + tex: Option>, + tex_dims: (u32, u32), + /// La frame remise au compositor. Ne possede aucun pixel : `data[0]` pointe + /// le carrier `VkFrameTex`. + present: *mut AVFrame, +} + +impl CpuFrames { + pub(crate) fn new(gpu: &Gpu) -> Result { + let present = unsafe { av_frame_alloc() }; + let nv12 = unsafe { av_frame_alloc() }; + if present.is_null() || nv12.is_null() { + bail!("av_frame_alloc (linux_frames)"); + } + Ok(CpuFrames { + device: gpu.device.clone(), + queue: gpu.context.clone(), + sws: ptr::null_mut(), + sws_key: (0, 0, -1), + nv12, + tex: None, + tex_dims: (0, 0), + present, + }) + } + + /// Convertit `src` (sortie decodeur, memoire systeme) en NV12, l'uploade dans + /// les textures wgpu, et rend la frame de presentation dont `data[0]` est un + /// carrier `Box`. Le pointeur reste valide jusqu'au prochain + /// `present()` — meme contrat que `mac_frames::present`. + pub(crate) unsafe fn present(&mut self, src: *mut AVFrame) -> Result<*mut AVFrame> { + if src.is_null() { + bail!("linux_frames::present: frame source nulle"); + } + let w = (*src).width; + let h = (*src).height; + if w <= 0 || h <= 0 { + bail!("frame decodee sans dimensions ({w}x{h})"); + } + self.ensure_sws(w, h, (*src).format)?; + self.ensure_nv12(w, h)?; + self.ensure_textures(w as u32, h as u32)?; + + let converted = sws_scale( + self.sws, + (*src).data.as_ptr() as *const *const u8, + (*src).linesize.as_ptr(), + 0, + h, + (*self.nv12).data.as_ptr(), + (*self.nv12).linesize.as_ptr(), + ); + if converted <= 0 { + bail!("sws_scale a converti {converted} lignes"); + } + + self.upload()?; + self.attach_carrier(w, h)?; + // Contrat lu par le compositor : sentinel + timestamps recopies (sinon la + // timeline se croit a t=0). + (*self.present).format = AVPixelFormat::AV_PIX_FMT_D3D11 as i32; + (*self.present).pts = (*src).pts; + (*self.present).best_effort_timestamp = (*src).best_effort_timestamp; + Ok(self.present) + } + + unsafe fn ensure_sws(&mut self, w: i32, h: i32, src_fmt: i32) -> Result<()> { + let key = (w, h, src_fmt); + if self.sws_key == key && !self.sws.is_null() { + return Ok(()); + } + if !self.sws.is_null() { + sws_freeContext(self.sws); + } + self.sws = sws_getContext( + w, + h, + src_fmt as AVPixelFormat::Type, + w, + h, + AVPixelFormat::AV_PIX_FMT_NV12, + SWS_POINT, + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + ); + if self.sws.is_null() { + bail!("sws_getContext {w}x{h} fmt {src_fmt} -> NV12"); + } + self.sws_key = key; + Ok(()) + } + + unsafe fn ensure_nv12(&mut self, w: i32, h: i32) -> Result<()> { + if (*self.nv12).width == w + && (*self.nv12).height == h + && (*self.nv12).format == AVPixelFormat::AV_PIX_FMT_NV12 as i32 + { + return Ok(()); + } + av_frame_unref(self.nv12); + (*self.nv12).width = w; + (*self.nv12).height = h; + (*self.nv12).format = AVPixelFormat::AV_PIX_FMT_NV12 as i32; + if av_frame_get_buffer(self.nv12, 32) < 0 { + bail!("av_frame_get_buffer NV12 {w}x{h}"); + } + Ok(()) + } + + fn ensure_textures(&mut self, w: u32, h: u32) -> Result<()> { + // NV12 impose des dimensions paires pour le chroma : arrondi au-dessus pour + // les textures, `present.width/height` reste aux dimensions visibles (meme + // ecart texture/visible que l'alignement macrobloc D3D11VA, 1080 -> 1088). + let dims = ((w + 1) & !1, (h + 1) & !1); + if let Some(tex) = &self.tex { + if tex.width == dims.0 && tex.height == dims.1 { + return Ok(()); + } + } + let y = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("nv12-y"), + size: wgpu::Extent3d { + width: dims.0, + height: dims.1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let uv = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("nv12-uv"), + size: wgpu::Extent3d { + width: dims.0 / 2, + height: dims.1 / 2, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rg8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.tex = Some(Box::new(VkFrameTex { + y, + uv, + width: dims.0, + height: dims.1, + })); + self.tex_dims = dims; + Ok(()) + } + + /// Upload du NV12 swscale dans les deux textures wgpu. `linesize[0]/[1]` sont + /// les strides memoire (paddes SIMD par swscale), passes tels quels a + /// `bytes_per_row`. + unsafe fn upload(&mut self) -> Result<()> { + let tex = match self.tex.as_ref() { + Some(t) => t, + None => bail!("upload avant ensure_textures"), + }; + let y_stride = (*self.nv12).linesize[0] as usize; + let uv_stride = (*self.nv12).linesize[1] as usize; + let y_size = y_stride * tex.height as usize; + let uv_size = uv_stride * tex.height.div_ceil(2) as usize; + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex.y, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + std::slice::from_raw_parts((*self.nv12).data[0], y_size), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(y_stride as u32), + rows_per_image: Some(tex.height), + }, + wgpu::Extent3d { + width: tex.width, + height: tex.height, + depth_or_array_layers: 1, + }, + ); + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex.uv, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + std::slice::from_raw_parts((*self.nv12).data[1], uv_size), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(uv_stride as u32), + rows_per_image: Some(tex.height / 2), + }, + wgpu::Extent3d { + width: tex.width / 2, + height: tex.height / 2, + depth_or_array_layers: 1, + }, + ); + Ok(()) + } + + /// Attache le carrier `Box` a `present.data[0]` et fixe les + /// dimensions visibles (avant padding pair). + unsafe fn attach_carrier(&mut self, w: i32, h: i32) -> Result<()> { + let tex = match self.tex.as_ref() { + Some(t) => t, + None => bail!("attach_carrier avant ensure_textures"), + }; + // Libere un carrier precedent eventuel avant de le remplacer. + if !(*self.present).data[0].is_null() { + let _ = Box::from_raw((*self.present).data[0] as *mut VkFrameTex); + } + (*self.present).data[0] = pack_carrier(Box::new(VkFrameTex { + y: tex.y.clone(), + uv: tex.uv.clone(), + width: tex.width, + height: tex.height, + })); + (*self.present).data[1] = ptr::null_mut(); + (*self.present).width = w; + (*self.present).height = h; + Ok(()) + } + + /// La frame de presentation courante (jamais nulle) — symetrie d'API avec + /// `mac_frames::CpuFrames::current`. + pub(crate) fn current(&self) -> *mut AVFrame { + self.present + } +} + +/// Dimensions (texture, padded pair) du carrier `frame.data[0]`. `(1, 1)` si nul. +pub(crate) unsafe fn carrier_dims(frame: *const AVFrame) -> (u32, u32) { + if (*frame).data[0].is_null() { + return (1, 1); + } + let tex = unpack_carrier((*frame).data[0]); + (tex.width, tex.height) +} + +/// Equivalent Linux de `nv12_srvs` : retourne les deux `TextureView` samplables +/// depuis le carrier `frame.data[0]`. Appele par `compositor_linux`. +pub(crate) unsafe fn nv12_planes( + frame: *const AVFrame, +) -> Result<(wgpu::TextureView, wgpu::TextureView)> { + if (*frame).data[0].is_null() { + bail!("nv12_planes: carrier nul dans data[0]"); + } + let tex = unpack_carrier((*frame).data[0]); + Ok(( + tex.y.create_view(&wgpu::TextureViewDescriptor::default()), + tex.uv.create_view(&wgpu::TextureViewDescriptor::default()), + )) +} + +impl Drop for CpuFrames { + fn drop(&mut self) { + unsafe { + if !self.sws.is_null() { + sws_freeContext(self.sws); + } + if !(*self.present).data[0].is_null() { + let _ = Box::from_raw((*self.present).data[0] as *mut VkFrameTex); + (*self.present).data[0] = ptr::null_mut(); + } + av_frame_free(&mut self.present); + av_frame_free(&mut self.nv12); + } + } +} diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 0addf916f1..49199a6999 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -529,6 +529,26 @@ fn same_source_path(a: &str, b: &str) -> bool { a.eq_ignore_ascii_case(b) } +/// True when the active clip really has a camera to draw. +/// +/// TWO ways the app says "no camera", and both must be caught here, because the +/// webcam decoder is opened either way — `open_and_seek_clip` falls back to the +/// SCREEN file when the webcam path won't open, so `wdec` always yields frames. +/// Whether those frames are the camera or a second copy of the screen is decided +/// HERE and nowhere else. +/// +/// - the empty string, which is what `sceneDescription.ts` and +/// `NativeCompositorOverlay` send for an asset with no `cameraTrack`; +/// - the screen's own path, the older convention kept working for scenes that +/// still use it. +/// +/// Missing the empty-string case is what put the screen recording inside the PiP +/// box: `"" != "/…/recording.mp4"`, so the box was drawn, and the decoder behind +/// it was the screen fallback. +fn webcam_is_real(webcam_path: &str, screen_path: &str) -> bool { + !webcam_path.trim().is_empty() && !same_source_path(webcam_path, screen_path) +} + fn scene_clip_matches( clip: &crate::scene::SceneClip, screen_path: &str, @@ -1328,11 +1348,9 @@ unsafe fn render_thread( cfg.bg_blur = ip.bg_blur; cfg.mblur_n = ip.mblur_taps; cfg.cursor = ip.cursor_show; - // TS falls `webcamPath` back to the screen asset's own path when a clip has no real - // camera (so the decoder pipeline always has something valid to open) — if we drew the - // PiP box in that case it would just duplicate the screen video into its own corner. - // `same_source_path` already exists for exactly this comparison (scene/clip matching). - let has_real_webcam = !same_source_path(&active_webcam_path, &active_screen_path); + // A clip with no camera must not draw the PiP box — the decoder behind it is the + // screen video, so drawing it duplicates the recording into its own corner. + let has_real_webcam = webcam_is_real(&active_webcam_path, &active_screen_path); comp.set_live_params(LiveParams { bg_color: ip.bg_color, shadow_scale: ip.shadow_scale, @@ -1778,6 +1796,23 @@ mod tests { assert_eq!(webcam_seek_time(0.5, 1.25), 0.0); } + #[test] + fn a_clip_without_a_camera_has_no_webcam_to_draw() { + // What the app actually sends for an asset with no `cameraTrack`. Treating + // this as a real camera drew the screen recording inside the PiP box, since + // the webcam decoder falls back to the screen file when the path won't open. + assert!(!webcam_is_real("", "/rec/recording-1.mp4")); + assert!(!webcam_is_real(" ", "/rec/recording-1.mp4")); + // The older sentinel: webcam path == screen path. + assert!(!webcam_is_real("/rec/recording-1.mp4", "/rec/recording-1.mp4")); + assert!(!webcam_is_real("/rec/RECORDING-1.mp4", "/rec/recording-1.mp4")); + } + + #[test] + fn a_clip_with_a_camera_draws_it() { + assert!(webcam_is_real("/rec/recording-1-webcam.webm", "/rec/recording-1.mp4")); + } + // --- transport handed to an export and back ------------------------------- // A real `LiveView` needs a D3D device and a decoder; the transport is the only // part an export touches, so these exercise it through `PreviewTransport` alone. diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs new file mode 100644 index 0000000000..2e8af625f9 --- /dev/null +++ b/crates/compositor/src/pipeline_linux.rs @@ -0,0 +1,532 @@ +//! Pipeline Linux (PR #183) : decode software (`linux_decode::SwDecoder`) + +//! upload NV12-split (`linux_frames::CpuFrames`). +//! +//! Equivalent Linux de `pipeline_windows.rs` / `pipeline_macos.rs` : meme +//! surface publique consommee par le code partage (`Decoder`, `ClipSource`, +//! `ExportCodec`, `ExportParams`, `Stats`, `run_composited_multi`). +//! +//! **Export (WP6).** `run_composited_multi` encode + mux un MP4 **vidéo** : +//! encodeur SOFTWARE (`libopenh264` H264 / `libkvazaar` H265 -- les seuls du +//! build LGPL BtbN qui marchent sans device HW, VAAPI/Vulkan-encode = suivi), +//! la frame composée est relue en RGBA (ring de staging à 2, cf. +//! `Compositor::set_readback_depth`) puis convertie +//! YUV420P par `sws_scale`. La marche de timeline est PARTAGÉE +//! (`timeline_walk::walk_composited_timeline`) et le muxer passe par le shim C +//! `sn_fmt_set_pb` (comme Windows/macOS). **L'audio AAC n'est pas encore muxé** +//! (increment suivant : `audio.rs` + `AacEncoder` sont déjà partagés). + +use anyhow::{bail, Result}; +use std::collections::HashMap; +use std::ffi::CString; +use std::ptr; + +use crate::audio::{ + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, +}; +use crate::config::Cfg; +use crate::d3d::Gpu; +use crate::ffi::AVFrame; +use crate::linux_decode::SwDecoder; +use crate::linux_frames::CpuFrames; + +/// `SWS_POINT` (plus proche voisin). Bindgen ne genere pas les `SWS_*` (macros), +/// valeur figee par l'ABI de libswscale -- comme `linux_frames::SWS_POINT`. +const SWS_POINT: i32 = 0x10; + +/// Bilan d'un run d'export. Memes champs que `pipeline_macos::Stats`. +pub struct Stats { + pub frames: u64, + pub wall_s: f64, + pub fps: f64, + pub video_duration_s: f64, +} + +/// Un clip de la timeline. Memes champs que `pipeline_macos::ClipSource`. +pub struct ClipSource { + pub screen: String, + pub webcam: String, + pub source_start_sec: f64, + pub source_end_sec: f64, + pub webcam_offset_sec: f64, + pub has_audio: bool, +} + +/// Codec cible. Memes variantes que `pipeline_macos::ExportCodec`. +#[derive(Clone, Copy, Debug)] +pub enum ExportCodec { + H264, + H265, +} + +/// Params d'export. Memes champs que `pipeline_macos::ExportParams`. +pub struct ExportParams { + pub width: u32, + pub height: u32, + pub fps: Option, + pub codec: ExportCodec, +} + +impl Default for ExportParams { + fn default() -> Self { + Self { + width: 1920, + height: 1080, + fps: None, + codec: ExportCodec::H264, + } + } +} + +/// Decodeur Linux : software decode (`SwDecoder`) + upload NV12-split +/// (`CpuFrames`). Meme surface que `pipeline_macos::Decoder` +/// (`open`/`seek_to`/`next`/`cur_frame`/`cur_time_sec`/`fps`) pour que `live.rs` +/// le pilote sans connaitre la plateforme. +pub struct Decoder { + sw: SwDecoder, + frames: CpuFrames, + cur: *mut AVFrame, + /// Index de la prochaine frame a decoder (sequentiel). + next_idx: u32, + fps: f64, +} + +// SAFETY : les pointeurs FFI n'ont pas d'affinite thread ; le caller uphold la +// regle « un thread a la fois » (idem `pipeline_macos::Decoder`). +unsafe impl Send for Decoder {} + +impl Decoder { + pub fn open(path: &str, gpu: &Gpu) -> Result { + let sw = SwDecoder::open(path)?; + let fps = sw.fps(); + let frames = CpuFrames::new(gpu)?; + Ok(Decoder { + sw, + frames, + cur: ptr::null_mut(), + next_idx: 0, + fps, + }) + } + + /// Decode la frame a `seconds` (seek), la presente en carrier, la retourne. + pub unsafe fn seek_to(&mut self, seconds: f64) -> Result<*mut AVFrame> { + let idx = (seconds.max(0.0) * self.fps).round() as u32; + self.decode_present(idx) + } + + /// Decode la frame SEQUENTIELLE suivante — pompage `next_frame`, PAS de seek. + /// La frame rendue appartient au decodeur (valide jusqu'au prochain appel), + /// donc elle ne se libere pas ici, contrairement au chemin `decode_at`. + pub unsafe fn next(&mut self) -> Result<*mut AVFrame> { + let raw = self.sw.next_frame()?; + if raw.is_null() { + self.cur = ptr::null_mut(); + return Ok(ptr::null_mut()); + } + let carrier = self.frames.present(raw)?; + self.cur = carrier; + self.next_idx = self.next_idx.saturating_add(1); + Ok(carrier) + } + + unsafe fn decode_present(&mut self, idx: u32) -> Result<*mut AVFrame> { + let raw = self.sw.decode_at(idx)?; + let carrier = self.frames.present(raw)?; + SwDecoder::free_frame(raw); + self.cur = carrier; + self.next_idx = idx + 1; + Ok(carrier) + } + + pub unsafe fn cur_frame(&self) -> *mut AVFrame { + self.cur + } + + /// Temps source (secondes) de la frame courante — pts REEL du decodeur, avec + /// repli sur le compteur d'index si le flux ne porte pas de pts. + pub unsafe fn cur_time_sec(&self) -> f64 { + if let Some(t) = self.sw.cur_time_sec() { + return t.max(0.0); + } + if self.next_idx == 0 || self.fps <= 0.0 { + 0.0 + } else { + (self.next_idx as f64 - 1.0) / self.fps + } + } + + pub unsafe fn fps(&self) -> f64 { + self.fps + } + + /// Duree du flux (secondes). Pendant de + /// `pipeline_macos::Decoder::available_duration_sec` ; consomme par + /// `timeline_walk` pour borner la marche d'export. + pub unsafe fn available_duration_sec(&self) -> Option { + self.sw.duration_sec() + } +} + +/// Encodeur video SOFTWARE (`libopenh264` / `libkvazaar`). Pas de zero-copy HW +/// (VAAPI/Vulkan-encode = suivi) : la frame composee est relue RGBA par +/// l'appelant puis convertie YUV420P par `sws_scale`. Surface +/// `open`/`send_rgba`/`flush` alignee sur le chemin software de +/// `pipeline_macos::VideoEncoder`. +pub struct VideoEncoder { + ctx: *mut crate::ffi::AVCodecContext, + /// AVFrame YUV420P envoyee a l'encodeur. + sw: *mut AVFrame, + /// RGBA (sortie compositeur) -> YUV420P. Cree paresseusement (dims du readback). + sws: *mut crate::ffi::SwsContext, + w: i32, + h: i32, +} + +// SAFETY : pointeurs FFI sans affinite thread ; caller mono-thread (idem Decoder). +unsafe impl Send for VideoEncoder {} + +impl VideoEncoder { + /// Encodeurs software candidats du build LGPL, par codec. La premiere qui + /// ouvre gagne ; `OPENSCREEN_EXPORT_ENCODER=` force un choix. + fn candidate_names(codec: &ExportCodec) -> &'static [&'static str] { + match codec { + ExportCodec::H264 => &["libopenh264"], + ExportCodec::H265 => &["libkvazaar"], + } + } + + pub fn open(codec: &ExportCodec, w: i32, h: i32, fps: i32, bit_rate: i64) -> Result { + let forced = std::env::var("OPENSCREEN_EXPORT_ENCODER").ok(); + let mut refused: Vec = Vec::new(); + // Liste par defaut, plus l'encodeur force s'il n'y figure pas (ex. h264_vaapi). + let defaults = Self::candidate_names(codec); + let extra: Vec<&str> = forced + .as_deref() + .filter(|f| !defaults.contains(f)) + .into_iter() + .collect(); + for &name in defaults.iter().chain(extra.iter()) { + if forced.as_deref().is_some_and(|f| f != name) { + continue; + } + match unsafe { Self::try_open(name, w, h, fps, bit_rate) } { + Ok(enc) => { + eprintln!("[pipeline] encodeur video : {name} (software YUV420P)"); + return Ok(enc); + } + Err(e) => refused.push(format!("{name}: {e}")), + } + } + match forced { + Some(name) => bail!("OPENSCREEN_EXPORT_ENCODER={name} inutilisable : {}", refused.join(" ; ")), + None => bail!("aucun encodeur video utilisable : {}", refused.join(" ; ")), + } + } + + unsafe fn try_open(name: &str, w: i32, h: i32, fps: i32, bit_rate: i64) -> Result { + use crate::ffi::*; + let cname = CString::new(name)?; + let enc = avcodec_find_encoder_by_name(cname.as_ptr()); + if enc.is_null() { + bail!("absent de ce build ffmpeg"); + } + let mut ctx = avcodec_alloc_context3(enc); + if ctx.is_null() { + bail!("avcodec_alloc_context3"); + } + (*ctx).width = w; + (*ctx).height = h; + (*ctx).pix_fmt = AVPixelFormat::AV_PIX_FMT_YUV420P; + (*ctx).time_base = AVRational { num: 1, den: fps }; + (*ctx).framerate = AVRational { num: fps, den: 1 }; + (*ctx).bit_rate = bit_rate; + // MP4 : header global dans l'extradata (pas par-paquet). + (*ctx).flags |= AV_CODEC_FLAG_GLOBAL_HEADER as i32; + if let Err(e) = averr(avcodec_open2(ctx, enc, ptr::null_mut()), "avcodec_open2(enc)") { + avcodec_free_context(&mut ctx); + return Err(e); + } + match alloc_sw_frame(AVPixelFormat::AV_PIX_FMT_YUV420P, w, h) { + Ok(sw) => Ok(VideoEncoder { ctx, sw, sws: ptr::null_mut(), w, h }), + Err(e) => { + avcodec_free_context(&mut ctx); + Err(e) + } + } + } + + /// Envoie une frame composee DEJA RELUE (RGBA) a l'encodeur, en YUV420P. + /// + /// La relecture est sortie d'ici : avec la ring de staging, la frame rendue + /// par `readback_submit` n'est pas celle qui vient d'etre composee mais la + /// precedente, donc l'appelant doit apparier lui-meme la frame et son pts + /// (cf. `run_composited_multi`). + pub unsafe fn send_rgba(&mut self, rgba: &[u8], rw: i32, rh: i32, pts: i64) -> Result<()> { + use crate::ffi::*; + if self.sws.is_null() { + self.sws = sws_getContext( + rw, + rh, + AVPixelFormat::AV_PIX_FMT_RGBA, + self.w, + self.h, + AVPixelFormat::AV_PIX_FMT_YUV420P, + // POINT : le compositeur est dimensionne a la sortie -> pas de + // mise a l'echelle, donc echantillonnage exact (cf. mac_frames). + SWS_POINT, + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + ); + if self.sws.is_null() { + bail!("sws_getContext {rw}x{rh} RGBA -> {}x{} YUV420P", self.w, self.h); + } + } + averr(av_frame_make_writable(self.sw), "make_writable")?; + // RGBA est un plan unique : data[0] + stride rw*4, les autres nuls. + let src_data: [*const u8; 4] = [rgba.as_ptr(), ptr::null(), ptr::null(), ptr::null()]; + let src_stride: [i32; 4] = [rw * 4, 0, 0, 0]; + let converted = sws_scale( + self.sws, + src_data.as_ptr(), + src_stride.as_ptr(), + 0, + rh, + (*self.sw).data.as_ptr() as *const *mut u8, + (*self.sw).linesize.as_ptr(), + ); + if converted <= 0 { + bail!("sws_scale RGBA->YUV420P : {converted} lignes"); + } + (*self.sw).pts = pts; + averr(avcodec_send_frame(self.ctx, self.sw), "send_frame") + } + + /// Flush : une frame nulle finalise le bitstream de l'encodeur. + pub unsafe fn flush(&mut self) -> Result<()> { + crate::ffi::averr( + crate::ffi::avcodec_send_frame(self.ctx, ptr::null_mut()), + "send_frame_flush", + ) + } +} + +impl Drop for VideoEncoder { + fn drop(&mut self) { + unsafe { + crate::ffi::avcodec_free_context(&mut self.ctx); + if !self.sw.is_null() { + crate::ffi::av_frame_free(&mut self.sw); + } + if !self.sws.is_null() { + crate::ffi::sws_freeContext(self.sws); + } + } + } +} + +/// Alloue une AVFrame systeme au format demande. Symetrique de +/// `pipeline_macos::alloc_sw_frame`. +unsafe fn alloc_sw_frame(pix_fmt: crate::ffi::AVPixelFormat::Type, w: i32, h: i32) -> Result<*mut AVFrame> { + let mut frame = crate::ffi::av_frame_alloc(); + if frame.is_null() { + bail!("av_frame_alloc (encodeur)"); + } + (*frame).format = pix_fmt as i32; + (*frame).width = w; + (*frame).height = h; + if crate::ffi::av_frame_get_buffer(frame, 32) < 0 { + crate::ffi::av_frame_free(&mut frame); + bail!("av_frame_get_buffer {w}x{h} pix_fmt={pix_fmt}"); + } + Ok(frame) +} + +/// Draine les paquets de l'encodeur vers le muxer. Symetrique de +/// `pipeline_macos::drain_encoder`. +unsafe fn drain_encoder( + ectx: *mut crate::ffi::AVCodecContext, + octx: *mut crate::ffi::AVFormatContext, + ostream: *mut crate::ffi::AVStream, + opkt: *mut crate::ffi::AVPacket, +) -> Result<()> { + use crate::ffi::*; + loop { + let r = avcodec_receive_packet(ectx, opkt); + if r == AVERROR_EOF || r == AVERROR_EAGAIN { + return Ok(()); + } + averr(r, "receive_packet")?; + av_packet_rescale_ts(opkt, (*ectx).time_base, (*ostream).time_base); + averr(av_interleaved_write_frame(octx, opkt), "interleaved_write_frame")?; + av_packet_unref(opkt); + } +} + +/// Export multiclip VIDEO (WP6). Encode software + mux MP4. Audio AAC = suivi +/// (`audio.rs`/`AacEncoder` partages, il ne manque que le branchement du 2e flux +/// + l'assemblage PCM par clip, cf. `pipeline_macos::run_composited_multi`). +/// +/// La marche de timeline est PARTAGEE (`walk_composited_timeline`) : elle compose +/// chaque frame de sortie (vitesse/fenetrage/curseur inclus) puis appelle +/// `on_frame(n)`, ou on relit + encode + draine. +pub fn run_composited_multi( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &crate::compositor::Compositor, + cfg: &Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), +) -> Result { + if clips.is_empty() { + bail!("run_composited_multi: aucun clip a exporter"); + } + let (out_w, out_h) = (params.width, params.height); + let out_fps = params.fps.unwrap_or(30) as i32; + // bitrate proportionnel a la surface (reference : 8 Mbps @ 1920x1080). + let bit_rate = ((out_w as i64 * out_h as i64 * 8_000_000) / (1920 * 1080)).max(2_000_000); + let t0 = std::time::Instant::now(); + + let mut enc = VideoEncoder::open(¶ms.codec, out_w as i32, out_h as i32, out_fps, bit_rate)?; + let ectx = enc.ctx; + + let mut screen_decs: HashMap = HashMap::new(); + let mut webcam_decs: HashMap = HashMap::new(); + + // ---- muxer MP4 (flux video + flux AAC) ---- + let outc = CString::new(out)?; + let mut octx: *mut crate::ffi::AVFormatContext = ptr::null_mut(); + let mut pb: *mut crate::ffi::AVIOContext = ptr::null_mut(); + let ostream; + let opkt; + let mut audio_encoder; + unsafe { + crate::ffi::averr( + crate::ffi::avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), + "alloc_output_context2", + )?; + ostream = crate::ffi::avformat_new_stream(octx, ptr::null()); + if ostream.is_null() { + bail!("avformat_new_stream"); + } + crate::ffi::averr( + crate::ffi::avcodec_parameters_from_context((*ostream).codecpar, ectx), + "params_from_ctx", + )?; + (*ostream).time_base = (*ectx).time_base; + crate::ffi::averr( + crate::ffi::avio_open(&mut pb, outc.as_ptr(), crate::ffi::AVIO_FLAG_WRITE as i32), + "avio_open", + )?; + crate::ffi::sn_fmt_set_pb(octx, pb); + // Le flux AAC doit exister AVANT l'en-tete (le muxer y fige sa table de flux). + // Meme si aucun clip n'a d'audio, on ecrit une piste silencieuse -- parite + // avec Windows/macOS, qui muxent toujours l'AAC. + audio_encoder = AacEncoder::open(octx)?; + crate::ffi::averr( + crate::ffi::avformat_write_header(octx, ptr::null_mut()), + "write_header", + )?; + opkt = crate::ffi::av_packet_alloc(); + } + + // Un PCM par clip, assemble apres la marche video (elle seule dit combien de + // frames chaque clip a produit, donc combien d'audio lui revient). + let mut clip_pcm: Vec> = (0..clips.len()).map(|_| None).collect(); + let mut clip_frame_counts: Vec = vec![0; clips.len()]; + + let scene = comp.scene_snapshot(); + // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence + // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour + // la raison pour laquelle la preview, elle, reste a 1. + comp.set_readback_depth(2)?; + // pts d'encodage : DECOUPLE de l'index de marche `n`, puisque la frame + // recoltee a l'iteration n est celle composee a n-1. Il reste contigu (les + // frames sortent de la ring dans l'ordre de composition), donc le fichier + // produit est identique a celui du chemin synchrone. + let mut encoded_pts: i64 = 0; + let frames = unsafe { + crate::timeline_walk::walk_composited_timeline( + clips, + gpu, + comp, + cfg, + out_fps, + &scene, + &mut screen_decs, + &mut webcam_decs, + &mut |n| { + // Soumet la copie de la frame n SANS l'attendre et recolte la + // precedente : c'est tout le pipelining. Pendant que le CPU + // passe ses ~12,6 ms dans sws_scale + avcodec_send_frame sur la + // frame n-1, le GPU finit la composition et la copie de n. + if let Some((rw, rh, rgba)) = comp.readback_submit()? { + enc.send_rgba(&rgba, rw as i32, rh as i32, encoded_pts)?; + encoded_pts += 1; + drain_encoder(ectx, octx, ostream, opkt)?; + } + // Progression = frames COMPOSEES (inchangee) : la barre ne doit + // pas reculer d'une frame parce que l'encodage a un tour de + // retard. + progress(n + 1); + Ok(()) + }, + &mut |clip_index, source_end_sec, frames_in_clip, speed_segments| { + clip_frame_counts[clip_index] = frames_in_clip; + let clip = &clips[clip_index]; + if clip.has_audio && frames_in_clip > 0 { + match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { + Ok(Some(pcm)) => { + clip_pcm[clip_index] = + Some(stretch_clip_pcm_by_speed(&pcm, speed_segments, out_fps as f64)); + } + Ok(None) => eprintln!( + "[pipeline] warning: clip #{clip_index} declare audio mais sans flux decodable; silence", + ), + Err(error) => eprintln!( + "[pipeline] warning: decodage audio clip #{clip_index} echoue ({error:#}); silence", + ), + } + } + Ok(()) + }, + )? + }; + + unsafe { + // Drain de la ring AVANT le flush de l'encodeur : les `depth - 1` + // dernieres copies sont encore en vol, et sans ce drain la derniere + // frame composee ne serait jamais encodee (video amputee d'une frame). + while let Some((rw, rh, rgba)) = comp.readback_take()? { + enc.send_rgba(&rgba, rw as i32, rh as i32, encoded_pts)?; + encoded_pts += 1; + drain_encoder(ectx, octx, ostream, opkt)?; + } + // Le compositeur peut survivre a l'export (l'appelant le possede) : on + // lui rend sa profondeur par defaut plutot que de lui laisser une ring + // a 2 et le buffer de 8 Mo qui va avec. + comp.set_readback_depth(1)?; + enc.flush()?; + drain_encoder(ectx, octx, ostream, opkt)?; + // Audio : le plan part des frames REELLEMENT produites par clip (un clip + // raccourci voit son audio raccourci d'autant), puis un seul encode AAC. + let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); + let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); + audio_encoder.encode(&assemble_concatenated_pcm(&clip_pcm, &plan), octx)?; + crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; + crate::ffi::avio_closep(&mut pb); + crate::ffi::avformat_free_context(octx); + let mut opkt = opkt; + crate::ffi::av_packet_free(&mut opkt); + } + + let wall_s = t0.elapsed().as_secs_f64(); + Ok(Stats { + frames, + wall_s, + fps: if wall_s > 0.0 { frames as f64 / wall_s } else { 0.0 }, + video_duration_s: frames as f64 / out_fps as f64, + }) +} diff --git a/crates/compositor/src/remux.rs b/crates/compositor/src/remux.rs new file mode 100644 index 0000000000..e147f8e39e --- /dev/null +++ b/crates/compositor/src/remux.rs @@ -0,0 +1,248 @@ +//! Remux « stream copy » vers le muxer MATROSKA — réécrit un fichier en gardant +//! ses paquets bit-pour-bit, uniquement pour lui donner un index de seek. +//! +//! # Le problème +//! +//! `MediaRecorder` (Chromium) écrit le WebM comme un flux **live** : il n'a pas +//! le droit de revenir en arrière pour remplir un index, donc le fichier final +//! n'a NI `Cues` NI `SeekHead`. Vérifié sur un vrai enregistrement de l'app : +//! les deux magic bytes sont absents du début comme de la fin du fichier. Sans +//! `Cues`, `av_seek_frame` n'a aucun point d'entrée et échoue pour tout +//! timestamp non nul — d'où le repli « rembobine et scanne » linéaire de +//! `linux_decode.rs`. +//! +//! # Le correctif +//! +//! Relire les paquets et les réécrire par le muxer matroska, qui lui connaît la +//! taille finale du fichier et écrit donc `Cues` + `SeekHead` en fin de course +//! (`av_write_trailer` revient au début patcher les offsets). Aucun ré-encodage : +//! les paquets sont copiés tels quels, seuls les timestamps sont rebasés sur la +//! timebase du flux de sortie. Mesuré sur un enregistrement réel de 7,8 Mo : +//! 0,084 s et +378 octets. +//! +//! # Pourquoi `matroska` et pas `webm` +//! +//! Le muxer `webm` REFUSE ce fichier : « Only VP8 or VP9 or AV1 video and Vorbis +//! or Opus audio ... are supported for WebM ». Chromium produit du **H.264 dans +//! du WebM**, une combinaison hors spec que seul lui écrit. On force donc le +//! muxer matroska par son nom (2e argument d'`avformat_alloc_output_context2`), +//! ce qui court-circuite la déduction par extension — le fichier de sortie garde +//! son nom `.webm` alors que son contenu est du Matroska, ce qui est un +//! sur-ensemble strict et décrit le contenu plus honnêtement que ne le faisait +//! `MediaRecorder`. L'extension NE CHANGE PAS : elle alimente le nom du sidecar +//! curseur, le JSON de session et la persistance projet côté TS. +//! +//! # Effet de bord utile : la `Duration` +//! +//! Le muxer matroska recalcule la `Duration` à partir des timestamps réels des +//! paquets, sans lire celle de l'entrée. Confronté à une entrée dont la +//! `Duration` avait été forcée à 999999, il a écrit 16977 — la vraie valeur. Ce +//! remux subsume donc le patch de `Duration` que `webm-duration.ts` applique +//! par ailleurs (`MediaRecorder` ne l'écrit pas non plus). + +use anyhow::{bail, Context, Result}; +use std::ffi::CString; +use std::ptr; + +use crate::ffi::{ + av_interleaved_write_frame, av_packet_alloc, av_packet_free, av_packet_rescale_ts, + av_packet_unref, av_read_frame, av_write_trailer, avcodec_parameters_copy, + avformat_alloc_output_context2, avformat_close_input, avformat_find_stream_info, + avformat_free_context, avformat_new_stream, avformat_open_input, avformat_write_header, + averr, avio_closep, avio_open, sn_fmt_nb_streams, sn_fmt_set_pb, sn_fmt_stream, AVIOContext, + AVFormatContext, AVMediaType, AVPacket, AVIO_FLAG_WRITE, +}; + +/// Bilan d'un remux, remonté jusqu'à la glue TS pour la journalisation. +#[derive(Debug)] +pub struct RemuxStats { + /// Nombre de paquets recopiés (toutes pistes confondues). + pub packets: u64, + /// Nombre de pistes conservées dans la sortie. + pub streams: u32, + /// Durée du remux en secondes. + pub wall_s: f64, +} + +/// Ferme les ressources libav* quel que soit le chemin de sortie (`?` compris). +/// +/// Sans ça, chaque `?` du corps de `remux_to_seekable_matroska` fuiterait un +/// `AVFormatContext` et un descripteur de fichier. `Drop` ne peut pas faillir, +/// donc les erreurs de fermeture sont ignorées — on est déjà en train de rendre +/// une erreur au caller quand ça arrive. +struct RemuxGuard { + ictx: *mut AVFormatContext, + octx: *mut AVFormatContext, + pb: *mut AVIOContext, + pkt: *mut AVPacket, +} + +impl Drop for RemuxGuard { + fn drop(&mut self) { + unsafe { + if !self.pkt.is_null() { + av_packet_free(&mut self.pkt); + } + if !self.ictx.is_null() { + avformat_close_input(&mut self.ictx); + } + if !self.pb.is_null() { + avio_closep(&mut self.pb); + } + if !self.octx.is_null() { + avformat_free_context(self.octx); + self.octx = ptr::null_mut(); + } + } + } +} + +/// Recopie `input` vers `output` par le muxer matroska, sans ré-encoder. +/// +/// `output` DOIT être un chemin temporaire distinct de `input` : le caller +/// (`electron/recording/webm-seek-index.ts`) ne renomme par-dessus l'original +/// qu'une fois le remux terminé, pour qu'un échec laisse l'enregistrement +/// d'origine intact. Écrire directement sur `input` détruirait la seule copie +/// des pixels dès la première erreur d'écriture. +/// +/// Les pistes autres que vidéo/audio/sous-titre sont ignorées : `MediaRecorder` +/// n'en produit pas, et un flux `DATA` ou `ATTACHMENT` inattendu ferait échouer +/// `avformat_write_header` plutôt que de dégrader proprement. +pub fn remux_to_seekable_matroska(input: &str, output: &str) -> Result { + if input == output { + bail!("remux : entrée et sortie identiques ({input}) — le caller doit passer un chemin temporaire"); + } + let t0 = std::time::Instant::now(); + let cin = CString::new(input).context("chemin d'entrée non convertible en CString")?; + let cout = CString::new(output).context("chemin de sortie non convertible en CString")?; + // Nom du muxer, PAS une extension : c'est l'équivalent de `-f matroska`. + let cfmt = CString::new("matroska").expect("littéral sans NUL"); + + let mut guard = RemuxGuard { + ictx: ptr::null_mut(), + octx: ptr::null_mut(), + pb: ptr::null_mut(), + pkt: ptr::null_mut(), + }; + + unsafe { + averr( + avformat_open_input(&mut guard.ictx, cin.as_ptr(), ptr::null_mut(), ptr::null_mut()), + "avformat_open_input", + )?; + averr( + avformat_find_stream_info(guard.ictx, ptr::null_mut()), + "avformat_find_stream_info", + )?; + + averr( + avformat_alloc_output_context2( + &mut guard.octx, + ptr::null(), + cfmt.as_ptr(), + cout.as_ptr(), + ), + "avformat_alloc_output_context2(matroska)", + )?; + if guard.octx.is_null() { + bail!("avformat_alloc_output_context2 n'a pas alloué de contexte matroska"); + } + + // `stream_map[i]` = index de sortie de la piste d'entrée `i`, ou -1 si + // elle est ignorée. Les index de sortie sont réattribués en séquence, + // donc ils ne coïncident pas forcément avec ceux de l'entrée. + let nb_in = sn_fmt_nb_streams(guard.ictx); + let mut stream_map: Vec = vec![-1; nb_in as usize]; + let mut nb_out: i32 = 0; + for i in 0..nb_in { + let istream = sn_fmt_stream(guard.ictx, i as i32); + if istream.is_null() { + continue; + } + let codec_type = (*(*istream).codecpar).codec_type; + if codec_type != AVMediaType::AVMEDIA_TYPE_VIDEO + && codec_type != AVMediaType::AVMEDIA_TYPE_AUDIO + && codec_type != AVMediaType::AVMEDIA_TYPE_SUBTITLE + { + continue; + } + let ostream = avformat_new_stream(guard.octx, ptr::null()); + if ostream.is_null() { + bail!("avformat_new_stream a rendu NULL pour la piste {i}"); + } + averr( + avcodec_parameters_copy((*ostream).codecpar, (*istream).codecpar), + "avcodec_parameters_copy", + )?; + // Le codec_tag est propre au conteneur d'origine ; le garder ferait + // écrire à matroska un tag qu'il ne reconnaît pas. 0 = « au muxer de + // choisir », c'est ce que fait `ffmpeg -c copy`. + (*(*ostream).codecpar).codec_tag = 0; + stream_map[i as usize] = nb_out; + nb_out += 1; + } + if nb_out == 0 { + bail!("remux : aucune piste vidéo/audio/sous-titre dans {input}"); + } + + averr( + avio_open(&mut guard.pb, cout.as_ptr(), AVIO_FLAG_WRITE as i32), + "avio_open", + )?; + sn_fmt_set_pb(guard.octx, guard.pb); + averr( + avformat_write_header(guard.octx, ptr::null_mut()), + "avformat_write_header", + )?; + + guard.pkt = av_packet_alloc(); + if guard.pkt.is_null() { + bail!("av_packet_alloc a rendu NULL"); + } + + let mut packets: u64 = 0; + loop { + let r = av_read_frame(guard.ictx, guard.pkt); + if r < 0 { + // Fin de fichier ou flux tronqué : dans les deux cas on écrit le + // trailer sur ce qu'on a. Un enregistrement coupé net (crash, + // batterie) reste lisible et devient seekable jusqu'à sa coupure. + break; + } + let in_idx = (*guard.pkt).stream_index; + let out_idx = stream_map + .get(in_idx as usize) + .copied() + .unwrap_or(-1); + if out_idx < 0 { + av_packet_unref(guard.pkt); + continue; + } + let istream = sn_fmt_stream(guard.ictx, in_idx); + let ostream = sn_fmt_stream(guard.octx, out_idx); + if istream.is_null() || ostream.is_null() { + av_packet_unref(guard.pkt); + continue; + } + av_packet_rescale_ts(guard.pkt, (*istream).time_base, (*ostream).time_base); + (*guard.pkt).stream_index = out_idx; + // `pos` décrit un offset dans le fichier d'ENTRÉE ; le laisser + // induirait le muxer en erreur sur la sortie. + (*guard.pkt).pos = -1; + let w = av_interleaved_write_frame(guard.octx, guard.pkt); + // `av_interleaved_write_frame` prend possession du paquet (il le + // déréférence lui-même), d'où l'absence d'`av_packet_unref` ici. + averr(w, "av_interleaved_write_frame")?; + packets += 1; + } + + // C'est CE trailer qui écrit `Cues` puis revient patcher `SeekHead`. + averr(av_write_trailer(guard.octx), "av_write_trailer")?; + + Ok(RemuxStats { + packets, + streams: nb_out as u32, + wall_s: t0.elapsed().as_secs_f64(), + }) + } +} diff --git a/crates/compositor/src/text_linux.rs b/crates/compositor/src/text_linux.rs new file mode 100644 index 0000000000..64c590c743 --- /dev/null +++ b/crates/compositor/src/text_linux.rs @@ -0,0 +1,485 @@ +//! Rasterisation de texte Linux (PR #183) -- `cosmic-text` (rustybuzz + swash + +//! fontdb) au lieu de DirectWrite (Windows) / CoreText (macOS). +//! +//! Equivalent Linux de `text_windows.rs` / `text_macos.rs` : meme surface +//! publique (`TextSpec` + `cache_key`, `TextRasterizer::new()`, +//! `rasterize(&self, gpu, spec)`) pour que `compositor` (cfg-re-exporte) et +//! `text_anim` (partage) l'utilisent sans connaitre la plateforme. +//! +//! **Difference de format.** macOS/Windows bakent la couleur dans une texture +//! BGRA premultipliee (CoreText/Direct2D). Ici on produit un **atlas de +//! couverture R8** (alpha) que le shader WGSL (`layer.wgsl` mode 11) teinte par +//! `layer.color` -- meme resultat visuel, et le contrat d'iso-render porte sur +//! la GEOMETRIE (`frame_geometry::plan_frame`), pas sur la rasterisation texte +//! (dont l'ecart d'antialiasing est deja exclu des goldens cross-backend). + +use anyhow::{bail, Result}; +use std::cell::RefCell; + +use cosmic_text::{Attrs, Buffer, FontSystem, Metrics, Shaping, SwashCache}; + +use crate::d3d::Gpu; + +/// Tout ce dont le rendu d'un texte depend. Meme structure et meme `cache_key` +/// que `text_windows::TextSpec` / `text_macos::TextSpec` : la cle est partagee +/// entre plateformes, donc deux specs identiques produisent la meme texture. +#[derive(Clone, PartialEq)] +pub struct TextSpec { + pub content: String, + /// RGBA 0..1 (deja parse depuis la chaine CSS cote appelant). + pub color: [f32; 4], + /// RGBA 0..1 ; alpha 0 = pas de fond (le CSS `transparent`). + pub background: [f32; 4], + pub font_size_px: f32, + pub font_family: String, + pub bold: bool, + pub italic: bool, + pub underline: bool, + /// "left" | "center" | "right". + pub align: String, + /// Taille de la boite en px de sortie. + pub box_px: [u32; 2], +} + +impl TextSpec { + /// FNV-1a sur les memes octets, dans le meme ordre, que + /// `text_macos::TextSpec::cache_key` / `text_windows` -- la policy est + /// partagee (cache cross-plateforme coherent). + pub fn cache_key(&self) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + let mut mix = |bytes: &[u8]| { + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x100_0000_01b3); + } + }; + mix(self.content.as_bytes()); + mix(self.font_family.as_bytes()); + mix(&self.font_size_px.to_bits().to_le_bytes()); + for c in self.color.iter().chain(self.background.iter()) { + mix(&c.to_bits().to_le_bytes()); + } + mix(&[self.bold as u8, self.italic as u8, self.underline as u8]); + mix(self.align.as_bytes()); + mix(&self.box_px[0].to_le_bytes()); + mix(&self.box_px[1].to_le_bytes()); + h + } +} + +/// Le resultat d'une rasterisation : la texture R8 de couverture + ses dims. +pub struct RasterizedGlyphs { + pub view: wgpu::TextureView, + pub width: u32, + pub height: u32, +} + +/// Le rasterizer Linux. `fontdb` lit `/usr/share/fonts` a la construction du +/// `FontSystem`. Etat (font_system, swash_cache) en `RefCell` pour que +/// `rasterize(&self, ...)` matche la signature `&self` des autres plateformes +/// (le compositor tient un `Option` et l'appelle sur `&self`). +pub struct TextRasterizer { + font_system: RefCell, + swash_cache: RefCell, +} + +impl TextRasterizer { + pub fn new() -> Result { + Ok(TextRasterizer { + font_system: RefCell::new(FontSystem::new()), + swash_cache: RefCell::new(SwashCache::new()), + }) + } + + /// Rasterise `spec` dans une texture R8Unorm (couverture alpha) et rend sa + /// view. `gpu` fournit le device/queue wgpu (passe au rasterize comme cote + /// macOS). Le cache par `cache_key()` est gere par le caller (compositor). + pub fn rasterize(&self, gpu: &Gpu, spec: &TextSpec) -> Result { + let (w, h) = (spec.box_px[0].max(1), spec.box_px[1].max(1)); + let atlas = self.build_atlas(spec)?; + + Self::upload(gpu, &atlas, w, h) + } + + /// La moitie CPU de [`Self::rasterize`] : shaping + placement des glyphes + /// dans un atlas R8 de `spec.box_px`. + /// + /// Separee du GPU EXPRES. Le placement des glyphes est de l'arithmetique + /// pure, et c'est exactement la ou le portage s'etait trompe (origine au + /// coin au lieu de la ligne de base, `line_top` ajoute au X, signe de + /// `placement.top` inverse). Tant que ce code vivait derriere un `&Gpu`, il + /// etait intestable sans peripherique. Il ne l'est plus. + pub fn build_atlas(&self, spec: &TextSpec) -> Result> { + let (w, h) = (spec.box_px[0].max(1), spec.box_px[1].max(1)); + if spec.content.is_empty() { + bail!("text_linux::rasterize: texte vide"); + } + + let mut font_system = self.font_system.borrow_mut(); + let mut swash_cache = self.swash_cache.borrow_mut(); + + let font_size = spec.font_size_px.max(1.0); + let line_height = font_size * 1.4; // heuristique standard. + let metrics = Metrics::new(font_size, line_height); + let mut buffer = Buffer::new(&mut font_system, metrics); + buffer.set_size(Some(w as f32), Some(h as f32)); + + let mut attrs = Attrs::new(); + attrs = attrs.family(cosmic_text::Family::Name(&spec.font_family)); + if spec.bold { + attrs = attrs.weight(cosmic_text::Weight::BOLD); + } + if spec.italic { + attrs = attrs.style(cosmic_text::Style::Italic); + } + if spec.underline { + attrs = attrs.underline(cosmic_text::UnderlineStyle::Single); + } + buffer.set_text(&spec.content, &attrs, Shaping::Advanced, None); + // L'alignement se pose PAR LIGNE, apres set_text (qui reconstruit les + // lignes) et avant le shaping. Sans ca tout le texte sort ferre a + // gauche alors que le defaut de l'editeur est « center ». + let align = match spec.align.as_str() { + "center" => Some(cosmic_text::Align::Center), + "right" | "end" => Some(cosmic_text::Align::Right), + "justify" => Some(cosmic_text::Align::Justified), + // `None` laisse cosmic-text suivre la direction du script, ce qui + // est le bon defaut pour "left"/"start" et pour une valeur inconnue. + _ => None, + }; + for line in &mut buffer.lines { + line.set_align(align); + } + buffer.shape_until_scroll(&mut font_system, false); + + // CENTRAGE VERTICAL. L'overlay web pose `alignItems: center` sur le + // conteneur de l'annotation, et Windows reproduit ca avec + // `DWRITE_PARAGRAPH_ALIGNMENT_CENTER` (text_windows.rs:175-176). macOS + // ne le fait pas, et le portage Linux avait copie macOS : le texte + // collait en haut de sa boite. Les deux references natives divergent + // reellement ici ; c'est le web qui porte l'intention produit. + // + // La hauteur du bloc est prise sur la DERNIERE ligne posee plutot que + // sur un compte de lignes x line_height : cosmic-text peut replier une + // ligne logique en plusieurs runs, donc compter les runs surestimerait + // des que le texte deborde en largeur. + let text_h = buffer + .layout_runs() + .map(|run| run.line_top + run.line_height) + .fold(0.0f32, f32::max); + // `max(0)` : un texte plus haut que sa boite reste ancre en haut plutot + // que de sortir par le dessus, ou il serait entierement rogne. + let y_offset = (((h as f32) - text_h) * 0.5).max(0.0).round() as i32; + + // Atlas R8 : on n'ecrit que le canal alpha (couverture). Le tint par + // `spec.color` se fait cote shader (mode 11). + let mut atlas: Vec = vec![0u8; (w * h) as usize]; + for run in buffer.layout_runs() { + for glyph in run.glyphs.iter() { + // L'ORIGINE EST LA LIGNE DE BASE, PAS LE COIN. C'est la + // convention de cosmic-text : `Buffer::draw` appelle + // `glyph.physical((0., run.line_y), 1.0)` et son rasteriseur + // pose ensuite le haut du bitmap a `y - placement.top`. + // + // Le portage passait `(0.0, 0.0)` — donc sans ligne de base — + // et ajoutait `run.line_top`, une quantite VERTICALE, au X. + // Resultat mesure sur « Agjo Hxy » en 40px : aucune ligne de + // base commune, le 'A' 14 px SOUS le 'o', et sur du multi-ligne + // la 2e ligne redessinee sur les memes rangees que la 1re mais + // decalee de `line_top` px vers la droite. + let physical = glyph.physical((0.0, run.line_y), 1.0); + let img = swash_cache.get_image(&mut font_system, physical.cache_key); + let glyph_x = physical.x; + let glyph_y = physical.y; + let Some(img) = img else { continue }; + let placement = img.placement; + let (img_w, img_h) = (placement.width, placement.height); + if img_w == 0 || img_h == 0 { + continue; + } + let stride = match img.content { + cosmic_text::SwashContent::Mask => img_w as usize, + cosmic_text::SwashContent::Color => img_w as usize * 4, + _ => continue, + }; + let alpha_offset = if matches!(img.content, cosmic_text::SwashContent::Color) { + 3 + } else { + 0 + }; + let bpp = if alpha_offset == 0 { 1 } else { 4 }; + // `placement.top` est la hauteur de l'encre AU-DESSUS de la + // ligne de base, donc la premiere rangee du bitmap est a + // `baseline - top`. Le signe etait inverse. + let ink_top = glyph_y - placement.top + y_offset; + let ink_left = glyph_x + placement.left; + for row in 0..img_h as i32 { + let dest_y = ink_top + row; + if dest_y < 0 || dest_y >= h as i32 { + continue; + } + // `placement.left` est negatif sur les glyphes qui debordent + // a gauche de leur avance ('j' en DejaVu Sans : -3). Sans ce + // rattrapage, `dest_x as usize` enroule en release et l'encre + // se retrouve collee au bord droit de la rangee PRECEDENTE ; + // en debug c'est une panique d'overflow. On saute plutot les + // colonnes SOURCE hors cadre. + let (dest_x, skip_cols) = if ink_left < 0 { + (0i32, (-ink_left) as usize) + } else { + (ink_left, 0usize) + }; + if dest_x >= w as i32 || skip_cols >= img_w as usize { + continue; + } + let copy_len = ((img_w as usize - skip_cols) as i32) + .min(w as i32 - dest_x) + .max(0) as usize; + if copy_len == 0 { + continue; + } + let src_row = &img.data[(row as usize) * stride..(row as usize + 1) * stride]; + let atlas_row_start = (dest_y as usize) * w as usize; + for col in 0..copy_len { + let atlas_idx = atlas_row_start + (dest_x as usize + col); + let src_idx = (col + skip_cols) * bpp + alpha_offset; + if src_idx < src_row.len() { + // `max` et non affectation : deux glyphes peuvent se + // chevaucher (accents, ligatures, italiques) et + // ecraser ferait disparaitre l'encre du premier. + atlas[atlas_idx] = atlas[atlas_idx].max(src_row[src_idx]); + } + } + } + } + } + + Ok(atlas) + } + + /// Televerse un atlas R8 deja construit et rend sa view. + fn upload(gpu: &Gpu, atlas: &[u8], w: u32, h: u32) -> Result { + let texture = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("text-atlas"), + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + gpu.context.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &atlas, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(w), + rows_per_image: Some(h), + }, + wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + ); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + Ok(RasterizedGlyphs { + view, + width: w, + height: h, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec(content: &str, align: &str) -> TextSpec { + TextSpec { + content: content.to_owned(), + color: [1.0, 1.0, 1.0, 1.0], + background: [0.0, 0.0, 0.0, 0.0], + font_size_px: 40.0, + // Vide = cosmic-text prend la police par defaut du systeme. Nommer + // une famille precise rendrait le test dependant des polices + // installees sur la machine qui l'execute. + font_family: String::new(), + bold: false, + italic: false, + underline: false, + align: align.to_owned(), + box_px: [400, 200], + } + } + + /// Rangees d'atlas contenant de l'encre, pour la tranche `x0..x1`. + fn ink_rows(atlas: &[u8], w: usize, x0: usize, x1: usize) -> Vec { + let h = atlas.len() / w; + (0..h) + .filter(|y| (x0..x1.min(w)).any(|x| atlas[y * w + x] > 16)) + .collect() + } + + fn ink_cols(atlas: &[u8], w: usize) -> Vec { + let h = atlas.len() / w; + (0..w) + .filter(|x| (0..h).any(|y| atlas[y * w + x] > 16)) + .collect() + } + + #[test] + fn glyphs_of_different_heights_share_one_baseline() { + // LE test de ce fichier. Le portage posait chaque glyphe contre le HAUT + // de sa propre boite d'encre au lieu de la ligne de base commune, avec + // en prime le signe de `placement.top` inverse. Mesure d'alors sur + // « Agjo Hxy » en 40px : le 'A' finissait 14 px SOUS le 'o'. + // + // On compare le bas de l'encre d'un 'H' (qui descend jusqu'a la ligne + // de base) et celui d'un 'x' (idem). S'ils partagent une ligne de base, + // leurs dernieres rangees d'encre coincident a l'antialiasing pres. + let raster = TextRasterizer::new().expect("rasterizer"); + let atlas = raster.build_atlas(&spec("Hx", "left")).expect("atlas"); + let w = 400usize; + + let cols = ink_cols(&atlas, w); + assert!(!cols.is_empty(), "aucune encre : le texte n'a pas ete rasterise"); + // Coupe entre les deux glyphes : le plus grand trou horizontal. + let split = cols + .windows(2) + .max_by_key(|p| p[1] - p[0]) + .map(|p| (p[0] + p[1]) / 2) + .expect("deux glyphes attendus"); + + let left = ink_rows(&atlas, w, cols[0], split); + let right = ink_rows(&atlas, w, split, *cols.last().unwrap() + 1); + assert!(!left.is_empty() && !right.is_empty(), "un des deux glyphes est vide"); + + let (h_bottom, x_bottom) = (*left.last().unwrap(), *right.last().unwrap()); + assert!( + h_bottom.abs_diff(x_bottom) <= 2, + "pas de ligne de base commune : bas du 'H' = {h_bottom}, bas du 'x' = {x_bottom}" + ); + } + + #[test] + fn a_second_line_sits_below_the_first() { + // L'autre moitie du meme bug : `run.line_top` etait ajoute au X, donc la + // 2e ligne se dessinait sur les MEMES rangees que la 1re, decalee vers + // la droite. Ici elle doit etre strictement plus bas, et commencer a peu + // pres a la meme abscisse. + let raster = TextRasterizer::new().expect("rasterizer"); + let atlas = raster.build_atlas(&spec("ab\ncd", "left")).expect("atlas"); + let w = 400usize; + + let rows = ink_rows(&atlas, w, 0, w); + assert!(rows.len() > 4, "trop peu d'encre pour deux lignes"); + // Un trou vertical separe les deux lignes. + let gap = rows + .windows(2) + .max_by_key(|p| p[1] - p[0]) + .expect("deux lignes attendues"); + assert!( + gap[1] - gap[0] > 2, + "les deux lignes se chevauchent : aucune separation verticale trouvee" + ); + } + + /// Chaque texte par defaut d'annotation produit-il de l'encre sur CETTE + /// machine ? + /// + /// L'app propose un texte localise a la creation ("Hello", "你好", + /// "مرحبا"...). Si la police systeme ne couvre pas le script, cosmic-text + /// rend du tofu ou rien du tout, et l'utilisateur voit une annotation vide + /// qu'il n'a pas ecrite. Le test ne PEUT pas garantir la couverture d'une + /// machine inconnue — il documente ce qui manque sur celle qui l'execute, + /// ce qui est exactement l'information utile quand quelqu'un rapporte + /// « mon annotation est invisible ». + #[test] + fn the_localised_default_texts_render_on_this_machine() { + let raster = TextRasterizer::new().expect("rasterizer"); + let samples = [ + ("en", "Hello"), ("fr", "Bonjour"), ("es", "Hola"), ("it", "Ciao"), + ("pt-BR", "Olá"), ("ru", "Привет"), ("tr", "Merhaba"), ("vi", "Xin chào"), + ("ar", "مرحبا"), ("ja-JP", "こんにちは"), ("ko-KR", "안녕하세요"), + ("zh-CN", "你好"), ("zh-TW", "你好"), + ]; + let mut blank = Vec::new(); + for (locale, text) in samples { + let atlas = raster.build_atlas(&spec(text, "center")).expect("atlas"); + let ink = atlas.iter().filter(|byte| **byte > 16).count(); + println!(" {locale:6} {text:12} -> {ink} px d'encre"); + if ink == 0 { + blank.push(locale); + } + } + assert!( + blank.is_empty(), + "aucune police installee ne couvre: {blank:?} — l'annotation par defaut y serait invisible" + ); + } + + #[test] + fn one_short_line_is_centred_vertically_in_its_box() { + // L'overlay web pose `alignItems: center` et Windows fait pareil + // (DWRITE_PARAGRAPH_ALIGNMENT_CENTER) ; macOS non, et le portage avait + // copie macOS. Une ligne de 40px dans une boite de 200px doit laisser a + // peu pres autant de vide au-dessus qu'en dessous. + let raster = TextRasterizer::new().expect("rasterizer"); + let atlas = raster.build_atlas(&spec("Hx", "center")).expect("atlas"); + let (w, h) = (400usize, 200usize); + let rows = ink_rows(&atlas, w, 0, w); + assert!(!rows.is_empty(), "aucune encre"); + + let (top, bottom) = (rows[0], *rows.last().unwrap()); + let above = top as i32; + let below = (h - 1 - bottom) as i32; + assert!( + (above - below).abs() <= 12, + "texte non centre verticalement : {above}px au-dessus, {below}px en dessous" + ); + } + + #[test] + fn centering_moves_the_ink_off_the_left_edge() { + // `spec.align` n'etait jamais applique : tout sortait ferre a gauche + // alors que le defaut de l'editeur est « center ». + let raster = TextRasterizer::new().expect("rasterizer"); + let w = 400usize; + let left = ink_cols(&raster.build_atlas(&spec("hi", "left")).expect("atlas"), w); + let centered = ink_cols(&raster.build_atlas(&spec("hi", "center")).expect("atlas"), w); + + assert!(!left.is_empty() && !centered.is_empty()); + assert!( + centered[0] > left[0] + 20, + "le centrage n'a pas bouge le texte : gauche debute a {}, centre a {}", + left[0], + centered[0] + ); + } + + #[test] + fn a_glyph_overhanging_to_the_left_does_not_wrap_around() { + // 'j' a un `placement.left` negatif en DejaVu Sans. Avant, `dest_x as + // usize` enroulait : l'encre atterrissait au bord DROIT de la rangee + // precedente en release, et paniquait en debug. Ce test tourne en debug + // sous `cargo test`, donc il attrape la panique directement. + let raster = TextRasterizer::new().expect("rasterizer"); + let atlas = raster.build_atlas(&spec("jazz", "left")).expect("pas de panique"); + let w = 400usize; + // Rien ne doit avoir atterri contre le bord droit d'une rangee. + let h = atlas.len() / w; + let right_edge_ink = (0..h).filter(|y| atlas[y * w + (w - 1)] > 16).count(); + assert_eq!(right_edge_ink, 0, "de l'encre a enroule jusqu'au bord droit"); + } +} diff --git a/crates/compositor/src/vk_shaders/blur.wgsl b/crates/compositor/src/vk_shaders/blur.wgsl new file mode 100644 index 0000000000..c5f8f88e64 --- /dev/null +++ b/crates/compositor/src/vk_shaders/blur.wgsl @@ -0,0 +1,132 @@ +// Tranche verticale WP4 — Kawase blur (mode 9+10 du HLSL) porté en WGSL. +// +// Le Kawase blur est une approximation gaussienne en 6 passes : down 3x +// (RT→½→¼→⅛) puis up 3x (⅛→¼→½→RT). Chaque passe est un 5-tap linéaire +// à offset 2.2 px (cf. HLSL `ps_kawase_down` / `ps_kawase_up`). Le résultat +// est visuellement équivalent à un flou gaussien ~30-50 px (selon la +// taille de la pyramide) à un coût constant 6×5 = 30 taps — vs 49 taps +// pour une passe gaussienne équivalente. Cf. HLSL `Compositor::blur_bg`. +// +// Bindings : la passe de down lit d'une texture RGBA8 et écrit dans +// une texture RGBA8 plus petite ; la passe d'up fait l'inverse. Toutes +// les passes partagent le même bind group layout, seule la constante +// `texel_offset` (dans LayerCB `fx`) change entre les passes. + +struct Layer { + dst: vec4, + src: vec4, + quad_px: vec2, + radius_px: f32, + mode: f32, + color: vec4, + fx: vec4, // .x = texel offset (2.2 pour Kawase) + src_prev: vec4, + dst_prev: vec4, + mb: vec4, +} + +@group(0) @binding(0) var layer: Layer; +@group(0) @binding(1) var tex: texture_2d; +@group(0) @binding(2) var samp: sampler; + +struct VsOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + // Fullscreen triangle — un seul triangle couvre tout l'écran, plus + // efficace qu'un quad en termes de pixels shaders émis. + let pos = array, 3>( + vec2(-1.0, -1.0), + vec2( 3.0, -1.0), + vec2(-1.0, 3.0), + ); + var o: VsOut; + // `pos[vid]` UNE SEULE FOIS, puis on réutilise `p`. Ce n'est pas du style : + // indexer deux fois le même tableau local avec un indice dynamique fait + // émettre à naga 24 du SPIR-V INVALIDE. Son `spilled_composites` est indexé + // par le handle de l'expression de base, donc le second accès écrase l'entrée + // du premier : un seul `OpVariable` est émis, et les `OpStore`/`OpAccessChain` + // du premier accès référencent un id sans définition. Le module viole alors + // VUID-VkShaderModuleCreateInfo-pCode-08737, donc le comportement du pilote + // est indéfini — RADV déréférence l'id fantôme et segfault à la création du + // pipeline, lavapipe survit par chance. Cf. gfx-rs/wgpu#7048, corrigé par + // #7239 (wgpu 25) ; il n'existe pas de patch 24.0.x, donc tant qu'on est sur + // wgpu 24 c'est au shader de ne pas déclencher le bug. + let p = pos[vid]; + o.pos = vec4(p, 0.0, 1.0); + o.uv = p * 0.5 + vec2(0.5, 0.5); + // Note : on inverse Y parce que wgpu NDC y-up mais l'image source est + // y-down (cf. le Y-flip dans le VS du layer.wgsl principal). + o.uv.y = 1.0 - o.uv.y; + return o; +} + +// Triangle plein écran pour une COPIE 1:1, séparé de `vs_main` à dessein. +// +// Les deux mappings sont aujourd'hui identiques, et c'est précisément le piège : +// `vs_main` appartient à la chaîne Kawase, qui enchaîne SIX passes. Une +// inversion en Y y serait invisible (six inversions se compensent), donc rien +// dans cette chaîne ne défend l'orientation. Une copie unique, elle, la porte +// entière. Dupliquer les quatre lignes coûte moins qu'une traînée de curseur +// retournée le jour où quelqu'un ajuste la convention du Kawase. +// +// Orientation : en NDC wgpu (calqué sur D3D/Metal) y=+1 est le HAUT de la cible +// et v=0 la PREMIÈRE ligne de la texture, donc v doit croître quand y décroît. +// Vérifié par `compose_linux_trainee_de_curseur`, qui échoue en trouvant la +// traînée dans la bande miroir si on écrit `0.5 + p.y * 0.5`. +@vertex +fn vs_fullscreen(@builtin(vertex_index) vid: u32) -> VsOut { + let pos = array, 3>( + vec2(-1.0, -1.0), + vec2( 3.0, -1.0), + vec2(-1.0, 3.0), + ); + // Un seul accès indexé, cf. la note naga 24 / RADV dans `vs_main`. + let p = pos[vid]; + var o: VsOut; + o.pos = vec4(p, 0.0, 1.0); + o.uv = vec2(p.x * 0.5 + 0.5, 0.5 - p.y * 0.5); + return o; +} + +// Copie telle quelle. La source est en alpha PRÉMULTIPLIÉ (tout le compositeur +// l'est), donc le blend « over » de la pipeline la composite sans reconversion. +@fragment +fn fs_copy(i: VsOut) -> @location(0) vec4 { + return textureSample(tex, samp, i.uv); +} + +// Kawase down : 5-tap linéaire à offset `texel_offset` en coords source. +// `texel_offset` est 2.2 typiquement (le spread mesuré du filtre). +@fragment +fn fs_kawase_down(i: VsOut) -> @location(0) vec4 { + let o = layer.fx.x; + let c = textureSample(tex, samp, i.uv).rgb; + let s1 = textureSample(tex, samp, i.uv + vec2( o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s2 = textureSample(tex, samp, i.uv + vec2(-o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s3 = textureSample(tex, samp, i.uv + vec2( o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s4 = textureSample(tex, samp, i.uv + vec2(-o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + return vec4((c + s1 + s2 + s3 + s4) * 0.2, layer.color.a); +} + +// Kawase up : interpolation linéaire entre la texture de destination +// (`tex`) et l'échantillon à offset `texel_offset` dans la même texture. +// C'est l'algorithme Kawase « up » original — moins connu que le down +// mais c'est ce qui donne le look "soft glow" mesuré sur le banc. +// +// On interpole entre la valeur au centre et les 4 voisins à offset `o`. +@fragment +fn fs_kawase_up(i: VsOut) -> @location(0) vec4 { + let o = layer.fx.x; + let c = textureSample(tex, samp, i.uv).rgb; + let s1 = textureSample(tex, samp, i.uv + vec2( o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s2 = textureSample(tex, samp, i.uv + vec2(-o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s3 = textureSample(tex, samp, i.uv + vec2( o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s4 = textureSample(tex, samp, i.uv + vec2(-o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + // Pondération (1.0 centre, 0.5 chaque voisin) — 1+4×0.5 = 3.0, /3 = 1/3 par + // échantillon. Le rendu Kawase up est plus doux que le down. + return vec4((c + (s1 + s2 + s3 + s4) * 0.5) / 3.0, layer.color.a); +} diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl new file mode 100644 index 0000000000..6fb2a73ed8 --- /dev/null +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -0,0 +1,443 @@ +// Tranche verticale WP3 — port 1:1 des modes 0 (vidéo NV12) et 1 (couleur pleine) +// du `ps_main` HLSL (`crates/compositor/src/shaders.hlsl`). Le mode 2 (ombre +// portée) partage la même SDF et le même feather que les autres modes, donc on +// l'inclut aussi pour parité. +// +// Les constantes YUV (BT.709 limited) sont reprises à l'identique du HLSL : +// Yf = (Y * 255 − 16) / 219 +// Cb = (UV.x * 255 − 128) / 224 +// Cr = (UV.y * 255 − 128) / 224 +// R = Yf + 1.5748 · Cr +// G = Yf − 0.1873 · Cb − 0.4681 · Cr +// B = Yf + 1.8556 · Cb +// Mesuré en S1 (cf. doc §7 E1). Une déviation > 0/255 entre HLSL et WGSL ici +// indiquerait une différence de précision fp32 ; IEEE-754 round-to-nearest est +// identique sur les deux backends. +// +// Le rendu est en alpha PRÉMULTIPLIÉ (cf. commentaire HLSL), convention qu'on +// retrouve dans tous les autres modes du compositeur (texte, curseur, ombre). + +struct Layer { + dst: vec4, // x,y,w,h sortie 0..1 (origine haut-gauche) + src: vec4, // u0,v0,u1,v1 source 0..1 + quad_px: vec2, // taille du quad en px de sortie (pour la SDF isotrope) + radius_px: f32, + mode: f32, // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre, 8 = écran tilté, 9 = flèche, 10 = flou/mosaïque, 12 = ombre du quad tilté, 13 = curseur tilté + color: vec4, + fx: vec4, // mode 2 : spread ombre en px ; modes 8/12/13 : coins TL,TR du quad projeté ; mode 9 : hampe de la flèche ; mode 10 : (flou?, rayon/bloc px, ovale?, teinté?) + src_prev: vec4, // modes 8/12/13 : coins BR,BL du quad projeté ; mode 9 : barbe 1 + dst_prev: vec4, // mode 8 : taille du plan en px AVANT projection (le rayon y vit) ; mode 13 : rect de clip ; mode 9 : barbe 2 + mb: vec4, // mode 12 : mb.y = spread de la pénombre en px ; mode 9 : mb.y = demi-épaisseur du trait en px +} + +@group(0) @binding(0) var layer: Layer; +@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r +@group(0) @binding(2) var texUV: texture_2d; // Rg8Unorm, sample .rg +@group(0) @binding(3) var samp: sampler; + +struct VsOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, // UV d'échantillonnage source + @location(1) local: vec2, // pixel local dans le quad (SDF) + @location(2) pout: vec2, // position 0..1 sortie +}; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + // strip 4 vertices : (0,0)(1,0)(0,1)(1,1) + let c = vec2(f32(vid & 1u), f32((vid >> 1u) & 1u)); + let p = layer.dst.xy + c * layer.dst.zw; + let ndc = vec2(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0); + var o: VsOut; + o.pos = vec4(ndc, 0.0, 1.0); + o.uv = layer.src.xy + c * (layer.src.zw - layer.src.xy); + o.local = c * layer.quad_px; + o.pout = p; + return o; +} + +fn yuv709_limited(y: f32, cbcr: vec2) -> vec3 { + let Yf = (y * 255.0 - 16.0) / 219.0; + let Cb = (cbcr.x * 255.0 - 128.0) / 224.0; + let Cr = (cbcr.y * 255.0 - 128.0) / 224.0; + return clamp(vec3( + Yf + 1.5748 * Cr, + Yf - 0.1873 * Cb - 0.4681 * Cr, + Yf + 1.8556 * Cb, + ), vec3(0.0), vec3(1.0)); +} + +fn sample_yuv(uv: vec2) -> vec3 { + let y = textureSample(texY, samp, uv).r; + let cbcr = textureSample(texUV, samp, uv).rg; + return yuv709_limited(y, cbcr); +} + +// SDF rectangle à coins arrondis (< 0 dedans). Identique au HLSL. +fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { + let q = abs(p) - halfsz + vec2(r); + return length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0) - r; +} + +// ---- Primitives du tilt 3D (modes 8 et 12), portees de `shaders.metal` ---- + +// SDF segment a bouts ronds. +fn sd_segment(p: vec2, a: vec2, b: vec2) -> f32 { + let pa = p - a; + let ba = b - a; + let h = clamp(dot(pa, ba) / max(dot(ba, ba), 1e-6), 0.0, 1.0); + return length(pa - ba * h); +} + +// Intersection de deux droites donnees par (normale, offset) : n.x = d. Cramer. +fn line_cross(n1: vec2, d1: f32, n2: vec2, d2: f32) -> vec2 { + let det = n1.x * n2.y - n1.y * n2.x; + if abs(det) < 1e-6 { + return vec2(0.0, 0.0); + } + return vec2(d1 * n2.y - d2 * n1.y, d2 * n1.x - d1 * n2.x) / det; +} + +// Contribution d'une arete a la SDF du quad : .x = distance signee au demi-plan +// porte par l'arete (>0 dehors), .y = distance au SEGMENT. +fn quad_edge(p: vec2, a: vec2, b: vec2) -> vec2 { + let e = b - a; + // Division par la longueur plutot que `normalize` : une arete degeneree + // donnerait un NaN qui effacerait le calque entier. + let n = vec2(e.y, -e.x) / max(length(e), 1e-6); + return vec2(dot(p - a, n), sd_segment(p, a, b)); +} + +// Distance signee EXACTE a un quadrilatere convexe (<0 dedans). La boucle `k` +// du MSL est deroulee : elle indexait un tableau local avec un indice runtime, +// ce que naga 24 traduit en SPIR-V invalide (cf. `blur.wgsl`). +fn sd_convex_quad(p: vec2, v0: vec2, v1: vec2, v2: vec2, v3: vec2) -> f32 { + let e0 = quad_edge(p, v0, v1); + let e1 = quad_edge(p, v1, v2); + let e2 = quad_edge(p, v2, v3); + let e3 = quad_edge(p, v3, v0); + let inside = max(max(e0.x, e1.x), max(e2.x, e3.x)); + let border = min(min(e0.y, e1.y), min(e2.y, e3.y)); + if inside < 0.0 { + return -border; + } + return border; +} + +// Coin d'un quad rentre de `r` : intersection des deux aretes adjacentes, +// chacune decalee de `r` vers l'interieur. Meme deroulement que ci-dessus. +fn inset_corner(prev: vec2, cur: vec2, next: vec2, r: f32) -> vec2 { + let ep = cur - prev; + let ec = next - cur; + // TL->TR->BR->BL tourne dans le sens horaire en y-bas, donc (e.y, -e.x) sort du quad. + let np = vec2(ep.y, -ep.x) / max(length(ep), 1e-6); + let nc = vec2(ec.y, -ec.x) / max(length(ec), 1e-6); + return line_cross(np, dot(prev, np) - r, nc, dot(cur, nc) - r); +} + +// (s, t, ok) du warp inverse du mode 8 pour une racine `t` donnee. +fn quad_st_for_root(t: f32, e: vec2, f: vec2, g: vec2, h: vec2) -> vec3 { + let denom_x = e.x + g.x * t; + let denom_y = e.y + g.y * t; + var s: f32; + if abs(denom_x) > abs(denom_y) { + s = (h.x - f.x * t) / denom_x; + } else { + s = (h.y - f.y * t) / denom_y; + } + // Tolerance de 2 % reprise telle quelle du MSL : sans elle une rangee de + // pixels du bord tombe hors du quad par arrondi et l'ecran se liseree. + var ok = 0.0; + if s >= -0.02 && s <= 1.02 && t >= -0.02 && t <= 1.02 { + ok = 1.0; + } + return vec3(s, t, ok); +} + +// (s, t, ok) du point `P` dans le quad c00->c10->c11->c01 : le warp bilineaire INVERSE. +fn quad_inverse_bilinear(P: vec2, c00: vec2, c10: vec2, c11: vec2, c01: vec2) -> vec3 { + let e = c10 - c00; + let f = c01 - c00; + let g = c00 - c10 - c01 + c11; + let h = P - c00; + let k2 = g.x * f.y - g.y * f.x; + let k1 = e.x * f.y - e.y * f.x + h.x * g.y - h.y * g.x; + let k0 = h.x * e.y - h.y * e.x; + // Quad quasi affine (rotation Y pure, p.ex.) : le terme quadratique s'evanouit + // et resoudre la quadratique diviserait par ~0. + if abs(k2) < 1e-5 * abs(k1) { + var t = 0.0; + if abs(k1) >= 1e-6 { + t = -k0 / k1; + } + return quad_st_for_root(t, e, f, g, h); + } + let disc = k1 * k1 - 4.0 * k2 * k0; + if disc < 0.0 { + return vec3(0.0, 0.0, 0.0); + } + // Forme stable de la quadratique : additionner deux termes de meme signe evite + // l'annulation catastrophique que `(-k1 +- sqrt(disc)) / (2 k2)` produit quand + // `disc` approche `k1^2`. `sign()` de WGSL rend 0 en 0, la ou le ternaire MSL + // rend +1 : d'ou le signe explicite. + var sgn = 1.0; + if k1 < 0.0 { + sgn = -1.0; + } + let q = -0.5 * (k1 + sgn * sqrt(disc)); + let r0 = quad_st_for_root(q / k2, e, f, g, h); + var t1 = q / k2; + if abs(q) > 0.0 { + t1 = k0 / q; + } + let r1 = quad_st_for_root(t1, e, f, g, h); + if r0.z > 0.5 { + return r0; + } + return r1; +} + +@fragment +fn fs_main(i: VsOut) -> @location(0) vec4 { + var rgb: vec3; + var alpha: f32; + + if layer.mode < 0.5 { + // Mode 0 — vidéo NV12 + flou de mouvement par vélocité (§8), port 1:1 du + // HLSL/MSL. Pour CE pixel de sortie, l'UV qu'il occupait à la frame + // précédente se retrouve en le remappant par (dst_prev, src_prev) : on + // floute le long de ce segment, ce qui capture la translation ET le zoom + // du calque sans avoir à transporter un champ de vitesse. + let taps = i32(layer.mb.x); + // `taps` d'abord : un draw qui a oublié `dst_prev` le laisse à zéro, et + // la division par `dst_prev.zw` produirait des UV infinis. Dégrader vers + // le chemin net est le seul échec acceptable pour un effet cosmétique. + if taps <= 1 || layer.dst_prev.z <= 0.0 || layer.dst_prev.w <= 0.0 { + rgb = sample_yuv(i.uv); + } else { + let localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; + let uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); + let duv = i.uv - uv_prev; + if dot(duv, duv) < 1e-9 { + rgb = sample_yuv(i.uv); + } else { + // Borne 16 en dur, identique au HLSL et au MSL : `taps` vient d'un + // uniform et une boucle sans borne statique ne se déroule pas. + // L'échelle de l'inspector s'arrête pile à 16 (1 + 15·blur), donc + // c'est `taps` qui coupe, jamais la borne. + var acc = vec3(0.0); + let step = 1.0 / f32(taps - 1); + for (var k: i32 = 0; k < 16; k = k + 1) { + if k >= taps { break; } + acc = acc + sample_yuv(uv_prev + duv * (f32(k) * step)); + } + rgb = acc / f32(taps); + } + } + } else if layer.mode < 1.5 { + // Mode 1 — couleur pleine. + rgb = layer.color.rgb; + } else if layer.mode > 4.5 && layer.mode < 5.5 { + // Mode 5 -- gradient lineaire : color (c0) -> src.rgb (c1) le long de + // la direction fx.xy (sin, -cos de l'angle). Parite avec le HLSL/MSL. + let t = clamp(dot(i.pout - vec2(0.5), layer.fx.xy) + 0.5, 0.0, 1.0); + rgb = mix(layer.color.rgb, layer.src.rgb, t); + } else if layer.mode > 10.5 && layer.mode < 11.5 { + // Mode 11 : texte. texY est l'atlas R8 (couverture alpha au canal .r, + // produit par text_cosmic::TextRasterizer), teinte par layer.color. + // Sortie en alpha premultiplie, comme les autres modes. + let cov = textureSample(texY, samp, i.uv).r; + let a = layer.color.a * cov; + return vec4(layer.color.rgb * a, a); + } else if layer.mode > 8.5 && layer.mode < 9.5 { + // Mode 9 -- annotation « figure » : une fleche. Parite EXACTE avec + // `ArrowSvgs.tsx`, dont chaque direction est un trace de trois segments a + // bouts ronds dans un viewBox 0..100 : une hampe et deux barbes. Trois + // `sd_segment` et un `min` reproduisent la forme telle quelle, pas une + // approximation. Les extremites arrivent deja converties en px locaux du + // quad par `regions::arrow_local_geometry` (echelle uniforme centree, + // comme le `preserveAspectRatio` par defaut du SVG), donc ce shader n'a + // aucune geometrie a deviner. + // + // fx = hampe (a.xy, b.xy), src_prev = barbe 1, dst_prev = barbe 2 ; + // mb.y = demi-epaisseur en px. + var d = sd_segment(i.local, layer.fx.xy, layer.fx.zw); + d = min(d, sd_segment(i.local, layer.src_prev.xy, layer.src_prev.zw)); + d = min(d, sd_segment(i.local, layer.dst_prev.xy, layer.dst_prev.zw)); + // Couverture sur ~1 px : le trait reste net sans crenelage, et une fleche + // fine ne disparait pas quand la demi-epaisseur descend sous le pixel. + let a = clamp(layer.mb.y - d + 0.5, 0.0, 1.0) * layer.color.a; + return vec4(layer.color.rgb * a, a); + } else if layer.mode > 9.5 && layer.mode < 10.5 { + // Mode 10 -- annotation « flou » : masque la zone en reutilisant l'image + // DEJA composee, qui arrive sur texY (recopie mipmappee du render target + // -- on ne peut pas echantillonner la cible sur laquelle on dessine). + // `i.pout` donne directement l'UV de sortie, donc aucun mapping a refaire. + // + // fx.x = 0 mosaique / 1 flou ; fx.y = taille de bloc px (mosaique) ou + // rayon px (flou) ; fx.z = 0 rectangle / 1 ovale ; fx.w = 1 si teinte. + let n = i.local / max(layer.quad_px, vec2(1e-6)); + var cov = 1.0; + if layer.fx.z > 0.5 { + // Ovale inscrit : distance au centre en unites de demi-axes, adoucie + // sur ~1px. + let dc = (n - vec2(0.5)) * 2.0; + let r = length(dc); + let aa = 2.0 / max(min(layer.quad_px.x, layer.quad_px.y), 1.0); + cov = 1.0 - smoothstep(1.0 - aa, 1.0, r); + } + if cov <= 0.0 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + var masked: vec3; + if layer.fx.x > 0.5 { + // Flou : on echantillonne un niveau de mip de l'image composee. + // `log2(rayon)` donne le niveau dont un texel couvre a peu pres le + // rayon demande, et le filtrage trilineaire lisse la transition entre + // deux niveaux quand le rayon varie. + // + // Un noyau de quelques taps espaces du rayon ne floute PAS : il + // superpose autant de copies decalees, ce qui se voit comme du texte + // fantome. Atteindre un vrai lissage par taps demanderait un tap par + // pixel de rayon ; la pyramide de mips donne le meme resultat a cout + // constant, et c'est le GPU qui l'a construite. + let lod = log2(max(layer.fx.y, 1.0)); + masked = textureSampleLevel(texY, samp, i.pout, lod).rgb; + } else { + // Mosaique : on quantifie l'UV sur une grille de `fx.y` px, alignee + // sur le quad pour que les blocs ne rampent pas quand l'annotation + // bouge. + let px_uv = layer.dst.zw / max(layer.quad_px, vec2(1e-6)); + let block = max(layer.fx.y, 1.0) * px_uv; + let origin = layer.dst.xy; + let q = origin + (floor((i.pout - origin) / block) + vec2(0.5)) * block; + // Niveau 0 explicite : l'UV quantifie est une marche d'escalier, donc + // ses derivees explosent en bord de bloc et le choix automatique de + // mip ramollirait justement les aretes qui font la mosaique. + masked = textureSampleLevel(texY, samp, q, 0.0).rgb; + } + if layer.fx.w > 0.5 { + // Teinte blanc/noir : la couleur choisie, melee a moitie, garde la + // forme lisible sans effacer completement ce qu'il y a dessous. + masked = mix(masked, layer.color.rgb, 0.5); + } + let a = cov * layer.color.a; + return vec4(masked * a, a); + } else if layer.mode > 6.5 && layer.mode < 7.5 { + // Mode 7 -- sprite curseur (PNG RGBA, alpha droite) echantillonne sur + // texY (comme le mode 11 y lie son atlas). `fx` = rect de clip "Clip to + // canvas" [x,y,w,h] en sortie 0..1 (= s_dst si actif, sinon un rect + // englobant : sans effet). Sortie en alpha premultiplie. + if i.pout.x < layer.fx.x || i.pout.x > layer.fx.x + layer.fx.z + || i.pout.y < layer.fx.y || i.pout.y > layer.fx.y + layer.fx.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + let s = textureSample(texY, samp, i.uv); + let ca = s.a * layer.color.a; + return vec4(s.rgb * ca, ca); + } else if layer.mode > 5.5 && layer.mode < 6.5 { + // Mode 6 -- fond image (wallpaper RGBA) cover-fit, echantillonne sur + // texY. `src` porte le rect UV cover-fit (calcule cote Rust). Opaque : + // le fond couvre tout le cadre. + return vec4(textureSample(texY, samp, i.uv).rgb, 1.0); + } else if layer.mode > 7.5 && layer.mode < 8.5 { + // Mode 8 -- ecran tilte (rotation 3D des zoom regions). Le quad projete est + // dessine dans sa BBOX (le VS ne sait tracer qu'un rect) et chaque fragment + // remonte au (s,t) du plan par warp bilineaire inverse. + // + // PAS de test de clip sur `dst_prev` : en mode 8 `dst_prev.xy` porte + // `plane_px`, la taille du plan en PIXELS (~1600), la ou `i.pout` vit dans + // [0,1]. Un clip la-dessus serait vrai partout et n'afficherait rien. + let r = quad_inverse_bilinear( + i.local, layer.fx.xy, layer.fx.zw, layer.src_prev.xy, layer.src_prev.zw, + ); + if r.z < 0.5 { + return vec4(0.0, 0.0, 0.0, 0.0); // hors du quad projete + } + // La coupe source s'applique ICI : `r` est une position DANS le plan (0..1), + // pas une coordonnee de texture. Echantillonner `r` directement ignorerait le + // crop utilisateur et le zoom. + let uv = vec2( + mix(layer.src.x, layer.src.z, clamp(r.x, 0.0, 1.0)), + mix(layer.src.y, layer.src.w, clamp(r.y, 0.0, 1.0)), + ); + // Coins arrondis DANS LE REPERE DU PLAN : le rayon reste constant le long du + // bord, la ou un arrondi calcule dans la bbox s'etirerait avec la perspective. + // Inconditionnel, rayon 0 compris -- `sd_round_rect` degenere en SDF de + // rectangle et le feather de 1,5 px subsiste, ce qui fait lire une arete + // inclinee COMME une arete plutot que comme un escalier. + let plane_px = layer.dst_prev.xy; + let p = vec2(r.x, r.y) * plane_px - plane_px * 0.5; + let d = sd_round_rect(p, plane_px * 0.5, max(layer.radius_px, 0.0)); + let tilt_a = 1.0 - smoothstep(0.0, 1.5, d); + // L'alpha est cette couverture, pas `color.a` : les draws du mode 8 laissent + // `color` a zero, donc s'en servir rendrait un plan totalement transparent. + return vec4(sample_yuv(uv) * tilt_a, tilt_a); + } else if layer.mode > 11.5 && layer.mode < 12.5 { + // Mode 12 -- ombre du quad projete. La penombre suit le QUADRILATERE, pas son + // rect englobant : un rect droit derriere un ecran incline se lit comme une + // seconde surface, pas comme son ombre. + // + // `fx`/`src_prev` portent les COINS (en px locaux a la bbox, comme `i.local`), + // et le spread vit dans `mb.y` -- pas dans `fx.x` comme au mode 2. + let tl = layer.fx.xy; + let tr = layer.fx.zw; + let br = layer.src_prev.xy; + let bl = layer.src_prev.zw; + // Coins arrondis du meme rayon que le plan : une ombre a coins vifs derriere un + // ecran arrondi depasse en pointe a chaque coin, d'autant plus que le rayon monte. + let r = max(layer.radius_px, 0.0); + let v0 = inset_corner(bl, tl, tr, r); + let v1 = inset_corner(tl, tr, br, r); + let v2 = inset_corner(tr, br, bl, r); + let v3 = inset_corner(br, bl, tl, r); + let d = sd_convex_quad(i.local, v0, v1, v2, v3) - r; + let spread = max(layer.mb.y, 1e-3); + let a = layer.color.a * (1.0 - smoothstep(0.0, spread, d)); + return vec4(layer.color.rgb * a, a); + } else if layer.mode > 12.5 && layer.mode < 13.5 { + // Mode 13 -- sprite de curseur POSE sur l'ecran incline : ses quatre coins + // ont traverse la meme projection que la video, et le fragment remonte a sa + // position dans le sprite par le meme warp inverse que le mode 8. + // + // Le rect de clip « Clip to canvas » est ici dans `dst_prev` (en sortie + // 0..1, [x,y,w,h]) et NON dans `fx` comme au mode 7 : `fx` porte les coins. + if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z + || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + let r = quad_inverse_bilinear( + i.local, layer.fx.xy, layer.fx.zw, layer.src_prev.xy, layer.src_prev.zw, + ); + if r.z < 0.5 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + // Sprite RGBA a alpha DROITE sur texY (comme le mode 7 y lie le sien) : + // on premultiplie ici. + let s = textureSample(texY, samp, clamp(vec2(r.x, r.y), vec2(0.0), vec2(1.0))); + let ca = s.a * layer.color.a; + return vec4(s.rgb * ca, ca); + } else { + // Mode 2 — ombre portée (SDF d'un quad arrondi élargi de `fx.x`). + let spread = layer.fx.x; + let halfsz = layer.quad_px * 0.5 - vec2(spread); + let p = i.local - layer.quad_px * 0.5; + let d = sd_round_rect(p, halfsz, layer.radius_px); + let a = layer.color.a * (1.0 - smoothstep(0.0, spread, d)); + return vec4(layer.color.rgb * a, a); + } + + alpha = layer.color.a; + + if layer.radius_px > 0.0 { + // Feather ~1.5 px sur le bord du quad — parité exacte avec le HLSL + // (`smoothstep(0.0, 1.5, d)`). Le shader HLSL inclut `quad_px` en px de + // SORTIE ; on reproduit la même chose ici. + let halfsz = layer.quad_px * 0.5; + let p = i.local - layer.quad_px * 0.5; + let d = sd_round_rect(p, halfsz, layer.radius_px); + alpha *= 1.0 - smoothstep(0.0, 1.5, d); + } + + return vec4(rgb * alpha, alpha); // alpha prémultiplié +} diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs new file mode 100644 index 0000000000..9aa60d928b --- /dev/null +++ b/crates/compositor/tests/compose_linux.rs @@ -0,0 +1,1470 @@ +//! Verifie que le port Linux reconciliie sur v1.8.0 REND une frame : +//! `d3d::Gpu` -> `compositor::Compositor` -> `pipeline::Decoder` (les modules +//! Linux, via les alias cfg) -> `compose_frame` (geometrie partagee +//! `plan_frame`) -> `readback_direct`. Bypass le render-thread de `live.rs` +//! pour isoler la chaine de rendu elle-meme. +//! +//! Opt-in (rend sur GPU) : `OPENSCREEN_LINUX_COMPOSE=1` + la fixture +//! `crates/fixture/screen.mp4`. Sinon skip (le teardown Vulkan/Mesa segfault a +//! l'exit apres le rendu -- verifier via la sortie, pas l'exit code). + +// Linux UNIQUEMENT, comme `warp_device_cannot_decode.rs` l'est a Windows. Les +// fichiers de `tests/` sont compiles quelle que soit la plateforme : sans cette +// porte, `cargo check` sous Windows resout `pipeline::Decoder` vers +// `pipeline_windows::Decoder`, qui est `pub(crate)` -- et le check Windows casse +// sur un test qui ne s'y executera jamais. +#![cfg(target_os = "linux")] + +use std::path::Path; + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config::Cfg; +use openscreen_compositor::cursor::CursorTrack; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::pipeline::{ + run_composited_multi, ClipSource, Decoder, ExportCodec, ExportParams, +}; +use openscreen_compositor::scene::Scene; + +const FIXTURE: &str = "../fixture/screen.mp4"; +const W: u32 = 960; +const H: u32 = 540; + +/// Ecrit un PPM P6 dans `OPENSCREEN_VK_OUT` (defaut `target`) pour inspection. +fn write_ppm(name: &str, w: u32, h: u32, rgba: &[u8]) { + use std::io::Write; + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let path = format!("{out}/{name}.ppm"); + let mut f = std::fs::File::create(&path).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + println!("wrote {path}"); +} + +#[test] +fn compose_linux_rend_une_frame() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // Scene : fond gradient + padding (l'ecran est inset -> le fond floute se + // voit tout autour) pour valider visuellement le blur du background. + let scene_json = r##"{"clips":[],"layout":{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0.18,"blur":true,"shadow":0,"roundnessFrac":0.05,"motionBlur":0},"background":{"kind":"gradient","angleDeg":45,"stops":["#ff3b6b","#3b6bff"]},"zoomRegions":[],"annotations":[],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1920,"height":1080,"fps":30}}"##; + comp.set_scene(Some(Scene::from_json(scene_json).expect("scene json"))); + + let (w, h, rgba) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let mut cfg = Cfg::c8(); + cfg.bg_blur = true; + // webcam = screen (mon compose coeur ne dessine que l'ecran). + comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + + let n = (rgba.len() / 4) as f32; + let mut sum = 0u64; + for px in rgba.chunks_exact(4) { + sum += px[0] as u64; + } + let mean_r = sum as f32 / n; + println!("compose_linux : {w}x{h} bytes={} mean_R={:.1}", rgba.len(), mean_r); + + // PPM P6 pour inspection visuelle. + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let ppm = format!("{out}/compose_linux.ppm"); + { + use std::io::Write; + let mut f = std::fs::File::create(&ppm).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + println!("wrote {ppm}"); + + assert_eq!(rgba.len(), (W * H * 4) as usize); + assert!( + mean_r > 5.0 && mean_r < 250.0, + "mean R={mean_r} hors plage plausible (5..250) — frame vide ?" + ); +} + +// --------------------------------------------------------------------------- +// Rotation 3D (modes 8 et 12) +// --------------------------------------------------------------------------- + +/// Scene « ecran seul sur fond plat magenta », avec ou sans preset de rotation. +/// Le fond est une couleur SATUREE que l'enregistrement d'ecran de la fixture ne +/// produit nulle part : c'est ce qui permet de separer l'ecran du fond au pixel +/// pres, donc de mesurer la forme reellement dessinee. +fn tilt_scene_json(rotation: &str, shadow: u32, roundness: f32) -> String { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.2,"blur":false,"shadow":{shadow},"roundnessFrac":{roundness},"motionBlur":0}},"background":{{"kind":"color","color":"#ff00ff"}},"zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":6,"scale":1.0,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) +} + +/// `true` si le pixel n'est PAS le fond magenta. Seuil large : le feather des +/// bords et le degrade du sampler ne doivent pas compter comme du fond. +fn not_bg(px: &[u8]) -> bool { + !(px[0] > 200 && px[1] < 60 && px[2] > 200) +} + +/// Pour chaque colonne, la premiere ligne non-fond. `None` = colonne entierement +/// de fond. C'est la trace du BORD HAUT de ce qui est dessine : horizontale pour +/// un ecran droit, oblique pour un ecran incline. +fn top_edge(rgba: &[u8], w: u32, h: u32) -> Vec> { + (0..w) + .map(|x| { + (0..h).find(|&y| { + let i = ((y * w + x) * 4) as usize; + not_bg(&rgba[i..i + 4]) + }) + }) + .collect() +} + +/// Ecart max du bord haut, mesure sur les colonnes centrales uniquement : aux +/// deux extremites le bord haut d'un quad incline bascule sur le bord LATERAL, +/// ce qui ajouterait une variation qui n'est pas celle qu'on veut mesurer. +fn top_edge_swing(edge: &[Option]) -> u32 { + let n = edge.len(); + let seen: Vec = edge[n / 4..3 * n / 4].iter().flatten().copied().collect(); + match (seen.iter().min(), seen.iter().max()) { + (Some(&lo), Some(&hi)) => hi - lo, + _ => 0, + } +} + +/// Ecran incline (mode 8). Rend DEUX fois la meme scene, seule la rotation +/// change, et compare la silhouette obtenue. +/// +/// L'assertion porte sur la GEOMETRIE, pas sur la presence d'un fichier : le +/// bord haut de l'ecran droit est horizontal a moins de 2 px pres, celui de +/// l'ecran incline balaie des dizaines de lignes. Un mode 8 non branche cote +/// Rust, un warp inverse faux, ou un `quad_st_for_root` qui rejetterait tout +/// casse l'une des trois bornes. +#[test] +fn compose_linux_ecran_tilte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux tilt: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let mut cfg = Cfg::c8(); + cfg.shadow = false; + let (w, h, upright, tilted) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + // `frame` = 90 -> source_t = 3 s, au coeur de la region [0, 6] : la rampe + // d'entree est finie, la rotation est a pleine force. + let render = |json: String| { + let scene = Scene::from_json(&json).expect("scene json"); + // Le padding transite par les live_params, pas la scene brute. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp.compose_frame(sf, sf, 90.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + let (w, h, upright) = render(tilt_scene_json("null", 0, 0.0)); + let tilted: Vec<(&str, Vec)> = ["iso", "left", "right"] + .iter() + .map(|p| (*p, render(tilt_scene_json(&format!("\"{p}\""), 0, 0.0)).2)) + .collect(); + (w, h, upright, tilted) + }; + + write_ppm("compose_linux_tilt_upright", w, h, &upright); + + let up_swing = top_edge_swing(&top_edge(&upright, w, h)); + let up_area = upright.chunks_exact(4).filter(|p| not_bg(p)).count(); + println!("compose_linux tilt : {w}x{h} droit bord_haut={up_swing}px aire={up_area}"); + + // Garde-fou du detecteur lui-meme : si le fond magenta ne separait pas + // proprement l'ecran, le bord de la reference droite ne serait pas plat et + // toute la mesure serait du bruit. + assert!( + up_swing <= 2, + "reference droite : bord haut non horizontal ({up_swing} px) — le detecteur de fond derape" + ); + + // Les TROIS presets. Ils ne donnent pas le meme quadrilatere : iso penche le + // plus, left/right sont dominés par leur rotateY, donc leur quad approche le + // cas quasi affine que `quad_inverse_bilinear` traite par une branche a part. + for (preset, tilted) in &tilted { + write_ppm(&format!("compose_linux_tilt_{preset}"), w, h, tilted); + let swing = top_edge_swing(&top_edge(tilted, w, h)); + let area = tilted.chunks_exact(4).filter(|p| not_bg(p)).count(); + println!("compose_linux tilt {preset} : bord_haut={swing}px aire={area}"); + + // Chaque preset combine un rotateX et un rotateZ non nuls : sur une largeur + // d'ecran de ~600 px le bord haut ne peut pas rester horizontal. + assert!( + swing >= 15, + "{preset} : bord haut plat a {swing} px — mode 8 pas dessine (rect droit ?)" + ); + // Le containment reduit le plan pour qu'il tienne dans le rect d'origine : + // l'aire couverte baisse. La borne basse attrape le cas « mode 8 ne rend + // rien » (quad_inverse_bilinear qui rejette tout, alpha a zero...). + assert!( + area > up_area * 4 / 10 && area < up_area * 95 / 100, + "{preset} : aire {area} hors de (0.40, 0.95) x {up_area} — mode 8 vide ou inopérant" + ); + } +} + +/// Ombre du quad projete (mode 12). L'ombre doit suivre le QUADRILATERE : si +/// elle retombait sur le mode 2 (rect arrondi axis-aligned), sa bordure exterieure +/// serait horizontale en haut. On isole l'ombre en soustrayant le meme rendu sans +/// ombre, puis on mesure la pente de la bordure de la zone assombrie. +#[test] +fn compose_linux_ombre_du_quad_tilte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux ombre tiltee: opt-in. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let (w, h, sans, avec) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let render = |json: String, shadow: bool| { + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + let mut cfg = Cfg::c8(); + cfg.shadow = shadow; + comp.compose_frame(sf, sf, 90.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + // Rayon non nul : c'est la seule facon d'exercer `inset_corner`/`line_cross` + // (le rentrant des coins de l'ombre) et l'arrondi en repere PLAN du mode 8. + let (w, h, sans) = render(tilt_scene_json("\"iso\"", 0, 0.04), false); + let (_, _, avec) = render(tilt_scene_json("\"iso\"", 1, 0.04), true); + (w, h, sans, avec) + }; + + write_ppm("compose_linux_tilt_shadow", w, h, &avec); + + // Masque de l'ombre : pixels du FOND assombris par le calque 12. On ignore + // l'ecran lui-meme (l'ombre passe dessous, il n'y change rien). + let mut mask = vec![false; (w * h) as usize]; + let mut count = 0usize; + for p in 0..(w * h) as usize { + let i = p * 4; + let dark = sans[i] as i32 - avec[i] as i32 > 12 && !not_bg(&sans[i..i + 4]); + mask[p] = dark; + count += dark as usize; + } + // Bordure HAUTE de la penombre, colonne par colonne. + let edge: Vec> = (0..w) + .map(|x| (0..h).find(|&y| mask[(y * w + x) as usize])) + .collect(); + let swing = top_edge_swing(&edge); + println!("compose_linux ombre tiltee : {count} px assombris, bordure haute swing={swing}px"); + + assert!(count > 3000, "ombre absente ({count} px assombris) — mode 12 pas dessine ?"); + // Un repli sur le mode 2 donnerait une bordure haute rigoureusement plate. + assert!( + swing >= 15, + "bordure haute de l'ombre plate a {swing} px — l'ombre est un rect droit, pas le quad projete" + ); +} + +/// Curseur pose sur l'ecran incline (mode 13). Le curseur est place HORS du +/// centre : c'est la que le plan incline le deplace vraiment. Au centre, la +/// position tiltee et la position droite coincident et le test ne prouverait rien. +/// +/// L'assertion est que le sprite BOUGE quand on incline. Un repli sur le +/// placement droit (mode 7) laisserait les deux barycentres au meme endroit ; un +/// mode 13 absent ferait disparaitre le curseur (compte a zero). +#[test] +fn compose_linux_curseur_sur_ecran_tilte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux curseur tilte: opt-in. Skip."); + return; + } + // Sprite vert 16x16 opaque (le meme que le test du mode 7). + const SPRITE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAACXBIWXMAAAABAAAAAQBPJcTWAAAAGElEQVR4nGNk+MdAEmAhTfmohlENQ0kDAGoRATwbkCdPAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let track_path = std::env::temp_dir().join("os_cursor_track_tilt.json"); + std::fs::write( + &track_path, + r#"{"samples":[{"timeMs":3000,"cx":0.22,"cy":0.24,"cursorType":"arrow"}]}"#, + ) + .expect("write track"); + let track = CursorTrack::load(track_path.to_str().unwrap(), 0.0, 6.0).expect("CursorTrack::load"); + comp.set_cursor(track); + comp.set_cursor_time(Some(3.0)); + + let scene_json = |rotation: &str| { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.2,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#ff00ff"}},"zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":6,"scale":1.0,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}],"annotations":[],"cursor":{{"show":true,"size":4,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default","cursorSprites":{{"arrow":{{"path":"{SPRITE}","hotspotX":0.5,"hotspotY":0.5}}}}}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let (w, h, upright, tilted) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let render = |json: String| { + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp.compose_frame(sf, sf, 90.0, &Cfg::c8()).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + let (w, h, upright) = render(scene_json("null")); + let (_, _, tilted) = render(scene_json("\"iso\"")); + (w, h, upright, tilted) + }; + + write_ppm("compose_linux_tilt_cursor", w, h, &tilted); + + // Barycentre des pixels verts du sprite. + let centroid = |rgba: &[u8]| -> (f32, f32, usize) { + let (mut sx, mut sy, mut n) = (0.0f32, 0.0f32, 0usize); + for y in 0..h { + for x in 0..w { + let i = ((y * w + x) * 4) as usize; + if rgba[i + 1] > 180 && rgba[i] < 120 && rgba[i + 2] < 120 { + sx += x as f32; + sy += y as f32; + n += 1; + } + } + } + (sx / n.max(1) as f32, sy / n.max(1) as f32, n) + }; + let (ux, uy, un) = centroid(&upright); + let (tx, ty, tn) = centroid(&tilted); + let shift = ((tx - ux).powi(2) + (ty - uy).powi(2)).sqrt(); + println!( + "compose_linux curseur tilte : droit=({ux:.1},{uy:.1}) n={un} \ + incline=({tx:.1},{ty:.1}) n={tn} deplacement={shift:.1}px" + ); + + assert!(un > 50, "curseur droit absent (n={un}) — la scene de reference est cassee"); + assert!(tn > 50, "curseur absent sous rotation (n={tn}) — mode 13 pas dessine"); + assert!( + shift >= 12.0, + "curseur deplace de {shift:.1}px seulement — il est reste sur le rect droit (mode 7 ?)" + ); +} + +/// Curseur : sprite thematise (mode 7) dessine au centre. Sprite VERT (data URI +/// PNG) distinct du fond sombre et de l'ecran, pour l'affirmer sans ambiguite. +#[test] +fn compose_linux_dessine_le_curseur() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux curseur: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + // Sprite vert 16x16 opaque en data URI (decode_data_uri -> crate image). + const SPRITE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAACXBIWXMAAAABAAAAAQBPJcTWAAAAGElEQVR4nGNk+MdAEmAhTfmohlENQ0kDAGoRATwbkCdPAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // Scene : curseur visible (size 3 pour un sprite bien lisible), sprite "arrow". + let scene_json = format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0.03,"motionBlur":0}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":true,"size":3,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default","cursorSprites":{{"arrow":{{"path":"{SPRITE}","hotspotX":0.5,"hotspotY":0.5}}}}}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ); + comp.set_scene(Some(Scene::from_json(&scene_json).expect("scene json"))); + + // Piste curseur : un echantillon au centre (0.5, 0.5). `load` lit un fichier. + let track_path = std::env::temp_dir().join("os_cursor_track.json"); + std::fs::write( + &track_path, + r#"{"samples":[{"timeMs":0,"cx":0.5,"cy":0.5,"cursorType":"arrow"}]}"#, + ) + .expect("write track"); + let track = CursorTrack::load(track_path.to_str().unwrap(), 0.0, 2.0).expect("CursorTrack::load"); + comp.set_cursor(track); + comp.set_cursor_time(Some(0.0)); + + let (w, h, rgba) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + + // Le sprite vert doit apparaitre franchement (G haut, R/B bas). + let green = rgba + .chunks_exact(4) + .filter(|p| p[1] > 180 && p[0] < 120 && p[2] < 120) + .count(); + println!("compose_linux curseur : {w}x{h} pixels verts={green}"); + + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let ppm = format!("{out}/compose_linux_cursor.ppm"); + { + use std::io::Write; + let mut f = std::fs::File::create(&ppm).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + println!("wrote {ppm}"); + + assert!(green > 50, "sprite curseur vert absent (verts={green}) — mode 7 ?"); +} + +/// Fond image (mode 6 wallpaper) : un PNG orange en data URI remplit le fond +/// (cover-fit) autour de l'ecran inset (padding). Distinct de l'ecran et du +/// gris par defaut, pour l'affirmer sans ambiguite. +#[test] +fn compose_linux_fond_image() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux fond image: opt-in. Skip."); + return; + } + const BG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAACXBIWXMAAAABAAAAAQBPJcTWAAAAKklEQVR4nO3NwQ0AAAQAMRJ721wswa83wDWn47X63QMAAAAAAAAAAIC7FhLfAfuIQEbyAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let scene_json = format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.4,"blur":false,"shadow":0,"roundnessFrac":0.05,"motionBlur":0}},"background":{{"kind":"image","path":"{BG}"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ); + let scene = Scene::from_json(&scene_json).expect("scene json"); + // Le padding (et les autres effets) transitent par les live_params, pas la + // scene brute -> sans ca l'ecran remplit tout le cadre et masque le fond. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + + let (w, h, rgba) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + // Orange (255,128,0) : R haut, G moyen, B bas. + let orange = rgba + .chunks_exact(4) + .filter(|p| p[0] > 200 && p[1] > 90 && p[1] < 170 && p[2] < 70) + .count(); + println!("compose_linux fond image : {w}x{h} pixels orange={orange}"); + + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + { + use std::io::Write; + let mut f = std::fs::File::create(format!("{out}/compose_linux_bgimage.ppm")).expect("ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + assert!(orange > 2000, "fond image absent (orange={orange}) — mode 6 ?"); +} + +/// Flou de mouvement par VELOCITE du calque ecran (mode 0 du shader). +/// +/// La velocite vient d'un zoom en pleine rampe : `plan_frame` calcule alors un +/// `s_dst_prev` different de `s_dst`, et le shader floute chaque pixel le long +/// du segment qui relie son UV d'avant a son UV d'aujourd'hui. +/// +/// La MEME frame decodee est composee deux fois, seul `effects.motionBlur` +/// change — toute difference mesuree ne peut donc venir que de l'effet. Deux +/// assertions, parce que « les deux images different » ne dirait pas dans quel +/// SENS : on verifie aussi que la version floutee a moins de detail haute +/// frequence. Un cablage errone de `src_prev`/`dst_prev` ferait bien differer +/// les images, mais pas forcement dans ce sens-la. +#[test] +fn compose_linux_flou_de_velocite_ecran() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux flou de velocite: opt-in. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // La region de zoom demarre a 2 s ; sa rampe d'entree commence ~1 s plus tot + // (`ZOOM_IN_TRANSITION_WINDOW_S`). A t = 66/60 = 1,1 s on est donc en pleine + // montee : `plan_frame` y donne s_dst 1,629 contre s_dst_prev 1,559, soit + // 4,5 % d'echelle en une frame — largement de quoi etaler le calque. + let scene_of = |mblur: f32| { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":{mblur}}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[{{"id":"z1","startSec":2,"endSec":4,"scale":2.5,"focusX":0.5,"focusY":0.5,"focusMode":"manual","rotation":null}}],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + // UN SEUL `seek_to` : les deux rendus partagent la meme AVFrame, donc le + // decodeur ne peut pas introduire de difference qu'on prendrait pour l'effet. + let (sharp, blurred) = unsafe { + let sf = dec.seek_to(1.1).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + let render = |mblur: f32| { + comp.set_scene(Some(Scene::from_json(&scene_of(mblur)).expect("scene json"))); + comp.compose_frame(sf, sf, 66.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + }; + (render(0.0), render(1.0)) + }; + + write_ppm("compose_linux_mb_screen_off", W, H, &sharp); + write_ppm("compose_linux_mb_screen_on", W, H, &blurred); + + let mut diff_sum = 0u64; + for (a, b) in sharp.chunks_exact(4).zip(blurred.chunks_exact(4)) { + for c in 0..3 { + diff_sum += (a[c] as i32 - b[c] as i32).unsigned_abs() as u64; + } + } + let mean_diff = diff_sum as f32 / (W * H * 3) as f32; + + // Detail haute frequence : somme des gradients voisins sur le canal vert. + let sharpness = |img: &[u8]| -> f32 { + let g = |x: u32, y: u32| img[((y * W + x) * 4 + 1) as usize] as i32; + let mut acc = 0u64; + for y in 0..H - 1 { + for x in 0..W - 1 { + acc += (g(x + 1, y) - g(x, y)).unsigned_abs() as u64; + acc += (g(x, y + 1) - g(x, y)).unsigned_abs() as u64; + } + } + acc as f32 / ((W - 1) * (H - 1) * 2) as f32 + }; + let (s_sharp, s_blur) = (sharpness(&sharp), sharpness(&blurred)); + println!( + "compose_linux flou de velocite : mean_diff={mean_diff:.2} gradient net={s_sharp:.2} floute={s_blur:.2}" + ); + + // Mesure observee : mean_diff 12,5 et gradient 7,9 -> 3,0. Les seuils gardent + // de la marge tout en restant loin du « ca a bouge d'un poil ». + assert!( + mean_diff > 4.0, + "motionBlur 0 vs 1 rend (quasi) la MEME image (mean_diff={mean_diff:.3}) — mb/src_prev/dst_prev non cables ?" + ); + assert!( + s_blur < s_sharp * 0.7, + "le rendu floute n'est pas plus doux (gradient {s_blur:.2} vs {s_sharp:.2}) — le flou ne suit pas la velocite" + ); +} + +/// Meme flou de velocite, mais sur le calque CAMERA — un draw distinct, avec son +/// propre `src_prev` (qui doit suivre le cover-crop et le miroir) et son propre +/// `dst_prev`. +/// +/// La velocite vient d'une region « Full Camera » en pleine ouverture : la boite +/// camera passe de 0,526 a 0,579 de large en une frame pendant que `s_dst` ne +/// bouge PAS d'un pouce. C'est ce qui rend le test concluant — une difference +/// mesuree dans la boite camera ne peut pas venir du calque ecran, et +/// l'assertion sur le coin haut-gauche (hors boite) le verifie explicitement. +#[test] +fn compose_linux_flou_de_velocite_camera() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux flou de velocite camera: opt-in. Skip."); + return; + } + let webcam_fixture = "../fixture/webcam.mp4"; + if !Path::new(webcam_fixture).is_file() { + eprintln!("compose_linux flou de velocite camera: pas de fixture webcam. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + let mut cam = Decoder::open(webcam_fixture, &gpu).expect("Decoder::open webcam"); + + // `webcamMirror: true` : le miroir inverse les bornes u de `src`, et + // `src_prev` doit inverser les MEMES. S'il gardait l'ancien [0,0,1,1] la + // reprojection viserait une zone de texture jamais affichee. + let scene_of = |mblur: f32| { + format!( + r##"{{"clips":[],"layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":true,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":{mblur}}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[],"cameraFullscreenRegions":[{{"startSec":2,"endSec":4}}],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let (sharp, blurred) = unsafe { + let sf = screen.seek_to(2.1).expect("seek screen"); + let wf = cam.seek_to(2.1).expect("seek webcam"); + let cfg = Cfg::c8(); + let render = |mblur: f32| { + let scene = Scene::from_json(&scene_of(mblur)).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + // frame 126 = t 2,1 s : la camera est a mi-ouverture. + comp.compose_frame(sf, wf, 126.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + }; + (render(0.0), render(1.0)) + }; + + write_ppm("compose_linux_mb_camera_off", W, H, &sharp); + write_ppm("compose_linux_mb_camera_on", W, H, &blurred); + + let mean_diff = |x0: u32, x1: u32, y0: u32, y1: u32| -> f32 { + let mut sum = 0u64; + for y in y0..y1 { + for x in x0..x1 { + let i = ((y * W + x) * 4) as usize; + for c in 0..3 { + sum += (sharp[i + c] as i32 - blurred[i + c] as i32).unsigned_abs() as u64; + } + } + } + sum as f32 / ((x1 - x0) * (y1 - y0) * 3) as f32 + }; + // Boite camera a t = 2,1 s : [0,411 ; 0,403 ; 0,579 ; 0,579] de la sortie, + // soit x 394..950 et y 217..530 en pixels — retrecie ici pour rester loin des + // bords adoucis. Le coin haut-gauche, lui, ne montre que l'ecran. + let inside = mean_diff(420, 920, 245, 505); + let outside = mean_diff(0, 300, 0, 150); + println!("compose_linux flou de velocite camera : dans la boite={inside:.2} hors boite={outside:.4}"); + + assert!( + inside > 4.0, + "la camera n'est pas floutee (diff={inside:.3}) — src_prev/dst_prev du calque webcam non cables ?" + ); + assert!( + outside < 0.01, + "l'ecran a bouge aussi (diff={outside:.3}) — le test ne prouve alors rien sur la camera" + ); +} + +/// Trainee fantome du curseur (accumulation temporelle, pas un mode de shader). +/// +/// Le curseur traverse le cadre ; a `cursor.motionBlur = 1` `plan_cursor` rend +/// 11 taps entre sa position d'il y a 8 frames (8/60 s) et sa position courante. +/// On compare au meme rendu sans trainee : la seule difference possible etant le +/// curseur, un exces de vert la ou le curseur N'EST PAS (mais est PASSE) est la +/// signature de la trainee. +/// +/// « Exces de vert » = G - max(R,B), pas G brut : une copie a 1/taps d'opacite +/// sur un fond CLAIR fait a peine monter le vert (le fond y est deja) mais fait +/// nettement chuter le rouge et le bleu. Mesurer G seul rendrait le test +/// dependant de ce qui passe sous le curseur dans la video. +/// +/// Le trajet est volontairement hors de l'axe median (cy = 0,28) : une passe de +/// composition qui retournerait `accum` verticalement enverrait la trainee dans +/// la bande miroir, ce que la seconde assertion interdit. A cy = 0,5 le defaut +/// serait invisible. +#[test] +fn compose_linux_trainee_de_curseur() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux trainee curseur: opt-in. Skip."); + return; + } + const SPRITE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAACXBIWXMAAAABAAAAAQBPJcTWAAAAGElEQVR4nGNk+MdAEmAhTfmohlENQ0kDAGoRATwbkCdPAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // Piste : deplacement horizontal regulier cx 0,1 -> 0,9 en 0,4 s, a cy fixe. + // Assez rapide pour que les 8/60 s de recul de la trainee separent nettement + // les deux extremites (~256 px a 960 de large) : sans quoi la trainee se + // superpose au curseur lui-meme et on ne pourrait plus les distinguer. + let mut samples = String::new(); + for k in 0..=8 { + let (ms, cx) = (k * 50, 0.1 + 0.1 * k as f32); + if k > 0 { + samples.push(','); + } + samples.push_str(&format!( + r#"{{"timeMs":{ms},"cx":{cx},"cy":0.28,"cursorType":"arrow"}}"# + )); + } + let track_path = std::env::temp_dir().join("os_cursor_trail_track.json"); + std::fs::write(&track_path, format!(r#"{{"samples":[{samples}]}}"#)).expect("write track"); + let track = CursorTrack::load(track_path.to_str().unwrap(), 0.0, 2.0).expect("CursorTrack::load"); + comp.set_cursor(track); + comp.set_cursor_time(Some(0.35)); + + let scene_of = |mblur: f32| { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":true,"size":3,"smoothing":0,"motionBlur":{mblur},"clickBounce":0,"clipToBounds":false,"theme":"default","cursorSprites":{{"arrow":{{"path":"{SPRITE}","hotspotX":0.5,"hotspotY":0.5}}}}}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let (sharp, trail) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + let render = |mblur: f32| { + let scene = Scene::from_json(&scene_of(mblur)).expect("scene json"); + // `cursor.motionBlur` ET `cursor.size` transitent par les LiveParams, + // pas par la scene brute : sans ca `plan_cursor` verrait toujours 0. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp.compose_frame(sf, sf, 15.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + }; + (render(0.0), render(1.0)) + }; + + write_ppm("compose_linux_cursor_trail_off", W, H, &sharp); + write_ppm("compose_linux_cursor_trail_on", W, H, &trail); + + // A t = 0,35 s le curseur est en cx 0,8 (x ~ 768 px) et 8/60 s plus tot en + // cx ~ 0,533 (x ~ 512 px) ; le sprite fait 51 px de cote a size 3 (34/1080 + // de frame_min_px, x3), donc le curseur COURANT occupe x = 742..794. La + // fenetre ci-dessous couvre le milieu du trajet, franchement a sa gauche : + // sans trainee il n'y a rien du tout. La bande miroir est son reflet par + // rapport a l'axe horizontal de l'image (cy = 0,28 est hors de cet axe + // exprès), donc un `accum` composite a l'envers y atterrirait. + let greener = |x0: u32, x1: u32, y0: u32, y1: u32| -> usize { + let excess = |img: &[u8], i: usize| { + img[i + 1] as i32 - (img[i] as i32).max(img[i + 2] as i32) + }; + let mut n = 0; + for y in y0..y1 { + for x in x0..x1 { + let i = ((y * W + x) * 4) as usize; + if excess(&trail, i) - excess(&sharp, i) > 10 { + n += 1; + } + } + } + n + }; + let on_path = greener(530, 700, 130, 172); + let mirrored = greener(530, 700, 368, 410); + println!("compose_linux trainee curseur : sur le trajet={on_path} bande miroir={mirrored}"); + + // La fenetre fait 170x42 = 7140 px et la trainee la remplit entierement. + // Le seuil a 4000 laisse de la marge tout en refusant une trainee qui ne + // couvrirait qu'un bout du trajet. + assert!( + on_path > 4000, + "pas de trainee au milieu du trajet ({on_path} px plus verts) — le curseur n'est dessine qu'a sa position courante" + ); + assert!( + mirrored < 50, + "trainee dans la bande MIROIR ({mirrored} px) — la passe de composition d'accum retourne l'image en Y" + ); +} + +/// Export (WP6) : ~1s de la fixture -> MP4 H264 software. Verifie que la marche +/// de timeline + l'encodeur + le muxer produisent un fichier non trivial. Le +/// contenu est re-validable par ffprobe (cf. la commande dans le run manuel). +#[test] +fn export_linux_mp4() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("export_linux: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + // Petite sortie : l'export est un smoke test, pas un bench. + let comp = Compositor::new_sized(&gpu, 640, 360).expect("Compositor::new_sized"); + + let out = std::env::var("OPENSCREEN_EXPORT_OUT") + .unwrap_or_else(|_| std::env::temp_dir().join("os_export_linux.mp4").to_string_lossy().into()); + let clips = vec![ClipSource { + screen: FIXTURE.to_string(), + webcam: FIXTURE.to_string(), + source_start_sec: 0.0, + source_end_sec: 1.0, + webcam_offset_sec: 0.0, + has_audio: true, + }]; + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + }; + + let mut last = 0u64; + let stats = run_composited_multi( + &clips, + &out, + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |n| last = n, + ) + .expect("run_composited_multi"); + println!( + "export_linux : {} frames, {:.1} fps encode, {:.2}s video, progress={last} -> {out}", + stats.frames, stats.fps, stats.video_duration_s + ); + + assert!(stats.frames > 0, "aucune frame exportee"); + let meta = std::fs::metadata(&out).expect("mp4 metadata"); + assert!(meta.len() > 2000, "mp4 trop petit ({} octets) — muxer ?", meta.len()); +} + +/// Rend une frame qui exerce EN MEME TEMPS les trois corrections de cette +/// serie : ombre portee (ecran + camera), cover-crop de la webcam sous un +/// masque CERCLE (le cas ou l'etirement etait le plus violent : la boite est +/// forcee carree, donc une camera 16:9 s'ecrasait de 1,78x), et une annotation +/// texte avec un fond. +/// +/// Opt-in comme les autres tests de ce fichier ; ecrit un PPM a inspecter. +#[test] +fn compose_linux_ombre_webcam_ronde_et_texte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux ombre/webcam/texte: opt-in. Skip."); + return; + } + let webcam_fixture = "../fixture/webcam.mp4"; + if !Path::new(webcam_fixture).is_file() { + eprintln!("compose_linux ombre/webcam/texte: pas de fixture webcam. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + let mut cam = Decoder::open(webcam_fixture, &gpu).expect("Decoder::open webcam"); + + // `shadow: 1` + camera en cercle + une annotation texte visible a t=1s. + let scene_json = r##"{"clips":[],"layout":{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"circle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0.14,"blur":false,"shadow":1,"roundnessFrac":0.04,"motionBlur":0},"background":{"kind":"gradient","angleDeg":45,"stops":["#1f2933","#3b6bff"]},"zoomRegions":[],"annotations":[{"id":"a1","kind":"text","x":0.08,"y":0.08,"w":0.5,"h":0.14,"startSec":0,"endSec":10,"zIndex":1,"text":{"content":"Ombre + fond","color":"#ffffff","backgroundColor":"#e0245e","fontSizeRel":0.09,"fontFamily":"","fontWeight":"normal","fontStyle":"normal","textDecoration":"none","textAlign":"center"}}],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1920,"height":1080,"fps":30}}"##; + let parsed = Scene::from_json(scene_json).expect("scene json"); + // Cf. le commentaire dans compose_linux_forme_webcam_cercle : sans les + // LiveParams, la scene est parsee et ignoree. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + + let (w, h, rgba) = unsafe { + let sf = screen.seek_to(1.0).expect("seek screen"); + let wf = cam.seek_to(1.0).expect("seek webcam"); + let mut cfg = Cfg::c8(); + cfg.shadow = true; + comp.compose_frame(sf, wf, 1.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let ppm = format!("{out}/compose_linux_shadow_webcam_text.ppm"); + { + use std::io::Write; + let mut f = std::fs::File::create(&ppm).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + println!("wrote {ppm}"); + + // L'annotation a un fond ROSE (#e0245e) : il doit exister des pixels + // nettement rouges-magenta dans le quart haut-gauche, ce qui n'etait pas le + // cas quand la plaque n'etait pas dessinee du tout. + let mut plate_px = 0usize; + for y in 0..(h / 3) { + for x in 0..(w / 2) { + let i = ((y * w + x) * 4) as usize; + let (r, g_, b) = (rgba[i] as i32, rgba[i + 1] as i32, rgba[i + 2] as i32); + if r > 140 && g_ < 90 && b > 40 && b < 140 { + plate_px += 1; + } + } + } + assert!( + plate_px > 200, + "fond d'annotation introuvable ({plate_px} px roses) — la plaque n'est pas dessinee" + ); +} + +/// Forme de la webcam : `rectangle` contre `circle`. +/// +/// Le masque n'est pas un mode de shader dedie — il sort de `radius_px`, que +/// `plan_frame` met a la moitie du cote pour `circle`. Ce test le MESURE au +/// lieu de le supposer. +/// +/// La methode : rendre trois fois la meme scene — sans camera, camera +/// rectangle, camera cercle — et diffe chacune des deux dernieres contre la +/// premiere. Le diff EST l'empreinte de la camera, masque compris, sans avoir +/// a deviner ou `plan_frame` l'a posee ni a distinguer la camera du fond. Un +/// disque remplit pi/4 ~= 0,785 de sa boite englobante, un rectangle la +/// remplit entierement : le taux de remplissage separe les deux sans ambiguite. +#[test] +fn compose_linux_forme_webcam_cercle() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux forme webcam: opt-in. Skip."); + return; + } + let webcam_fixture = "../fixture/webcam.mp4"; + if !Path::new(webcam_fixture).is_file() { + eprintln!("compose_linux forme webcam: pas de fixture webcam. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + let mut cam = Decoder::open(webcam_fixture, &gpu).expect("Decoder::open webcam"); + + let scene = |preset: &str, shape: &str| { + format!( + r##"{{"clips":[],"layout":{{"preset":"{preset}","webcamSize":1.6,"webcamShape":"{shape}","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.1,"blur":false,"shadow":0,"roundnessFrac":0.0,"motionBlur":0}},"background":{{"kind":"color","color":"#00ff00"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let mut render = |preset: &str, shape: &str| -> Vec { + let parsed = Scene::from_json(&scene(preset, shape)).expect("scene json"); + // OBLIGATOIRE. `compose_frame` lit les LiveParams, PAS la scene brute : + // la forme webcam, le padding et les effets y transitent. Sans cette + // ligne la scene est parsee mais ignoree, et le test mesure la forme par + // defaut ("rounded") en croyant mesurer celle qu'il a demandee. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek screen"); + let wf = cam.seek_to(1.0).expect("seek webcam"); + let mut cfg = Cfg::c8(); + cfg.shadow = false; + comp.compose_frame(sf, wf, 1.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + let none = render("no-webcam", "rectangle"); + let rect = render("picture-in-picture", "rectangle"); + let circle = render("picture-in-picture", "circle"); + write_ppm("compose_linux_webcam_rect", W, H, &rect); + write_ppm("compose_linux_webcam_circle", W, H, &circle); + + // Empreinte = pixels qui changent quand la camera apparait. + let footprint = |with: &[u8]| -> Vec { + with.chunks_exact(4) + .zip(none.chunks_exact(4)) + .map(|(a, b)| { + (a[0] as i32 - b[0] as i32).abs() + + (a[1] as i32 - b[1] as i32).abs() + + (a[2] as i32 - b[2] as i32).abs() + > 24 + }) + .collect() + }; + // Taux de remplissage de la boite englobante de l'empreinte. + let fill = |mask: &[bool], label: &str| -> f32 { + let (mut x0, mut y0, mut x1, mut y1) = (W, H, 0u32, 0u32); + let mut n = 0u32; + for y in 0..H { + for x in 0..W { + if mask[(y * W + x) as usize] { + x0 = x0.min(x); y0 = y0.min(y); x1 = x1.max(x); y1 = y1.max(y); n += 1; + } + } + } + assert!(x1 > x0 && y1 > y0, "{label} : aucune empreinte de camera"); + let (bw, bh) = (x1 - x0 + 1, y1 - y0 + 1); + let ar = bw as f32 / bh as f32; + let f = n as f32 / (bw * bh) as f32; + println!("{label} : boite {bw}x{bh} (AR {ar:.3}), {n} px, remplissage {f:.3}"); + f + }; + + let rect_fill = fill(&footprint(&rect), "rectangle"); + let circle_fill = fill(&footprint(&circle), "cercle"); + + assert!(rect_fill > 0.95, "le rectangle devrait remplir sa boite ({rect_fill:.3})"); + // pi/4 = 0,785 ; on tolere l'antialiasing du SDF sur le pourtour. + assert!( + (circle_fill - 0.785).abs() < 0.06, + "le masque cercle ne rogne pas comme un disque (remplissage {circle_fill:.3}, attendu ~0.785)" + ); +} + +/// Un enregistrement SANS camera ne doit rien dessiner dans la boite PiP. +/// +/// Le cas est reproduit tel quel : le decodeur « webcam » recoit la frame de +/// l'ECRAN, ce que `open_and_seek_clip` fait en production des que le chemin +/// webcam est vide ou illisible. Seul `LiveParams::has_webcam` distingue alors +/// une vraie camera d'une seconde copie de l'ecran, et ce backend ne le lisait +/// pas : l'enregistrement d'ecran apparaissait dans sa propre vignette. +/// +/// L'assertion est une egalite stricte avec le rendu « no-webcam » : pas un +/// seuil, parce qu'il ne s'agit pas de mesurer une empreinte plus petite mais +/// de verifier qu'il n'y en a aucune. +#[test] +fn compose_linux_sans_camera_ne_dessine_pas_de_vignette() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux sans camera: opt-in. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + + let scene_json = r##"{"clips":[],"layout":{"preset":"picture-in-picture","webcamSize":1.6,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0.1,"blur":false,"shadow":0,"roundnessFrac":0.0,"motionBlur":0},"background":{"kind":"color","color":"#00ff00"},"zoomRegions":[],"annotations":[],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1920,"height":1080,"fps":30}}"##; + + let mut render = |has_webcam: bool| -> Vec { + let parsed = Scene::from_json(scene_json).expect("scene json"); + let mut lp = openscreen_compositor::compositor::live_params_from_scene(&parsed); + lp.has_webcam = has_webcam; + comp.set_live_params(lp); + comp.set_scene(Some(parsed)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek screen"); + let mut cfg = Cfg::c8(); + cfg.shadow = false; + // La frame ecran passee AUSSI comme webcam : le repli exact de + // `open_and_seek_clip` quand il n'y a pas de fichier camera. + comp.compose_frame(sf, sf, 1.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + let with_camera = render(true); + let without_camera = render(false); + write_ppm("compose_linux_sans_camera", W, H, &without_camera); + + let differing = with_camera + .chunks_exact(4) + .zip(without_camera.chunks_exact(4)) + .filter(|(a, b)| { + (a[0] as i32 - b[0] as i32).abs() + + (a[1] as i32 - b[1] as i32).abs() + + (a[2] as i32 - b[2] as i32).abs() + > 24 + }) + .count(); + println!("vignette ecran-dans-la-camera : {differing} px"); + assert!( + differing > 1000, + "le rendu de controle ne dessine aucune vignette — le test ne prouve rien ({differing} px)" + ); + + // Reference : le preset qui ne veut pas de camera du tout. Il pose le meme + // rectangle d'ecran (`plan_frame` : « no-webcam » et « picture-in-picture » + // partagent `full_screen`), donc seule la vignette peut les separer. + let no_webcam_preset = { + let parsed = Scene::from_json(&scene_json.replace( + r#""preset":"picture-in-picture""#, + r#""preset":"no-webcam""#, + )) + .expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek screen"); + let mut cfg = Cfg::c8(); + cfg.shadow = false; + comp.compose_frame(sf, sf, 1.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + let residual = without_camera + .chunks_exact(4) + .zip(no_webcam_preset.chunks_exact(4)) + .filter(|(a, b)| { + (a[0] as i32 - b[0] as i32).abs() + + (a[1] as i32 - b[1] as i32).abs() + + (a[2] as i32 - b[2] as i32).abs() + > 24 + }) + .count(); + assert_eq!( + residual, 0, + "sans camera, le rendu devrait etre celui du preset no-webcam ({residual} px d'ecart)" + ); +} + +// --------------------------------------------------------------------------- +// Annotations : figure (fleche), flou/mosaique, image, et animations du texte. +// +// Ces quatre familles existaient dans le schema et arrivaient jusqu'au +// compositeur, mais le chemin Linux ne dessinait QUE le texte -- tout le reste +// etait ignore en silence. Les tests ci-dessous les MESURENT sur le GPU au lieu +// de supposer qu'un draw ajoute suffit : la methode est toujours la meme, rendre +// deux fois la meme scene (avec et sans l'annotation) et lire l'empreinte dans +// le diff. Elle ne demande de connaitre ni ou `plan_frame` a pose l'ecran, ni +// quelle couleur la video porte a cet endroit. +// --------------------------------------------------------------------------- + +/// Scene de base des tests d'annotation : fond uni, pas d'effets, une liste +/// d'annotations injectee telle quelle. +fn annotation_scene(annotations: &str) -> String { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.1,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#00ff00"}},"zoomRegions":[],"annotations":[{annotations}],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) +} + +/// Indices des pixels qui different entre deux rendus. C'est l'empreinte exacte +/// de ce que l'annotation a ajoute. +fn changed_pixels(a: &[u8], b: &[u8]) -> Vec { + a.chunks_exact(4) + .zip(b.chunks_exact(4)) + .enumerate() + .filter(|(_, (p, q))| { + // Seuil 6/255 : au-dessus du bruit de quantification du YUV->RGB, + // bien en-dessous de tout trait ou masque reel. + (0..3).any(|c| (p[c] as i32 - q[c] as i32).abs() > 6) + }) + .map(|(i, _)| i) + .collect() +} + +/// Boite englobante (x0, y0, x1, y1) inclusive d'une liste d'indices de pixels. +fn bbox(px: &[usize], w: u32) -> (u32, u32, u32, u32) { + let (mut x0, mut y0, mut x1, mut y1) = (u32::MAX, u32::MAX, 0u32, 0u32); + for &i in px { + let (x, y) = (i as u32 % w, i as u32 / w); + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x); + y1 = y1.max(y); + } + (x0, y0, x1, y1) +} + +/// Une fleche est un TRACE, pas un aplat — et elle suit sa direction. +/// +/// Deux pieges que ce test ferme. Le premier : ne rien dessiner du tout, ce que +/// faisait le chemin Linux. Le second, plus sournois : dessiner le quad entier +/// (un mode inconnu tombe sur la branche « ombre » du shader et remplit la +/// boite), ce qui se voit comme un rectangle colore et non comme une fleche. +/// L'aire couverte les separe : trois segments d'epaisseur fixe ne peuvent pas +/// remplir la moitie de leur boite. +#[test] +fn compose_linux_annotation_fleche() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux annotation fleche: opt-in. Skip."); + return; + } + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let mut render = |annotations: &str| -> Vec { + let parsed = Scene::from_json(&annotation_scene(annotations)).expect("scene json"); + // Cf. compose_linux_forme_webcam_cercle : sans les LiveParams la scene + // est parsee puis ignoree, et le test mesure les valeurs par defaut. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + // Le 3e argument de `compose_frame` est un NUMERO DE FRAME (source_t = + // frame / 60), pas des secondes. `set_timeline_time` fixe directement + // l'instant que lit la fenetre temporelle des annotations -- sans lui, + // une annotation a `startSec: 1` ne serait tout simplement pas visible, + // et un test d'animation mesurerait sa propre erreur de cadrage. + comp.set_timeline_time(Some(1.0)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek"); + comp.compose_frame(sf, std::ptr::null(), 60.0, &Cfg::c8()).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + let figure = |direction: &str| { + format!( + r##"{{"id":"f1","kind":"figure","x":0.2,"y":0.2,"w":0.4,"h":0.4,"startSec":0,"endSec":10,"zIndex":1,"figure":{{"direction":"{direction}","color":"#ff0000","strokeWidth":8}}}}"## + ) + }; + let none = render(""); + let right = render(&figure("right")); + let up = render(&figure("up")); + + let right_px = changed_pixels(&none, &right); + let up_px = changed_pixels(&none, &up); + assert!( + right_px.len() > 200, + "aucune fleche dessinee ({} px changes) — le mode 9 n'atteint pas le shader", + right_px.len() + ); + + // La boite fait 0.4 x 0.4 du rect ecran ; la fleche s'y inscrit en carre + // (preserveAspectRatio). Un trait de 8/100 d'epaisseur sur trois segments + // couvre nettement moins de la moitie de ce carre. + let (x0, y0, x1, y1) = bbox(&right_px, W); + let box_area = ((x1 - x0 + 1) * (y1 - y0 + 1)) as f64; + let fill = right_px.len() as f64 / box_area; + assert!( + fill < 0.5, + "la fleche remplit {fill:.2} de sa boite — c'est un aplat, pas un trace" + ); + + // La direction est vraiment lue : « right » et « up » sont deux tracés + // differents, donc leurs empreintes ne peuvent pas coincider. + let common = right_px + .iter() + .collect::>() + .intersection(&up_px.iter().collect::>()) + .count(); + let overlap = common as f64 / right_px.len().min(up_px.len()) as f64; + assert!( + overlap < 0.75, + "« right » et « up » se recouvrent a {overlap:.2} — la direction est ignoree" + ); +} + +/// Le flou floute vraiment, et la mosaique fait des blocs. +/// +/// Mesure sur l'ENERGIE HAUTE FREQUENCE (somme des ecarts entre voisins +/// horizontaux) dans la zone masquee. Compter des pixels changes ne suffirait +/// pas : un masque qui recopierait la frame telle quelle changerait aussi des +/// pixels au bord et passerait. Ce qu'on veut prouver, c'est que le detail a +/// DISPARU — c'est la seule propriete qui rend l'annotation utile. +#[test] +fn compose_linux_annotation_flou_et_mosaique() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux annotation flou: opt-in. Skip."); + return; + } + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let mut render = |annotations: &str| -> Vec { + let parsed = Scene::from_json(&annotation_scene(annotations)).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + // Le 3e argument de `compose_frame` est un NUMERO DE FRAME (source_t = + // frame / 60), pas des secondes. `set_timeline_time` fixe directement + // l'instant que lit la fenetre temporelle des annotations -- sans lui, + // une annotation a `startSec: 1` ne serait tout simplement pas visible, + // et un test d'animation mesurerait sa propre erreur de cadrage. + comp.set_timeline_time(Some(1.0)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek"); + comp.compose_frame(sf, std::ptr::null(), 60.0, &Cfg::c8()).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + // Boite bien a l'interieur du rect ecran, sur de la video (pas sur le fond). + let blur_ann = |style: &str, amount: f32| { + format!( + r##"{{"id":"b1","kind":"blur","x":0.25,"y":0.25,"w":0.5,"h":0.5,"startSec":0,"endSec":10,"zIndex":1,"blur":{{"style":"{style}","shape":"rectangle","color":"white","intensity":{amount},"blockSize":{amount}}}}}"## + ) + }; + let none = render(""); + let blurred = render(&blur_ann("blur", 24.0)); + let mosaic = render(&blur_ann("mosaic", 16.0)); + + let changed = changed_pixels(&none, &blurred); + assert!( + changed.len() > 2000, + "le flou n'a rien change ({} px) — le mode 10 n'atteint pas le shader", + changed.len() + ); + let (x0, y0, x1, y1) = bbox(&changed, W); + + // Energie haute frequence sur le canal vert, a l'interieur de la zone, en + // s'ecartant du bord (les 2 px de bord melangent masque et image nette). + let hf = |px: &[u8]| -> f64 { + let mut sum = 0f64; + let mut n = 0usize; + for y in (y0 + 2)..=(y1 - 2) { + for x in (x0 + 2)..(x1 - 2) { + let i = ((y * W + x) * 4) as usize; + let j = i + 4; + sum += (px[i + 1] as i32 - px[j + 1] as i32).abs() as f64; + n += 1; + } + } + sum / n.max(1) as f64 + }; + let sharp_hf = hf(&none); + let blur_hf = hf(&blurred); + assert!( + sharp_hf > 1.0, + "la fixture est trop plate a cet endroit ({sharp_hf:.2}) pour mesurer un flou" + ); + assert!( + blur_hf < sharp_hf * 0.5, + "detail toujours present sous le flou : {blur_hf:.2} contre {sharp_hf:.2} sans masque" + ); + + // Mosaique : a l'interieur d'un bloc les pixels sont IDENTIQUES, donc la + // proportion de voisins strictement egaux explose par rapport a l'image + // nette. C'est la signature d'un aplat par blocs, qu'un simple flou n'a pas. + let flat_ratio = |px: &[u8]| -> f64 { + let (mut eq, mut n) = (0usize, 0usize); + for y in (y0 + 2)..=(y1 - 2) { + for x in (x0 + 2)..(x1 - 2) { + let i = ((y * W + x) * 4) as usize; + let j = i + 4; + if px[i..i + 3] == px[j..j + 3] { + eq += 1; + } + n += 1; + } + } + eq as f64 / n.max(1) as f64 + }; + let sharp_flat = flat_ratio(&none); + let mosaic_flat = flat_ratio(&mosaic); + assert!( + mosaic_flat > sharp_flat + 0.3, + "pas de blocs : {mosaic_flat:.2} de voisins egaux contre {sharp_flat:.2} sans masque" + ); +} + +/// Une image d'annotation tient dans sa boite SANS etre etiree. +/// +/// C'est le meme defaut que celui corrige pour la webcam : coller la source au +/// rect deforme tout ce qui n'a pas exactement son rapport. On rend une image +/// 4:1 dans une boite qui ne l'est pas, et on mesure le rapport de l'empreinte +/// — il doit rester 4:1, quelle que soit la boite. +#[test] +fn compose_linux_annotation_image() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux annotation image: opt-in. Skip."); + return; + } + // Aplat magenta 400x100 : un rapport 4:1 franc, et une couleur que la + // fixture ne porte pas. + let img_path = std::env::temp_dir().join("openscreen-annotation-4x1.png"); + let img = image::RgbaImage::from_pixel(400, 100, image::Rgba([255, 0, 255, 255])); + img.save(&img_path).expect("ecrire le png de test"); + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let mut render = |annotations: &str| -> Vec { + let parsed = Scene::from_json(&annotation_scene(annotations)).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + // Le 3e argument de `compose_frame` est un NUMERO DE FRAME (source_t = + // frame / 60), pas des secondes. `set_timeline_time` fixe directement + // l'instant que lit la fenetre temporelle des annotations -- sans lui, + // une annotation a `startSec: 1` ne serait tout simplement pas visible, + // et un test d'animation mesurerait sa propre erreur de cadrage. + comp.set_timeline_time(Some(1.0)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek"); + comp.compose_frame(sf, std::ptr::null(), 60.0, &Cfg::c8()).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + let none = render(""); + // Boite carree en fraction du rect ecran — donc PAS carree en pixels, et + // dans tous les cas pas 4:1. + let with_image = render(&format!( + r##"{{"id":"i1","kind":"image","x":0.25,"y":0.3,"w":0.4,"h":0.4,"startSec":0,"endSec":10,"zIndex":1,"imagePath":"{}"}}"##, + img_path.display() + )); + + let changed = changed_pixels(&none, &with_image); + assert!( + changed.len() > 500, + "aucune image dessinee ({} px changes)", + changed.len() + ); + let (x0, y0, x1, y1) = bbox(&changed, W); + let (bw, bh) = ((x1 - x0 + 1) as f64, (y1 - y0 + 1) as f64); + let aspect = bw / bh; + assert!( + (aspect - 4.0).abs() < 0.25, + "image etiree : empreinte {bw}x{bh} (rapport {aspect:.2}), attendu 4:1" + ); + + // Et c'est bien l'image qui est peinte, pas un aplat de la couleur du bord : + // le centre doit etre magenta. + let (cx, cy) = ((x0 + x1) / 2, (y0 + y1) / 2); + let i = ((cy * W + cx) * 4) as usize; + let (r, g_, b) = (with_image[i] as i32, with_image[i + 1] as i32, with_image[i + 2] as i32); + assert!( + r > 200 && g_ < 80 && b > 200, + "centre de l'empreinte non magenta : ({r}, {g_}, {b})" + ); + let _ = std::fs::remove_file(&img_path); +} + +/// Les animations d'apparition du texte sont JOUEES. +/// +/// `text_anim.rs` etait porte, teste unitairement et transporte par la scene, +/// mais aucun appelant Linux ne l'invoquait : une annotation animee s'affichait +/// simplement d'un bloc. On rend la MEME frame video a deux instants differents +/// de l'animation en deplaçant `startSec` — le seul ecart possible entre les +/// deux rendus est donc l'animation elle-meme. +#[test] +fn compose_linux_animation_texte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux animation texte: opt-in. Skip."); + return; + } + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let mut render = |annotations: &str| -> Vec { + let parsed = Scene::from_json(&annotation_scene(annotations)).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + // Le 3e argument de `compose_frame` est un NUMERO DE FRAME (source_t = + // frame / 60), pas des secondes. `set_timeline_time` fixe directement + // l'instant que lit la fenetre temporelle des annotations -- sans lui, + // une annotation a `startSec: 1` ne serait tout simplement pas visible, + // et un test d'animation mesurerait sa propre erreur de cadrage. + comp.set_timeline_time(Some(1.0)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek"); + comp.compose_frame(sf, std::ptr::null(), 60.0, &Cfg::c8()).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + // `startSec` deplace l'instant DANS l'animation sans toucher la frame video : + // a t=1.0s, start=1.0 donne 0 ms ecoulees, start=0.0 en donne 1000 (fini). + let text = |animation: &str, start: f32| { + format!( + r##"{{"id":"t1","kind":"text","x":0.1,"y":0.1,"w":0.6,"h":0.2,"startSec":{start},"endSec":10,"zIndex":1,"text":{{"content":"Hello","color":"#ffffff","backgroundColor":"transparent","fontSizeRel":0.12,"fontFamily":"","fontWeight":"bold","fontStyle":"normal","textDecoration":"none","textAlign":"center","animation":"{animation}"}}}}"## + ) + }; + let none = render(""); + let settled = render(&text("fade", 0.0)); + let ink_settled = changed_pixels(&none, &settled).len(); + assert!( + ink_settled > 200, + "aucun texte rendu ({ink_settled} px) — le test ne mesure rien" + ); + + // Fondu a 0 ms : opacite 0, donc RIEN ne doit apparaitre. Sans l'animation + // le texte serait deja a pleine opacite et ce compte vaudrait `ink_settled`. + let starting = render(&text("fade", 1.0)); + let ink_starting = changed_pixels(&none, &starting).len(); + assert!( + ink_starting < ink_settled / 10, + "le fondu n'est pas applique : {ink_starting} px a 0 ms contre {ink_settled} px a la fin" + ); + + // Machine a ecrire a mi-course (350 ms sur 700) : la moitie gauche du bloc + // est revelee, donc l'empreinte est nettement plus etroite qu'a la fin. + let typed_full = render(&text("typewriter", 0.0)); + let typed_half = render(&text("typewriter", 0.65)); + let full_px = changed_pixels(&none, &typed_full); + let half_px = changed_pixels(&none, &typed_half); + assert!(!half_px.is_empty(), "la machine a ecrire n'a rien revele a mi-course"); + let full_right = bbox(&full_px, W).2; + let half_right = bbox(&half_px, W).2; + assert!( + half_right < full_right, + "le texte n'est pas revele progressivement : bord droit {half_right} a mi-course \ + contre {full_right} a la fin" + ); +} diff --git a/crates/compositor/tests/remux_seek_index.rs b/crates/compositor/tests/remux_seek_index.rs new file mode 100644 index 0000000000..5e209bc823 --- /dev/null +++ b/crates/compositor/tests/remux_seek_index.rs @@ -0,0 +1,164 @@ +//! Le remux doit doter un Matroska « live » (sans index) de ses `Cues`. +//! +//! C'est le filet du correctif « les enregistrements Linux ne sont pas +//! seekables » : `MediaRecorder` écrit son WebM en flux live, donc sans `Cues` +//! ni `SeekHead`, et `av_seek_frame` échoue alors pour tout timestamp non nul. +//! +//! Le test est AUTONOME — il ne dépend ni d'un binaire ffmpeg ni d'une fixture +//! média (celles de `crates/fixture/` ne sont pas versionnées, cf. +//! `export_timing.rs` qui doit être opt-in pour cette raison). Il fabrique son +//! entrée avec le muxer matroska lui-même en mode `live=1`, qui est précisément +//! le mode « je ne peux pas revenir en arrière » que subit `MediaRecorder` : +//! aucun `Cues` n'est écrit. Une piste `rawvideo` porte quelques octets +//! arbitraires — le test est de niveau CONTENEUR, la charge utile n'est jamais +//! décodée. + +use openscreen_compositor::ffi::*; +use openscreen_compositor::remux::remux_to_seekable_matroska; +use std::ffi::CString; +use std::ptr; + +/// ID EBML de `Cues` (0x1C53BB6B) et de `SeekHead` (0x114D9B74), en big-endian +/// tels qu'ils apparaissent tels quels dans les octets du fichier. +const CUES_ID: &[u8] = &[0x1C, 0x53, 0xBB, 0x6B]; +const SEEKHEAD_ID: &[u8] = &[0x11, 0x4D, 0x9B, 0x74]; + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) +} + +/// Écrit un Matroska minimal SANS `Cues`, en forçant l'option `live` du muxer. +/// +/// Retourne le nombre de paquets écrits, pour que le test puisse vérifier que le +/// remux les retrouve tous. +fn write_live_matroska(path: &str, packets: i64) -> i64 { + let cpath = CString::new(path).unwrap(); + let cfmt = CString::new("matroska").unwrap(); + unsafe { + let mut octx: *mut AVFormatContext = ptr::null_mut(); + assert!( + avformat_alloc_output_context2( + &mut octx, + ptr::null(), + cfmt.as_ptr(), + cpath.as_ptr() + ) >= 0, + "alloc_output_context2" + ); + + let st = avformat_new_stream(octx, ptr::null()); + assert!(!st.is_null(), "avformat_new_stream"); + // VP8 : matroska le range en `V_VP8` sans jamais regarder la charge + // utile (contrairement à H264, qui exige un `extradata` avcC valide, et + // à RAWVIDEO, que le muxer refuse). Le test reste donc de niveau + // conteneur, sans encodeur ni bitstream réel. + (*(*st).codecpar).codec_type = AVMediaType::AVMEDIA_TYPE_VIDEO; + (*(*st).codecpar).codec_id = AVCodecID::AV_CODEC_ID_VP8; + (*(*st).codecpar).width = 16; + (*(*st).codecpar).height = 16; + // 1 ms par tick : les timestamps du test sont alors directement des ms. + (*st).time_base = AVRational { num: 1, den: 1000 }; + + let mut pb: *mut AVIOContext = ptr::null_mut(); + assert!( + avio_open(&mut pb, cpath.as_ptr(), AVIO_FLAG_WRITE as i32) >= 0, + "avio_open" + ); + sn_fmt_set_pb(octx, pb); + + // `live=1` : le muxer se comporte comme s'il ne pouvait pas revenir en + // arrière — pas de Cues. C'est la contrainte que subit MediaRecorder. + let mut opts: *mut AVDictionary = ptr::null_mut(); + let k = CString::new("live").unwrap(); + let v = CString::new("1").unwrap(); + av_dict_set(&mut opts, k.as_ptr(), v.as_ptr(), 0); + assert!(avformat_write_header(octx, &mut opts) >= 0, "write_header"); + av_dict_free(&mut opts); + + // Contenu arbitraire : jamais décodé, seul le conteneur est testé. + let payload = vec![0x42u8; 256]; + let mut pkt = av_packet_alloc(); + for i in 0..packets { + assert!(av_new_packet(pkt, payload.len() as i32) >= 0, "av_new_packet"); + ptr::copy_nonoverlapping(payload.as_ptr(), (*pkt).data, payload.len()); + (*pkt).stream_index = 0; + (*pkt).pts = i * 40; // 25 fps en timebase 1/1000 + (*pkt).dts = i * 40; + (*pkt).duration = 40; + (*pkt).flags = AV_PKT_FLAG_KEY as i32; + assert!( + av_interleaved_write_frame(octx, pkt) >= 0, + "interleaved_write_frame" + ); + } + av_packet_free(&mut pkt); + assert!(av_write_trailer(octx) >= 0, "write_trailer"); + avio_closep(&mut pb); + avformat_free_context(octx); + } + packets +} + +#[test] +fn remux_adds_cues_to_a_live_matroska() { + // `avformat_find_stream_info` tente de décoder la charge utile bidon pour + // deviner les paramètres du flux et crache un « Invalid sync code » par + // paquet. C'est attendu ici (et sans effet : les `codecpar` sont déjà + // renseignés par le muxer d'entrée) ; on coupe le log pour que la sortie du + // test reste lisible. + unsafe { av_log_set_level(AV_LOG_QUIET) }; + + let dir = std::env::temp_dir().join(format!("openscreen-remux-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let input = dir.join("live.webm"); + let output = dir.join("indexed.webm"); + let input_s = input.to_str().unwrap().to_string(); + let output_s = output.to_str().unwrap().to_string(); + + let written = write_live_matroska(&input_s, 25); + + // Prémisse du correctif : l'entrée n'a PAS d'index. Si cette assertion + // tombe un jour, c'est le mode `live` du muxer qui a changé, et le test ne + // prouve plus rien — mieux vaut qu'il échoue ici que silencieusement. + let before = std::fs::read(&input).unwrap(); + assert!( + !contains(&before, CUES_ID), + "l'entrée de test ne doit pas avoir de Cues (mode live)" + ); + + let stats = remux_to_seekable_matroska(&input_s, &output_s).expect("remux"); + + let after = std::fs::read(&output).unwrap(); + assert!(contains(&after, CUES_ID), "la sortie doit contenir des Cues"); + assert!( + contains(&after, SEEKHEAD_ID), + "la sortie doit contenir un SeekHead" + ); + assert_eq!(stats.packets, written as u64, "tous les paquets recopiés"); + assert_eq!(stats.streams, 1); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn remux_refuses_to_write_over_its_own_input() { + // Garde-fou : écrire sur l'entrée détruirait la seule copie des pixels dès + // la première erreur d'écriture. Le contrat « passe un chemin temporaire » + // est vérifié, pas seulement documenté. + let err = remux_to_seekable_matroska("/tmp/same.webm", "/tmp/same.webm") + .expect_err("doit refuser entrée == sortie"); + assert!(err.to_string().contains("identiques"), "message: {err}"); +} + +#[test] +fn remux_reports_an_error_for_a_missing_input() { + // Le caller TS traite toute erreur comme « garde l'original » ; encore + // faut-il qu'une entrée absente en produise une plutôt que de paniquer. + let out = std::env::temp_dir().join("openscreen-remux-never-written.webm"); + let err = remux_to_seekable_matroska( + "/nonexistent/openscreen/no-such-recording.webm", + out.to_str().unwrap(), + ) + .expect_err("doit échouer sur une entrée absente"); + assert!(err.to_string().contains("avformat_open_input"), "message: {err}"); +} diff --git a/crates/compositor/wrapper_linux.h b/crates/compositor/wrapper_linux.h new file mode 100644 index 0000000000..e01a31a97b --- /dev/null +++ b/crates/compositor/wrapper_linux.h @@ -0,0 +1,14 @@ +/* Linux ffmpeg wrapper for bindgen (PR #183). + * + * Software/VAAPI decode+encode only: no D3D11VA (Windows) nor VideoToolbox + * (macOS) hwcontext headers, which pull platform-specific system headers + * (d3d11.h / CoreVideo) that don't exist on Linux. The generic hwcontext.h is + * kept for AVHWDeviceContext should the VAAPI path need it later. */ +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/crates/poc-d3d/Cargo.toml b/crates/poc-d3d/Cargo.toml index 34ba4e7ce4..4c42f58200 100644 --- a/crates/poc-d3d/Cargo.toml +++ b/crates/poc-d3d/Cargo.toml @@ -15,4 +15,7 @@ path = "src/main.rs" [dependencies] openscreen-compositor.workspace = true anyhow.workspace = true + +# GUI Win32 : Windows-only (le POC ne se compile qu'en no-op ailleurs). +[target.'cfg(windows)'.dependencies] windows.workspace = true diff --git a/crates/poc-d3d/src/main.rs b/crates/poc-d3d/src/main.rs index 6a8a43f163..9d5365faf9 100644 --- a/crates/poc-d3d/src/main.rs +++ b/crates/poc-d3d/src/main.rs @@ -1,9 +1,19 @@ //! Binaire du POC de mesure : GUI Win32 de preview/export, harnais de bench fps, mode live. //! Tout le rendu vient d'`openscreen-compositor` — ce crate ne fait que le piloter et le mesurer. +// GUI Win32 + bench encodeurs Windows : Windows-only. Sur les autres plateformes +// le POC n'a rien a piloter (le rendu/bench passe par les tests du crate compositor). +#[cfg(windows)] mod app; +#[cfg(windows)] mod bench; +#[cfg(windows)] fn main() -> anyhow::Result<()> { bench::run() } + +#[cfg(not(windows))] +fn main() { + eprintln!("poc-d3d : bench GUI Win32 (Windows-only), rien a faire sur cette plateforme."); +} diff --git a/electron-builder.json5 b/electron-builder.json5 index 72ee1ea420..b121858774 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -108,7 +108,13 @@ { "from": "electron/native/bin", "to": "electron/native/bin", - "filter": ["linux-*/*"] + // `**`, not `*`: the capture helper's ffmpeg libraries live one level + // deeper, in `linux-x64/ffmpeg/`. They are in their own directory + // because `linux-x64/` also holds the compositor addon's copies of the + // same sonames with every symbol renamed to `osff_*`, and the helper + // must not find those — see scripts/build-linux-pipewire-helper.mjs. + // A single-level filter ships a helper that cannot start. + "filter": ["linux-*/**"] } ] }, diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 0cad8d4c12..dd761b0b05 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -185,6 +185,45 @@ interface Window { message?: string; error?: string; }>; + isNativeLinuxCaptureAvailable: () => Promise<{ + success: boolean; + available: boolean; + helperPath?: string; + reason?: "unsupported-platform" | "missing-helper" | string; + error?: string; + }>; + startNativeLinuxRecording: ( + request: import("../src/lib/nativeLinuxRecording").NativeLinuxRecordingRequest, + ) => Promise; + pauseNativeLinuxRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + resumeNativeLinuxRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + stopNativeLinuxRecording: (discard?: boolean) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + discarded?: boolean; + error?: string; + }>; + attachNativeLinuxWebcamRecording: (payload: { + screenVideoPath: string; + recordingId: number; + webcam: import("../src/lib/recordingSession").RecordedVideoAssetInput; + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode; + webcamOffsetMs?: number; + }) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + error?: string; + }>; discardCursorTelemetry: (recordingId: number) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 7dc8c1092b..4a285394d3 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -16,6 +16,10 @@ import { shell, systemPreferences, } from "electron"; +import { + type NativeLinuxRecordingRequest, + portalCursorMode, +} from "../../src/lib/nativeLinuxRecording"; import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { @@ -52,10 +56,13 @@ import { mainLogBuffer } from "../diagnostics/main-log-buffer"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; import { findMediaLinksByFingerprint, registerMediaLinks } from "../media/mediaLinksRegistry"; +import { LinuxNativeCaptureSession } from "../native-bridge/capture/linuxNativeCaptureSession"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; import { requestMacCursorAccessibilityAccess } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; +import { findPipeWireCursorHelperPath } from "../native-bridge/cursor/recording/pipeWireCursorRecordingSession"; import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; import { patchWebmDurationOnDisk } from "../recording/webm-duration"; +import { reindexRecordingOnDisk } from "../recording/webm-seek-index"; import { registerNativeBridgeHandlers } from "./nativeBridge"; import { RecordingStreamRegistry, registerRecordingStreamHandlers } from "./recordingStream"; @@ -307,6 +314,37 @@ async function finalizeRecordingFile( return streamed; } +/** + * Give a finalized recording a usable container, by whichever route works. + * + * MediaRecorder omits the `Duration` header, so without repair the editor sees + * `duration = Infinity` and cannot scale its timeline. + * + * Two routes, in order of quality: + * 1. A full container remux through libavformat's matroska muxer (Linux, where + * capture goes through MediaRecorder). The muxer derives `Duration` from + * the real packet timestamps rather than from the renderer's wall-clock + * measurement, and writes `Cues`/`SeekHead` on the way past. + * 2. The EBML header patch, which splices the caller's `durationMs` into the + * Info section. Used on Windows/macOS, and on Linux whenever the remux is + * unavailable (no addon, or a `.node` predating it) or fails. + * + * Both rewrite the file exactly once, so route 1 is not the more expensive one — + * it is the same I/O for a strictly better result. Either way a failure leaves + * the original file intact; a recording is never lost to a failed repair. + */ +async function repairRecordingContainer(filePath: string, durationMs: number): Promise { + const reindexed = await reindexRecordingOnDisk(filePath); + if (reindexed.reindexed) { + console.info( + `[recording] re-muxed ${path.basename(filePath)}: ${reindexed.packets} packets, ` + + `${reindexed.streams} streams, ${reindexed.wallS.toFixed(3)}s`, + ); + return; + } + await patchWebmDurationOnDisk(filePath, durationMs); +} + async function getApprovedProjectSession( project: unknown, projectFilePath?: string, @@ -466,6 +504,51 @@ let nativeMacPauseRanges: Array<{ startMs: number; endMs: number }> = []; let nativeMacIsPaused = false; // Global frame of the region captured by the SCK helper (see getSelectedSourceBounds). let activeMacCaptureBounds: Rectangle | null = null; +let linuxNativeCaptureSession: LinuxNativeCaptureSession | null = null; +let linuxNativeCaptureRecordingId: number | null = null; +let linuxNativeCaptureCursorMode: CursorCaptureMode = "editable-overlay"; +/** + * The portal's restore token, kept between recordings. + * + * Without it the compositor raises its source picker on EVERY recording, which + * on Wayland is the single most intrusive thing about capturing at all. The + * portal issues the token only after a successful session and honours it until + * the user revokes it in system settings, so it is safe to persist and useless + * to anyone else. + */ +const LINUX_RESTORE_TOKEN_FILE = "linux-capture-restore-token.json"; + +async function readLinuxRestoreToken(): Promise { + try { + const raw = await fs.readFile( + path.join(app.getPath("userData"), LINUX_RESTORE_TOKEN_FILE), + "utf-8", + ); + const parsed = JSON.parse(raw) as { restoreToken?: unknown }; + return typeof parsed.restoreToken === "string" && parsed.restoreToken + ? parsed.restoreToken + : undefined; + } catch { + // Absent or unreadable: the picker appears, which is the old behaviour + // and not worth failing a recording over. + return undefined; + } +} + +async function writeLinuxRestoreToken(restoreToken?: string) { + if (!restoreToken) { + return; + } + try { + await fs.writeFile( + path.join(app.getPath("userData"), LINUX_RESTORE_TOKEN_FILE), + JSON.stringify({ restoreToken }, null, 2), + "utf-8", + ); + } catch (error) { + console.warn("Could not persist the Linux portal restore token:", error); + } +} function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { if (!sample || typeof sample !== "object") { @@ -866,6 +949,16 @@ async function resolveDirectShowWebcamClsid(deviceName?: string) { } async function startCursorRecording(recordingId?: number) { + // On Linux the native capture helper already produces cursor samples, from + // the SAME portal session that produces the pixels. Spawning the cursor-only + // helper alongside it would open a SECOND portal session — and since + // SelectSources may be called once per session, that means a second picker + // in front of the user, for a stream nothing consumes. The double prompt is + // precisely what the native path exists to remove. + if (linuxNativeCaptureSession) { + return; + } + if (cursorRecordingSession) { pendingCursorRecordingData = await cursorRecordingSession.stop(); cursorRecordingSession = null; @@ -1779,6 +1872,187 @@ export function registerIpcHandlers( : { success: true, available: false, reason: "missing-helper" }; }); + ipcMain.handle("is-native-linux-capture-available", async () => { + if (process.platform !== "linux") { + return { success: true, available: false, reason: "unsupported-platform" }; + } + + const helperPath = findPipeWireCursorHelperPath(); + return helperPath + ? { success: true, available: true, helperPath } + : { success: true, available: false, reason: "missing-helper" }; + }); + + ipcMain.handle( + "start-native-linux-recording", + async (_, request: NativeLinuxRecordingRequest) => { + try { + if (process.platform !== "linux") { + return { success: false, error: "Native Linux capture requires Linux." }; + } + if (linuxNativeCaptureSession) { + return { success: false, error: "Native Linux capture is already running." }; + } + if (!findPipeWireCursorHelperPath()) { + return { success: false, error: "Native Linux capture helper is not available." }; + } + + const recordingId = + typeof request?.recordingId === "number" && Number.isFinite(request.recordingId) + ? request.recordingId + : Date.now(); + const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + const cursorCaptureMode = + normalizeCursorCaptureMode(request?.cursor?.mode) ?? "editable-overlay"; + + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + const restoreToken = await readLinuxRestoreToken(); + + const session = new LinuxNativeCaptureSession({ + outputPath, + cursorMode: portalCursorMode(cursorCaptureMode), + fps: request.video.fps, + ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), + audio: { + system: { enabled: request.audio.system.enabled }, + microphone: { + enabled: request.audio.microphone.enabled, + ...(request.audio.microphone.deviceName + ? { deviceName: request.audio.microphone.deviceName } + : {}), + gain: request.audio.microphone.gain, + }, + }, + maxCursorSamples: MAX_CURSOR_SAMPLES, + ...(restoreToken ? { restoreToken } : {}), + }); + + console.info("[native-linux] starting capture", { + outputPath, + cursor: { mode: cursorCaptureMode }, + audio: request.audio, + video: request.video, + }); + + await session.start(); + // Blocks until the user answers the portal picker, which has no + // upper bound — the countdown UI is already showing by now. + await session.waitUntilCapturing(); + + linuxNativeCaptureSession = session; + linuxNativeCaptureRecordingId = recordingId; + linuxNativeCaptureCursorMode = cursorCaptureMode; + + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(true, source.name); + } + + return { success: true, recordingId, path: outputPath }; + } catch (error) { + console.error("Failed to start native Linux recording:", error); + linuxNativeCaptureSession = null; + linuxNativeCaptureRecordingId = null; + linuxNativeCaptureCursorMode = "editable-overlay"; + return { success: false, error: String(error) }; + } + }, + ); + + ipcMain.handle("pause-native-linux-recording", async () => { + if (!linuxNativeCaptureSession) { + return { success: false, error: "Native Linux capture is not running." }; + } + linuxNativeCaptureSession.pause(); + return { success: true }; + }); + + ipcMain.handle("resume-native-linux-recording", async () => { + if (!linuxNativeCaptureSession) { + return { success: false, error: "Native Linux capture is not running." }; + } + linuxNativeCaptureSession.resume(); + return { success: true }; + }); + + ipcMain.handle("stop-native-linux-recording", async (_, discard?: boolean) => { + const session = linuxNativeCaptureSession; + const recordingId = linuxNativeCaptureRecordingId ?? Date.now(); + const cursorCaptureMode = linuxNativeCaptureCursorMode; + + if (!session) { + return { success: false, error: "Native Linux capture is not running." }; + } + + try { + if (discard) { + session.discard(); + const discarded = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + await Promise.all([ + fs.rm(discarded, { force: true }), + fs.rm(`${discarded}.cursor.json`, { force: true }), + ]); + return { success: true, discarded: true }; + } + + const result = await session.stop(); + await writeLinuxRestoreToken(result.restoreToken); + + // The helper collects cursor samples itself, from the same portal + // session that produced the pixels, so there is no separate sampler + // to stop and no clock offset to correct — the two are one recording. + if (cursorCaptureMode === "editable-overlay" && result.cursor.samples.length > 0) { + await fs.writeFile( + `${result.path}.cursor.json`, + JSON.stringify(result.cursor, null, 2), + "utf-8", + ); + } + + const session_: RecordingSession = { + screenVideoPath: result.path, + createdAt: recordingId, + cursorCaptureMode, + }; + setCurrentRecordingSessionState(session_); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(result.path).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session_, null, 2), "utf-8"); + await registerRecordingMediaLinks(result.path, { cursorCaptureMode }); + + console.info("[native-linux] capture stored", { + path: result.path, + frames: result.frames, + droppedFrames: result.droppedFrames, + durationMs: result.durationMs, + videoEncoder: result.videoEncoder, + cursorSamples: result.cursor.samples.length, + }); + + return { + success: true, + path: result.path, + session: session_, + message: "Native Linux recording session stored successfully", + }; + } catch (error) { + console.error("Failed to stop native Linux recording:", error); + return { success: false, error: String(error) }; + } finally { + linuxNativeCaptureSession = null; + linuxNativeCaptureRecordingId = null; + linuxNativeCaptureCursorMode = "editable-overlay"; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(false, source.name); + } + } + }); + ipcMain.handle( "start-native-windows-recording", async (_, request: NativeWindowsRecordingRequest) => { @@ -2394,26 +2668,40 @@ export function registerIpcHandlers( } }); - ipcMain.handle( - "attach-native-mac-webcam-recording", - async (_, payload: AttachNativeMacWebcamRecordingInput) => { - try { - if (process.platform !== "darwin") { - return { success: false, error: "Native macOS webcam attachment requires macOS." }; - } - + /** + * Writes a browser-recorded webcam clip next to a natively-recorded screen + * video and rewrites the session manifest to include both. + * + * Shared by macOS and Linux, whose native helpers both leave the camera to + * the renderer's `MediaRecorder`: opening the device a second time from the + * helper would fight the preview for an exclusive claim, and buys nothing + * that the screen capture needs. (Windows is the exception — its helper takes + * the webcam through DirectShow so both streams share one start clock.) + * + * Nothing in here is platform-specific; the two `ipcMain.handle` calls below + * differ only in which platform they accept. + */ + const attachNativeWebcamRecording = async ( + platformLabel: string, + payload: AttachNativeMacWebcamRecordingInput, + ) => { + try { + { const screenVideoPath = normalizeVideoSourcePath(payload.screenVideoPath); if (!screenVideoPath || !isPathWithinDir(screenVideoPath, RECORDINGS_DIR)) { return { success: false, - error: "Native macOS webcam attachment requires a recording output path.", + error: `Native ${platformLabel} webcam attachment requires a recording output path.`, }; } await fs.access(screenVideoPath, fsConstants.R_OK); if (!payload.webcam?.fileName || !payload.webcam.videoData) { - return { success: false, error: "Native macOS webcam attachment is missing video data." }; + return { + success: false, + error: `Native ${platformLabel} webcam attachment is missing video data.`, + }; } const webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); @@ -2452,15 +2740,35 @@ export function registerIpcHandlers( success: true, path: screenVideoPath, session, - message: "Native macOS webcam recording attached successfully", - }; - } catch (error) { - console.error("Failed to attach native macOS webcam recording:", error); - return { - success: false, - error: error instanceof Error ? error.message : String(error), + message: `Native ${platformLabel} webcam recording attached successfully`, }; } + } catch (error) { + console.error(`Failed to attach native ${platformLabel} webcam recording:`, error); + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }; + + ipcMain.handle( + "attach-native-mac-webcam-recording", + async (_, payload: AttachNativeMacWebcamRecordingInput) => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS webcam attachment requires macOS." }; + } + return attachNativeWebcamRecording("macOS", payload); + }, + ); + + ipcMain.handle( + "attach-native-linux-webcam-recording", + async (_, payload: AttachNativeMacWebcamRecordingInput) => { + if (process.platform !== "linux") { + return { success: false, error: "Native Linux webcam attachment requires Linux." }; + } + return attachNativeWebcamRecording("Linux", payload); }, ); @@ -2539,15 +2847,15 @@ export function registerIpcHandlers( } // Streamed files lack the WebM Duration header (renderer no longer holds the - // blob), so patch on disk for the editor's seek bar and timeline. Best-effort, - // independent per file, so they run together. + // blob), so repair the container on disk for the editor's seek bar and timeline. + // Best-effort, independent per file, so they run together. if (isValidDurationMs(payload.durationMs)) { const patches: Promise[] = []; if (screenStreamed) { - patches.push(patchWebmDurationOnDisk(screenVideoPath, payload.durationMs)); + patches.push(repairRecordingContainer(screenVideoPath, payload.durationMs)); } if (webcamStreamed && webcamVideoPath) { - patches.push(patchWebmDurationOnDisk(webcamVideoPath, payload.durationMs)); + patches.push(repairRecordingContainer(webcamVideoPath, payload.durationMs)); } await Promise.all(patches); } diff --git a/electron/native-bridge/capture/linuxNativeCaptureSession.ts b/electron/native-bridge/capture/linuxNativeCaptureSession.ts new file mode 100644 index 0000000000..96c5917cff --- /dev/null +++ b/electron/native-bridge/capture/linuxNativeCaptureSession.ts @@ -0,0 +1,403 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import type { Readable, Writable } from "node:stream"; +import type { CursorRecordingData } from "../../../src/native/contracts"; +import { + NdjsonLineReader, + PipeWireCursorAccumulator, + type PipeWireHelperEvent, +} from "../cursor/recording/pipeWireCursorAccumulator"; +import { findPipeWireCursorHelperPath } from "../cursor/recording/pipeWireCursorRecordingSession"; + +/** + * Drives the Linux capture helper for a full recording: video, audio and cursor + * telemetry from ONE portal session. + * + * WHY ONE SESSION MATTERS ENOUGH TO BUILD THIS. `SelectSources` may be called + * once per portal session, so a second process means a second picker. Before + * this, Linux users picked a source twice — once for Chromium's + * `getDisplayMedia` and once for the cursor helper — and the two captures could + * legitimately be of different things, because each picker was answered + * separately. That is also why the cursor overlay never lined up: the telemetry + * described one stream and the pixels came from another. + * + * TIMEOUTS. `ready` arrives in milliseconds and means the helper loaded + * libpipewire and reached the portal. `capture-started` cannot arrive until a + * human has clicked through the compositor's dialog, which has no upper bound — + * so it is deliberately NOT on a timer. The caller shows its own UI for that + * wait; a timeout here would cancel recordings whose user was merely reading the + * dialog. + */ + +const READY_TIMEOUT_MS = 5_000; +const STOP_TIMEOUT_MS = 15_000; + +export interface LinuxCaptureConfig { + outputPath: string; + cursorMode: "metadata" | "embedded"; + fps: number; + bitrate?: number; + audio: { + system: { enabled: boolean }; + microphone: { enabled: boolean; deviceName?: string; gain: number }; + }; + maxCursorSamples: number; + /** From a previous recording. Lets the portal skip its picker. */ + restoreToken?: string; +} + +export interface LinuxCaptureResult { + path: string; + durationMs: number; + frames: number; + droppedFrames: number; + cursor: CursorRecordingData; + /** Present when the portal issued one; persist it for the next recording. */ + restoreToken?: string; + videoEncoder?: string; +} + +export class LinuxNativeCaptureSession { + private readonly cursor: PipeWireCursorAccumulator; + private readonly lines = new NdjsonLineReader(); + private process: ChildProcessByStdio | null = null; + + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + + private startedResolve: (() => void) | null = null; + private startedReject: ((error: Error) => void) | null = null; + + private stopped: LinuxCaptureResult | null = null; + private stoppedResolve: (() => void) | null = null; + private stoppedReject: ((error: Error) => void) | null = null; + + private restoreToken: string | undefined; + private videoEncoder: string | undefined; + private lastError: string | null = null; + private paused = false; + + constructor(private readonly config: LinuxCaptureConfig) { + this.cursor = new PipeWireCursorAccumulator(config.maxCursorSamples); + } + + /** + * Spawns the helper and resolves once it reports `ready` — which is BEFORE + * the portal picker is raised. Call [`waitUntilCapturing`] to wait for the + * user to answer it. + */ + async start(): Promise { + const helperPath = findPipeWireCursorHelperPath(); + if (!helperPath) { + throw new Error( + "The Linux capture helper is not available. Build it with `npm run build:native:linux`.", + ); + } + + this.cursor.reset(Date.now()); + this.lines.reset(); + + const request = { + outputPath: this.config.outputPath, + cursorMode: this.config.cursorMode, + video: { + fps: this.config.fps, + ...(this.config.bitrate ? { bitrate: this.config.bitrate } : {}), + }, + audio: this.config.audio, + ...(this.config.restoreToken ? { restoreToken: this.config.restoreToken } : {}), + }; + + const child = spawn(helperPath, [JSON.stringify(request)], { + stdio: ["pipe", "pipe", "pipe"], + }); + this.process = child; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.handleStdout(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + console.error("[capture-linux]", message); + } + }); + child.once("exit", (code, signal) => { + this.process = null; + const reason = + this.lastError ?? + `the Linux capture helper exited (code=${code ?? "null"}, signal=${signal ?? "null"})`; + this.rejectReady(new Error(reason)); + this.startedReject?.(new Error(reason)); + this.startedReject = null; + this.startedResolve = null; + // A clean exit after `capture-stopped` is the normal path; anything + // else means the file may not have its trailer. + if (this.stopped) { + this.stoppedResolve?.(); + } else { + this.stoppedReject?.(new Error(reason)); + } + this.stoppedResolve = null; + this.stoppedReject = null; + }); + child.once("error", (error) => { + this.process = null; + this.rejectReady(error); + }); + + try { + await new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for the Linux capture helper")); + }, READY_TIMEOUT_MS); + }); + } catch (error) { + this.kill(); + throw error; + } + } + + /** + * Resolves when the first frame has been encoded — i.e. once the user has + * answered the portal picker. No timeout, on purpose: see the class doc. + */ + waitUntilCapturing(): Promise { + if (!this.process) { + return Promise.reject(new Error("The Linux capture helper is not running.")); + } + return new Promise((resolve, reject) => { + this.startedResolve = resolve; + this.startedReject = reject; + }); + } + + pause() { + if (this.paused) { + return; + } + this.paused = true; + this.write("pause"); + } + + resume() { + if (!this.paused) { + return; + } + this.paused = false; + this.write("resume"); + } + + get isPaused() { + return this.paused; + } + + /** + * Asks the helper to write the trailer and exit, then returns what it made. + * + * Waits for the PROCESS to exit rather than for `capture-stopped` alone: the + * MP4's moov atom is written during teardown, and returning a path to a file + * whose index is still being written would hand the app an unplayable + * recording. + */ + async stop(): Promise { + const child = this.process; + if (!child) { + if (this.stopped) { + return this.stopped; + } + throw new Error("The Linux capture helper is not running."); + } + + const exited = new Promise((resolve, reject) => { + this.stoppedResolve = resolve; + this.stoppedReject = reject; + }); + this.write("stop"); + try { + child.stdin.end(); + } catch { + // Already closed if the helper died on its own; the exit handler runs. + } + + const timer = setTimeout(() => { + console.error("[capture-linux] helper did not exit in time; terminating"); + this.kill(); + }, STOP_TIMEOUT_MS); + try { + await exited; + } finally { + clearTimeout(timer); + } + + if (!this.stopped) { + throw new Error(this.lastError ?? "The Linux capture helper produced no output file."); + } + return this.stopped; + } + + /** Kills the helper and discards whatever it wrote. */ + discard() { + this.kill(); + } + + private write(command: string) { + try { + this.process?.stdin.write(`${command}\n`); + } catch { + // The helper is gone; stop()/exit already reports that. + } + } + + private kill() { + const child = this.process; + if (!child || child.killed) { + return; + } + child.kill("SIGTERM"); + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 500).unref(); + } + + private handleStdout(chunk: string) { + this.lines.push(chunk, (line) => { + let payload: PipeWireHelperEvent; + try { + payload = JSON.parse(line) as PipeWireHelperEvent; + } catch (error) { + console.error("Failed to parse Linux capture helper output:", error, line); + return; + } + this.handleEvent(payload); + }); + } + + private handleEvent(payload: PipeWireHelperEvent) { + switch (payload.event) { + case "ready": + this.resolveReady(); + return; + + case "stream-started": + if (payload.restoreToken) { + this.restoreToken = payload.restoreToken; + } + console.info( + "[capture-linux] portal stream started", + JSON.stringify({ width: payload.width, height: payload.height }), + ); + return; + + case "audio-source": + // Logged unconditionally: when someone reports "that is not my + // microphone", this line is the answer. + console.info( + "[capture-linux] audio source", + JSON.stringify({ + role: payload.role, + requested: payload.requested ?? null, + node: payload.node ?? null, + }), + ); + if (payload.requested && !payload.node) { + console.warn( + `[capture-linux] no PipeWire node matched ${JSON.stringify(payload.requested)}; ` + + "the session default source was recorded instead", + ); + } + return; + + case "encoder-selection": + this.videoEncoder = payload.video; + console.info( + "[capture-linux] encoder", + JSON.stringify({ video: payload.video, rejected: payload.rejected ?? [] }), + ); + return; + + case "capture-started": + // Cursor samples started flowing when the helper did, which was + // before the portal picker. The recording's zero is HERE, so the + // telemetry is re-based onto it and anything from during the + // picker is dropped rather than left pinned to the start. + this.cursor.rebase(payload.timestampMs); + console.info( + "[capture-linux] capture started", + JSON.stringify({ width: payload.width, height: payload.height, fps: payload.fps }), + ); + this.startedResolve?.(); + this.startedResolve = null; + this.startedReject = null; + return; + + case "cursor-sample": + this.cursor.addSample(payload); + return; + + case "capture-stopped": + this.stopped = { + path: payload.path, + durationMs: payload.durationMs, + frames: payload.frames, + droppedFrames: payload.dropped, + cursor: this.cursor.toRecordingData(), + ...(this.restoreToken ? { restoreToken: this.restoreToken } : {}), + ...(this.videoEncoder ? { videoEncoder: this.videoEncoder } : {}), + }; + console.info( + "[capture-linux] capture stopped", + JSON.stringify({ + frames: payload.frames, + dropped: payload.dropped, + durationMs: payload.durationMs, + convertMs: payload.convertMs, + uploadMs: payload.uploadMs, + encodeMs: payload.encodeMs, + }), + ); + return; + + case "warning": + console.warn(`[capture-linux] ${payload.code}: ${payload.message}`); + return; + + case "error": + console.error(`[capture-linux] ${payload.code}: ${payload.message}`); + // Kept so the process-exit handler can report the CAUSE rather + // than just the exit code, which on its own says nothing useful. + this.lastError = payload.message; + this.rejectReady(new Error(payload.message)); + return; + + case "debug": + console.info("[capture-linux][debug]", JSON.stringify(payload)); + return; + } + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 9a82ffd55c..facf1f078e 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -1,5 +1,6 @@ import type { Rectangle } from "electron"; import { MacNativeCursorRecordingSession } from "./macNativeCursorRecordingSession"; +import { PipeWireCursorRecordingSession } from "./pipeWireCursorRecordingSession"; import type { CursorRecordingSession } from "./session"; import { TelemetryRecordingSession } from "./telemetryRecordingSession"; import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; @@ -35,8 +36,23 @@ export function createCursorRecordingSession( }); } - // Linux: capture cursor positions via Electron's `screen` API on an interval. - // No cursor sprites/assets and no clicks, just position telemetry. + if (options.platform === "linux") { + // The ScreenCast portal's METADATA cursor mode is the only source of a real + // pointer position on Wayland: `screen.getCursorScreenPoint()` returns + // {0,0} there, so TelemetryRecordingSession produced well-formed recordings + // with every sample pinned to the top-left corner. The helper throws when + // it is unavailable rather than falling back here — the caller then records + // without cursor data, which beats a file full of {0,0}. + return new PipeWireCursorRecordingSession({ + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + startTimeMs: options.startTimeMs, + }); + } + + // Anything else (a future platform, or a test harness): capture cursor + // positions via Electron's `screen` API on an interval. No cursor + // sprites/assets and no clicks, just position telemetry. return new TelemetryRecordingSession({ getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts new file mode 100644 index 0000000000..69ae80e3ad --- /dev/null +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from "vitest"; +import { NdjsonLineReader, PipeWireCursorAccumulator } from "./pipeWireCursorAccumulator"; + +function sample(timestampMs: number, x: number, y: number, extra: Record = {}) { + return { + event: "cursor-sample" as const, + timestampMs, + x, + y, + width: 1920, + height: 1080, + visible: true, + ...extra, + }; +} + +describe("NdjsonLineReader", () => { + it("reassembles a line split across chunks", () => { + // The helper writes one JSON object per line and flushes per line, but a + // pipe read can still land mid-line. Splitting on newline per chunk and + // discarding the remainder would drop a sample every time that happened. + const reader = new NdjsonLineReader(); + const lines: string[] = []; + reader.push('{"event":"a"}\n{"ev', (line) => lines.push(line)); + expect(lines).toEqual(['{"event":"a"}']); + reader.push('ent":"b"}\n', (line) => lines.push(line)); + expect(lines).toEqual(['{"event":"a"}', '{"event":"b"}']); + }); + + it("emits nothing for a chunk with no newline", () => { + const reader = new NdjsonLineReader(); + const lines: string[] = []; + reader.push('{"partial"', (line) => lines.push(line)); + expect(lines).toEqual([]); + }); + + it("drops a half-line on reset so a new recording cannot inherit it", () => { + const reader = new NdjsonLineReader(); + const lines: string[] = []; + reader.push('{"stale', (line) => lines.push(line)); + reader.reset(); + reader.push('{"event":"fresh"}\n', (line) => lines.push(line)); + expect(lines).toEqual(['{"event":"fresh"}']); + }); +}); + +describe("PipeWireCursorAccumulator", () => { + it("normalises positions against the stream size the helper reports", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(1_000); + accumulator.addSample(sample(1_500, 960, 540)); + + const [point] = accumulator.toRecordingData().samples; + expect(point.cx).toBeCloseTo(0.5, 5); + expect(point.cy).toBeCloseTo(0.5, 5); + expect(point.timeMs).toBe(500); + }); + + it("reports every sample as a move, because Wayland exposes no buttons", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(0); + accumulator.addSample(sample(10, 5, 5)); + expect(accumulator.toRecordingData().samples[0].interactionType).toBe("move"); + }); + + it("re-bases onto the video's start and drops what came before it", () => { + // This is the single-session case. Cursor samples start flowing as soon + // as the helper does, but the video's frame 0 is only stamped once the + // user has answered the portal picker — an unbounded wait. Samples taken + // during it belong to no part of the recording. + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(1_000); + accumulator.addSample(sample(1_200, 100, 100)); // during the picker + accumulator.addSample(sample(2_000, 200, 200)); // exactly at capture start + accumulator.addSample(sample(2_500, 300, 300)); // after + + accumulator.rebase(2_000); + + const { samples } = accumulator.toRecordingData(); + expect(samples).toHaveLength(2); + expect(samples[0].timeMs).toBe(0); + expect(samples[1].timeMs).toBe(500); + }); + + it("leaves samples alone when the origin has not moved", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(1_000); + accumulator.addSample(sample(1_400, 10, 10)); + accumulator.rebase(1_000); + expect(accumulator.toRecordingData().samples[0].timeMs).toBe(400); + }); + + it("reports provider 'none' until a sprite arrives", () => { + // The editor uses this to decide whether it has real cursor art to draw + // or must fall back. A recording with positions but no bitmap is the + // normal state for the first few frames. + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(0); + accumulator.addSample(sample(10, 5, 5)); + expect(accumulator.toRecordingData().provider).toBe("none"); + + accumulator.addSample( + sample(20, 6, 6, { + assetId: "abc", + asset: { + id: "abc", + imageDataUrl: "data:image/png;base64,AAAA", + width: 24, + height: 24, + hotspotX: 4, + hotspotY: 2, + }, + }), + ); + const data = accumulator.toRecordingData(); + expect(data.provider).toBe("native"); + expect(data.assets).toHaveLength(1); + expect(data.assets[0]).toMatchObject({ platform: "linux", hotspotX: 4, scaleFactor: 1 }); + }); + + it("keeps the first sprite when the same id is sent again", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(0); + const asset = { + id: "same", + imageDataUrl: "data:image/png;base64,FIRST", + width: 24, + height: 24, + hotspotX: 1, + hotspotY: 1, + }; + accumulator.addSample(sample(10, 1, 1, { assetId: "same", asset })); + accumulator.addSample( + sample(20, 2, 2, { + assetId: "same", + asset: { ...asset, imageDataUrl: "data:image/png;base64,SECOND" }, + }), + ); + const { assets } = accumulator.toRecordingData(); + expect(assets).toHaveLength(1); + expect(assets[0].imageDataUrl).toBe("data:image/png;base64,FIRST"); + }); + + it("caps the sample list by dropping the oldest", () => { + const accumulator = new PipeWireCursorAccumulator(3); + accumulator.reset(0); + for (let index = 0; index < 5; index++) { + accumulator.addSample(sample(index * 10, index, index)); + } + const { samples } = accumulator.toRecordingData(); + expect(samples).toHaveLength(3); + expect(samples[0].timeMs).toBe(20); + }); + + it("clamps positions outside the stream instead of emitting them", () => { + // A cursor can legitimately be reported outside a window capture's + // bounds. Un-clamped, cx/cy leave 0..1 and the editor draws the overlay + // off-canvas rather than at the edge. + const accumulator = new PipeWireCursorAccumulator(10); + accumulator.reset(0); + accumulator.addSample(sample(10, -50, 5000)); + const [point] = accumulator.toRecordingData().samples; + expect(point.cx).toBe(0); + expect(point.cy).toBe(1); + }); +}); diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts new file mode 100644 index 0000000000..6cbeb9ee1c --- /dev/null +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts @@ -0,0 +1,200 @@ +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "../../../../src/native/contracts"; +import { clamp } from "../../../../src/utils/math"; + +/** + * Turns the Linux helper's NDJSON stdout into `CursorRecordingData`. + * + * Extracted from `PipeWireCursorRecordingSession` because there are now two + * callers and only one helper. The cursor-only session spawns the helper for + * telemetry alone; the native capture session spawns it for video AND telemetry, + * from the same portal session. Sharing this class is what keeps them from + * drifting — and, more importantly, is what lets the second one exist at all: + * `SelectSources` may be called once per portal session, so two processes would + * mean two pickers, which is exactly the double-prompt this replaces. + */ + +export interface PipeWireCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +export type PipeWireHelperEvent = + | { event: "ready"; timestampMs: number; pipewireVersion?: string | null } + | { + event: "stream-started"; + timestampMs: number; + nodeId: number; + width: number; + height: number; + restoreToken?: string | null; + } + | { + event: "cursor-sample"; + timestampMs: number; + x: number; + y: number; + width: number; + height: number; + visible: boolean; + assetId?: string; + asset?: PipeWireCursorAssetPayload; + } + | { + event: "audio-source"; + role: string; + requested?: string | null; + /** The PipeWire node the stream was linked to. `null` means the session + * default was used, which is often NOT the device the user picked. */ + node?: string | null; + } + | { event: "encoder-selection"; video: string; rejected?: string[] } + | { + event: "capture-started"; + timestampMs: number; + path: string; + width: number; + height: number; + fps: number; + } + | { + event: "capture-stopped"; + timestampMs: number; + path: string; + durationMs: number; + frames: number; + dropped: number; + convertMs?: number; + uploadMs?: number; + encodeMs?: number; + } + | { event: "warning"; code: string; message: string } + | { event: "error"; code: string; message: string } + | { event: "debug"; code: string; [key: string]: unknown }; + +/** Splits a stdout stream into whole NDJSON lines across chunk boundaries. */ +export class NdjsonLineReader { + private buffer = ""; + + push(chunk: string, onLine: (line: string) => void) { + this.buffer += chunk; + const lines = this.buffer.split(/\r?\n/); + // The last element is whatever came after the final newline — a partial + // line, or "" when the chunk ended cleanly. Either way it is not ready. + this.buffer = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed) { + onLine(trimmed); + } + } + } + + reset() { + this.buffer = ""; + } +} + +export class PipeWireCursorAccumulator { + private samples: CursorRecordingSample[] = []; + private assets = new Map(); + private startTimeMs = 0; + + constructor(private readonly maxSamples: number) {} + + reset(startTimeMs: number) { + this.samples = []; + this.assets.clear(); + this.startTimeMs = startTimeMs; + } + + /** + * Re-bases every sample already collected onto a new time origin. + * + * Needed because the helper's cursor stream starts before its video does: the + * portal picker, the format negotiation and the encoder open all happen first, + * and only then does `capture-started` fire. Samples taken in between belong + * to the recording's negative time, so the origin moves to the video's and + * anything now before zero is dropped rather than clamped — a clamped sample + * would pin the pointer to wherever it happened to be during the picker. + */ + rebase(startTimeMs: number) { + const shiftMs = startTimeMs - this.startTimeMs; + this.startTimeMs = startTimeMs; + if (shiftMs === 0) { + return; + } + this.samples = this.samples + .map((sample) => ({ ...sample, timeMs: sample.timeMs - shiftMs })) + .filter((sample) => sample.timeMs >= 0); + } + + addSample(payload: Extract) { + this.rememberAsset(payload.asset); + + // Normalised against the stream's own dimensions, which the helper repeats + // on every sample. Electron's display bounds are deliberately NOT used: + // they are in DIPs, whereas the portal reports stream pixels, and the + // portal's source is whatever the user picked in its own dialog, which + // need not be the display the app thinks it is recording. + const width = Math.max(1, payload.width); + const height = Math.max(1, payload.height); + + this.samples.push({ + // Clamped, matching the behaviour this was extracted from: a sample + // stamped a few milliseconds before the recording's own start is a + // clock artefact, not a sample from before the recording. + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: clamp(payload.x / width, 0, 1), + cy: clamp(payload.y / height, 0, 1), + visible: payload.visible, + // Wayland exposes no click events to an unprivileged process. + interactionType: "move", + ...(payload.assetId ? { assetId: payload.assetId } : {}), + }); + + if (this.samples.length > this.maxSamples) { + this.samples.shift(); + } + } + + private rememberAsset(asset: PipeWireCursorAssetPayload | undefined) { + if (!asset?.id || this.assets.has(asset.id)) { + return; + } + + this.assets.set(asset.id, { + id: asset.id, + platform: "linux", + imageDataUrl: asset.imageDataUrl, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + // The portal reports cursor bitmaps in stream pixels, the same space + // the positions are in, so no extra scaling applies. + scaleFactor: 1, + }); + } + + /** Drops samples taken before zero, which `rebase` can produce. */ + get sampleCount() { + return this.samples.length; + } + + toRecordingData(): CursorRecordingData { + return { + version: 2, + provider: this.assets.size > 0 ? "native" : "none", + samples: this.samples, + assets: [...this.assets.values()], + }; + } +} diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.test.ts b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.test.ts new file mode 100644 index 0000000000..0058079d39 --- /dev/null +++ b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.test.ts @@ -0,0 +1,347 @@ +import { EventEmitter } from "node:events"; +import { PassThrough, Writable } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * The cast on `actual` is written out in each factory rather than shared in a + * helper: `vi.mock` calls are HOISTED above every top-level statement, so a + * module-scope helper is still in its temporal dead zone when the factory runs + * and the mock dies with "Cannot access 'x' before initialization". + * + * The cast itself is needed because these modules DO carry a default export at + * runtime — the CJS namespace, which the code under test may import — while + * `@types/node` declares only the named ones. Spreading it keeps that shape + * intact instead of dropping it. + */ +type WithDefault = { default?: Record }; + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + const spawn = vi.fn(); + return { ...actual, spawn, default: { ...((actual as WithDefault).default ?? {}), spawn } }; +}); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + // The helper binary does not exist in a test checkout; pretend the first + // candidate path is executable so path resolution is not what is under test. + return { + ...actual, + accessSync: vi.fn(), + default: { ...((actual as WithDefault).default ?? {}), accessSync: vi.fn() }, + }; +}); + +import { spawn } from "node:child_process"; +import { PipeWireCursorRecordingSession } from "./pipeWireCursorRecordingSession"; + +/** Minimal stand-in for the helper process: stdio pipes plus kill bookkeeping. */ +class FakeHelper extends EventEmitter { + stdout = new PassThrough(); + stderr = new PassThrough(); + stdinWrites: string[] = []; + killed = false; + signals: (string | undefined)[] = []; + stdin: Writable; + + constructor() { + super(); + const writes = this.stdinWrites; + this.stdin = new Writable({ + write(chunk, _encoding, callback) { + writes.push(chunk.toString()); + callback(); + }, + }); + } + + kill(signal?: string) { + this.signals.push(signal); + this.killed = true; + return true; + } + + /** Feeds one NDJSON line, the way the real helper emits them. */ + emitEvent(event: Record) { + this.stdout.write(`${JSON.stringify({ schemaVersion: 1, ...event })}\n`); + } +} + +const spawnMock = vi.mocked(spawn); +let helper: FakeHelper; + +function newSession(overrides: Partial<{ maxSamples: number; sampleIntervalMs: number }> = {}) { + return new PipeWireCursorRecordingSession({ + maxSamples: overrides.maxSamples ?? 100, + sampleIntervalMs: overrides.sampleIntervalMs ?? 33, + startTimeMs: 1_000, + }); +} + +/** Starts a session that becomes ready as soon as the helper says so. */ +async function startReady(session: PipeWireCursorRecordingSession) { + const started = session.start(); + // The listeners are attached synchronously inside start(), so a microtask is + // enough before the fake helper speaks. + await Promise.resolve(); + helper.emitEvent({ event: "ready", timestampMs: 1_000, pipewireVersion: "1.0.5" }); + await started; +} + +/** stdout is a stream: give the "data" listeners a turn before asserting. */ +function flushStdout() { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +beforeEach(() => { + helper = new FakeHelper(); + spawnMock.mockReset(); + spawnMock.mockReturnValue(helper as unknown as ReturnType); + const silence = () => { + // The session logs every helper diagnostic; keep the test output readable. + }; + vi.spyOn(console, "info").mockImplementation(silence); + vi.spyOn(console, "warn").mockImplementation(silence); + vi.spyOn(console, "error").mockImplementation(silence); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("PipeWireCursorRecordingSession", () => { + it("passes the sample interval to the helper as a JSON request argument", async () => { + await startReady(newSession({ sampleIntervalMs: 20 })); + const [, args] = spawnMock.mock.calls[0]; + expect(JSON.parse((args as string[])[0])).toEqual({ sampleIntervalMs: 20 }); + }); + + it("resolves start() on `ready`, before the portal picker has been answered", async () => { + const session = newSession(); + let resolved = false; + const started = session.start().then(() => { + resolved = true; + }); + await Promise.resolve(); + expect(resolved).toBe(false); + helper.emitEvent({ event: "ready", timestampMs: 1_000 }); + await started; + expect(resolved).toBe(true); + }); + + it("rejects start() when the helper reports an error instead of becoming ready", async () => { + const session = newSession(); + const started = session.start(); + await Promise.resolve(); + helper.emitEvent({ + event: "error", + code: "cursor-metadata-unsupported", + message: "no METADATA cursor mode", + }); + await expect(started).rejects.toThrow("no METADATA cursor mode"); + }); + + it("rejects start() when the helper exits before becoming ready", async () => { + const session = newSession(); + const started = session.start(); + await Promise.resolve(); + helper.emit("exit", 1, null); + await expect(started).rejects.toThrow(/exited before ready/); + }); + + it("normalises cursor positions against the stream size, not a display", async () => { + const session = newSession(); + await startReady(session); + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 2_000, + x: 960, + y: 270, + width: 1920, + height: 1080, + visible: true, + }); + await flushStdout(); + + const data = await session.stop(); + expect(data.samples).toHaveLength(1); + expect(data.samples[0]).toEqual({ + timeMs: 1_000, + cx: 0.5, + cy: 0.25, + visible: true, + interactionType: "move", + }); + }); + + it("clamps out-of-range positions and preserves the helper's visibility flag", async () => { + const session = newSession(); + await startReady(session); + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 2_000, + x: -40, + y: 5_000, + width: 1920, + height: 1080, + visible: false, + }); + await flushStdout(); + + const [sample] = (await session.stop()).samples; + expect(sample.cx).toBe(0); + expect(sample.cy).toBe(1); + expect(sample.visible).toBe(false); + }); + + it("never reports a click: Wayland exposes no mouse buttons to this process", async () => { + const session = newSession(); + await startReady(session); + for (let i = 0; i < 3; i++) { + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 2_000 + i, + x: i, + y: i, + width: 100, + height: 100, + visible: true, + }); + } + await flushStdout(); + + const { samples } = await session.stop(); + expect(samples.map((sample) => sample.interactionType)).toEqual(["move", "move", "move"]); + }); + + it("collects each cursor sprite once and reports the native provider", async () => { + const session = newSession(); + await startReady(session); + const asset = { + id: "sha-1", + imageDataUrl: "data:image/png;base64,AAAA", + width: 24, + height: 24, + hotspotX: 4, + hotspotY: 3, + }; + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 2_000, + x: 0, + y: 0, + width: 100, + height: 100, + visible: true, + assetId: "sha-1", + asset, + }); + // The helper only ships the payload once; later samples reference the id. + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 2_100, + x: 1, + y: 1, + width: 100, + height: 100, + visible: true, + assetId: "sha-1", + }); + await flushStdout(); + + const data = await session.stop(); + expect(data.provider).toBe("native"); + expect(data.assets).toEqual([ + { + id: "sha-1", + platform: "linux", + imageDataUrl: "data:image/png;base64,AAAA", + width: 24, + height: 24, + hotspotX: 4, + hotspotY: 3, + scaleFactor: 1, + }, + ]); + expect(data.samples.map((sample) => sample.assetId)).toEqual(["sha-1", "sha-1"]); + }); + + it("reports the `none` provider when no sprite ever arrived", async () => { + const session = newSession(); + await startReady(session); + const data = await session.stop(); + expect(data.provider).toBe("none"); + expect(data.assets).toEqual([]); + expect(data.version).toBe(2); + }); + + it("keeps only the newest samples once maxSamples is exceeded", async () => { + const session = newSession({ maxSamples: 2 }); + await startReady(session); + for (let i = 0; i < 4; i++) { + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 1_000 + i, + x: i, + y: 0, + width: 100, + height: 100, + visible: true, + }); + } + await flushStdout(); + + const { samples } = await session.stop(); + expect(samples.map((sample) => sample.timeMs)).toEqual([2, 3]); + }); + + it("tolerates a JSON object split across two stdout chunks", async () => { + const session = newSession(); + await startReady(session); + const line = JSON.stringify({ + event: "cursor-sample", + schemaVersion: 1, + timestampMs: 1_500, + x: 10, + y: 20, + width: 100, + height: 200, + visible: true, + }); + helper.stdout.write(line.slice(0, 25)); + await flushStdout(); + helper.stdout.write(`${line.slice(25)}\n`); + await flushStdout(); + + const { samples } = await session.stop(); + expect(samples).toHaveLength(1); + expect(samples[0].cx).toBeCloseTo(0.1); + }); + + it("ignores unparseable output rather than losing the rest of the stream", async () => { + const session = newSession(); + await startReady(session); + helper.stdout.write("not json at all\n"); + helper.emitEvent({ + event: "cursor-sample", + timestampMs: 1_100, + x: 50, + y: 50, + width: 100, + height: 100, + visible: true, + }); + await flushStdout(); + + const { samples } = await session.stop(); + expect(samples).toHaveLength(1); + }); + + it("asks the helper to stop over stdin before resorting to a signal", async () => { + const session = newSession(); + await startReady(session); + await session.stop(); + expect(helper.stdinWrites.join("")).toBe("stop\n"); + expect(helper.killed).toBe(false); + }); +}); diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts new file mode 100644 index 0000000000..16815071c6 --- /dev/null +++ b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts @@ -0,0 +1,255 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { accessSync, constants as fsConstants } from "node:fs"; +import path from "node:path"; +import type { Readable, Writable } from "node:stream"; +import type { CursorRecordingData } from "../../../../src/native/contracts"; +import { + NdjsonLineReader, + PipeWireCursorAccumulator, + type PipeWireHelperEvent, +} from "./pipeWireCursorAccumulator"; +import type { CursorRecordingSession } from "./session"; + +/** + * Linux cursor recording via the ScreenCast portal's METADATA cursor mode. + * + * Why this exists at all: `screen.getCursorScreenPoint()` returns {0,0} under + * Wayland, so `TelemetryRecordingSession` silently produced recordings whose + * every sample sat in the top-left corner. The portal is the only source of a + * real pointer position on Wayland — the compositor keeps the cursor out of the + * captured pixels and attaches it to each frame as metadata instead. + * + * Two consequences the caller should know about: + * + * * `interactionType` is always "move". Wayland exposes no portal for mouse + * buttons and /dev/input/event* is root:input, so clicks are unobtainable. + * * The helper raises its own portal picker. On Wayland, Electron's + * `desktopCapturer` already raised one, so the user currently picks a source + * twice. Merging the two is the job of the capture stage that will reuse this + * same portal session — SelectSources may only be called once per session, + * so it has to be one session doing both. + * + * If the helper is missing or fails, this throws rather than falling back to + * `TelemetryRecordingSession`. The caller (`startCursorRecording` in + * electron/ipc/handlers.ts) catches that and records without cursor data, which + * is the honest outcome: no cursor file beats a file full of {0,0}. + */ + +const HELPER_NAME = "openscreen-pipewire-helper"; +const READY_TIMEOUT_MS = 5_000; +const STOP_GRACE_MS = 500; + +interface PipeWireCursorRecordingSessionOptions { + maxSamples: number; + sampleIntervalMs: number; + startTimeMs?: number; +} + +function platformArchTag() { + return process.arch === "arm64" ? "linux-arm64" : "linux-x64"; +} + +function helperCandidates() { + const envPath = process.env.OPENSCREEN_LINUX_CURSOR_HELPER_EXE?.trim(); + const appRoot = process.env.APP_ROOT ? path.resolve(process.env.APP_ROOT) : process.cwd(); + const archTag = platformArchTag(); + const resourceRoot = + typeof process.resourcesPath === "string" + ? process.resourcesPath + : path.join(appRoot, "resources"); + + return [ + envPath, + path.join(appRoot, "electron", "native", "pipewire-capture", "build", HELPER_NAME), + path.join(appRoot, "electron", "native", "bin", archTag, HELPER_NAME), + path.join(resourceRoot, "electron", "native", "bin", archTag, HELPER_NAME), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +export function findPipeWireCursorHelperPath() { + for (const candidate of helperCandidates()) { + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next helper location. + } + } + + return null; +} + +export class PipeWireCursorRecordingSession implements CursorRecordingSession { + private readonly cursor: PipeWireCursorAccumulator; + private readonly lines = new NdjsonLineReader(); + private process: ChildProcessByStdio | null = null; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + + constructor(private readonly options: PipeWireCursorRecordingSessionOptions) { + this.cursor = new PipeWireCursorAccumulator(options.maxSamples); + } + + async start(): Promise { + this.cursor.reset(this.options.startTimeMs ?? Date.now()); + this.lines.reset(); + + const helperPath = findPipeWireCursorHelperPath(); + if (!helperPath) { + throw new Error( + "Linux cursor helper is not available. Build it with `npm run build:native:linux`.", + ); + } + + const child = spawn( + helperPath, + [JSON.stringify({ sampleIntervalMs: this.options.sampleIntervalMs })], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + this.process = child; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.handleStdoutChunk(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + console.error("[cursor-linux]", message); + } + }); + child.once("exit", (code, signal) => { + this.rejectReady( + new Error(`Linux cursor helper exited before ready (code=${code}, signal=${signal})`), + ); + this.process = null; + }); + child.once("error", (error) => { + this.rejectReady(error); + this.process = null; + }); + + try { + await this.waitUntilReady(); + } catch (error) { + this.killHelperProcess(child); + this.process = null; + throw error; + } + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + if (child) { + // `stop` on stdin lets the helper close the portal session cleanly; + // SIGTERM is the fallback if it does not exit promptly. + try { + child.stdin.write("stop\n"); + child.stdin.end(); + } catch { + // The pipe may already be closed if the helper died on its own. + } + this.killHelperProcess(child, STOP_GRACE_MS); + } + + return this.cursor.toRecordingData(); + } + + private handleStdoutChunk(chunk: string) { + this.lines.push(chunk, (line) => { + try { + this.handleEvent(JSON.parse(line) as PipeWireHelperEvent); + } catch (error) { + console.error("Failed to parse Linux cursor helper output:", error, line); + } + }); + } + + private handleEvent(payload: PipeWireHelperEvent) { + switch (payload.event) { + case "ready": + this.resolveReady(); + return; + case "stream-started": + console.info( + "[cursor-linux] portal stream started", + JSON.stringify({ + nodeId: payload.nodeId, + width: payload.width, + height: payload.height, + }), + ); + return; + case "cursor-sample": + this.cursor.addSample(payload); + return; + case "warning": + console.warn(`[cursor-linux] ${payload.code}: ${payload.message}`); + return; + case "error": + console.error(`[cursor-linux] ${payload.code}: ${payload.message}`); + this.rejectReady(new Error(payload.message)); + return; + case "debug": + console.info("[cursor-linux][debug]", JSON.stringify(payload)); + return; + } + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for Linux cursor helper")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private killHelperProcess(child: ChildProcessByStdio, graceMs = 0) { + if (child.killed) { + return; + } + + const terminate = () => { + if (!child.killed) { + child.kill("SIGTERM"); + } + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 500).unref(); + }; + + if (graceMs > 0) { + setTimeout(terminate, graceMs).unref(); + return; + } + terminate(); + } +} diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 7400b4a6d0..9efc302ecd 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -15,6 +15,7 @@ import type { GifExportStats, GifParamsInput, NativeFramePacket, + RemuxStats, } from "../../native/compositor-view/addon"; /** @@ -575,4 +576,23 @@ export class CompositorViewService { onProgress, ); } + + /** Stream-copy `inputPath` to `outputPath` through libavformat's matroska muxer. + * No re-encode: the packets are copied verbatim and only the container is rebuilt, + * which is what gives the output a real `Duration`, `Cues` and `SeekHead`. + * + * `outputPath` must be a TEMPORARY path — the caller renames it over the original + * only once this resolves, so a failure leaves the recording untouched. See + * `electron/recording/webm-seek-index.ts`, the only caller. + * + * Returns null when the addon is absent, or when it predates this export (a stale + * `.node` from an earlier build): the caller then keeps the file as it is rather + * than treating a missing optimisation as a failed save. */ + async remuxSeekable(inputPath: string, outputPath: string): Promise { + const addon = this.ensureAddon(); + if (!addon?.remuxSeekable) { + return null; + } + return addon.remuxSeekable(inputPath, outputPath); + } } diff --git a/electron/native/README.md b/electron/native/README.md index 9c450339d6..5b38de88e0 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -116,6 +116,102 @@ npm run test:wgc-mic:win Remove-Item Env:OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME ``` +## Linux + +Linux cursor recording is handled by `openscreen-pipewire-helper` +(`electron/native/pipewire-capture`). Stage 1 emits **cursor samples only** — no +video capture, no encoding. + +It exists because `screen.getCursorScreenPoint()` returns `{0,0}` under Wayland. +`TelemetryRecordingSession` therefore produced well-formed recordings whose every +cursor sample sat in the top-left corner of the screen. The ScreenCast portal's +METADATA cursor mode is the only source of a real pointer position on Wayland: +the compositor keeps the cursor out of the captured pixels and attaches it to +each frame as `SPA_META_Cursor` instead. + +Helper locations, in resolution order: + +1. `OPENSCREEN_LINUX_CURSOR_HELPER_EXE`, for local development and diagnostics. +2. `electron/native/pipewire-capture/build/openscreen-pipewire-helper`, for a locally built binary. +3. `electron/native/bin/linux-x64/openscreen-pipewire-helper` (or `linux-arm64`), for packaged prebuilt helpers. + +Build it with: + +```bash +npm run build:native:linux +``` + +The build needs `cargo` and a C compiler and **nothing else** — in particular not +`libpipewire-0.3-dev`, which Ubuntu does not ship by default and which needs root +to install. The C shim compiles against headers vendored in +`electron/native/pipewire-capture/vendor/` and resolves `libpipewire-0.3.so.0` +with `dlopen` at runtime. On non-Linux hosts the command exits successfully. + +The contract matches the other helpers: one JSON argument, newline-delimited JSON +events on stdout, `stop\n` (or EOF) on stdin. Events are `ready`, `stream-started`, +`cursor-sample`, `warning`, `error` and `debug`, each carrying `"schemaVersion": 1`. +`ready` is emitted before the portal picker is raised, so the app's readiness +timeout does not have to accommodate a human clicking a dialog; `stream-started` +marks the point where samples begin. + +### Manual verification + +**This raises the GNOME/portal source picker and streams until you stop it.** + +```bash +OPENSCREEN_PIPEWIRE_DEBUG=1 electron/native/bin/linux-x64/openscreen-pipewire-helper '{"sampleIntervalMs":100}' +``` + +Pick a monitor in the dialog, move the mouse, and `cursor-sample` lines with real +coordinates should stream out. Type `stop` and press Enter, or Ctrl-D, to end it. + +`OPENSCREEN_PIPEWIRE_DEBUG=1` additionally reports stream state transitions, the +negotiated SPA buffer data type, the full list of metadata blocks that survived +negotiation, and whether `SPA_META_Cursor` carries a sprite bitmap. Every line +carries `timestampMs`, so a log shows how long the stream actually lived. + +### Field notes from the first real GNOME run + +Two facts, and two bugs that run exposed. Both bugs are fixed; both are pinned by +tests that run without a portal. + +- **mutter negotiates `MemFd`**, not DMA-BUF, even when the helper advertises + `MemPtr|MemFd|DmaBuf`. Worth knowing for the video stage: the pixels arrive as + a mappable fd, so a CPU path is viable and no DMA-BUF import is required. +- **`SPA_META_Cursor` was absent from every buffer.** Producers declare + `SPA_PARAM_META_size` as a *fixed* value — mutter 46.2 uses + `CURSOR_META_SIZE(384, 384)` = 589872 bytes — and PipeWire intersects it with + the consumer's declaration. The helper's range was capped at 256×256 = 262192 + bytes, so the intersection was empty, the whole `ParamMeta` object was dropped, + and the buffers arrived with no cursor metadata. There is no error for this: the + stream negotiates and runs perfectly while reporting nothing. The ceiling is now + 1024×1024, matching OBS. A range that does not *contain* the producer's constant + is the failure mode to remember. +- **The stream was also dying on its own**, reporting `target not found`. That + string comes from WirePlumber's `policy-node.lua`, in the branch taken when + `node.dont-reconnect` is set — which `PW_STREAM_FLAG_DONT_RECONNECT` sets. That + branch destroys the node outright, turning a transient link failure into a + permanent, silent end of capture. The flag is gone. + +To exercise everything except the picker — dlopen, D-Bus, and the portal's +cursor-mode check — run the non-interactive probe. This is also what +`npm run build:native:linux` runs after a successful build: + +```bash +electron/native/bin/linux-x64/openscreen-pipewire-helper '{"probeOnly":true}' +``` + +### Known gaps + +- **Mouse clicks are unobtainable.** Wayland exposes no portal for input events + and `/dev/input/event*` is `root:input`, so every sample's `interactionType` is + `"move"`. +- **The user picks a source twice.** Electron's `desktopCapturer` raises its own + portal dialog for the video, and this helper raises a second one for the cursor. + Collapsing them requires one portal session serving both, which is why the + capture stage has to reuse this helper's session — `SelectSources` may only be + called once per session. + ## STT helper The speech-to-text helper (`whisper-stt-server`) is built separately from the diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index da0172c314..dc5fe599a1 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -56,6 +56,15 @@ export interface GifExportStats { fileBytes: number; } +/** Outcome of a container-only remux — see `remuxSeekable`. */ +export interface RemuxStats { + /** Packets copied verbatim, all streams combined. */ + packets: number; + /** Streams kept in the output (video/audio/subtitle; anything else is dropped). */ + streams: number; + wallS: number; +} + /** Sortie GIF native : taille, cadence, loop, dither. Tout optionnel — * absent → 854×480, 12 fps, boucle infinie, pas de dithering. */ export interface GifParamsInput { @@ -172,6 +181,15 @@ export interface CompositorViewAddon { params?: GifParamsInput, onProgress?: (frames: number) => void, ): Promise; + + /** Stream-copy `inputPath` to `outputPath` through the matroska muxer, rebuilding the + * container (real `Duration` computed from the packet timestamps, plus `Cues` and + * `SeekHead`) without re-encoding a single frame. + * + * Optional at the type level: a `.node` built before this export exists in the wild + * (dev trees keep a stale binary until the next `build-linux-compositor-addon.mjs`), + * and the caller degrades to "leave the file alone" rather than failing the save. */ + remuxSeekable?(inputPath: string, outputPath: string): Promise; } /** diff --git a/electron/native/pipewire-capture/Cargo.lock b/electron/native/pipewire-capture/Cargo.lock new file mode 100644 index 0000000000..f8fa27a588 --- /dev/null +++ b/electron/native/pipewire-capture/Cargo.lock @@ -0,0 +1,1677 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "ashpd" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8bd58b44ea371b48d21cdc217380cfcafd4b2bb1ad50d27514ec5beca71a2d" +dependencies = [ + "async-fs", + "async-net", + "enumflags2", + "futures-channel", + "futures-util", + "rand", + "serde", + "serde_repr", + "url", + "zbus", +] + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openscreen-pipewire-helper" +version = "0.0.0" +dependencies = [ + "ashpd", + "base64", + "bindgen", + "cc", + "png", + "pollster", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys 0.61.2", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix", + "ordered-stream", + "rand", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "url", + "zvariant_derive", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/electron/native/pipewire-capture/Cargo.toml b/electron/native/pipewire-capture/Cargo.toml new file mode 100644 index 0000000000..604da16afa --- /dev/null +++ b/electron/native/pipewire-capture/Cargo.toml @@ -0,0 +1,64 @@ +# OpenScreen Linux capture helper (PipeWire + xdg-desktop-portal). +# +# WHY IT LIVES HERE AND NOT IN crates/. +# +# `crates/` is the compositor workspace: a rendering library plus the napi addon +# the app links, sharing wgpu/ffmpeg/windows dependencies and a French comment +# convention. This is not that. It is a sidecar executable spawned over stdio, +# exactly like `electron/native/screencapturekit/` (Swift) and +# `electron/native/wgc-capture/` (C++). Putting it in the compositor workspace +# would drag its dependency tree (ashpd/zbus, cc, png) into every +# `cargo build -p compositor-view-napi` resolution for no benefit, and would put +# a Linux-only binary behind a workspace whose default member is a Windows POC. +# +# IT DOES SHARE FFMPEG — and only ffmpeg. Stage 2 encodes H.264 here, against the +# same vendored tree the compositor uses (crates/thirdparty/). The crucial +# difference from the addon is that this is a separate PROCESS: it never enters +# Electron's address space, so the flat-namespace collision with Chromium's +# `libffmpeg.so` (see scripts/build-linux-compositor-addon.mjs) does not apply +# and no symbol prefixing is needed. Linking it normally is safe HERE and only +# here. +# +# It is therefore a standalone package. `[workspace]` below makes that explicit +# so that a future root Cargo.toml cannot silently adopt it. + +[package] +name = "openscreen-pipewire-helper" +version = "0.0.0" +edition = "2021" +publish = false +license = "MIT" + +[workspace] + +[[bin]] +name = "openscreen-pipewire-helper" +path = "src/main.rs" + +[dependencies] +# Pure-Rust D-Bus. Deliberately NOT the `pipewire` feature (that one pulls +# pipewire-rs, which needs libpipewire-0.3-dev at build time — the exact +# dependency this crate exists to avoid) and NOT `gtk4`. +ashpd = { version = "0.9", default-features = false, features = ["async-std"] } +# ashpd's async-std feature routes zbus through async-io, whose reactor lives on +# its own thread, so a plain block_on is all the executor we need. +pollster = "0.4" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# Cursor sprites: the renderer wants a PNG data URL (see NativeCursorAsset in +# src/native/contracts.ts). Both crates are pure Rust, no C dependency. +png = "0.17" +base64 = "0.22" +sha2 = "0.10" + +[build-dependencies] +cc = "1" +# Same generator the compositor uses on its copy of these headers. Hand-writing +# AVCodecContext/AVFrame layouts is the one thing guaranteed to break silently on +# an ffmpeg bump, so the layouts are always generated, never typed out. +bindgen = "0.70" + +[profile.release] +opt-level = 2 +lto = true +strip = true diff --git a/electron/native/pipewire-capture/build.rs b/electron/native/pipewire-capture/build.rs new file mode 100644 index 0000000000..5aecdb9413 --- /dev/null +++ b/electron/native/pipewire-capture/build.rs @@ -0,0 +1,185 @@ +// Two build inputs, with deliberately different linkage strategies. +// +// PIPEWIRE is compiled from csrc/pw_shim.c against the vendored headers and NOT +// linked: the shim resolves every libpipewire symbol with dlsym at runtime, so +// the only build requirement is a C compiler. That is what makes this crate +// buildable on a machine with no libpipewire-0.3-dev, which is most machines — +// Ubuntu ships only the runtime .so.0. +// +// FFMPEG is linked normally against the tree the app already vendors for the +// compositor. See Cargo.toml for why that is safe here and nowhere else: this is +// a separate process, so Chromium's `libffmpeg.so` is not in its address space +// and the symbol collision that forces the addon's `osff_` renaming cannot +// happen. RUNPATH points at $ORIGIN so the packaged binary finds the .so files +// staged beside it, and at the vendored lib dir so a dev build runs from the +// repo without LD_LIBRARY_PATH. + +use std::path::{Path, PathBuf}; + +fn main() { + let root = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + build_pipewire_shim(&root); + link_ffmpeg(&root); +} + +fn build_pipewire_shim(root: &Path) { + let vendor = root.join("vendor/pipewire-1.0.5/include"); + let sources = ["csrc/pw_shim.c", "csrc/pw_audio.c"]; + + assert!( + vendor.join("pipewire/pipewire.h").is_file(), + "vendored PipeWire headers are missing at {} — see vendor/README.md", + vendor.display() + ); + + let mut build = cc::Build::new(); + for source in sources { + build.file(root.join(source)); + } + build + .include(&vendor) + .include(root.join("csrc")) + .flag_if_supported("-std=gnu11") + .warnings(true) + .compile("openscreen_pw_shim"); + + for source in sources { + println!("cargo:rerun-if-changed={}", root.join(source).display()); + } + println!("cargo:rerun-if-changed={}", root.join("csrc/pw_shim.h").display()); + println!("cargo:rerun-if-changed={}", root.join("csrc/pw_internal.h").display()); + println!("cargo:rerun-if-changed={}", vendor.display()); + + // dlopen/dlsym. + println!("cargo:rustc-link-lib=dl"); +} + +/// Extra `-I` flags so clang can find the freestanding headers it normally ships +/// with itself (`stddef.h`, `limits.h`, `stdint.h`). +/// +/// Ubuntu splits libclang: `libclang.so.1` comes from the runtime package while +/// the builtin header directory comes from `libclang-N-dev`. With only the +/// former installed — the common case — parsing any real header fails on +/// `/usr/include/limits.h: 'limits.h' file not found`, because glibc's copy +/// `#include_next`s the compiler's and there is none. gcc's copies are +/// interchangeable for this purpose, so point clang at those instead of making +/// every contributor install a second toolchain. Mirrors `bindgenClangArgs()` in +/// scripts/build-linux-compositor-addon.mjs, but lives here so that a bare +/// `cargo build` in this directory works too. +fn freestanding_header_args() -> Vec { + if let Ok(extra) = std::env::var("BINDGEN_EXTRA_CLANG_ARGS") { + // Already configured by the caller; bindgen picks that up on its own. + if !extra.trim().is_empty() { + return Vec::new(); + } + } + let gcc_root = Path::new("/usr/lib/gcc/x86_64-linux-gnu"); + let Ok(entries) = std::fs::read_dir(gcc_root) else { + return Vec::new(); + }; + let mut dirs: Vec = entries + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path().join("include")) + .filter(|dir| dir.join("limits.h").is_file() && dir.join("stddef.h").is_file()) + .collect(); + // Highest gcc version first, so a machine with several picks the newest. + dirs.sort(); + dirs.reverse(); + dirs.into_iter() + .take(1) + .map(|dir| format!("-I{}", dir.display())) + .collect() +} + +/// The vendored ffmpeg tree, five directories up from this crate. `FFMPEG_DIR` +/// overrides it for CI or a cross build. +fn ffmpeg_dir(root: &Path) -> PathBuf { + if let Some(dir) = std::env::var_os("FFMPEG_DIR") { + return PathBuf::from(dir); + } + root.join("../../../crates/thirdparty/ffmpeg-linux64-lgpl-shared") +} + +fn link_ffmpeg(root: &Path) { + let dir = ffmpeg_dir(root); + let include = dir.join("include"); + let lib = dir.join("lib"); + + assert!( + include.join("libavcodec/avcodec.h").is_file(), + "vendored ffmpeg headers are missing at {} — run `npm run fetch:ffmpeg` or set FFMPEG_DIR", + include.display() + ); + + println!("cargo:rustc-link-search=native={}", lib.display()); + for name in ["avcodec", "avformat", "avutil", "swscale", "swresample"] { + println!("cargo:rustc-link-lib={name}"); + } + // `$ORIGIN/ffmpeg`, NOT `$ORIGIN`. The helper is staged into + // electron/native/bin/linux-x64/, and that directory ALREADY contains + // libavcodec.so.62 and friends — the copies whose every symbol was renamed + // to `osff_*` by scripts/build-linux-compositor-addon.mjs so the compositor + // addon does not collide with Chromium's ffmpeg inside Electron. Searching + // `$ORIGIN` finds those first and the helper dies at startup with + // "undefined symbol: avcodec_send_frame, version LIBAVCODEC_62". The + // renaming trick is what makes the ADDON work and what would break the + // HELPER, so the two sets of libraries must not share a directory. + // + // The absolute vendored path comes second so `cargo run` works straight out + // of the repo. `--disable-new-dtags` is what makes these RUNPATH entries + // apply to the transitive ffmpeg libs too; with the default DT_RUNPATH they + // would not. + println!("cargo:rustc-link-arg=-Wl,--disable-new-dtags"); + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/ffmpeg"); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib.display()); + + let mut builder = bindgen::Builder::default(); + for arg in freestanding_header_args() { + builder = builder.clang_arg(arg); + } + let bindings = builder + .header_contents( + "ffmpeg.h", + r#" + #include + #include + #include + #include + #include + #include + #include + #include + "#, + ) + .clang_arg(format!("-I{}", include.display())) + // An allowlist rather than the whole tree: it keeps the generated file + // to what this helper calls, and keeps `cargo build` from regenerating + // thousands of items nothing references. + .allowlist_function("av_.*") + .allowlist_function("avcodec_.*") + .allowlist_function("avformat_.*") + .allowlist_function("avio_.*") + .allowlist_function("sws_.*") + .allowlist_function("swr_.*") + .allowlist_type("AV.*") + .allowlist_type("Sws.*") + .allowlist_type("SwrContext") + .allowlist_var("AV_.*") + .allowlist_var("AVERROR.*") + .allowlist_var("FF_.*") + .allowlist_var("SWS_.*") + .derive_default(true) + .prepend_enum_name(false) + .layout_tests(false) + // ffmpeg's headers are riddled with doc comments that are not valid + // rustdoc; without this every build emits hundreds of warnings. + .generate_comments(false) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("bindgen failed on the ffmpeg headers"); + + let out = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR")).join("ffmpeg_sys.rs"); + bindings.write_to_file(&out).expect("could not write ffmpeg bindings"); + + println!("cargo:rerun-if-env-changed=FFMPEG_DIR"); +} diff --git a/electron/native/pipewire-capture/csrc/pw_audio.c b/electron/native/pipewire-capture/csrc/pw_audio.c new file mode 100644 index 0000000000..9c86c673ed --- /dev/null +++ b/electron/native/pipewire-capture/csrc/pw_audio.c @@ -0,0 +1,417 @@ +/* + * PipeWire audio capture: the system mix and the microphone. + * + * WHY THIS IS A SEPARATE CONNECTION FROM THE VIDEO. + * + * The screen arrives over the fd the portal handed us from OpenPipeWireRemote. + * That remote is deliberately restricted: it exposes the one screencast node the + * user approved and nothing else. There is no audio on it, and there is no + * portal that grants audio the way ScreenCast grants pixels — the ScreenCast + * interface has no audio option at all. + * + * So audio comes from the session's own PipeWire daemon, over the ordinary + * socket, with pw_context_connect rather than pw_context_connect_fd. That works + * for a .deb/AppImage build, which runs unsandboxed. Inside a Flatpak it would + * need the `--socket=pipewire` permission; there is no way to ask for it at + * runtime, so a sandboxed build simply gets no audio and says so rather than + * recording silence. + * + * SYSTEM AUDIO IS A SINK MONITOR. `PW_KEY_STREAM_CAPTURE_SINK=true` turns a + * capture stream into one that links to a SINK's monitor ports instead of a + * source — that is, to what is being played rather than to a microphone. It is + * the same mechanism `pw-record --target` uses and what OBS's audio capture + * does; without it the stream links to the default source and records the + * microphone twice. + */ + +#include "pw_shim.h" + +#include "pw_internal.h" + +#include +#include + +#include +#include +#include + +struct osc_pw_audio { + struct pw_thread_loop *loop; + struct pw_context *context; + struct pw_core *core; + struct pw_stream *stream; + struct spa_hook stream_listener; + struct osc_pw_audio_callbacks callbacks; + uint32_t channels; +}; + +static void osc_audio_on_state_changed(void *userdata, enum pw_stream_state old, + enum pw_stream_state state, const char *error) +{ + struct osc_pw_audio *audio = userdata; + + (void)old; + if (audio->callbacks.on_state != NULL) { + audio->callbacks.on_state(audio->callbacks.user, + osc_audio_api.stream_state_as_string(state), error); + } +} + +static void osc_audio_on_param_changed(void *userdata, uint32_t id, const struct spa_pod *param) +{ + struct osc_pw_audio *audio = userdata; + struct spa_audio_info_raw info; + + if (param == NULL || id != SPA_PARAM_Format) { + return; + } + memset(&info, 0, sizeof(info)); + if (spa_format_audio_raw_parse(param, &info) < 0) { + return; + } + /* + * Recorded so osc_audio_on_process knows how many floats make one frame. + * The stream adapter honours what we asked for, but reading back what was + * actually negotiated is what keeps a mono device from being interpreted as + * interleaved stereo and played back at double speed. + */ + audio->channels = info.channels; +} + +static void osc_audio_on_process(void *userdata) +{ + struct osc_pw_audio *audio = userdata; + struct pw_buffer *b; + + while ((b = osc_audio_api.stream_dequeue_buffer(audio->stream)) != NULL) { + struct spa_data *data = &b->buffer->datas[0]; + + if (b->buffer->n_datas > 0 && data->data != NULL && data->chunk != NULL && + audio->callbacks.on_samples != NULL) { + uint32_t offset; + uint32_t size; + + /* Same untrusted-producer clamping as the video path: `chunk` lives + * in memory another process writes. */ + offset = SPA_MIN(data->chunk->offset, data->maxsize); + size = SPA_MIN(data->chunk->size, data->maxsize - offset); + if (size >= sizeof(float)) { + const float *samples = SPA_PTROFF(data->data, offset, const float); + + audio->callbacks.on_samples(audio->callbacks.user, samples, + size / (uint32_t)sizeof(float)); + } + } + osc_audio_api.stream_queue_buffer(audio->stream, b); + } +} + +static const struct pw_stream_events osc_audio_stream_events = { + PW_VERSION_STREAM_EVENTS, + .state_changed = osc_audio_on_state_changed, + .param_changed = osc_audio_on_param_changed, + .process = osc_audio_on_process, +}; + +struct osc_pw_audio *osc_pw_audio_start(const char *target_object, int capture_sink, + uint32_t rate, uint32_t channels, + const struct osc_pw_audio_callbacks *callbacks, char *err, + size_t err_len) +{ + struct osc_pw_audio *audio; + struct pw_properties *props; + uint8_t storage[1024]; + struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(storage, sizeof(storage)); + const struct spa_pod *params[1]; + struct spa_audio_info_raw info; + int result; + + if (osc_audio_api.stream_new == NULL) { + osc_pw_set_error(err, err_len, "osc_pw_audio_start called before osc_pw_load"); + return NULL; + } + + audio = calloc(1, sizeof(*audio)); + if (audio == NULL) { + osc_pw_set_error(err, err_len, "out of memory"); + return NULL; + } + audio->callbacks = *callbacks; + audio->channels = channels; + + audio->loop = osc_audio_api.thread_loop_new("openscreen-audio", NULL); + if (audio->loop == NULL) { + osc_pw_set_error(err, err_len, "pw_thread_loop_new failed"); + free(audio); + return NULL; + } + + osc_audio_api.thread_loop_lock(audio->loop); + + audio->context = + osc_audio_api.context_new(osc_audio_api.thread_loop_get_loop(audio->loop), NULL, 0); + if (audio->context == NULL) { + osc_pw_set_error(err, err_len, "pw_context_new failed"); + goto fail; + } + + /* The session's own daemon, not the portal's restricted remote. */ + audio->core = osc_audio_api.context_connect(audio->context, NULL, 0); + if (audio->core == NULL) { + osc_pw_set_error(err, err_len, + "pw_context_connect failed: no PipeWire session is reachable " + "(in a sandbox this needs the pipewire socket permission)"); + goto fail; + } + + props = osc_audio_api.properties_new(PW_KEY_MEDIA_TYPE, "Audio", PW_KEY_MEDIA_CATEGORY, + "Capture", PW_KEY_MEDIA_ROLE, "Production", NULL); + if (props == NULL) { + osc_pw_set_error(err, err_len, "pw_properties_new failed"); + goto fail; + } + if (capture_sink) { + /* Link to a SINK's monitor ports — what is being played — rather than + * to a source. Without this the stream records the microphone. */ + osc_audio_api.properties_set(props, PW_KEY_STREAM_CAPTURE_SINK, "true"); + } + if (target_object != NULL && target_object[0] != '\0') { + osc_audio_api.properties_set(props, PW_KEY_TARGET_OBJECT, target_object); + } + + audio->stream = osc_audio_api.stream_new(audio->core, "openscreen-audio", props); + if (audio->stream == NULL) { + osc_pw_set_error(err, err_len, "pw_stream_new failed"); + goto fail; + } + + osc_audio_api.stream_add_listener(audio->stream, &audio->stream_listener, + &osc_audio_stream_events, audio); + + /* + * A single fixed format, not a choice. pw_stream inserts an adapter node + * that resamples and remixes whatever the device runs at, so asking for + * exactly what the AAC encoder wants costs nothing and removes every + * conversion from this process. + */ + memset(&info, 0, sizeof(info)); + info.format = SPA_AUDIO_FORMAT_F32; + info.rate = rate; + info.channels = channels; + params[0] = spa_format_audio_raw_build(&builder, SPA_PARAM_EnumFormat, &info); + if (params[0] == NULL) { + osc_pw_set_error(err, err_len, "the audio EnumFormat POD did not fit its builder"); + goto fail; + } + + /* No PW_STREAM_FLAG_RT_PROCESS: osc_audio_on_process takes a mutex on the + * Rust side, which is not allowed on PipeWire's realtime thread. */ + result = osc_audio_api.stream_connect(audio->stream, PW_DIRECTION_INPUT, PW_ID_ANY, + PW_STREAM_FLAG_AUTOCONNECT | + PW_STREAM_FLAG_MAP_BUFFERS, + params, 1); + if (result < 0) { + osc_pw_set_error(err, err_len, "pw_stream_connect failed: %s", spa_strerror(result)); + goto fail; + } + + osc_audio_api.thread_loop_unlock(audio->loop); + + if (osc_audio_api.thread_loop_start(audio->loop) < 0) { + osc_pw_set_error(err, err_len, "pw_thread_loop_start failed"); + osc_audio_api.thread_loop_lock(audio->loop); + goto fail; + } + + return audio; + +fail: + osc_audio_api.thread_loop_unlock(audio->loop); + if (audio->stream != NULL) { + osc_audio_api.stream_destroy(audio->stream); + } + if (audio->core != NULL) { + osc_audio_api.core_disconnect(audio->core); + } + if (audio->context != NULL) { + osc_audio_api.context_destroy(audio->context); + } + osc_audio_api.thread_loop_destroy(audio->loop); + free(audio); + return NULL; +} + +void osc_pw_audio_stop(struct osc_pw_audio *audio) +{ + if (audio == NULL) { + return; + } + /* Stop the loop BEFORE destroying anything it might still be running a + * callback against; this joins the thread. */ + osc_audio_api.thread_loop_stop(audio->loop); + if (audio->stream != NULL) { + osc_audio_api.stream_destroy(audio->stream); + } + if (audio->core != NULL) { + osc_audio_api.core_disconnect(audio->core); + } + if (audio->context != NULL) { + osc_audio_api.context_destroy(audio->context); + } + osc_audio_api.thread_loop_destroy(audio->loop); + free(audio); +} + +/* --------------------------------------------------------------------------- + * Enumeration of audio sources + * + * WHY THIS IS NEEDED AT ALL. The app's microphone picker lists Chromium's + * `MediaDeviceInfo`, whose `deviceId` is an opaque hash and whose `label` is a + * human string. Neither is a PipeWire `node.name`, and `PW_KEY_TARGET_OBJECT` + * accepts only a node name (or a serial). With no target, pw_stream connects to + * the session DEFAULT source — which is why a user who picked their built-in + * microphone in the HUD got the empty headphone jack recorded instead. + * + * So the helper enumerates the graph itself and resolves the label against + * `node.description`, which is exactly the string Chromium surfaces as the + * device label on a PipeWire system. + * ------------------------------------------------------------------------- */ + +struct osc_pw_enum { + struct pw_main_loop *loop; + struct pw_context *context; + struct pw_core *core; + struct pw_registry *registry; + struct spa_hook registry_listener; + struct spa_hook core_listener; + int sync_seq; + /* Caller's buffer, filled as "name\037description\036" records. */ + char *out; + size_t out_len; + size_t out_used; +}; + +static void osc_enum_append(struct osc_pw_enum *e, const char *name, const char *desc) +{ + size_t need; + + if (name == NULL || name[0] == '\0') { + return; + } + if (desc == NULL) { + desc = ""; + } + /* +2 for the two separators, +1 for the trailing NUL the Rust side reads. */ + need = strlen(name) + strlen(desc) + 3; + if (e->out_used + need > e->out_len) { + return; /* Buffer full: report what fits rather than truncating a record. */ + } + e->out_used += (size_t)snprintf(e->out + e->out_used, e->out_len - e->out_used, + "%s\037%s\036", name, desc); +} + +static void osc_enum_on_global(void *data, uint32_t id, uint32_t permissions, const char *type, + uint32_t version, const struct spa_dict *props) +{ + struct osc_pw_enum *e = data; + const char *media_class; + + (void)id; + (void)permissions; + (void)version; + if (props == NULL || type == NULL || strcmp(type, PW_TYPE_INTERFACE_Node) != 0) { + return; + } + media_class = spa_dict_lookup(props, PW_KEY_MEDIA_CLASS); + /* Only real capture devices. "Audio/Source" covers microphones and line-in; + * a sink's monitor is "Audio/Sink" and is reached via capture_sink instead, + * so listing it here would offer the user a duplicate of system audio. */ + if (media_class == NULL || strcmp(media_class, "Audio/Source") != 0) { + return; + } + osc_enum_append(e, spa_dict_lookup(props, PW_KEY_NODE_NAME), + spa_dict_lookup(props, PW_KEY_NODE_DESCRIPTION)); +} + +static const struct pw_registry_events osc_enum_registry_events = { + PW_VERSION_REGISTRY_EVENTS, + .global = osc_enum_on_global, +}; + +static void osc_enum_on_core_done(void *data, uint32_t id, int seq) +{ + struct osc_pw_enum *e = data; + + /* The registry replays every existing global BEFORE answering our sync, so + * a matching `done` means the initial dump is complete. Without this we + * would have to guess at a timeout and would race a busy graph. */ + if (id == PW_ID_CORE && seq == e->sync_seq) { + osc_audio_api.main_loop_quit(e->loop); + } +} + +static const struct pw_core_events osc_enum_core_events = { + PW_VERSION_CORE_EVENTS, + .done = osc_enum_on_core_done, +}; + +int osc_pw_list_audio_sources(char *out, size_t out_len, char *err, size_t err_len) +{ + struct osc_pw_enum e; + int result = -1; + + if (osc_audio_api.main_loop_new == NULL) { + osc_pw_set_error(err, err_len, "osc_pw_list_audio_sources called before osc_pw_load"); + return -1; + } + memset(&e, 0, sizeof(e)); + e.out = out; + e.out_len = out_len; + if (out_len > 0) { + out[0] = '\0'; + } + + /* A plain main loop, not the thread loop the streams use: this call is + * synchronous by design — it runs the loop until the registry dump is done + * and then returns, so there is no thread to hand off to. */ + e.loop = osc_audio_api.main_loop_new(NULL); + if (e.loop == NULL) { + osc_pw_set_error(err, err_len, "pw_main_loop_new failed"); + return -1; + } + e.context = osc_audio_api.context_new(osc_audio_api.main_loop_get_loop(e.loop), NULL, 0); + if (e.context == NULL) { + osc_pw_set_error(err, err_len, "pw_context_new failed"); + goto out; + } + e.core = osc_audio_api.context_connect(e.context, NULL, 0); + if (e.core == NULL) { + osc_pw_set_error(err, err_len, "pw_context_connect failed: no PipeWire session reachable"); + goto out; + } + e.registry = pw_core_get_registry(e.core, PW_VERSION_REGISTRY, 0); + if (e.registry == NULL) { + osc_pw_set_error(err, err_len, "pw_core_get_registry failed"); + goto out; + } + + pw_registry_add_listener(e.registry, &e.registry_listener, &osc_enum_registry_events, &e); + pw_core_add_listener(e.core, &e.core_listener, &osc_enum_core_events, &e); + e.sync_seq = pw_core_sync(e.core, PW_ID_CORE, 0); + + osc_audio_api.main_loop_run(e.loop); + result = 0; + +out: + if (e.registry != NULL) { + osc_audio_api.proxy_destroy((struct pw_proxy *)e.registry); + } + if (e.core != NULL) { + osc_audio_api.core_disconnect(e.core); + } + if (e.context != NULL) { + osc_audio_api.context_destroy(e.context); + } + osc_audio_api.main_loop_destroy(e.loop); + return result; +} diff --git a/electron/native/pipewire-capture/csrc/pw_internal.h b/electron/native/pipewire-capture/csrc/pw_internal.h new file mode 100644 index 0000000000..a78f087796 --- /dev/null +++ b/electron/native/pipewire-capture/csrc/pw_internal.h @@ -0,0 +1,66 @@ +/* + * Shared between pw_shim.c and pw_audio.c. NOT part of the Rust-facing ABI. + * + * pw_shim.h is deliberately free of PipeWire and SPA types — that is what lets + * src/shim.rs mirror it by hand without restating a single upstream struct + * layout. This header is the opposite: it is full of them, and no Rust ever + * sees it. + */ + +#ifndef OPENSCREEN_PW_INTERNAL_H +#define OPENSCREEN_PW_INTERNAL_H + +#include +#include + +#include + +/* + * The subset of libpipewire that pw_audio.c calls, resolved once by + * osc_pw_load alongside the video half's table. One dlopen and one symbol + * table for the whole helper means a missing symbol is reported at load, not + * at the first audio stream — by which time the user is already recording. + */ +struct osc_pw_audio_api { + struct pw_thread_loop *(*thread_loop_new)(const char *name, const struct spa_dict *props); + void (*thread_loop_destroy)(struct pw_thread_loop *loop); + struct pw_loop *(*thread_loop_get_loop)(struct pw_thread_loop *loop); + int (*thread_loop_start)(struct pw_thread_loop *loop); + void (*thread_loop_stop)(struct pw_thread_loop *loop); + void (*thread_loop_lock)(struct pw_thread_loop *loop); + void (*thread_loop_unlock)(struct pw_thread_loop *loop); + struct pw_context *(*context_new)(struct pw_loop *loop, struct pw_properties *props, + size_t user_data_size); + void (*context_destroy)(struct pw_context *context); + struct pw_core *(*context_connect)(struct pw_context *context, struct pw_properties *props, + size_t user_data_size); + int (*core_disconnect)(struct pw_core *core); + struct pw_properties *(*properties_new)(const char *key, ...); + int (*properties_set)(struct pw_properties *props, const char *key, const char *value); + struct pw_stream *(*stream_new)(struct pw_core *core, const char *name, + struct pw_properties *props); + void (*stream_destroy)(struct pw_stream *stream); + void (*stream_add_listener)(struct pw_stream *stream, struct spa_hook *listener, + const struct pw_stream_events *events, void *data); + int (*stream_connect)(struct pw_stream *stream, enum pw_direction direction, + uint32_t target_id, enum pw_stream_flags flags, + const struct spa_pod **params, uint32_t n_params); + struct pw_buffer *(*stream_dequeue_buffer)(struct pw_stream *stream); + int (*stream_queue_buffer)(struct pw_stream *stream, struct pw_buffer *buffer); + const char *(*stream_state_as_string)(enum pw_stream_state state); + /* Enumeration only (osc_pw_list_audio_sources): a synchronous main loop + * rather than the thread loop the streams use. */ + struct pw_main_loop *(*main_loop_new)(const struct spa_dict *props); + void (*main_loop_destroy)(struct pw_main_loop *loop); + struct pw_loop *(*main_loop_get_loop)(struct pw_main_loop *loop); + int (*main_loop_run)(struct pw_main_loop *loop); + int (*main_loop_quit)(struct pw_main_loop *loop); + void (*proxy_destroy)(struct pw_proxy *proxy); +}; + +extern struct osc_pw_audio_api osc_audio_api; + +/* Shared so both files report failures in the same shape. */ +void osc_pw_set_error(char *err, size_t err_len, const char *format, ...); + +#endif /* OPENSCREEN_PW_INTERNAL_H */ diff --git a/electron/native/pipewire-capture/csrc/pw_shim.c b/electron/native/pipewire-capture/csrc/pw_shim.c new file mode 100644 index 0000000000..d12665f35d --- /dev/null +++ b/electron/native/pipewire-capture/csrc/pw_shim.c @@ -0,0 +1,889 @@ +/* + * PipeWire glue for the OpenScreen Linux capture helper. + * + * WHY THIS FILE IS C AND NOT RUST. + * + * Two thirds of the PipeWire/SPA API a consumer needs is not in the shared + * object at all: `spa_pod_builder_*`, the `SPA_POD_CHOICE_*` macros, + * `spa_format_video_raw_parse` and `spa_buffer_find_meta` are `static inline` + * in the headers. Rust cannot call them, and re-implementing the POD builder in + * Rust would mean restating a binary serialisation format by hand — exactly the + * kind of thing AGENTS.md flags as security-sensitive. Compiling the real + * headers keeps every struct layout and every POD byte in the hands of the + * upstream code that defines them. + * + * WHY dlopen AND NOT -lpipewire-0.3. + * + * Ubuntu ships `libpipewire-0.3.so.0` in the base system but the `.so` link and + * the headers only come with `libpipewire-0.3-dev`. Linking normally would put + * a dev package in every contributor's and CI runner's critical path, and would + * bake a hard DT_NEEDED into the helper so that it could not even start to + * print a clean "PipeWire is not available" error. dlopen at runtime gives us + * the vendored headers' ABI at compile time and a recoverable failure at run + * time — the same trade the compositor addon makes with ffmpeg. + * + * The headers under ../vendor/ are PipeWire 1.0.5, MIT, unmodified. The ABI has + * been stable across 0.3.x/1.x for everything used here (pw_stream, pw_context, + * spa_meta_cursor), so a runtime with a different minor version is fine. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "pw_shim.h" + +#include "pw_internal.h" + +#define OSC_PW_SONAME "libpipewire-0.3.so.0" + +/* + * Cursor metadata budget: `struct spa_meta_cursor` + `struct spa_meta_bitmap` + + * w*h*4 bytes of pixels. + * + * THE UPPER BOUND IS LOAD-BEARING, AND GETTING IT WRONG FAILS SILENTLY. + * + * PipeWire negotiates SPA_PARAM_Meta by intersecting the two ports' PODs + * (spa_pod_filter). Producers declare SPA_PARAM_META_size as a FIXED + * SPA_POD_Int, not a range — mutter 46.2 declares exactly + * SPA_POD_Int(CURSOR_META_SIZE(384, 384)) = 589872 bytes + * (src/backends/meta-screen-cast-stream-src.c). A consumer range that does not + * CONTAIN that constant intersects to nothing, the whole ParamMeta object is + * dropped, and the buffers simply arrive with no cursor metadata at all. There + * is no error, no warning, and no clue: the stream negotiates and runs happily. + * + * This bit us. The original 256x256 ceiling (copied from PipeWire's own + * video-play.c example, which targets cameras and never meets mutter) caps at + * 262192 bytes — below mutter's 589872 — so every buffer came back with + * hasCursorMeta=false. 1024x1024 = 4194352 bytes covers mutter's 384x384 and + * leaves headroom for compositors with larger cursor planes; the range is only + * an upper bound on what we accept, not an allocation we pay for. + * + * osc_pw_cursor_meta_accepts_producer_size() below turns this into something a + * unit test can assert instead of something a maintainer rediscovers. + */ +#define OSC_CURSOR_META_SIZE(w, h) \ + (sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + (size_t)(w) * (h) * 4) + +/* + * How many buffers to describe before going quiet. + * + * One is not enough to answer the question this instrumentation exists for. + * Metadata layout is fixed per buffer SET, so in theory one sample suffices — + * but a single line cannot distinguish "the stream delivered one buffer and + * died" from "the stream ran for a minute", and that ambiguity cost a debugging + * round trip. A handful of lines makes stream liveness visible without turning + * stdout into a firehose. + */ +#define OSC_BUFFER_INFO_REPORTS 5 + +/* Every libpipewire entry point the helper touches, resolved once by dlsym. */ +struct osc_pw_api { + void (*init)(int *argc, char ***argv); + const char *(*get_library_version)(void); + struct pw_thread_loop *(*thread_loop_new)(const char *name, const struct spa_dict *props); + void (*thread_loop_destroy)(struct pw_thread_loop *loop); + struct pw_loop *(*thread_loop_get_loop)(struct pw_thread_loop *loop); + int (*thread_loop_start)(struct pw_thread_loop *loop); + void (*thread_loop_stop)(struct pw_thread_loop *loop); + void (*thread_loop_lock)(struct pw_thread_loop *loop); + void (*thread_loop_unlock)(struct pw_thread_loop *loop); + struct pw_context *(*context_new)(struct pw_loop *loop, struct pw_properties *props, + size_t user_data_size); + void (*context_destroy)(struct pw_context *context); + struct pw_core *(*context_connect_fd)(struct pw_context *context, int fd, + struct pw_properties *props, size_t user_data_size); + int (*core_disconnect)(struct pw_core *core); + struct pw_properties *(*properties_new)(const char *key, ...); + struct pw_stream *(*stream_new)(struct pw_core *core, const char *name, + struct pw_properties *props); + void (*stream_destroy)(struct pw_stream *stream); + void (*stream_add_listener)(struct pw_stream *stream, struct spa_hook *listener, + const struct pw_stream_events *events, void *data); + int (*stream_connect)(struct pw_stream *stream, enum pw_direction direction, + uint32_t target_id, enum pw_stream_flags flags, + const struct spa_pod **params, uint32_t n_params); + int (*stream_disconnect)(struct pw_stream *stream); + struct pw_buffer *(*stream_dequeue_buffer)(struct pw_stream *stream); + int (*stream_queue_buffer)(struct pw_stream *stream, struct pw_buffer *buffer); + int (*stream_update_params)(struct pw_stream *stream, const struct spa_pod **params, + uint32_t n_params); + const char *(*stream_state_as_string)(enum pw_stream_state state); +}; + +static struct osc_pw_api api; +static void *api_handle; + +struct osc_pw_session { + struct pw_thread_loop *loop; + struct pw_context *context; + struct pw_core *core; + struct pw_stream *stream; + struct spa_hook stream_listener; + struct osc_pw_callbacks callbacks; + struct spa_video_info_raw format; + int buffer_info_reports; + int want_video; +}; + +struct osc_pw_audio_api osc_audio_api; + +/* The shared spelling pw_audio.c uses; `osc_set_error` below is the local + * alias this file has always called. */ +void osc_pw_set_error(char *err, size_t err_len, const char *format, ...) +{ + va_list args; + + if (err == NULL || err_len == 0) { + return; + } + va_start(args, format); + vsnprintf(err, err_len, format, args); + va_end(args); + err[err_len - 1] = '\0'; +} + +static void osc_set_error(char *err, size_t err_len, const char *format, ...) +{ + va_list args; + + if (err == NULL || err_len == 0) { + return; + } + va_start(args, format); + vsnprintf(err, err_len, format, args); + va_end(args); +} + +int osc_pw_load(char *err, size_t err_len) +{ + if (api_handle != NULL) { + return 0; + } + + api_handle = dlopen(OSC_PW_SONAME, RTLD_NOW | RTLD_LOCAL); + if (api_handle == NULL) { + osc_set_error(err, err_len, "%s could not be loaded: %s", OSC_PW_SONAME, dlerror()); + return -1; + } + +/* The `*(void **)&field` dance is the POSIX-blessed way to assign a dlsym + * result to a function pointer without tripping -Wpedantic. */ +#define OSC_LOAD(field, symbol) \ + do { \ + *(void **)(&api.field) = dlsym(api_handle, symbol); \ + if (api.field == NULL) { \ + osc_set_error(err, err_len, "%s is missing symbol %s", OSC_PW_SONAME, \ + symbol); \ + dlclose(api_handle); \ + api_handle = NULL; \ + return -1; \ + } \ + } while (0) + + OSC_LOAD(init, "pw_init"); + OSC_LOAD(get_library_version, "pw_get_library_version"); + OSC_LOAD(thread_loop_new, "pw_thread_loop_new"); + OSC_LOAD(thread_loop_destroy, "pw_thread_loop_destroy"); + OSC_LOAD(thread_loop_get_loop, "pw_thread_loop_get_loop"); + OSC_LOAD(thread_loop_start, "pw_thread_loop_start"); + OSC_LOAD(thread_loop_stop, "pw_thread_loop_stop"); + OSC_LOAD(thread_loop_lock, "pw_thread_loop_lock"); + OSC_LOAD(thread_loop_unlock, "pw_thread_loop_unlock"); + OSC_LOAD(context_new, "pw_context_new"); + OSC_LOAD(context_destroy, "pw_context_destroy"); + OSC_LOAD(context_connect_fd, "pw_context_connect_fd"); + OSC_LOAD(core_disconnect, "pw_core_disconnect"); + OSC_LOAD(properties_new, "pw_properties_new"); + OSC_LOAD(stream_new, "pw_stream_new"); + OSC_LOAD(stream_destroy, "pw_stream_destroy"); + OSC_LOAD(stream_add_listener, "pw_stream_add_listener"); + OSC_LOAD(stream_connect, "pw_stream_connect"); + OSC_LOAD(stream_disconnect, "pw_stream_disconnect"); + OSC_LOAD(stream_dequeue_buffer, "pw_stream_dequeue_buffer"); + OSC_LOAD(stream_queue_buffer, "pw_stream_queue_buffer"); + OSC_LOAD(stream_update_params, "pw_stream_update_params"); + OSC_LOAD(stream_state_as_string, "pw_stream_state_as_string"); + +/* The audio half's table. Same dlopen, same failure path: a libpipewire too old + * to have one of these should be reported here, at load, and not halfway into a + * recording. */ +#define OSC_LOAD_AUDIO(field, symbol) \ + do { \ + *(void **)(&osc_audio_api.field) = dlsym(api_handle, symbol); \ + if (osc_audio_api.field == NULL) { \ + osc_set_error(err, err_len, "%s is missing symbol %s", OSC_PW_SONAME, \ + symbol); \ + dlclose(api_handle); \ + api_handle = NULL; \ + return -1; \ + } \ + } while (0) + + OSC_LOAD_AUDIO(thread_loop_new, "pw_thread_loop_new"); + OSC_LOAD_AUDIO(thread_loop_destroy, "pw_thread_loop_destroy"); + OSC_LOAD_AUDIO(thread_loop_get_loop, "pw_thread_loop_get_loop"); + OSC_LOAD_AUDIO(thread_loop_start, "pw_thread_loop_start"); + OSC_LOAD_AUDIO(thread_loop_stop, "pw_thread_loop_stop"); + OSC_LOAD_AUDIO(thread_loop_lock, "pw_thread_loop_lock"); + OSC_LOAD_AUDIO(thread_loop_unlock, "pw_thread_loop_unlock"); + OSC_LOAD_AUDIO(context_new, "pw_context_new"); + OSC_LOAD_AUDIO(context_destroy, "pw_context_destroy"); + OSC_LOAD_AUDIO(context_connect, "pw_context_connect"); + OSC_LOAD_AUDIO(core_disconnect, "pw_core_disconnect"); + OSC_LOAD_AUDIO(properties_new, "pw_properties_new"); + OSC_LOAD_AUDIO(properties_set, "pw_properties_set"); + OSC_LOAD_AUDIO(stream_new, "pw_stream_new"); + OSC_LOAD_AUDIO(stream_destroy, "pw_stream_destroy"); + OSC_LOAD_AUDIO(stream_add_listener, "pw_stream_add_listener"); + OSC_LOAD_AUDIO(stream_connect, "pw_stream_connect"); + OSC_LOAD_AUDIO(stream_dequeue_buffer, "pw_stream_dequeue_buffer"); + OSC_LOAD_AUDIO(stream_queue_buffer, "pw_stream_queue_buffer"); + OSC_LOAD_AUDIO(stream_state_as_string, "pw_stream_state_as_string"); + OSC_LOAD_AUDIO(main_loop_new, "pw_main_loop_new"); + OSC_LOAD_AUDIO(main_loop_destroy, "pw_main_loop_destroy"); + OSC_LOAD_AUDIO(main_loop_get_loop, "pw_main_loop_get_loop"); + OSC_LOAD_AUDIO(main_loop_run, "pw_main_loop_run"); + OSC_LOAD_AUDIO(main_loop_quit, "pw_main_loop_quit"); + OSC_LOAD_AUDIO(proxy_destroy, "pw_proxy_destroy"); + +#undef OSC_LOAD_AUDIO +#undef OSC_LOAD + + api.init(NULL, NULL); + return 0; +} + +const char *osc_pw_library_version(void) +{ + return api_handle != NULL ? api.get_library_version() : NULL; +} + +void osc_pw_constants(struct osc_pw_constants *out) +{ + out->video_format_rgbx = SPA_VIDEO_FORMAT_RGBx; + out->video_format_bgrx = SPA_VIDEO_FORMAT_BGRx; + out->video_format_xrgb = SPA_VIDEO_FORMAT_xRGB; + out->video_format_xbgr = SPA_VIDEO_FORMAT_xBGR; + out->video_format_rgba = SPA_VIDEO_FORMAT_RGBA; + out->video_format_bgra = SPA_VIDEO_FORMAT_BGRA; + out->video_format_argb = SPA_VIDEO_FORMAT_ARGB; + out->video_format_abgr = SPA_VIDEO_FORMAT_ABGR; + out->data_mem_ptr = SPA_DATA_MemPtr; + out->data_mem_fd = SPA_DATA_MemFd; + out->data_dma_buf = SPA_DATA_DmaBuf; +} + +/* + * Publish what we accept. The pixel format list is broad on purpose: Stage 1 + * never reads a pixel, and narrowing it would only make the compositor refuse + * formats that Stage 2 may well want. What matters here is that the stream + * negotiates at all, because SPA_META_Cursor rides on the video buffers. + */ +static const struct spa_pod *osc_build_enum_format(struct spa_pod_builder *builder) +{ + return spa_pod_builder_add_object( + builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, SPA_FORMAT_mediaType, + SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype, + SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), SPA_FORMAT_VIDEO_format, + /* SPA_POD_CHOICE_ENUM counts the DEFAULT plus the alternatives, and the + * default is repeated as the first alternative. BGRx appearing twice is + * the idiom, not a typo: 5 values = default BGRx + {BGRx,RGBx,BGRA,RGBA}. */ + SPA_POD_CHOICE_ENUM_Id(5, SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_BGRx, + SPA_VIDEO_FORMAT_RGBx, SPA_VIDEO_FORMAT_BGRA, + SPA_VIDEO_FORMAT_RGBA), + SPA_FORMAT_VIDEO_size, + SPA_POD_CHOICE_RANGE_Rectangle(&SPA_RECTANGLE(1920, 1080), &SPA_RECTANGLE(1, 1), + &SPA_RECTANGLE(16384, 16384)), + SPA_FORMAT_VIDEO_framerate, + SPA_POD_CHOICE_RANGE_Fraction(&SPA_FRACTION(30, 1), &SPA_FRACTION(0, 1), + &SPA_FRACTION(240, 1))); +} + +/* + * The consumer side of the SPA_META_Cursor negotiation, in one place so the + * bytes a unit test checks are literally the bytes sent on the wire. + */ +static const struct spa_pod *osc_build_cursor_meta(struct spa_pod_builder *builder) +{ + return spa_pod_builder_add_object( + builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Cursor), SPA_PARAM_META_size, + SPA_POD_CHOICE_RANGE_Int((int32_t)OSC_CURSOR_META_SIZE(64, 64), + (int32_t)OSC_CURSOR_META_SIZE(1, 1), + (int32_t)OSC_CURSOR_META_SIZE(1024, 1024))); +} + +int osc_pw_cursor_meta_accepts_producer_size(uint32_t width, uint32_t height) +{ + uint8_t ours_storage[512]; + uint8_t theirs_storage[512]; + uint8_t result_storage[512]; + struct spa_pod_builder ours = SPA_POD_BUILDER_INIT(ours_storage, sizeof(ours_storage)); + struct spa_pod_builder theirs = SPA_POD_BUILDER_INIT(theirs_storage, sizeof(theirs_storage)); + struct spa_pod_builder result = SPA_POD_BUILDER_INIT(result_storage, sizeof(result_storage)); + struct spa_pod *filtered = NULL; + const struct spa_pod *consumer; + const struct spa_pod *producer; + + consumer = osc_build_cursor_meta(&ours); + /* Exactly how a compositor declares it: a FIXED size, not a range. */ + producer = spa_pod_builder_add_object( + &theirs, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Cursor), SPA_PARAM_META_size, + SPA_POD_Int((int32_t)OSC_CURSOR_META_SIZE(width, height))); + if (consumer == NULL || producer == NULL) { + return -1; + } + + /* The same call pw_impl_link uses to intersect the two ports' params. A + * negative result means the objects do not intersect, which is precisely the + * failure mode that leaves buffers with no cursor metadata. */ + return spa_pod_filter(&result, &filtered, producer, consumer) < 0 ? 0 : 1; +} + +static void osc_on_param_changed(void *userdata, uint32_t id, const struct spa_pod *param) +{ + struct osc_pw_session *session = userdata; + uint8_t buffer[1024]; + struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer)); + const struct spa_pod *params[3]; + struct osc_pw_format reported; + uint32_t media_type; + uint32_t media_subtype; + + if (param == NULL || id != SPA_PARAM_Format) { + return; + } + if (spa_format_parse(param, &media_type, &media_subtype) < 0) { + return; + } + if (media_type != SPA_MEDIA_TYPE_video || media_subtype != SPA_MEDIA_SUBTYPE_raw) { + return; + } + if (spa_format_video_raw_parse(param, &session->format) < 0) { + return; + } + reported.width = (int32_t)session->format.size.width; + reported.height = (int32_t)session->format.size.height; + reported.video_format = session->format.format; + reported.framerate_num = (int32_t)session->format.framerate.num; + reported.framerate_denom = (int32_t)session->format.framerate.denom; + if (session->callbacks.on_format != NULL) { + session->callbacks.on_format(session->callbacks.user, &reported); + } + + /* + * No `size`/`stride` constraint is published: the compositor's own choice is + * fine, and osc_read_frame validates whatever comes back. + * + * The dataType set differs by mode, and the difference is load-bearing. + * Cursor-only advertises everything, so that on_buffer_info reports what the + * compositor would PREFER rather than what we forced it into. Video mode + * advertises shared memory only: pw_stream does not map DmaBuf even with + * PW_STREAM_FLAG_MAP_BUFFERS, so accepting one would leave `datas[0].data` + * NULL and produce a recording of nothing. Importing DmaBuf properly is its + * own piece of work; until then, not offering it is what makes the + * compositor fall back to memfd instead. + */ + params[0] = spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, SPA_PARAM_BUFFERS_buffers, + SPA_POD_CHOICE_RANGE_Int(4, 2, 16), SPA_PARAM_BUFFERS_blocks, SPA_POD_Int(1), + SPA_PARAM_BUFFERS_dataType, + SPA_POD_CHOICE_FLAGS_Int(session->want_video + ? ((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd)) + : ((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd) | + (1 << SPA_DATA_DmaBuf)))); + + params[1] = spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Header), SPA_PARAM_META_size, + SPA_POD_Int(sizeof(struct spa_meta_header))); + + params[2] = osc_build_cursor_meta(&builder); + + /* The builder returns NULL if its fixed buffer overflowed. 1 KiB is far more + * than these three objects need, but handing NULLs to update_params would be + * a null deref inside libpipewire, so it is checked rather than assumed. */ + if (params[0] == NULL || params[1] == NULL || params[2] == NULL) { + return; + } + + /* Buffers are reallocated on renegotiation, and the metadata layout comes + * with them. Re-arm the reports so the instrumentation describes the buffer + * set actually in use rather than a set that no longer exists. */ + session->buffer_info_reports = 0; + + api.stream_update_params(session->stream, params, 3); +} + +static void osc_on_state_changed(void *userdata, enum pw_stream_state old, + enum pw_stream_state state, const char *error) +{ + struct osc_pw_session *session = userdata; + + (void)old; + if (session->callbacks.on_state != NULL) { + session->callbacks.on_state(session->callbacks.user, api.stream_state_as_string(state), + error); + } +} + +/* + * Extract the cursor metadata, or return 0 if this buffer has none. + * + * This uses spa_buffer_find_meta rather than the more usual + * spa_buffer_find_meta_data because the latter returns only the pointer and + * throws away `spa_meta::size`. That size is the sole bound available for + * validating the bitmap offsets below, so it has to be kept. The size check + * find_meta_data would have performed is done explicitly instead. + * + * Every offset in `spa_meta_cursor` is attacker-controlled from this process's + * point of view (it comes from another process's shared memory), so each one is + * validated against the metadata block's declared size before it is followed. + * The arithmetic is done in uint64_t so that a hostile 32-bit offset cannot + * wrap past the bound. + */ +static int osc_read_cursor(const struct spa_buffer *buffer, struct osc_pw_cursor *out, + uint32_t *meta_size_out) +{ + struct spa_meta *meta; + struct spa_meta_cursor *cursor; + struct spa_meta_bitmap *bitmap; + uint64_t meta_size; + uint64_t bitmap_offset; + uint64_t pixels_offset; + uint64_t pixels_len; + + memset(out, 0, sizeof(*out)); + + meta = spa_buffer_find_meta(buffer, SPA_META_Cursor); + if (meta == NULL) { + return 0; + } + *meta_size_out = meta->size; + meta_size = meta->size; + if (meta_size < sizeof(struct spa_meta_cursor) || meta->data == NULL) { + return 0; + } + + cursor = meta->data; + if (!spa_meta_cursor_is_valid(cursor)) { + /* id == 0 means "nothing new"; the previous position still stands. */ + return 0; + } + + out->id = cursor->id; + out->flags = cursor->flags; + out->x = cursor->position.x; + out->y = cursor->position.y; + out->hotspot_x = cursor->hotspot.x; + out->hotspot_y = cursor->hotspot.y; + + bitmap_offset = cursor->bitmap_offset; + if (bitmap_offset < sizeof(struct spa_meta_cursor) || + bitmap_offset + sizeof(struct spa_meta_bitmap) > meta_size) { + return 1; + } + + bitmap = SPA_PTROFF(cursor, (size_t)bitmap_offset, struct spa_meta_bitmap); + if (!spa_meta_bitmap_is_valid(bitmap)) { + return 1; + } + if (bitmap->stride <= 0 || bitmap->size.width == 0 || bitmap->size.height == 0) { + return 1; + } + + pixels_offset = bitmap_offset + bitmap->offset; + if (bitmap->offset < sizeof(struct spa_meta_bitmap) || pixels_offset >= meta_size) { + return 1; + } + pixels_len = (uint64_t)bitmap->stride * (uint64_t)bitmap->size.height; + if (pixels_len == 0 || pixels_len > meta_size - pixels_offset) { + return 1; + } + + out->has_bitmap = 1; + out->bitmap_format = bitmap->format; + out->bitmap_width = (int32_t)bitmap->size.width; + out->bitmap_height = (int32_t)bitmap->size.height; + out->bitmap_stride = bitmap->stride; + out->bitmap_data = SPA_PTROFF(cursor, (size_t)pixels_offset, const uint8_t); + out->bitmap_len = (size_t)pixels_len; + return 1; +} + +/* + * Extracts the pixels of one buffer. Returns 1 when `out` describes a frame, 0 + * when this buffer carries none. + * + * The offset/size clamping against `maxsize` is the standard PipeWire consumer + * idiom and is not paranoia: `chunk` lives in memory the PRODUCER writes, so its + * fields are untrusted input from another process. A compositor bug — or a + * malicious one — that reports a size past the end of the mapping would + * otherwise be a read straight off the end of the shared memory. + */ +static int osc_read_frame(struct osc_pw_session *session, const struct spa_buffer *buffer, + struct osc_pw_frame *out) +{ + struct spa_data *data; + struct spa_meta_header *header; + uint32_t offset; + uint32_t size; + int32_t stride; + int32_t height; + + memset(out, 0, sizeof(*out)); + out->pts_ns = -1; + + if (buffer->n_datas < 1) { + return 0; + } + data = &buffer->datas[0]; + /* + * NULL means the buffer was never mapped: either this is a cursor-only + * session (no PW_STREAM_FLAG_MAP_BUFFERS) or the compositor handed us a + * DmaBuf, which pw_stream does not map even with the flag. Neither is an + * error here — the format negotiation excludes DmaBuf when want_video is + * set, so in practice this is the cursor-only case. + */ + if (data->data == NULL || data->chunk == NULL) { + return 0; + } + /* A zero-sized chunk is how a compositor ships a cursor update with no new + * frame attached. Not an error, just not a frame. */ + if (data->chunk->size == 0) { + return 0; + } + + offset = SPA_MIN(data->chunk->offset, data->maxsize); + size = SPA_MIN(data->chunk->size, data->maxsize - offset); + + height = (int32_t)session->format.size.height; + stride = data->chunk->stride; + if (stride <= 0 || height <= 0) { + return 0; + } + /* One short row is one row of garbage in the recording; refuse the whole + * frame instead, and let the caller count it as dropped. */ + if ((uint64_t)stride * (uint64_t)height > (uint64_t)size) { + return 0; + } + + out->data = SPA_PTROFF(data->data, offset, const uint8_t); + out->size = size; + out->stride = stride; + out->width = (int32_t)session->format.size.width; + out->height = height; + out->video_format = session->format.format; + + header = spa_buffer_find_meta_data(buffer, SPA_META_Header, sizeof(*header)); + if (header != NULL) { + out->pts_ns = header->pts; + } + return 1; +} + +static const char *osc_meta_type_name(uint32_t type) +{ + switch (type) { + case SPA_META_Header: + return "Header"; + case SPA_META_VideoCrop: + return "VideoCrop"; + case SPA_META_VideoDamage: + return "VideoDamage"; + case SPA_META_Bitmap: + return "Bitmap"; + case SPA_META_Cursor: + return "Cursor"; + case SPA_META_Control: + return "Control"; + case SPA_META_Busy: + return "Busy"; + case SPA_META_VideoTransform: + return "VideoTransform"; + default: + return "?"; + } +} + +/* + * "Header:12,Cursor:589872" — every metadata block the negotiation actually + * produced, with its size. + * + * This exists because a missing SPA_META_Cursor used to be reported as a bare + * `hasCursorMeta: false`, which cannot distinguish "our ParamMeta never reached + * the negotiation" from "it reached it and was filtered out on size". Listing + * the survivors answers that in one line: if Header is present and Cursor is + * not, the params were sent and the cursor object lost the intersection. + */ +static void osc_describe_metas(const struct spa_buffer *buffer, char *out, size_t out_len) +{ + size_t used = 0; + uint32_t i; + + if (out_len == 0) { + return; + } + out[0] = '\0'; + for (i = 0; i < buffer->n_metas; i++) { + int written = snprintf(out + used, out_len - used, "%s%s:%u", used > 0 ? "," : "", + osc_meta_type_name(buffer->metas[i].type), buffer->metas[i].size); + if (written < 0 || (size_t)written >= out_len - used) { + /* Truncated: leave what fits, NUL-terminated by snprintf. */ + return; + } + used += (size_t)written; + } +} + +static void osc_inspect_buffer(struct osc_pw_session *session, const struct spa_buffer *buffer) +{ + struct osc_pw_cursor cursor; + uint32_t meta_size = 0; + + if (session->buffer_info_reports < OSC_BUFFER_INFO_REPORTS && + session->callbacks.on_buffer_info != NULL) { + uint32_t data_type = buffer->n_datas > 0 ? buffer->datas[0].type : 0; + struct spa_meta *cursor_meta = spa_buffer_find_meta(buffer, SPA_META_Cursor); + char metas[256]; + + osc_describe_metas(buffer, metas, sizeof(metas)); + session->buffer_info_reports++; + session->callbacks.on_buffer_info(session->callbacks.user, data_type, buffer->n_datas, + cursor_meta != NULL, + cursor_meta != NULL ? cursor_meta->size : 0, metas); + } + + if (osc_read_cursor(buffer, &cursor, &meta_size) && session->callbacks.on_cursor != NULL) { + session->callbacks.on_cursor(session->callbacks.user, &cursor); + } + + /* + * Cursor first, then pixels. The order matters for the frame the cursor + * shape changes on: the consumer stamps its cursor track from the latest + * sample, and reading the cursor after the frame would attribute the new + * position to the NEXT frame instead of this one. + */ + if (session->want_video && session->callbacks.on_frame != NULL) { + struct osc_pw_frame frame; + + if (osc_read_frame(session, buffer, &frame)) { + session->callbacks.on_frame(session->callbacks.user, &frame); + } + } +} + +static void osc_on_process(void *userdata) +{ + struct osc_pw_session *session = userdata; + struct pw_buffer *b; + + /* + * EVERY dequeued buffer is inspected, in arrival order. + * + * The obvious optimisation — drain to the newest buffer and drop the rest — + * is wrong here, and was the code this replaced. Cursor updates are not + * guaranteed to be attached to buffers that also carry a video frame: a + * compositor may deliver a buffer whose chunk size is 0 purely to ship a new + * SPA_META_Cursor (KWin is documented as doing this). Dropping "stale" + * buffers would silently throw those away, which is indistinguishable from + * the cursor never moving. + * + * Reading metadata is a handful of struct field loads, so doing it per + * buffer costs nothing. + * + * Since Stage 2 this loop DOES touch pixels, and the same reasoning still + * holds — but for a different reason. The frame callback copies into a + * single-slot mailbox on the Rust side where a newer frame overwrites an + * unconsumed older one, so a backlog is dropped there, at the point that + * knows whether the encoder is keeping up. Dropping here instead would also + * throw away the cursor metadata riding on the same buffers. + */ + while ((b = api.stream_dequeue_buffer(session->stream)) != NULL) { + osc_inspect_buffer(session, b->buffer); + api.stream_queue_buffer(session->stream, b); + } +} + +static const struct pw_stream_events osc_stream_events = { + PW_VERSION_STREAM_EVENTS, + .state_changed = osc_on_state_changed, + .param_changed = osc_on_param_changed, + .process = osc_on_process, +}; + +struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video, + const struct osc_pw_callbacks *callbacks, char *err, + size_t err_len) +{ + struct osc_pw_session *session; + uint8_t buffer[1024]; + struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer)); + const struct spa_pod *params[1]; + int result; + + if (api_handle == NULL) { + osc_set_error(err, err_len, "osc_pw_start called before osc_pw_load"); + close(fd); + return NULL; + } + + session = calloc(1, sizeof(*session)); + if (session == NULL) { + osc_set_error(err, err_len, "out of memory"); + close(fd); + return NULL; + } + session->callbacks = *callbacks; + session->want_video = want_video; + + session->loop = api.thread_loop_new("openscreen-pipewire", NULL); + if (session->loop == NULL) { + osc_set_error(err, err_len, "pw_thread_loop_new failed"); + close(fd); + free(session); + return NULL; + } + + /* + * Everything below runs with the loop lock held. The loop is not started + * yet, so nothing can race us, but taking the lock keeps the teardown path + * (which does run concurrently) symmetric with the setup path. + */ + api.thread_loop_lock(session->loop); + + session->context = api.context_new(api.thread_loop_get_loop(session->loop), NULL, 0); + if (session->context == NULL) { + osc_set_error(err, err_len, "pw_context_new failed"); + close(fd); + goto fail; + } + + /* Takes ownership of fd, on success and on failure alike. */ + session->core = api.context_connect_fd(session->context, fd, NULL, 0); + if (session->core == NULL) { + osc_set_error(err, err_len, "pw_context_connect_fd failed: %s", strerror(errno)); + goto fail; + } + + session->stream = api.stream_new(session->core, "openscreen-cursor", + api.properties_new(PW_KEY_MEDIA_TYPE, "Video", + PW_KEY_MEDIA_CATEGORY, "Capture", + PW_KEY_MEDIA_ROLE, "Screen", NULL)); + if (session->stream == NULL) { + osc_set_error(err, err_len, "pw_stream_new failed"); + goto fail; + } + + api.stream_add_listener(session->stream, &session->stream_listener, &osc_stream_events, + session); + + params[0] = osc_build_enum_format(&builder); + if (params[0] == NULL) { + osc_set_error(err, err_len, "the EnumFormat POD did not fit its builder"); + goto fail; + } + + /* + * Flags. PW_STREAM_FLAG_MAP_BUFFERS only when the caller wants pixels: + * mapping a full-screen framebuffer on every buffer is pure waste for a + * cursor-only session. Metadata is unaffected either way — pw_stream maps the + * buffer skeleton (and therefore every spa_meta) regardless of this flag; + * MAP_BUFFERS only governs whether `datas[i].data` is populated. + * + * And one flag that is absent on purpose: + * + * No PW_STREAM_FLAG_DONT_RECONNECT, which this code used to set. That flag + * killed the first real GNOME run, and the chain is worth writing down + * because the symptom names nothing that appears in our source: + * + * 1. pw_stream turns the flag into the node property `node.dont-reconnect` + * (pipewire 1.0.5 src/pipewire/stream.c:2020). + * 2. WirePlumber reads it as `reconnect = not node.dont-reconnect` + * (/usr/share/wireplumber/scripts/policy-node.lua:653). + * 3. When the session manager cannot resolve a target, the `not reconnect` + * branch reports the error string "target not found" — the exact text we + * saw — where the reconnecting branch would merely say "no target node + * available" and wait (policy-node.lua:807). + * 4. That same branch then DESTROYS the node (policy-node.lua:812), so the + * stream is gone for good rather than re-linking. + * + * In other words the flag converts a transient link failure into a silent, + * permanent end of capture. OBS's linux-pipewire plugin — the reference + * implementation that works on GNOME — does not set it either. + * + * `node_id` is passed as the connect target rather than through + * PW_KEY_TARGET_OBJECT. That is the older of the two spellings, but it is what + * OBS does and it is verified working here (see the ignored integration test, + * which connects by numeric id and reaches `streaming`). + */ + result = api.stream_connect(session->stream, PW_DIRECTION_INPUT, node_id, + want_video ? (PW_STREAM_FLAG_AUTOCONNECT | + PW_STREAM_FLAG_MAP_BUFFERS) + : PW_STREAM_FLAG_AUTOCONNECT, + params, 1); + if (result < 0) { + osc_set_error(err, err_len, "pw_stream_connect failed: %s", spa_strerror(result)); + goto fail; + } + + api.thread_loop_unlock(session->loop); + + if (api.thread_loop_start(session->loop) < 0) { + osc_set_error(err, err_len, "pw_thread_loop_start failed"); + api.thread_loop_lock(session->loop); + goto fail; + } + + return session; + +fail: + api.thread_loop_unlock(session->loop); + if (session->stream != NULL) { + api.stream_destroy(session->stream); + } + if (session->core != NULL) { + api.core_disconnect(session->core); + } + if (session->context != NULL) { + api.context_destroy(session->context); + } + api.thread_loop_destroy(session->loop); + free(session); + return NULL; +} + +void osc_pw_stop(struct osc_pw_session *session) +{ + if (session == NULL) { + return; + } + + /* pw_thread_loop_stop must be called WITHOUT the lock: it joins the thread. */ + api.thread_loop_stop(session->loop); + + if (session->stream != NULL) { + api.stream_disconnect(session->stream); + api.stream_destroy(session->stream); + } + if (session->core != NULL) { + api.core_disconnect(session->core); + } + if (session->context != NULL) { + api.context_destroy(session->context); + } + api.thread_loop_destroy(session->loop); + free(session); +} diff --git a/electron/native/pipewire-capture/csrc/pw_shim.h b/electron/native/pipewire-capture/csrc/pw_shim.h new file mode 100644 index 0000000000..f67cd33f0c --- /dev/null +++ b/electron/native/pipewire-capture/csrc/pw_shim.h @@ -0,0 +1,223 @@ +/* + * Flat C ABI between the Rust helper and libpipewire. + * + * Every declaration here is mirrored by hand in `src/shim.rs`. The two files + * are a matched pair: changing a struct on one side without the other is an ABI + * break the compiler cannot catch, so keep the field order and the integer + * widths identical. + * + * Only plain scalars and pointers cross this boundary — no SPA or PipeWire type + * appears in a signature — so Rust never has to restate a PipeWire struct + * layout. That is deliberate: the layouts live in the vendored headers and are + * consumed by the C compiler that also compiled libpipewire's own users. + */ + +#ifndef OPENSCREEN_PW_SHIM_H +#define OPENSCREEN_PW_SHIM_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* One SPA_META_Cursor observation, already bounds-checked by the shim. */ +struct osc_pw_cursor { + /* Cursor position in stream pixels, top-left origin of the captured region. */ + int32_t x; + int32_t y; + /* Hotspot inside the bitmap. Meaningless when `has_bitmap` is 0. */ + int32_t hotspot_x; + int32_t hotspot_y; + /* Compositor-assigned shape id. 0 means "no cursor data in this frame". */ + uint32_t id; + uint32_t flags; + /* 1 when this frame carried a new shape bitmap. Compositors only send one + * when the shape changes, so most frames have has_bitmap == 0. */ + int32_t has_bitmap; + uint32_t bitmap_format; /* enum spa_video_format */ + int32_t bitmap_width; + int32_t bitmap_height; + int32_t bitmap_stride; + /* Borrowed for the duration of the callback only. Copy before returning. */ + const uint8_t *bitmap_data; + size_t bitmap_len; +}; + +/* + * One video frame, already bounds-checked and mapped. + * + * Only produced when `osc_pw_start` was asked for video; a cursor-only session + * never maps a pixel. Buffers that carry no frame (chunk size 0 — how some + * compositors ship a cursor update on its own) are skipped rather than reported + * as a zero-sized frame. + */ +struct osc_pw_frame { + /* Borrowed for the duration of the callback only. Copy before returning: + * the buffer is re-queued to the compositor as soon as it returns. */ + const uint8_t *data; + size_t size; + int32_t stride; + int32_t width; + int32_t height; + uint32_t video_format; /* enum spa_video_format */ + /* SPA_META_Header pts in nanoseconds on the compositor's monotonic clock, + * or -1 when the buffer carried no header. Far better than the callback's + * arrival time: it is stamped when the frame was composited, not when this + * process got round to looking at it. */ + int64_t pts_ns; +}; + +/* The negotiated video format. Reported once, from param_changed. */ +struct osc_pw_format { + int32_t width; + int32_t height; + uint32_t video_format; /* enum spa_video_format */ + int32_t framerate_num; + int32_t framerate_denom; +}; + +/* + * Callbacks fire on the PipeWire thread, never on the caller's thread. They + * must not block: the Rust side only pushes onto an unbounded channel. + */ +struct osc_pw_callbacks { + void *user; + void (*on_format)(void *user, const struct osc_pw_format *format); + void (*on_cursor)(void *user, const struct osc_pw_cursor *cursor); + /* Only ever called when osc_pw_start was given want_video != 0. */ + void (*on_frame)(void *user, const struct osc_pw_frame *frame); + /* Emitted once per negotiated buffer set. `data_type` is the SPA_DATA_* of + * datas[0]; `metas` is a borrowed "Header:12,Cursor:589872" listing of every + * metadata block that survived negotiation, which is what distinguishes a + * ParamMeta that was never sent from one that lost the size intersection. */ + void (*on_buffer_info)(void *user, uint32_t data_type, uint32_t n_datas, + int32_t has_cursor_meta, uint32_t cursor_meta_size, + const char *metas); + void (*on_state)(void *user, const char *state, const char *error); +}; + +/* + * SPA enum values, read out of the vendored headers rather than restated in + * Rust. Hardcoding `SPA_VIDEO_FORMAT_BGRA == 12` on the Rust side would work + * today and rot silently the day upstream inserts a value; this cannot. + */ +struct osc_pw_constants { + uint32_t video_format_rgbx; + uint32_t video_format_bgrx; + uint32_t video_format_xrgb; + uint32_t video_format_xbgr; + uint32_t video_format_rgba; + uint32_t video_format_bgra; + uint32_t video_format_argb; + uint32_t video_format_abgr; + uint32_t data_mem_ptr; + uint32_t data_mem_fd; + uint32_t data_dma_buf; +}; + +void osc_pw_constants(struct osc_pw_constants *out); + +/* + * Would our SPA_META_Cursor declaration survive negotiation against a producer + * that declares a FIXED size of `width` x `height` pixels? 1 yes, 0 no, -1 if + * the PODs could not be built. + * + * Compositors declare that size as a constant (mutter 46.2: 384x384), so our + * accepted range has to contain it or the metadata is silently dropped from the + * buffers. Exposed so a unit test can assert the bound without a portal, a + * compositor, or a screen. + */ +int osc_pw_cursor_meta_accepts_producer_size(uint32_t width, uint32_t height); + +struct osc_pw_session; + +/* + * dlopen("libpipewire-0.3.so.0") and resolve every symbol used below. Returns 0 + * on success, -1 with a NUL-terminated message in `err` otherwise. Idempotent. + */ +int osc_pw_load(char *err, size_t err_len); + +/* Runtime library version string, or NULL before a successful osc_pw_load. */ +const char *osc_pw_library_version(void); + +/* + * Connect to the PipeWire remote behind `fd` (from the portal's + * OpenPipeWireRemote) and start consuming `node_id`. + * + * Takes ownership of `fd`: once handed to pw_context_connect_fd, libpipewire's + * loop owns and closes it. Failures before that point close it here. In the one + * narrow case where pw_context_connect_fd itself fails, upstream may or may not + * have taken it, so it is deliberately leaked rather than risking a double + * close — the caller is on its way to reporting a fatal error either way. + * + * `want_video` decides whether pixels are mapped at all. With it set the stream + * asks for PW_STREAM_FLAG_MAP_BUFFERS and restricts itself to shared-memory + * buffer types; without it neither happens, and a cursor-only session never pays + * to map a full-screen framebuffer per frame. + * + * Returns NULL on failure, with a message in `err`. + */ +struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video, + const struct osc_pw_callbacks *callbacks, char *err, + size_t err_len); + +/* Stops the thread loop, joins it, and frees everything. Safe with NULL. */ +void osc_pw_stop(struct osc_pw_session *session); + +/* --------------------------------------------------------------------------- + * Audio (csrc/pw_audio.c) + * + * A second, independent PipeWire connection — to the session's own daemon, not + * to the portal's restricted remote, which carries no audio. See the header of + * pw_audio.c for why that is the only option. + * ------------------------------------------------------------------------- */ + +struct osc_pw_audio; + +struct osc_pw_audio_callbacks { + void *user; + /* `interleaved` holds `n_samples` floats — that is n_samples/channels frames + * — and is borrowed for the duration of the call. */ + void (*on_samples)(void *user, const float *interleaved, uint32_t n_samples); + void (*on_state)(void *user, const char *state, const char *error); +}; + +/* + * Opens one capture stream. + * + * `capture_sink` non-zero links to a SINK's monitor ports (what is being played) + * rather than to a source; that is the difference between recording the system + * mix and recording the microphone. `target_object` names a specific node, or is + * NULL/"" for the session default. + * + * Returns NULL on failure with a message in `err`. + */ +struct osc_pw_audio *osc_pw_audio_start(const char *target_object, int capture_sink, + uint32_t rate, uint32_t channels, + const struct osc_pw_audio_callbacks *callbacks, char *err, + size_t err_len); + +/* Stops the thread loop, joins it, and frees everything. Safe with NULL. */ +void osc_pw_audio_stop(struct osc_pw_audio *audio); + +/* + * Lists the graph's audio CAPTURE nodes into `out` as records + * "node.name\037node.description\036", NUL-terminated. + * + * Synchronous: it runs a main loop until the registry has replayed every + * existing global, then returns. Needed because the app's microphone picker + * carries Chromium device labels, and PW_KEY_TARGET_OBJECT only accepts a + * PipeWire node name — without this the stream falls back to the session + * default source, which is rarely the microphone the user picked. + * + * Returns 0 on success, -1 with a message in `err`. + */ +int osc_pw_list_audio_sources(char *out, size_t out_len, char *err, size_t err_len); + +#ifdef __cplusplus +} +#endif + +#endif /* OPENSCREEN_PW_SHIM_H */ diff --git a/electron/native/pipewire-capture/src/bitmap.rs b/electron/native/pipewire-capture/src/bitmap.rs new file mode 100644 index 0000000000..f7d6e43da8 --- /dev/null +++ b/electron/native/pipewire-capture/src/bitmap.rs @@ -0,0 +1,359 @@ +//! SPA cursor bitmap → PNG data URL. +//! +//! The renderer consumes `NativeCursorAsset.imageDataUrl` (see +//! src/native/contracts.ts) exactly as the macOS helper produces it: a base64 +//! `data:image/png` string, keyed by the SHA-256 of the PNG bytes so the same +//! shape is only ever serialised once. +//! +//! VERIFIED PREMULTIPLIED. Wayland cursor buffers carry premultiplied alpha and +//! PNG expects straight alpha, so the channels have to be divided back out. This +//! was a guess in Stage 1 and is now a measurement: a captured GNOME sprite had +//! 126 semi-transparent pixels and not one where any colour channel exceeded its +//! own alpha, which is only true of premultiplied data. Copying the channels +//! through unchanged darkened every antialiased edge. + +use base64::Engine; +use sha2::{Digest, Sha256}; + +use crate::shim::{Constants, CursorBitmap}; + +/// Refuse anything larger. Kept in step with the cursor metadata ceiling +/// negotiated in pw_shim.c (OSC_CURSOR_META_SIZE(1024, 1024)): a bitmap bigger +/// than the block we agreed to accept means the producer and this process +/// disagree about the buffer. For reference, mutter's own plane is 384×384 and +/// a real GNOME cursor is 24–96 px. +const MAX_DIMENSION: u32 = 1024; + +#[derive(Debug, PartialEq, Eq)] +pub struct EncodedCursor { + pub id: String, + pub image_data_url: String, + pub width: u32, + pub height: u32, +} + +/// Byte offsets of R, G, B, A within one source pixel, or `None` when the +/// format is not a 32-bit packed one we understand. +fn channel_order(constants: &Constants, format: u32) -> Option<[usize; 4]> { + // Written as a lookup against the values the C shim read out of the + // vendored headers, never against hardcoded enum numbers. + let table = [ + (constants.video_format_rgbx, [0, 1, 2, 3]), + (constants.video_format_bgrx, [2, 1, 0, 3]), + (constants.video_format_xrgb, [1, 2, 3, 0]), + (constants.video_format_xbgr, [3, 2, 1, 0]), + (constants.video_format_rgba, [0, 1, 2, 3]), + (constants.video_format_bgra, [2, 1, 0, 3]), + (constants.video_format_argb, [1, 2, 3, 0]), + (constants.video_format_abgr, [3, 2, 1, 0]), + ]; + table + .iter() + .find(|(id, _)| *id == format) + .map(|(_, order)| *order) +} + +/// Divides a premultiplied colour channel back out by its alpha. +/// +/// `+ alpha / 2` rounds to nearest instead of truncating, which matters at the +/// low alphas that make up an antialiased edge: at a=8 the truncating form loses +/// up to 6% of the recovered channel. The `min` is not defensive noise — a +/// producer that sends c > a (or a colour-keyed sprite) would otherwise overflow +/// past 255 and wrap when cast back to u8. +fn unpremultiply(channel: u8, alpha: u8) -> u8 { + if alpha == 0 { + // Fully transparent: the colour carries no information, and every value + // is equally "correct". Zero keeps identical sprites hashing alike. + return 0; + } + let alpha = u16::from(alpha); + (((u16::from(channel) * 255 + alpha / 2) / alpha).min(255)) as u8 +} + +/// True when the format has no alpha channel, so the 4th byte is padding and +/// must be replaced by opaque rather than copied. +fn is_opaque(constants: &Constants, format: u32) -> bool { + format == constants.video_format_rgbx + || format == constants.video_format_bgrx + || format == constants.video_format_xrgb + || format == constants.video_format_xbgr +} + +pub fn encode(constants: &Constants, bitmap: &CursorBitmap) -> Result { + if bitmap.width <= 0 || bitmap.height <= 0 { + return Err(format!( + "cursor bitmap has a non-positive size ({}x{})", + bitmap.width, bitmap.height + )); + } + let width = bitmap.width as u32; + let height = bitmap.height as u32; + if width > MAX_DIMENSION || height > MAX_DIMENSION { + return Err(format!("cursor bitmap is implausibly large ({width}x{height})")); + } + + let order = channel_order(constants, bitmap.format) + .ok_or_else(|| format!("unsupported cursor bitmap format id {}", bitmap.format))?; + let opaque = is_opaque(constants, bitmap.format); + + let stride = usize::try_from(bitmap.stride).map_err(|_| "negative stride".to_owned())?; + let row_bytes = width as usize * 4; + if stride < row_bytes { + return Err(format!("stride {stride} is shorter than one {width}px row")); + } + // The C side already bounds-checked this against the metadata block; repeat + // it here so the slicing below cannot panic even if that ever regresses. + let needed = stride + .checked_mul(height as usize) + .ok_or_else(|| "cursor bitmap size overflows".to_owned())?; + if bitmap.pixels.len() < needed { + return Err(format!( + "cursor bitmap is truncated: {} bytes for {}x{} at stride {}", + bitmap.pixels.len(), + width, + height, + stride + )); + } + + let mut rgba = Vec::with_capacity(row_bytes * height as usize); + for row in 0..height as usize { + let line = &bitmap.pixels[row * stride..row * stride + row_bytes]; + for pixel in line.chunks_exact(4) { + // An opaque format's 4th byte is padding, so there is nothing to + // divide by and the channels are already straight. + if opaque { + rgba.push(pixel[order[0]]); + rgba.push(pixel[order[1]]); + rgba.push(pixel[order[2]]); + rgba.push(0xff); + continue; + } + let alpha = pixel[order[3]]; + rgba.push(unpremultiply(pixel[order[0]], alpha)); + rgba.push(unpremultiply(pixel[order[1]], alpha)); + rgba.push(unpremultiply(pixel[order[2]], alpha)); + rgba.push(alpha); + } + } + + let png = encode_png(width, height, &rgba)?; + let id = Sha256::digest(&png) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let image_data_url = format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&png) + ); + + Ok(EncodedCursor { + id, + image_data_url, + width, + height, + }) +} + +fn encode_png(width: u32, height: u32, rgba: &[u8]) -> Result, String> { + let mut png = Vec::new(); + { + let mut encoder = png::Encoder::new(&mut png, width, height); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + let mut writer = encoder + .write_header() + .map_err(|error| format!("png header: {error}"))?; + writer + .write_image_data(rgba) + .map_err(|error| format!("png data: {error}"))?; + } + Ok(png) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn constants() -> Constants { + // Read from the compiled C shim, so the test also proves the Rust view + // of the SPA enum matches the vendored headers. + crate::shim::constants() + } + + fn bitmap(format: u32, width: i32, height: i32, stride: i32, pixels: Vec) -> CursorBitmap { + CursorBitmap { + format, + width, + height, + stride, + pixels, + } + } + + fn decode(data_url: &str) -> (png::OutputInfo, Vec) { + let base64 = data_url + .strip_prefix("data:image/png;base64,") + .expect("data url prefix"); + let bytes = base64::engine::general_purpose::STANDARD + .decode(base64) + .expect("base64"); + let decoder = png::Decoder::new(std::io::Cursor::new(bytes)); + let mut reader = decoder.read_info().expect("png info"); + let mut buffer = vec![0; reader.output_buffer_size()]; + let info = reader.next_frame(&mut buffer).expect("png frame"); + buffer.truncate(info.buffer_size()); + (info, buffer) + } + + #[test] + fn spa_format_ids_are_distinct() { + // Guards against the constants struct being mis-ordered between C and Rust: + // a field-order mismatch would collapse several of these onto one value. + let c = constants(); + let ids = [ + c.video_format_rgbx, + c.video_format_bgrx, + c.video_format_xrgb, + c.video_format_xbgr, + c.video_format_rgba, + c.video_format_bgra, + c.video_format_argb, + c.video_format_abgr, + ]; + let mut sorted = ids.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(sorted.len(), ids.len(), "duplicate SPA video format ids: {ids:?}"); + assert_ne!(c.data_mem_ptr, c.data_dma_buf); + } + + #[test] + fn bgra_channels_are_reordered_to_rgba() { + let c = constants(); + // Opaque on purpose: un-premultiplying by a=255 is the identity, so this + // test stays about channel ORDER and nothing else. + let encoded = encode(&c, &bitmap(c.video_format_bgra, 1, 1, 4, vec![1, 2, 3, 0xff])) + .expect("encode"); + let (info, pixels) = decode(&encoded.image_data_url); + assert_eq!((info.width, info.height), (1, 1)); + assert_eq!(&pixels[..4], &[3, 2, 1, 0xff]); + } + + #[test] + fn argb_channels_are_reordered_to_rgba() { + let c = constants(); + // One pixel: A=255, R=1, G=2, B=3. + let encoded = encode(&c, &bitmap(c.video_format_argb, 1, 1, 4, vec![0xff, 1, 2, 3])) + .expect("encode"); + let (_, pixels) = decode(&encoded.image_data_url); + assert_eq!(&pixels[..4], &[1, 2, 3, 0xff]); + } + + #[test] + fn premultiplied_channels_are_divided_back_out() { + let c = constants(); + // A half-transparent white pixel arrives premultiplied as 128/128/128 + // at a=128. Straight alpha puts the channels back at ~255. + let encoded = encode(&c, &bitmap(c.video_format_bgra, 1, 1, 4, vec![128, 128, 128, 128])) + .expect("encode"); + let (_, pixels) = decode(&encoded.image_data_url); + assert_eq!(&pixels[..4], &[255, 255, 255, 128]); + } + + #[test] + fn fully_transparent_pixels_keep_no_colour() { + let c = constants(); + // a=0 leaves the colour undefined; zeroing it is what makes two visually + // identical sprites hash to the same asset id. + let encoded = encode(&c, &bitmap(c.video_format_bgra, 1, 1, 4, vec![9, 9, 9, 0])) + .expect("encode"); + let (_, pixels) = decode(&encoded.image_data_url); + assert_eq!(&pixels[..4], &[0, 0, 0, 0]); + } + + #[test] + fn unpremultiply_cannot_exceed_opaque() { + // A producer that sends c > a would wrap past 255 without the clamp. + assert_eq!(unpremultiply(200, 100), 255); + assert_eq!(unpremultiply(0, 0), 0); + assert_eq!(unpremultiply(37, 255), 37); + } + + #[test] + fn opaque_formats_are_not_divided() { + let c = constants(); + // BGRx: the 4th byte is padding, not alpha. Dividing by it — or by the + // 0xff we substitute — must leave the colours untouched. + let encoded = encode(&c, &bitmap(c.video_format_bgrx, 1, 1, 4, vec![10, 20, 30, 0])) + .expect("encode"); + let (_, pixels) = decode(&encoded.image_data_url); + assert_eq!(&pixels[..4], &[30, 20, 10, 0xff]); + } + + #[test] + fn padding_byte_becomes_opaque_for_formats_without_alpha() { + let c = constants(); + let encoded = encode(&c, &bitmap(c.video_format_bgrx, 1, 1, 4, vec![1, 2, 3, 0])) + .expect("encode"); + let (_, pixels) = decode(&encoded.image_data_url); + assert_eq!(&pixels[..4], &[3, 2, 1, 0xff]); + } + + #[test] + fn row_padding_is_skipped() { + let c = constants(); + // 1px wide, 2 rows, stride 8: bytes 4..8 and 12..16 are padding. Both + // pixels are opaque so the assertion is about stride, not alpha. + let pixels = vec![ + 1, 2, 3, 0xff, 0xaa, 0xaa, 0xaa, 0xaa, 5, 6, 7, 0xff, 0xbb, 0xbb, 0xbb, 0xbb, + ]; + let encoded = + encode(&c, &bitmap(c.video_format_bgra, 1, 2, 8, pixels)).expect("encode"); + let (_, decoded) = decode(&encoded.image_data_url); + assert_eq!(&decoded[..8], &[3, 2, 1, 0xff, 7, 6, 5, 0xff]); + } + + #[test] + fn identical_bitmaps_hash_to_the_same_asset_id() { + let c = constants(); + let first = encode(&c, &bitmap(c.video_format_bgra, 1, 1, 4, vec![1, 2, 3, 4])).unwrap(); + let second = encode(&c, &bitmap(c.video_format_bgra, 1, 1, 4, vec![1, 2, 3, 4])).unwrap(); + let other = encode(&c, &bitmap(c.video_format_bgra, 1, 1, 4, vec![9, 9, 9, 9])).unwrap(); + assert_eq!(first.id, second.id); + assert_ne!(first.id, other.id); + assert_eq!(first.id.len(), 64, "sha-256 hex, like the macOS helper"); + } + + #[test] + fn truncated_buffers_are_rejected_rather_than_sliced() { + let c = constants(); + let error = encode(&c, &bitmap(c.video_format_bgra, 4, 4, 16, vec![0; 16])) + .expect_err("must reject"); + assert!(error.contains("truncated"), "{error}"); + } + + #[test] + fn short_stride_is_rejected() { + let c = constants(); + let error = encode(&c, &bitmap(c.video_format_bgra, 4, 1, 8, vec![0; 64])) + .expect_err("must reject"); + assert!(error.contains("shorter"), "{error}"); + } + + #[test] + fn implausible_dimensions_are_rejected() { + let c = constants(); + let error = encode(&c, &bitmap(c.video_format_bgra, 4096, 1, 16384, vec![0; 16384])) + .expect_err("must reject"); + assert!(error.contains("implausibly large"), "{error}"); + } + + #[test] + fn unknown_formats_are_reported_not_guessed() { + let c = constants(); + let error = + encode(&c, &bitmap(u32::MAX, 1, 1, 4, vec![0; 4])).expect_err("must reject"); + assert!(error.contains("unsupported"), "{error}"); + } +} diff --git a/electron/native/pipewire-capture/src/capture.rs b/electron/native/pipewire-capture/src/capture.rs new file mode 100644 index 0000000000..aaed892783 --- /dev/null +++ b/electron/native/pipewire-capture/src/capture.rs @@ -0,0 +1,727 @@ +//! The video half of Stage 2: PipeWire frames in, MP4 on disk out. +//! +//! CONSTANT FRAME RATE, DRIVEN BY A CLOCK, NOT BY ARRIVALS. +//! +//! A compositor delivers frames on damage. Nothing moves on screen, no frames +//! arrive — mutter will happily go seconds without one while the user reads a +//! page. Writing one output frame per delivered frame would therefore produce a +//! file whose playback speed depends on how busy the screen was, which is not a +//! recording of anything. +//! +//! So the output rate comes from a monotonic clock instead. [`Capture::advance`] +//! asks what frame index the wall clock is on and encodes forward to it, holding +//! the last staged picture across the gap. That is why [`crate::encoder`] splits +//! conversion from encoding: a held frame costs an upload and an encode (1.4 ms +//! here) but not the colour conversion (3.6 ms), which is the expensive part. +//! +//! The clock is ours, not the compositor's. `SPA_META_Header.pts` is more precise +//! per frame, but pause/resume and — once Stage 2's audio lands — the audio +//! epoch all live on this process's monotonic clock, and quantising to 1/60 s +//! makes the difference between the two immaterial. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::encoder::{ + AudioEncoder, Backend, EncodeStats, Muxer, TrackId, VideoEncoder, VideoParams, + AUDIO_CHANNELS, AUDIO_SAMPLE_RATE, +}; +use crate::ffmpeg as ff; +use crate::shim::{self, AudioRing}; + +/// An audio capture to mux alongside the video. +pub struct AudioSource { + /// "system" or "microphone" — the label a warning names. + pub label: &'static str, + pub ring: Arc, + /// Linear multiplier applied before encoding. 1.0 for the system mix; the + /// microphone carries the UI's boost. + pub gain: f32, + pub bitrate: i64, +} + +/// One capture feeding the mix. +struct AudioInput { + label: &'static str, + ring: Arc, + gain: f32, + /// Samples drained but not yet mixed, because the other inputs had fewer. + pending: Vec, +} + +/// The single AAC track, and everything mixed into it. +/// +/// ONE TRACK, NOT ONE PER SOURCE. The first shape of this code followed macOS +/// and wrote system audio and the microphone as two separate AAC tracks, on the +/// grounds that the export mixes every track it finds +/// (`crates/compositor/src/audio.rs`). What that missed is that THE PREVIEW DOES +/// NOT: it plays an HTML5 `