Ship YouTube's challenge solver without costing anyone a reinstall

app/__init__.py

@@ -0,0 +1,31 @@
+"""StemDeck.
+
+The vendored path insert below runs before anything else in the package, which
+is the only place it can work: yt-dlp resolves its optional dependencies at
+import time, and `app.pipeline.download` imports yt_dlp at module level.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+# yt-dlp's YouTube challenge solver (#432, #438).
+#
+# Vendored rather than declared as a dependency, and the reason is the desktop
+# updater, not preference. runtimeId is sha256(uv.lock), and the updater stands
+# down when it changes because it can replace `backend/` but never `python/`.
+# A new dependency would therefore have sent every existing desktop install to
+# a full manual reinstall to get this.
+#
+# This package is not one StemDeck imports. It is a 53 KB pure-Python payload
+# with no dependencies that yt-dlp discovers at runtime, so it belongs in the
+# app layer that the updater does replace. Existing installs get the solver
+# through the ordinary in-app update instead.
+#
+# Refresh with: python scripts/update_vendored_ejs.py
+_VENDOR = Path(__file__).resolve().parent / "_vendor"
+if _VENDOR.is_dir():
+    _path = str(_VENDOR)
+    if _path not in sys.path:
+        sys.path.insert(0, _path)

app/_vendor/LICENSE.yt-dlp-ejs

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org/>

app/_vendor/yt_dlp_ejs/__init__.py

@@ -0,0 +1,3 @@
+from yt_dlp_ejs._version import version
+
+__all__ = ["version"]

app/_vendor/yt_dlp_ejs/_version.py

@@ -0,0 +1,33 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Union
+
+    VERSION_TUPLE = tuple[int | str, ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = "0.8.0"
+__version_tuple__ = version_tuple = (0, 8, 0)
+
+__commit_id__ = commit_id = None

app/_vendor/yt_dlp_ejs/yt/solver/__init__.py

@@ -0,0 +1,21 @@
+import importlib.resources
+
+import yt_dlp_ejs.yt.solver
+
+
+def core() -> str:
+    """
+    Read the contents of the JavaScript core solver bundle as string.
+    """
+    return (importlib.resources.files(yt_dlp_ejs.yt.solver) / "core.min.js").read_text(
+        encoding="utf-8"
+    )
+
+
+def lib() -> str:
+    """
+    Read the contents of the JavaScript library solver bundle as string.
+    """
+    return (importlib.resources.files(yt_dlp_ejs.yt.solver) / "lib.min.js").read_text(
+        encoding="utf-8"
+    )

app/core/config.py

@@ -140,23 +140,47 @@ def _stored_jobs_dir() -> Path | None:
 _JS_RUNTIME_BINARIES = (("deno", "deno"), ("node", "node"), ("quickjs", "qjs"))
 
 
+def _js_runtime_dirs() -> tuple[Path, ...]:
+    """Where a bundled JS runtime can live, in priority order.
+
+    Two layouts, because the packages are not built the same way. Windows and
+    Linux stage a whole tree and put the binary in `data/jsruntime`, which is
+    JS_RUNTIME_DIR's default. macOS downloads a runtime pack and keeps its own
+    data directory in ~/Library/Application Support, so the binary rides in the
+    pack next to `backend/` instead.
+
+    Checking both here rather than setting an env var from the desktop shell
+    keeps this a Python-side fact that can be tested, instead of a contract
+    split across two languages that only breaks on one platform.
+    """
+    dirs = [JS_RUNTIME_DIR]
+    # app/core/config.py -> app/core -> app -> backend/ (or the repo root).
+    backend_root = Path(__file__).resolve().parents[2]
+    dirs.append(backend_root / "jsruntime")
+    dirs.append(backend_root.parent / "jsruntime")
+    seen: set[Path] = set()
+    return tuple(d for d in dirs if not (d in seen or seen.add(d)))  # type: ignore[func-returns-value]
+
+
 def bundled_js_runtime() -> tuple[str, Path] | None:
     """The JS runtime shipped with this install, as (yt-dlp name, path).
 
-    None when nothing is bundled, which is the normal case outside a portable
-    build -- yt-dlp then falls back to its own PATH lookup. Never raises: a
-    missing or unreadable directory just means "not bundled".
+    None when nothing is bundled, which is the normal case outside a packaged
+    build -- yt-dlp then falls back to its own PATH lookup, which is how Docker
+    finds the deno it ships. Never raises: a missing or unreadable directory
+    just means "not bundled".
     """
-    try:
-        if not JS_RUNTIME_DIR.is_dir():
-            return None
-        suffix = ".exe" if sys.platform.startswith("win") else ""
-        for name, stem in _JS_RUNTIME_BINARIES:
-            exe = JS_RUNTIME_DIR / f"{stem}{suffix}"
-            if exe.is_file():
-                return name, exe
-    except OSError:
-        return None
+    suffix = ".exe" if sys.platform.startswith("win") else ""
+    for directory in _js_runtime_dirs():
+        try:
+            if not directory.is_dir():
+                continue
+            for name, stem in _JS_RUNTIME_BINARIES:
+                exe = directory / f"{stem}{suffix}"
+                if exe.is_file():
+                    return name, exe
+        except OSError:
+            continue
     return None
 
 

pyproject.toml

@@ -115,7 +115,10 @@ packages = ["app"]
 target-version = "py310"
 line-length = 100
 # app/_version.py is generated by hatch-vcs at build/sync time — don't lint it.
-exclude = ["jobs", ".run", "static/vendor", "app/_version.py"]
+# app/_vendor is a third-party payload copied verbatim from PyPI. Linting or
+# reformatting it would create a diff against upstream that the refresh script
+# would then fight every time it runs.
+exclude = ["jobs", ".run", "static/vendor", "app/_version.py", "app/_vendor"]
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "I", "UP", "B", "SIM"]

scripts/linux/make-portable.sh

@@ -80,6 +80,36 @@ echo "==> Cleaning stage"
 rm -rf "$STAGE" "$ARCHIVE_PATH" "$CHECKSUM_PATH"
 mkdir -p "$STAGE" "$BACKEND_DIR" "$PYTHON_DIR"
 
+# QuickJS, for YouTube's signature/n-challenge solver (#438). yt-dlp ships the
+# solver script (the yt-dlp-ejs dependency) but needs a JavaScript engine to
+# run it, and a portable install has nothing on PATH.
+#
+# quickjs-ng rather than deno: 2.6 MB against deno's ~110 MB, times two
+# bundles, against a 2 GiB release asset cap this project has already hit once
+# (#222, #225).
+#
+# It lives in backend/, not data/. The in-app updater replaces the executable
+# and backend/ and leaves data/ alone, so a binary in data/ would only ever
+# reach a fresh install. Here it arrives through an ordinary update, which is
+# the whole point: shipping this must not cost existing users a reinstall.
+#
+# Pinned by version and SHA256, the same rule as the macOS FFmpeg download
+# (#172). An unverified binary fetched at package time is a supply-chain hole
+# whether or not it is small.
+QJS_VERSION="v0.16.2"
+QJS_SHA256="c5e1b16adfa36def7ac523d6ba54edc77ef66a4dfd65d73e6eae19025f9b7b0a"
+QJS_URL="https://github.com/quickjs-ng/quickjs/releases/download/${QJS_VERSION}/qjs-linux-x86_64"
+QJS_DIR="${BACKEND_DIR}/jsruntime"
+mkdir -p "$QJS_DIR"
+echo "==> Fetching QuickJS ${QJS_VERSION}"
+curl -fsSL --retry 3 -o "${QJS_DIR}/qjs" "$QJS_URL"
+echo "${QJS_SHA256}  ${QJS_DIR}/qjs" | sha256sum -c - >/dev/null || {
+    rm -f "${QJS_DIR}/qjs"
+    echo "QuickJS checksum mismatch" >&2
+    exit 1
+}
+chmod +x "${QJS_DIR}/qjs"
+
 # Copy the entire PBS install into python/ (-a preserves symlinks/permissions).
 echo "==> Bundling Python runtime from ${PBS_BASE_PREFIX}"
 cp -a "$PBS_BASE_PREFIX/." "$PYTHON_DIR/"

scripts/macos/make-runtime-pack.sh

@@ -166,6 +166,35 @@ cat > "$BACKEND_DIR/static/version.json" <<JSON
 }
 JSON
 
+# QuickJS, for YouTube's signature/n-challenge solver (#438). yt-dlp ships the
+# solver script (the yt-dlp-ejs dependency) but needs a JavaScript engine to
+# run it, and a packaged install has nothing on PATH.
+#
+# It rides inside backend/, the same place Windows and Linux put it, so all
+# three packages agree and config.bundled_js_runtime() has one layout to find.
+# Not the app's data directory: on macOS that lives in ~/Library/Application
+# Support and is user-owned, so a binary there would have to be installed at
+# first run rather than shipped.
+#
+# Pinned by version and SHA256, the same rule as the macOS FFmpeg download
+# (#172).
+QJS_VERSION="v0.16.2"
+case "$ARCH" in
+  arm64)  QJS_ASSET="qjs-darwin-arm64";  QJS_SHA256="f6200e9856c45578a5d42ac873a32f3f994b421e29df9f63b452d9c7145015fc" ;;
+  x86_64) QJS_ASSET="qjs-darwin-x86_64"; QJS_SHA256="4448991c0500dbe40c7b2f91ba39275995413aa4ee59db3b513b68350908a413" ;;
+  *) echo "unknown arch for QuickJS: $ARCH" >&2; exit 1 ;;
+esac
+QJS_DIR="${BACKEND_DIR}/jsruntime"
+mkdir -p "$QJS_DIR"
+echo "==> Fetching QuickJS ${QJS_VERSION} (${QJS_ASSET})"
+curl -fsSL --retry 3 -o "${QJS_DIR}/qjs"   "https://github.com/quickjs-ng/quickjs/releases/download/${QJS_VERSION}/${QJS_ASSET}"
+echo "${QJS_SHA256}  ${QJS_DIR}/qjs" | shasum -a 256 -c - >/dev/null || {
+    rm -f "${QJS_DIR}/qjs"
+    echo "QuickJS checksum mismatch" >&2
+    exit 1
+}
+chmod +x "${QJS_DIR}/qjs"
+
 echo "==> Capturing dependency inventory"
 mkdir -p "$RUNTIME_DIR/licenses"
 uv pip list --system --python "$PYTHON_DIR/bin/python" --format=json > "$RUNTIME_DIR/licenses/pip-list.json"