From 332758c614da7714c7a4d8897801c6ca4da286bf Mon Sep 17 00:00:00 2001 From: Thales <> Date: Tue, 25 Aug 2026 14:03:06 +0100 Subject: [PATCH 01/11] A byte we cannot decode is not a reason to fail a separation Reported against a real import: 'charmap' codec can't decode byte 0x8f in position 20 text=True on its own decodes a child's output with the locale encoding. On Windows that is cp1252, so one byte outside it in Demucs' progress output killed the whole job. The output in question is a progress bar and some echoed metadata. Diagnostic text, never worth failing a separation over. Both halves have to agree or the mismatch just moves, so the parents now read utf-8 with errors=replace and the children are told to write utf-8. Three call sites: the Demucs worker, the vocal-split worker, and ffprobe on upload, where the same trap would have failed an upload rather than a separation. Pre-existing and unrelated to any feature work. It surfaced now only because error_detail started carrying the message (#434) instead of the bare word 'unknown'. --- app/api/jobs.py | 5 +++ app/pipeline/separate.py | 13 ++++++ app/pipeline/vocal_split.py | 13 ++++++ tests/test_subprocess_encoding.py | 68 +++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 tests/test_subprocess_encoding.py diff --git a/app/api/jobs.py b/app/api/jobs.py index 247c68f0..bc68918f 100644 --- a/app/api/jobs.py +++ b/app/api/jobs.py @@ -76,7 +76,12 @@ def _probe_duration(path: Path) -> float: str(path), ], capture_output=True, + # See the note in pipeline/separate.py: text=True alone decodes with the + # Windows locale encoding and a stray byte in ffprobe's output would + # fail the upload outright. text=True, + encoding="utf-8", + errors="replace", timeout=30, ) if result.returncode != 0: diff --git a/app/pipeline/separate.py b/app/pipeline/separate.py index b086643b..609ff3d0 100644 --- a/app/pipeline/separate.py +++ b/app/pipeline/separate.py @@ -64,6 +64,12 @@ def _get_worker(device: str) -> subprocess.Popen: # exits when we are gone, so it cannot be left holding a GPU after a kill # that ran no cleanup (SIGKILL, Force Quit, Task Manager, a crash). env["STEMDECK_PARENT_PID"] = str(os.getpid()) + # Pin the child's stdio encoding to match what the parent now decodes with. + # Without it a Windows child writes cp1252 while the parent reads utf-8, so + # the mismatch simply moves rather than being fixed. Demucs and audio- + # separator both emit progress bars and can echo track metadata, neither of + # which is guaranteed to be cp1252-safe. + env["PYTHONIOENCODING"] = "utf-8:replace" try: import certifi @@ -77,7 +83,14 @@ def _get_worker(device: str) -> subprocess.Popen: stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + # utf-8/replace explicitly. text=True alone decodes with the locale + # encoding, which on Windows is cp1252, and a single byte outside it + # in a child's output kills the whole job with a UnicodeDecodeError + # ("'charmap' codec can't decode byte 0x8f"). This is diagnostic text: + # a byte we cannot read is never a reason to fail a separation. text=True, + encoding="utf-8", + errors="replace", bufsize=1, env=env, ) diff --git a/app/pipeline/vocal_split.py b/app/pipeline/vocal_split.py index e007a274..65cb224e 100644 --- a/app/pipeline/vocal_split.py +++ b/app/pipeline/vocal_split.py @@ -56,6 +56,12 @@ def split_vocals(job: Job, stems_dir: Path) -> list[str]: device = get_demucs_device() env = os.environ.copy() + # Pin the child's stdio encoding to match what the parent now decodes with. + # Without it a Windows child writes cp1252 while the parent reads utf-8, so + # the mismatch simply moves rather than being fixed. Demucs and audio- + # separator both emit progress bars and can echo track metadata, neither of + # which is guaranteed to be cp1252-safe. + env["PYTHONIOENCODING"] = "utf-8:replace" try: import certifi @@ -68,7 +74,14 @@ def split_vocals(job: Job, stems_dir: Path) -> list[str]: _spawn_cmd(device, vocals_path, stems_dir), stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + # utf-8/replace explicitly. text=True alone decodes with the locale + # encoding, which on Windows is cp1252, and a single byte outside it + # in a child's output kills the whole job with a UnicodeDecodeError + # ("'charmap' codec can't decode byte 0x8f"). This is diagnostic text: + # a byte we cannot read is never a reason to fail a separation. text=True, + encoding="utf-8", + errors="replace", bufsize=1, env=env, ) diff --git a/tests/test_subprocess_encoding.py b/tests/test_subprocess_encoding.py new file mode 100644 index 00000000..f0806e4d --- /dev/null +++ b/tests/test_subprocess_encoding.py @@ -0,0 +1,68 @@ +"""Child process output must never be able to fail a job by being unreadable. + +`text=True` on its own decodes with the locale encoding. On Windows that is +cp1252, so a single byte outside it in Demucs' or ffprobe's output raised + + 'charmap' codec can't decode byte 0x8f in position 20 + +and killed the whole separation. The output in question is a progress bar and +some echoed metadata: diagnostic text that is never worth failing a job over. + +Both halves have to agree. The parent reads utf-8 with errors="replace", and +the child is told to write utf-8, or the mismatch just moves. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +SITES = [ + ("app/pipeline/separate.py", "the Demucs worker"), + ("app/pipeline/vocal_split.py", "the vocal-split worker"), + ("app/api/jobs.py", "ffprobe on upload"), +] + + +@pytest.mark.parametrize(("path", "what"), SITES) +def test_decoding_is_pinned_not_left_to_the_locale(path: str, what: str): + src = Path(path).read_text(encoding="utf-8") + assert "text=True" in src, f"{path} no longer spawns a text-mode child; update this test" + assert 'encoding="utf-8"' in src, f"{what} decodes with the locale encoding, not utf-8" + assert 'errors="replace"' in src, f"{what} would still raise on an undecodable byte" + + +@pytest.mark.parametrize("path", ["app/pipeline/separate.py", "app/pipeline/vocal_split.py"]) +def test_children_are_told_to_write_utf8(path: str): + """The parent reading utf-8 is only half of it.""" + src = Path(path).read_text(encoding="utf-8") + assert "PYTHONIOENCODING" in src, f"{path} lets the child pick its own stdio encoding" + + +def test_a_byte_outside_cp1252_survives_a_round_trip(): + """The actual failure, reproduced. 0x8f is undefined in cp1252, so the old + configuration raised here instead of returning a string.""" + payload = b"progress: \x8f\x9d\x81 50%\n" + + with pytest.raises(UnicodeDecodeError): + payload.decode("cp1252") + + # What the code does now. + assert payload.decode("utf-8", errors="replace") + + +def test_ffprobe_duration_survives_undecodable_output(): + """A stray byte on stderr must not stop a valid duration being read.""" + from app.api import jobs as jobs_mod + + def fake_run(cmd, **kwargs): + assert kwargs.get("encoding") == "utf-8" + assert kwargs.get("errors") == "replace" + # Exactly what a text-mode pipe hands back once errors="replace" is on. + return subprocess.CompletedProcess(cmd, 0, "212.5\n", "warn ��\n") + + with patch.object(jobs_mod.subprocess, "run", fake_run): + assert jobs_mod._probe_duration(Path("whatever.mp3")) == 212.5 From c1001a2dc8802aaed766c2b522209d97c87795b2 Mon Sep 17 00:00:00 2001 From: Thales <> Date: Tue, 25 Aug 2026 14:03:24 +0100 Subject: [PATCH 02/11] Search YouTube and SoundCloud from the topbar The box already took a pasted link. Typing anything that is not a link now searches instead, so finding a track no longer means leaving StemDeck, finding it in a browser, and coming back with a URL. Three tabs: YouTube songs, YouTube playlists, SoundCloud songs. SoundCloud playlists is deliberately absent, because yt-dlp exposes exactly one SoundCloud search key (scsearch, tracks only) and a tab that can only ever be empty is worse than no tab. Cost, measured rather than assumed. A search is one flat extraction, about 1.1 s for YouTube and 2.0 s for SoundCloud, and it returns titles, durations, uploaders and thumbnails for the whole page at once. Requests fire on a word boundary rather than a keystroke, so typing a full phrase costs one request, not one per character. An AbortController cancels the superseded request so results cannot land out of order. A 60 s server cache absorbs the repeats that backspacing produces. A semaphore caps concurrent yt-dlp searches, because an aborted fetch does not stop a thread that has already started. The duration limit is part of the cache key, not just the payload. It decides each result's too_long verdict and it is a live setting: raising it has to un-grey the rows now, not once a 60 s entry expires. Over that limit the pipeline refuses the job outright, so those rows are not selectable and say so. This reads the user's configured value, which is anywhere from 1 to 60 minutes, not a hardcoded 20. The SSRF boundary from #173 is unchanged. Each search gets the narrowest extractor allowlist that can serve it, generic stays out of all of them, and every result goes back through validate_youtube_url or validate_playlist_url before it can reach the pipeline. Anything that fails is dropped rather than shown. SoundCloud needs webpage_url rather than url for this: its search returns an api.soundcloud.com endpoint that is not on the allowlisted host set, so reading url first (as expand_playlist does) drops every result. Picking a result fills the box and stops there. Extraction is minutes of work, so it stays behind a deliberate press of Split stems rather than starting on a click in a list the user may still be reading. The panel lives on body rather than in the composer, which sets overflow:hidden for its rounded pill and clipped the dropdown out of existence. The topbar input is type=text now, since type=url marks a search query invalid and blocks submission; its styling keys off #url rather than the type so that cannot silently unstyle it again. --- app/api/router.py | 2 + app/api/search.py | 234 ++++++++++++++++++ app/pipeline/search.py | 175 +++++++++++++ static/css/daw.css | 114 ++++++++- static/index.html | 9 +- static/js/i18n.js | 112 ++++++++- static/js/main.js | 16 ++ static/js/search.js | 542 +++++++++++++++++++++++++++++++++++++++++ tests/test_search.py | 391 +++++++++++++++++++++++++++++ 9 files changed, 1582 insertions(+), 13 deletions(-) create mode 100644 app/api/search.py create mode 100644 app/pipeline/search.py create mode 100644 static/js/search.js create mode 100644 tests/test_search.py diff --git a/app/api/router.py b/app/api/router.py index f0d46929..3232a5a6 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -8,6 +8,7 @@ from app.api.playlist import router as playlist_router from app.api.qr import router as qr_router from app.api.queue import router as queue_router +from app.api.search import router as search_router from app.api.stems import router as stems_router router = APIRouter() @@ -18,3 +19,4 @@ router.include_router(qr_router, tags=["qr"]) router.include_router(queue_router, prefix="/queue", tags=["queue"]) router.include_router(playlist_router, prefix="/playlist", tags=["playlist"]) +router.include_router(search_router, prefix="/search", tags=["search"]) diff --git a/app/api/search.py b/app/api/search.py new file mode 100644 index 00000000..0ea64890 --- /dev/null +++ b/app/api/search.py @@ -0,0 +1,234 @@ +"""Search endpoint for the topbar. + +Fired while the user types, so the two things that matter are not spending a +round trip when an identical one just happened, and not letting a burst of +typing spawn unbounded work. + + * A short TTL cache absorbs the repeats. Word-boundary triggering means + "daft", "daft punk", "daft punk around" are three separate queries, and + backspacing walks straight back through them. A 60 s window turns the + revisits into cache hits. + * A semaphore caps how many yt-dlp searches can be in flight. The client + aborts superseded requests, but an abort does not stop a thread that has + already started, so the cap is what actually bounds the work. + +Neither is a substitute for the client debouncing. Both exist because the +client cannot be trusted to be the only caller. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +import urllib.error +import urllib.request + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field + +from app.core.settings import get_max_duration_sec +from app.pipeline.download import InvalidYouTubeURL +from app.pipeline.preview import PreviewUnavailable +from app.pipeline.preview import resolve as resolve_preview +from app.pipeline.search import ( + DEFAULT_LIMIT, + KINDS, + MAX_LIMIT, + MIN_QUERY_LEN, + SOURCES, + UnsupportedSearch, + search, + supported, +) + +logger = logging.getLogger("stemdeck.api") + +router = APIRouter(tags=["search"]) + +# yt-dlp searches are ~1-2 s of blocking network I/O each. Three at once keeps +# a fast typist responsive without turning the thread pool into a queue of +# results nobody is waiting for any more. +_MAX_CONCURRENT = 3 +_semaphore = asyncio.Semaphore(_MAX_CONCURRENT) + +_CACHE_TTL_SEC = 60.0 +_CACHE_MAX = 128 +# key -> (expires_at, payload). Plain dict rather than functools.lru_cache: the +# entries expire on time, not on eviction pressure alone, and search results go +# stale in a way that a pure LRU would happily serve forever. +_cache: dict[tuple, tuple[float, dict]] = {} + + +class SearchRequest(BaseModel): + query: str = Field(min_length=MIN_QUERY_LEN, max_length=200) + source: str = "youtube" + kind: str = "track" + limit: int = Field(default=DEFAULT_LIMIT, ge=1, le=MAX_LIMIT) + + +def _cache_get(key: tuple) -> dict | None: + hit = _cache.get(key) + if hit is None: + return None + expires_at, payload = hit + if expires_at < time.monotonic(): + _cache.pop(key, None) + return None + return payload + + +def _cache_put(key: tuple, payload: dict) -> None: + if len(_cache) >= _CACHE_MAX: + # Drop whatever expires soonest. Cheap at this size, and it sheds the + # entries closest to being useless rather than an arbitrary one. + oldest = min(_cache, key=lambda k: _cache[k][0]) + _cache.pop(oldest, None) + _cache[key] = (time.monotonic() + _CACHE_TTL_SEC, payload) + + +def _clear_cache() -> None: + """Test hook. Results are per-process and disposable, so nothing else needs + to reach in here.""" + _cache.clear() + + +@router.post("") +async def search_sources(request: Request) -> dict: + try: + body = await request.json() + except Exception as e: + raise HTTPException(status_code=422, detail=f"Invalid JSON: {e}") from e + try: + payload = SearchRequest(**body) + except Exception as e: + raise HTTPException(status_code=422, detail=str(e)) from e + + if payload.source not in SOURCES or payload.kind not in KINDS: + raise HTTPException(status_code=422, detail="unknown source or kind") + if not supported(payload.source, payload.kind): + raise HTTPException( + status_code=422, + detail=f"{payload.source} has no {payload.kind} search", + ) + + query = payload.query.strip() + # The duration limit is part of the key, not just part of the payload. It + # decides each result's too_long verdict, and it is a live setting: raising + # it in Settings has to un-grey the rows now, not once a 60 s entry expires. + key = ( + payload.source, + payload.kind, + query.casefold(), + payload.limit, + get_max_duration_sec(), + ) + cached = _cache_get(key) + if cached is not None: + return {**cached, "cached": True} + + try: + async with _semaphore: + # Re-check: while queued behind the semaphore, an identical search + # may have finished and filled the cache. Common when a burst of + # keystrokes produces the same query twice. + cached = _cache_get(key) + if cached is not None: + return {**cached, "cached": True} + result = await asyncio.to_thread( + search, query, payload.source, payload.kind, payload.limit + ) + except UnsupportedSearch as e: + raise HTTPException(status_code=422, detail=str(e)) from e + except Exception as e: + logger.exception("search failed") + raise HTTPException(status_code=502, detail="Could not reach that service") from e + + _cache_put(key, result) + return {**result, "cached": False} + + +@router.get("/sources") +def search_sources_available() -> dict: + """Which source/kind pairs actually return anything, so the UI does not + offer a SoundCloud playlist tab that can only ever be empty.""" + return { + "sources": [{"source": s, "kinds": [k for k in KINDS if supported(s, k)]} for s in SOURCES] + } + + +# Preview audio is proxied, never redirected: the CSP allows `media-src 'self'` +# only, and widening it so an