From 50ffc505abe818dc3a3ad15bf4d9a55fe4d35f52 Mon Sep 17 00:00:00 2001 From: emranemran Date: Fri, 1 May 2026 09:48:17 -0700 Subject: [PATCH 1/7] fix --- src/scope/cloud/livepeer_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scope/cloud/livepeer_app.py b/src/scope/cloud/livepeer_app.py index 7f310dc8f..13277de97 100644 --- a/src/scope/cloud/livepeer_app.py +++ b/src/scope/cloud/livepeer_app.py @@ -1546,6 +1546,7 @@ async def websocket_endpoint(ws: WebSocket) -> None: if control_task is not None: await _shutdown_task(control_task, task_name="control_channel") _connection_active = False + logger.info("XXX: WebSocket client disconnected") set_connection_id(None) From 72ffc3d37fa44f695270a1c6211097a847a85b7c Mon Sep 17 00:00:00 2001 From: emranemran Date: Fri, 1 May 2026 12:32:53 -0700 Subject: [PATCH 2/7] perf(cloud): shrink fal cold start by ~20s Three changes that target the cold-start path measured in fal.log: - livepeer_fal_app.py: drop `requirements = [...]` so fal isolate stops provisioning a separate venv on top of the image's. The image already has websockets/httpx/aiokafka via the kafka extra below. - Dockerfile.cloud: pre-build the venv with `uv sync --extra livepeer --extra kafka --no-dev` so the runtime `uv run --extra livepeer --extra kafka livepeer-runner` is a no-op instead of fetching + installing aiokafka, uvloop, and re-resolving ~50 packages. - livepeer_fal_app.py dockerfile_str: re-sync after `COPY src/` so the daydream-scope editable install is refreshed at image build time, eliminating the "Built daydream-scope @ file:///app" rebuild on every cold start. - livepeer_app.py lifespan: pre-warm the pipeline registry so the torch/diffusers/transformers/torchao import cascade runs at runner startup instead of on the first cloud-proxy call (~8s shifted off the user-perceived connect path). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: emranemran --- Dockerfile.cloud | 6 ++++++ src/scope/cloud/livepeer_app.py | 6 ++++++ src/scope/cloud/livepeer_fal_app.py | 16 +++++++++++----- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Dockerfile.cloud b/Dockerfile.cloud index b3f5664fa..d52b67ac4 100644 --- a/Dockerfile.cloud +++ b/Dockerfile.cloud @@ -45,6 +45,12 @@ RUN uv python install && \ # Copy project files COPY src/ /app/src/ +# Pre-build the venv with the runtime extras the cloud wrapper invokes +# (`uv run --extra livepeer --extra kafka livepeer-runner`). Without this +# the first cold start has to fetch + install aiokafka, uvicorn[standard], +# and re-resolve the entire dep graph at runtime — adds ~10s per cold start. +RUN uv sync --extra livepeer --extra kafka --no-dev + # Pre-install bundled plugins (cannot be removed by users) ENV DAYDREAM_SCOPE_BUNDLED_PLUGINS_FILE="/app/bundled-plugins.txt" RUN echo "git+https://github.com/daydreamlive/scope-ltx-2.git@d13b5f9d94130b975989cd820eedbef5b3a8f165" > /app/bundled-plugins.txt diff --git a/src/scope/cloud/livepeer_app.py b/src/scope/cloud/livepeer_app.py index 13277de97..f52a54e4e 100644 --- a/src/scope/cloud/livepeer_app.py +++ b/src/scope/cloud/livepeer_app.py @@ -70,6 +70,12 @@ async def lifespan(_app: FastAPI): """Initialize embedded Scope app lifespan and ASGI client.""" global scope_client async with scope_lifespan(scope_app): + # Pre-warm the pipeline registry so the import cascade + # (torch / diffusers / transformers / torchao / per-pipeline modules) + # runs at runner startup instead of on the first cloud-proxy request. + # Shaves the registry-init delay out of the user-perceived connect path. + import scope.core.pipelines.registry # noqa: F401 + scope_client = httpx.AsyncClient( transport=httpx.ASGITransport(app=scope_app), base_url="http://runner", diff --git a/src/scope/cloud/livepeer_fal_app.py b/src/scope/cloud/livepeer_fal_app.py index 1d234a0b6..6785604f4 100644 --- a/src/scope/cloud/livepeer_fal_app.py +++ b/src/scope/cloud/livepeer_fal_app.py @@ -262,11 +262,17 @@ def _get_git_sha() -> str: GIT_SHA = _get_git_sha() DOCKER_IMAGE = f"daydreamlive/scope:{GIT_SHA}" +# Re-sync after COPY so the daydream-scope editable install picks up the +# freshly-copied src/ at image build time. Without this, the first `uv run` +# at cold start sees a stale path source and rebuilds + reinstalls the +# project (the ~10s "Built daydream-scope @ file:///app" + Uninstalled/Installed +# block visible in cold-start logs). dockerfile_str = f""" FROM {DOCKER_IMAGE} WORKDIR /app COPY pyproject.toml uv.lock README.md patches.pth /app/ COPY src/ /app/src/ +RUN uv sync --extra livepeer --extra kafka --no-dev """ custom_image = ContainerImage.from_dockerfile_str( dockerfile_str, @@ -386,11 +392,11 @@ class LivepeerScopeApp(fal.App, keep_alive=300): image = custom_image machine_type = "GPU-H100" - requirements = [ - "websockets", - "httpx", - "aiokafka", - ] + # Empty `requirements` so fal isolate doesn't provision a SECOND venv + # alongside the image's pre-built one. websockets, httpx, and aiokafka + # are baked into the container image (httpx + websockets via base deps; + # aiokafka via the `kafka` extra installed in Dockerfile.cloud's uv sync). + requirements: list[str] = [] def setup(self): """Start the Livepeer runner as a background subprocess.""" From 1f94168e95916cc2ef2fb40e4c2ec77ee4781fcb Mon Sep 17 00:00:00 2001 From: emranemran Date: Fri, 1 May 2026 13:05:37 -0700 Subject: [PATCH 3/7] fix(cloud): restore fal isolate requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empty `requirements` broke the wrapper at first websocket — fal isolate runs the App's setup() / websocket handler in a venv separate from the image's /app/.venv (under /usr/local/lib/python3.12/dist-packages), where httpx wasn't available, so check_runner_readiness raised ModuleNotFoundError on every connect. The ~10s cold-start tax that motivated emptying requirements was actually `uv run --extra livepeer --extra kafka` resyncing the image venv, not isolate. The Dockerfile.cloud `uv sync --extra livepeer --extra kafka --no-dev` and the fal-side dockerfile_str re-sync from the previous commit already address that on their own. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: emranemran --- src/scope/cloud/livepeer_fal_app.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/scope/cloud/livepeer_fal_app.py b/src/scope/cloud/livepeer_fal_app.py index 6785604f4..24a1162f8 100644 --- a/src/scope/cloud/livepeer_fal_app.py +++ b/src/scope/cloud/livepeer_fal_app.py @@ -392,11 +392,18 @@ class LivepeerScopeApp(fal.App, keep_alive=300): image = custom_image machine_type = "GPU-H100" - # Empty `requirements` so fal isolate doesn't provision a SECOND venv - # alongside the image's pre-built one. websockets, httpx, and aiokafka - # are baked into the container image (httpx + websockets via base deps; - # aiokafka via the `kafka` extra installed in Dockerfile.cloud's uv sync). - requirements: list[str] = [] + # These are required in fal isolate's runtime context (separate from + # the image's /app/.venv), where the App's setup() and websocket + # handler actually run. They're cheap — the real cold-start savings + # come from the image-level `uv sync --extra livepeer --extra kafka` + # in Dockerfile.cloud and the fal-side dockerfile_str re-sync, which + # together skip the ~10s `uv run --extra` resync that previously fired + # when the wrapper invoked `livepeer-runner`. + requirements = [ + "websockets", + "httpx", + "aiokafka", + ] def setup(self): """Start the Livepeer runner as a background subprocess.""" From 6a7f8432bb615e1420f9ddf30375f1c78b9803a2 Mon Sep 17 00:00:00 2001 From: emranemran Date: Mon, 4 May 2026 13:16:21 -0700 Subject: [PATCH 4/7] perf(cloud): shrink image via runtime base + venv strip Two low-risk image-size reductions stacked together: 1. Switch base from `cuda:12.8.0-cudnn-runtime-ubuntu24.04` to plain `cuda:12.8.0-runtime-ubuntu24.04`. The pyproject.toml override `nvidia-cudnn-cu12>=9.15` already ships cuDNN via pip into the venv, so the base image's cuDNN was dead weight (~700 MB). 2. After `uv sync` and the bundled-plugin install, strip files that are never read at runtime: C/C++ headers/sources (only used to compile extensions), package tests/ and docs/ directories, .pdb debug symbols, and the leftover uv/apt caches. Keep .pyc / __pycache__ so first import isn't slowed by recompilation, and keep examples/ because some packages import from it. Estimated combined savings: ~700 MB (cuDNN duplication) + 200-500 MB (strip) = ~1 GB compressed off the 7.43 GB image. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: emranemran --- Dockerfile.cloud | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Dockerfile.cloud b/Dockerfile.cloud index d52b67ac4..ae415460c 100644 --- a/Dockerfile.cloud +++ b/Dockerfile.cloud @@ -1,4 +1,7 @@ -FROM nvidia/cuda:12.8.0-cudnn-runtime-ubuntu24.04 +# Use the plain `runtime` variant (no cuDNN). pyproject.toml overrides +# `nvidia-cudnn-cu12>=9.15` so cuDNN ships in the venv anyway, and the +# base copy was dead weight (~700 MB). +FROM nvidia/cuda:12.8.0-runtime-ubuntu24.04 ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 @@ -56,6 +59,20 @@ ENV DAYDREAM_SCOPE_BUNDLED_PLUGINS_FILE="/app/bundled-plugins.txt" RUN echo "git+https://github.com/daydreamlive/scope-ltx-2.git@d13b5f9d94130b975989cd820eedbef5b3a8f165" > /app/bundled-plugins.txt RUN uv run daydream-scope install git+https://github.com/daydreamlive/scope-ltx-2.git@d13b5f9d94130b975989cd820eedbef5b3a8f165 +# Strip never-at-runtime files from the venv to shrink the image. +# Keep .pyc/__pycache__ so first import isn't slowed by recompilation. +# Drop C/C++ headers/sources (only used at compile time), package +# tests/ and docs/ directories, .pdb debug symbols, and the uv/apt +# caches we don't need at runtime. Keep examples/ — some packages +# import from it. +RUN find /app/.venv -depth -type d \ + \( -name 'tests' -o -name 'test' -o -name 'doc' -o -name 'docs' \) \ + -exec rm -rf {} + 2>/dev/null || true && \ + find /app/.venv -type f \ + \( -name '*.h' -o -name '*.c' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.pdb' \) \ + -delete 2>/dev/null || true && \ + rm -rf /tmp/uv-cache /root/.cache /var/cache/apt/archives/*.deb + # Expose port 8000 for RunPod HTTP proxy EXPOSE 8000 From 4cfe5d79c2074042bc068b22ee9b5efeb1a8a316 Mon Sep 17 00:00:00 2001 From: emranemran Date: Tue, 5 May 2026 14:02:10 -0700 Subject: [PATCH 5/7] perf(cloud): multi-stage build + strip duplicated CUDA libs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two aggressive image-size reductions stacked: 1. Multi-stage build: build deps (curl/git/build-essential/ software-properties-common/python3-dev) live in the builder stage and never reach the final image. Final stage installs only the runtime libs (libgl1/libglib2.0-0/libsm6/libxext6/libxrender-dev/ libgomp1) and copies the venv + uv binary + uv-managed Python from the builder. 2. Strip duplicated CUDA libs from the venv. Torch on Linux brings in ~2 GB of nvidia-*-cu12 packages whose .so files are already in the base image at /usr/local/cuda/lib64. We `rm -rf` the lib dirs for cublas, cufft, curand, cusolver, cusparse, cuda_runtime, and cuda_nvrtc — keeping their dist-info so uv treats them as installed in the fal-side `uv sync`. LD_LIBRARY_PATH is set so torch finds them in /usr/local/cuda/lib64 at runtime. Kept in the venv: cuDNN (override-pinned newer than the base ships), cusparselt (not in base), nccl/cupti/nvtx/nvjitlink/nvshmem (small or version-sensitive). Estimated combined savings: ~1.2-1.5 GB compressed off the 6.75 GB post-base-swap image. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: emranemran --- Dockerfile.cloud | 102 ++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 36 deletions(-) diff --git a/Dockerfile.cloud b/Dockerfile.cloud index ae415460c..6f415dae9 100644 --- a/Dockerfile.cloud +++ b/Dockerfile.cloud @@ -1,80 +1,110 @@ -# Use the plain `runtime` variant (no cuDNN). pyproject.toml overrides -# `nvidia-cudnn-cu12>=9.15` so cuDNN ships in the venv anyway, and the -# base copy was dead weight (~700 MB). -FROM nvidia/cuda:12.8.0-runtime-ubuntu24.04 +# --- Builder stage --------------------------------------------------------- +# Build-only deps live here (build-essential, git, python3-dev, etc.) so the +# final stage can drop them entirely. +FROM nvidia/cuda:12.8.0-runtime-ubuntu24.04 AS builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 -ENV CUDA_VISIBLE_DEVICES=0 -ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility -ENV DAYDREAM_SCOPE_LOGS_DIR=/workspace/logs -ENV DAYDREAM_SCOPE_MODELS_DIR=/workspace/models WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ - # System dependencies curl \ git \ build-essential \ software-properties-common \ - # Dependencies required for OpenCV - libgl1 \ - libglib2.0-0 \ - libsm6 \ - libxext6 \ - libxrender-dev \ - libgomp1 \ python3-dev \ - # Cleanup && rm -rf /var/lib/apt/lists/* -# Install uv (Python package manager) to system-wide location ENV UV_INSTALL_DIR="/usr/local/bin" RUN curl -LsSf https://astral.sh/uv/0.9.11/install.sh | sh && \ - # Create symlink for fal.ai deploy which hardcodes $HOME/.local/bin/uv mkdir -p /root/.local/bin && \ ln -s /usr/local/bin/uv /root/.local/bin/uv -# Python/uv configuration - use default location but make world-readable ENV UV_CACHE_DIR="/tmp/uv-cache" COPY pyproject.toml uv.lock README.md .python-version LICENSE.md patches.pth . -# Pre-install Python to uv's default location and make it world-readable RUN uv python install && \ chmod -R a+rX /root/.local/share/uv -# Copy project files COPY src/ /app/src/ -# Pre-build the venv with the runtime extras the cloud wrapper invokes -# (`uv run --extra livepeer --extra kafka livepeer-runner`). Without this -# the first cold start has to fetch + install aiokafka, uvicorn[standard], -# and re-resolve the entire dep graph at runtime — adds ~10s per cold start. +# Build the venv with all runtime extras the cloud wrapper invokes +# (`uv run --extra livepeer --extra kafka livepeer-runner`). RUN uv sync --extra livepeer --extra kafka --no-dev -# Pre-install bundled plugins (cannot be removed by users) +# Pre-install bundled plugins (cannot be removed by users). ENV DAYDREAM_SCOPE_BUNDLED_PLUGINS_FILE="/app/bundled-plugins.txt" RUN echo "git+https://github.com/daydreamlive/scope-ltx-2.git@d13b5f9d94130b975989cd820eedbef5b3a8f165" > /app/bundled-plugins.txt RUN uv run daydream-scope install git+https://github.com/daydreamlive/scope-ltx-2.git@d13b5f9d94130b975989cd820eedbef5b3a8f165 -# Strip never-at-runtime files from the venv to shrink the image. -# Keep .pyc/__pycache__ so first import isn't slowed by recompilation. -# Drop C/C++ headers/sources (only used at compile time), package -# tests/ and docs/ directories, .pdb debug symbols, and the uv/apt -# caches we don't need at runtime. Keep examples/ — some packages -# import from it. +# Strip never-at-runtime files from the venv, plus duplicated CUDA libs that +# the base image already ships under /usr/local/cuda/lib64. Keep cuDNN +# (pyproject `override-dependencies` pins it newer than the base for a Conv3D +# regression), cusparselt (not in base), nccl/cupti/nvtx/nvjitlink/nvshmem +# (small or version-sensitive). Leave the dist-info directories so uv's later +# `sync` (in the fal-side image extension) treats these packages as +# installed and skips reinstall. RUN find /app/.venv -depth -type d \ \( -name 'tests' -o -name 'test' -o -name 'doc' -o -name 'docs' \) \ -exec rm -rf {} + 2>/dev/null || true && \ find /app/.venv -type f \ \( -name '*.h' -o -name '*.c' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.pdb' \) \ -delete 2>/dev/null || true && \ + rm -rf \ + /app/.venv/lib/python3.12/site-packages/nvidia/cublas \ + /app/.venv/lib/python3.12/site-packages/nvidia/cufft \ + /app/.venv/lib/python3.12/site-packages/nvidia/curand \ + /app/.venv/lib/python3.12/site-packages/nvidia/cusolver \ + /app/.venv/lib/python3.12/site-packages/nvidia/cusparse \ + /app/.venv/lib/python3.12/site-packages/nvidia/cuda_runtime \ + /app/.venv/lib/python3.12/site-packages/nvidia/cuda_nvrtc && \ rm -rf /tmp/uv-cache /root/.cache /var/cache/apt/archives/*.deb -# Expose port 8000 for RunPod HTTP proxy + +# --- Runtime stage --------------------------------------------------------- +# Minimal: copies the venv + uv-managed Python from the builder, no build deps. +FROM nvidia/cuda:12.8.0-runtime-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV CUDA_VISIBLE_DEVICES=0 +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility +ENV DAYDREAM_SCOPE_LOGS_DIR=/workspace/logs +ENV DAYDREAM_SCOPE_MODELS_DIR=/workspace/models + +WORKDIR /app + +# Runtime-only system libs (no build-essential / git / python3-dev — those +# stayed in the builder stage). +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgl1 \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +# uv binary + uv-managed Python interpreter (the venv's bin/python is a +# symlink into /root/.local/share/uv/python/...). +COPY --from=builder /usr/local/bin/uv /usr/local/bin/uv +COPY --from=builder /root/.local/share/uv /root/.local/share/uv +RUN mkdir -p /root/.local/bin && ln -s /usr/local/bin/uv /root/.local/bin/uv +ENV UV_INSTALL_DIR="/usr/local/bin" +ENV UV_CACHE_DIR="/tmp/uv-cache" + +# App + venv from the builder. .venv has stripped nvidia-* lib dirs whose +# .so files now come from the base image's /usr/local/cuda/lib64. +COPY --from=builder /app /app + +ENV DAYDREAM_SCOPE_BUNDLED_PLUGINS_FILE="/app/bundled-plugins.txt" + +# Make torch find the base image's CUDA libs for the stripped deps. Keep +# venv's cuDNN first so the override-pinned newer version wins. +ENV LD_LIBRARY_PATH="/app/.venv/lib/python3.12/site-packages/nvidia/cudnn/lib:/usr/local/cuda/lib64" + EXPOSE 8000 -# Default command to run the application CMD ["uv", "run", "daydream-scope", "--host", "0.0.0.0", "--port", "8000"] From 14d06fe0019cf7cf136a10e2ca28adec35c3474b Mon Sep 17 00:00:00 2001 From: emranemran Date: Tue, 5 May 2026 15:06:45 -0700 Subject: [PATCH 6/7] fix(cloud): add python3.12 to runtime stage for fal's image check fal's image extension runs `check_python.sh 3.12 python3.12` which looks for python3.12 on PATH. The multi-stage rewrite dropped python3-dev (which used to pull python3.12 into /usr/bin), causing the deploy to fail with "the Docker image does not have a python3.12 executable". Re-add just python3.12 (the apt package, not python3-dev) so the check passes; the actual app keeps using the uv-managed Python from /root/.local/share/uv via uv run. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: emranemran --- Dockerfile.cloud | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.cloud b/Dockerfile.cloud index 6f415dae9..2a6054b71 100644 --- a/Dockerfile.cloud +++ b/Dockerfile.cloud @@ -77,8 +77,10 @@ ENV DAYDREAM_SCOPE_MODELS_DIR=/workspace/models WORKDIR /app # Runtime-only system libs (no build-essential / git / python3-dev — those -# stayed in the builder stage). +# stayed in the builder stage). python3.12 is required for fal's image +# check (`check_python.sh 3.12 python3.12` looks for it on PATH). RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.12 \ libgl1 \ libglib2.0-0 \ libsm6 \ From 8d78c9db3c8c737543780f43ea9fc253e3bd8afa Mon Sep 17 00:00:00 2001 From: emranemran Date: Tue, 5 May 2026 15:25:23 -0700 Subject: [PATCH 7/7] fix(cloud): add curl + ca-certificates for fal's install_uv.sh After dropping curl/git from the runtime stage in the multi-stage rewrite, fal's image-extension `install_uv.sh` failed at deploy time with "curl is not installed". fal seems to (re)install uv even when the image already ships one, so curl + ca-certs are mandatory. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: emranemran --- Dockerfile.cloud | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile.cloud b/Dockerfile.cloud index 2a6054b71..ab2c943d3 100644 --- a/Dockerfile.cloud +++ b/Dockerfile.cloud @@ -78,9 +78,13 @@ WORKDIR /app # Runtime-only system libs (no build-essential / git / python3-dev — those # stayed in the builder stage). python3.12 is required for fal's image -# check (`check_python.sh 3.12 python3.12` looks for it on PATH). +# check (`check_python.sh 3.12 python3.12` looks for it on PATH). curl + +# ca-certificates are needed by fal's `install_uv.sh` (it tries to +# (re)install uv even though we already shipped one in the image). RUN apt-get update && apt-get install -y --no-install-recommends \ python3.12 \ + curl \ + ca-certificates \ libgl1 \ libglib2.0-0 \ libsm6 \