fix: dependency installation gaps in Plugin Store and first-time install

first_time_install.sh

@@ -726,23 +726,27 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then
 
         if command -v timeout >/dev/null 2>&1; then
             # Use timeout if available (10 minutes = 600 seconds)
-            if timeout 600 python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
+            # --ignore-installed: apt-managed packages (e.g. python3-requests)
+            # ship no pip RECORD file, so upgrading them would otherwise abort
+            # with "uninstall-no-record-file"; this lays the new version down
+            # alongside instead of trying to uninstall the apt copy first.
+            if timeout 600 python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
                 INSTALL_SUCCESS=true
             else
                 EXIT_CODE=$?
                 if [ "$EXIT_CODE" -eq 124 ]; then
                     echo "✗ Timeout (10 minutes) installing: $line"
                     echo "  This package may require building from source, which can be slow on Raspberry Pi."
                     echo "  You can try installing it manually later with:"
-                    echo "    python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose '$line'"
+                    echo "    python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose '$line'"
                 else
                     echo "✗ Failed to install: $line (exit code: $EXIT_CODE)"
                 fi
             fi
         else
             # No timeout command available, install without timeout
             echo "  Note: timeout command not available, installation may take a while..."
-            if python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
+            if python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
                 INSTALL_SUCCESS=true
             else
                 EXIT_CODE=$?
@@ -794,7 +798,7 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then
         echo "  1. Ensure you have enough disk space: df -h"
         echo "  2. Check available memory: free -h"
         echo "  3. Try installing failed packages individually with verbose output:"
-        echo "     python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose <package>"
+        echo "     python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose <package>"
         echo "  4. For packages that build from source (like numpy), consider:"
         echo "     - Installing pre-built wheels: python3 -m pip install --only-binary :all: <package>"
         echo "     - Or installing via apt if available: sudo apt install python3-<package>"
@@ -816,7 +820,10 @@ echo ""
 # Install web interface dependencies
 echo "Installing web interface dependencies..."
 if [ -f "$PROJECT_ROOT_DIR/web_interface/requirements.txt" ]; then
-    if python3 -m pip install --break-system-packages --prefer-binary -r "$PROJECT_ROOT_DIR/web_interface/requirements.txt"; then
+    # --ignore-installed: apt-managed packages (e.g. python3-requests) ship no
+    # pip RECORD file, so upgrading them to the version pinned here would
+    # otherwise abort the whole install with "uninstall-no-record-file".
+    if python3 -m pip install --break-system-packages --prefer-binary --ignore-installed -r "$PROJECT_ROOT_DIR/web_interface/requirements.txt"; then
         echo "✓ Web interface dependencies installed"
         # Create marker file to indicate dependencies are installed
         touch "$PROJECT_ROOT_DIR/.web_deps_installed"
@@ -977,7 +984,9 @@ else
     else
         echo "Using pip to install dependencies..."
         if [ -f "$PROJECT_ROOT_DIR/requirements_web_v2.txt" ]; then
-            python3 -m pip install --break-system-packages --prefer-binary -r requirements_web_v2.txt
+            # --ignore-installed: see the Step 5 web_interface/requirements.txt
+            # install above — same apt/pip RECORD-file conflict applies here.
+            python3 -m pip install --break-system-packages --prefer-binary --ignore-installed -r requirements_web_v2.txt
         else
             echo "⚠ requirements_web_v2.txt not found; skipping web dependency install"
         fi

src/common/permission_utils.py

@@ -10,6 +10,7 @@
 import logging
 import shutil as _shutil
 import subprocess
+import sys
 from pathlib import Path
 from typing import Optional
 
@@ -287,3 +288,104 @@ def sudo_remove_directory(path: Path, allowed_bases: Optional[list] = None) -> b
         logger.error(f"Unexpected error during sudo helper for {path}: {e}")
         return False
 
+
+def install_requirements_file(req_file: Path, timeout: int = 300) -> subprocess.CompletedProcess:
+    """
+    Install a requirements.txt file for a plugin (or the project itself).
+
+    Prefers the vetted sudo wrapper (scripts/fix_perms/safe_pip_install.sh) so
+    packages end up visible to root-run ledmatrix.service, not just to
+    whichever non-root user happens to run the calling process (e.g. the web
+    interface). Falls back to installing with the calling process's own
+    interpreter if the wrapper isn't set up yet (the admin hasn't run
+    scripts/install/configure_web_sudo.sh), so dependency installation still
+    does *something* useful rather than hard-failing.
+
+    Always installs with the interpreter that will actually run the code
+    (``sys.executable`` in the fallback path, the wrapper's ``python3`` in the
+    sudo path) rather than a bare ``pip``/``pip3`` off PATH, which can
+    silently resolve to a different Python installation (e.g. system Python
+    vs. a virtualenv) than the one importing the package at runtime.
+
+    Args:
+        req_file: Path to a requirements.txt file
+        timeout: Subprocess timeout in seconds
+
+    Returns:
+        subprocess.CompletedProcess from the pip (or wrapper) invocation.
+        Never raises on a non-zero exit; callers should check ``returncode``.
+        ``stdout`` is prefixed with an explanatory note when the root wrapper
+        was unavailable and the fallback path was used.
+    """
+    project_root = Path(__file__).resolve().parent.parent.parent
+    wrapper = project_root / "scripts" / "fix_perms" / "safe_pip_install.sh"
+
+    if wrapper.exists():
+        # See sudo_remove_directory / configure_web_sudo.sh for why bash must
+        # be invoked with an explicit, known path rather than relying on the
+        # wrapper's shebang: sudoers matches the exact command line.
+        bash_candidates = []
+        for candidate in ("/usr/bin/bash", "/bin/bash", _shutil.which("bash")):
+            if candidate and candidate not in bash_candidates:
+                bash_candidates.append(candidate)
+
+        result = None
+        for bash_path in bash_candidates:
+            # bash_path and wrapper are fixed, known-good paths, and
+            # safe_pip_install.sh independently re-validates req_file is an
+            # allowed requirements.txt before installing anything as root.
+            result = subprocess.run(  # nosec B603 - no shell invoked (list-form argv)  # nosemgrep
+                ["sudo", "-n", bash_path, str(wrapper), str(req_file)],
+                capture_output=True, text=True, timeout=timeout, cwd=str(project_root)
+            )
+            if result.returncode == 0:
+                return result
+            # Distinguish "sudo rejected this exact command line" (worth
+            # trying the next bash candidate) from "sudo ran it but pip
+            # itself failed" (a real error — stop and surface it).
+            denied = any(
+                phrase in result.stderr
+                for phrase in ("a password is required", "is not allowed to run", "no tty present")
+            )
+            if not denied:
+                logger.warning(
+                    "Root pip install failed (rc=%s) for %s: %s",
+                    result.returncode, req_file, result.stderr.strip()[:500],
+                )
+                return result
+
+        logger.warning(
+            "Root pip install wrapper denied via sudo for %s; falling back to "
+            "user-level install: %s",
+            req_file, result.stderr.strip()[:500] if result else "no bash candidates found",
+        )
+        note = (
+            f"[Root install unavailable ({(result.stderr.strip() if result else 'sudo denied') or 'sudo denied'}); "
+            "installed for the current process's user only. Packages may not be "
+            "visible to ledmatrix.service if it runs as a different user — "
+            "run scripts/install/configure_web_sudo.sh to fix this.]\n"
+        )
+    else:
+        logger.warning(
+            "safe_pip_install.sh not found; falling back to user-level install for %s",
+            req_file,
+        )
+        note = (
+            "[safe_pip_install.sh not found; installed for the current process's "
+            "user only. Run scripts/install/configure_web_sudo.sh to enable "
+            "root installs visible to ledmatrix.service.]\n"
+        )
+
+    # sys.executable is this process's own interpreter (not
+    # attacker-influenced), and req_file is a Path built internally by callers
+    # (store_manager.py plugin paths, PROJECT_ROOT/requirements.txt), never
+    # raw external/user input. --ignore-installed matches safe_pip_install.sh:
+    # apt-managed packages (e.g. python3-requests) ship no pip RECORD file, so
+    # upgrading them would otherwise abort with "uninstall-no-record-file".
+    result = subprocess.run(  # nosec B603 - no shell invoked (list-form argv)  # nosemgrep
+        [sys.executable, "-m", "pip", "install", "--break-system-packages", "--ignore-installed", "-r", str(req_file)],
+        capture_output=True, text=True, timeout=timeout, cwd=str(project_root)
+    )
+    result.stdout = note + (result.stdout or "")
+    return result
+

src/plugin_system/store_manager.py

@@ -25,7 +25,7 @@
 
 from urllib.parse import urlparse
 
-from src.common.permission_utils import sudo_remove_directory
+from src.common.permission_utils import sudo_remove_directory, install_requirements_file
 
 try:
     from jsonschema import Draft7Validator, ValidationError
@@ -1915,13 +1915,19 @@ def _install_dependencies(self, plugin_path: Path) -> bool:
 
         try:
             self.logger.info(f"Installing dependencies for {plugin_path.name}")
-            subprocess.run(
-                ['pip3', 'install', '--break-system-packages', '-r', str(requirements_file)],
-                check=True,
-                capture_output=True,
-                text=True,
-                timeout=300
-            )
+            # Routed through the shared root-visible installer (same one the
+            # web UI's "Reinstall Plugin Deps" tool uses) rather than a bare
+            # `pip`/`pip3` off PATH: a bare pip binary can silently resolve to
+            # a different Python installation than the one that actually runs
+            # ledmatrix.service, so pip reports success while the package
+            # stays invisible to the running plugin (e.g. missing `astral`
+            # for the weather plugin even though "install" succeeded).
+            result = install_requirements_file(requirements_file, timeout=300)
+            if result.returncode != 0:
+                self.logger.error(
+                    f"Error installing dependencies for {plugin_path.name}: {result.stderr}"
+                )
+                return False
             self.logger.info(f"Dependencies installed successfully for {plugin_path.name}")
             # Write hash marker so plugin_loader skips redundant pip run on next startup
             try:
@@ -1930,10 +1936,7 @@ def _install_dependencies(self, plugin_path: Path) -> bool:
             except OSError as marker_err:
                 self.logger.debug("Could not write dependency marker for %s: %s", plugin_path.name, marker_err)
             return True
-
-        except subprocess.CalledProcessError as e:
-            self.logger.error(f"Error installing dependencies: {e.stderr}")
-            return False
+
         except subprocess.TimeoutExpired:
             self.logger.error("Dependency installation timed out")
             return False

web_interface/blueprints/api_v3.py

@@ -26,6 +26,7 @@
     validate_file_upload
 )
 from src.error_aggregator import get_error_aggregator
+from src.common.permission_utils import install_requirements_file
 
 _SUDO = shutil.which('sudo')
 _JOURNALCTL = shutil.which('journalctl')
@@ -50,83 +51,12 @@ def _pip_install_requirements(req_file: Path, timeout: int) -> subprocess.Comple
     for the current process only if the wrapper isn't set up yet (i.e. the
     admin hasn't run scripts/install/configure_web_sudo.sh since upgrading),
     so the button still does *something* useful rather than hard-failing.
-    """
-    wrapper = PROJECT_ROOT / 'scripts' / 'fix_perms' / 'safe_pip_install.sh'
-    if wrapper.exists():
-        # Must invoke via an explicit `bash <path>` — matching both the
-        # sudoers rule configure_web_sudo.sh provisions ($BASH_PATH
-        # $SAFE_PIP_INSTALL_PATH *) and the existing safe_plugin_rm.sh call
-        # in src/common/permission_utils.py. Calling the script path directly
-        # (relying on its shebang) makes sudo check a different command line
-        # than what's actually allowlisted, so `sudo -n` denies it on any
-        # install that only has the specific rules this script provisions —
-        # it only appeared to work in prior testing because that device also
-        # had a broader, non-standard NOPASSWD: ALL grant.
-        #
-        # $BASH_PATH is resolved once at setup time (configure_web_sudo.sh's
-        # `command -v bash`) and baked into the static sudoers file as a
-        # literal path; sudo requires an exact string match against that, so
-        # if this process's own PATH resolves bash somewhere else, the
-        # sudoers rule won't match here either. Try the standard Debian/
-        # Raspberry Pi OS locations first, then this process's own
-        # resolution, so a divergence in just one of them doesn't break this.
-        bash_candidates = []
-        for candidate in ('/usr/bin/bash', '/bin/bash', shutil.which('bash')):
-            if candidate and candidate not in bash_candidates:
-                bash_candidates.append(candidate)
-
-        result = None
-        for bash_path in bash_candidates:
-            result = subprocess.run(
-                ['sudo', '-n', bash_path, str(wrapper), str(req_file)],
-                capture_output=True, text=True, timeout=timeout, cwd=str(PROJECT_ROOT)
-            )
-            if result.returncode == 0:
-                return result
-            # Best-effort distinction between "sudo rejected this exact
-            # command line" (no matching NOPASSWD rule for this bash path —
-            # worth trying the next candidate) and "sudo ran it but the
-            # wrapper/pip itself failed" (a real error — stop and surface it
-            # rather than uselessly retrying other bash paths or doubling up
-            # with a redundant non-root install attempt).
-            denied = any(
-                phrase in result.stderr
-                for phrase in ('a password is required', 'is not allowed to run', 'no tty present')
-            )
-            if not denied:
-                logger.warning(
-                    "[Pip Install] Root install failed (rc=%s) for %s: %s",
-                    result.returncode, req_file, result.stderr.strip()[:500],
-                )
-                return result
 
-        logger.warning(
-            "[Pip Install] Root wrapper denied via sudo for %s; falling back "
-            "to user-level install: %s",
-            req_file, result.stderr.strip()[:500] if result else 'no bash candidates found',
-        )
-        note = (
-            f"[Root install unavailable ({(result.stderr.strip() if result else 'sudo denied') or 'sudo denied'}); "
-            "installed for the web service's user only. Packages may not be "
-            "visible to ledmatrix.service if it runs as a different user — "
-            "run scripts/install/configure_web_sudo.sh to fix this.]\n"
-        )
-    else:
-        logger.warning(
-            "[Pip Install] safe_pip_install.sh not found; falling back to user-level install for %s",
-            req_file,
-        )
-        note = (
-            "[safe_pip_install.sh not found; installed for the web service's "
-            "user only. Run scripts/install/configure_web_sudo.sh to enable "
-            "root installs visible to ledmatrix.service.]\n"
-        )
-    result = subprocess.run(
-        [sys.executable, '-m', 'pip', 'install', '--break-system-packages', '-r', str(req_file)],
-        capture_output=True, text=True, timeout=timeout, cwd=str(PROJECT_ROOT)
-    )
-    result.stdout = note + (result.stdout or '')
-    return result
+    Thin wrapper around the shared implementation in permission_utils so the
+    Plugin Store's own dependency installation (store_manager.py) follows the
+    exact same root-visible install path instead of a divergent one.
+    """
+    return install_requirements_file(req_file, timeout=timeout)
 
 
 def _scrub_git_remote_url(url: str) -> str: