docs: add June 2026 security review

docs/troubleshooting.md

@@ -200,7 +200,10 @@ The secret is printed once. Then point a client at it:
 ```bash
 export BERTH_TOKEN=sk-...
 export BERTH_URL=https://127.0.0.1:11500
-curl -k "$BERTH_URL/v1/models" -H "Authorization: Bearer $BERTH_TOKEN"
+# The token is a secret, so verify TLS by pinning berth's CA rather than using
+# `-k` (which would leak the Bearer token to a MITM).
+curl --cacert ~/.berth/ca/ca.crt "$BERTH_URL/v1/models" \
+  -H "Authorization: Bearer $BERTH_TOKEN"
 ```
 
 If a key that used to work suddenly fails everywhere, check that

examples/README.md

@@ -14,13 +14,17 @@ couple of POSTs. They're here so you don't have to write the JSON from scratch.
 To apply the profile and its route, point `$BERTH_URL` and `$BERTH_TOKEN` at your
 daemon and POST the two files:
 
+These requests carry an admin Bearer token, so verify TLS — pin berth's CA with
+`--cacert` instead of disabling verification with `-k` (which would expose the
+token to any MITM). Adjust the path if your CA lives elsewhere.
+
 ```bash
-curl -k -X POST "$BERTH_URL/admin/service-profiles" \
+curl --cacert ~/.berth/ca/ca.crt -X POST "$BERTH_URL/admin/service-profiles" \
   -H "Authorization: Bearer $BERTH_TOKEN" \
   -H "Content-Type: application/json" \
   --data @examples/service-profile-qwen.json
 
-curl -k -X POST "$BERTH_URL/admin/routes" \
+curl --cacert ~/.berth/ca/ca.crt -X POST "$BERTH_URL/admin/routes" \
   -H "Authorization: Bearer $BERTH_TOKEN" \
   -H "Content-Type: application/json" \
   --data @examples/service-route-chat.json

packaging/berth.service

@@ -34,12 +34,41 @@ RestartSec=5
 StandardOutput=journal
 StandardError=journal
 
-# Modest hardening. Loosen these if your deployments need additional
-# capabilities (e.g. ReadWritePaths if model weights live elsewhere).
+# Process hardening (kept in sync with the installer-generated unit in
+# scripts/setup-leader-vps.sh). The daemon listens on high ports and talks to
+# Docker over /var/run/docker.sock; it never needs kernel modules, devices,
+# raw sockets, or capabilities of any kind. Loosen these only if a deployment
+# genuinely requires extra access (e.g. add a ReadWritePaths entry if model
+# weights live elsewhere).
 NoNewPrivileges=true
 PrivateTmp=true
-ProtectSystem=full
+PrivateDevices=true
+ProtectSystem=strict
 ProtectHome=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectKernelLogs=true
+ProtectControlGroups=true
+ProtectClock=true
+ProtectHostname=true
+ProtectProc=invisible
+ProcSubset=pid
+RestrictNamespaces=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
+LockPersonality=true
+RemoveIPC=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged @resources
+CapabilityBoundingSet=
+AmbientCapabilities=
+UMask=0077
+# NOTE: granting write access to /var/run/docker.sock makes the `berth` service
+# user effectively root-equivalent on the host (the Docker API can mount the
+# host filesystem and launch privileged containers). Drop this entry if the
+# daemon does not need to manage Docker on this node.
 ReadWritePaths=/var/lib/berth /var/run/docker.sock
 
 [Install]

scripts/install.sh

@@ -8,7 +8,9 @@ set -euo pipefail
 #   curl -fsSL https://example.com/install.sh | bash -s -- --verbose
 #
 # What it does:
-#   1. Installs `uv` if missing (https://docs.astral.sh/uv/).
+#   1. Installs `uv` if missing (https://docs.astral.sh/uv/), pinned to
+#      $UV_VERSION below. Operators should review this script before piping it
+#      into a shell, and may override the pin via UV_VERSION=x.y.z.
 #   2. `uv tool install` the `berth` package (or editable, if run in a checkout).
 #   3. Runs `berth doctor`.
 #   4. Prints next steps.
@@ -18,6 +20,10 @@ case "${1:-}" in
   -v|--verbose) VERBOSE=1 ;;
 esac
 
+# Pin the uv installer to a known version rather than tracking latest. The
+# astral installer honours UV_INSTALL_VERSION; operators may override this.
+UV_VERSION="${UV_VERSION:-0.5.11}"
+
 # ── output harness ──────────────────────────────────────────────────────────
 # Unlike the leader installer, steps run in the *current* shell (no subshell):
 # installing uv mutates PATH for the install-berth step that follows, so the
@@ -84,7 +90,9 @@ fi
 
 ensure_uv() {
   command -v uv >/dev/null 2>&1 && return 0
-  curl -LsSf https://astral.sh/uv/install.sh | sh || return 1
+  # Pin to $UV_VERSION; the installer reads UV_INSTALL_VERSION from the env.
+  curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \
+    | env UV_INSTALL_VERSION="$UV_VERSION" sh || return 1
   # shellcheck source=/dev/null
   [ -f "$HOME/.local/share/uv/env" ] && . "$HOME/.local/share/uv/env"
   export PATH="$HOME/.local/bin:$PATH"

scripts/setup-leader-vps.sh

@@ -139,6 +139,11 @@ _fail() {
   exit 1
 }
 
+_redact_secrets() {
+  # Mask minted API keys (sk-...) so they never persist in $LOG.
+  sed 's/sk-[A-Za-z0-9_-]\{8,\}/sk-***REDACTED***/g'
+}
+
 step() {
   local label="$1"; shift
   _step_n=$(( _step_n + 1 ))
@@ -150,13 +155,18 @@ step() {
   # a mid-step failure would slip through. `step` is always called as a plain
   # statement, which keeps -e honoured all the way down.
   set +e
+  # Defense in depth: redact any sk-... secret before it reaches $LOG so the
+  # admin key never lands on disk even if the file's perms are loosened later.
+  # (The operator-facing one-time display reads $BOOTSTRAP_OUT, which is 0600
+  # and removed on EXIT, so it is unaffected.)
   if [[ $VERBOSE == 1 ]]; then
     printf '\n'
-    ( set -eo pipefail; "$@" ) 2>&1 | tee -a "$LOG"
+    ( set -eo pipefail; "$@" ) 2>&1 \
+      | tee >(_redact_secrets >>"$LOG")
     rc=${PIPESTATUS[0]}
   else
-    ( set -eo pipefail; "$@" ) >>"$LOG" 2>&1
-    rc=$?
+    ( set -eo pipefail; "$@" ) 2>&1 | _redact_secrets >>"$LOG"
+    rc=${PIPESTATUS[0]}
   fi
   set -e
   (( rc == 0 )) || _fail
@@ -522,7 +532,9 @@ do_start_services() {
 }
 
 print_header() {
-  : > "$LOG"
+  # Create the log private (0600). The bootstrap step's stdout carries the
+  # freshly minted sk-... admin key; a world-readable log would leak it.
+  install -m 0600 /dev/null "$LOG"
   printf '\n  %sberth%s leader installer    %s%s%s\n' \
     "$_c_accent" "$_c_off" "$_c_dim" "$BASE_DOMAIN" "$_c_off"
   _rule

src/berth/auth/middleware.py

@@ -11,9 +11,11 @@
 
 
 def _is_local_control_request(request: Request) -> bool:
-    return request.scope.get("client") is None or bool(
-        getattr(request.app.state, "local_control_surface", False)
-    )
+    # Trust the explicit per-app flag only (set True solely on the UDS app).
+    # Inferring locality from ``scope['client'] is None`` is a fragile,
+    # silently-failing heuristic that could disable auth on a TCP listener;
+    # the explicit flag fails closed. See _is_uds_request in daemon/admin.py.
+    return bool(getattr(request.app.state, "local_control_surface", False))
 
 
 def _extract_bearer(authorization: str | None) -> str | None:
@@ -57,6 +59,14 @@ def require_auth_dep(request: Request) -> api_keys.ApiKey | None:
 
     tier_cfg: dict[str, Limits] = request.app.state.tier_cfg
     usage_event_id: int | None = None
+    # Rate-limit semantics: request-per-window limits (rpm/rpd) are enforced
+    # hard at admission. Token-per-window limits (tpm/tpd) are necessarily
+    # *advisory / post-hoc*: a request's own token cost is unknown at admission
+    # and is backfilled later via key_usage.set_tokens, so the check evaluates
+    # only previously-completed requests. A single request can therefore exceed
+    # a token budget; the budget reasserts on the next request. Only /v1/* calls
+    # record a usage event, so other authenticated routes are intentionally
+    # unmetered (admin/control traffic is not billed against tenant quotas).
     with db.locked(conn):
         decision = limiter.check(conn, key=key, tier_cfg=tier_cfg)
         if isinstance(decision, limiter.Denied):
@@ -76,12 +86,15 @@ def require_auth_dep(request: Request) -> api_keys.ApiKey | None:
 
 
 def require_metrics_key(request: Request) -> api_keys.ApiKey | None:
-    """Light-weight bearer-auth for /metrics on the public listener.
-
-    Any non-revoked key (no tier requirement) is accepted — the only goal
-    is to keep the deployment inventory / engine URLs / key counts off
-    public scrapers. UDS callers bypass (so local control commands over the
-    local socket still works).
+    """Bearer-auth for /metrics on the public listener — admin tier only.
+
+    /metrics exposes the deployment inventory, scraped engine metrics, active
+    key counts, and per-node cluster topology/labels. That is operational
+    detail useful for reconnaissance, so it is restricted to admin-tier keys
+    rather than any valid tenant key — a low-tier tenant must not be able to
+    map internal deployments/nodes. Point Prometheus (etc.) at an admin key.
+    UDS callers bypass (so local control commands over the local socket still
+    work).
     """
     if _is_local_control_request(request):
         return None  # UDS — operator surface
@@ -99,4 +112,9 @@ def require_metrics_key(request: Request) -> api_keys.ApiKey | None:
             status.HTTP_401_UNAUTHORIZED,
             detail="invalid or revoked API key",
         )
+    if key.tier != "admin":
+        raise HTTPException(
+            status.HTTP_403_FORBIDDEN,
+            detail="admin tier required for /metrics",
+        )
     return key

src/berth/backends/backends.yaml

@@ -1,9 +1,21 @@
 # Pinned engine images. Update via `berth update-engines` or
 # `berth config set engine.<name>.image <tag>`.
+#
+# Optional digest pinning (defense against mutable tags): add a
+# `pinned_digest: sha256:<64-hex>` field to any backend below to require the
+# launched container's image to match that immutable content digest. When set,
+# DockerClient.verify_image_digest enforces it at launch and refuses to mark the
+# deployment ready on mismatch. Resolve the digest for a tag with, e.g.:
+#   docker pull vllm/vllm-openai:v0.20.2
+#   docker inspect --format '{{index .RepoDigests 0}}' vllm/vllm-openai:v0.20.2
+#   # or, to pin the local image id:
+#   docker inspect --format '{{.Id}}' vllm/vllm-openai:v0.20.2
+# Left unset by default, so behavior is unchanged until an operator pins one.
 
 vllm:
   image: vllm/vllm-openai
   pinned_tag: v0.20.2
+  # pinned_digest: sha256:...  # optional; see header comment above
   health_path: /health
   openai_base: /v1
   metrics_path: /metrics

src/berth/backends/base.py

@@ -1,13 +1,34 @@
 from __future__ import annotations
 
+from importlib.resources import files
 from typing import ClassVar, Protocol
 
+import yaml
 from docker.types import Ulimit  # type: ignore[import-untyped]
 
 from berth.backends.manifest import EngineManifest, Headroom, load_manifest
 from berth.lifecycle.plan import DeploymentPlan
 
 
+def _load_pinned_digest(name: str) -> str | None:
+    """Read the optional `pinned_digest` for a backend from backends.yaml.
+
+    The shipped manifest leaves this unset; operators add it to pin an image
+    to an immutable digest. Kept here (rather than on EngineManifest) so the
+    digest-pin feature lives entirely within the backends layer. Best-effort:
+    a malformed/missing file yields None (no enforcement) rather than blocking
+    a deployment on parse failure.
+    """
+    try:
+        text = files("berth.backends").joinpath("backends.yaml").read_text()
+        raw = yaml.safe_load(text) or {}
+        entry = raw.get(name) or {}
+        digest = entry.get("pinned_digest")
+        return str(digest) if digest else None
+    except Exception:
+        return None
+
+
 class Backend(Protocol):
     @property
     def name(self) -> str: ...
@@ -21,6 +42,8 @@ def adapter_unload_path(self) -> str: ...
     @property
     def image_default(self) -> str: ...
     @property
+    def pinned_digest(self) -> str | None: ...
+    @property
     def health_path(self) -> str: ...
     @property
     def openai_base(self) -> str: ...
@@ -62,6 +85,19 @@ def __init__(self, manifest: EngineManifest | None = None):
     def image_default(self) -> str:
         return self.manifest.image_default
 
+    @property
+    def pinned_digest(self) -> str | None:
+        """Optional content-addressable digest (`sha256:...`) an operator has
+        pinned for this engine image in backends.yaml. When set, the running
+        container's image id is verified against it at launch (see
+        DockerClient.verify_image_digest) and a mismatch refuses the load.
+
+        `EngineManifest` intentionally does not carry this field, so it is read
+        from the raw backends.yaml here. Returns None (no enforcement) when
+        absent - the default - keeping behavior unchanged until pinned.
+        """
+        return _load_pinned_digest(self.name)
+
     @property
     def health_path(self) -> str:
         return self.manifest.health_path
@@ -105,6 +141,12 @@ def container_env(self, plan: DeploymentPlan) -> dict[str, str]:
         return {}
 
     def container_kwargs(self, plan: DeploymentPlan) -> dict[str, object]:
+        # No `ipc_mode: host`: host IPC weakens container/host isolation
+        # (a compromised engine could reach other processes' SysV/POSIX shm).
+        # The explicit private `shm_size` below covers the single-container
+        # case, which is what we run. If a future tensor-parallel path needs
+        # shared-memory IPC across separate containers, make host IPC opt-in
+        # rather than the unconditional default.
         return {
             "device_requests": [
                 {
@@ -113,7 +155,6 @@ def container_kwargs(self, plan: DeploymentPlan) -> dict[str, object]:
                     "Capabilities": [["gpu"]],
                 }
             ],
-            "ipc_mode": "host",
             "shm_size": "2g",
             "ulimits": [Ulimit(name="memlock", soft=-1, hard=-1)],
         }

src/berth/cli/agent_cmd.py

@@ -386,10 +386,12 @@ def install_service(
 
 @agent_app.command("register")
 def register(
-    uri: str = typer.Option(
-        ..., "--uri",
+    uri: str | None = typer.Option(
+        None, "--uri",
         help="Single-paste enrollment URI from `berth nodes enroll` "
-             "(format: berth://enroll?leader=...&token=...&ca_fp=...).",
+             "(format: berth://enroll?leader=...&token=...&ca_fp=...). "
+             "Omit to be prompted (hidden input) or read from BERTH_ENROLL_URI; "
+             "this keeps the embedded token out of argv and shell history.",
     ),
     reachable_as: str | None = typer.Option(
         None, "--reachable-as",
@@ -399,7 +401,15 @@ def register(
     """Exchange a one-time enrollment token for a durable agent certificate.
 
     The URI bundles the leader URL, token, and CA fingerprint so the
-    agent can detect a swapped CA during bootstrap."""
+    agent can detect a swapped CA during bootstrap.
+
+    The token is a secret, so prefer not to pass it on the command line where it
+    lands in argv and shell history. Omit ``--uri`` to paste it at a hidden
+    prompt, or supply it via the ``BERTH_ENROLL_URI`` environment variable."""
+    if uri is None:
+        uri = os.environ.get("BERTH_ENROLL_URI") or typer.prompt(
+            "Enrollment URI", hide_input=True,
+        )
     leader_url, token_val, ca_fp = parse_enrollment_uri(uri)
     ca_pem = _fetch_ca_pinned(leader_url, ca_fp)
     _do_register(

src/berth/cli/backup_cmd.py

@@ -33,13 +33,19 @@ def create_backup(
 ) -> None:
     """Tarball db.sqlite (consistent .backup snapshot), ca/, key_pepper, config.toml."""
     dest_path = Path(dest)
-    dest_path.parent.mkdir(parents=True, exist_ok=True)
+    # The backup parent may hold the snapshot and the final tarball, both of
+    # which contain CA keys / pepper. Create it private (0700) so a freshly
+    # made backup dir is never world-traversable.
+    dest_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
 
     # Take a hot-snapshot of the sqlite file. Using .backup avoids the
     # well-known WAL-tail truncation bug of a naive `cp db.sqlite`.
     snapshot_path = (
         config.BERTH_DIR / f".db-backup-{int(time.time())}.sqlite"
     )
+    # Pre-create the snapshot file with 0600 before sqlite opens it, so the
+    # intermediate DB copy is never world-readable under the process umask.
+    os.close(os.open(snapshot_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600))
     src = sqlite3.connect(config.DB_PATH)
     dst = sqlite3.connect(snapshot_path)
     try: