compose.yaml

name: llm-fleet

services:
  llama:
    image: ghcr.io/ggml-org/llama.cpp:${BASE_TAG}
    restart: unless-stopped

    ports:
      - "${HOST_PORT}:8080"

    # --- AMD GPU passthrough (Vulkan) ---
    devices:
      - "/dev/dri:/dev/dri"          # Vulkan render node
    group_add:
      - "${RENDER_GID}"              # access to /dev/dri/renderD*
      - "${VIDEO_GID}"

    # environment:                   # uncomment to pin a Vulkan device
    #   GGML_VK_VISIBLE_DEVICES: "${GGML_VK_VISIBLE_DEVICES}"

    volumes:
      - "${CACHE_DIR}:/cache"        # slot-save snapshots (rw)
      - "${HF_CACHE_DIR}:/root/.cache/llama.cpp"  # HF download cache (reused from host)

    # All runtime flags. ${VAR} comes from .env via Compose interpolation.
    # --swa-full: keep for sliding-window models (Qwen3.5/3.6, Gemma) if you use
    # slot save/restore; harmless otherwise.
    # Files are downloaded once into HF_CACHE_DIR and reused on restart.
    command: >
      --hf-repo ${HF_REPO}
      --hf-file ${HF_FILE}
      --host 0.0.0.0 --port 8080
      -ngl ${NGL}
      --ctx-size ${CTX}
      -fa ${FLASH_ATTN}
      --cache-type-k ${KV_TYPE} --cache-type-v ${KV_TYPE}
      --jinja
      --swa-full
      --slot-save-path /cache
      --metrics

    healthcheck:
      # NOTE: needs curl IN the image; the slim server image may not ship it.
      # If the check never passes, delete this block (rely on restart) or add
      # curl with a 2-line Dockerfile FROM this image.
      test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s            # allow a ~18 GB model to finish loading