compose.yml

name: monitoring

# Central observability stack.
#   OTel Collector  →  ingestion gateway (OTLP gRPC 4317 / HTTP 4318)
#   Loki            →  logs
#   Tempo           →  traces
#   Prometheus      →  metrics
#   Grafana         →  UI
#
# Do NOT publish :4317/:4318 to the public internet. The ports below bind to
# 127.0.0.1; expose them via Cloudflare Tunnel, Tailscale, WireGuard, or similar.

x-logging: &default-logging
  driver: json-file
  options:
    max-size: "10m"
    max-file: "3"

# Every service drops all capabilities and carries a pids limit. Each mem_limit
# is a ceiling sized from observed usage, so one runaway component cannot OOM a
# host that also runs production. Services that need a tighter pids_limit
# override it after the merge.
x-hardened: &hardened
  cap_drop: [ALL]
  security_opt:
    - no-new-privileges:true
  pids_limit: 1024
  logging: *default-logging

x-healthcheck: &healthcheck
  interval: 15s
  timeout: 3s
  retries: 5
  start_period: 30s
  # Probe often while starting so `up --wait` returns on the first pass.
  start_interval: 2s

services:
  # A fresh named volume is root-owned and the collector image is distroless, so
  # it cannot chown its own queue directory at startup. This one-shot service
  # does it first. It lives here rather than in the justfile so that a plain
  # `docker compose up` works, and so the volume stays Compose-managed: a
  # `docker volume create` from outside leaves it unlabelled, and then
  # `down --volumes` never removes it.
  otel-queue-init:
    image: alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
    command: ["chown", "10001:10001", "/q"]
    restart: "no"
    network_mode: none
    volumes:
      - otel_queue:/q
    cap_drop: [ALL]
    cap_add: [CHOWN]
    security_opt:
      - no-new-privileges:true
    pids_limit: 16
    mem_limit: 32m
    logging: *default-logging

  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.160.0@sha256:799dc6cf12c96192af37b5bdba804da8c10b3bc563b43cb90c3f3c58d9572ad6
    restart: unless-stopped
    depends_on:
      otel-queue-init:
        condition: service_completed_successfully
      loki:
        condition: service_started
      tempo:
        condition: service_started
      prometheus:
        condition: service_healthy
    ports:
      - "127.0.0.1:4317:4317" # OTLP gRPC
      - "127.0.0.1:4318:4318" # OTLP HTTP
    volumes:
      - ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro
      # File-backed exporter queue. Must be owned by uid 10001; otel-queue-init
      # chowns it before this service starts.
      - otel_queue:/var/lib/otelcol/queue
    command: ["--config=/etc/otelcol/config.yaml"]
    environment:
      OTLP_AUTH_TOKEN: ${OTLP_AUTH_TOKEN:?set OTLP_AUTH_TOKEN in .env}
    # Keep in step with memory_limiter in config/otel-collector.yaml.
    mem_limit: 512m
    !!merge <<: *hardened
    pids_limit: 512
    networks: [default, backend]

  loki:
    image: grafana/loki:3.7.7@sha256:d70e4659623f3e109af669cae76fe2a5dd5be54e2298fe8aed380d982fbc2500
    restart: unless-stopped
    volumes:
      - ./config/loki.yaml:/etc/loki/loki.yaml:ro
      - loki_data:/loki
    command: ["-config.file=/etc/loki/loki.yaml"]
    # No healthcheck: the image is distroless, with no shell or wget to probe with.
    mem_limit: 2g
    !!merge <<: *hardened
    networks: [backend]

  tempo:
    image: grafana/tempo:3.0.3@sha256:0296560ac66f8a3600d7fb3014a52c189d4d9c3549ad6ff441bf2409855d68d5
    restart: unless-stopped
    volumes:
      - ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro
      - tempo_data:/var/tempo
    command: ["-config.file=/etc/tempo/tempo.yaml"]
    # Distroless too, so no healthcheck.
    mem_limit: 2g
    !!merge <<: *hardened
    networks: [backend]

  prometheus:
    image: prom/prometheus:v3.14.0@sha256:5ce7540c3c00ef4ab0c9d2c995c6a5b9c421f44b4a115d97a2c7af3b1c21cbb0
    restart: unless-stopped
    volumes:
      - ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
      - prometheus_data:/prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yaml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=30d
      # Prometheus 3.x: OTLP ingestion is its own flag, not an --enable-feature entry.
      - --web.enable-otlp-receiver
      - --enable-feature=native-histograms,exemplar-storage
      # Whichever limit hits first wins; the disk-space alert is the backstop.
      - --storage.tsdb.retention.size=15GB
    healthcheck:
      !!merge <<: *healthcheck
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"]
    mem_limit: 2g
    !!merge <<: *hardened
    networks: [backend]

  node-exporter:
    image: prom/node-exporter:v1.12.1@sha256:1b4e4438faca4dd7e001dd445d161a4a2091b0fededa84093b3a8dfeae1f1be0
    restart: unless-stopped
    # Same collector allowlist as the spokes' Alloy (templates/alloy/config.alloy).
    # The defaults add ~1000 series on this host (cpufreq, thermal, cooling,
    # per-collector scrape stats) that no dashboard, alert or runbook reads.
    command:
      - --path.rootfs=/host
      - --collector.disable-defaults
      - --collector.cpu
      - --collector.diskstats
      - --collector.filesystem
      - --collector.hwmon
      - --collector.loadavg
      - --collector.meminfo
      - --collector.netdev
      - --collector.stat
      - --collector.uname
    pid: host
    mem_limit: 128m
    !!merge <<: *hardened
    # Holds pid:host plus the whole host filesystem, hence the tightest limits.
    read_only: true
    pids_limit: 64
    volumes:
      # No rslave (unsupported on Docker Desktop) and no network_mode:host, so
      # node_network_* describes the container veth, not host NICs.
      - /:/host:ro
    networks: [backend]

  grafana:
    image: grafana/grafana:13.2.1@sha256:f772d434e8fab0049deb2b1b30abd43342bcfca1537614aa8d36080232cf4283
    restart: unless-stopped
    depends_on:
      loki:
        condition: service_started
      tempo:
        condition: service_started
      prometheus:
        condition: service_healthy
    ports:
      - "127.0.0.1:3000:3000"
    volumes:
      - ./config/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro
      - ./config/grafana/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro
      - ./config/grafana/alerting:/etc/grafana/provisioning/alerting:ro
      - ./dashboards:/var/lib/grafana/dashboards:ro
      - grafana_data:/var/lib/grafana
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env}
      # Expanded by Grafana into the provisioned contact points.
      ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-}
      HEARTBEAT_URL: ${HEARTBEAT_URL:-}
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000}
      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/stack-health.json
      # Secure cookies break plain-http localhost logins, so opt-in. With the
      # tunnel overlay active, `just up` refuses to start without it.
      GF_SECURITY_COOKIE_SECURE: ${GRAFANA_COOKIE_SECURE:-false}
      GF_SECURITY_COOKIE_SAMESITE: strict
      # Per-user identity from Cloudflare Access. Off until GRAFANA_JWT_AUTH=true
      # plus CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD are set (the exposure guards
      # enforce the pair). No fallback team domain: an unset name must fail
      # closed, not fetch signing keys from a claimable cloudflareaccess.com
      # subdomain. The aud pin is required because the JWK set is team-wide.
      GF_AUTH_JWT_ENABLED: ${GRAFANA_JWT_AUTH:-false}
      GF_AUTH_JWT_HEADER_NAME: Cf-Access-Jwt-Assertion
      GF_AUTH_JWT_JWK_SET_URL: https://${CF_ACCESS_TEAM_DOMAIN:-}.cloudflareaccess.com/cdn-cgi/access/certs
      GF_AUTH_JWT_EXPECT_CLAIMS: '{"aud":"${CF_ACCESS_AUD:-}"}'
      GF_AUTH_JWT_EMAIL_CLAIM: email
      GF_AUTH_JWT_USERNAME_CLAIM: email
      GF_AUTH_JWT_AUTO_SIGN_UP: "true"
    healthcheck:
      !!merge <<: *healthcheck
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
    mem_limit: 1g
    !!merge <<: *hardened
    networks: [default, backend]

# Loki, Prometheus and Tempo have no authentication of their own, so only
# Grafana and the collector can reach them. cloudflared stays on `default`: its
# ingress list comes from Cloudflare at runtime, so an edited ingress cannot
# publish a backend that skips the collector's token gate.
networks:
  backend:
    internal: true

volumes:
  loki_data:
  tempo_data:
  prometheus_data:
  grafana_data:
  # Not in `just backup`: seconds of in-flight telemetry, worthless by restore time.
  otel_queue: