diff --git a/.env.example b/.env.example index a581ecf..35e7199 100644 --- a/.env.example +++ b/.env.example @@ -1,22 +1,49 @@ # Copy to .env and fill in. The .env file is gitignored. +# Which compose overlays this host runs; every `just` recipe acts on the same +# set. Unset = the core stack only. +# production: COMPOSE_FILE=compose.yml:compose.tunnel.yml +# local demo: COMPOSE_FILE=compose.yml:compose.demo.yml +#COMPOSE_FILE= + GRAFANA_ADMIN_PASSWORD=change-me # Set to the public URL Cloudflare Tunnel exposes Grafana on, e.g.: # GRAFANA_ROOT_URL=https://grafana.example.com GRAFANA_ROOT_URL=http://localhost:3000 +# Set to true whenever GRAFANA_ROOT_URL is https; `just up` with the tunnel +# overlay refuses to run without it. Keep false for plain-http localhost use, +# or logins break. +GRAFANA_COOKIE_SECURE=false + # Bearer token every telemetry sender must present (Authorization: Bearer ). -# The default only suits local use — generate a real one for production, e.g.: openssl rand -hex 32 +# The default only suits local use. Generate a real one for production: +# openssl rand -hex 32 OTLP_AUTH_TOKEN=local-dev-token -# Where Alertmanager delivers alert notifications (any webhook: ntfy, Slack, …). -# Leave empty to run without delivery; failures are logged and harmless. +# Where Grafana delivers alert notifications (any webhook: ntfy, Slack, …). +# Not optional: an empty value drops every alert silently while the heartbeat +# below keeps reporting healthy. With the tunnel overlay active, `just up` +# refuses to start without it. ALERT_WEBHOOK_URL= + # Dead man's switch ping target (e.g. https://hc-ping.com/). The Watchdog # alert posts here every 5m; alert externally when pings stop. HEARTBEAT_URL= -# Only needed for `just up-tunnel` (production exposure via Cloudflare Tunnel). -# From: Cloudflare Zero Trust → Networks → Tunnels → → Configure → token +# Optional. Lets `./bootstrap.sh` create a project's healthchecks.io checks. +# Must be the project's READ-WRITE API key (Project Settings -> API keys). +HEALTHCHECKS_API_KEY= + +# Only needed with the tunnel overlay: cd infra && tofu output -raw tunnel_token CLOUDFLARE_TUNNEL_TOKEN= + +# Set to true to let Cloudflare Access sign users into Grafana individually. +# Needs both values below; `just up` with the tunnel overlay enforces that. +GRAFANA_JWT_AUTH=false +# Your Zero Trust team name, the in https://.cloudflareaccess.com: +# cd infra && tofu output -raw grafana_access_team_domain +CF_ACCESS_TEAM_DOMAIN= +# The Grafana Access application's aud tag: cd infra && tofu output -raw grafana_access_aud +CF_ACCESS_AUD= diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6d6cc41..f0e0153 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,23 +1,52 @@ version: 2 updates: + # Hub images: patch bumps ride together; a minor or major comes on its own, + # so a red PR names the one image that broke. - package-ecosystem: "docker-compose" directory: "/" schedule: interval: "weekly" groups: - monitoring-stack-updates: - patterns: - - "*" + hub-patches: + update-types: ["patch"] + # The spoke images every project host runs. + - package-ecosystem: "docker-compose" + directory: "/templates" + schedule: + interval: "weekly" + groups: + spoke-patches: + update-types: ["patch"] + + # The demo: one PR per month for all of it. - package-ecosystem: "docker" directory: "/demo" schedule: interval: "monthly" + groups: + demo: + patterns: ["*"] - package-ecosystem: "pip" directory: "/demo" schedule: interval: "monthly" + groups: + demo: + patterns: ["*"] + # `groups` only cover version updates; without this every advisory opens + # its own PR. Only pip gets Dependabot security advisories here. + demo-security: + applies-to: security-updates + patterns: ["*"] + + # Cloudflare provider for infra/. Dependabot bumps the constraint in + # main.tf but not the hashes in .terraform.lock.hcl; see the runbook. + - package-ecosystem: "terraform" + directory: "/infra" + schedule: + interval: "monthly" - package-ecosystem: "github-actions" directory: "/" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 982f69c..6f6ee90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,29 +1,51 @@ -# Thin wrapper around the `just` contract: everything CI runs, you can run -# locally with the same command. +# Everything CI runs, you can run locally with the same `just` command. `lint` +# has its own job so a lint failure reports without pulling the stack images. name: CI +# Pull requests only: main moves through PRs, so a push run would repeat the +# check. No path filter: gitleaks must see every PR, and a skipped workflow +# never reports the checks a PR needs. infra/ also has its own (infra.yml). on: - push: - branches: [main] pull_request: + workflow_dispatch: permissions: contents: read +# A push to an open PR supersedes the run in flight. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - check: + lint: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - uses: actions/checkout@v7 - - uses: extractions/setup-just@v4 - - run: cp .env.example .env - - run: just check + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # gitleaks scans git history; a shallow clone would pass vacuously. + with: + fetch-depth: 0 + - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 + # 600, or the exposure guards refuse it as world-readable. + - run: install -m 600 .env.example .env + - run: just lint smoke: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v7 - - uses: extractions/setup-just@v4 - - run: cp .env.example .env + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 + # 600, or the exposure guards refuse it as world-readable. + - run: install -m 600 .env.example .env + - run: just validate + # SMOKE_ALERTS adds the TargetDown round trip (about 3 minutes). - run: just smoke - - run: just down + env: + SMOKE_ALERTS: "1" + # The smoke failure messages point at these logs. + - if: failure() + run: just smoke-logs + - if: always() + run: just smoke-down diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml new file mode 100644 index 0000000..8f5c130 --- /dev/null +++ b/.github/workflows/infra.yml @@ -0,0 +1,25 @@ +# OpenTofu validation for infra/, on its own trigger: it downloads the +# provider every run and nothing outside infra/ can change its result. +name: infra + +on: + pull_request: + paths: ["infra/**"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 + - run: install -m 600 .env.example .env # 600 or the exposure guards reject it + - run: just infra-validate diff --git a/.gitignore b/.gitignore index 6968e63..666d257 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ backups/ infra/.terraform/ infra/terraform.tfstate* +infra/imports.tf infra/*.tfvars !infra/*.tfvars.example diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..333638e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +# Installed by `just hooks` (prek). +# pre-commit gitleaks on the staged diff +# commit-msg Conventional Commits shape on the first line +# pre-push `just check`, plus `just infra-validate` when infra/ changed +# `prek run` runs the pre-commit stage on demand; `--hook-stage pre-push` the rest. +repos: + - repo: local + hooks: + - id: gitleaks + name: gitleaks (staged) + entry: just _gitleaks-staged + language: system + pass_filenames: false + always_run: true + stages: [pre-commit] + - id: commit-msg + name: conventional commit message + entry: >- + sh -c 'head -n1 "$1" + | grep -Eq "^((build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)(\([a-z0-9._/-]+\))?!?: + [a-z].{0,81}|(Merge|Revert|fixup!|squash!) .*)$" + || { echo "commit message must be type(scope): lower-case summary, at most 82 chars" >&2; exit 1; }' -- + language: system + stages: [commit-msg] + - id: check + name: just check + entry: just check + language: system + pass_filenames: false + always_run: true + stages: [pre-push] + - id: infra-validate + name: just infra-validate + entry: just infra-validate + language: system + pass_filenames: false + files: ^infra/ + stages: [pre-push] diff --git a/.yamlfmt b/.yamlfmt new file mode 100644 index 0000000..7ac61d8 --- /dev/null +++ b/.yamlfmt @@ -0,0 +1,6 @@ +# Keep line breaks inside folded (>) block scalars: the default re-joins them +# onto one line, which un-wraps the long PromQL exprs and alert descriptions +# in config/grafana/alerting/ every time yamlfmt runs. +formatter: + scan_folded_as_literal: true + retain_line_breaks_single: true diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d0c34..db1d9b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,168 @@ # Changelog -Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); -versions follow [SemVer](https://semver.org/). +Notable changes to this stack. Format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow +[SemVer](https://semver.org/). + +## [0.3.0] - 2026-09-06 + +The hub runs the department's telemetry in production, with one spoke on the +contract (hostname, labels, templates pinned by tag). This release hardens the +stack around that contract and makes the checks prove the data paths, not just +the config syntax. 1.0 waits for a second consumer to confirm the contract. + +### Upgrade + +- Point every spoke's `OTEL_EXPORTER_OTLP_ENDPOINT` at `otel.`. The + old `otlp.` name is gone. +- Set `COMPOSE_FILE` in `.env` to name the overlays this host runs + (`compose.yml:compose.tunnel.yml` in production). `just up-tunnel` is gone; + `just up` runs the exposure guards whenever the tunnel overlay is active. +- Remove the orphaned `alertmanager_data` volume when convenient. Alerting is + Grafana-managed (ADR 0002). +- Re-vendor the templates on each spoke at `v0.3.0` (`bootstrap.sh` prints + the commands): the Alloy agent gains a memory limiter, keeps only the four + cAdvisor metrics this stack reads, and drops its own histogram buckets; the + overlay declares the `egress` network it joins. +- Loki streams keep their old label set until they age out (30 days). +- Set `project` as well as `env` in every sender's + `OTEL_RESOURCE_ATTRIBUTES`. A sender that omits either is now counted as + `unknown` and raises `ProjectsUncovered` instead of arriving unattributed + and unnoticed. + +### Added + +- **Per-user Grafana logins** through Cloudflare Access: `GRAFANA_JWT_AUTH` + with `CF_ACCESS_TEAM_DOMAIN` and `CF_ACCESS_AUD`. The exposure guards + enforce the pair, and the smoke test boots with it on. +- **Durable export queue**: the collector's send queues live on a file-backed + `otel_queue` volume, so telemetry buffered during an outage survives a + collector restart. A one-shot `otel-queue-init` service owns the volume for + the collector, so a plain `docker compose up` works and `down --volumes` + reclaims it. +- **Self-monitoring covers every service**: Prometheus scrapes Grafana, Loki + and Tempo too. `HostDiskFilling` (full within 3 days at the current rate) + and `PrometheusCardinalityHigh` (over 100k active series) warn ahead of the + 80% disk backstop. +- **Per-project ingest counters**: the gateway counts what arrives and emits + `telemetry_{datapoints,logs,spans}_total` by project and environment + (`metrics` and `spanevents` are counted once, without labels). The rules + match `telemetry_.+_total`; `ProjectTelemetrySilent` and `ProjectsUncovered` + key on those instead of scanning every project-labelled series, so their cost is flat in + fleet size, and a project that sends only logs or only traces is covered at + last: it reached no Prometheus series before. Stack Health gains an Ingest + by Project panel. +- **`grafana_access_team_domain`** joins the OpenTofu outputs, so neither + Access value is copied out of the dashboard by hand. Grafana builds its JWK + set URL from the team name, and an unset one fetches signing keys from a + claimable subdomain. +- **`infra/generate-imports.sh`** emits OpenTofu `import` blocks for the + tunnel, DNS records and Access app built by hand, so the first plan does + not create duplicates. +- **`just smoke` proves the stack works, not that it boots**: every dashboard + and alert rule provisioned, uid for uid, none paused; contact points carry + the exact URLs given; Grafana honours the JWT settings and refuses a forged + token; every scrape target is up; a metric and a log round-trip through + the collector into Prometheus and Loki with the `project`, `env` and + `department` labels. With `SMOKE_ALERTS=1` (on in CI) it also stops a + scrape target and waits for `TargetDown` to fire. `just restore-check` + rehearses backup and restore on the smoke volumes. +- **`just check` is `lint` plus `validate`**. `lint` covers compose files, + the rendered alert templates, YAML formatting, workflows, shell scripts, + the demo's Python, OpenTofu formatting, dashboard JSON and the datasource + uids it names, and git history for secrets. `validate` runs each stack + config through the exact image the stack uses. `just hooks` installs git + hooks via prek: gitleaks at commit, a Conventional Commits check on the + message, `check` at push, `infra-validate` at push when `infra/` changed. +- Dependabot watches the spoke images in `templates/` and the Cloudflare + provider in `infra/`. Patch bumps arrive grouped. + +### Changed + +- **Loki indexes only the identity labels** (`service.name`, `department`, + `project`, `env`, `host.name`). Everything else is structured metadata, so + a sender restart no longer mints a new stream. +- **Half the series are gone.** Prometheus drops the histogram buckets from + the Grafana, Loki and Tempo self-scrapes, the hub's node-exporter runs the + same nine-collector allowlist the spokes' Alloy does, and the agent keeps + only the four cAdvisor metrics this stack reads and drops its own buckets. + Measured on the hub with one spoke: 14.1k active series to 7.0k, 438 + samples/s to 206. Nothing that a dashboard, alert or runbook reads was + dropped; `_sum` and `_count` survive, so latency is still there to explore. + `PrometheusCardinalityHigh`'s 100k ceiling is ~14x the baseline now. +- **Dashboards are provisioned, not editable**: `dashboards/*.json` is the + source of truth; UI saves are off. +- **Tempo stores traces and nothing else**: its metrics generator is gone. + RED comes from the applications' own OTLP metrics (ADR 0002). +- CI runs on pull requests only and validates `infra/` on its own workflow. + `lint` and `validate` + `smoke` run on separate jobs. `just demo-build` is + gone, and nothing in CI builds the demo image; `just demo` is the check. +- `just smoke` and `just demo` each run under their own compose project and + Grafana port (`compose.sandbox.yml`), so neither can touch a running stack. +- Image bumps: Grafana 13.1.4, Loki 3.7.6, Tempo 3.0.3, Prometheus 3.13.2, + the collector 0.156.0, node-exporter 1.12.1, cloudflared 2026.8.2, and the + demo's Python dependencies. The `relab-api` dashboard is dropped; the + `$service` picker on Service Health covers it. + +### Fixed + +- **`HighErrorRate` measured the wrong thing twice**: it counted all spans, + so child spans diluted the ratio, and it aggregated by job alone, so a + healthy prod service masked a broken staging one. It now reads the HTTP + server metrics, keyed on job, project and env. +- Both onboarding templates and the demo set `env` but not `project` in + `OTEL_RESOURCE_ATTRIBUTES`, so an app that followed them shipped telemetry + no alert or dashboard could attribute to a project. +- Trace links from the latency panel resolved to nothing: exemplars carry + `traceID`, the datasource looked for `trace_id`. +- The Infrastructure Logs dashboard queried labels this stack never set. The + GPU dashboard's host picker keyed on `instance`, identical on every host. +- The spoke overlay joined an `egress` network it never declared, so the + documented `up -d` failed on a host without one. `just lint` renders both + spoke overlays now. +- With `GRAFANA_JWT_AUTH=true` and no team domain, Grafana fetched signing + keys from a placeholder `cloudflareaccess.com` subdomain any customer could + claim. There is no fallback now, and the guards refuse to start. +- Checks that could pass without checking: the exposure-guard self-test + dropped the status of its positive half; the smoke provisioning check + accepted an empty dashboard list; `just backup` exited 0 with the stack + still paused when the unpause failed; `bootstrap.sh` printed the hash of + empty input for a template missing from the tag; `generate-imports.sh` + reported a failed DNS API call as "no record". `bootstrap.sh` now reads + its rule back from Grafana after the restart, and says so when the admin + password is missing or rejected instead of blaming the rule. +- `TargetDown` and `HostDiskSpaceLow` could never fire on the state they + watch: `up == 0` and a 0% free ratio both evaluate to 0, and the threshold + node fires on value > 0. `HostDiskFilling` had the same defect from a + negative projection. All three carry `bool` now, and the smoke alert + round trip guards the contract. +- `just backup` lost tar's exit status behind the gzip pipe; a partial + archive reported success. `just restore-check` no longer inherits the + host's `COMPOSE_FILE`. +- The stack-health dashboard's host panels averaged spoke node metrics into + the hub's CPU, memory and disk. They are scoped to the hub's `node` job. +- Grafana and the collector wait for Prometheus to report ready instead of + racing it on a cold start. The demo load generator's error rate matches + its advertised one in ten. + +### Security + +- Every service drops all capabilities, runs with `no-new-privileges`, and + carries `mem_limit` and `pids_limit` sized from observed usage. + node-exporter and cloudflared run read-only; the demo runs as `nobody`. +- Loki, Tempo, Prometheus and node-exporter sit on an internal `backend` + network only Grafana and the collector join. cloudflared stays outside it, + so an ingress edited in the Cloudflare dashboard cannot reach a backend + with no authentication of its own. +- `GRAFANA_COOKIE_SECURE` marks the session cookie Secure with strict + SameSite. With the tunnel overlay, `just up` refuses to start without it, + an `https://` root URL, non-default credentials, a webhook URL, and with + `.env` or the OpenTofu state readable by other local users. + `just lint` runs those guards both ways. +- Hub images are digest-pinned, GitHub Actions are pinned to commit SHAs, the + tunnel token reaches cloudflared via environment rather than argv, the + smoke test's ingest token reaches curl through its stdin config, and + `bootstrap.sh` prints `sha256sum -c` lines for the files a spoke vendors. ## [0.2.0] - 2026-07-05 @@ -39,7 +200,7 @@ First tagged release: the stack is runnable, demoable, and validated in CI. ### Added -- `just demo`: one-command demo: an auto-instrumented FastAPI service under +- `just demo`: a one-command demo. An auto-instrumented FastAPI service under constant load populates Grafana with correlated traces, metrics, and logs. - `just check`: validation gate (compose syntax, Prometheus config + alert rules, collector config, YAML, workflows, dashboard JSON), all in pinned diff --git a/README.md b/README.md index 3f761d5..53f3333 100644 --- a/README.md +++ b/README.md @@ -6,45 +6,49 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE) Central monitoring for CML's research software. One host runs Grafana, Loki, -Tempo, Prometheus, and an OpenTelemetry Collector, wired together so logs, -traces, and metrics cross-reference each other. Projects send their telemetry -here over OTLP (the OpenTelemetry protocol) and it lands in one place, -queryable side by side. +Tempo, Prometheus, and an OpenTelemetry Collector. Projects send logs, traces, +and metrics here over OTLP (the OpenTelemetry protocol), and Grafana shows +them side by side. ## Try it in one command +You need Docker with the [Compose plugin](https://docs.docker.com/compose/install/) +and [`just`](https://github.com/casey/just#installation). Everything else runs +in containers. + ```sh cp .env.example .env just demo ``` -This starts the full stack plus a small FastAPI service under constant -artificial load (`compose.demo.yml`). The service is instrumented with -OpenTelemetry auto-instrumentation and fails about one request in ten, on -purpose. Give it a minute, then open Grafana at -(admin / change-me) and look at: +This starts the stack plus a small FastAPI service under constant load +(`compose.demo.yml`). The service is auto-instrumented and fails one request +in ten, so the error panels and the error-rate alert have something to show. +Give it a minute, then open Grafana at +(admin / change-me): -- **Dashboards → Service Health (RED)** — request rate, error rate, and - latency. The dots on the latency panel are exemplars: click one and Grafana - opens the exact trace behind that measurement. -- **Dashboards → Logs Overview** — log volume by service and level, an error - feed, and a live tail of everything arriving over OTLP. -- **Alerting → Alert rules** — the stack-health and error-rate rules Prometheus - is evaluating. +- **Dashboards → Service Health (RED)**: request rate, error rate, and + latency. The dots on the latency panel are exemplars. Click one to open the + trace behind that measurement. +- **Dashboards → Logs**: log volume by service and level, an error feed, and + a live tail. +- **Alerting → Alert rules**: the rules Grafana evaluates. `HighErrorRate` + trips on the demo service after five minutes. ![Service Health (RED) dashboard](docs/img/service-health.png) -`just demo-down` removes the demo services again; the rest of the stack keeps -running. +`just demo-down` removes the demo services. `just demo-destroy` removes the +whole demo stack. + +The demo runs under its own compose project on its own port, so it never +touches a stack already running on the host. The same holds for `just smoke` +on :3001. ## How it works -Everything enters through a single gateway, the OpenTelemetry Collector. A -project only ever configures one endpoint, and we can swap a storage backend -later without touching any application. Tempo also derives request-rate, -error-rate, and duration ("RED") metrics from the traces it receives, so a -service that sends nothing but traces still gets a working dashboard and error -alerting. +Everything enters through one gateway, the OpenTelemetry Collector. A project +configures a single endpoint, and a storage backend can be swapped later +without touching any application. ```mermaid flowchart LR @@ -53,81 +57,111 @@ flowchart LR cf --> otel cf --> grafana - subgraph host["Monitoring host — Docker Compose, ports bound to 127.0.0.1"] + subgraph host["Monitoring host: Docker Compose, ports bound to 127.0.0.1"] otel["OTel Collector
(ingestion gateway)"] otel -->|logs| loki["Loki"] otel -->|traces| tempo["Tempo"] otel -->|metrics| prom["Prometheus"] - tempo -->|"span metrics (RED)"| prom grafana["Grafana"] -. queries .-> loki & tempo & prom end ``` -Solid arrows show telemetry being written; dotted arrows show Grafana reading -at query time. Locally (`just up` or `just demo`) there is no tunnel involved: -everything talks over the compose network and Grafana is at `localhost:3000`. +Solid arrows write telemetry. Dotted arrows are Grafana reading at query +time. Locally there is no tunnel, and Grafana is at `localhost:3000` for +`just up` or `localhost:3002` for `just demo`. -The stack deliberately runs on a single host. At CML's telemetry volume, -distributed ingestion would add operational weight for no gain. The reasoning, -and the alternatives we considered, are recorded in -[ADR 0001](docs/adr/0001-observability-stack.md). +The stack runs on a single host. [ADR 0001](docs/adr/0001-observability-stack.md) +records why, and the alternatives. [ADR 0002](docs/adr/0002-hub-and-spoke-observability.md) +describes the hub-and-spoke design that serves multiple projects. ## Run it for real ```sh cp .env.example .env # set GRAFANA_ADMIN_PASSWORD -just up # core stack, local only -just up-tunnel # production: core stack + Cloudflare Tunnel +just up just check # validate every config in the repo ``` -Grafana: (admin / whatever you set). +Grafana: . + +`COMPOSE_FILE` in `.env` names the overlays a host runs. In production, set +`COMPOSE_FILE=compose.yml:compose.tunnel.yml`. Every recipe (`up`, `logs`, +`ps`, `backup`) then acts on that set. -In production the stack sits behind a Cloudflare Tunnel, and that edge is code -too. The tunnel, its hostnames, DNS, and the Cloudflare Access rule that puts -an email one-time-PIN in front of Grafana all live in `infra/` as a small -OpenTofu configuration. Applying it produces the `CLOUDFLARE_TUNNEL_TOKEN` that -`just up-tunnel` needs; bootstrap steps are at the top of -[infra/main.tf](infra/main.tf). +With the tunnel overlay active, `just up` refuses to start until five settings +are real: a generated `OTLP_AUTH_TOKEN`, a changed `GRAFANA_ADMIN_PASSWORD`, +`GRAFANA_ROOT_URL` pointing at the tunnel hostname, `GRAFANA_COOKIE_SECURE=true`, +and a non-empty `ALERT_WEBHOOK_URL`. An empty `HEARTBEAT_URL` only warns. -`just check` validates compose files, Prometheus config and alert rules, the -collector config, YAML, workflows, and dashboard JSON. Every validator runs in -a pinned container, so nothing needs to be installed on the host. CI runs the -same command on every push and pull request, plus a smoke test that boots the -stack and waits for Grafana to come up healthy. +The Cloudflare edge is code too. The tunnel, its hostnames, DNS, and the +Access rule that puts an email one-time PIN in front of Grafana live in +`infra/` as an OpenTofu configuration. Applying it produces the +`CLOUDFLARE_TUNNEL_TOKEN` the overlay needs. The bootstrap steps are at the +top of [infra/main.tf](infra/main.tf). + +### Checks + +| Recipe | What it does | +| --- | --- | +| `just lint` | Static checks: compose files, rendered alert templates, YAML, workflows, shell, Python, OpenTofu formatting, dashboard JSON, git history for secrets | +| `just validate` | Runs each stack config through the exact image the stack uses | +| `just check` | `lint` plus `validate` | +| `just smoke` | Boots an isolated copy of the stack and asserts it works end to end | +| `just restore-check` | Rehearses backup and restore on the smoke stack | +| `just hooks` | Installs the git hooks: gitleaks at commit, a Conventional Commits check on the message, `check` at push | + +Every check runs in a pinned container. Nothing is installed on the host. + +`just smoke` covers what has no offline validator. It boots the stack in +production shape with JWT auth on, then asserts that every dashboard and +alert rule provisioned, that the contact points carry the URLs from the +environment, that Grafana refuses a forged token, that every scrape target is +up, and that a metric and a log posted through the collector come back out of +Prometheus and Loki with their labels. `SMOKE_ALERTS=1 just smoke` also stops +a scrape target and waits for `TargetDown` to fire, about three minutes more. +The assertions are in [scripts/smoke.sh](scripts/smoke.sh). + +CI runs `lint` on one job and `validate` plus `smoke` (with the alert round +trip) on another, on every pull request. ## Sending telemetry from a project -You need three things: the OTLP endpoint, the bearer token (`OTLP_AUTH_TOKEN`), -and a few naming conventions. Copy-paste templates for each route — zero-code -Python/FastAPI, plain OTLP environment variables, the Loki Docker driver, and -Grafana Alloy for log files — are in -**[docs/ONBOARDING.md](docs/ONBOARDING.md)**. +You need the OTLP endpoint, the bearer token, and a few naming conventions. +[docs/ONBOARDING.md](docs/ONBOARDING.md) has copy-paste templates for +applications. [templates/README.md](templates/README.md) covers the host +agent that ships container logs and host metrics. > [!WARNING] > Never publish ports 4317/4318 to the internet. The compose file binds them to -> `127.0.0.1`; the tunnel is the way in. +> `127.0.0.1`. The tunnel is the way in. ## Alerting -Prometheus evaluates the rules in `config/alerts/`: scrape target down, OTel -export failures, error rate above 5%, disk above 80%. Alertmanager delivers -them to whatever webhook you set in `ALERT_WEBHOOK_URL` (ntfy, Slack, and so -on). +Grafana evaluates and delivers the rules in `config/grafana/alerting/`: +telemetry silent per project, container crash-looping or OOM-killed, scrape +target down, OTel export failures, alert delivery failing, error rate above +5%, disk above 80%, disk projected full within 3 days, and Prometheus head +series above 100k. There is no Alertmanager. Grafana rules can query Loki as +well as Prometheus, and one engine means one answer to "who gets told". + +Notifications go to the webhook in `ALERT_WEBHOOK_URL` (ntfy, Slack, and so +on). One rule, `Watchdog`, fires permanently and posts to `HEARTBEAT_URL` +every five minutes. Point that at a dead man's switch such as healthchecks.io, +which alerts when the pings stop. That is the only way to notice the host +itself dying. -One rule, `Watchdog`, fires permanently by design and posts to `HEARTBEAT_URL` -every five minutes. Point that at a dead man's switch such as healthchecks.io — -a service that alerts when the pings *stop* — and you will also hear about the -one failure the host cannot report itself: its own death. Both variables are -optional; leave them unset and alerts are simply visible in Grafana. +Set both variables. An unset `ALERT_WEBHOOK_URL` drops every alert while the +heartbeat keeps reporting healthy. With the tunnel overlay, `just up` refuses +to start without it. ## Storage -Everything persists to local Docker volumes (`loki_data`, `tempo_data`, -`prometheus_data`, `grafana_data`). When local disk stops fitting, Loki and -Tempo can move to any S3-compatible object store (Backblaze B2, Cloudflare R2, -Hetzner, MinIO); `compose.storage-s3.yml` documents the concrete shape of that -change. +Everything persists to local Docker volumes, captured by `just backup`. The +one exception is `otel_queue`, the collector's on-disk export queue. It holds +seconds of in-flight telemetry and is not worth restoring. + +When local disk stops fitting, Loki and Tempo can move to any S3-compatible +object store. The appendix of ADR 0001 documents that change. ## Layout @@ -135,26 +169,34 @@ change. compose.yml # core services compose.tunnel.yml # production overlay: Cloudflare Tunnel compose.demo.yml # demo overlay: sample telemetry source +compose.sandbox.yml # isolation overlay for `just demo` and `just smoke` demo/ # the demo FastAPI service +scripts/smoke.sh # what `just smoke` asserts against the booted stack config/ otel-collector.yaml # ingestion gateway (OTLP in → Loki/Tempo/Prometheus out) loki.yaml # logs tempo.yaml # traces prometheus.yaml # metrics - alertmanager.yaml # alert routing (webhook + watchdog heartbeat) - alerts/ # Prometheus alert rules - grafana/ # provisioned datasources + dashboard loader -dashboards/ # drop JSON dashboards here; Grafana auto-loads them -docs/ # runbook, onboarding templates, ADRs, screenshots + grafana/ # provisioned datasources, dashboards, and alerting +dashboards/ # JSON dashboards; Grafana loads them from here +docs/ # runbook, onboarding, ADRs, screenshots infra/ # OpenTofu: Cloudflare tunnel, ingress routes, DNS +templates/ # files a project host vendors: agent config, overlays ``` +`dashboards/*.json` is the source of truth. The directory is mounted read-only +and UI saves are disabled, so a change made in the browser lasts until the +page reloads. Edit the JSON and provisioning picks it up within 30 seconds. To +keep something built interactively, export it as JSON and paste it into the +file. + ## Documentation | Document | What it covers | | --- | --- | -| [docs/ONBOARDING.md](docs/ONBOARDING.md) | Connecting a project: endpoint, token, copy-paste templates | -| [docs/RUNBOOK.md](docs/RUNBOOK.md) | Day-to-day ops: rotating tokens, disk pressure, backup and restore | +| [docs/ONBOARDING.md](docs/ONBOARDING.md) | Connecting an application: endpoint, token, copy-paste templates | +| [templates/README.md](templates/README.md) | Onboarding a project host: the agent, GPU hosts, removing a project | +| [docs/RUNBOOK.md](docs/RUNBOOK.md) | Operations: rotating secrets, disk pressure, backup and restore | | [docs/adr/](docs/adr/) | Why the stack looks like this | | [CHANGELOG.md](CHANGELOG.md) | Release history | diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..8a49aa7 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Onboard one project/environment onto this monitoring stack. +# +# ./bootstrap.sh +# +# 1. renders the ProjectTelemetrySilent rule and reloads Grafana; +# 2. regenerates the coverage rule that catches projects never bootstrapped; +# 3. creates the project's healthchecks.io checks (if an API key is present); +# 4. prints the `.env` block for the project host and the curls that vendor the +# templates at a pinned tag. +# +# Idempotent. +set -euo pipefail + +project="${1:-}" +env_name="${2:-}" +if [[ -z "$project" || -z "$env_name" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi +# These become label values, a rule uid, and a filename; a quote or brace in a +# label value produces a rule that silently never matches. +if [[ ! "$project" =~ ^[a-z0-9][a-z0-9-]*$ || ! "$env_name" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "error: project and env must match [a-z0-9][a-z0-9-]* (lowercase, no spaces)" >&2 + exit 2 +fi + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "$root" + +# A bare ./bootstrap.sh does not load .env, so read single keys out of it. +# Sourcing the whole file would drag the stack's secrets into scope. +env_get() { [[ -f .env ]] && sed -n "s/^$1=//p" .env | tail -1; } + +if [[ -z "${HEALTHCHECKS_API_KEY:-}" ]]; then + HEALTHCHECKS_API_KEY="$(env_get HEALTHCHECKS_API_KEY)" + export HEALTHCHECKS_API_KEY +fi +# Flat, not a subdirectory: Grafana's alerting provisioner does not recurse, and +# skips a nested directory with only a warning. BOOTSTRAP_OUT_DIR is for `just +# lint`; with it set, nothing past the rendering runs. +out_dir="${BOOTSTRAP_OUT_DIR:-config/grafana/alerting}" +# The vendored templates and the paths they are printed under. +templates="alloy/config.alloy:deploy/alloy/config.alloy compose.telemetry.yml:compose.telemetry.yml compose.telemetry.gpu.yml:compose.telemetry.gpu.yml run_scheduled.sh:scripts/run_scheduled.sh" + +if [[ -z "${BOOTSTRAP_OUT_DIR:-}" ]]; then + # Pinned tag, no fallback to a branch: two projects must never vendor two + # different configs under the same name. + tag="$(git -C "$root" describe --tags --abbrev=0 2>/dev/null)" \ + || { echo "error: no release tag to pin the vendoring curls to; tag a release first" >&2; exit 1; } + # The tag must contain every template, or a curl 404s and the hash printed + # for it below is the hash of nothing. + for pair in $templates; do + git -C "$root" rev-parse -q --verify "${tag}:templates/${pair%%:*}" >/dev/null \ + || { echo "error: tag ${tag} predates templates/${pair%%:*}; tag a new release before onboarding" >&2; exit 1; } + done + repo_raw="https://raw.githubusercontent.com/CMLPlatform/monitoring/${tag}/templates" +fi + +# --------------------------------------------------------------- 1. the keystone rules +# Every covered project/env pair, read back from the COVERS marker in each +# rendered file, plus the pair being bootstrapped now. +pairs="$({ sed -n 's/^# COVERS: //p' "$out_dir"/project-*.yaml 2>/dev/null || true + echo "$project $env_name"; } | sort -u)" + +# All pairs are re-rendered, so a template fix reaches every project. +while read -r p e; do + rendered="${out_dir}/project-${p}-${e}.yaml" + sed -e "s/__PROJECT__/${p}/g" \ + -e "s/__ENV__/${e}/g" \ + templates/alerting/project.yaml.tmpl > "$rendered" + echo "rendered $rendered" +done <<<"$pairs" + +# ------------------------------------------------------------ 2. the coverage backstop +covered="$(echo "$pairs" | sed 's| |/|' | paste -sd',' - | sed 's/,/, /g')" +covered_expr="$(echo "$pairs" \ + | sed 's@^\([^ ]*\) \([^ ]*\)$@{__name__=~"telemetry_.+_total", project="\1",env="\2"}@' \ + | paste -sd'@' - | sed 's/@/ or /g')" +sed -e "s@__COVERED__@${covered}@" \ + -e "s@__COVERED_EXPR__@${covered_expr}@" \ + templates/alerting/coverage.yaml.tmpl > "${out_dir}/coverage.yaml" +echo "rendered ${out_dir}/coverage.yaml (covering: ${covered})" +[[ -z "${BOOTSTRAP_OUT_DIR:-}" ]] || exit 0 + +# ------------------------------------------------------------------ 3. reload Grafana +if docker compose ps --status running --services 2>/dev/null | grep -qx grafana; then + # This recreates an exposed Grafana, so the `just up` guards apply here too. + just _guard-if-exposed || exit 1 + # A restart, not SIGHUP: Grafana re-reads alert provisioning only at startup. + docker compose up -d --force-recreate grafana >/dev/null 2>&1 \ + && echo "reloaded grafana (restarted)" \ + || echo "WARNING: could not restart grafana; run 'docker compose up -d --force-recreate grafana'" >&2 + # Grafana skips a malformed alert group with only a log line, so read the + # rule back. + if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then + GRAFANA_ADMIN_PASSWORD="$(env_get GRAFANA_ADMIN_PASSWORD)" + fi + [[ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]] \ + || { echo "error: GRAFANA_ADMIN_PASSWORD is not set and not readable from .env; cannot verify the rule" >&2; exit 1; } + # curl's config parser unescapes \ and " inside a quoted value, so escape both. + gpw="${GRAFANA_ADMIN_PASSWORD//\\/\\\\}" + gpw="${gpw//\"/\\\"}" + uid="proj-silent-${project}-${env_name}" + for _ in $(seq 30); do + sleep 2 + # Password on stdin, never argv. A wrong password would otherwise poll + # for a minute and then report the rule as missing. + code="$(printf 'user = "admin:%s"\n' "$gpw" \ + | curl -s -o /dev/null -w '%{http_code}' -K - "http://localhost:3000/api/v1/provisioning/alert-rules/${uid}")" || continue + case "$code" in + 200) + echo "verified rule ${uid} is provisioned" + uid="" + break + ;; + 401 | 403) + echo "error: grafana rejected the admin credentials (HTTP ${code}); check GRAFANA_ADMIN_PASSWORD" >&2 + exit 1 + ;; + esac + done + [[ -z "$uid" ]] || { echo "error: grafana restarted but rule ${uid} is not provisioned; check 'just logs grafana' for the rejected file" >&2; exit 1; } +else + echo "note grafana is not running; the rules apply next time it starts" +fi + +# ------------------------------------------------------- 4. healthchecks.io + printout +if [[ -n "${HEALTHCHECKS_API_KEY:-}" ]]; then + # Default job set; override per project with HC_JOBS="backup nightly-sync" etc. + for job in ${HC_JOBS:-backup watchdog restore-check}; do + # Key via curl's stdin config, not -H: argv is readable in `ps`. + printf 'header = "X-Api-Key: %s"\n' "$HEALTHCHECKS_API_KEY" \ + | curl -fsS -K - -X POST https://healthchecks.io/api/v3/checks/ \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"${project}-${env_name}-${job}\",\"slug\":\"${project}-${env_name}-${job}\",\"unique\":[\"name\"],\"timeout\":93600,\"grace\":3600,\"channels\":\"*\"}" \ + | sed -n 's/.*"ping_url": *"\([^"]*\)".*/ PING_'"$(echo "$job" | tr 'a-z-' 'A-Z_')"'=\1/p' + done +else + echo "note HEALTHCHECKS_API_KEY unset; create these by hand at https://healthchecks.io and note their ping URLs" +fi + +# Computed before the heredoc: a substitution inside `cat < +OTLP_AUTH_TOKEN= +TELEMETRY_EDGE_KEY= # only if the project's egress crosses a WAF +GPU_METRICS= # 1 on a host with an NVIDIA card, read by your deploy + # tooling to include compose.telemetry.gpu.yml + +──────────────────────────────────────────────────────────────────────────── +Vendor the templates on the project host (pinned at ${tag}) +──────────────────────────────────────────────────────────────────────────── +mkdir -p deploy/alloy scripts +${curls} + +Verify before executing anything (the hashes come from the ${tag} tag): +sha256sum -c <<'SUM' +${hashes} +SUM +chmod +x scripts/run_scheduled.sh + +Then include the overlay and bring it up: + docker compose -f compose.yml -f compose.telemetry.yml up -d + +Verify from the monitoring host, within ~2 minutes: + count(telemetry_datapoints_total{project="${project}",env="${env_name}"}) -> non-zero + count(container_start_time_seconds{project="${project}",name!=""}) -> one per container +SUMMARY diff --git a/compose.demo.yml b/compose.demo.yml index 16c100f..2ce0fd9 100644 --- a/compose.demo.yml +++ b/compose.demo.yml @@ -1,6 +1,6 @@ -# Demo overlay: a tiny auto-instrumented FastAPI service plus a load -# generator, so the stack has real traces, metrics, and correlated logs -# to show. Not for production — no restart-across-reboots on purpose. +# Demo overlay: an auto-instrumented FastAPI service plus a load generator, so +# the stack has traces, metrics, and correlated logs to show. Does not survive a +# reboot. # # just demo # core stack + this overlay # just demo-down # remove just the demo services @@ -15,33 +15,31 @@ services: demo-api: build: ./demo restart: on-failure - depends_on: [ otel-collector ] + depends_on: [otel-collector] environment: OTEL_SERVICE_NAME: demo-api - OTEL_RESOURCE_ATTRIBUTES: env=demo + OTEL_RESOURCE_ATTRIBUTES: project=demo,env=demo OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf OTEL_EXPORTER_OTLP_HEADERS: Authorization=Bearer ${OTLP_AUTH_TOKEN:?set OTLP_AUTH_TOKEN in .env} - # Traces/metrics/logs exporters all default to otlp via opentelemetry-distro. # Ship Python log records via OTLP with trace context attached. OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: "true" OTEL_METRIC_EXPORT_INTERVAL: "10000" - # Emit the *stable* HTTP semconv (http.server.request.duration in seconds, - # http.route) rather than the legacy names. Stable metric names are frozen, - # so the Service Health dashboard's app-SDK panel survives library bumps. + # Stable HTTP semconv (http.server.request.duration, http.route), which the + # Service Health dashboard queries. OTEL_SEMCONV_STABILITY_OPT_IN: http logging: *demo-logging demo-load: - image: curlimages/curl:8.21.0 + image: curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13 restart: on-failure - depends_on: [ demo-api ] + depends_on: [demo-api] command: - sh - -c - | while true; do - curl -s -o /dev/null demo-api:8000/ + curl -s -o /dev/null demo-api:8000/work curl -s -o /dev/null demo-api:8000/work sleep 1 done diff --git a/compose.sandbox.yml b/compose.sandbox.yml new file mode 100644 index 0000000..41c2670 --- /dev/null +++ b/compose.sandbox.yml @@ -0,0 +1,12 @@ +# Isolation overlay shared by `just smoke` and `just demo`. +# +# A separate compose project gives them their own containers and volumes, but +# not their own host ports. This overlay drops the ingestion ports (the demo app +# and the smoke assertions reach the collector over the project network) and +# moves Grafana to SANDBOX_PORT, so neither can collide with a running stack. +services: + otel-collector: + ports: !override [] + + grafana: + ports: !override ["127.0.0.1:${SANDBOX_PORT:-3001}:3000"] diff --git a/compose.storage-s3.yml b/compose.storage-s3.yml deleted file mode 100644 index 921a8f7..0000000 --- a/compose.storage-s3.yml +++ /dev/null @@ -1,54 +0,0 @@ -# STUB — the storage scale-out path (not wired up; see ADR 0001). -# -# When local volumes stop fitting, Loki and Tempo move their object storage -# to any S3-compatible backend (Cloudflare R2, Backblaze B2, Hetzner, MinIO) -# without touching the collector, Prometheus, or any client project. -# -# This is deliberately a commented stub: it documents the concrete shape of -# the change so the README's scale-out claim is real, but it is not meant to -# be started until credentials and a bucket exist. To activate: -# -# 1. Create s3 variants of the configs, e.g. config/loki.s3.yaml replacing -# `common.storage.filesystem` with: -# -# common: -# storage: -# s3: -# endpoint: ${S3_ENDPOINT} # e.g. .r2.cloudflarestorage.com -# bucketnames: cml-loki -# access_key_id: ${S3_ACCESS_KEY_ID} -# secret_access_key: ${S3_SECRET_ACCESS_KEY} -# s3forcepathstyle: true -# -# and config/tempo.s3.yaml replacing `storage.trace.backend: local` with: -# -# storage: -# trace: -# backend: s3 -# s3: -# endpoint: ${S3_ENDPOINT} -# bucket: cml-tempo -# access_key: ${S3_ACCESS_KEY_ID} -# secret_key: ${S3_SECRET_ACCESS_KEY} -# -# 2. Uncomment the overlay below and run: -# docker compose -f compose.yml -f compose.storage-s3.yml up -d -# -# services: -# loki: -# volumes: -# - ./config/loki.s3.yaml:/etc/loki/loki.yaml:ro -# - loki_data:/loki -# environment: -# S3_ENDPOINT: ${S3_ENDPOINT:?set S3_ENDPOINT in .env} -# S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} -# S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} -# -# tempo: -# volumes: -# - ./config/tempo.s3.yaml:/etc/tempo/tempo.yaml:ro -# - tempo_data:/var/tempo -# environment: -# S3_ENDPOINT: ${S3_ENDPOINT:?set S3_ENDPOINT in .env} -# S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} -# S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} diff --git a/compose.tunnel.yml b/compose.tunnel.yml index 4932e01..fa3934d 100644 --- a/compose.tunnel.yml +++ b/compose.tunnel.yml @@ -1,18 +1,25 @@ -# Production overlay: expose Grafana (and the OTLP endpoints) via Cloudflare -# Tunnel. Requires CLOUDFLARE_TUNNEL_TOKEN in .env. +# Production overlay: expose Grafana and the OTLP endpoint via Cloudflare Tunnel. # -# just up-tunnel +# COMPOSE_FILE=compose.yml:compose.tunnel.yml in .env, then `just up` # -# The tunnel, its public hostnames, and DNS are managed as code in infra/ -# (OpenTofu); `tofu output -raw tunnel_token` yields the token for .env. -# See infra/main.tf for the bootstrap steps. - +# The tunnel, its hostnames, and DNS are OpenTofu in infra/; +# `tofu output -raw tunnel_token` yields CLOUDFLARE_TUNNEL_TOKEN for .env. services: cloudflared: - image: cloudflare/cloudflared:2026.7.0 + image: cloudflare/cloudflared:2026.8.2@sha256:0aa26e284f05e6c77ae375b8c9c11d9eb6a448fb7bcd8d40f31cb6176189eb38 restart: unless-stopped - depends_on: [ grafana, otel-collector ] - command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN:?set CLOUDFLARE_TUNNEL_TOKEN in .env} + depends_on: [grafana, otel-collector] + # Token via env, not --token: argv is readable from the host and from any + # container with pid:host (node-exporter has it). + command: tunnel --no-autoupdate run + environment: + TUNNEL_TOKEN: ${CLOUDFLARE_TUNNEL_TOKEN:?set CLOUDFLARE_TUNNEL_TOKEN in .env} + mem_limit: 256m + pids_limit: 256 + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true logging: driver: json-file options: diff --git a/compose.yml b/compose.yml index f56343a..f9a1cff 100644 --- a/compose.yml +++ b/compose.yml @@ -3,14 +3,12 @@ name: monitoring # Central observability stack. # OTel Collector → ingestion gateway (OTLP gRPC 4317 / HTTP 4318) # Loki → logs -# Tempo → traces (OTLP-native; Grafana has first-class support) -# Prometheus → metrics (native OTLP remote-write-receiver) +# Tempo → traces +# Prometheus → metrics # Grafana → UI # -# Projects ship telemetry to this host via OTLP. Do NOT publish :4317/:4318 -# directly to the public internet — expose via Cloudflare Tunnel, Tailscale, -# WireGuard, or similar. The port bindings below are bound to 127.0.0.1 so -# they are only reachable from the host / tunnel sidecar. +# Do NOT publish :4317/:4318 to the public internet. The ports below bind to +# 127.0.0.1; expose them via Cloudflare Tunnel, Tailscale, WireGuard, or similar. x-logging: &default-logging driver: json-file @@ -18,114 +16,213 @@ x-logging: &default-logging max-size: "10m" max-file: "3" +# Every service drops all capabilities and carries a pids limit. Each mem_limit +# is a ceiling sized from observed usage, so one runaway component cannot OOM a +# host that also runs production. Services that need a tighter pids_limit +# override it after the merge. +x-hardened: &hardened + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + pids_limit: 1024 + logging: *default-logging + +x-healthcheck: &healthcheck + interval: 15s + timeout: 3s + retries: 5 + start_period: 30s + # Probe often while starting so `up --wait` returns on the first pass. + start_interval: 2s + services: + # A fresh named volume is root-owned and the collector image is distroless, so + # it cannot chown its own queue directory at startup. This one-shot service + # does it first. It lives here rather than in the justfile so that a plain + # `docker compose up` works, and so the volume stays Compose-managed: a + # `docker volume create` from outside leaves it unlabelled, and then + # `down --volumes` never removes it. + otel-queue-init: + image: alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b + command: ["chown", "10001:10001", "/q"] + restart: "no" + network_mode: none + volumes: + - otel_queue:/q + cap_drop: [ALL] + cap_add: [CHOWN] + security_opt: + - no-new-privileges:true + pids_limit: 16 + mem_limit: 32m + logging: *default-logging + otel-collector: - image: otel/opentelemetry-collector-contrib:0.156.0 + image: otel/opentelemetry-collector-contrib:0.156.0@sha256:125bdbeb7590cc1952c5b3430ecf14063568980c2c93d5b38676cc0446ed8108 restart: unless-stopped - depends_on: [ loki, tempo, prometheus ] + depends_on: + otel-queue-init: + condition: service_completed_successfully + loki: + condition: service_started + tempo: + condition: service_started + prometheus: + condition: service_healthy ports: - "127.0.0.1:4317:4317" # OTLP gRPC - "127.0.0.1:4318:4318" # OTLP HTTP volumes: - ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro - command: [ "--config=/etc/otelcol/config.yaml" ] + # File-backed exporter queue. Must be owned by uid 10001; otel-queue-init + # chowns it before this service starts. + - otel_queue:/var/lib/otelcol/queue + command: ["--config=/etc/otelcol/config.yaml"] environment: OTLP_AUTH_TOKEN: ${OTLP_AUTH_TOKEN:?set OTLP_AUTH_TOKEN in .env} - logging: *default-logging + # Keep in step with memory_limiter in config/otel-collector.yaml. + mem_limit: 512m + !!merge <<: *hardened + pids_limit: 512 + networks: [default, backend] loki: - image: grafana/loki:3.7.3 + image: grafana/loki:3.7.6@sha256:efd47c67f9bac88ca29bcf8cb997d9ab29d1848bd0aff579282295542a745952 restart: unless-stopped volumes: - ./config/loki.yaml:/etc/loki/loki.yaml:ro - loki_data:/loki - command: [ "-config.file=/etc/loki/loki.yaml" ] - logging: *default-logging + command: ["-config.file=/etc/loki/loki.yaml"] + # No healthcheck: the image is distroless, with no shell or wget to probe with. + mem_limit: 2g + !!merge <<: *hardened + networks: [backend] tempo: - image: grafana/tempo:3.0.2 + image: grafana/tempo:3.0.3@sha256:0296560ac66f8a3600d7fb3014a52c189d4d9c3549ad6ff441bf2409855d68d5 restart: unless-stopped volumes: - ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo_data:/var/tempo - command: [ "-config.file=/etc/tempo/tempo.yaml" ] - logging: *default-logging + command: ["-config.file=/etc/tempo/tempo.yaml"] + # Distroless too, so no healthcheck. + mem_limit: 2g + !!merge <<: *hardened + networks: [backend] prometheus: - image: prom/prometheus:v3.13.0 + image: prom/prometheus:v3.13.2@sha256:508729e0e2d18e11fd742a5a5ca70e557b940a93948c3c95fd0123a6fd538b69 restart: unless-stopped volumes: - ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - - ./config/alerts:/etc/prometheus/alerts:ro - prometheus_data:/prometheus command: - --config.file=/etc/prometheus/prometheus.yaml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=30d - - --web.enable-remote-write-receiver - # Prometheus 3.x: OTLP ingestion is its own flag, no longer a feature flag. + # Prometheus 3.x: OTLP ingestion is its own flag, not an --enable-feature entry. - --web.enable-otlp-receiver - --enable-feature=native-histograms,exemplar-storage - # Whichever hits first wins; the disk-space alert is the backstop. + # Whichever limit hits first wins; the disk-space alert is the backstop. - --storage.tsdb.retention.size=15GB - logging: *default-logging - - alertmanager: - image: prom/alertmanager:v0.33.1 - restart: unless-stopped - volumes: - - ./config/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro - - alertmanager_data:/alertmanager - environment: - ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-} - HEARTBEAT_URL: ${HEARTBEAT_URL:-} - # Alertmanager can't expand env vars in its config; write the webhook - # URLs to the url_file paths the config points at, then start. tmpfs on - # purpose: the URLs often carry secrets and must not end up in the - # persistent volume (which `just backup` archives). - tmpfs: [ /run/am ] - entrypoint: [ "/bin/sh", "-c" ] - command: - - | - printf '%s' "$$ALERT_WEBHOOK_URL" > /run/am/webhook_url - printf '%s' "$$HEARTBEAT_URL" > /run/am/heartbeat_url - exec /bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yaml --storage.path=/alertmanager - logging: *default-logging + healthcheck: + !!merge <<: *healthcheck + test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"] + mem_limit: 2g + !!merge <<: *hardened + networks: [backend] node-exporter: - image: prom/node-exporter:v1.11.1 + image: prom/node-exporter:v1.12.1@sha256:1b4e4438faca4dd7e001dd445d161a4a2091b0fededa84093b3a8dfeae1f1be0 restart: unless-stopped - command: [ "--path.rootfs=/host" ] + # Same collector allowlist as the spokes' Alloy (templates/alloy/config.alloy). + # The defaults add ~1000 series on this host (cpufreq, thermal, cooling, + # per-collector scrape stats) that no dashboard, alert or runbook reads. + command: + - --path.rootfs=/host + - --collector.disable-defaults + - --collector.cpu + - --collector.diskstats + - --collector.filesystem + - --collector.hwmon + - --collector.loadavg + - --collector.meminfo + - --collector.netdev + - --collector.stat + - --collector.uname pid: host + mem_limit: 128m + !!merge <<: *hardened + # Holds pid:host plus the whole host filesystem, hence the tightest limits. + read_only: true + pids_limit: 64 volumes: - # No rslave: unsupported on Docker Desktop; only affects mounts added after start. - # No network_mode:host either, so node_network_* describes the container - # veth, not host NICs — fine while nothing alerts on network metrics. + # No rslave (unsupported on Docker Desktop) and no network_mode:host, so + # node_network_* describes the container veth, not host NICs. - /:/host:ro - logging: *default-logging + networks: [backend] grafana: - image: grafana/grafana:13.1.0 + image: grafana/grafana:13.1.4@sha256:9be3a3ccdb06bcbb127f888b0c4c1d151837443e478887897a63a27d7b348043 restart: unless-stopped - depends_on: [ loki, tempo, prometheus ] + depends_on: + loki: + condition: service_started + tempo: + condition: service_started + prometheus: + condition: service_healthy ports: - "127.0.0.1:3000:3000" volumes: - ./config/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro - ./config/grafana/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro + - ./config/grafana/alerting:/etc/grafana/provisioning/alerting:ro - ./dashboards:/var/lib/grafana/dashboards:ro - grafana_data:/var/lib/grafana environment: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} + # Expanded by Grafana into the provisioned contact points. + ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-} + HEARTBEAT_URL: ${HEARTBEAT_URL:-} GF_USERS_ALLOW_SIGN_UP: "false" GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000} - # Land on Stack Health instead of the empty welcome page. GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/stack-health.json - logging: *default-logging + # Secure cookies break plain-http localhost logins, so opt-in. With the + # tunnel overlay active, `just up` refuses to start without it. + GF_SECURITY_COOKIE_SECURE: ${GRAFANA_COOKIE_SECURE:-false} + GF_SECURITY_COOKIE_SAMESITE: strict + # Per-user identity from Cloudflare Access. Off until GRAFANA_JWT_AUTH=true + # plus CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD are set (the exposure guards + # enforce the pair). No fallback team domain: an unset name must fail + # closed, not fetch signing keys from a claimable cloudflareaccess.com + # subdomain. The aud pin is required because the JWK set is team-wide. + GF_AUTH_JWT_ENABLED: ${GRAFANA_JWT_AUTH:-false} + GF_AUTH_JWT_HEADER_NAME: Cf-Access-Jwt-Assertion + GF_AUTH_JWT_JWK_SET_URL: https://${CF_ACCESS_TEAM_DOMAIN:-}.cloudflareaccess.com/cdn-cgi/access/certs + GF_AUTH_JWT_EXPECT_CLAIMS: '{"aud":"${CF_ACCESS_AUD:-}"}' + GF_AUTH_JWT_EMAIL_CLAIM: email + GF_AUTH_JWT_USERNAME_CLAIM: email + GF_AUTH_JWT_AUTO_SIGN_UP: "true" + healthcheck: + !!merge <<: *healthcheck + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"] + mem_limit: 1g + !!merge <<: *hardened + networks: [default, backend] + +# Loki, Prometheus and Tempo have no authentication of their own, so only +# Grafana and the collector can reach them. cloudflared stays on `default`: its +# ingress list comes from Cloudflare at runtime, so an edited ingress cannot +# publish a backend that skips the collector's token gate. +networks: + backend: + internal: true volumes: loki_data: tempo_data: prometheus_data: grafana_data: - alertmanager_data: + # Not in `just backup`: seconds of in-flight telemetry, worthless by restore time. + otel_queue: diff --git a/config/alertmanager.yaml b/config/alertmanager.yaml deleted file mode 100644 index 72b3289..0000000 --- a/config/alertmanager.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Alert routing. Webhook URLs are supplied at runtime via url_file (written -# from ALERT_WEBHOOK_URL / HEARTBEAT_URL by the compose entrypoint) because -# Alertmanager does not expand env vars in its config. With the env vars -# unset, delivery fails with a logged error and nothing else — the stack -# runs fine without them. - -route: - receiver: webhook - group_by: [alertname] - routes: - # Watchdog is always firing; its delivery is the heartbeat. Wire - # HEARTBEAT_URL to a dead man's switch (e.g. healthchecks.io) that - # alerts when pings STOP arriving. - - matchers: [ 'alertname = "Watchdog"' ] - receiver: heartbeat - group_wait: 0s - repeat_interval: 5m - -receivers: - - name: webhook - webhook_configs: - - url_file: /run/am/webhook_url - - - name: heartbeat - webhook_configs: - - url_file: /run/am/heartbeat_url - send_resolved: false diff --git a/config/alerts/stack.yaml b/config/alerts/stack.yaml deleted file mode 100644 index 07d80bd..0000000 --- a/config/alerts/stack.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Prometheus alert rules for the stack itself plus RED-style alerts on the -# span metrics Tempo derives from traces. Evaluated by Prometheus, delivered -# by Alertmanager (config/alertmanager.yaml → ALERT_WEBHOOK_URL/HEARTBEAT_URL), -# and visible in Grafana under Alerting → Alert rules. -# Validated by `just check` (promtool check config, which loads these rules). - -groups: - - name: meta - rules: - # Always firing on purpose: its arrival at the heartbeat receiver is - # the proof the whole alerting pipeline works. Silence = broken. - - alert: Watchdog - expr: vector(1) - labels: - severity: none - annotations: - summary: "Alerting-pipeline heartbeat (always firing)" - description: "Route this to a dead man's switch; investigate if pings stop." - - - name: stack-health - rules: - - alert: TargetDown - expr: up == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "Scrape target {{ $labels.job }} is down" - description: "Prometheus cannot scrape {{ $labels.instance }} (job {{ $labels.job }}) for 2 minutes." - - - alert: OtelExportFailures - expr: sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m])) > 0 - or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[5m])) > 0 - or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m])) > 0 - for: 5m - labels: - severity: warning - annotations: - summary: "OTel Collector failing to export via {{ $labels.exporter }}" - description: "The collector has been failing to deliver telemetry to a backend for 5 minutes. Check `just logs otel-collector`." - - - name: capacity - rules: - - alert: HostDiskSpaceLow - expr: > - (node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay"} - / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|overlay"}) < 0.2 - for: 15m - labels: - severity: warning - annotations: - summary: "Filesystem {{ $labels.mountpoint }} is over 80% full" - description: "Loki and Tempo have no total-size cap, so this alert is the storage backstop. Free space or lower retention (see RUNBOOK)." - - - name: service-red - rules: - - alert: HighErrorRate - expr: > - sum by (service) (rate(traces_spanmetrics_calls_total{status_code="STATUS_CODE_ERROR"}[5m])) - / sum by (service) (rate(traces_spanmetrics_calls_total[5m])) > 0.05 - for: 5m - labels: - severity: warning - annotations: - summary: "{{ $labels.service }} error rate above 5%" - description: "More than 5% of spans from {{ $labels.service }} report errors (from Tempo span metrics)." diff --git a/config/grafana/alerting/contact-points.yaml b/config/grafana/alerting/contact-points.yaml new file mode 100644 index 0000000..621bdb2 --- /dev/null +++ b/config/grafana/alerting/contact-points.yaml @@ -0,0 +1,26 @@ +# Notification targets. Grafana expands $VAR from the environment. +# +# An unset variable leaves the URL empty and every notification fails silently: +# AlertDeliveryFailing routes to this same webhook, and the heartbeat stays +# green. The exposure guard in `just up` is the only control. + +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: webhook + receivers: + - uid: cp-webhook-default + type: webhook + settings: + url: $ALERT_WEBHOOK_URL + + - orgId: 1 + name: heartbeat + receivers: + - uid: cp-heartbeat-dms + type: webhook + settings: + url: $HEARTBEAT_URL + # A resolved-notification would ping the switch and mask an outage. + disableResolveMessage: true diff --git a/config/grafana/alerting/coverage.yaml b/config/grafana/alerting/coverage.yaml new file mode 100644 index 0000000..8c62d06 --- /dev/null +++ b/config/grafana/alerting/coverage.yaml @@ -0,0 +1,53 @@ +# GENERATED by bootstrap.sh from templates/alerting/coverage.yaml.tmpl. Do not edit +# the rendered file; it is regenerated on every run. +# +# Fires on any series whose project/env pair has no rendered rule file: a project that +# ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule. +# +# Covered right now: relab/staging + +apiVersion: 1 + +groups: + - orgId: 1 + name: _coverage + folder: Stack alerts + interval: 5m + rules: + - uid: projects-uncovered + title: ProjectsUncovered + condition: FIRING + for: 15m + noDataState: OK + execErrState: Error + isPaused: false + data: + - refId: QUERY + datasourceUid: prometheus + relativeTimeRange: + from: 3600 + to: 0 + model: + refId: QUERY + instant: true + editorMode: code + # The collector's count connector (config/otel-collector.yaml) mints these + # counters for every sender, whatever signal it sends. A bare + # {project!=""} would scan every project-labelled series in the TSDB on + # each evaluation, and would still miss a logs-only or traces-only project. + expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="relab",env="staging"})) + - refId: FIRING + datasourceUid: __expr__ + model: + refId: FIRING + type: threshold + expression: QUERY + conditions: + - evaluator: + type: gt + params: [0] + labels: + severity: warning + annotations: + summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules" + description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host." diff --git a/config/grafana/alerting/notification-policies.yaml b/config/grafana/alerting/notification-policies.yaml new file mode 100644 index 0000000..dda18e5 --- /dev/null +++ b/config/grafana/alerting/notification-policies.yaml @@ -0,0 +1,16 @@ +# Everything reaches the webhook except Watchdog, whose delivery is the +# heartbeat: it goes to the dead man's switch on a short repeat and never waits. + +apiVersion: 1 + +policies: + - orgId: 1 + receiver: webhook + group_by: [alertname] + routes: + - receiver: heartbeat + object_matchers: + - [alertname, =, Watchdog] + group_wait: 0s + group_interval: 1m + repeat_interval: 5m diff --git a/config/grafana/alerting/project-relab-staging.yaml b/config/grafana/alerting/project-relab-staging.yaml new file mode 100644 index 0000000..fda2de9 --- /dev/null +++ b/config/grafana/alerting/project-relab-staging.yaml @@ -0,0 +1,57 @@ +# Rendered by bootstrap.sh into config/grafana/alerting/. Do not edit the rendered +# files by hand; edit this template so every project gets the fix. +# +# Nothing on a project host can detect its own absence, so this rule lives here. +# +# bootstrap.sh reads the marker below to regenerate coverage.yaml. It carries the pair, +# not the filename, because both halves may contain a dash. +# COVERS: relab staging + +apiVersion: 1 + +groups: + - orgId: 1 + name: project-relab-staging + folder: Stack alerts + interval: 1m + rules: + # absent() yields nothing while telemetry flows, so NoData is the HEALTHY state + # and must map to OK. + - uid: proj-silent-relab-staging + title: ProjectTelemetrySilent + condition: FIRING + for: 15m + noDataState: OK + execErrState: Error + isPaused: false + data: + - refId: QUERY + datasourceUid: prometheus + relativeTimeRange: + from: 3600 + to: 0 + model: + refId: QUERY + instant: true + editorMode: code + # The collector's ingest counters, as in coverage.yaml.tmpl. absent() loads + # every series its selector matches, once a minute per project, so it is + # bounded to a few per service rather than everything the project sends. + expr: absent({__name__=~"telemetry_.+_total", project="relab",env="staging"}) + - refId: FIRING + datasourceUid: __expr__ + model: + refId: FIRING + type: threshold + expression: QUERY + conditions: + - evaluator: + type: gt + params: [0] + labels: + severity: critical + project: relab + env: staging + annotations: + summary: "No telemetry from relab/staging in 15m" + description: "Host down, Docker down, the agent down, tunnel down, token rotated wrong, or the collector is rejecting this project's data. Nothing on the relab side can detect this on its own. Expect this ~20m after the last sample: absent() needs the series to go stale first." diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml new file mode 100644 index 0000000..dbdc80f --- /dev/null +++ b/config/grafana/alerting/rules.yaml @@ -0,0 +1,258 @@ +# Grafana-managed alert rules (ADR 0002). Every rule has the same shape: the +# PromQL carries its own comparison and the threshold node fires on value > 0. +# A comparison whose matching value can be 0 or negative (up == 0, a ratio, +# predict_linear) needs `bool`, or the rule can never fire. +# +# Keep the total around a dozen. The per-project rules beside this file are +# rendered by bootstrap.sh. +apiVersion: 1 + +# Anchors only. Grafana reads apiVersion and groups; every rule below repeats +# these four blocks verbatim apart from `for` and `expr`, so they live here once +# and are merged in. Nothing here defines a rule. +_defaults: + - &rule_defaults + condition: FIRING + noDataState: OK + execErrState: Error + isPaused: false + - &query_node + refId: QUERY + datasourceUid: prometheus + relativeTimeRange: + from: 3600 + to: 0 + - &query_model + refId: QUERY + instant: true + editorMode: code + - &firing + refId: FIRING + datasourceUid: __expr__ + model: + refId: FIRING + type: threshold + expression: QUERY + conditions: + - evaluator: + type: gt + params: [0] + +groups: + - orgId: 1 + name: meta + folder: Stack alerts + interval: 1m + rules: + # Always firing. execErrState is Error so a broken query stops the + # heartbeat and the switch alarms. + - uid: watchdog-heartbeat + title: Watchdog + !!merge <<: *rule_defaults + for: 0m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: vector(1) + - *firing + labels: + severity: none + annotations: + summary: "Alerting-pipeline heartbeat (always firing)" + description: "Routed to a dead man's switch. Investigate if the pings stop." + # Cannot page when the webhook itself is broken; the heartbeat covers that. + - uid: alert-delivery-failing + title: AlertDeliveryFailing + !!merge <<: *rule_defaults + for: 10m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: rate(grafana_alerting_notifications_failed_total[10m]) > 0 + - *firing + labels: + severity: critical + annotations: + summary: "Grafana cannot deliver {{ $labels.integration }} notifications" + description: >- + Alerts are firing and going nowhere. Most likely an unset ALERT_WEBHOOK_URL, + otherwise the receiver is rejecting. Check `just logs grafana`. + - orgId: 1 + name: stack-health + folder: Stack alerts + interval: 1m + rules: + - uid: target-down + title: TargetDown + !!merge <<: *rule_defaults + for: 2m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: up == bool 0 + - *firing + labels: + severity: critical + annotations: + summary: "Scrape target {{ $labels.job }} is down" + description: "Prometheus cannot scrape {{ $labels.instance }} (job {{ $labels.job }}) for 2 minutes." + - uid: otel-export-failures + title: OtelExportFailures + !!merge <<: *rule_defaults + for: 5m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: > + sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m])) > 0 + or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[5m])) > 0 + or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m])) > 0 + - *firing + labels: + severity: warning + annotations: + summary: "OTel Collector failing to export via {{ $labels.exporter }}" + description: >- + The collector has been failing to deliver telemetry to a backend for + 5 minutes. Check `just logs otel-collector`. + - orgId: 1 + name: capacity + folder: Stack alerts + # Every rule here waits 15m or more before firing, and the disk fit reads 6h + # of samples per filesystem; evaluating that once a minute bought no warning + # time. 5m is 5x less work for the same alert. + interval: 5m + rules: + - uid: host-disk-space-low + title: HostDiskSpaceLow + !!merge <<: *rule_defaults + for: 15m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: > + (node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay"} + / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|overlay"}) < bool 0.2 + - *firing + labels: + severity: warning + annotations: + # Spoke series carry host_name; the central node-exporter only instance. + summary: "Filesystem {{ $labels.mountpoint }} on {{ or $labels.host_name $labels.instance }} is over 80% full" + description: >- + Loki and Tempo have no total-size cap, so this alert is the storage + backstop. Free space or lower retention (see RUNBOOK). + # Linear fit over 6h, projected 3 days out. Filesystems under 10G (boot, + # EFI, snap) are skipped: they cross zero on noise. + - uid: host-disk-filling + title: HostDiskFilling + !!merge <<: *rule_defaults + for: 30m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: > + predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|overlay"}[6h], 3 * 86400) < bool 0 + and node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|overlay"} > 10e9 + - *firing + labels: + severity: warning + annotations: + summary: >- + Filesystem {{ $labels.mountpoint }} on {{ or $labels.host_name $labels.instance }} + fills up within 3 days at the current rate + description: >- + Extrapolated from the last 6 hours. Find what is growing (a spoke + shipping more than before, a log loop, a backup that stopped + rotating) before HostDiskSpaceLow makes it urgent. + # ~14x the baseline of ~7k series with one spoke (measured after the scrape + # trimming in config/prometheus.yaml and the agent config); raise it as spokes + # are onboarded. + - uid: prometheus-cardinality + title: PrometheusCardinalityHigh + !!merge <<: *rule_defaults + for: 30m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: prometheus_tsdb_head_series > 100000 + - *firing + labels: + severity: warning + annotations: + summary: "Prometheus holds {{ $values.QUERY }} active series, over the 100k ceiling" + description: >- + Find the label that exploded with + `topk(10, count by (__name__) ({__name__=~".+"}))` and + `topk(10, count by (job) ({__name__=~".+"}))`, then drop it at the + spoke's Alloy config or the collector. Retention is 15GB; at this + rate it fills. + - orgId: 1 + name: container-lifecycle + folder: Stack alerts + interval: 1m + rules: + - uid: container-restarting + title: ContainerRestarting + !!merge <<: *rule_defaults + for: 10m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: changes(container_start_time_seconds{name!=""}[1h]) > 3 + - *firing + labels: + severity: warning + annotations: + summary: "Container {{ $labels.name }} is crash-looping" + description: "{{ $labels.name }} on {{ $labels.host_name }} has restarted more than 3 times in the last hour." + - uid: container-oom-killed + title: ContainerOOMKilled + !!merge <<: *rule_defaults + for: 0m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: increase(container_oom_events_total[5m]) > 0 + - *firing + labels: + severity: warning + annotations: + summary: "Container {{ $labels.name }} was OOM-killed" + description: "{{ $labels.name }} on {{ $labels.host_name }} hit an OOM kill in the last 5 minutes." + - orgId: 1 + name: service-red + folder: Stack alerts + interval: 1m + rules: + - uid: high-error-rate + title: HighErrorRate + !!merge <<: *rule_defaults + for: 5m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + # Keyed on (job, project, env): on job alone, a healthy prod + # service dilutes a broken staging one below the threshold. + expr: > + sum by (job, project, env) + (rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[5m])) + / sum by (job, project, env) (rate(http_server_request_duration_seconds_count[5m])) > 0.05 + - *firing + labels: + severity: warning + annotations: + summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} error rate above 5%" + description: >- + More than 5% of requests to {{ $labels.job }} + ({{ $labels.project }}/{{ $labels.env }}) returned 5xx over the last 5 minutes. diff --git a/config/grafana/dashboards.yaml b/config/grafana/dashboards.yaml index dacd0fa..2b5d842 100644 --- a/config/grafana/dashboards.yaml +++ b/config/grafana/dashboards.yaml @@ -4,9 +4,7 @@ providers: - name: default folder: "" type: file - disableDeletion: false - allowUiUpdates: true + allowUiUpdates: false updateIntervalSeconds: 30 options: path: /var/lib/grafana/dashboards - foldersFromFilesStructure: true diff --git a/config/grafana/datasources.yaml b/config/grafana/datasources.yaml index 5a2dcf4..daefee2 100644 --- a/config/grafana/datasources.yaml +++ b/config/grafana/datasources.yaml @@ -10,7 +10,7 @@ datasources: jsonData: timeInterval: 30s exemplarTraceIdDestinations: - - name: trace_id + - name: traceID datasourceUid: tempo - name: Loki @@ -19,7 +19,7 @@ datasources: access: proxy url: http://loki:3100 jsonData: - # Click a trace_id in a log line → jump to the trace in Tempo. + # trace_id in a log line links to the trace in Tempo. derivedFields: - name: trace_id matcherRegex: "[tT]race[_-]?[iI][dD][\"=:\\s]+([A-Fa-f0-9]+)" @@ -38,17 +38,11 @@ datasources: access: proxy url: http://tempo:3200 jsonData: + # No tracesToMetrics/serviceMap/nodeGraph: they need span-metrics this + # stack does not generate, and would render an empty service graph. tracesToLogsV2: datasourceUid: loki filterByTraceID: true tags: - key: service.name value: service_name - tracesToMetrics: - datasourceUid: prometheus - serviceMap: - datasourceUid: prometheus - nodeGraph: - enabled: true - search: - hide: false diff --git a/config/loki.yaml b/config/loki.yaml index 96d3512..faff492 100644 --- a/config/loki.yaml +++ b/config/loki.yaml @@ -1,5 +1,4 @@ -# Single-binary Loki with filesystem storage. Swap `common.storage.filesystem` -# for `s3:` (B2 / R2 / Hetzner / MinIO) when you outgrow local disk. +# Single-binary Loki with filesystem storage. auth_enabled: false @@ -34,11 +33,25 @@ limits_config: retention_period: 30d allow_structured_metadata: true volume_enabled: true - # Most log lines under ~100KB. Bump if you log large payloads. max_line_size: 256kb - # Keep ingestion generous for a single-tenant homelab. ingestion_rate_mb: 16 ingestion_burst_size_mb: 32 + # Index only the identity labels below; everything else is structured + # metadata. Loki's default mapping also indexes service.instance.id, which + # mints a fresh stream on every sender restart. These five must be real + # stream labels: dashboard template variables use label_values(), which + # cannot see structured metadata. + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: + - service.name + - department + - project + - env + - host.name compactor: working_directory: /loki/compactor diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index 338ab99..523aceb 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -1,12 +1,14 @@ -# Ingestion gateway. Receives OTLP from all projects and fans out to the -# appropriate backend. Keep this config dumb — do shaping/enrichment at the -# app edge (per-project collector) where it has context. +# Ingestion gateway: receives OTLP from all projects and fans out to the +# backends. Keep it dumb; shaping and enrichment belong at the app edge. -# All senders must present "Authorization: Bearer ". The -# OTLP hostnames are public behind the tunnel, so ingestion needs auth. extensions: bearertokenauth: token: ${env:OTLP_AUTH_TOKEN} + # Backs the exporter queues so buffered telemetry survives a collector + # restart. The directory must be writable by uid 10001; `just up` chowns it. + file_storage: + directory: /var/lib/otelcol/queue + create_directory: true receivers: otlp: @@ -20,43 +22,114 @@ receivers: auth: authenticator: bearertokenauth +connectors: + # A per-(project, env) ingest counter for every signal. The coverage and silence + # rules ask "is this project sending anything?" against a handful of series per + # project instead of scanning every project-labelled series in the TSDB, and they + # see a logs-only or traces-only project, which reaches no other Prometheus series. + # Attributes are read from the resource too, which is where project and env live. + # + # Every data type is named, including the two that carry no project: a type left + # undefined emits the connector's own default metric instead. The rules match + # telemetry_.+_total and key on the project label, so the two without it never + # reach them. + count: + datapoints: + telemetry.datapoints: &by_project + description: Telemetry received, by project and environment. + # default_value, or a sender that never set project is counted nowhere and + # stays invisible: no series, so nothing for the coverage rule to fire on. + # Counted as "unknown" it shows up as an uncovered project instead. + attributes: + - key: project + default_value: unknown + - key: env + default_value: unknown + logs: + telemetry.logs: *by_project + spans: + telemetry.spans: *by_project + # Neither of these is read by a rule. A metric stream has no attributes of its + # own (its datapoints carry them), and span events are only ever counted + # alongside their spans, so one series each is enough. + spanevents: + telemetry.spanevents: + description: Span events received. + metrics: + telemetry.metrics: + description: Metric streams received. + processors: + # Stamped here with upsert, so a sender cannot claim another department. + # Literal rather than an env knob: one value has ever been correct, and a knob here + # is a silent-failure surface. A hub brought up with the wrong value mislabels every + # signal and mints a parallel set of Loki streams, with nothing to catch it. Make it + # a variable when a second department runs its own hub; if one ever federates into + # this one, the upsert itself is what has to go. + resource/department: + attributes: + - key: department + value: cml + action: upsert + + # The count connector emits delta sums and the Prometheus OTLP receiver refuses + # them ("invalid temporality and type combination"). Converting here keeps the + # conversion state to the handful of counter streams, rather than turning on + # Prometheus's experimental global delta handling. + deltatocumulative: {} + + # Sized to the container's mem_limit (512m in compose.yml); keep the two in step. memory_limiter: check_interval: 2s - limit_percentage: 80 - spike_limit_percentage: 15 + limit_mib: 400 + spike_limit_mib: 100 batch: timeout: 5s send_batch_size: 1024 +# The RUNBOOK's "buffers about five minutes" promise is max_elapsed_time below. +# Defined once on the loki exporter and aliased onto the other two. exporters: - # Loki 3.x accepts OTLP natively — no Promtail / loki exporter needed. otlp_http/loki: endpoint: http://loki:3100/otlp + sending_queue: &queue + queue_size: 1000 + storage: file_storage + retry_on_failure: &retry + max_elapsed_time: 300s otlp_grpc/tempo: endpoint: tempo:4317 tls: insecure: true + sending_queue: *queue + retry_on_failure: *retry - # Prometheus native OTLP receiver (requires otlp-write-receiver feature flag). + # Needs --web.enable-otlp-receiver in compose.yml. otlp_http/prometheus: endpoint: http://prometheus:9090/api/v1/otlp + sending_queue: *queue + retry_on_failure: *retry service: - extensions: [bearertokenauth] + extensions: [bearertokenauth, file_storage] pipelines: logs: receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [otlp_http/loki] + processors: [memory_limiter, resource/department, batch] + exporters: [otlp_http/loki, count] traces: receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [otlp_grpc/tempo] + processors: [memory_limiter, resource/department, batch] + exporters: [otlp_grpc/tempo, count] metrics: receivers: [otlp] - processors: [memory_limiter, batch] + processors: [memory_limiter, resource/department, batch] + exporters: [otlp_http/prometheus, count] + # The counters themselves. Same limiter and batcher as every other pipeline. + metrics/count: + receivers: [count] + processors: [memory_limiter, deltatocumulative, batch] exporters: [otlp_http/prometheus] telemetry: metrics: diff --git a/config/prometheus.yaml b/config/prometheus.yaml index 5df8860..92e2fdc 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -4,15 +4,19 @@ global: external_labels: origin: monitoring-host -rule_files: - - /etc/prometheus/alerts/*.yaml +storage: + tsdb: + # Required for OTLP ingestion: without it, late batches are silently dropped. + out_of_order_time_window: 30m -alerting: - alertmanagers: - - static_configs: - - targets: ["alertmanager:9093"] +otlp: + # Otherwise these stay on target_info only and no alert or dashboard sees + # them. service.name and service.instance.id become job/instance regardless. + promote_resource_attributes: [department, project, env, host.name] -# Apps push metrics via OTLP (through the collector). Scrape only infra we host. +# No rule_files and no alerting block: Grafana owns both (ADR 0002). + +# Apps push metrics via OTLP. Scrape only what this stack hosts. scrape_configs: - job_name: prometheus static_configs: @@ -25,3 +29,24 @@ scrape_configs: - job_name: node static_configs: - targets: ["node-exporter:9100"] + + # Only `up` and grafana_alerting_notifications_failed_total are read from these + # three; their histogram buckets were 5.2k of 14.1k head series. The _sum and + # _count series stay, so latency is still visible in Explore. + - job_name: grafana + static_configs: + - targets: ["grafana:3000"] + metric_relabel_configs: &drop_buckets + - source_labels: [__name__] + regex: .*_bucket + action: drop + + - job_name: loki + static_configs: + - targets: ["loki:3100"] + metric_relabel_configs: *drop_buckets + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] + metric_relabel_configs: *drop_buckets diff --git a/config/tempo.yaml b/config/tempo.yaml index 245ef57..20d260b 100644 --- a/config/tempo.yaml +++ b/config/tempo.yaml @@ -1,8 +1,4 @@ -# Tempo 3.x single-binary. Traces arrive only from the collector -# (collector → tempo:4317) so external apps never talk to Tempo directly. -# Migrated from 2.x: the ingester/compactor blocks and the local-blocks -# processor are gone (handled internally by the live-store/backend worker); -# block retention now lives under overrides. +# Tempo single-binary. Traces arrive only from the collector. server: http_listen_port: 3200 @@ -11,7 +7,6 @@ distributor: receivers: otlp: protocols: - # gRPC only: the collector is the sole sender (otlp_grpc/tempo). grpc: endpoint: 0.0.0.0:4317 @@ -23,24 +18,12 @@ storage: local: path: /var/tempo/blocks -# Derive RED metrics (requests/errors/duration) + a service graph from traces. -# These land in Prometheus and power the Service Health dashboard. -metrics_generator: - registry: - external_labels: - source: tempo - storage: - path: /var/tempo/generator/wal - remote_write: - - url: http://prometheus:9090/api/v1/write - send_exemplars: true +# No metrics_generator: RED comes from the applications' own OTLP metrics (ADR 0002). overrides: defaults: - metrics_generator: - processors: [service-graphs, span-metrics] compaction: - block_retention: 168h # 7 days — traces are bulky, tune to disk budget + block_retention: 168h # 7 days usage_report: reporting_enabled: false diff --git a/dashboards/gpu.json b/dashboards/gpu.json new file mode 100644 index 0000000..90f570d --- /dev/null +++ b/dashboards/gpu.json @@ -0,0 +1,4230 @@ +{ + "__elements": {}, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Imported from grafana.com dashboard 14574 (Nvidia GPU Metrics), revised 2026-08-04, for the nvidia_gpu_exporter this stack ships. Kept close to upstream on purpose: the panels encode NVML domain knowledge \u2014 throttle-reason bitmasks, clock domains, XID handling \u2014 that is not worth reproducing by hand. Local changes: the datasource is pinned instead of picked, and the variable chain is scoped by project/env so one dashboard serves every GPU host. Panel expressions are upstream's, untouched.", + "editable": false, + "fiscalYearStartMonth": 0, + "gnetId": 14574, + "graphTooltip": 0, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "nvidia_gpu_exporter" + ], + "targetBlank": false, + "title": "Related dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Recovery action the driver recommends for this GPU. Healthy means none. Color reflects how disruptive the action is: Drain P2P degrades peer-to-peer but keeps computing, GPU Reset interrupts workloads, Node Reboot and Drain & Reset are the heaviest. Requires a recent driver, older drivers do not report this.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Healthy", + "color": "#56A64B", + "index": 0 + }, + "1": { + "text": "GPU Reset", + "color": "#FF9830", + "index": 1 + }, + "2": { + "text": "Node Reboot", + "color": "#C4162A", + "index": 2 + }, + "3": { + "text": "Drain P2P", + "color": "#F2CC0C", + "index": 3 + }, + "4": { + "text": "Drain & Reset", + "color": "#E02F44", + "index": 4 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "", + "noValue": "Not reported by driver" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 35, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_smi_gpu_recovery_action{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Recovery Action", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of performance-limiting throttle reasons currently active: power cap, SW/HW thermal slowdown, HW power brake. Idle and configuration states (application clocks, sync boost) are not counted, and a flag the driver does not report counts as inactive. See the Throttle Reasons history panel for which reason and when.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "None", + "color": "#56A64B", + "index": 0 + }, + "1": { + "text": "1 reason active", + "color": "#FFB357", + "index": 1 + }, + "2": { + "text": "2 reasons active", + "color": "#FF9830", + "index": 2 + }, + "3": { + "text": "3 reasons active", + "color": "#E02F44", + "index": 3 + }, + "4": { + "text": "4 reasons active", + "color": "#C4162A", + "index": 4 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 40, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum by(uuid) (nvidia_smi_clocks_event_reasons_sw_power_cap{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_sw_power_cap{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or (sum by(uuid) (nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0)) + (sum by(uuid) (nvidia_smi_clocks_event_reasons_sw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_sw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or (sum by(uuid) (nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0)) + (sum by(uuid) (nvidia_smi_clocks_event_reasons_hw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_hw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or (sum by(uuid) (nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0)) + (sum by(uuid) (nvidia_smi_clocks_event_reasons_hw_power_brake_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_hw_power_brake_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or (sum by(uuid) (nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Throttling", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Lifetime uncorrected ECC error count. Uses the aggregate total when the driver reports one, otherwise the sum of all reported aggregate uncorrected counters (some GPUs report subcounters without a total). Any value above zero means the GPU memory has produced uncorrectable errors and the card needs attention. Shows 'No ECC reported' when the driver reports no uncorrected-error counters at all (most GeForce cards).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "None", + "color": "#56A64B", + "index": 0 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#6E7B8B", + "value": null + }, + { + "color": "#E02F44", + "value": 1 + } + ] + }, + "unit": "", + "noValue": "No ECC reported" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 41, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by(uuid) (nvidia_smi_ecc_errors_uncorrected_aggregate_total{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or sum by(uuid) ({__name__=~\"nvidia_smi_ecc_errors_uncorrected_aggregate_.+\", uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "ECC Errors (uncorrected)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 49, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current performance state, P0 (maximum performance) to P15 (minimum). Lower P means busier. Vivid blue = working hard, pale blue = idle. This is operational state, not a health signal.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "P0 \u00b7 Max perf", + "color": "#2B62D9", + "index": 0 + }, + "1": { + "text": "P1", + "color": "#3469DB", + "index": 1 + }, + "2": { + "text": "P2", + "color": "#3E70DD", + "index": 2 + }, + "3": { + "text": "P3", + "color": "#4777DE", + "index": 3 + }, + "4": { + "text": "P4", + "color": "#507FE0", + "index": 4 + }, + "5": { + "text": "P5", + "color": "#5986E2", + "index": 5 + }, + "6": { + "text": "P6", + "color": "#638DE4", + "index": 6 + }, + "7": { + "text": "P7", + "color": "#6C94E6", + "index": 7 + }, + "8": { + "text": "P8 \u00b7 Idle", + "color": "#759BE7", + "index": 8 + }, + "9": { + "text": "P9", + "color": "#7EA2E9", + "index": 9 + }, + "10": { + "text": "P10", + "color": "#88A9EB", + "index": 10 + }, + "11": { + "text": "P11", + "color": "#91B0ED", + "index": 11 + }, + "12": { + "text": "P12 \u00b7 Deep idle", + "color": "#9AB8EF", + "index": 12 + }, + "13": { + "text": "P13", + "color": "#A3BFF0", + "index": 13 + }, + "14": { + "text": "P14", + "color": "#ADC6F2", + "index": 14 + }, + "15": { + "text": "P15 \u00b7 Min", + "color": "#B6CDF4", + "index": 15 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 22, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_pstate{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "P-State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Static identity of the selected GPU as reported by the driver.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Field" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8A94A6" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 42, + "options": { + "showHeader": false, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "Value": true, + "instance": true, + "job": true, + "uuid": true, + "index": true, + "driver_model_current": true, + "driver_model_pending": true, + "pci_sub_device_id": true, + "serial": true + }, + "indexByName": { + "name": 0, + "driver_version": 1, + "cuda_version": 2, + "vbios_version": 3, + "compute_cap": 4, + "pci_bus_id": 5 + }, + "renameByName": { + "name": "GPU", + "driver_version": "Driver", + "vbios_version": "VBIOS", + "compute_cap": "Compute Cap", + "pci_bus_id": "PCI Bus", + "cuda_version": "CUDA" + } + } + }, + { + "id": "transpose", + "options": {} + } + ], + "title": "GPU Info", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Compute mode is a configuration, not a health state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Default", + "color": "#5B7A9D", + "index": 0 + }, + "1": { + "text": "Exclusive Thread", + "color": "#8A5CD1", + "index": 1 + }, + "2": { + "text": "Prohibited", + "color": "#B455A0", + "index": 2 + }, + "3": { + "text": "Exclusive Process", + "color": "#3E9E9E", + "index": 3 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 36, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_smi_compute_mode{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Compute Mode", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percent of time over the past sample period during which one or more kernels was executing on the GPU.\nThe sample period may be between 1 second and 1/6 second depending on the product. MIG-enabled cards do not report a whole-GPU number, which shows as N/A; per-instance activity is in the MIG panels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "N/A", + "color": "#6E7B8B", + "index": 0 + } + } + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 4 + }, + "id": 6, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum(nvidia_smi_utilization_gpu_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or sum(nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0 - 1", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "GPU Util", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Board power draw as a share of the power limit in force: the enforced limit when the driver reports one, otherwise the software or default limit.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 4 + }, + "id": 21, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_power_draw_watts{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} / (nvidia_smi_enforced_power_limit_watts{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_power_limit_watts{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_power_default_limit_watts{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Power", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The fan speed value is the percent of the product's maximum noise tolerance fan speed that the device's fan is currently intended to run at. This value may exceed 100% in certain cases. Note: The reported speed is the intended fan speed. If the fan is physically blocked and unable to spin, this output will not match the actual fan speed. Many parts do not report fan speeds because they rely on cooling via fans in the surrounding enclosure.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "N/A", + "color": "#6E7B8B", + "index": 0 + } + } + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 4 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum(nvidia_smi_fan_speed_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or sum(nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0 - 1", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Fan", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Core GPU temperature. in degrees C.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#56A64B", + "value": null + }, + { + "color": "#F2CC0C", + "value": 70 + }, + { + "color": "#E02F44", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 4 + }, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_temperature_gpu{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Temperature", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current frequency of graphics (shader) clock\n/\nMaximum frequency of graphics (shader) clock.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 9 + }, + "id": 20, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_clocks_current_graphics_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} / nvidia_smi_clocks_max_graphics_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "GPU Clock", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current frequency of memory clock / Maximum frequency of memory clock", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 9 + }, + "id": 33, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_clocks_current_memory_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} / nvidia_smi_clocks_max_memory_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory Clock", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total memory allocated by active contexts / Total installed GPU memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "#F2CC0C", + "value": 0.85 + }, + { + "color": "#E02F44", + "value": 0.95 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 9 + }, + "id": 25, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_memory_used_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} / nvidia_smi_memory_total_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory Alloc", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percent of time over the past sample period during which global (device) memory was being read or written.\nThe sample period may be between 1 second and 1/6 second depending on the product. MIG-enabled cards do not report a whole-GPU number, which shows as N/A; per-instance activity is in the MIG panels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "N/A", + "color": "#6E7B8B", + "index": 0 + } + } + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 9 + }, + "id": 7, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum(nvidia_smi_utilization_memory_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or sum(nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0 - 1", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory Util", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percent of time over the past sample period during which global (device) memory was being read or written.\nThe sample period may be between 1 second and 1/6 second depending on the product.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit", + "noValue": "Not reported by driver" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_utilization_memory_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Memory Utilization %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percent of time over the past sample period during which one or more kernels was executing on the GPU.\nThe sample period may be between 1 second and 1/6 second depending on the product.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit", + "noValue": "Not reported by driver" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_utilization_gpu_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "GPU Utilization %", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 50, + "panels": [], + "title": "State History", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Throttle reasons over time. A colored band means the reason was active. Hardware slowdowns (thermal, power brake) are serious, software power cap is routine under sustained load, and Idle just means the GPU had nothing to do.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "custom": { + "fillOpacity": 80, + "lineWidth": 0, + "spanNulls": false, + "insertNulls": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Active", + "color": "transparent", + "index": 0 + }, + "1": { + "text": "Active", + "color": "#6E7B8B", + "index": 1 + } + } + } + ], + "noValue": "-" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "SW Power Cap" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Active", + "color": "transparent", + "index": 0 + }, + "1": { + "text": "Active", + "color": "#F2CC0C", + "index": 1 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SW Thermal Slowdown" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Active", + "color": "transparent", + "index": 0 + }, + "1": { + "text": "Active", + "color": "#FF9830", + "index": 1 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HW Thermal Slowdown" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Active", + "color": "transparent", + "index": 0 + }, + "1": { + "text": "Active", + "color": "#E02F44", + "index": 1 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HW Power Brake" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Active", + "color": "transparent", + "index": 0 + }, + "1": { + "text": "Active", + "color": "#E02F44", + "index": 1 + } + } + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 38, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.85, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_gpu_idle{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_gpu_idle{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "Idle", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_sw_power_cap{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_sw_power_cap{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "SW Power Cap", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_sw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_sw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "SW Thermal Slowdown", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_hw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_hw_thermal_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "HW Thermal Slowdown", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_hw_power_brake_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_hw_power_brake_slowdown{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "HW Power Brake", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_applications_clocks_setting{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_applications_clocks_setting{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "App Clocks Setting", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_clocks_event_reasons_sync_boost{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or nvidia_smi_clocks_throttle_reasons_sync_boost{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "Sync Boost", + "refId": "G" + } + ], + "title": "Throttle Reasons (history)", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Performance state over time. P0 is maximum performance, P15 minimum. Lower P means busier. This is operational state, not a health signal.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "custom": { + "fillOpacity": 80, + "lineWidth": 0, + "spanNulls": false, + "insertNulls": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "P0 \u00b7 Max perf", + "color": "#2B62D9", + "index": 0 + }, + "1": { + "text": "P1", + "color": "#3469DB", + "index": 1 + }, + "2": { + "text": "P2", + "color": "#3E70DD", + "index": 2 + }, + "3": { + "text": "P3", + "color": "#4777DE", + "index": 3 + }, + "4": { + "text": "P4", + "color": "#507FE0", + "index": 4 + }, + "5": { + "text": "P5", + "color": "#5986E2", + "index": 5 + }, + "6": { + "text": "P6", + "color": "#638DE4", + "index": 6 + }, + "7": { + "text": "P7", + "color": "#6C94E6", + "index": 7 + }, + "8": { + "text": "P8 \u00b7 Idle", + "color": "#759BE7", + "index": 8 + }, + "9": { + "text": "P9", + "color": "#7EA2E9", + "index": 9 + }, + "10": { + "text": "P10", + "color": "#88A9EB", + "index": 10 + }, + "11": { + "text": "P11", + "color": "#91B0ED", + "index": 11 + }, + "12": { + "text": "P12 \u00b7 Deep idle", + "color": "#9AB8EF", + "index": 12 + }, + "13": { + "text": "P13", + "color": "#A3BFF0", + "index": 13 + }, + "14": { + "text": "P14", + "color": "#ADC6F2", + "index": 14 + }, + "15": { + "text": "P15 \u00b7 Min", + "color": "#B6CDF4", + "index": 15 + } + } + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 39, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.85, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_pstate{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "P-State", + "refId": "A" + } + ], + "title": "P-State (history)", + "type": "state-timeline" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 45, + "panels": [], + "title": "Power & Thermals", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total memory allocated by active contexts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_memory_used_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Memory Allocation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Core GPU temperature. in degrees C.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "orange" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#E02F44", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_temperature_gpu{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Temperature", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The last measured power draw for the entire board, in watts. Only available if power management is supported; the reading is accurate to within +/- 5 watts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "yellow" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_power_draw_watts{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Power Draw", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The fan speed value is the percent of the product's maximum noise tolerance fan speed that the device's fan is currently intended to run at. This value may exceed 100% in certain cases. Note: The reported speed is the intended fan speed. If the fan is physically blocked and unable to spin, this output will not match the actual fan speed. Many parts do not report fan speeds because they rely on cooling via fans in the surrounding enclosure.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "orange" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "percentunit", + "noValue": "N/A" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_fan_speed_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Fan Speed %", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 46, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current frequency of graphics (shader) clock.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_clocks_current_graphics_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Graphics Clock Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current frequency of video encoder/decoder clock.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 28 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_clocks_current_video_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Video Clock Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current frequency of SM (Streaming Multiprocessor) clock.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 28 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_clocks_current_sm_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "SM Clock Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current frequency of memory clock.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_clocks_current_memory_clock_hz{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{uuid}}", + "refId": "A" + } + ], + "title": "Memory Clock Speed", + "type": "timeseries" + } + ], + "title": "Clocks", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 47, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Processes with a compute context on the selected GPU, with used memory and its share of total VRAM. Requires the exporter to run with --collect.compute-apps. A memory value of 0 B means the platform does not report per-process memory (e.g. Windows in WDDM mode).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PID" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Share" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "custom.width", + "value": 140 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 29, + "w": 9, + "h": 8 + }, + "id": 43, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Memory", + "desc": true + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_compute_app_used_memory_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or (nvidia_smi_compute_app_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} * 0)", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(nvidia_smi_compute_app_used_memory_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} or (nvidia_smi_compute_app_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"} * 0)) / on(uuid) group_left() nvidia_smi_memory_total_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "B" + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "uuid": true, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": { + "process_name": 0, + "pid": 1, + "Value #A": 2, + "Value #B": 3 + }, + "renameByName": { + "process_name": "Process", + "pid": "PID", + "Value #A": "Memory", + "Value #B": "Share" + } + } + } + ], + "title": "GPU Processes", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "GPU memory per process over time, stacked. Processes appear and disappear as they open and close compute contexts. Empty when --collect.compute-apps is off or the platform reports no per-process memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 35, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "normal", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "bytes", + "noValue": "No process data" + }, + "overrides": [] + }, + "gridPos": { + "x": 9, + "y": 29, + "w": 15, + "h": 11 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_compute_app_used_memory_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "legendFormat": "{{process_name}} ({{pid}})", + "refId": "A" + } + ], + "title": "Process Memory Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Health of the per-process collection on the selected node. OK means the last collection succeeded, so an empty table really means no compute processes. Failing means nvidia-smi could not deliver per-process data. Not enabled means the exporter runs without --collect.compute-apps.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Failing", + "color": "#E02F44", + "index": 0 + }, + "1": { + "text": "OK", + "color": "#56A64B", + "index": 1 + } + } + } + ], + "noValue": "Not enabled (--collect.compute-apps)", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#6E7B8B", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 37, + "w": 9, + "h": 3 + }, + "id": 52, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "wideLayout": false, + "text": { + "valueSize": 24 + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nvidia_smi_compute_apps_last_collect_success{host_name=\"$node\", job=\"$job\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Process Collection", + "type": "stat" + } + ], + "title": "Processes", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 48, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "NVLink fabric registration state. Completed is the healthy steady state. In Progress is normal briefly at boot, but persistent In Progress or Not Started means the GPU is not registered with the fabric manager and NVLink workloads will fail.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Supported", + "color": "#6E7B8B", + "index": 0 + }, + "1": { + "text": "Not Started", + "color": "#FF9830", + "index": 1 + }, + "2": { + "text": "In Progress", + "color": "#F2CC0C", + "index": 2 + }, + "3": { + "text": "Completed", + "color": "#56A64B", + "index": 3 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "", + "noValue": "No NVLink fabric" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 30 + }, + "id": 37, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nvidia_smi_fabric_state{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Fabric State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "All uncorrected ECC error counters (aggregate = lifetime, volatile = since boot, broken down by memory subsystem). Empty on GPUs without ECC memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "none", + "noValue": "No ECC reported" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 30 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "label_replace({__name__=~\"nvidia_smi_ecc_errors_uncorrected_.+\", uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}, \"kind\", \"$1\", \"__name__\", \"nvidia_smi_ecc_errors_uncorrected_(.+)\")", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "title": "ECC Uncorrected (detail)", + "type": "timeseries" + } + ], + "title": "Datacenter Health", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 53, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total XID errors observed on this GPU since the exporter started (earlier history cannot be replayed, so a restart resets the count). Any value above zero deserves a look at the per-code panel next to this one. Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "None", + "color": "#56A64B", + "index": 0 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#6E7B8B", + "value": null + }, + { + "color": "#E02F44", + "value": 1 + } + ] + }, + "unit": "", + "noValue": "No data yet" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 31 + }, + "id": 54, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(nvidia_smi_xid_errors_total{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or sum(nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0", + "legendFormat": "", + "refId": "A" + } + ], + "title": "XID Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Seconds since the most recent XID error on this GPU. 'Never' means none were observed since the exporter started. This is the signal to alert on: unlike increase() on the error counter, it also sees each series' first event (see METRICS.md). Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "Never", + "color": "#56A64B", + "index": 0 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#E02F44", + "value": null + }, + { + "color": "#EAB839", + "value": 3600 + }, + { + "color": "#56A64B", + "value": 86400 + } + ] + }, + "unit": "s", + "noValue": "No data yet" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 31 + }, + "id": 61, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - max(nvidia_smi_xid_last_timestamp_seconds{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) or (sum(nvidia_smi_gpu_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * 0 - 1)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Last XID", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Cumulative count per XID code since the exporter started. A series appears when its first event arrives, so an empty panel means no errors were observed yet. Do not alert on increase() of this counter: a series' first event is invisible to it, alert on the last-timestamp metric instead (see METRICS.md). Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "none", + "decimals": 0, + "noValue": "No XID data" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_xid_errors_total{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "XID {{xid}}", + "refId": "A" + } + ], + "title": "XID Errors by Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "PCIe throughput per direction, sampled by the driver over 20ms windows. Requires --collect.pcie-throughput on the nvml backend. Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "Bps", + "noValue": "No PCIe data" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_pcie_throughput_tx_bytes_per_second{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "TX", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_pcie_throughput_rx_bytes_per_second{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "RX", + "refId": "B" + } + ], + "title": "PCIe Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Compares power integrated from the energy counter against the sampled power draw. The energy counter is NVML-only, so the default backend draws only the sampled line.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(nvidia_smi_energy_joules_total{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "from energy counter", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "nvidia_smi_power_draw_watts{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}", + "interval": "", + "legendFormat": "sampled power draw", + "refId": "B" + } + ], + "title": "Power (energy counter vs sampled)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Energy the GPU consumed over the trailing 24 hours, from the driver's cumulative counter (consumption actually measured, not a projection of the current rate). Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#6E7B8B", + "value": null + } + ] + }, + "unit": "kwatth", + "noValue": "No energy data" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 36 + }, + "id": 62, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(nvidia_smi_energy_joules_total{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}[24h])) / 3.6e6", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Energy (trailing 24h)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Framebuffer memory used per MIG GPU instance (memory is a GPU-instance-level resource, shared by its compute instances). Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "bytes", + "noValue": "No MIG memory data" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "(nvidia_smi_mig_memory_used_bytes{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * on(uuid, gpu_instance_id) group_left(profile) (max by (uuid, gpu_instance_id, profile) (label_replace(nvidia_smi_mig_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}, \"profile\", \"$1\", \"profile\", \"^[0-9]+c\\\\.(.*)$\")))", + "interval": "", + "legendFormat": "GI {{gpu_instance_id}} ({{profile}})", + "refId": "A" + } + ], + "title": "MIG Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SM activity per MIG GPU instance. Empty on GPUs without MIG partitions; the first collection that sees an instance serves nothing (the sampling needs a pair of collections). Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "percentunit", + "noValue": "No MIG activity data" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "(nvidia_smi_mig_sm_activity_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * on(uuid, gpu_instance_id) group_left(profile) (max by (uuid, gpu_instance_id, profile) (label_replace(nvidia_smi_mig_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}, \"profile\", \"$1\", \"profile\", \"^[0-9]+c\\\\.(.*)$\")))", + "interval": "", + "legendFormat": "GI {{gpu_instance_id}} ({{profile}})", + "refId": "A" + } + ], + "title": "MIG SM Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Tensor pipe activity per MIG GPU instance, the signal that shows whether an inference workload actually exercises the tensor cores. Served only by the nvml and demo backends; absent under the default exec backend.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "percentunit", + "noValue": "No MIG activity data" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "(nvidia_smi_mig_tensor_activity_ratio{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}) * on(uuid, gpu_instance_id) group_left(profile) (max by (uuid, gpu_instance_id, profile) (label_replace(nvidia_smi_mig_info{uuid=\"$gpu\", host_name=\"$node\", job=\"$job\"}, \"profile\", \"$1\", \"profile\", \"^[0-9]+c\\\\.(.*)$\")))", + "interval": "", + "legendFormat": "GI {{gpu_instance_id}} ({{profile}})", + "refId": "A" + } + ], + "title": "MIG Tensor Activity", + "type": "timeseries" + } + ], + "title": "XID / MIG / Power / PCIe (NVML mode only)", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "gpu", + "project" + ], + "templating": { + "list": [ + { + "name": "project", + "label": "Project", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(nvidia_smi_index, project)", + "query": { + "qryType": 1, + "query": "label_values(nvidia_smi_index, project)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "env", + "label": "Environment", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(nvidia_smi_index{project=\"$project\"}, env)", + "query": { + "qryType": 1, + "query": "label_values(nvidia_smi_index{project=\"$project\"}, env)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "job", + "label": "Job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(nvidia_smi_index{project=\"$project\", env=\"$env\"}, job)", + "query": { + "qryType": 1, + "query": "label_values(nvidia_smi_index{project=\"$project\", env=\"$env\"}, job)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "node", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(nvidia_smi_index{project=\"$project\", env=\"$env\", job=\"$job\"}, host_name)", + "query": { + "qryType": 1, + "query": "label_values(nvidia_smi_index{project=\"$project\", env=\"$env\", job=\"$job\"}, host_name)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "query_result(label_join(label_replace(label_join(nvidia_smi_gpu_info{host_name=\"$node\", job=\"$job\"}, \"ni\", \" #\", \"name\", \"index\"), \"us\", \"($1)\", \"uuid\", \"^(........).*\"), \"gpu_label\", \" \", \"ni\", \"us\"))", + "hide": 0, + "includeAll": false, + "label": "GPU", + "multi": false, + "name": "gpu", + "options": [], + "query": { + "query": "query_result(label_join(label_replace(label_join(nvidia_smi_gpu_info{host_name=\"$node\", job=\"$job\"}, \"ni\", \" #\", \"name\", \"index\"), \"us\", \"($1)\", \"uuid\", \"^(........).*\"), \"gpu_label\", \" \", \"ni\", \"us\"))", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/gpu_label=\"(?[^\"]+)|uuid=\"(?[^\"]+)/g", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU", + "uid": "gpu", + "weekStart": "" +} diff --git a/dashboards/host-containers.json b/dashboards/host-containers.json new file mode 100644 index 0000000..e66e7aa --- /dev/null +++ b/dashboards/host-containers.json @@ -0,0 +1,314 @@ +{ + "uid": "host-containers", + "title": "Host & Containers", + "tags": [ + "host", + "containers", + "project" + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "refresh": "30s", + "schemaVersion": 39, + "editable": null, + "description": "Per-host and per-container resources for any onboarded project, from the Alloy agent's node-exporter and cAdvisor. Container panels are empty unless the agent runs with cgroup: host \u2014 without it cAdvisor reports the root cgroup alone.", + "templating": { + "list": [ + { + "name": "project", + "label": "Project", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_uname_info, project)", + "query": { + "qryType": 1, + "query": "label_values(node_uname_info, project)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "env", + "label": "Environment", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_uname_info{project=\"$project\"}, env)", + "query": { + "qryType": 1, + "query": "label_values(node_uname_info{project=\"$project\"}, env)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_uname_info{project=\"$project\", env=\"$env\"}, host_name)", + "query": { + "qryType": 1, + "query": "label_values(node_uname_info{project=\"$project\", env=\"$env\"}, host_name)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Host CPU", + "description": "All cores, aggregated.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{project=\"$project\", env=\"$env\", host_name=\"$host\", mode=\"idle\"}[$__rate_interval])))", + "legendFormat": "cpu", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Host Memory Used", + "description": "Available, not free \u2014 page cache is not pressure.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "100 * (1 - node_memory_MemAvailable_bytes{project=\"$project\", env=\"$env\", host_name=\"$host\"} / node_memory_MemTotal_bytes{project=\"$project\", env=\"$env\", host_name=\"$host\"})", + "legendFormat": "memory", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Filesystem Used", + "description": "HostDiskSpaceLow fires at 80%.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "100 * (1 - node_filesystem_avail_bytes{project=\"$project\", env=\"$env\", host_name=\"$host\", fstype!~\"tmpfs|ramfs|overlay\"} / node_filesystem_size_bytes{project=\"$project\", env=\"$env\", host_name=\"$host\", fstype!~\"tmpfs|ramfs|overlay\"})", + "legendFormat": "{{mountpoint}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Container CPU", + "description": "Per container, by name.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{project=\"$project\", env=\"$env\", host_name=\"$host\", name!=\"\"}[$__rate_interval]))", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Container Memory", + "description": "Working set, which is what the OOM killer reads.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "sum by (name) (container_memory_working_set_bytes{project=\"$project\", env=\"$env\", host_name=\"$host\", name!=\"\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Container Restarts (1h)", + "description": "ContainerRestarting fires above 3.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "changes(container_start_time_seconds{project=\"$project\", env=\"$env\", host_name=\"$host\", name!=\"\"}[1h])", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Host Network", + "description": "Receive and transmit, all non-loopback interfaces.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "sum by (device) (rate(node_network_receive_bytes_total{project=\"$project\", env=\"$env\", host_name=\"$host\", device!=\"lo\"}[$__rate_interval]))", + "legendFormat": "rx {{device}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + } + ] +} diff --git a/dashboards/infrastructure-logs.json b/dashboards/infrastructure-logs.json deleted file mode 100644 index f49dbaa..0000000 --- a/dashboards/infrastructure-logs.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "title": "RELab — Container Logs", - "uid": "relab-infra-logs", - "schemaVersion": 39, - "version": 1, - "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, - "graphTooltip": 1, - "tags": ["relab"], - "templating": { - "list": [ - { - "name": "env", - "label": "Environment", - "type": "custom", - "query": "staging,prod", - "current": { "text": "staging", "value": "staging" }, - "options": [ - { "text": "staging", "value": "staging", "selected": true }, - { "text": "prod", "value": "prod", "selected": false } - ], - "hide": 0 - }, - { - "name": "service", - "label": "Service", - "type": "query", - "datasource": { "type": "loki", "uid": "loki" }, - "definition": "label_values({env=\"$env\"}, service)", - "query": { "type": "labelValues", "label": "service", "stream": "{env=\"$env\"}" }, - "refresh": 2, - "sort": 1, - "multi": true, - "includeAll": true, - "allValue": ".+", - "current": { "text": "All", "value": "$__all" }, - "options": [], - "hide": 0 - } - ] - }, - "panels": [ - { - "type": "timeseries", - "title": "Log Rate by Service", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, - "id": 1, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "sum by (service) (rate({env=\"$env\", service=~\"$service\"}[$__auto]))", - "legendFormat": "{{service}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "custom": { "lineWidth": 2, "fillOpacity": 10 } - } - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "type": "logs", - "title": "All Container Logs", - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 8 }, - "id": 2, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "{env=\"$env\", service=~\"$service\"}", - "refId": "A" - } - ], - "options": { - "dedupStrategy": "none", - "showLabels": true, - "wrapLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending" - } - }, - { - "type": "logs", - "title": "Errors & Warnings", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 20 }, - "id": 3, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "{env=\"$env\", service=~\"$service\"} |~ `(?i)(error|fatal|panic|exception|warn)`", - "refId": "A" - } - ], - "options": { - "dedupStrategy": "none", - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, - "sortOrder": "Descending" - } - } - ], - "annotations": { "list": [] } -} diff --git a/dashboards/logs-overview.json b/dashboards/logs-overview.json deleted file mode 100644 index 90410e9..0000000 --- a/dashboards/logs-overview.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "title": "Logs Overview", - "uid": "logs-overview", - "schemaVersion": 39, - "version": 1, - "refresh": "30s", - "time": { "from": "now-3h", "to": "now" }, - "graphTooltip": 1, - "tags": ["logs", "otlp"], - "templating": { - "list": [ - { - "name": "service", - "label": "Service", - "type": "query", - "datasource": { "type": "loki", "uid": "loki" }, - "definition": "label_values(service_name)", - "query": { "type": "labelValues", "label": "service_name" }, - "refresh": 2, - "sort": 1, - "multi": true, - "includeAll": true, - "allValue": ".+", - "current": { "text": "All", "value": "$__all" }, - "options": [], - "hide": 0 - } - ] - }, - "panels": [ - { - "type": "timeseries", - "title": "Log Rate by Service", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, - "id": 1, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "sum by (service_name) (rate({service_name=~\"$service\"}[$__auto]))", - "legendFormat": "{{service_name}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "logs/s", - "custom": { "lineWidth": 2, "fillOpacity": 10 } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Log Rate by Level", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, - "id": 2, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "sum by (detected_level) (rate({service_name=~\"$service\"}[$__auto]))", - "legendFormat": "{{detected_level}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "logs/s", - "custom": { "lineWidth": 2, "fillOpacity": 10 } - }, - "overrides": [ - { - "matcher": { "id": "byRegexp", "options": "(?i)error" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] - }, - { - "matcher": { "id": "byRegexp", "options": "(?i)(critical|fatal)" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-red" } }] - }, - { - "matcher": { "id": "byRegexp", "options": "(?i)warn.*" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] - }, - { - "matcher": { "id": "byRegexp", "options": "(?i)info" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] - } - ] - } - }, - { - "type": "logs", - "title": "Errors", - "description": "Error-and-worse log lines. Expand one and follow trace_id to the trace in Tempo.", - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 8 }, - "id": 3, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "{service_name=~\"$service\"} | detected_level=~\"(?i)(error|critical|fatal)\"", - "refId": "A" - } - ], - "options": { - "showTime": true, - "wrapLogMessage": true, - "enableLogDetails": true, - "sortOrder": "Descending" - } - }, - { - "type": "logs", - "title": "All Logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 17 }, - "id": 4, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "{service_name=~\"$service\"}", - "refId": "A" - } - ], - "options": { - "showTime": true, - "wrapLogMessage": true, - "enableLogDetails": true, - "sortOrder": "Descending" - } - } - ] -} diff --git a/dashboards/logs.json b/dashboards/logs.json new file mode 100644 index 0000000..73f1caa --- /dev/null +++ b/dashboards/logs.json @@ -0,0 +1,195 @@ +{ + "uid": "logs", + "title": "Logs", + "tags": [ + "logs", + "project" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "refresh": "30s", + "schemaVersion": 39, + "editable": null, + "description": "Every project's container logs. Scoped by the project/env template variables, so a new project appears here the moment its first log line lands \u2014 nobody edits this JSON to onboard one.", + "templating": { + "list": [ + { + "name": "project", + "label": "Project", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values(project)", + "query": "label_values(project)", + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "env", + "label": "Environment", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({project=\"$project\"}, env)", + "query": "label_values({project=\"$project\"}, env)", + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({project=\"$project\", env=\"$env\"}, service_name)", + "query": "label_values({project=\"$project\", env=\"$env\"}, service_name)", + "refresh": 1, + "sort": 1, + "current": { + "text": "All", + "value": "$__all" + }, + "options": [], + "hide": 0, + "includeAll": true, + "allValue": ".+" + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Log Rate by Service", + "description": "Which services are talking, and how much.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "expr": "sum by (service_name) (rate({project=\"$project\", env=\"$env\", service_name=~\"$service\"}[$__auto]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "logs/s", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Log Rate by Level", + "description": "A rising error line is the cheapest early warning here.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "expr": "sum by (detected_level) (rate({project=\"$project\", env=\"$env\", service_name=~\"$service\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "logs/s", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + } + }, + { + "type": "logs", + "title": "Errors & Warnings", + "description": "Level is detected by Loki at ingest, not parsed here.", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "expr": "{project=\"$project\", env=\"$env\", service_name=~\"$service\"} | detected_level=~\"(?i)(error|critical|fatal|warn)\"", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } + }, + { + "type": "logs", + "title": "All Logs", + "description": "Everything, newest first.", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 4, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "expr": "{project=\"$project\", env=\"$env\", service_name=~\"$service\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } + } + ] +} diff --git a/dashboards/relab-api.json b/dashboards/relab-api.json deleted file mode 100644 index b0abee7..0000000 --- a/dashboards/relab-api.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "title": "RELab — API", - "uid": "relab-api", - "schemaVersion": 39, - "version": 1, - "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, - "graphTooltip": 1, - "tags": ["relab"], - "panels": [ - { - "type": "row", - "title": "Logs", - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 1 - }, - { - "type": "timeseries", - "title": "Log Rate by Level", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, - "id": 2, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "sum by (detected_level) (rate({service_name=\"relab-api\"}[$__auto]))", - "legendFormat": "{{detected_level}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "custom": { "lineWidth": 2, "fillOpacity": 10 } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "error" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] - }, - { - "matcher": { "id": "byName", "options": "critical" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-red" } }] - }, - { - "matcher": { "id": "byName", "options": "warning" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] - }, - { - "matcher": { "id": "byName", "options": "info" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] - } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "type": "logs", - "title": "Log Stream", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, - "id": 3, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "{service_name=\"relab-api\"}", - "refId": "A" - } - ], - "options": { - "dedupStrategy": "none", - "showLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending" - } - }, - { - "type": "logs", - "title": "Errors", - "gridPos": { "h": 7, "w": 24, "x": 0, "y": 9 }, - "id": 6, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "{service_name=\"relab-api\"} | detected_level=~\"error|critical\"", - "refId": "A" - } - ], - "options": { - "dedupStrategy": "none", - "showLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true, - "sortOrder": "Descending" - } - }, - { - "type": "row", - "title": "Traces", - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, - "id": 4 - }, - { - "type": "traces", - "title": "Recent Traces", - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 17 }, - "id": 5, - "datasource": { "type": "tempo", "uid": "tempo" }, - "targets": [ - { - "queryType": "traceql", - "query": "{resource.service.name=\"relab-api\"}", - "refId": "A", - "tableType": "traces" - } - ], - "options": { "frameType": "TraceqlSearch" } - }, - { - "type": "traces", - "title": "Failed Requests", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 29 }, - "id": 7, - "datasource": { "type": "tempo", "uid": "tempo" }, - "targets": [ - { - "queryType": "traceql", - "query": "{resource.service.name=\"relab-api\" && status=error}", - "refId": "A", - "tableType": "traces" - } - ], - "options": { "frameType": "TraceqlSearch" } - } - ], - "templating": { "list": [] }, - "annotations": { "list": [] } -} diff --git a/dashboards/service-health.json b/dashboards/service-health.json index d0d6c73..3dd9390 100644 --- a/dashboards/service-health.json +++ b/dashboards/service-health.json @@ -4,21 +4,74 @@ "schemaVersion": 39, "version": 1, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "graphTooltip": 1, - "tags": ["red", "otlp"], + "tags": [ + "red", + "otlp" + ], "templating": { "list": [ + { + "name": "project", + "label": "Project", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_request_duration_seconds_count, project)", + "query": { + "qryType": 1, + "query": "label_values(http_server_request_duration_seconds_count, project)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "env", + "label": "Environment", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_request_duration_seconds_count{project=\"$project\"}, env)", + "query": { + "qryType": 1, + "query": "label_values(http_server_request_duration_seconds_count{project=\"$project\"}, env)", + "refId": "var" + }, + "refresh": 1, + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, { "name": "service", "label": "Service", "type": "query", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(traces_spanmetrics_calls_total, service)", - "query": { "qryType": 1, "query": "label_values(traces_spanmetrics_calls_total, service)", "refId": "var" }, - "refresh": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\"}, job)", + "query": { + "qryType": 1, + "query": "label_values(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\"}, job)", + "refId": "var" + }, + "refresh": 1, "sort": 1, - "current": { "text": "demo-api", "value": "demo-api" }, + "current": {}, "options": [], "hide": 0 } @@ -27,22 +80,33 @@ "panels": [ { "type": "timeseries", - "title": "Request Rate by Endpoint", - "description": "Server spans per second, from Tempo span metrics — works for any service that sends traces.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "title": "Request Rate by Route", + "description": "Requests per second per route, from the app's own OTLP metrics.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, "id": 1, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", - "legendFormat": "{{span_name}}", + "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval]))", + "legendFormat": "{{http_route}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "reqps", - "custom": { "lineWidth": 2, "fillOpacity": 10 } + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } }, "overrides": [] } @@ -51,12 +115,20 @@ "type": "timeseries", "title": "Error Rate", "description": "Share of server spans with STATUS_CODE_ERROR.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "100 * sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "expr": "100 * sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\", http_response_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval]))", "legendFormat": "errors", "refId": "A" } @@ -65,35 +137,57 @@ "defaults": { "unit": "percent", "min": 0, - "color": { "mode": "fixed", "fixedColor": "red" }, - "custom": { "lineWidth": 2, "fillOpacity": 10 }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] } + "color": { + "mode": "fixed", + "fixedColor": "red" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } }, "overrides": [] } }, { "type": "timeseries", - "title": "Latency (from traces, with exemplars)", - "description": "Quantiles over Tempo span-metrics latency. Dots are exemplars — click one to open the exact trace in Tempo.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "title": "Latency percentiles", + "description": "Quantiles over Tempo span-metrics latency. Dots are exemplars \u2014 click one to open the exact trace in Tempo.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, "id": 3, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "exemplar": true }, { - "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B", "exemplar": true }, { - "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C", "exemplar": true @@ -102,21 +196,32 @@ "fieldConfig": { "defaults": { "unit": "s", - "custom": { "lineWidth": 2, "fillOpacity": 5 } + "custom": { + "lineWidth": 2, + "fillOpacity": 5 + } }, "overrides": [] } }, { "type": "timeseries", - "title": "HTTP Server Duration p95 (app SDK metrics)", - "description": "From the service's own OTLP metrics (OTel SDK), i.e. the collector → Prometheus OTLP path.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "title": "Latency p95 by route", + "description": "From the service's own OTLP metrics (OTel SDK), i.e. the collector \u2192 Prometheus OTLP path.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", "legendFormat": "{{http_route}}", "refId": "A" } @@ -124,7 +229,10 @@ "fieldConfig": { "defaults": { "unit": "s", - "custom": { "lineWidth": 2, "fillOpacity": 10 } + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } }, "overrides": [] } @@ -133,12 +241,20 @@ "type": "logs", "title": "Logs", "description": "OTLP logs for the service. Expand a line and follow trace_id to jump to the trace in Tempo.", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 16 + }, "id": 5, - "datasource": { "type": "loki", "uid": "loki" }, + "datasource": { + "type": "loki", + "uid": "loki" + }, "targets": [ { - "expr": "{service_name=\"$service\"}", + "expr": "{project=\"$project\", env=\"$env\", service_name=~\"$service\"}", "refId": "A" } ], @@ -149,5 +265,6 @@ "sortOrder": "Descending" } } - ] + ], + "description": "RED for any onboarded project, from the applications' own OTLP metrics (ADR 0002 moved this off Tempo's span-metrics generator). The project/env variables mean a new service appears here without anyone editing JSON." } diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json index 422bfc0..7700725 100644 --- a/dashboards/stack-health.json +++ b/dashboards/stack-health.json @@ -4,20 +4,33 @@ "schemaVersion": 39, "version": 1, "refresh": "30s", - "time": { "from": "now-3h", "to": "now" }, + "time": { + "from": "now-3h", + "to": "now" + }, "graphTooltip": 1, - "tags": ["infra"], + "tags": [ + "infra" + ], "panels": [ { "type": "timeseries", "title": "Filesystem Used", - "description": "Loki and Tempo have no total-size cap — the HostDiskSpaceLow alert fires at 80%.", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, + "description": "Loki and Tempo have no total-size cap \u2014 the HostDiskSpaceLow alert fires at 80%.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, "id": 1, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|ramfs|overlay\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|ramfs|overlay\"})", + "expr": "100 * (1 - node_filesystem_avail_bytes{job=\"node\",fstype!~\"tmpfs|ramfs|overlay\"} / node_filesystem_size_bytes{job=\"node\",fstype!~\"tmpfs|ramfs|overlay\"})", "legendFormat": "{{mountpoint}}", "refId": "A" } @@ -27,8 +40,23 @@ "unit": "percent", "min": 0, "max": 100, - "custom": { "lineWidth": 2, "fillOpacity": 10 }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 80 }] } + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, "overrides": [] } @@ -36,12 +64,20 @@ { "type": "timeseries", "title": "Host CPU", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",mode=\"idle\"}[$__rate_interval])))", "legendFormat": "used", "refId": "A" } @@ -51,7 +87,10 @@ "unit": "percent", "min": 0, "max": 100, - "custom": { "lineWidth": 2, "fillOpacity": 10 } + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } }, "overrides": [] } @@ -59,12 +98,20 @@ { "type": "timeseries", "title": "Host Memory Used", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, "id": 3, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "expr": "100 * (1 - node_memory_MemAvailable_bytes{job=\"node\"} / node_memory_MemTotal_bytes{job=\"node\"})", "legendFormat": "used", "refId": "A" } @@ -74,7 +121,10 @@ "unit": "percent", "min": 0, "max": 100, - "custom": { "lineWidth": 2, "fillOpacity": 10 } + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } }, "overrides": [] } @@ -83,9 +133,17 @@ "type": "timeseries", "title": "Collector Ingest Rate", "description": "Items accepted by the OTLP receivers, per signal.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", @@ -106,7 +164,10 @@ "fieldConfig": { "defaults": { "unit": "ops", - "custom": { "lineWidth": 2, "fillOpacity": 10 } + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } }, "overrides": [] } @@ -114,10 +175,18 @@ { "type": "timeseries", "title": "Collector Export Failures", - "description": "Anything above zero means a backend is rejecting or unreachable — the OtelExportFailures alert covers this.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "description": "Anything above zero means a backend is rejecting or unreachable \u2014 the OtelExportFailures alert covers this.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, "id": 5, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", @@ -129,47 +198,85 @@ "defaults": { "unit": "ops", "min": 0, - "color": { "mode": "fixed", "fixedColor": "red" }, - "custom": { "lineWidth": 2, "fillOpacity": 10 } + "color": { + "mode": "fixed", + "fixedColor": "red" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } }, "overrides": [] } }, { - "type": "table", - "title": "Active Alerts", - "description": "Pending and firing alerts (Watchdog excluded — it fires by design).", - "gridPos": { "h": 6, "w": 24, "x": 0, "y": 16 }, - "id": 6, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "type": "timeseries", + "title": "Ingest by Project", + "description": "Telemetry accepted per project and environment, from the gateway's count connector. A project missing here is sending nothing, whichever signal it uses.", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { - "expr": "ALERTS{alertname!=\"Watchdog\"}", - "instant": true, - "format": "table", + "expr": "sum by (project, env) (rate(telemetry_datapoints_total[$__rate_interval]))", + "legendFormat": "{{project}}/{{env}} metric points/s", "refId": "A" - } - ], - "transformations": [ + }, { - "id": "organize", - "options": { - "excludeByName": { "Time": true, "Value": true, "__name__": true }, - "indexByName": { "alertname": 0, "alertstate": 1, "severity": 2 } - } + "expr": "sum by (project, env) (rate(telemetry_logs_total[$__rate_interval]))", + "legendFormat": "{{project}}/{{env}} log records/s", + "refId": "B" + }, + { + "expr": "sum by (project, env) (rate(telemetry_spans_total[$__rate_interval]))", + "legendFormat": "{{project}}/{{env}} spans/s", + "refId": "C" } ], "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { "id": "byName", "options": "alertstate" }, - "properties": [ - { "id": "custom.cellOptions", "value": { "type": "color-text" } }, - { "id": "mappings", "value": [{ "type": "value", "options": { "firing": { "color": "red" }, "pending": { "color": "orange" } } }] } - ] + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 } - ] + }, + "overrides": [] + } + }, + { + "type": "alertlist", + "title": "Active Alerts", + "id": 6, + "description": "Grafana-managed alerts. The old panel queried Prometheus's ALERTS series, which only exists while Prometheus evaluates rules \u2014 it stopped existing when alerting moved to Grafana (ADR 0002).", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 24 + }, + "options": { + "viewMode": "list", + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "alertInstanceLabelFilter": "{alertname!=\"Watchdog\"}", + "stateFilter": { + "firing": true, + "pending": true, + "noData": false, + "normal": false, + "error": true + } } } ] diff --git a/demo/Dockerfile b/demo/Dockerfile index 5b429ee..847e9f8 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,13 +1,14 @@ -FROM python:3.14-slim AS builder -COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /bin/ +FROM python:3.14.7-slim@sha256:cad9a2c871761c413caa6fdd6441c783451e740a48aaeba60ae62a8b53525ef6 AS builder +COPY --from=ghcr.io/astral-sh/uv:0.12.1 /uv /bin/ WORKDIR /app COPY pyproject.toml . RUN uv pip install --system --no-cache -r pyproject.toml -FROM python:3.14-slim +FROM python:3.14.7-slim@sha256:cad9a2c871761c413caa6fdd6441c783451e740a48aaeba60ae62a8b53525ef6 COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages COPY --from=builder /usr/local/bin /usr/local/bin WORKDIR /app -COPY app.py . +COPY --chown=nobody:nogroup app.py . +USER nobody CMD ["opentelemetry-instrument", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/demo/__init__.py b/demo/__init__.py deleted file mode 100644 index fd3aec1..0000000 --- a/demo/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Demo telemetry source for the monitoring stack (see compose.demo.yml).""" diff --git a/demo/app.py b/demo/app.py index a43c6ac..43ad079 100644 --- a/demo/app.py +++ b/demo/app.py @@ -1,7 +1,6 @@ """Minimal FastAPI service for the demo overlay. -All telemetry (traces, metrics, logs with trace context) comes from OTel -auto-instrumentation — see compose.demo.yml. No OTel code needed here. +All telemetry comes from OTel auto-instrumentation; see compose.demo.yml. """ import logging @@ -10,9 +9,9 @@ from fastapi import FastAPI, HTTPException -ERROR_RATE = 0.1 # fixed error rate, enough to light up RED panels +ERROR_RATE = 0.1 -logging.basicConfig(level=logging.INFO) # root logger defaults to WARNING; we want the INFO lines too +logging.basicConfig(level=logging.INFO) log = logging.getLogger("demo-api") app = FastAPI() @@ -26,7 +25,7 @@ def root() -> dict[str, bool]: @app.get("/work") def work() -> dict[str, bool]: """Simulate variable-latency work that sometimes fails.""" - time.sleep(random.uniform(0.02, 0.3)) # noqa: S311 — not crypto, just jitter + time.sleep(random.uniform(0.02, 0.3)) # noqa: S311 if random.random() < ERROR_RATE: # noqa: S311 log.error("work failed: upstream flaked") raise HTTPException(status_code=500, detail="upstream flaked") diff --git a/demo/pyproject.toml b/demo/pyproject.toml index 7a86bbc..97bea80 100644 --- a/demo/pyproject.toml +++ b/demo/pyproject.toml @@ -1,11 +1,23 @@ [project] -name = "demo" -version = "0.1.0" -requires-python = ">=3.14" -dependencies = [ - "fastapi==0.139.0", - "uvicorn==0.50.0", - "opentelemetry-distro==0.64b0", - "opentelemetry-exporter-otlp==1.43.0", - "opentelemetry-instrumentation-fastapi==0.64b0", -] + dependencies = [ + "fastapi==0.141.1", + "uvicorn==0.52.4", + "opentelemetry-distro==0.65b0", + "opentelemetry-exporter-otlp==1.44.0", + "opentelemetry-instrumentation-fastapi==0.65b0", + ] + name = "demo" + requires-python = ">=3.14" + version = "0.1.0" + +# The ruff config lives here so lint results do not depend on whatever config the +# contributor has at home. target-version is explicit because requires-python is +# ahead of what released ruff knows about. +[tool.ruff] + target-version = "py313" + + [tool.ruff.lint] + pydocstyle.convention = "google" + select = ["ALL"] + # COM812 fights the formatter (`ruff format --check` runs in `just lint`). + ignore = ["COM812"] diff --git a/docs/ONBOARDING.md b/docs/ONBOARDING.md index e6e0a88..eebdc7d 100644 --- a/docs/ONBOARDING.md +++ b/docs/ONBOARDING.md @@ -1,42 +1,44 @@ -# Sending telemetry from your project +# Sending telemetry from your application -This stack collects logs, traces, and metrics from CML projects and shows -them side by side in one Grafana. Getting your project in takes an -endpoint, a token, and a couple of naming conventions. Then pick the -template that matches how your project runs — if it's a Python/FastAPI -service, Template 1 needs no code changes at all. +You need an endpoint, a token, and a few naming conventions. Then pick the +template that matches how your application runs. A Python/FastAPI service +needs no code changes. ## The endpoint | | | | --- | --- | -| Production (via tunnel) | `https://otlp.` — OTLP **HTTP** (`http/protobuf`) only | +| Production (via tunnel) | `https://otel.`, OTLP **HTTP** (`http/protobuf`) only | | Private network / same host | `:4317` (gRPC) or `:4318` (HTTP) | | Auth | `Authorization: Bearer ` (ask the stack operator) | -The tunnel only routes HTTPS to the collector's HTTP receiver; there is no -public gRPC path. When sending through it, set -`OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`. gRPC works on private paths -only (VPN/WireGuard, or the same Docker network). Never expose 4317/4318 -directly. +The tunnel routes HTTPS to the collector's HTTP receiver only. Set +`OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` when you send through it. gRPC +works on private paths only (VPN, WireGuard, or the same Docker network). +Never expose 4317/4318 directly. ## The conventions - **`service.name`** is required: one stable name per deployable unit (`relab-api`, not `relab-api-prod-2`). Dashboards key on it. -- **`env`** is `prod`, `staging`, or `dev`, set as a resource attribute. -- **Keep labels low-cardinality.** Loki and Prometheus index labels, and - every distinct value creates a new stream or series. User IDs, request - IDs, and timestamps therefore don't belong in resource attributes or log - labels. Put them in the log line or in span attributes instead — you can - still filter on them at query time, without the storage blowing up. - -Traces are the most valuable signal to send. Tempo derives request-rate, -error-rate, and duration ("RED") metrics from them, so a service that -sends only traces already gets the Service Health dashboard and error -alerting. Start with traces; everything else is a bonus. - -## Template 1 — Python/FastAPI, zero code changes +- **`project`** and **`env`** are both required resource attributes: `project` + is the name `bootstrap.sh` was run with, `env` is `prod`, `staging`, or + `dev`. Together they are what every alert and dashboard filters on, and a + sender that omits them is counted as `unknown` and alerts as an uncovered + project. +- **Keep labels low-cardinality.** Prometheus turns every distinct label + value into a series. User IDs, request IDs, and timestamps belong in the + log line or in span attributes, not in resource attributes or metric labels. +- **Loki indexes only the identity labels**: `service.name`, `department`, + `project`, `env`, `host.name` (the list is in `config/loki.yaml`). Every + other attribute is structured metadata. Select the stream first, then + filter: `{service_name="my-service", env="prod"} | service_instance_id="..."`. + +The Service Health dashboard and the error-rate alert read the standard HTTP +server metrics (`http_server_request_duration_seconds`). The +auto-instrumentation below emits them. + +## Template 1: Python/FastAPI, zero code changes ```sh pip install opentelemetry-distro opentelemetry-exporter-otlp opentelemetry-instrumentation-fastapi @@ -44,8 +46,8 @@ pip install opentelemetry-distro opentelemetry-exporter-otlp opentelemetry-instr ```sh export OTEL_SERVICE_NAME=my-service -export OTEL_RESOURCE_ATTRIBUTES=env=prod -export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.example.org +export OTEL_RESOURCE_ATTRIBUTES=project=my-project,env=prod +export OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.org export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true @@ -54,81 +56,25 @@ export OTEL_SEMCONV_STABILITY_OPT_IN=http opentelemetry-instrument uvicorn app:app --host 0.0.0.0 --port 8000 ``` -That's the whole integration: traces, RED metrics, and logs that carry -their trace context, without a line of OTel code in the app. The working -example is this repo's own [`demo/`](../demo/) service plus -[`compose.demo.yml`](../compose.demo.yml). +That gives traces, RED metrics, and logs with trace context, with no OTel +code in the app. A working example is this repo's [`demo/`](../demo/) service +plus [`compose.demo.yml`](../compose.demo.yml). -## Template 2 — any language, plain OTLP +## Template 2: any language, plain OTLP -Every OpenTelemetry SDK understands the same four environment variables: +Every OpenTelemetry SDK reads the same four environment variables: ```sh OTEL_SERVICE_NAME=my-service -OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.example.org +OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.org OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " -OTEL_RESOURCE_ATTRIBUTES=env=prod -``` - -## Template 3 — Docker container logs (Loki driver) - -This ships container stdout/stderr without touching the app. It needs a -Loki push URL, which **this stack does not expose by default**: Loki has -no authentication of its own, so a push hostname must first be added to -`infra/main.tf` and protected with a Cloudflare Access service token, or -reached over a private network path (VPN/WireGuard). If in doubt, use the -OTLP log path from Templates 1–2 instead. - -```sh -# once per host -docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions -``` - -```yaml -# per service, in compose.yml -logging: - driver: loki - options: - loki-url: ${LOKI_URL} # e.g. https://logs.example.org/loki/api/v1/push - loki-external-labels: service={{.Name}},env=prod,host=myhost +OTEL_RESOURCE_ATTRIBUTES=project=my-project,env=prod ``` -RELab's `compose.logging.loki.yml` overlay, auto-included when `LOKI_URL` -is set, is the reference implementation of this pattern. - -## Template 4 — host or file logs (Grafana Alloy) - -For log files that live outside containers. (Promtail is end-of-life; -Alloy is its successor.) - -```alloy -// alloy/config.alloy -local.file_match "app" { - path_targets = [{ __path__ = "/var/log/myapp/*.log", service = "myapp", env = "prod", host = "myhost" }] -} - -loki.source.file "app" { - targets = local.file_match.app.targets - forward_to = [loki.write.central.receiver] -} - -loki.write "central" { - endpoint { - url = "https://logs.example.org/loki/api/v1/push" - } -} -``` - -```yaml -# compose service -alloy: - image: grafana/alloy:v1.13.0 - restart: unless-stopped - command: [ "run", "/etc/alloy/config.alloy" ] - volumes: - - ./alloy/config.alloy:/etc/alloy/config.alloy:ro - - /var/log/myapp:/var/log/myapp:ro -``` +## Container logs and host metrics -The same caveat as Template 3 applies: the Loki push URL needs a protected -network path. +An application cannot report other containers' stdout, host resources, or its +own crash loops. One Grafana Alloy agent per host ships those, over the same +endpoint and token. Run `./bootstrap.sh ` on the monitoring +host and follow what it prints. Details are in +[templates/README.md](../templates/README.md). diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 9bf3fb9..4d4c146 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -1,48 +1,57 @@ # Runbook -How to operate this stack. All commands run from the repo root on the -monitoring host. Start with `just ps` and the **Stack Health** dashboard; -between them they answer most "what is wrong" questions. +All commands run from the repo root on the monitoring host. Start with +`just ps` and the **Stack Health** dashboard. Between them they answer most +"what is wrong" questions. ![Stack Health dashboard](img/stack-health.png) -That capture is honest, not staged: the red export-failure spike is a real -Tempo outage after a bad major-version bump, and the gap in the ingest -panel is a backup/restore drill. Both procedures are below. +In that capture, the red export-failure spike is a Tempo outage and the gap in +the ingest panel is a backup/restore drill. ## A service is down or misbehaving ```sh just ps # what's running, what's restarting -just logs # follow logs (otel-collector, loki, tempo, prometheus, grafana, alertmanager) +just logs # follow logs (otel-collector, loki, tempo, prometheus, grafana) just restart ``` -Two alerts point here. `TargetDown` fires after two minutes when -Prometheus can't scrape the collector, node-exporter, or itself. -`OtelExportFailures` means the collector is up but a backend is rejecting -its data, so look at that backend's logs, not the collector's. +Two alerts point here. `TargetDown` fires after two minutes when Prometheus +cannot scrape a service. It scrapes every service, so the alert names the one +that went quiet. `OtelExportFailures` means the collector is up but a backend +is rejecting its data. Read that backend's logs, not the collector's. ## Disk filling up (`HostDiskSpaceLow`) -Retention is only partially size-bounded, and that's by design rather than -oversight: +Retention is only partially size-bounded: | Data | Time limit | Size limit | | --- | --- | --- | -| Container stdout logs | — | json-file 10m × 3 per service | +| Container stdout logs | none | json-file 10m × 3 per service | | Prometheus TSDB | 30d | 15GB (`--storage.tsdb.retention.size`) | -| Loki chunks | 30d | none — Loki cannot cap total size | +| Loki chunks | 30d | none | | Tempo blocks | 7d | none | -Loki and Tempo simply have no total-size knob, which is why the disk alert -at 80% is the real backstop. When it fires: check the Filesystem panel on -Stack Health, then either free space or shorten a retention window -(`retention_period` in `config/loki.yaml`, `block_retention` in -`config/tempo.yaml`, the `--storage.tsdb.retention.*` flags in -`compose.yml`) and restart the affected service. If disk pressure keeps -coming back, the durable fix is moving Loki and Tempo to object storage — -see `compose.storage-s3.yml`. +Loki and Tempo cannot cap their total size, so the disk alert at 80% is the +backstop. Two warnings fire earlier. `HostDiskFilling` means a 6-hour linear +fit says a filesystem is full within 3 days. `PrometheusCardinalityHigh` +means active series passed 100k, about 14x the baseline of ~7k with one +spoke. Series count, not time, is what grows the TSDB. + +When one fires: + +1. Check the Filesystem panel on Stack Health. +2. Free space, or shorten a retention window and restart the service: + `retention_period` in `config/loki.yaml`, `block_retention` in + `config/tempo.yaml`, or the `--storage.tsdb.retention.*` flags in + `compose.yml`. +3. If disk pressure keeps returning, move Loki and Tempo to object storage. + The appendix of ADR 0001 describes the change. + +Memory is bounded per service: every service has a `mem_limit` in +`compose.yml`, sized from observed usage. Raise it there if a component +legitimately grows into its ceiling. ## Backup and restore @@ -52,56 +61,161 @@ just restore backups/monitoring-.tar.gz # stops the stack, wipes volumes, just up ``` -Backups are crash-consistent: restoring one is equivalent to recovering -from a power loss, which every component in the stack does cleanly via its -write-ahead log. Two practical notes: +Backups are crash-consistent. Restoring one is like recovering from a power +loss, which every component handles through its write-ahead log. -- The tarball is mode 0600 and contains secrets (the Grafana database - among them). Copy it off-host over a private channel — a backup on the - disk it protects is a decoration. -- During the pause the collector keeps accepting telemetry and buffers it - for about five minutes. A backup that takes longer than that will drop - data, so on large volumes run it at a quiet hour. +- The tarball is mode 0600 and contains secrets, including the Grafana + database. Copy it off-host over a private channel. +- `just restore` refuses to run if the stack's volumes do not exist yet. On a + fresh host, run `just up` once to create them, then restore. +- It covers the docker volumes only. The OpenTofu state for the Cloudflare + edge is not in it. See "Where every secret lives" below. +- During the pause the collector buffers incoming telemetry for five minutes + (`retry_on_failure.max_elapsed_time` in `config/otel-collector.yaml`). A + backup that runs longer drops data. On large volumes, run it at a quiet + hour. -How much history you can lose equals how often you run it. A daily cron on -the host is the intended setup. +Worst-case loss equals the interval between backups. Run it from a daily cron +on the host. ## Rotating secrets -- **OTLP token:** set the new `OTLP_AUTH_TOKEN` in `.env`, then +- **OTLP token:** set the new `OTLP_AUTH_TOKEN` in `.env`, run `docker compose up -d otel-collector`, then update every sender's - `OTEL_EXPORTER_OTLP_HEADERS`. Senders still on the old token get 401s - (visible as export errors on their side) until they're updated. A - running demo overlay counts as a sender: re-run `just demo` to recreate - it with the new token. -- **Tunnel token:** rotate in Cloudflare Zero Trust, set the new - `CLOUDFLARE_TUNNEL_TOKEN` in `.env`, then `just up-tunnel`. + `OTEL_EXPORTER_OTLP_HEADERS`. Senders on the old token get 401s until + updated. A running demo overlay is a sender too: re-run `just demo`. + The token is shared, and the collector does not check `project` or `env` + against the sender. Every project host can therefore spoof another + project's labels. +- **Tunnel token:** rotate the tunnel secret in Cloudflare Zero Trust, then + run `cd infra && tofu apply` to refresh the token data source, then read + the new value with `tofu output -raw tunnel_token`. To rotate from code + instead, run `tofu apply -replace=cloudflare_zero_trust_tunnel_cloudflared.monitoring`. + That builds a new tunnel and repoints both CNAMEs at it, and ingestion and + Grafana are unreachable for about a minute. Either way, put the new token + in `CLOUDFLARE_TUNNEL_TOKEN` in `.env`, then `just up`. - **Grafana admin password:** change `GRAFANA_ADMIN_PASSWORD` in `.env`, then `docker compose up -d grafana`. +- **Alert webhook and heartbeat URLs:** the URL is the credential. Mint a new + topic or check at the provider, put it in `ALERT_WEBHOOK_URL` or + `HEARTBEAT_URL`, then `docker compose up -d grafana`. Grafana reads the + contact points only at startup. +- **healthchecks.io API key:** regenerate it in the project's settings and + set `HEALTHCHECKS_API_KEY` in `.env`. Only `bootstrap.sh` reads it, so + nothing restarts. +- **Cloudflare API token:** it is never stored here. Create a new one with + the same three permissions, revoke the old one, and export the new value + before the next `tofu` run. + +### Where every secret lives + +Three gitignored files hold everything. `just backup` archives none of them, +but the tarball contains Grafana's database, and Grafana stores the expanded +contact points there. So the webhook and heartbeat URLs are inside every +backup. Copy `.env` and `infra/terraform.tfstate` off-host together with the +backups and treat all three the same way. + +| Secret | Lives in | Comes from | +| --- | --- | --- | +| `OTLP_AUTH_TOKEN` | `.env` | `openssl rand -hex 32` | +| `GRAFANA_ADMIN_PASSWORD` | `.env` | you | +| `ALERT_WEBHOOK_URL`, `HEARTBEAT_URL` | `.env` | the notification provider | +| `HEALTHCHECKS_API_KEY` | `.env` | healthchecks.io project settings | +| `CLOUDFLARE_TUNNEL_TOKEN` | `.env` | `tofu output -raw tunnel_token` | +| `CF_ACCESS_AUD` | `.env` (not secret, but paired) | `tofu output -raw grafana_access_aud` | +| Tunnel secret, API responses | `infra/terraform.tfstate` | written by every `tofu apply` | +| `CLOUDFLARE_API_TOKEN` | your shell, per session | Cloudflare dashboard | + +`infra/terraform.tfvars` holds identifiers only (account, zone, domain, the +Access email list). It is gitignored for privacy, not because it holds a +credential. ## Alert delivery -Prometheus evaluates the rules; Alertmanager delivers them. Two -environment variables control where: +Grafana evaluates and delivers the rules. Rules, contact points, and the +routing tree are provisioned from `config/grafana/alerting/`, so the UI shows +them read-only. Edit the YAML. + +- `ALERT_WEBHOOK_URL` receives all alerts. +- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five minutes. + Point it at a dead man's switch that raises the alarm when pings stop. -- `ALERT_WEBHOOK_URL` receives all alerts (any webhook: ntfy, Slack, …). -- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five - minutes. Point it at a dead man's switch (e.g. healthchecks.io) that - raises the alarm when pings **stop** — that is the "monitoring host is - dead" signal nothing inside the host can send. +Leaving either empty is not safe. Delivery fails silently while the heartbeat +keeps pinging, so the switch reads healthy and every real alert is dropped. +With the tunnel overlay active, `just up` refuses to start without +`ALERT_WEBHOOK_URL`, and `AlertDeliveryFailing` fires on a failing notifier. -Leaving both empty is fine: nothing is delivered, Alertmanager logs one -notify error per cycle (expected, harmless), and alerts remain visible in -Grafana. After changing either variable, `docker compose up -d -alertmanager`. +After changing either variable, run `docker compose up -d grafana`. A plain +`restart` keeps the old environment. Compose rebuilds a container's +environment only on `up`. + +## Changing the Cloudflare edge + +The tunnel, its ingress rules, both DNS records, and the Access policy in +front of Grafana are all OpenTofu in `infra/`. Change them there, not in the +Zero Trust dashboard. The next apply reverts anything clicked in by hand. + +```sh +just infra-validate # tofu init + validate, in a container +cd infra && tofu plan # needs CLOUDFLARE_API_TOKEN exported +cd infra && tofu apply +``` + +- **Granting or revoking Grafana access:** edit `grafana_allowed_emails` in + `infra/terraform.tfvars` and apply. That list is the entire allowlist. A + removed address keeps working until its Access session expires (24h). For + an urgent revocation, also revoke the session in Zero Trust. The variable + rejects an empty list, which would lock everyone out. Before adding people, + check your plan's Zero Trust seat limit in the Cloudflare dashboard. +- **Per-user Grafana logins:** by default everyone who clears Access shares + the admin password. Set `GRAFANA_JWT_AUTH=true`, + `CF_ACCESS_TEAM_DOMAIN=`, and `CF_ACCESS_AUD` (from + `tofu output -raw grafana_access_aud`) in `.env`. Grafana then verifies + the Access JWT, and each address signs in as itself with the Viewer role. + The aud pin is required. The JWK set is team-wide, so without it a token + minted for any other Access app in the team would be accepted here. +- **Adding a hostname:** add an `ingress` entry pointing at the service's + container port, plus a matching `cloudflare_dns_record`. The catch-all + `http_status:404` entry stays last, or it swallows everything after it. +- **First apply against an edge built by hand:** an empty state plans the + existing tunnel, DNS records, and Access app as "create", and applying + that mints duplicates. Run `infra/generate-imports.sh > infra/imports.tf` + first, check the plan reads 0 to add for the imported resources, apply, + then delete `imports.tf`. +- **State lives on this host only, and it is a secret.** It stores the + tunnel secret and every API response in plain text. Copy + `infra/terraform.tfstate` off-host next to the backups. Losing it orphans + the Cloudflare resources: they keep running, but the next apply creates + duplicates. Recovery is the same import path as the first apply above, + `infra/generate-imports.sh > infra/imports.tf`, which reads the live objects + back out of the Cloudflare API and adopts them into a fresh state. ## Upgrading images -Dependabot opens PRs that bump the pinned versions, and CI runs `just -check` on each one. The validators (promtool, otelcol, amtool) read their -image versions from `compose.yml`, so every bump is checked with the exact -binaries the stack will run — when a new version changes its config -syntax, CI fails loudly before the change reaches the host. That is the -point. +Dependabot opens PRs that bump the pinned versions, and CI runs `just validate` +and `just smoke` on each one. The validators read their image versions from +`compose.yml`, so every bump is checked with the exact binaries the stack +will run. Patch bumps arrive grouped. A minor or major comes on its own, so a +red PR names the one image that broke. + +Dependabot also watches the Cloudflare provider in `infra/`. Those PRs need +one manual step: it bumps the constraint in `main.tf` but not the hashes in +`.terraform.lock.hcl`. Check the branch out, run `cd infra && tofu init +-upgrade`, then `just infra-validate` and `tofu plan` against the real +account. Only a plan proves the provider still maps the config to the same +resources. + +Nothing watches the tool images pinned in the `justfile` (yamllint, +actionlint, shellcheck, ruff, gitleaks, OpenTofu, jq, Alloy, alpine). Bump +those by hand. The alpine pin appears twice: in the `justfile` and on +`otel-queue-init` in `compose.yml`. Bump both together. + +After merging, on the host: + +```sh +git pull && just pull && just up +``` -After merging: on the host, `git pull && just pull && just up`. +`just up` adds the exposure guards, but a plain `docker compose up -d` is now +safe too: the `otel-queue-init` service chowns the collector's queue volume to +uid 10001 before the collector starts. diff --git a/docs/adr/0001-observability-stack.md b/docs/adr/0001-observability-stack.md index bf0357c..73ff9a5 100644 --- a/docs/adr/0001-observability-stack.md +++ b/docs/adr/0001-observability-stack.md @@ -5,9 +5,9 @@ Date: 2026-07-03. Status: accepted (records a decision already in production). ## Context CML runs several long-lived research platforms (RELab and others) that need -their logs, traces, and metrics in one place. The volume is modest — a handful -of services at single-digit requests per second — but one small team maintains -all of it, so whatever we run must stay cheap and auditable. +their logs, traces, and metrics in one place. The volume is modest: a handful +of services at single-digit requests per second. One small team maintains all of +it, so whatever we run must stay cheap and auditable. ## Decision @@ -38,11 +38,53 @@ endpoints via Cloudflare Tunnel; bind everything else to `127.0.0.1`. ## Consequences -- The host is a single point of failure — acceptable, because the monitored +- The host is a single point of failure. That is acceptable: the monitored platforms degrade gracefully when telemetry stops (OTLP export is - fire-and-forget) and the stack rebuilds from this repo in minutes. + fire-and-forget), and the stack rebuilds from this repo in minutes. - Local disk bounds retention (30d logs/metrics, 7d traces). The escape hatch, reached before any move to distributed ingest, is S3-compatible storage for - Loki and Tempo — see `compose.storage-s3.yml`. + Loki and Tempo (see the appendix below). - Every image is pinned and validated by `just check` in CI, so the stack stays reproducible. + +## Appendix: the S3 storage escape hatch + +When local volumes stop fitting, Loki and Tempo move their object storage to +any S3-compatible backend (Cloudflare R2, Backblaze B2, Hetzner, MinIO) +without touching the collector, Prometheus, or any client project. It is not +wired up: don't start until credentials and a bucket exist. The concrete shape: + +1. Create s3 variants of the configs. `config/loki.s3.yaml` replaces + `common.storage.filesystem` with: + + ```yaml + common: + storage: + s3: + endpoint: ${S3_ENDPOINT} # e.g. .r2.cloudflarestorage.com + bucketnames: cml-loki + access_key_id: ${S3_ACCESS_KEY_ID} + secret_access_key: ${S3_SECRET_ACCESS_KEY} + s3forcepathstyle: true + ``` + + and `config/tempo.s3.yaml` replaces `storage.trace.backend: local` with: + + ```yaml + storage: + trace: + backend: s3 + s3: + endpoint: ${S3_ENDPOINT} + bucket: cml-tempo + access_key: ${S3_ACCESS_KEY_ID} + secret_key: ${S3_SECRET_ACCESS_KEY} + ``` + +2. Add a `compose.storage-s3.yml` overlay to `COMPOSE_FILE` in `.env` that + mounts the s3 config variants over the originals and re-declares each + service's `command` with `-config.expand-env=true` appended. Neither Loki + nor Tempo expands `${...}` in its config by default, and compose replaces + `command` wholesale rather than merging it. Pass `S3_ENDPOINT`, + `S3_ACCESS_KEY_ID`, and `S3_SECRET_ACCESS_KEY` through each service's + `environment` with `:?` guards, then `just up`. diff --git a/docs/adr/0002-hub-and-spoke-observability.md b/docs/adr/0002-hub-and-spoke-observability.md new file mode 100644 index 0000000..df9e716 --- /dev/null +++ b/docs/adr/0002-hub-and-spoke-observability.md @@ -0,0 +1,105 @@ +# ADR 0002: Hub-and-spoke observability for CML projects + +Date: 2026-08-20. Status: accepted; **migration complete 2026-08-28**. The +transitional HANDOVER.md that tracked it has been deleted: it described a +transition, not a system. Onboarding is [templates/README.md](../../templates/README.md). + +Supersedes one decision from ADR 0001: RED metrics move off Tempo's span-metrics +and onto the applications' native OTLP HTTP metrics. Everything else in ADR 0001 +stands. + +## Context + +This stack was built for RELab and must now serve multiple CML projects at very +different maturity levels, including GPU hosts for computer-vision work. One +part-time operator, Docker Compose everywhere, zero budget. The formative +incident: a backup container crash-looped 668 times over 19 hours while every +monitor read green. The failure modes that matter are the ones with no detector +at all. + +## Decision + +Three tiers, each owning distinct signals: + +- **Per-project host (spoke):** one Grafana Alloy agent per host, covering + container stdout, host metrics (node exporter), and container + lifecycle/resources (cAdvisor). The application's own OTel SDK adds traces and + app metrics. The agent config is one shared file published by this repo, + parameterised only by environment variables; no project ever edits it. systemd + timers run scheduled jobs (backups, checks) through a wrapper that pings a + per-job dead-man's switch. +- **Central host (hub):** this stack. One OTLP/HTTP endpoint, one bearer token, + no per-backend hostnames or credentials, ever. Grafana is the *single* home + for alert rules and notification (Alertmanager and Prometheus rule files go + away, because Grafana-managed rules can query Loki, which the most valuable + alerts need). +- **Outside everything:** healthchecks.io as the per-job dead-man's switch, and + an external HTTP prober for public reachability. These are the only detectors + whose default state is alarm; everything else fails silent, and silence is + indistinguishable from health. + +Contracts that make it scale: + +- **Five identity labels on every signal**: `project`, `env`, `service.name` and + `host_name` (Prometheus form; OTel form is `host.name`) come from the shared + agent config, and `department` is stamped by the gateway collector, which is + the only component that knows it and the only one a sender cannot override. + Cardinality rule: user ids, request ids and timestamps go in + bodies and span attributes, never labels. +- **Each signal has exactly one producer.** Alloy owns all container logs (SDK + log exporters stay off); native app metrics own RED; cAdvisor owns container + lifecycle; healthchecks.io owns "did the job run"; a host-local drift script + owns "is the deployed code the code we think"; nothing derives metrics from + logs or from traces. The one exception is accounting, not signal: the + gateway's count connector emits `telemetry_{datapoints,logs,spans}_total` per + project and environment, so the keystone alert below can see a project that + sends only logs or only traces, and can ask whether telemetry is arriving + without reading every series a project has ever produced. +- **`ProjectTelemetrySilent` per project/env is the keystone alert**: nothing + on a spoke can detect its own absence. Templated and provisioned by + `bootstrap.sh`, which is also what creates a project's healthchecks and prints + its `.env` block. Bootstrap is what creates the safety net, not the telemetry. +- **Onboarding is a copy, not a port:** vendor the template files at a pinned + tag (three, plus one for a GPU host), add six `.env` variables, include the + overlay, run `bootstrap.sh`. A GPU host is an ordinary host plus one opt-in + overlay (`nvidia_gpu_exporter` scraped by Alloy, not dcgm-exporter, whose + profiling fields are datacentre-only) and three GPU alert rules. + +## Alternatives considered + +- **Grafana Cloud free tier (no hub at all):** deletes this host, its backups + and its disk-full failure mode, and the free tier covers CML's volume. Rejected + on data protection, not economics: container logs cross the applications' + sanitization boundary (Postgres error lines can quote research and personal + data), and shipping them to a US-operated SaaS is a GDPR/university-policy + problem a department-run host does not have. Revisit only if that question is + formally cleared. +- **Per-project tokens and Loki multi-tenancy:** organisational controls for a + problem one operator does not have. The trigger for per-project tokens is the + first leaked-token incident; for multi-tenancy, the first dataset other CML + projects must not see. Both are a day of work, not a redesign. +- **Pushgateway for batch/ML jobs:** rejected on Prometheus's own guidance; + machine-level batch jobs use node_exporter's textfile collector plus a + dead-man's-switch check. +- **A second alerting engine, SLOs, paging rotations, per-project dashboards, + long retention:** all rejected. One operator, alert count capped around ten, + and dashboards carry a `project` template variable instead of per-project + copies. + +## Consequences + +- Tempo demotes to trace storage only; deleting its metrics-generator dependency + makes it disposable on its next breaking upgrade. +- The spokes' local watchdog checks (service health, snapshot age, timer state) + were deletable only once `ProjectTelemetrySilent` and the container-lifecycle + rules were live here. Both are, and both have been seen working on real traffic: + the keystone fired on an actual spoke outage on 2026-08-27 and resolved when the + host came back. That tripwire has therefore been released. +- Prometheus needs `out_of_order_time_window: 30m` for OTLP ingestion; without + it late batches drop silently. The OTLP receiver is documented as a + low-volume path. The revisit trigger is a host exceeding a few thousand active + series. +- Unverified at decision time: whether cAdvisor metric names survive the OTLP + round trip (a 15-minute experiment decides between importing dashboard 15798 + and adding a second ingestion path); Cloudflare free-plan Zero Trust seat + count. diff --git a/infra/generate-imports.sh b/infra/generate-imports.sh new file mode 100755 index 0000000..e674f5e --- /dev/null +++ b/infra/generate-imports.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Emit OpenTofu `import` blocks for the Cloudflare resources that already exist. +# +# A first plan against an empty state says "create" for objects already serving +# traffic, and applying it mints a second tunnel and duplicate DNS records. Import +# blocks make the adoption reviewable: read the generated file, then the plan, then +# apply. +# +# The ingestion record was renamed from `otlp.` to `otel.` (2026-09). +# On a state that predates the rename, the live record is imported as +# `cloudflare_dns_record.otel`, so the apply renames it in place. +# +# The generated imports.tf is a throwaway: delete it after the apply, or every plan +# re-runs the imports. +# +# Usage (run in this directory, with terraform.tfvars already filled in): +# export CLOUDFLARE_API_TOKEN=... # Tunnel:Read, DNS:Read, Access: Apps and Policies:Read +# ./generate-imports.sh > imports.tf +# tofu plan # expect "0 to add", only in-place changes +# tofu apply && rm imports.tf +set -euo pipefail + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + +die() { + echo "error: $*" >&2 + exit 1 +} + +api() { + # Token via curl config on stdin, not argv: /proc//cmdline is world-readable. + curl -fsS --config - "https://api.cloudflare.com/client/v4/$1" \ + <<<"header = \"Authorization: Bearer ${CLOUDFLARE_API_TOKEN}\"" +} + +command -v jq >/dev/null || die "jq is required" +: "${CLOUDFLARE_API_TOKEN:?is not set}" + +# Read the ids from terraform.tfvars; an exported TF_VAR_* still wins. +[[ -f terraform.tfvars ]] || die "no terraform.tfvars here; copy terraform.tfvars.example and fill it in" +tfvar() { + sed -n "s/^[[:space:]]*$1[[:space:]]*=[[:space:]]*\"\([^\"]*\)\".*/\1/p" terraform.tfvars | tail -1 +} +account="${TF_VAR_account_id:-$(tfvar account_id)}" +zone="${TF_VAR_zone_id:-$(tfvar zone_id)}" +domain="${TF_VAR_domain:-$(tfvar domain)}" + +# The example file's placeholders are valid-looking strings; catch them here. +for pair in "account_id:$account" "zone_id:$zone" "domain:$domain"; do + value="${pair#*:}" + [[ -n "$value" ]] || die "${pair%%:*} is empty in terraform.tfvars" + case "$value" in + your-* | example.org) die "${pair%%:*} is still the placeholder from terraform.tfvars.example" ;; + esac +done + +# A silently skipped import comes back as a "create" in the plan, so each lookup +# fails loudly when the resource is absent. +lookup_dns_record() { + local hostname="$1" id + # Type-filtered: an unfiltered .result[0] could bind a TXT record on the same + # name, and the apply would rewrite it into a CNAME. This runs in `$(...)`, so + # a `die` here would only exit the subshell and read as "no record": an API + # failure gets its own status 2, which every call site distinguishes from 1. + local body + body="$(api "zones/$zone/dns_records?name=$hostname&type=CNAME")" || return 2 + id="$(jq -r '.result[0].id // empty' <<<"$body")" + [[ -n "$id" ]] || return 1 + printf '%s' "$id" +} + +tunnel_name="${MONITORING_TUNNEL_NAME:-cml-monitoring}" +tunnels="$(api "accounts/$account/cfd_tunnel?is_deleted=false")" +tunnel_id="$(jq -r --arg name "$tunnel_name" '.result[] | select(.name == $name) | .id' <<<"$tunnels" | head -1)" +if [[ -z "$tunnel_id" ]]; then + # The tunnel is usually present under another name; list what is there. + echo "error: no tunnel named $tunnel_name in account $account" >&2 + echo "tunnels that do exist in this account:" >&2 + jq -r '.result[]? | " \(.name)\t\(.id)\tconnections=\(.connections | length)"' <<<"$tunnels" >&2 + echo "Re-run with MONITORING_TUNNEL_NAME='' once you know which one serves monitoring." >&2 + exit 1 +fi + +grafana_record="$(lookup_dns_record "grafana.$domain")" || { + [[ $? -ne 2 ]] || die "DNS lookup for grafana.$domain failed (token lacks DNS:Read?)" + die "no CNAME found for grafana.$domain" +} + +# otel. after the rename, otlp. before it. New name first, so a re-run is a no-op. +for host in "otel.$domain" "otlp.$domain"; do + if otel_record="$(lookup_dns_record "$host")"; then + ingestion_host="$host" + break + elif [[ $? -eq 2 ]]; then + die "DNS lookup for $host failed (token lacks DNS:Read?)" + fi +done +[[ -n "${ingestion_host:-}" ]] || die "no CNAME found for otel.$domain or otlp.$domain" + +# The Access app may legitimately not exist; then the apply should create it. A +# zone-scoped app (predating account-scoped Access) cannot be adopted by the +# account-scoped resource, so that case gets its own message. +access_apps="$(api "accounts/$account/access/apps?per_page=100")" +access_app_id="$(jq -r --arg d "grafana.$domain" '.result[] | select(.domain == $d) | .id' <<<"$access_apps" | head -1)" +if [[ -z "$access_app_id" ]]; then + zone_app_id="$(api "zones/$zone/access/apps?per_page=100" \ + | jq -r --arg d "grafana.$domain" '.result[]? | select(.domain == $d) | .id' | head -1)" || zone_app_id="" + if [[ -n "$zone_app_id" ]]; then + die "the Access app on grafana.$domain is zone-scoped ($zone_app_id); this root declares an + account-scoped one. Recreate it at the account level, or give the resource + zone_id instead of account_id, before importing." + fi + echo "note: no Access application on grafana.$domain; omitting its import block so the" >&2 + echo " apply CREATES it. Until that apply lands, Grafana's hostname is protected only" >&2 + echo " by its own login. Access apps that do exist in this account:" >&2 + jq -r '.result[]? | " \(.name)\t\(.domain)"' <<<"$access_apps" >&2 +fi + +# An app built in the dashboard usually carries an inline policy with no id to +# import; the apply then creates the reusable one and reattaches the app. +access_policies="$(api "accounts/$account/access/policies?per_page=100")" +access_policy_id="$(jq -r '.result[] | select(.name == "monitoring: allowed emails") | .id' <<<"$access_policies" | head -1)" + +echo "# Generated by generate-imports.sh on $(date -u +%Y-%m-%dT%H:%M:%SZ). Delete after a successful apply." +echo "# Ingestion record adopted from $ingestion_host as cloudflare_dns_record.otel." +cat <//'. +import { + to = cloudflare_zero_trust_access_application.grafana + id = "accounts/$account/$access_app_id" +} +EOF +fi + +if [[ -n "$access_policy_id" ]]; then + cat <&2 + echo " import block so the apply creates it. Check the plan's include{} list against" >&2 + echo " the emails the live app admits before applying." >&2 +fi diff --git a/infra/main.tf b/infra/main.tf index 9510b66..6bd548c 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,15 +1,19 @@ -# Cloudflare edge for the monitoring stack: the tunnel, its ingress rules, -# and DNS. This is the only part of the stack that otherwise lives as -# click-ops in the Zero Trust dashboard. +# Cloudflare edge for the monitoring stack: the tunnel, its ingress rules, DNS, +# and the Access policy in front of Grafana. # # Bootstrap (owner-run, once): # cp terraform.tfvars.example terraform.tfvars # then fill it in -# export CLOUDFLARE_API_TOKEN=... # needs Tunnel:Edit, DNS:Edit, Access:Edit -# cd infra && tofu init && tofu apply +# export CLOUDFLARE_API_TOKEN=... # needs Tunnel:Edit, DNS:Edit, Access:Edit, and +# # "Access: Organizations, Identity Providers, and +# # Groups: Read" for the team-domain data source +# cd infra && tofu init +# ./generate-imports.sh > imports.tf # the edge already exists: adopt it first +# tofu plan # expect "0 to add"; see the script's header +# tofu apply && rm imports.tf # tofu output -raw tunnel_token # → CLOUDFLARE_TUNNEL_TOKEN in ../.env +# tofu output -raw grafana_access_aud grafana_access_team_domain # → the CF_ACCESS_* pair # -# State is local (infra/terraform.tfstate, gitignored) — one host, one -# operator; move it to R2 the day a second operator exists. +# State is local (infra/terraform.tfstate, gitignored) and holds the tunnel secret. terraform { required_version = ">= 1.8" @@ -37,7 +41,7 @@ variable "zone_id" { variable "domain" { type = string - description = "Apex domain, e.g. example.org → grafana.example.org, otlp.example.org." + description = "Apex domain, e.g. example.org → grafana.example.org, otel.example.org." } variable "grafana_allowed_emails" { @@ -51,7 +55,8 @@ variable "grafana_allowed_emails" { resource "cloudflare_zero_trust_tunnel_cloudflared" "monitoring" { account_id = var.account_id - name = "monitoring" + # Must match the live tunnel's name; a different name here renames it on apply. + name = "cml-monitoring" config_src = "cloudflare" } @@ -67,7 +72,7 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "monitoring" { }, { # OTLP HTTP ingestion; the collector enforces bearer-token auth. - hostname = "otlp.${var.domain}" + hostname = "otel.${var.domain}" service = "http://otel-collector:4318" }, { @@ -87,19 +92,19 @@ resource "cloudflare_dns_record" "grafana" { ttl = 1 } -resource "cloudflare_dns_record" "otlp" { +# Renamed from `otlp` (2026-09). No `moved` block: nothing was ever in state under +# the old name, and generate-imports.sh adopts the live record straight into this one. +resource "cloudflare_dns_record" "otel" { zone_id = var.zone_id - name = "otlp.${var.domain}" + name = "otel.${var.domain}" type = "CNAME" content = "${cloudflare_zero_trust_tunnel_cloudflared.monitoring.id}.cfargotunnel.com" proxied = true ttl = 1 } -# Cloudflare Access in front of Grafana: email one-time-PIN at the edge, so -# the public hostname never reaches Grafana's login page unauthenticated. -# The OTLP hostname is NOT behind Access — machines authenticate with the -# bearer token instead. +# Email one-time PIN in front of Grafana. The ingestion hostname is not behind +# Access; machines authenticate with the bearer token. resource "cloudflare_zero_trust_access_application" "grafana" { account_id = var.account_id name = "Grafana (monitoring)" @@ -128,8 +133,25 @@ data "cloudflare_zero_trust_tunnel_cloudflared_token" "monitoring" { tunnel_id = cloudflare_zero_trust_tunnel_cloudflared.monitoring.id } +# The Zero Trust team name is account-wide and predates this config, so it is read, +# not managed. Grafana builds its JWK set URL from it, and an unset name would fetch +# signing keys from a subdomain anyone could claim, hence the exposure guards. +data "cloudflare_zero_trust_organization" "team" { + account_id = var.account_id +} + +output "grafana_access_team_domain" { + description = "Set as CF_ACCESS_TEAM_DOMAIN in ../.env. Grafana appends .cloudflareaccess.com." + value = trimsuffix(data.cloudflare_zero_trust_organization.team.auth_domain, ".cloudflareaccess.com") +} + +output "grafana_access_aud" { + description = "Set as CF_ACCESS_AUD in ../.env so Grafana rejects tokens minted for other Access apps." + value = cloudflare_zero_trust_access_application.grafana.aud +} + output "tunnel_token" { - description = "Set as CLOUDFLARE_TUNNEL_TOKEN in ../.env for just up-tunnel." + description = "Set as CLOUDFLARE_TUNNEL_TOKEN in ../.env for the tunnel overlay." value = data.cloudflare_zero_trust_tunnel_cloudflared_token.monitoring.token sensitive = true } diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 5ef4f33..9ddd21c 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -1,5 +1,5 @@ # Copy to terraform.tfvars (gitignored) and fill in. The Cloudflare API token -# is NOT set here — export it: `export CLOUDFLARE_API_TOKEN=...` +# is NOT set here. Export it: `export CLOUDFLARE_API_TOKEN=...` # (needs Tunnel:Edit, DNS:Edit, and Access: Apps and Policies:Edit). # Cloudflare account ID (dash.cloudflare.com → any domain → overview sidebar). @@ -8,7 +8,7 @@ account_id = "your-account-id" # Zone ID of the domain the hostnames live under (same overview sidebar). zone_id = "your-zone-id" -# Apex domain → grafana., otlp.. +# Apex domain → grafana., otel.. domain = "example.org" # Emails allowed through Cloudflare Access to Grafana (one-time PIN). diff --git a/justfile b/justfile index 7b6ce64..5eede5e 100644 --- a/justfile +++ b/justfile @@ -1,45 +1,104 @@ set dotenv-load -# Stateful services and their volumes, shared by backup/restore. The volume -# names assume the compose project name "monitoring" (see guard in backup). -stateful := "grafana prometheus loki tempo alertmanager" -backup_mounts := "-v monitoring_grafana_data:/data/grafana -v monitoring_prometheus_data:/data/prometheus -v monitoring_loki_data:/data/loki -v monitoring_tempo_data:/data/tempo -v monitoring_alertmanager_data:/data/alertmanager" - +# Stateful services; their volumes are __data. +stateful := "grafana prometheus loki tempo" + +# Helper and lint images, pinned once. +alpine := "alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" +jq := "ghcr.io/jqlang/jq:1.8.1@sha256:4f34c6d23f4b1372ac789752cc955dc67c2ae177eb1b5860b75cdc5091ce6f91" +yamllint := "pipelinecomponents/yamllint:0.35.13@sha256:5ab5eb7da0ed5e606b07c1723fc8b275e925189f70ac259b26b7329cb5f8f44d" +yamlfmt := "ghcr.io/google/yamlfmt:0.17.2@sha256:fa6874890092db69f35ece6a50e574522cae2a59b6148a1f6ac6d510e5bcf3cc" +actionlint := "rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667" +shellcheck := "koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f1782532571a34ed51fedf90dd25f925a8d" +ruff := "ghcr.io/astral-sh/ruff:0.14.2@sha256:636e27f3feb43800e44b0ad48c72811b500a2c6309d094b641a9bf2247f4dbff" +tofu := "ghcr.io/opentofu/opentofu:1.12.3@sha256:a0766d12f07b43e66f2ed40d7a8babe97d581d20339c68ad0ab561737af9a5b3" +gitleaks := "zricethezav/gitleaks:v8.30.1@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f" +lint_images := jq + " " + yamllint + " " + yamlfmt + " " + actionlint + " " + shellcheck + " " + ruff + " " + tofu + " " + gitleaks + +# `demo` and `smoke` each run a throwaway copy of the core stack under their own +# compose project and Grafana port (compose.sandbox.yml), so neither can touch a +# stack already running on this host. The explicit -f list keeps the host's +# COMPOSE_FILE out of both. +demo_project := "monitoring-demo" +demo_port := "3002" +compose_demo := "SANDBOX_PORT=" + demo_port + " docker compose -p " + demo_project + " -f compose.yml -f compose.demo.yml -f compose.sandbox.yml" + +# The project name compose will use, so the queue volume and backups target +# the running stack's volumes. +core_project := env("COMPOSE_PROJECT_NAME", "monitoring") + +# The smoke stack boots in production shape: JWT auth on and fixed notification +# URLs, so smoke.sh can assert exact values. The team domain has a dot, which no +# real Zero Trust team name can, so the JWK URL can never resolve to a team +# someone registers. +smoke_project := "monitoring-smoke" +smoke_port := "3001" +smoke_env := "SANDBOX_PORT=" + smoke_port + " GRAFANA_JWT_AUTH=true CF_ACCESS_TEAM_DOMAIN=smoke.invalid CF_ACCESS_AUD=smoke ALERT_WEBHOOK_URL=https://smoke.invalid/alerts HEARTBEAT_URL=https://smoke.invalid/heartbeat" +compose_smoke := smoke_env + " docker compose -p " + smoke_project + " -f compose.yml -f compose.sandbox.yml" + +# The spoke overlays interpolate these, so both rendering them and reading an +# image ref out of them needs the set. +spoke_env := "ENVIRONMENT=dummy PROJECT=dummy COMPOSE_PROJECT_NAME=dummy OTEL_EXPORTER_OTLP_ENDPOINT=https://dummy OTLP_AUTH_TOKEN=dummy" + +# dashboards/*.json as mounted at /dashboards. 2>/dev/null so an empty +# dashboards/ doesn't abort every recipe; `lint` refuses the empty list instead. +dash_paths := `ls dashboards/*.json 2>/dev/null | sed 's|^dashboards|/dashboards|' | tr '\n' ' '` + +# List the recipes. default: @just --list -# Core stack (no tunnel; Grafana at http://localhost:3000) -up: +# COMPOSE_FILE in .env names the overlays. With the tunnel overlay active, this +# refuses to start until the exposure guards pass. +# Start the stack (Grafana at http://localhost:3000). +up: _guard-if-exposed docker compose up -d -# Core stack + Cloudflare Tunnel (production; needs CLOUDFLARE_TUNNEL_TOKEN). -# Refuses to expose the stack with the documented default credentials. -up-tunnel: +# The exposure guards, only when the tunnel overlay is in play. +_guard-if-exposed: + @case "${COMPOSE_FILE:-}" in *compose.tunnel.yml*) just _expose-guards;; esac + +# Refuses to expose the stack with the documented default credentials or with +# secret files other local users can read. +_expose-guards: @[ "${OTLP_AUTH_TOKEN:-}" != "local-dev-token" ] || { echo "error: OTLP_AUTH_TOKEN is still the local default; generate one (openssl rand -hex 32) before exposing ingestion" >&2; exit 1; } @[ "${GRAFANA_ADMIN_PASSWORD:-}" != "change-me" ] || { echo "error: GRAFANA_ADMIN_PASSWORD is still the documented default; change it before exposing Grafana" >&2; exit 1; } - docker compose -f compose.yml -f compose.tunnel.yml up -d - + @case "${GRAFANA_ROOT_URL:-}" in https://*) ;; *) echo "error: GRAFANA_ROOT_URL must be the https:// tunnel hostname (got '${GRAFANA_ROOT_URL:-}'); every absolute URL Grafana generates comes from it" >&2; exit 1;; esac + @[ "${GRAFANA_COOKIE_SECURE:-false}" = "true" ] || { echo "error: GRAFANA_COOKIE_SECURE must be true when Grafana is served over HTTPS; set it in .env" >&2; exit 1; } + @[ "${GRAFANA_JWT_AUTH:-false}" != "true" ] || { [ -n "${CF_ACCESS_TEAM_DOMAIN:-}" ] && [ -n "${CF_ACCESS_AUD:-}" ]; } || { echo "error: GRAFANA_JWT_AUTH=true needs CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD in .env (cd infra && tofu output -raw grafana_access_team_domain grafana_access_aud)" >&2; exit 1; } + @[ -n "${HEARTBEAT_URL:-}" ] || echo "WARNING: HEARTBEAT_URL is empty; the stack goes live without a dead-man's switch" >&2 + @for f in .env infra/terraform.tfvars infra/terraform.tfstate infra/terraform.tfstate.backup; do [ ! -e "$f" ] || case "$(stat -c %a "$f")" in *00) ;; *) echo "error: $f is readable by other users (mode $(stat -c %a "$f")); it holds live secrets, run: chmod 600 $f" >&2; exit 1;; esac; done + @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; every alert would fire into an empty webhook URL and be dropped. The heartbeat keeps pinging either way, so this failure looks healthy from the outside. Set it, or comment out this guard" >&2; exit 1; } + +# Stop the stack; volumes stay. down: docker compose down --remove-orphans -# Core stack + a demo telemetry source (see compose.demo.yml), then look at -# Grafana: http://localhost:3000 +# Core stack plus a demo telemetry source, isolated from any running stack (:3002). demo: - docker compose -f compose.yml -f compose.demo.yml up -d --build + {{compose_demo}} up -d --build -# Remove only the demo services; the core stack keeps running. +# Stop the demo telemetry source; the demo project's core stack keeps running. demo-down: - docker compose -f compose.yml -f compose.demo.yml rm -sf demo-api demo-load + {{compose_demo}} rm -sf demo-api demo-load +# Tear down the whole demo stack and its throwaway volumes. +demo-destroy: + {{compose_demo}} down --remove-orphans --volumes + +# Follow logs, optionally of one service. logs service="": docker compose logs -f {{service}} +# Container status of the stack. ps: docker compose ps -restart service: +# Restart one service. +restart service: _guard-if-exposed docker compose restart {{service}} +# Pull the pinned images. pull: docker compose pull @@ -47,52 +106,132 @@ pull: tail service: docker compose logs -f --no-log-prefix {{service}} | jq -R 'fromjson? // .' -# Validate everything. All validators run in containers — no host installs. -# promtool/otelcol/amtool images are read from compose.yml so they can't -# drift from the versions the stack actually runs. -check: - docker compose config -q - CLOUDFLARE_TUNNEL_TOKEN=dummy docker compose -f compose.yml -f compose.tunnel.yml config -q - docker compose -f compose.yml -f compose.demo.yml config -q - docker run --rm -v ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro -v ./config/alerts:/etc/prometheus/alerts:ro --entrypoint promtool $(docker compose config --images | grep prom/prometheus) check config /etc/prometheus/prometheus.yaml - docker run --rm -e OTLP_AUTH_TOKEN=dummy -v ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro $(docker compose config --images | grep opentelemetry-collector) validate --config=/etc/otelcol/config.yaml - docker run --rm -v ./config/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro --entrypoint /bin/amtool $(docker compose config --images | grep prom/alertmanager) check-config /etc/alertmanager/alertmanager.yaml - docker run --rm -v .:/code:ro pipelinecomponents/yamllint:0.35.13 yamllint -d '{extends: relaxed, ignore: [.git/, backups/, infra/.terraform/]}' . - docker run --rm -v .:/repo:ro -w /repo rhysd/actionlint:1.7.12 -color - docker run --rm -v ./infra:/infra:ro -w /infra ghcr.io/opentofu/opentofu:1.12.3 fmt -check - docker run --rm -v ./dashboards:/dashboards:ro ghcr.io/jqlang/jq:1.8.1 empty $(ls dashboards/*.json | sed 's|^dashboards|/dashboards|') +# Every check runs in a container: no host installs, no network. +# Validate every config in the repo. +check: lint validate +# Static checks in tool images (~280 MB cold, seconds warm). +lint: + # Digest-pinned, so anything already local is current; only fetch what is missing. + @printf '%s\n' {{lint_images}} | xargs -P 8 -I{} sh -c 'docker image inspect {} >/dev/null 2>&1 || docker pull -q {} >/dev/null' + # Explicit -f, not the host's COMPOSE_FILE, so lint means the same here as in CI. + docker compose -f compose.yml config -q + CLOUDFLARE_TUNNEL_TOKEN=dummy docker compose -f compose.yml -f compose.tunnel.yml config -q + {{compose_demo}} config -q + # compose_smoke turns the JWT interpolation on. + {{compose_smoke}} config -q + # The spoke overlays, which otherwise first fail on a project host after vendoring. + {{spoke_env}} docker compose -f templates/compose.telemetry.yml -f templates/compose.telemetry.gpu.yml config -q + # The exposure guards, both ways: a fully set .env passes, and each + # documented default is refused on its own. + @good="OTLP_AUTH_TOKEN=t GRAFANA_ADMIN_PASSWORD=p GRAFANA_ROOT_URL=https://g.example GRAFANA_COOKIE_SECURE=true GRAFANA_JWT_AUTH=true CF_ACCESS_TEAM_DOMAIN=d CF_ACCESS_AUD=a HEARTBEAT_URL=https://h ALERT_WEBHOOK_URL=https://w"; \ + env $good just _expose-guards || { echo "error: exposure guards rejected a fully set environment" >&2; exit 1; }; \ + for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL=; do \ + ! env $good $bad just _expose-guards 2>/dev/null || { echo "error: exposure guards accepted $bad" >&2; exit 1; }; \ + done + # The rendered project-*/coverage rules are skipped (their expr lines grow + # with every project); their templates are checked by rendering them into + # a scratch dir instead. + docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/project-*.yaml, config/grafana/alerting/coverage.yaml]}' . + @d=$(mktemp -d) && BOOTSTRAP_OUT_DIR="$d" ./bootstrap.sh dummy dummy >/dev/null && docker run --rm --network none -v "$d":/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: disable}}' .; rc=$?; rm -rf "$d"; exit $rc + docker run --rm --network none -v .:/repo:ro -w /repo {{actionlint}} -color + docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh templates/run_scheduled.sh infra/generate-imports.sh + docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo + docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo + # `just fmt` is the fix. + docker run --rm --network none -v .:/code:ro -w /code {{yamlfmt}} -lint . + docker run --rm --network none -v ./infra:/infra:ro -w /infra {{tofu}} fmt -check + # Dashboards: valid JSON, and every datasource uid they name is provisioned. + # A typo provisions fine and renders empty panels. + @[ -n "{{dash_paths}}" ] || { echo "error: no dashboards/*.json to check" >&2; exit 1; } + docker run --rm --network none -v ./dashboards:/dashboards:ro {{jq}} empty {{dash_paths}} + @bad=$(docker run --rm --network none -v ./dashboards:/dashboards:ro {{jq}} -r '.. | objects | select(has("datasource")) | .datasource | (if type == "object" then .uid else . end) | strings' {{dash_paths}} | sort -u | grep -vxF "$(sed -n 's/^ *uid: *//p' config/grafana/datasources.yaml; echo grafana)"); \ + [ -z "$bad" ] || { echo "error: dashboards reference datasource uids that are not provisioned:" $bad >&2; exit 1; } + # Secrets in git history. Scans commits, not the working tree, so the + # gitignored .env never trips it. Needs full history (see ci.yml). + docker run --rm --network none -v .:/repo:ro {{gitleaks}} git --redact --no-banner /repo + # The agent config every project host vendors, checked by the Alloy build + # the template actually pins. + docker run --rm --network none -v ./templates/alloy/config.alloy:/etc/alloy/config.alloy:ro -e COMPOSE_PROJECT_NAME=dummy -e ENVIRONMENT=dummy -e PROJECT=dummy -e OTEL_EXPORTER_OTLP_ENDPOINT=https://dummy -e OTLP_AUTH_TOKEN=dummy -e TELEMETRY_EDGE_KEY= $({{spoke_env}} just _image templates/compose.telemetry.yml alloy) validate /etc/alloy/config.alloy + +# Image ref of one service in a compose file. (`config --images ` also +# lists the service's dependencies, hence the json route.) +_image file service: + @docker compose -f {{file}} config --format json | docker run --rm -i {{jq}} -er '.services["{{service}}"].image // error("no service {{service}} in {{file}}")' + +# Each config goes through the binary that will load it. +validate: + docker run --rm --network none -v ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --entrypoint promtool $(just _image compose.yml prometheus) check config /etc/prometheus/prometheus.yaml + docker run --rm --network none -e OTLP_AUTH_TOKEN=dummy -v ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro $(just _image compose.yml otel-collector) validate --config=/etc/otelcol/config.yaml + docker run --rm --network none -v ./config/loki.yaml:/etc/loki/loki.yaml:ro $(just _image compose.yml loki) -config.file=/etc/loki/loki.yaml -verify-config + docker run --rm --network none -v ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro $(just _image compose.yml tempo) -config.file=/etc/tempo/tempo.yaml -config.verify=true + +# Runs against a copy of the sources: state and tfvars never enter the +# container, which has network access to fetch the provider. # Full OpenTofu validation (downloads the provider, so not part of `check`). infra-validate: - docker run --rm --entrypoint sh -v ./infra:/infra -w /infra ghcr.io/opentofu/opentofu:1.12.3 -c 'tofu init -backend=false -input=false >/dev/null && tofu validate' + @d=$(mktemp -d) && cp infra/main.tf infra/.terraform.lock.hcl "$d"/ && docker run --rm --entrypoint sh -v "$d":/src:ro {{tofu}} -c 'mkdir /work && cp /src/main.tf /src/.terraform.lock.hcl /work && cd /work && tofu init -backend=false -input=false >/dev/null && tofu validate'; rc=$?; rm -rf "$d"; exit $rc + +# gitleaks over the staged diff (the pre-commit hook; see .pre-commit-config.yaml). +_gitleaks-staged: + @docker run --rm --network none -v .:/repo:ro {{gitleaks}} git --pre-commit --staged --redact --no-banner /repo -# Format YAML in place (needs yamlfmt on the host; optional). +# Format YAML in place (--user so the rewritten files stay yours). fmt: - yamlfmt . - -# Snapshot all stateful volumes to backups/.tar.gz (mode 0600 — -# it contains the Grafana DB and webhook secrets; copy it off-host, keep it -# private). Services are paused during the copy, and unpaused even if the -# copy fails. -backup: - @[ -z "${COMPOSE_PROJECT_NAME:-}" ] || { echo "error: COMPOSE_PROJECT_NAME is set; backup expects the monitoring_* volume names" >&2; exit 1; } - mkdir -p backups - -docker compose unpause {{stateful}} 2>/dev/null - docker compose pause {{stateful}} && { docker run --rm {{backup_mounts}} -v ./backups:/backups alpine:3.24 sh -c 'umask 077 && tar czf /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz -C /data .'; rc=$?; docker compose unpause {{stateful}}; exit $rc; } - @ls -lh backups/ | tail -1 + docker run --rm --network none --user "$(id -u):$(id -g)" -v .:/code -w /code {{yamlfmt}} . + +# Install the git hooks: gitleaks on commit, `just check` on push (see .pre-commit-config.yaml). +hooks: + prek install --hook-type pre-commit --hook-type commit-msg --hook-type pre-push + +# Snapshot all stateful volumes to backups/.tar.gz (mode 0600: it holds secrets). +backup: (_backup core_project "backups") + +# gzip -1: the stack is paused for as long as the tar runs, and the chunks are +# already compressed. The unpause runs unconditionally (`pause` can fail +# halfway), and a failed unpause fails the recipe. +_mounts project: + @for s in {{stateful}}; do printf -- '-v {{project}}_%s_data:/data/%s ' $s $s; done + +_backup project dir: + mkdir -p {{dir}} + @-docker compose -p {{project}} unpause {{stateful}} >/dev/null 2>&1 + @rc=0; m=$(just _mounts {{project}}); docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups {{alpine}} sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz' || rc=$?; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack is still paused" >&2; rc=1; }; exit $rc + @ls -lh {{dir}}/ | tail -1 # Restore a backup tarball into the volumes (stops the stack; wipes current state). -restore file: - @[ -f "{{file}}" ] || { echo "error: {{file}} not found" >&2; exit 1; } - docker run --rm -v {{absolute_path(file)}}:/backup.tar.gz:ro alpine:3.24 tar tzf /backup.tar.gz > /dev/null - docker compose down --remove-orphans - docker run --rm {{backup_mounts}} -v {{absolute_path(file)}}:/backup.tar.gz:ro alpine:3.24 sh -c 'for d in /data/*; do find "$d" -mindepth 1 -delete; done && tar xzf /backup.tar.gz -C /data' - @echo "Restored {{file}} — run 'just up' to start the stack." +restore file: (_restore core_project file "backups") + @echo "Restored {{file}}. Run 'just up' to start the stack." -# Boot the core stack, wait until Grafana reports healthy, and fail if any -# service is crash-looping. Used by CI. +# The current state is snapshotted to /pre-restore-*.tar.gz first: if the +# extract dies halfway, that is the only way back. +_restore project file dir: + @[ -f "{{file}}" ] || { echo "error: {{file}} not found" >&2; exit 1; } + @for s in {{stateful}}; do docker volume inspect {{project}}_${s}_data > /dev/null 2>&1 || { echo "error: volume {{project}}_${s}_data does not exist. 'docker run -v' would create it empty, so the pre-restore snapshot below would be a tarball of nothing. On a fresh host run 'just up' once first; otherwise check COMPOSE_PROJECT_NAME." >&2; exit 1; }; done + docker run --rm --network none -v {{absolute_path(file)}}:/backup.tar.gz:ro {{alpine}} tar tzf /backup.tar.gz > /dev/null + docker compose -p {{project}} down --remove-orphans + mkdir -p {{dir}} + m=$(just _mounts {{project}}); docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups {{alpine}} sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/pre-restore-$(date +%Y%m%d-%H%M%S).tar.gz' + m=$(just _mounts {{project}}); docker run --rm --network none $m -v {{absolute_path(file)}}:/backup.tar.gz:ro {{alpine}} sh -c 'for d in /data/*; do find "$d" -mindepth 1 -delete; done && tar xzf /backup.tar.gz -C /data' + +# Needs a booted smoke stack (`just smoke`). Run it after touching _backup/_restore. +# COMPOSE_FILE is pinned so the host's overlay list stays out, as for `smoke`. +# Round-trip backup and restore on the smoke stack. +restore-check: + @export COMPOSE_FILE=compose.yml:compose.sandbox.yml; d=$(mktemp -d) && just _backup {{smoke_project}} "$d" && f=$(ls "$d"/monitoring-*.tar.gz) && just _restore {{smoke_project}} "$f" "$d" && docker run --rm --network none -v {{smoke_project}}_grafana_data:/g:ro {{alpine}} test -s /g/grafana.db && echo "Backup round-trip ok"; rc=$?; rm -rf "$d"; exit $rc + +# `--wait` blocks on the healthchecks and fails if any container exits, so a +# crash-looping service is caught (after the full timeout). scripts/smoke.sh +# asserts what lands after that: provisioning, scrapes, the data paths. +# Boot an isolated copy of the core stack and assert it works end to end. smoke: - docker compose up -d - n=0; until curl -sf http://localhost:3000/api/health >/dev/null; do n=$((n+3)); [ $n -ge 120 ] && { echo "Grafana not healthy after 120s" >&2; exit 1; }; sleep 3; done - @[ -z "$(docker compose ps -q --status=restarting --status=exited)" ] || { echo "error: services not running:" >&2; docker compose ps >&2; exit 1; } - @echo "Stack healthy" + {{compose_smoke}} up -d --wait --wait-timeout 120 + {{smoke_env}} SMOKE_URL=http://localhost:{{smoke_port}} SMOKE_PROJECT={{smoke_project}} scripts/smoke.sh + +# Logs from the smoke stack (its own project, so `just logs` will not show it). +smoke-logs: + {{compose_smoke}} logs --no-color --tail=200 + +# Tear down the smoke stack and its throwaway volumes. +smoke-down: + {{compose_smoke}} down --remove-orphans --volumes -t 1 diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100755 index 0000000..4b9d02e --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Assertions `just smoke` runs against the stack it just booted. Everything +# here lands asynchronously after Grafana's /api/health answers, so each +# assertion polls (one second, up to a minute). +set -euo pipefail + +url="${SMOKE_URL:?}" +project="${SMOKE_PROJECT:?}" +command -v jq >/dev/null || { echo "error: jq is required" >&2; exit 1; } + +die() { echo "error: $*" >&2; exit 1; } +# Admin basic auth through curl's stdin config: argv is world-readable in ps. +gf() { printf 'user = "admin:%s"\n' "$GRAFANA_ADMIN_PASSWORD" | curl -sf -K - "$@"; } +promq() { gf -G --data-urlencode "query=$1" "$url/api/datasources/proxy/uid/prometheus/api/v1/query"; } +poll() { local n=0; until "$@"; do n=$((n + 1)); [[ $n -lt ${POLL_MAX:-60} ]] || return 1; sleep "${POLL_SLEEP:-1}"; done; } + +# ------------------------------------------------------------------ provisioning +# Grafana skips a broken dashboard or a malformed alert group silently, so the +# provisioned sets must equal what the repo holds, uid for uid. +want_dash="$(jq -r .uid dashboards/*.json | sort)" +rule_files=() +for f in config/grafana/alerting/*.yaml; do + case "$f" in */contact-points.yaml | */notification-policies.yaml) ;; *) rule_files+=("$f") ;; esac +done +want_rules="$(sed -n 's/^ *- uid: *//p' "${rule_files[@]}" | sort)" +[[ -n "$want_dash" && -n "$want_rules" ]] || die "no dashboards or alert rules in the repo to assert against" + +provisioned() { + { have_dash="$(gf "$url/api/search?type=dash-db&limit=5000" | jq -r '.[].uid' | sort)" \ + && rules="$(gf "$url/api/v1/provisioning/alert-rules")"; } \ + || die "Grafana API request failed. Check GRAFANA_ADMIN_PASSWORD, and that Grafana answers on $url" + have_rules="$(jq -r '.[].uid' <<<"$rules" | sort)" + [[ "$have_dash" == "$want_dash" && "$have_rules" == "$want_rules" ]] +} +poll provisioned || die "not provisioned after 60s (a malformed file provisions none of its group). See just smoke-logs +dashboards, want vs have: $(diff <(echo "$want_dash") <(echo "$have_dash") | grep '^[<>]' | tr '\n' ' ') +alert rules, want vs have: $(diff <(echo "$want_rules") <(echo "$have_rules") | grep '^[<>]' | tr '\n' ' ')" +jq -e 'all(.isPaused | not)' <<<"$rules" >/dev/null \ + || die "paused alert rules never fire: $(jq -r '.[] | select(.isPaused) | .uid' <<<"$rules" | tr '\n' ' ')" + +# --------------------------------------------------------------- contact points +# The exact URLs the stack was given must come back: an empty value would +# pass a "not literally $ALERT_WEBHOOK_URL" test and drop every alert. +gf "$url/api/v1/provisioning/contact-points" \ + | jq -e --arg a "$ALERT_WEBHOOK_URL" --arg h "$HEARTBEAT_URL" \ + '[.[] | select(.uid | startswith("cp-")) | .settings.url] | sort == ([$a, $h] | sort)' >/dev/null \ + || die "contact points do not carry ALERT_WEBHOOK_URL and HEARTBEAT_URL; Grafana did not expand the provisioning file, or a receiver is missing" + +# ------------------------------------------------------------------- JWT auth +# Grafana ignores an env key it no longer knows, so read the parsed setting +# back, and check that a forged Access header is refused. +gf "$url/api/admin/settings" | jq -e '.["auth.jwt"].enabled == "true"' >/dev/null \ + || die "Grafana did not enable JWT auth from GF_AUTH_JWT_*; the Cloudflare Access path is broken" +code="$(curl -s -o /dev/null -w '%{http_code}' -H 'Cf-Access-Jwt-Assertion: not-a-jwt' "$url/api/dashboards/home")" +[[ "$code" == 401 ]] || die "a forged Access token got HTTP $code from Grafana, expected 401" + +# ------------------------------------------------------------- scrape targets +# A renamed service leaves its scrape job silently empty, and TargetDown then +# matches nothing. Every job in prometheus.yaml must have scraped its target. +n_jobs="$(grep -c '^ *- job_name:' config/prometheus.yaml)" +all_up() { [[ "$(promq 'count(up == 1)' | jq -r '.data.result[0].value[1] // 0')" == "$n_jobs" ]]; } +poll all_up || die "not all $n_jobs scrape targets are up: $(promq up | jq -r '.data.result[] | "\(.metric.job)=\(.value[1])"' | tr '\n' ' ')" + +# ------------------------------------------------------------- data paths +# One metric and one log through the collector's bearer auth, read back with +# the identity labels the alert rules key on. Posted from inside the stack's +# network with Grafana's curl: the sandbox overlay publishes no ingestion ports. +ts="$(date +%s)000000000" +res='{"attributes":[{"key":"service.name","value":{"stringValue":"smoke"}},{"key":"project","value":{"stringValue":"smoke"}},{"key":"env","value":{"stringValue":"ci"}}]}' +# Token through curl's stdin config: argv is world-readable in ps, on the host +# and inside the container alike. +otlp() { + docker compose -p "$project" exec -T grafana sh -c \ + 'curl -sf -K - -o /dev/null -H "Content-Type: application/json" --data-binary "$1" "http://otel-collector:4318/v1/$2"' \ + _ "$2" "$1" <<<"header = \"Authorization: Bearer $OTLP_AUTH_TOKEN\"" +} +otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"1","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \ + || die "the collector refused an OTLP metric with the .env token" +otlp logs '{"resourceLogs":[{"resource":'"$res"',"scopeLogs":[{"logRecords":[{"timeUnixNano":"'"$ts"'","body":{"stringValue":"smoke"}}]}]}]}' \ + || die "the collector refused an OTLP log with the .env token" +smoke_metric() { promq "smoke_up{job=\"smoke\",project=\"smoke\",env=\"ci\",department=\"cml\"}" | jq -e '.data.result | length > 0' >/dev/null; } +smoke_log() { + gf -G --data-urlencode "query={project=\"smoke\",env=\"ci\",department=\"cml\"}" \ + "$url/api/datasources/proxy/uid/loki/loki/api/v1/query_range" \ + | jq -e 'any(.data.result[].values[][1]; . == "smoke")' >/dev/null +} +poll smoke_metric || die "the smoke metric never reached Prometheus with its project/env/department labels; see just smoke-logs" +poll smoke_log || die "the smoke log never reached Loki with its department label; see just smoke-logs" +# The keystone alert (ProjectTelemetrySilent) and the coverage backstop key on the +# gateway's ingest counters, so a renamed metric or a dropped attribute would +# silence both without any other assertion noticing. +counted() { promq "count by (project, env) ({__name__=~\"telemetry_.+_total\", project=\"smoke\", env=\"ci\"})" | jq -e '.data.result | length > 0' >/dev/null; } +poll counted || die "the count connector never produced telemetry_*_total{project=smoke,env=ci}; the alert rules that key on it would never fire" + +# ------------------------------------------------------------- alert pipeline +# Opt-in (SMOKE_ALERTS=1): stop one scrape target and wait for TargetDown to +# reach firing. Costs the rule's `for` (2m) plus an evaluation, so it is off +# for the local loop and on in CI. Guards the threshold-node contract in +# rules.yaml: a query whose matching value is 0 never fires without `bool`. +if [[ "${SMOKE_ALERTS:-}" == 1 ]]; then + docker compose -p "$project" stop -t 1 node-exporter >/dev/null + target_down_firing() { + gf "$url/api/prometheus/grafana/api/v1/rules" \ + | jq -e '.data.groups[].rules[] | select(.name == "TargetDown") | .state == "firing"' >/dev/null + } + POLL_MAX=60 POLL_SLEEP=5 poll target_down_firing || die "TargetDown did not fire within 5 minutes of stopping node-exporter" + docker compose -p "$project" start node-exporter >/dev/null + echo "TargetDown fired for the stopped node-exporter" +fi + +echo "Stack healthy" diff --git a/templates/README.md b/templates/README.md new file mode 100644 index 0000000..12ff9c0 --- /dev/null +++ b/templates/README.md @@ -0,0 +1,124 @@ +# Onboarding a project host + +Four files get vendored onto a project host at a pinned tag. **No project +edits them.** Everything that differs between projects arrives as an +environment variable, so the agent config on every host is byte-identical and +a fix here reaches all of them. + +| File | What it is | +| --- | --- | +| `alloy/config.alloy` | The agent config: container logs, host metrics, cAdvisor, optional GPU | +| `compose.telemetry.yml` | The Alloy agent and its Docker socket proxy | +| `compose.telemetry.gpu.yml` | Opt-in overlay: `nvidia_gpu_exporter`, discovered automatically | +| `run_scheduled.sh` | Dead man's switch wrapper for scheduled jobs | + +`alerting/*.tmpl` are not vendored. `bootstrap.sh` renders them into this +stack's own alert rules. + +## The checklist + +1. On the monitoring host, run `./bootstrap.sh `. It prints + the `.env` variables for the project host and the `curl` commands that + vendor the files at a pinned tag. +2. On the project host, paste the variables, run the curls, and bring the + overlay up: + + ```sh + docker compose -f compose.yml -f compose.telemetry.yml up -d + ``` + +3. Verify from the monitoring host (see below). + +Skipping `bootstrap.sh` leaves a project unmonitored with no error anywhere. +The rule that notices a host's *silence* lives on this stack, not on the +host. The backstop is `ProjectsUncovered`, regenerated on every bootstrap +run: it fires on any project the gateway counts telemetry for that has no +rendered rule file, whichever signal that project sends. + +## Two settings that silently produce nothing + +- **`cgroup: host` on the agent container.** The overlay sets it. cAdvisor + finds containers by walking the cgroup tree, so under Docker's default + private cgroup namespace it sees only its own cgroup. It then reports one + root series with no `name` label, and every container alert matches + nothing. No error is logged. +- **`OTEL_SEMCONV_STABILITY_OPT_IN=http` in an instrumented app.** Without + it the SDK emits the legacy HTTP metric names, whose `http_target` label + carries the raw request path: unbounded series on any API with path + parameters. The Service Health dashboard queries the stable names. + +## GPU hosts + +Include `compose.telemetry.gpu.yml` as well. The agent config discovers the +exporter by its Compose service label, so nothing else changes. + +A host without the NVIDIA container runtime cannot include the overlay at all +(`up` aborts with "could not select device driver"), so deploy tooling that +assembles its `-f` list from the host's `.env` needs a switch. Use +`GPU_METRICS=1` for it. None of the vendored files read that name; it is a +convention, so that one runbook covers every spoke. + +The exporter is `nvidia_gpu_exporter`, not dcgm-exporter. DCGM's profiling +fields are datacentre-only, so on a consumer card it offers nothing extra and +still requires `SYS_ADMIN`. + +`bootstrap.sh` does not provision GPU alert rules. The signals worth alerting +on per GPU host: + +- `nvidia_smi_gpu_recovery_action > 0`: the driver is asking for a reset. +- A thermal or power throttle flag. +- XID faults, with an explicit code allowlist, since most XIDs are + application faults: + + ```promql + time() - nvidia_smi_xid_last_timestamp_seconds{xid=~"48|62|64|74|79|95|119|120"} < 300 + ``` + +A stuck kernel, an uncorrectable memory fault, or a card that has fallen off +the bus are all invisible to utilisation graphs. XIDs are how they show. + +Both dashboards are provisioned centrally: `dashboards/gpu.json` (vendored +from [14574](https://grafana.com/grafana/dashboards/14574)) and +`dashboards/host-containers.json`. Nothing to import on the project host. + +## Removing a project + +Deleting `config/grafana/alerting/project--.yaml` is **not** +enough. Grafana provisioning never deletes a rule because its file vanished, +and the API refuses to delete a provisioned rule (409). The orphan keeps +evaluating and firing. + +1. Delete the project file. +2. Add a temporary provisioning file that drops the rule: + + ```yaml + # config/grafana/alerting/zz-delete.yaml (temporary) + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: proj-silent-- + ``` + +3. Restart Grafana and confirm the rule group is gone. +4. Remove `zz-delete.yaml` and restart Grafana again. Left in place, it + deletes the rule again the next time `bootstrap.sh` renders it. +5. Re-run `bootstrap.sh` for a project that remains, so `coverage.yaml` stops + listing the removed one as covered. + +## Limits + +The healthchecks.io free tier is 20 checks. `bootstrap.sh` creates three per +project/environment, so about six environments fit. + +## Verifying, from the monitoring host + +```promql +count(telemetry_datapoints_total{project="",env=""}) # non-zero within ~2 min +count(container_start_time_seconds{project="",name!=""}) # one per container +count({job=""}) # app's own SDK metrics +``` + +If the first is still zero after five minutes, `ProjectTelemetrySilent` +will say so: it fires after 15 minutes without data, or about 20 minutes +after the last sample once a project has sent any, because `absent()` needs +the series to go stale first. diff --git a/templates/alerting/coverage.yaml.tmpl b/templates/alerting/coverage.yaml.tmpl new file mode 100644 index 0000000..ae87b7f --- /dev/null +++ b/templates/alerting/coverage.yaml.tmpl @@ -0,0 +1,53 @@ +# GENERATED by bootstrap.sh from templates/alerting/coverage.yaml.tmpl. Do not edit +# the rendered file; it is regenerated on every run. +# +# Fires on any series whose project/env pair has no rendered rule file: a project that +# ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule. +# +# Covered right now: __COVERED__ + +apiVersion: 1 + +groups: + - orgId: 1 + name: _coverage + folder: Stack alerts + interval: 5m + rules: + - uid: projects-uncovered + title: ProjectsUncovered + condition: FIRING + for: 15m + noDataState: OK + execErrState: Error + isPaused: false + data: + - refId: QUERY + datasourceUid: prometheus + relativeTimeRange: + from: 3600 + to: 0 + model: + refId: QUERY + instant: true + editorMode: code + # The collector's count connector (config/otel-collector.yaml) mints these + # counters for every sender, whatever signal it sends. A bare + # {project!=""} would scan every project-labelled series in the TSDB on + # each evaluation, and would still miss a logs-only or traces-only project. + expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) (__COVERED_EXPR__)) + - refId: FIRING + datasourceUid: __expr__ + model: + refId: FIRING + type: threshold + expression: QUERY + conditions: + - evaluator: + type: gt + params: [0] + labels: + severity: warning + annotations: + summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules" + description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host. A literal `unknown` means the sender set no project or env resource attribute at all: fix the sender (docs/ONBOARDING.md), do not bootstrap a project by that name." diff --git a/templates/alerting/project.yaml.tmpl b/templates/alerting/project.yaml.tmpl new file mode 100644 index 0000000..3afcedc --- /dev/null +++ b/templates/alerting/project.yaml.tmpl @@ -0,0 +1,57 @@ +# Rendered by bootstrap.sh into config/grafana/alerting/. Do not edit the rendered +# files by hand; edit this template so every project gets the fix. +# +# Nothing on a project host can detect its own absence, so this rule lives here. +# +# bootstrap.sh reads the marker below to regenerate coverage.yaml. It carries the pair, +# not the filename, because both halves may contain a dash. +# COVERS: __PROJECT__ __ENV__ + +apiVersion: 1 + +groups: + - orgId: 1 + name: project-__PROJECT__-__ENV__ + folder: Stack alerts + interval: 1m + rules: + # absent() yields nothing while telemetry flows, so NoData is the HEALTHY state + # and must map to OK. + - uid: proj-silent-__PROJECT__-__ENV__ + title: ProjectTelemetrySilent + condition: FIRING + for: 15m + noDataState: OK + execErrState: Error + isPaused: false + data: + - refId: QUERY + datasourceUid: prometheus + relativeTimeRange: + from: 3600 + to: 0 + model: + refId: QUERY + instant: true + editorMode: code + # The collector's ingest counters, as in coverage.yaml.tmpl. absent() loads + # every series its selector matches, once a minute per project, so it is + # bounded to a few per service rather than everything the project sends. + expr: absent({__name__=~"telemetry_.+_total", project="__PROJECT__",env="__ENV__"}) + - refId: FIRING + datasourceUid: __expr__ + model: + refId: FIRING + type: threshold + expression: QUERY + conditions: + - evaluator: + type: gt + params: [0] + labels: + severity: critical + project: __PROJECT__ + env: __ENV__ + annotations: + summary: "No telemetry from __PROJECT__/__ENV__ in 15m" + description: "Host down, Docker down, the agent down, tunnel down, token rotated wrong, or the collector is rejecting this project's data. Nothing on the __PROJECT__ side can detect this on its own. Expect this ~20m after the last sample: absent() needs the series to go stale first." diff --git a/templates/alloy/config.alloy b/templates/alloy/config.alloy new file mode 100644 index 0000000..199c347 --- /dev/null +++ b/templates/alloy/config.alloy @@ -0,0 +1,317 @@ +// SHARED AGENT CONFIG, vendored from the CML monitoring repository at a pinned tag. +// Do not edit it per project: every deployment-specific value arrives as an environment +// variable (PROJECT, ENVIRONMENT, COMPOSE_PROJECT_NAME, OTEL_EXPORTER_OTLP_ENDPOINT, +// OTLP_AUTH_TOKEN, TELEMETRY_EDGE_KEY). If onboarding needs an edit here, that is a +// bug in the template. Report it upstream. +// +// Ships what the application cannot report about itself: other containers' stdout as +// logs, host resource metrics, and per-container lifecycle from cAdvisor, all over the +// same OTLP endpoint and token the application uses. + +// The host's name from its own /etc/hostname; the container's hostname is a docker id. +local.file "hostname" { + filename = "/rootfs/etc/hostname" +} + +// The Docker API is reached through the socket proxy (compose.telemetry.yml), never the +// socket itself, so this container cannot mutate the daemon. +discovery.docker "containers" { + host = "tcp://docker-socket-proxy:2375" + + // The default 1m lets a short-lived one-shot (a backup) start and exit inside one + // poll window, never discovered. 15s catches those without polling the socket + // proxy 12x more often than the default forever. + refresh_interval = "15s" +} + +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + // Only this Compose project; other projects on the host ship their own telemetry. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + regex = sys.env("COMPOSE_PROJECT_NAME") + action = "keep" + } + + // Never tail this pipeline's own containers: a broken export path would log an + // error per failed batch, queue it, fail again, with no exit. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + regex = "alloy|docker-socket-proxy" + action = "drop" + } + + // The Compose service name is the stable identifier dashboards key on. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service_name" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "container" + } +} + +loki.source.docker "containers" { + host = "tcp://docker-socket-proxy:2375" + targets = discovery.relabel.containers.output + labels = { + env = sys.env("ENVIRONMENT"), + project = sys.env("PROJECT"), + + // `instance` is identical on every host (Alloy scrapes its own in-container + // exporter), so this is the host identity. The coalesce handles a blank + // /etc/hostname. + host_name = coalesce(string.trim_space(local.file.hostname.content), constants.hostname), + } + forward_to = [otelcol.receiver.loki.containers.receiver] +} + +// Loki-shaped entries in, OpenTelemetry logs out. +otelcol.receiver.loki "containers" { + output { + logs = [otelcol.processor.transform.resource_attributes.input] + } +} + +// Promote the identity labels from log-record attributes to resource attributes. +// error_mode = "ignore" so a renamed attribute arrives unpromoted rather than dropped. +otelcol.processor.transform "resource_attributes" { + error_mode = "ignore" + + log_statements { + context = "log" + + statements = [ + `set(resource.attributes["service.name"], attributes["service_name"]) where attributes["service_name"] != nil`, + `set(resource.attributes["env"], attributes["env"]) where attributes["env"] != nil`, + `set(resource.attributes["project"], attributes["project"]) where attributes["project"] != nil`, + `set(resource.attributes["host.name"], attributes["host_name"]) where attributes["host_name"] != nil`, + ] + } + + output { + logs = [otelcol.processor.memory_limiter.default.input] + } +} + +// --------------------------------------------------------------------------- +// Host metrics: node_exporter's collectors, scraped in-process and converted to OTLP. +// set_collectors is an allowlist, which keeps the series count down. `hwmon` feeds a +// dashboard panel only; nothing alerts on temperature. +prometheus.exporter.unix "host" { + procfs_path = "/host/proc" + sysfs_path = "/host/sys" + rootfs_path = "/rootfs" + + set_collectors = [ + "cpu", + "diskstats", + "filesystem", + "hwmon", + "loadavg", + "meminfo", + "netdev", + "stat", + "uname", + ] + + filesystem { + // Pseudo-filesystems and per-container overlays. + mount_points_exclude = "^/(dev|proc|sys|run)($|/)|^/rootfs/(dev|proc|sys|run)($|/)|^/var/lib/docker/" + } +} + +// --------------------------------------------------------------------------- +// Per-container resource metrics. The monitoring stack detects a crash loop as +// container_start_time_seconds changing repeatedly, and container_oom_events_total says +// why. Needs `cgroup: host` on this container (see compose.telemetry.yml); without it +// cAdvisor reports one root series and the rule matches nothing. +prometheus.exporter.cadvisor "containers" { + docker_host = "tcp://docker-socket-proxy:2375" + + // Both defaults are cardinality traps: every container label and environment + // variable as a Prometheus label, plus raw systemd-slice cgroups. + store_container_labels = false + docker_only = true + + allowlisted_container_labels = [ + "com.docker.compose.project", + "com.docker.compose.service", + ] +} + +prometheus.scrape "cadvisor" { + targets = prometheus.exporter.cadvisor.containers.targets + forward_to = [prometheus.relabel.cadvisor_scope.receiver] + scrape_interval = "30s" + job_name = "cadvisor" +} + +// cAdvisor sees every container on the box, and the discovery.docker filter above does +// not reach it. Keep only this project's, or other stacks' containers arrive stamped +// with this project's labels. The unlabelled root-cgroup series drops out too. +prometheus.relabel "cadvisor_scope" { + forward_to = [prometheus.relabel.host.receiver] + + rule { + source_labels = ["container_label_com_docker_compose_project"] + regex = sys.env("COMPOSE_PROJECT_NAME") + action = "keep" + } + + // cAdvisor emits ~59 metric families per container (per-cpu, per-disk, per-NIC, + // TCP state); the stack reads four. The rest is the fastest-growing block in the + // series budget, since it scales per container per host. Add a name here when a + // dashboard or rule starts using one. + rule { + source_labels = ["__name__"] + regex = "container_(start_time_seconds|oom_events_total|cpu_usage_seconds_total|memory_working_set_bytes)" + action = "keep" + } +} + +prometheus.scrape "host" { + targets = prometheus.exporter.unix.host.targets + forward_to = [prometheus.relabel.host.receiver] + scrape_interval = "30s" + job_name = "host" +} + +// The same identity labels the logs carry. +prometheus.relabel "host" { + forward_to = [otelcol.receiver.prometheus.host.receiver] + + rule { + target_label = "env" + replacement = sys.env("ENVIRONMENT") + action = "replace" + } + + rule { + target_label = "project" + replacement = sys.env("PROJECT") + action = "replace" + } + + // See the note on the log labels. + rule { + target_label = "host_name" + replacement = coalesce(string.trim_space(local.file.hostname.content), constants.hostname) + action = "replace" + } +} + +// --------------------------------------------------------------------------- +// Alloy's own health: the only place that counts what this agent drops +// (otelcol_exporter_send_failed_*). +prometheus.exporter.self "alloy" { } + +prometheus.scrape "alloy" { + targets = prometheus.exporter.self.alloy.targets + forward_to = [prometheus.relabel.alloy_self.receiver] + scrape_interval = "30s" + job_name = "alloy" +} + +// Alloy is the only exporter in this file with histograms, and nothing central +// reads them: 345 of this job's 760 series. The _sum and _count series survive. +prometheus.relabel "alloy_self" { + forward_to = [prometheus.relabel.host.receiver] + + rule { + source_labels = ["__name__"] + regex = ".*_bucket" + action = "drop" + } +} + +// --------------------------------------------------------------------------- +// GPU metrics, present only on hosts that run compose.telemetry.gpu.yml. Without the +// exporter container this finds no targets and costs nothing. +discovery.relabel "gpu_exporter" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + regex = sys.env("COMPOSE_PROJECT_NAME") + action = "keep" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + regex = "nvidia-gpu-exporter" + action = "keep" + } + + rule { + target_label = "service_name" + replacement = "gpu" + action = "replace" + } +} + +prometheus.scrape "gpu" { + targets = discovery.relabel.gpu_exporter.output + forward_to = [prometheus.relabel.host.receiver] + + scrape_interval = "30s" + job_name = "gpu" +} + +otelcol.receiver.prometheus "host" { + output { + metrics = [otelcol.processor.memory_limiter.default.input] + } +} + +// Without this a long outage of the central stack grows the sending queue until the +// OOM killer takes the agent. Sized under the overlay's 512m mem_limit; keep the two +// in step. +otelcol.processor.memory_limiter "default" { + check_interval = "1s" + limit = "400MiB" + spike_limit = "100MiB" + + output { + logs = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.batch.default.input] + } +} + +otelcol.processor.batch "default" { + output { + logs = [otelcol.exporter.otlphttp.central.input] + metrics = [otelcol.exporter.otlphttp.central.input] + } +} + +otelcol.exporter.otlphttp "central" { + client { + // The tunnel routes HTTPS to the collector's HTTP receiver only, hence otlphttp. + // The endpoint MUST be https://: an http:// value ships the token and every log + // line in cleartext, and nothing here can tell. + endpoint = sys.env("OTEL_EXPORTER_OTLP_ENDPOINT") + + headers = { + Authorization = "Bearer " + sys.env("OTLP_AUTH_TOKEN"), + + // Second, weaker credential for projects whose egress crosses a WAF: the + // WAF-skip rule matches this header, so the bearer token never appears in a + // ruleset expression. Empty when unused. + "X-Telemetry-Key" = sys.env("TELEMETRY_EDGE_KEY"), + } + } + + // Retry for 15 minutes, hold up to 2000 batches in memory, then drop and count the + // drop. See the overlay's mem_limit before raising queue_size. + sending_queue { + queue_size = 2000 + } + + retry_on_failure { + max_elapsed_time = "15m" + } +} diff --git a/templates/compose.telemetry.gpu.yml b/templates/compose.telemetry.gpu.yml new file mode 100644 index 0000000..e3e8890 --- /dev/null +++ b/templates/compose.telemetry.gpu.yml @@ -0,0 +1,45 @@ +# compose.telemetry.gpu.yml, VENDORED from the CML monitoring repository at a pinned tag. +# +# OPTIONAL OVERLAY: GPU metrics on a host with NVIDIA hardware. The Alloy agent +# discovers and scrapes the exporter automatically. +# +# docker compose ... -f compose.telemetry.yml -f compose.telemetry.gpu.yml up -d +# +# Deploy tooling that builds its `-f` list from the host's `.env` should key on +# `GPU_METRICS=1`. Nothing in these templates reads it, but one name across every spoke +# keeps the runbooks identical, and the switch has to exist: including this overlay on a +# host without the NVIDIA container runtime makes `up` abort with "could not select +# device driver". +# +# nvidia_gpu_exporter, not dcgm-exporter: DCGM's profiling fields are datacentre-only, +# and on a consumer card it keeps only its SYS_ADMIN requirement. + +services: + nvidia-gpu-exporter: + image: utkuozdemir/nvidia_gpu_exporter:1.14.0-nvml@sha256:82acc3fc60a5a709846ea9757bbccb170fd141039fa47e5899c55fe9a60f56fe + restart: unless-stopped + # The NVML backend reports XID error counters; the nvidia-smi backend cannot. + environment: + NVIDIA_DRIVER_CAPABILITIES: utility + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + mem_limit: 256m + pids_limit: 256 + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + # No published ports: Alloy reaches it over the Compose network. + networks: + - egress + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/templates/compose.telemetry.yml b/templates/compose.telemetry.yml new file mode 100644 index 0000000..003d408 --- /dev/null +++ b/templates/compose.telemetry.yml @@ -0,0 +1,117 @@ +# compose.telemetry.yml, VENDORED from the CML monitoring repository at a pinned tag. +# Vendor it as-is; everything that differs between projects arrives as an environment +# variable. See templates/README.md. +# +# One Grafana Alloy agent per host, shipping what the application cannot report on +# itself to the same OTLP endpoint with the same token: other containers' stdout as +# logs, host resource metrics, and per-container lifecycle from cAdvisor. + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +services: + # Read-only gate between Alloy and the Docker API. The proxy holds the socket, so it + # is the root-equivalent component: haproxy with a static allowlist, on the internal + # socket network only. + docker-socket-proxy: + image: tecnativa/docker-socket-proxy:v0.5.0@sha256:1f5038b54f06c3e18422902cf00ba21803d1c97805aae032e5e6673d532d3459 + restart: unless-stopped + environment: + CONTAINERS: 1 # list/inspect/logs, for discovery.docker and loki.source.docker + NETWORKS: 1 # discovery.docker computes network labels; 403 here kills discovery + EVENTS: 1 # container lifecycle stream, watched by cAdvisor + INFO: 1 # machine info, read by cAdvisor + VERSION: 1 + PING: 1 + POST: 0 # no mutations, ever + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + security_opt: + - no-new-privileges:true + mem_limit: 64m + pids_limit: 128 + networks: + - docker_socket + logging: *default-logging + + alloy: + image: grafana/alloy:v1.18.1@sha256:0f4434c92b3e6cdac38bb129b344e1790c246f7b6e2eaffcc16a5fa363240e33 + restart: unless-stopped + # Root to read root-only files under the host mounts below. The Docker API is + # behind the proxy, so root here does not imply control of the daemon. + user: root + command: + - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy/data + # The diagnostics UI has no auth; keep it on loopback. + - --server.http.listen-addr=127.0.0.1:12345 + environment: + ENVIRONMENT: ${ENVIRONMENT:?telemetry overlay requires ENVIRONMENT} + # Must match the value bootstrap.sh was run with on the monitoring host. + PROJECT: ${PROJECT:?telemetry overlay requires PROJECT} + # The Compose project (docker's -p), not PROJECT above. Compose always injects + # it into interpolation, so Alloy's container filter follows the sibling + # containers' label. + COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:?required by the telemetry overlay} + OTLP_AUTH_TOKEN: ${OTLP_AUTH_TOKEN:?telemetry overlay requires OTLP_AUTH_TOKEN} + # Optional second credential for projects whose egress crosses a WAF; the + # WAF-skip rule matches this header instead of the bearer token. + TELEMETRY_EDGE_KEY: ${TELEMETRY_EDGE_KEY:-} + volumes: + - ./deploy/alloy/config.alloy:/etc/alloy/config.alloy:ro # vendored, never edited + # node_exporter reads the host's /proc, /sys and filesystems. /rootfs gives + # the container read access to the whole host filesystem, the price of host + # disk-usage metrics from a container. + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + # cAdvisor's filesystem reads; container metadata comes over the socket proxy. + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + # Read positions; without this every restart re-ships what is still on disk. + - alloy_data:/var/lib/alloy/data + # cAdvisor finds containers by walking the cgroup tree, not through the Docker + # API. Under Docker's default private cgroup namespace it sees only its own, as + # "/", so every container_* metric collapses to one root series with no `name` + # label and the container alerts match nothing. No error is logged. + cgroup: host + # OOM-kill events come from the kernel log. Without the device AND CAP_SYSLOG, + # container_oom_events_total never increments ("Could not configure a source for + # OOM detection" in the agent log). + devices: + - /dev/kmsg:/dev/kmsg + # DAC_READ_SEARCH reads root-only files under the host mounts without the + # write-side bypass DAC_OVERRIDE would grant. + cap_drop: + - ALL + cap_add: + - SYSLOG + - DAC_READ_SEARCH + security_opt: + - no-new-privileges:true + # During a collector outage the export queue grows in memory; without a ceiling + # the OOM killer picks by RSS and may take the application instead. Keep in step + # with the memory_limiter in config.alloy. + mem_limit: 512m + pids_limit: 512 + networks: + - egress + - docker_socket + # Never through itself: shipping a log about failing to ship logs has no exit. + logging: *default-logging + +volumes: + alloy_data: + +networks: + # Alloy's way out, and where the GPU exporter overlay attaches. A project that + # already defines `egress` keeps its definition; Compose merges by name. + egress: + # Only the proxied Docker API. No gateway. + docker_socket: + internal: true diff --git a/templates/run_scheduled.sh b/templates/run_scheduled.sh new file mode 100755 index 0000000..3237fa0 --- /dev/null +++ b/templates/run_scheduled.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# VENDORED from the CML monitoring repository at a pinned tag. +# +# Run one scheduled job and report the result to a dead man's switch (healthchecks.io). +# This is the one piece of monitoring that does not share fate with the observability +# stack: a push to an external endpoint still arrives when the telemetry pipeline is +# what broke, and its absence is the alarm. +# +# Usage: run_scheduled.sh +# job: selects the `just` recipe and the PING_ URL variable. +# env: passed through to the recipe. +# +# Ping URLs are capability URLs: they come from the host file the systemd units load, +# never from the repository. bootstrap.sh prints the PING_* block. An unset URL +# disables the ping for that job without failing it. +set -uo pipefail + +job="${1:-}" +env_name="${2:-}" + +if [[ -z "$job" || -z "$env_name" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi +# Job and env become part of a variable name below; a stray character would look up +# the wrong variable and silently disable the ping. +if [[ ! "$job" =~ ^[A-Za-z0-9_-]+$ || ! "$env_name" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "error: job and env must match [A-Za-z0-9_-]+" >&2 + exit 2 +fi + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" || exit 1 + +JUST_BIN="${JUST_BIN:-just}" + +# One recipe per job, named the same. Projects without `just` can point JUST_BIN at any +# runner with the same shape. +command=("$JUST_BIN" "$job" "$env_name") + +# One URL per job: a shared one would let a frequent job's pings mask a rare one's silence. +url_var="PING_${job//-/_}" +url_var="${url_var^^}" +ping_url="${!url_var:-}" + +output_file="$(mktemp)" +trap 'rm -f "$output_file"' EXIT + +# The job's output is the failure body, so the alert carries the reason. This puts job +# output in a third party's hands; keep credentials out of it. +ping_fail() { + curl -fsS -m 10 --retry 3 --data-binary "@${output_file}" "${ping_url}/fail" -o /dev/null \ + || echo "WARNING: failure ping to ${url_var} failed" >&2 +} + +# A killed job must still report: systemd's TimeoutStartSec TERMs the whole cgroup, and +# without this trap a hung job would send neither ping. The job runs in the background +# so `wait` can be interrupted; the child receives the same TERM from systemd. +# shellcheck disable=SC2329 # invoked via the TERM/INT traps below +on_terminate() { + local sig="$1" + echo "run_scheduled: received SIG${sig}; job killed (likely a systemd timeout)" >>"$output_file" + cat "$output_file" + if [[ -n "$ping_url" ]]; then + ping_fail + fi + rm -f "$output_file" + exit 143 +} +trap 'on_terminate TERM' TERM +trap 'on_terminate INT' INT + +status=0 +"${command[@]}" >"$output_file" 2>&1 & +wait $! || status=$? +cat "$output_file" + +if [[ -z "$ping_url" ]]; then + exit "$status" +fi + +if [[ "$status" -eq 0 ]]; then + curl -fsS -m 10 --retry 3 "$ping_url" -o /dev/null \ + || echo "WARNING: success ping to ${url_var} failed" >&2 +else + ping_fail +fi + +exit "$status"