diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f0e0153..c0f21d6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,15 @@ version: 2 updates: - # Hub images: patch bumps ride together; a minor or major comes on its own, - # so a red PR names the one image that broke. + # Hub images: patch and minor bumps ride together, a major comes on its own + # so a red PR names the one image that broke. `just check` runs on the group + # PR; bisect it by dropping images from the branch if it goes red. - package-ecosystem: "docker-compose" directory: "/" schedule: interval: "weekly" groups: - hub-patches: - update-types: ["patch"] + hub: + update-types: ["minor", "patch"] # The spoke images every project host runs. - package-ecosystem: "docker-compose" @@ -16,8 +17,8 @@ updates: schedule: interval: "weekly" groups: - spoke-patches: - update-types: ["patch"] + spoke: + update-types: ["minor", "patch"] # The demo: one PR per month for all of it. - package-ecosystem: "docker" @@ -52,3 +53,6 @@ updates: directory: "/" schedule: interval: "weekly" + groups: + actions: + patterns: ["*"] diff --git a/CHANGELOG.md b/CHANGELOG.md index db1d9b3..0019082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow [SemVer](https://semver.org/). +## [0.3.1] - 2026-09-07 + +### Fixed + +- **Prometheus rejected the ingest counters every few minutes**, so five of + eight `telemetry_logs_total` senders never reached it. Both coverage alerts + read that counter. Three faults: `batch` reordered timestamps after + `deltatocumulative`; the count connector emitted one sample per incoming + batch, which Prometheus rejects as duplicates; and the counters carried the + source telemetry's timestamp, so a log backlog fell outside + `out_of_order_time_window`. The pipeline now drops `batch`, collapses each + stream to one sample per 30s, and stamps samples on receipt. +- The ingest counters get their own `otlp_http/prometheus_count` exporter, so + their send failures are counted apart from a spoke's application metrics. +- `deltatocumulative` streams are capped at 5000. A service that mints a new + `service.instance.id` per restart added one each time. +- `ProjectsUncovered` no longer fires for `demo/demo`, the pair `just demo` + sets. A real project named `demo` is still caught. + +### Changed + +- **`PrometheusCardinalityHigh` fires at 30k active series, not 100k.** 100k + would have put Prometheus near its 2g `mem_limit` before the warning + arrived; 30k is ~18 spokes of room. +- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes: a + label that explodes, which a ceiling cannot catch. One spoke onboarding adds + ~1,400. + ## [0.3.0] - 2026-09-06 The hub runs the department's telemetry in production, with one spoke on the diff --git a/bootstrap.sh b/bootstrap.sh index 8a49aa7..e8d41d7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -20,7 +20,8 @@ if [[ -z "$project" || -z "$env_name" ]]; then fi # These become label values, a rule uid, and a filename; a quote or brace in a # label value produces a rule that silently never matches. -if [[ ! "$project" =~ ^[a-z0-9][a-z0-9-]*$ || ! "$env_name" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then +valid_pair() { [[ "$1" =~ ^[a-z0-9][a-z0-9-]*$ && "$2" =~ ^[a-z0-9][a-z0-9-]*$ ]]; } +if ! valid_pair "$project" "$env_name"; then echo "error: project and env must match [a-z0-9][a-z0-9-]* (lowercase, no spaces)" >&2 exit 2 fi @@ -62,6 +63,13 @@ fi # rendered file, plus the pair being bootstrapped now. pairs="$({ sed -n 's/^# COVERS: //p' "$out_dir"/project-*.yaml 2>/dev/null || true echo "$project $env_name"; } | sort -u)" +# The markers come off disk and land in a sed replacement and a PromQL label +# value. Refuse the run rather than render a rule that silently never matches; +# the fix is to delete the edited file. +while read -r p e; do + valid_pair "$p" "$e" \ + || { echo "error: bad '# COVERS:' marker in $out_dir: '$p $e'" >&2; exit 2; } +done <<<"$pairs" # All pairs are re-rendered, so a template fix reaches every project. while read -r p e; do diff --git a/config/grafana/alerting/coverage.yaml b/config/grafana/alerting/coverage.yaml index 8c62d06..58b2cb1 100644 --- a/config/grafana/alerting/coverage.yaml +++ b/config/grafana/alerting/coverage.yaml @@ -4,7 +4,7 @@ # Fires on any series whose project/env pair has no rendered rule file: a project that # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule. # -# Covered right now: relab/staging +# Covered right now: relab/staging (plus demo/demo, exempt: see below) apiVersion: 1 @@ -35,7 +35,12 @@ groups: # counters for every sender, whatever signal it sends. A bare # {project!=""} would scan every project-labelled series in the TSDB on # each evaluation, and would still miss a logs-only or traces-only project. - expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="relab",env="staging"})) + # + # demo/demo is this repo's own `just demo` overlay. Bootstrapping it + # would leave ProjectTelemetrySilent firing forever after `just + # demo-down`, so it is exempt as a pair; a real project named demo in + # a real env is still caught. + expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"})) - refId: FIRING datasourceUid: __expr__ model: @@ -50,4 +55,4 @@ groups: severity: warning annotations: summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules" - description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host." + description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host. A literal `unknown` means the sender set no project or env resource attribute at all: fix the sender (docs/ONBOARDING.md), do not bootstrap a project by that name." diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml index dbdc80f..4f1247c 100644 --- a/config/grafana/alerting/rules.yaml +++ b/config/grafana/alerting/rules.yaml @@ -171,9 +171,11 @@ groups: Extrapolated from the last 6 hours. Find what is growing (a spoke shipping more than before, a log loop, a backup that stopped rotating) before HostDiskSpaceLow makes it urgent. - # ~14x the baseline of ~7k series with one spoke (measured after the scrape + # ~4x the baseline of ~7k series with one spoke (measured after the scrape # trimming in config/prometheus.yaml and the agent config); raise it as spokes - # are onboarded. + # are onboarded. A spoke costs ~1,400 series, so this is ~18 spokes of room, + # and it fires near a quarter of Prometheus's 2g mem_limit rather than at + # the wall. Sudden growth is PrometheusCardinalitySpike's job, below. - uid: prometheus-cardinality title: PrometheusCardinalityHigh !!merge <<: *rule_defaults @@ -182,18 +184,44 @@ groups: - !!merge <<: *query_node model: !!merge <<: *query_model - expr: prometheus_tsdb_head_series > 100000 + expr: prometheus_tsdb_head_series > 30000 - *firing labels: severity: warning annotations: - summary: "Prometheus holds {{ $values.QUERY }} active series, over the 100k ceiling" + summary: "Prometheus holds {{ $values.QUERY }} active series, over the 30k ceiling" description: >- Find the label that exploded with `topk(10, count by (__name__) ({__name__=~".+"}))` and `topk(10, count by (job) ({__name__=~".+"}))`, then drop it at the spoke's Alloy config or the collector. Retention is 15GB; at this rate it fills. + + # A label that explodes, at any fleet size. Steady state on the hub moves + # under 100 series/hour and one spoke onboarding adds ~1,400, so 5,000 in + # 30m clears both. 30m fits inside the shared relativeTimeRange. A head-block + # truncation moves the delta negative, never positive, so it cannot false-fire. + - uid: prometheus-cardinality-spike + title: PrometheusCardinalitySpike + !!merge <<: *rule_defaults + for: 15m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: delta(prometheus_tsdb_head_series[30m]) > 5000 + - *firing + labels: + severity: warning + annotations: + summary: "Prometheus gained {{ $values.QUERY }} active series in 30 minutes" + description: >- + Far faster than onboarding a spoke explains, so a label has most + likely exploded. Find it with + `topk(10, count by (__name__) ({__name__=~".+"}))` and + `topk(10, count by (job) ({__name__=~".+"}))`, then drop it in the + spoke's Alloy config or the collector. Several spokes onboarded at + once also trips this; it clears on its own. - orgId: 1 name: container-lifecycle folder: Stack alerts diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index 523aceb..a909abe 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -74,9 +74,33 @@ processors: # The count connector emits delta sums and the Prometheus OTLP receiver refuses # them ("invalid temporality and type combination"). Converting here keeps the - # conversion state to the handful of counter streams, rather than turning on - # Prometheus's experimental global delta handling. - deltatocumulative: {} + # conversion state to the counter streams, rather than turning on Prometheus's + # experimental global delta handling. + # + # Streams are per sender resource, and a service that mints a new + # service.instance.id on each restart adds one every time. The default is + # unbounded. This cap is a runaway guard, not a budget: 28 streams cost ~51 MiB + # heap against the 400 MiB limiter (measured before `interval` was added). + deltatocumulative: + max_streams: 5000 + + # The count connector emits one datapoint per incoming batch, so 50 log batches + # become 50 samples of one series microseconds apart, and Prometheus rejects + # the whole write as duplicates. Keep the latest value per scrape interval; + # 30s matches scrape_interval in config/prometheus.yaml. + interval: + interval: 30s + + # Stamp the counters at receipt. Left alone they carry the source telemetry's + # timestamp, so a backlog of logs mints samples hours old and Prometheus + # rejects anything past out_of_order_time_window (30m in config/prometheus.yaml). + # Must run after `interval`: restamping several datapoints of one stream would + # make them duplicates again. + transform/count_ingest_time: + metric_statements: + - context: datapoint + statements: + - set(time, Now()) # Sized to the container's mem_limit (512m in compose.yml); keep the two in step. memory_limiter: @@ -111,6 +135,16 @@ exporters: sending_queue: *queue retry_on_failure: *retry + # Same endpoint, separate exporter, so otelcol_exporter_send_failed_metric_points + # tells rejected counters apart from a spoke's rejected application metrics. + # No file_storage: the counters resume after a restart, so in-flight samples + # are not worth a queue on disk. + otlp_http/prometheus_count: + endpoint: http://prometheus:9090/api/v1/otlp + sending_queue: + queue_size: 1000 + retry_on_failure: *retry + service: extensions: [bearertokenauth, file_storage] pipelines: @@ -126,11 +160,12 @@ service: receivers: [otlp] processors: [memory_limiter, resource/department, batch] exporters: [otlp_http/prometheus, count] - # The counters themselves. Same limiter and batcher as every other pipeline. + # The counters themselves. No batcher: it reordered timestamps ~1/min, which + # deltatocumulative refused as out of order. `interval` does the collapsing. metrics/count: receivers: [count] - processors: [memory_limiter, deltatocumulative, batch] - exporters: [otlp_http/prometheus] + processors: [memory_limiter, deltatocumulative, interval, transform/count_ingest_time] + exporters: [otlp_http/prometheus_count] telemetry: metrics: level: basic diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 4d4c146..c717ea1 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -36,8 +36,11 @@ Retention is only partially size-bounded: Loki and Tempo cannot cap their total size, so the disk alert at 80% is the backstop. Two warnings fire earlier. `HostDiskFilling` means a 6-hour linear fit says a filesystem is full within 3 days. `PrometheusCardinalityHigh` -means active series passed 100k, about 14x the baseline of ~7k with one -spoke. Series count, not time, is what grows the TSDB. +means active series passed 30k, about 4x the baseline of ~7k with one spoke, +which is roughly 18 spokes of room at ~1,400 series each. +`PrometheusCardinalitySpike` means 5,000 series appeared in 30 minutes, far +faster than onboarding explains, so a label has most likely exploded. Series +count, not time, is what grows the TSDB. When one fires: @@ -87,6 +90,9 @@ on the host. The token is shared, and the collector does not check `project` or `env` against the sender. Every project host can therefore spoof another project's labels. + + Bearer is the only scheme the collector accepts. A sender that cannot set a + raw `Authorization` header ships through the host's Alloy agent instead. - **Tunnel token:** rotate the tunnel secret in Cloudflare Zero Trust, then run `cd infra && tofu apply` to refresh the token data source, then read the new value with `tofu output -raw tunnel_token`. To rotate from code diff --git a/templates/alerting/coverage.yaml.tmpl b/templates/alerting/coverage.yaml.tmpl index ae87b7f..d337bf5 100644 --- a/templates/alerting/coverage.yaml.tmpl +++ b/templates/alerting/coverage.yaml.tmpl @@ -4,7 +4,7 @@ # Fires on any series whose project/env pair has no rendered rule file: a project that # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule. # -# Covered right now: __COVERED__ +# Covered right now: __COVERED__ (plus demo/demo, exempt: see below) apiVersion: 1 @@ -35,7 +35,12 @@ groups: # counters for every sender, whatever signal it sends. A bare # {project!=""} would scan every project-labelled series in the TSDB on # each evaluation, and would still miss a logs-only or traces-only project. - expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) (__COVERED_EXPR__)) + # + # demo/demo is this repo's own `just demo` overlay. Bootstrapping it + # would leave ProjectTelemetrySilent firing forever after `just + # demo-down`, so it is exempt as a pair; a real project named demo in + # a real env is still caught. + expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or __COVERED_EXPR__)) - refId: FIRING datasourceUid: __expr__ model: