From 269bc50ec1cb909e0b7bfdad1fad77bbc38d1088 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:30:46 +0000 Subject: [PATCH 1/8] ci(dependabot): group minor bumps and github-actions --- .github/dependabot.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f0e0153..c0f21d6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,15 @@ version: 2 updates: - # Hub images: patch bumps ride together; a minor or major comes on its own, - # so a red PR names the one image that broke. + # Hub images: patch and minor bumps ride together, a major comes on its own + # so a red PR names the one image that broke. `just check` runs on the group + # PR; bisect it by dropping images from the branch if it goes red. - package-ecosystem: "docker-compose" directory: "/" schedule: interval: "weekly" groups: - hub-patches: - update-types: ["patch"] + hub: + update-types: ["minor", "patch"] # The spoke images every project host runs. - package-ecosystem: "docker-compose" @@ -16,8 +17,8 @@ updates: schedule: interval: "weekly" groups: - spoke-patches: - update-types: ["patch"] + spoke: + update-types: ["minor", "patch"] # The demo: one PR per month for all of it. - package-ecosystem: "docker" @@ -52,3 +53,6 @@ updates: directory: "/" schedule: interval: "weekly" + groups: + actions: + patterns: ["*"] From dd101ba11d0aec62cc58a527885c366c7f8ec1ec Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:39:51 +0000 Subject: [PATCH 2/8] fix(collector): stop Prometheus rejecting the ingest counters Three faults stacked behind one HTTP 400 every five or six minutes, found by dumping the rejected payload through a debug exporter on metrics/count. - batch sat after deltatocumulative and handed it datapoints whose timestamps had regressed, ~1/min. Removing it took delta.ErrOutOfOrder from 55 per collector hour to none. - The count connector emits one ResourceMetrics per incoming batch and merges none of them, so a spoke shipping 50 log batches at once made 50 samples for one series microseconds apart. interval collapses a stream to one sample per 30s scrape. - The counters inherited the source telemetry's own timestamp, so a log backlog minted counter samples up to 11 hours old, past the 30m out_of_order_time_window. They are stamped at ingest now, which is what an ingest counter means. It has to run after interval, or several datapoints of one stream would restamp into duplicate samples. - deltatocumulative tracked an unbounded number of streams. The counters carry the sender's service.instance.id, so every service restart added one. Capped at 5000. - The counters export through their own otlp_http/prometheus_count, so send_failed_metric_points is attributable by pipeline rather than pooling counter drops with a spoke's application metrics. telemetry_logs_total now reports all eight senders, where five had never reached Prometheus at all, and none of them reset. That counter is what ProjectTelemetrySilent and ProjectsUncovered read. --- CHANGELOG.md | 27 +++++++++++++++++++ config/otel-collector.yaml | 54 +++++++++++++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db1d9b3..db75af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow [SemVer](https://semver.org/). +## [Unreleased] + +### Fixed + +- **Prometheus rejected the ingest counters every few minutes**, and three + separate faults were behind it. `batch` sat after `deltatocumulative` and + handed it datapoints whose timestamps had regressed, ~1/min; removing it + took `delta.ErrOutOfOrder` from 55 per collector hour to none. The count + connector emits one `ResourceMetrics` per incoming batch and merges none of + them, so a spoke shipping 50 log batches at once produced 50 samples for one + series microseconds apart; `interval` now collapses a stream to one sample + per 30s scrape. And the counters inherited the source telemetry's own + timestamp, so a backlog of logs minted counter samples up to 11 hours old, + past the 30m `out_of_order_time_window`; they are now stamped when the hub + receives them, which is what an ingest counter means. + + Five of the eight `telemetry_logs_total` senders had never appeared in + Prometheus at all, and the ones that did reset on their own. Both are what + `ProjectTelemetrySilent` and `ProjectsUncovered` read. +- The ingest counters export through their own `otlp_http/prometheus_count` + exporter. Same endpoint, but `otelcol_exporter_send_failed_metric_points` + is now labelled by pipeline, so a rejected batch of counters and a rejected + batch of a spoke's application metrics are no longer one number. +- `deltatocumulative` tracked an unbounded number of streams. Its counters + carry the sender's `service.instance.id`, so a service that mints a new id + on each restart added a stream every time. Capped at 5000. + ## [0.3.0] - 2026-09-06 The hub runs the department's telemetry in production, with one spoke on the diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index 523aceb..5cf51ed 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -74,9 +74,39 @@ processors: # The count connector emits delta sums and the Prometheus OTLP receiver refuses # them ("invalid temporality and type combination"). Converting here keeps the - # conversion state to the handful of counter streams, rather than turning on - # Prometheus's experimental global delta handling. - deltatocumulative: {} + # conversion state to the counter streams, rather than turning on Prometheus's + # experimental global delta handling. + # + # A stream is per sender resource, not per project: the counters carry the + # sender's service.instance.id, so one spoke already tracks ~28 and a service + # that mints a new instance id on each restart adds one every time. The default + # limit is unbounded. Cap it: at 28 streams the process sits at ~51 MiB heap + # against the 400 MiB limiter below, so this is a runaway guard, not a budget. + deltatocumulative: + max_streams: 5000 + + # The count connector emits one ResourceMetrics per incoming batch and never + # merges identical ones, so a spoke that ships 50 ResourceLogs at once produces + # 50 datapoints for one counter stream, microseconds apart. Prometheus sees + # duplicate samples for one series in one write and rejects the whole request + # with a 400. Collapsing each stream to its latest value per scrape interval is + # what the counter meant all along; 30s matches scrape_interval in + # config/prometheus.yaml. + interval: + interval: 30s + + # These counters answer "what has the hub received", so they are stamped when the + # hub receives it. Left alone, the count connector inherits the source telemetry's + # own timestamp: a spoke shipping a backlog of logs mints counter samples hours + # old, Prometheus refuses anything past out_of_order_time_window (30m in + # config/prometheus.yaml) and 400s the whole write. Must run after `interval`, + # which collapses a stream to one datapoint first; restamping several datapoints + # of one stream would make them duplicate samples instead. + transform/count_ingest_time: + metric_statements: + - context: datapoint + statements: + - set(time, Now()) # Sized to the container's mem_limit (512m in compose.yml); keep the two in step. memory_limiter: @@ -111,6 +141,14 @@ exporters: sending_queue: *queue retry_on_failure: *retry + # Same endpoint, separate instance, so otelcol_exporter_send_failed_metric_points + # is labelled by pipeline. Shared, a rejected batch of ingest counters and a + # rejected batch of a spoke's application metrics are one number. + otlp_http/prometheus_count: + endpoint: http://prometheus:9090/api/v1/otlp + sending_queue: *queue + retry_on_failure: *retry + service: extensions: [bearertokenauth, file_storage] pipelines: @@ -126,11 +164,15 @@ service: receivers: [otlp] processors: [memory_limiter, resource/department, batch] exporters: [otlp_http/prometheus, count] - # The counters themselves. Same limiter and batcher as every other pipeline. + # The counters themselves. No batcher, unlike every other pipeline: after + # deltatocumulative it handed on datapoints whose timestamps had regressed, + # ~1/min, which deltatocumulative then refused as out of order. Without it that + # stopped dead. `interval` then collapses each stream to one sample per scrape, + # which is what makes the export safe for Prometheus; see the processor. metrics/count: receivers: [count] - processors: [memory_limiter, deltatocumulative, batch] - exporters: [otlp_http/prometheus] + processors: [memory_limiter, deltatocumulative, interval, transform/count_ingest_time] + exporters: [otlp_http/prometheus_count] telemetry: metrics: level: basic From 4cb3830eb39cec7b1aeeb86624bde7d0b709e79c Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:40:01 +0000 Subject: [PATCH 3/8] fix(alerting): exempt demo/demo from ProjectsUncovered `just demo` runs a throwaway stack under its own compose project and is torn down again, so the rule fires against the demo's own hub. Bootstrapping the pair instead would leave a ProjectTelemetrySilent rule firing forever once the demo is removed. - Exempt as a pair, not by project name, so a real project called demo in a real environment is still caught. - Re-rendering through bootstrap.sh also picked up a coverage.yaml that had drifted from the template's `unknown` wording. --- CHANGELOG.md | 4 ++++ config/grafana/alerting/coverage.yaml | 12 +++++++++--- templates/alerting/coverage.yaml.tmpl | 10 ++++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db75af3..c79e8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ Notable changes to this stack. Format follows - `deltatocumulative` tracked an unbounded number of streams. Its counters carry the sender's `service.instance.id`, so a service that mints a new id on each restart added a stream every time. Capped at 5000. +- `ProjectsUncovered` no longer fires for `demo/demo`, the pair `just demo` + sets. Bootstrapping it would leave a `ProjectTelemetrySilent` rule firing + forever once the demo is torn down. Exempt as a pair, so a real project + named `demo` in a real environment is still caught. ## [0.3.0] - 2026-09-06 diff --git a/config/grafana/alerting/coverage.yaml b/config/grafana/alerting/coverage.yaml index 8c62d06..c2aa0d3 100644 --- a/config/grafana/alerting/coverage.yaml +++ b/config/grafana/alerting/coverage.yaml @@ -4,7 +4,7 @@ # Fires on any series whose project/env pair has no rendered rule file: a project that # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule. # -# Covered right now: relab/staging +# Covered right now: relab/staging (plus demo/demo, exempt: see below) apiVersion: 1 @@ -35,7 +35,13 @@ groups: # counters for every sender, whatever signal it sends. A bare # {project!=""} would scan every project-labelled series in the TSDB on # each evaluation, and would still miss a logs-only or traces-only project. - expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="relab",env="staging"})) + # + # demo/demo is exempt. It is this repo's own `just demo` overlay, which + # runs in its own throwaway stack and is torn down again; bootstrapping + # it would leave a ProjectTelemetrySilent rule firing forever after + # `just demo-down`. Exempt as a pair, so a real project named demo in a + # real env is still caught. + expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"})) - refId: FIRING datasourceUid: __expr__ model: @@ -50,4 +56,4 @@ groups: severity: warning annotations: summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules" - description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host." + description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host. A literal `unknown` means the sender set no project or env resource attribute at all: fix the sender (docs/ONBOARDING.md), do not bootstrap a project by that name." diff --git a/templates/alerting/coverage.yaml.tmpl b/templates/alerting/coverage.yaml.tmpl index ae87b7f..bb13ae1 100644 --- a/templates/alerting/coverage.yaml.tmpl +++ b/templates/alerting/coverage.yaml.tmpl @@ -4,7 +4,7 @@ # Fires on any series whose project/env pair has no rendered rule file: a project that # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule. # -# Covered right now: __COVERED__ +# Covered right now: __COVERED__ (plus demo/demo, exempt: see below) apiVersion: 1 @@ -35,7 +35,13 @@ groups: # counters for every sender, whatever signal it sends. A bare # {project!=""} would scan every project-labelled series in the TSDB on # each evaluation, and would still miss a logs-only or traces-only project. - expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) (__COVERED_EXPR__)) + # + # demo/demo is exempt. It is this repo's own `just demo` overlay, which + # runs in its own throwaway stack and is torn down again; bootstrapping + # it would leave a ProjectTelemetrySilent rule firing forever after + # `just demo-down`. Exempt as a pair, so a real project named demo in a + # real env is still caught. + expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or __COVERED_EXPR__)) - refId: FIRING datasourceUid: __expr__ model: From 46221e69fce5bf2d250cb35d1ea33585bc354040 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:43:07 +0000 Subject: [PATCH 4/8] fix(bootstrap): reject malformed COVERS markers The project and env arguments are validated, but the pairs read back from the rendered files' '# COVERS:' markers were not, and they land in a sed replacement and a PromQL label value. Fail the run instead of rendering a rule that silently never matches. --- bootstrap.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bootstrap.sh b/bootstrap.sh index 8a49aa7..ea13e02 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -62,6 +62,13 @@ fi # rendered file, plus the pair being bootstrapped now. pairs="$({ sed -n 's/^# COVERS: //p' "$out_dir"/project-*.yaml 2>/dev/null || true echo "$project $env_name"; } | sort -u)" +# The args were validated above, but these markers come off disk and land in a +# sed replacement and a PromQL label value. Refuse the run rather than render a +# rule that silently never matches; the fix is to delete the edited file. +while read -r p e; do + [[ "$p" =~ ^[a-z0-9][a-z0-9-]*$ && "$e" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || { echo "error: bad '# COVERS:' marker in $out_dir: '$p $e'" >&2; exit 2; } +done <<<"$pairs" # All pairs are re-rendered, so a template fix reaches every project. while read -r p e; do From 9fac4e949256a22d1e4803dcd4b6e8ced625141b Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:45:05 +0000 Subject: [PATCH 5/8] fix(runbook): clarify collector's authentication scheme and token handling --- docs/RUNBOOK.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 4d4c146..81cd160 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -87,6 +87,10 @@ on the host. The token is shared, and the collector does not check `project` or `env` against the sender. Every project host can therefore spoof another project's labels. + + Bearer is the only scheme the collector accepts; a second HTTP Basic + listener was considered and rejected. A sender that cannot set a raw + `Authorization` header ships through the host's Alloy agent instead. - **Tunnel token:** rotate the tunnel secret in Cloudflare Zero Trust, then run `cd infra && tofu apply` to refresh the token data source, then read the new value with `tofu output -raw tunnel_token`. To rotate from code From 77e339d9a3b0e0c1556d98f7ac189f90347c1abe Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:49:24 +0000 Subject: [PATCH 6/8] feat(alerting): catch cardinality spikes, lower the ceiling to 30k The ceiling was 67 spokes away at ~1,400 series each, and 100k would have put Prometheus near its 2g mem_limit before the warning arrived. A single static threshold also cannot separate the two things it was being asked to catch: onboarding is slow and planned, an exploding label is fast and is the failure that hurts. - PrometheusCardinalityHigh drops to 30k, ~4x the ~7k baseline and ~18 spokes of room, firing at roughly a quarter of Prometheus's memory limit. - New PrometheusCardinalitySpike on 5,000 new series in 30 minutes, which needs no re-tuning as the fleet grows. Calibrated on the hub: steady state moves under 100 series/hour, one spoke onboarding adds ~1,400. Measured over 30m so the window fits inside the shared relativeTimeRange, and a head-block truncation only ever moves the delta negative, so it cannot false-fire. - RUNBOOK quotes both numbers. --- CHANGELOG.md | 14 ++++++++++ config/grafana/alerting/rules.yaml | 42 +++++++++++++++++++++++++++--- docs/RUNBOOK.md | 7 +++-- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c79e8c1..f056a23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,20 @@ Notable changes to this stack. Format follows forever once the demo is torn down. Exempt as a pair, so a real project named `demo` in a real environment is still caught. +### Changed + +- **`PrometheusCardinalityHigh` fires at 30k active series, not 100k.** At + ~1,400 series per spoke the old ceiling was 67 spokes away, and 100k would + have put Prometheus near its 2g `mem_limit` before the warning arrived. 30k + is ~18 spokes of room and fires at about a quarter of that limit. +- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes. A + ceiling only catches slow growth and has to be raised as spokes are + onboarded; this catches the failure that actually hurts, a label that + explodes, and needs no re-tuning at any fleet size. Calibrated on the hub: + steady state moves under 100 series/hour and one spoke onboarding adds + ~1,400, so the threshold clears both by a wide margin. Head-block + truncations move the delta several thousand negative, never positive. + ## [0.3.0] - 2026-09-06 The hub runs the department's telemetry in production, with one spoke on the diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml index dbdc80f..504533b 100644 --- a/config/grafana/alerting/rules.yaml +++ b/config/grafana/alerting/rules.yaml @@ -171,9 +171,13 @@ groups: Extrapolated from the last 6 hours. Find what is growing (a spoke shipping more than before, a log loop, a backup that stopped rotating) before HostDiskSpaceLow makes it urgent. - # ~14x the baseline of ~7k series with one spoke (measured after the scrape + # ~4x the baseline of ~7k series with one spoke (measured after the scrape # trimming in config/prometheus.yaml and the agent config); raise it as spokes - # are onboarded. + # are onboarded. A spoke costs ~1,400 series, so this is ~18 spokes of room, + # and it fires while Prometheus is near a quarter of its 2g mem_limit rather + # than at the wall. The ceiling only ever catches slow growth; a label that + # explodes is caught by PrometheusCardinalitySpike below, whatever the + # fleet size. - uid: prometheus-cardinality title: PrometheusCardinalityHigh !!merge <<: *rule_defaults @@ -182,18 +186,48 @@ groups: - !!merge <<: *query_node model: !!merge <<: *query_model - expr: prometheus_tsdb_head_series > 100000 + expr: prometheus_tsdb_head_series > 30000 - *firing labels: severity: warning annotations: - summary: "Prometheus holds {{ $values.QUERY }} active series, over the 100k ceiling" + summary: "Prometheus holds {{ $values.QUERY }} active series, over the 30k ceiling" description: >- Find the label that exploded with `topk(10, count by (__name__) ({__name__=~".+"}))` and `topk(10, count by (job) ({__name__=~".+"}))`, then drop it at the spoke's Alloy config or the collector. Retention is 15GB; at this rate it fills. + + # The ceiling above only catches growth, and it has to be raised as spokes + # are onboarded. This one catches the failure that actually hurts, a label + # that explodes, and needs no re-tuning at any fleet size. Calibrated on the + # hub: steady state moves under 100 series/hour, one spoke onboarding adds + # ~1,400, so 5,000 in 30 minutes clears both by a wide margin. Measured over + # 30m, not an hour, so the window fits inside the shared relativeTimeRange. + # A head-block truncation moves this several thousand negative, never + # positive, so it cannot false-fire. + - uid: prometheus-cardinality-spike + title: PrometheusCardinalitySpike + !!merge <<: *rule_defaults + for: 15m + data: + - !!merge <<: *query_node + model: + !!merge <<: *query_model + expr: delta(prometheus_tsdb_head_series[30m]) > 5000 + - *firing + labels: + severity: warning + annotations: + summary: "Prometheus gained {{ $values.QUERY }} active series in 30 minutes" + description: >- + Series are appearing far faster than onboarding a spoke would explain, + so a label has most likely exploded. Find it with + `topk(10, count by (__name__) ({__name__=~".+"}))` and + `topk(10, count by (job) ({__name__=~".+"}))`, then drop it at the + spoke's Alloy config or the collector. If a batch of spokes really was + onboarded at once, this is expected and clears on its own. - orgId: 1 name: container-lifecycle folder: Stack alerts diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 81cd160..3de4929 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -36,8 +36,11 @@ Retention is only partially size-bounded: Loki and Tempo cannot cap their total size, so the disk alert at 80% is the backstop. Two warnings fire earlier. `HostDiskFilling` means a 6-hour linear fit says a filesystem is full within 3 days. `PrometheusCardinalityHigh` -means active series passed 100k, about 14x the baseline of ~7k with one -spoke. Series count, not time, is what grows the TSDB. +means active series passed 30k, about 4x the baseline of ~7k with one spoke, +which is roughly 18 spokes of room at ~1,400 series each. +`PrometheusCardinalitySpike` means 5,000 series appeared in 30 minutes, far +faster than onboarding explains, so a label has most likely exploded. Series +count, not time, is what grows the TSDB. When one fires: From 083c70add7f65fa2f51896255282d1bb531cdba4 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 02:53:23 +0000 Subject: [PATCH 7/8] chore(release): cut 0.3.1 - extract valid_pair() so argv and on-disk COVERS markers share one check - drop file_storage from the count exporter's queue; the counters resume after a restart, so in-flight samples are not worth a queue on disk - note that the max_streams heap figure predates the interval processor - condense the changelog and date the release --- CHANGELOG.md | 58 +++++++++++++++----------------------- bootstrap.sh | 5 ++-- config/otel-collector.yaml | 12 +++++--- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f056a23..e4885df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,50 +4,38 @@ Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow [SemVer](https://semver.org/). -## [Unreleased] +## [0.3.1] - 2026-09-07 ### Fixed -- **Prometheus rejected the ingest counters every few minutes**, and three - separate faults were behind it. `batch` sat after `deltatocumulative` and - handed it datapoints whose timestamps had regressed, ~1/min; removing it - took `delta.ErrOutOfOrder` from 55 per collector hour to none. The count - connector emits one `ResourceMetrics` per incoming batch and merges none of - them, so a spoke shipping 50 log batches at once produced 50 samples for one - series microseconds apart; `interval` now collapses a stream to one sample - per 30s scrape. And the counters inherited the source telemetry's own - timestamp, so a backlog of logs minted counter samples up to 11 hours old, - past the 30m `out_of_order_time_window`; they are now stamped when the hub - receives them, which is what an ingest counter means. - - Five of the eight `telemetry_logs_total` senders had never appeared in - Prometheus at all, and the ones that did reset on their own. Both are what - `ProjectTelemetrySilent` and `ProjectsUncovered` read. +- **Prometheus rejected the ingest counters every few minutes.** Three faults: + `batch` after `deltatocumulative` reordered timestamps (~1/min); the count + connector emitted one sample per incoming batch, so 50 log batches became 50 + samples of one series microseconds apart; and the counters carried the source + telemetry's timestamp, so a log backlog minted samples hours past the 30m + `out_of_order_time_window`. `batch` is gone from that pipeline, `interval` + collapses each stream to one sample per 30s, and samples are stamped on + receipt. Five of eight `telemetry_logs_total` senders had never reached + Prometheus; both `ProjectTelemetrySilent` and `ProjectsUncovered` read them. - The ingest counters export through their own `otlp_http/prometheus_count` - exporter. Same endpoint, but `otelcol_exporter_send_failed_metric_points` - is now labelled by pipeline, so a rejected batch of counters and a rejected - batch of a spoke's application metrics are no longer one number. -- `deltatocumulative` tracked an unbounded number of streams. Its counters - carry the sender's `service.instance.id`, so a service that mints a new id - on each restart added a stream every time. Capped at 5000. + exporter, so `otelcol_exporter_send_failed_metric_points` separates them per + exporter from a spoke's application metrics. +- `deltatocumulative` streams capped at 5000; a service that mints a new + `service.instance.id` per restart added one each time. - `ProjectsUncovered` no longer fires for `demo/demo`, the pair `just demo` - sets. Bootstrapping it would leave a `ProjectTelemetrySilent` rule firing - forever once the demo is torn down. Exempt as a pair, so a real project - named `demo` in a real environment is still caught. + sets; bootstrapping it would leave `ProjectTelemetrySilent` firing forever + after teardown. Exempt as a pair, so a real project named `demo` is still + caught. ### Changed - **`PrometheusCardinalityHigh` fires at 30k active series, not 100k.** At - ~1,400 series per spoke the old ceiling was 67 spokes away, and 100k would - have put Prometheus near its 2g `mem_limit` before the warning arrived. 30k - is ~18 spokes of room and fires at about a quarter of that limit. -- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes. A - ceiling only catches slow growth and has to be raised as spokes are - onboarded; this catches the failure that actually hurts, a label that - explodes, and needs no re-tuning at any fleet size. Calibrated on the hub: - steady state moves under 100 series/hour and one spoke onboarding adds - ~1,400, so the threshold clears both by a wide margin. Head-block - truncations move the delta several thousand negative, never positive. + ~1,400 series per spoke, 100k would have put Prometheus near its 2g + `mem_limit` before the warning arrived; 30k is ~18 spokes of room. +- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes. + Catches a label that explodes, which a ceiling cannot, and needs no + re-tuning as spokes are onboarded: steady state moves under 100 series/hour + and one onboarding adds ~1,400. ## [0.3.0] - 2026-09-06 diff --git a/bootstrap.sh b/bootstrap.sh index ea13e02..bf416de 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -20,7 +20,8 @@ if [[ -z "$project" || -z "$env_name" ]]; then fi # These become label values, a rule uid, and a filename; a quote or brace in a # label value produces a rule that silently never matches. -if [[ ! "$project" =~ ^[a-z0-9][a-z0-9-]*$ || ! "$env_name" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then +valid_pair() { [[ "$1" =~ ^[a-z0-9][a-z0-9-]*$ && "$2" =~ ^[a-z0-9][a-z0-9-]*$ ]]; } +if ! valid_pair "$project" "$env_name"; then echo "error: project and env must match [a-z0-9][a-z0-9-]* (lowercase, no spaces)" >&2 exit 2 fi @@ -66,7 +67,7 @@ pairs="$({ sed -n 's/^# COVERS: //p' "$out_dir"/project-*.yaml 2>/dev/null || tr # sed replacement and a PromQL label value. Refuse the run rather than render a # rule that silently never matches; the fix is to delete the edited file. while read -r p e; do - [[ "$p" =~ ^[a-z0-9][a-z0-9-]*$ && "$e" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + valid_pair "$p" "$e" \ || { echo "error: bad '# COVERS:' marker in $out_dir: '$p $e'" >&2; exit 2; } done <<<"$pairs" diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index 5cf51ed..bd3218e 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -80,8 +80,9 @@ processors: # A stream is per sender resource, not per project: the counters carry the # sender's service.instance.id, so one spoke already tracks ~28 and a service # that mints a new instance id on each restart adds one every time. The default - # limit is unbounded. Cap it: at 28 streams the process sits at ~51 MiB heap - # against the 400 MiB limiter below, so this is a runaway guard, not a budget. + # limit is unbounded. Cap it: at 28 streams the process sat at ~51 MiB heap + # (measured before `interval` below added its own per-stream state) against the + # 400 MiB limiter, so this is a runaway guard, not a budget. deltatocumulative: max_streams: 5000 @@ -143,10 +144,13 @@ exporters: # Same endpoint, separate instance, so otelcol_exporter_send_failed_metric_points # is labelled by pipeline. Shared, a rejected batch of ingest counters and a - # rejected batch of a spoke's application metrics are one number. + # rejected batch of a spoke's application metrics are one number. No file_storage: + # the counters resume counting after a restart, so in-flight samples are not + # worth a fourth queue on disk. otlp_http/prometheus_count: endpoint: http://prometheus:9090/api/v1/otlp - sending_queue: *queue + sending_queue: + queue_size: 1000 retry_on_failure: *retry service: From 8e3442559f9bee5b5ccdbae79ce453b4f7972a5b Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 7 Sep 2026 04:13:09 +0000 Subject: [PATCH 8/8] docs: shorten the comments and changelog added on this branch --- CHANGELOG.md | 41 ++++++++++------------ bootstrap.sh | 6 ++-- config/grafana/alerting/coverage.yaml | 9 +++-- config/grafana/alerting/rules.yaml | 28 ++++++--------- config/otel-collector.yaml | 49 +++++++++++---------------- docs/RUNBOOK.md | 5 ++- templates/alerting/coverage.yaml.tmpl | 9 +++-- 7 files changed, 61 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4885df..0019082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,34 +8,29 @@ Notable changes to this stack. Format follows ### Fixed -- **Prometheus rejected the ingest counters every few minutes.** Three faults: - `batch` after `deltatocumulative` reordered timestamps (~1/min); the count - connector emitted one sample per incoming batch, so 50 log batches became 50 - samples of one series microseconds apart; and the counters carried the source - telemetry's timestamp, so a log backlog minted samples hours past the 30m - `out_of_order_time_window`. `batch` is gone from that pipeline, `interval` - collapses each stream to one sample per 30s, and samples are stamped on - receipt. Five of eight `telemetry_logs_total` senders had never reached - Prometheus; both `ProjectTelemetrySilent` and `ProjectsUncovered` read them. -- The ingest counters export through their own `otlp_http/prometheus_count` - exporter, so `otelcol_exporter_send_failed_metric_points` separates them per - exporter from a spoke's application metrics. -- `deltatocumulative` streams capped at 5000; a service that mints a new +- **Prometheus rejected the ingest counters every few minutes**, so five of + eight `telemetry_logs_total` senders never reached it. Both coverage alerts + read that counter. Three faults: `batch` reordered timestamps after + `deltatocumulative`; the count connector emitted one sample per incoming + batch, which Prometheus rejects as duplicates; and the counters carried the + source telemetry's timestamp, so a log backlog fell outside + `out_of_order_time_window`. The pipeline now drops `batch`, collapses each + stream to one sample per 30s, and stamps samples on receipt. +- The ingest counters get their own `otlp_http/prometheus_count` exporter, so + their send failures are counted apart from a spoke's application metrics. +- `deltatocumulative` streams are capped at 5000. A service that mints a new `service.instance.id` per restart added one each time. - `ProjectsUncovered` no longer fires for `demo/demo`, the pair `just demo` - sets; bootstrapping it would leave `ProjectTelemetrySilent` firing forever - after teardown. Exempt as a pair, so a real project named `demo` is still - caught. + sets. A real project named `demo` is still caught. ### Changed -- **`PrometheusCardinalityHigh` fires at 30k active series, not 100k.** At - ~1,400 series per spoke, 100k would have put Prometheus near its 2g - `mem_limit` before the warning arrived; 30k is ~18 spokes of room. -- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes. - Catches a label that explodes, which a ceiling cannot, and needs no - re-tuning as spokes are onboarded: steady state moves under 100 series/hour - and one onboarding adds ~1,400. +- **`PrometheusCardinalityHigh` fires at 30k active series, not 100k.** 100k + would have put Prometheus near its 2g `mem_limit` before the warning + arrived; 30k is ~18 spokes of room. +- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes: a + label that explodes, which a ceiling cannot catch. One spoke onboarding adds + ~1,400. ## [0.3.0] - 2026-09-06 diff --git a/bootstrap.sh b/bootstrap.sh index bf416de..e8d41d7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -63,9 +63,9 @@ fi # rendered file, plus the pair being bootstrapped now. pairs="$({ sed -n 's/^# COVERS: //p' "$out_dir"/project-*.yaml 2>/dev/null || true echo "$project $env_name"; } | sort -u)" -# The args were validated above, but these markers come off disk and land in a -# sed replacement and a PromQL label value. Refuse the run rather than render a -# rule that silently never matches; the fix is to delete the edited file. +# The markers come off disk and land in a sed replacement and a PromQL label +# value. Refuse the run rather than render a rule that silently never matches; +# the fix is to delete the edited file. while read -r p e; do valid_pair "$p" "$e" \ || { echo "error: bad '# COVERS:' marker in $out_dir: '$p $e'" >&2; exit 2; } diff --git a/config/grafana/alerting/coverage.yaml b/config/grafana/alerting/coverage.yaml index c2aa0d3..58b2cb1 100644 --- a/config/grafana/alerting/coverage.yaml +++ b/config/grafana/alerting/coverage.yaml @@ -36,11 +36,10 @@ groups: # {project!=""} would scan every project-labelled series in the TSDB on # each evaluation, and would still miss a logs-only or traces-only project. # - # demo/demo is exempt. It is this repo's own `just demo` overlay, which - # runs in its own throwaway stack and is torn down again; bootstrapping - # it would leave a ProjectTelemetrySilent rule firing forever after - # `just demo-down`. Exempt as a pair, so a real project named demo in a - # real env is still caught. + # demo/demo is this repo's own `just demo` overlay. Bootstrapping it + # would leave ProjectTelemetrySilent firing forever after `just + # demo-down`, so it is exempt as a pair; a real project named demo in + # a real env is still caught. expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"})) - refId: FIRING datasourceUid: __expr__ diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml index 504533b..4f1247c 100644 --- a/config/grafana/alerting/rules.yaml +++ b/config/grafana/alerting/rules.yaml @@ -174,10 +174,8 @@ groups: # ~4x the baseline of ~7k series with one spoke (measured after the scrape # trimming in config/prometheus.yaml and the agent config); raise it as spokes # are onboarded. A spoke costs ~1,400 series, so this is ~18 spokes of room, - # and it fires while Prometheus is near a quarter of its 2g mem_limit rather - # than at the wall. The ceiling only ever catches slow growth; a label that - # explodes is caught by PrometheusCardinalitySpike below, whatever the - # fleet size. + # and it fires near a quarter of Prometheus's 2g mem_limit rather than at + # the wall. Sudden growth is PrometheusCardinalitySpike's job, below. - uid: prometheus-cardinality title: PrometheusCardinalityHigh !!merge <<: *rule_defaults @@ -199,14 +197,10 @@ groups: spoke's Alloy config or the collector. Retention is 15GB; at this rate it fills. - # The ceiling above only catches growth, and it has to be raised as spokes - # are onboarded. This one catches the failure that actually hurts, a label - # that explodes, and needs no re-tuning at any fleet size. Calibrated on the - # hub: steady state moves under 100 series/hour, one spoke onboarding adds - # ~1,400, so 5,000 in 30 minutes clears both by a wide margin. Measured over - # 30m, not an hour, so the window fits inside the shared relativeTimeRange. - # A head-block truncation moves this several thousand negative, never - # positive, so it cannot false-fire. + # A label that explodes, at any fleet size. Steady state on the hub moves + # under 100 series/hour and one spoke onboarding adds ~1,400, so 5,000 in + # 30m clears both. 30m fits inside the shared relativeTimeRange. A head-block + # truncation moves the delta negative, never positive, so it cannot false-fire. - uid: prometheus-cardinality-spike title: PrometheusCardinalitySpike !!merge <<: *rule_defaults @@ -222,12 +216,12 @@ groups: annotations: summary: "Prometheus gained {{ $values.QUERY }} active series in 30 minutes" description: >- - Series are appearing far faster than onboarding a spoke would explain, - so a label has most likely exploded. Find it with + Far faster than onboarding a spoke explains, so a label has most + likely exploded. Find it with `topk(10, count by (__name__) ({__name__=~".+"}))` and - `topk(10, count by (job) ({__name__=~".+"}))`, then drop it at the - spoke's Alloy config or the collector. If a batch of spokes really was - onboarded at once, this is expected and clears on its own. + `topk(10, count by (job) ({__name__=~".+"}))`, then drop it in the + spoke's Alloy config or the collector. Several spokes onboarded at + once also trips this; it clears on its own. - orgId: 1 name: container-lifecycle folder: Stack alerts diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index bd3218e..a909abe 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -77,32 +77,25 @@ processors: # conversion state to the counter streams, rather than turning on Prometheus's # experimental global delta handling. # - # A stream is per sender resource, not per project: the counters carry the - # sender's service.instance.id, so one spoke already tracks ~28 and a service - # that mints a new instance id on each restart adds one every time. The default - # limit is unbounded. Cap it: at 28 streams the process sat at ~51 MiB heap - # (measured before `interval` below added its own per-stream state) against the - # 400 MiB limiter, so this is a runaway guard, not a budget. + # Streams are per sender resource, and a service that mints a new + # service.instance.id on each restart adds one every time. The default is + # unbounded. This cap is a runaway guard, not a budget: 28 streams cost ~51 MiB + # heap against the 400 MiB limiter (measured before `interval` was added). deltatocumulative: max_streams: 5000 - # The count connector emits one ResourceMetrics per incoming batch and never - # merges identical ones, so a spoke that ships 50 ResourceLogs at once produces - # 50 datapoints for one counter stream, microseconds apart. Prometheus sees - # duplicate samples for one series in one write and rejects the whole request - # with a 400. Collapsing each stream to its latest value per scrape interval is - # what the counter meant all along; 30s matches scrape_interval in - # config/prometheus.yaml. + # The count connector emits one datapoint per incoming batch, so 50 log batches + # become 50 samples of one series microseconds apart, and Prometheus rejects + # the whole write as duplicates. Keep the latest value per scrape interval; + # 30s matches scrape_interval in config/prometheus.yaml. interval: interval: 30s - # These counters answer "what has the hub received", so they are stamped when the - # hub receives it. Left alone, the count connector inherits the source telemetry's - # own timestamp: a spoke shipping a backlog of logs mints counter samples hours - # old, Prometheus refuses anything past out_of_order_time_window (30m in - # config/prometheus.yaml) and 400s the whole write. Must run after `interval`, - # which collapses a stream to one datapoint first; restamping several datapoints - # of one stream would make them duplicate samples instead. + # Stamp the counters at receipt. Left alone they carry the source telemetry's + # timestamp, so a backlog of logs mints samples hours old and Prometheus + # rejects anything past out_of_order_time_window (30m in config/prometheus.yaml). + # Must run after `interval`: restamping several datapoints of one stream would + # make them duplicates again. transform/count_ingest_time: metric_statements: - context: datapoint @@ -142,11 +135,10 @@ exporters: sending_queue: *queue retry_on_failure: *retry - # Same endpoint, separate instance, so otelcol_exporter_send_failed_metric_points - # is labelled by pipeline. Shared, a rejected batch of ingest counters and a - # rejected batch of a spoke's application metrics are one number. No file_storage: - # the counters resume counting after a restart, so in-flight samples are not - # worth a fourth queue on disk. + # Same endpoint, separate exporter, so otelcol_exporter_send_failed_metric_points + # tells rejected counters apart from a spoke's rejected application metrics. + # No file_storage: the counters resume after a restart, so in-flight samples + # are not worth a queue on disk. otlp_http/prometheus_count: endpoint: http://prometheus:9090/api/v1/otlp sending_queue: @@ -168,11 +160,8 @@ service: receivers: [otlp] processors: [memory_limiter, resource/department, batch] exporters: [otlp_http/prometheus, count] - # The counters themselves. No batcher, unlike every other pipeline: after - # deltatocumulative it handed on datapoints whose timestamps had regressed, - # ~1/min, which deltatocumulative then refused as out of order. Without it that - # stopped dead. `interval` then collapses each stream to one sample per scrape, - # which is what makes the export safe for Prometheus; see the processor. + # The counters themselves. No batcher: it reordered timestamps ~1/min, which + # deltatocumulative refused as out of order. `interval` does the collapsing. metrics/count: receivers: [count] processors: [memory_limiter, deltatocumulative, interval, transform/count_ingest_time] diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 3de4929..c717ea1 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -91,9 +91,8 @@ on the host. against the sender. Every project host can therefore spoof another project's labels. - Bearer is the only scheme the collector accepts; a second HTTP Basic - listener was considered and rejected. A sender that cannot set a raw - `Authorization` header ships through the host's Alloy agent instead. + Bearer is the only scheme the collector accepts. A sender that cannot set a + raw `Authorization` header ships through the host's Alloy agent instead. - **Tunnel token:** rotate the tunnel secret in Cloudflare Zero Trust, then run `cd infra && tofu apply` to refresh the token data source, then read the new value with `tofu output -raw tunnel_token`. To rotate from code diff --git a/templates/alerting/coverage.yaml.tmpl b/templates/alerting/coverage.yaml.tmpl index bb13ae1..d337bf5 100644 --- a/templates/alerting/coverage.yaml.tmpl +++ b/templates/alerting/coverage.yaml.tmpl @@ -36,11 +36,10 @@ groups: # {project!=""} would scan every project-labelled series in the TSDB on # each evaluation, and would still miss a logs-only or traces-only project. # - # demo/demo is exempt. It is this repo's own `just demo` overlay, which - # runs in its own throwaway stack and is torn down again; bootstrapping - # it would leave a ProjectTelemetrySilent rule firing forever after - # `just demo-down`. Exempt as a pair, so a real project named demo in a - # real env is still caught. + # demo/demo is this repo's own `just demo` overlay. Bootstrapping it + # would leave ProjectTelemetrySilent firing forever after `just + # demo-down`, so it is exempt as a pair; a real project named demo in + # a real env is still caught. expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or __COVERED_EXPR__)) - refId: FIRING datasourceUid: __expr__