fix: stop Prometheus rejecting the ingest counters, cut 0.3.1

.github/dependabot.yml

@@ -1,23 +1,24 @@
 version: 2
 updates:
-  # Hub images: patch bumps ride together; a minor or major comes on its own,
-  # so a red PR names the one image that broke.
+  # Hub images: patch and minor bumps ride together, a major comes on its own
+  # so a red PR names the one image that broke. `just check` runs on the group
+  # PR; bisect it by dropping images from the branch if it goes red.
   - package-ecosystem: "docker-compose"
     directory: "/"
     schedule:
       interval: "weekly"
     groups:
-      hub-patches:
-        update-types: ["patch"]
+      hub:
+        update-types: ["minor", "patch"]
 
   # The spoke images every project host runs.
   - package-ecosystem: "docker-compose"
     directory: "/templates"
     schedule:
       interval: "weekly"
     groups:
-      spoke-patches:
-        update-types: ["patch"]
+      spoke:
+        update-types: ["minor", "patch"]
 
   # The demo: one PR per month for all of it.
   - package-ecosystem: "docker"
@@ -52,3 +53,6 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    groups:
+      actions:
+        patterns: ["*"]

CHANGELOG.md

@@ -4,6 +4,34 @@ Notable changes to this stack. Format follows
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow
 [SemVer](https://semver.org/).
 
+## [0.3.1] - 2026-09-07
+
+### Fixed
+
+- **Prometheus rejected the ingest counters every few minutes**, so five of
+  eight `telemetry_logs_total` senders never reached it. Both coverage alerts
+  read that counter. Three faults: `batch` reordered timestamps after
+  `deltatocumulative`; the count connector emitted one sample per incoming
+  batch, which Prometheus rejects as duplicates; and the counters carried the
+  source telemetry's timestamp, so a log backlog fell outside
+  `out_of_order_time_window`. The pipeline now drops `batch`, collapses each
+  stream to one sample per 30s, and stamps samples on receipt.
+- The ingest counters get their own `otlp_http/prometheus_count` exporter, so
+  their send failures are counted apart from a spoke's application metrics.
+- `deltatocumulative` streams are capped at 5000. A service that mints a new
+  `service.instance.id` per restart added one each time.
+- `ProjectsUncovered` no longer fires for `demo/demo`, the pair `just demo`
+  sets. A real project named `demo` is still caught.
+
+### Changed
+
+- **`PrometheusCardinalityHigh` fires at 30k active series, not 100k.** 100k
+  would have put Prometheus near its 2g `mem_limit` before the warning
+  arrived; 30k is ~18 spokes of room.
+- **New `PrometheusCardinalitySpike`**, on 5,000 new series in 30 minutes: a
+  label that explodes, which a ceiling cannot catch. One spoke onboarding adds
+  ~1,400.
+
 ## [0.3.0] - 2026-09-06
 
 The hub runs the department's telemetry in production, with one spoke on the

bootstrap.sh

@@ -20,7 +20,8 @@ if [[ -z "$project" || -z "$env_name" ]]; then
 fi
 # These become label values, a rule uid, and a filename; a quote or brace in a
 # label value produces a rule that silently never matches.
-if [[ ! "$project" =~ ^[a-z0-9][a-z0-9-]*$ || ! "$env_name" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
+valid_pair() { [[ "$1" =~ ^[a-z0-9][a-z0-9-]*$ && "$2" =~ ^[a-z0-9][a-z0-9-]*$ ]]; }
+if ! valid_pair "$project" "$env_name"; then
     echo "error: project and env must match [a-z0-9][a-z0-9-]* (lowercase, no spaces)" >&2
     exit 2
 fi
@@ -62,6 +63,13 @@ fi
 # rendered file, plus the pair being bootstrapped now.
 pairs="$({ sed -n 's/^# COVERS: //p' "$out_dir"/project-*.yaml 2>/dev/null || true
            echo "$project $env_name"; } | sort -u)"
+# The markers come off disk and land in a sed replacement and a PromQL label
+# value. Refuse the run rather than render a rule that silently never matches;
+# the fix is to delete the edited file.
+while read -r p e; do
+    valid_pair "$p" "$e" \
+        || { echo "error: bad '# COVERS:' marker in $out_dir: '$p $e'" >&2; exit 2; }
+done <<<"$pairs"
 
 # All pairs are re-rendered, so a template fix reaches every project.
 while read -r p e; do

config/grafana/alerting/coverage.yaml

@@ -4,7 +4,7 @@
 # Fires on any series whose project/env pair has no rendered rule file: a project that
 # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule.
 #
-# Covered right now: relab/staging
+# Covered right now: relab/staging (plus demo/demo, exempt: see below)
 
 apiVersion: 1
 
@@ -35,7 +35,12 @@ groups:
               # counters for every sender, whatever signal it sends. A bare
               # {project!=""} would scan every project-labelled series in the TSDB on
               # each evaluation, and would still miss a logs-only or traces-only project.
-              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="relab",env="staging"}))
+              #
+              # demo/demo is this repo's own `just demo` overlay. Bootstrapping it
+              # would leave ProjectTelemetrySilent firing forever after `just
+              # demo-down`, so it is exempt as a pair; a real project named demo in
+              # a real env is still caught.
+              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"}))
           - refId: FIRING
             datasourceUid: __expr__
             model:
@@ -50,4 +55,4 @@ groups:
           severity: warning
         annotations:
           summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules"
-          description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host."
+          description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host. A literal `unknown` means the sender set no project or env resource attribute at all: fix the sender (docs/ONBOARDING.md), do not bootstrap a project by that name."

config/grafana/alerting/rules.yaml

@@ -171,9 +171,11 @@ groups:
             Extrapolated from the last 6 hours. Find what is growing (a spoke
             shipping more than before, a log loop, a backup that stopped
             rotating) before HostDiskSpaceLow makes it urgent.
-      # ~14x the baseline of ~7k series with one spoke (measured after the scrape
+      # ~4x the baseline of ~7k series with one spoke (measured after the scrape
       # trimming in config/prometheus.yaml and the agent config); raise it as spokes
-      # are onboarded.
+      # are onboarded. A spoke costs ~1,400 series, so this is ~18 spokes of room,
+      # and it fires near a quarter of Prometheus's 2g mem_limit rather than at
+      # the wall. Sudden growth is PrometheusCardinalitySpike's job, below.
       - uid: prometheus-cardinality
         title: PrometheusCardinalityHigh
         !!merge <<: *rule_defaults
@@ -182,18 +184,44 @@ groups:
           - !!merge <<: *query_node
             model:
               !!merge <<: *query_model
-              expr: prometheus_tsdb_head_series > 100000
+              expr: prometheus_tsdb_head_series > 30000
           - *firing
         labels:
           severity: warning
         annotations:
-          summary: "Prometheus holds {{ $values.QUERY }} active series, over the 100k ceiling"
+          summary: "Prometheus holds {{ $values.QUERY }} active series, over the 30k ceiling"
           description: >-
             Find the label that exploded with
             `topk(10, count by (__name__) ({__name__=~".+"}))` and
             `topk(10, count by (job) ({__name__=~".+"}))`, then drop it at the
             spoke's Alloy config or the collector. Retention is 15GB; at this
             rate it fills.
+
+      # A label that explodes, at any fleet size. Steady state on the hub moves
+      # under 100 series/hour and one spoke onboarding adds ~1,400, so 5,000 in
+      # 30m clears both. 30m fits inside the shared relativeTimeRange. A head-block
+      # truncation moves the delta negative, never positive, so it cannot false-fire.
+      - uid: prometheus-cardinality-spike
+        title: PrometheusCardinalitySpike
+        !!merge <<: *rule_defaults
+        for: 15m
+        data:
+          - !!merge <<: *query_node
+            model:
+              !!merge <<: *query_model
+              expr: delta(prometheus_tsdb_head_series[30m]) > 5000
+          - *firing
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus gained {{ $values.QUERY }} active series in 30 minutes"
+          description: >-
+            Far faster than onboarding a spoke explains, so a label has most
+            likely exploded. Find it with
+            `topk(10, count by (__name__) ({__name__=~".+"}))` and
+            `topk(10, count by (job) ({__name__=~".+"}))`, then drop it in the
+            spoke's Alloy config or the collector. Several spokes onboarded at
+            once also trips this; it clears on its own.
   - orgId: 1
     name: container-lifecycle
     folder: Stack alerts

config/otel-collector.yaml

@@ -74,9 +74,33 @@ processors:
 
   # The count connector emits delta sums and the Prometheus OTLP receiver refuses
   # them ("invalid temporality and type combination"). Converting here keeps the
-  # conversion state to the handful of counter streams, rather than turning on
-  # Prometheus's experimental global delta handling.
-  deltatocumulative: {}
+  # conversion state to the counter streams, rather than turning on Prometheus's
+  # experimental global delta handling.
+  #
+  # Streams are per sender resource, and a service that mints a new
+  # service.instance.id on each restart adds one every time. The default is
+  # unbounded. This cap is a runaway guard, not a budget: 28 streams cost ~51 MiB
+  # heap against the 400 MiB limiter (measured before `interval` was added).
+  deltatocumulative:
+    max_streams: 5000
+
+  # The count connector emits one datapoint per incoming batch, so 50 log batches
+  # become 50 samples of one series microseconds apart, and Prometheus rejects
+  # the whole write as duplicates. Keep the latest value per scrape interval;
+  # 30s matches scrape_interval in config/prometheus.yaml.
+  interval:
+    interval: 30s
+
+  # Stamp the counters at receipt. Left alone they carry the source telemetry's
+  # timestamp, so a backlog of logs mints samples hours old and Prometheus
+  # rejects anything past out_of_order_time_window (30m in config/prometheus.yaml).
+  # Must run after `interval`: restamping several datapoints of one stream would
+  # make them duplicates again.
+  transform/count_ingest_time:
+    metric_statements:
+      - context: datapoint
+        statements:
+          - set(time, Now())
 
   # Sized to the container's mem_limit (512m in compose.yml); keep the two in step.
   memory_limiter:
@@ -111,6 +135,16 @@ exporters:
     sending_queue: *queue
     retry_on_failure: *retry
 
+  # Same endpoint, separate exporter, so otelcol_exporter_send_failed_metric_points
+  # tells rejected counters apart from a spoke's rejected application metrics.
+  # No file_storage: the counters resume after a restart, so in-flight samples
+  # are not worth a queue on disk.
+  otlp_http/prometheus_count:
+    endpoint: http://prometheus:9090/api/v1/otlp
+    sending_queue:
+      queue_size: 1000
+    retry_on_failure: *retry
+
 service:
   extensions: [bearertokenauth, file_storage]
   pipelines:
@@ -126,11 +160,12 @@ service:
       receivers: [otlp]
       processors: [memory_limiter, resource/department, batch]
       exporters: [otlp_http/prometheus, count]
-    # The counters themselves. Same limiter and batcher as every other pipeline.
+    # The counters themselves. No batcher: it reordered timestamps ~1/min, which
+    # deltatocumulative refused as out of order. `interval` does the collapsing.
     metrics/count:
       receivers: [count]
-      processors: [memory_limiter, deltatocumulative, batch]
-      exporters: [otlp_http/prometheus]
+      processors: [memory_limiter, deltatocumulative, interval, transform/count_ingest_time]
+      exporters: [otlp_http/prometheus_count]
   telemetry:
     metrics:
       level: basic

docs/RUNBOOK.md

@@ -36,8 +36,11 @@ Retention is only partially size-bounded:
 Loki and Tempo cannot cap their total size, so the disk alert at 80% is the
 backstop. Two warnings fire earlier. `HostDiskFilling` means a 6-hour linear
 fit says a filesystem is full within 3 days. `PrometheusCardinalityHigh`
-means active series passed 100k, about 14x the baseline of ~7k with one
-spoke. Series count, not time, is what grows the TSDB.
+means active series passed 30k, about 4x the baseline of ~7k with one spoke,
+which is roughly 18 spokes of room at ~1,400 series each.
+`PrometheusCardinalitySpike` means 5,000 series appeared in 30 minutes, far
+faster than onboarding explains, so a label has most likely exploded. Series
+count, not time, is what grows the TSDB.
 
 When one fires:
 
@@ -87,6 +90,9 @@ on the host.
   The token is shared, and the collector does not check `project` or `env`
   against the sender. Every project host can therefore spoof another
   project's labels.
+
+  Bearer is the only scheme the collector accepts. A sender that cannot set a
+  raw `Authorization` header ships through the host's Alloy agent instead.
 - **Tunnel token:** rotate the tunnel secret in Cloudflare Zero Trust, then
   run `cd infra && tofu apply` to refresh the token data source, then read
   the new value with `tofu output -raw tunnel_token`. To rotate from code

templates/alerting/coverage.yaml.tmpl

@@ -4,7 +4,7 @@
 # Fires on any series whose project/env pair has no rendered rule file: a project that
 # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule.
 #
-# Covered right now: __COVERED__
+# Covered right now: __COVERED__ (plus demo/demo, exempt: see below)
 
 apiVersion: 1
 
@@ -35,7 +35,12 @@ groups:
               # counters for every sender, whatever signal it sends. A bare
               # {project!=""} would scan every project-labelled series in the TSDB on
               # each evaluation, and would still miss a logs-only or traces-only project.
-              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) (__COVERED_EXPR__))
+              #
+              # demo/demo is this repo's own `just demo` overlay. Bootstrapping it
+              # would leave ProjectTelemetrySilent firing forever after `just
+              # demo-down`, so it is exempt as a pair; a real project named demo in
+              # a real env is still caught.
+              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or __COVERED_EXPR__))
           - refId: FIRING
             datasourceUid: __expr__
             model: