From d41df7727729137af2257ba5133a551180c2dd95 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Sun, 16 Aug 2026 10:52:01 +0200
Subject: [PATCH 01/10] split fleet probes so a dependency blip stops
 restarting healthy pods

---
 charts/fleet/templates/deployment.yaml        | 20 +++++++++-
 charts/fleet/values.yaml                      | 19 ++++++++++
 openframe/docs/helm-chart.md                  | 38 ++++++++++++++++++-
 .../docs/upstream-sync-conflict-resolution.md |  2 +-
 4 files changed, 75 insertions(+), 4 deletions(-)

diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml
index 1d65f2105e1..064e1415a05 100644
--- a/charts/fleet/templates/deployment.yaml
+++ b/charts/fleet/templates/deployment.yaml
@@ -526,13 +526,27 @@ spec:
           {{- toYaml . | nindent 10 }}
         {{- end }}
         # <<< OPENFRAME(hardening)
-        livenessProbe:
+        # >>> OPENFRAME(helm): /healthz reports MySQL and Redis, so liveness uses /version instead — openframe/docs/helm-chart.md
+        startupProbe:
           httpGet:
             path: /healthz
             port: {{ .Values.fleet.listenPort }}
             {{- if .Values.fleet.tls.enabled }}
             scheme: HTTPS
             {{- end }}
+          periodSeconds: {{ .Values.fleet.probes.startup.periodSeconds }}
+          failureThreshold: {{ .Values.fleet.probes.startup.failureThreshold }}
+          timeoutSeconds: {{ .Values.fleet.probes.startup.timeoutSeconds }}
+        livenessProbe:
+          httpGet:
+            path: /version
+            port: {{ .Values.fleet.listenPort }}
+            {{- if .Values.fleet.tls.enabled }}
+            scheme: HTTPS
+            {{- end }}
+          periodSeconds: {{ .Values.fleet.probes.liveness.periodSeconds }}
+          failureThreshold: {{ .Values.fleet.probes.liveness.failureThreshold }}
+          timeoutSeconds: {{ .Values.fleet.probes.liveness.timeoutSeconds }}
         readinessProbe:
           httpGet:
             path: /healthz
@@ -540,6 +554,10 @@ spec:
             {{- if .Values.fleet.tls.enabled }}
             scheme: HTTPS
             {{- end }}
+          periodSeconds: {{ .Values.fleet.probes.readiness.periodSeconds }}
+          failureThreshold: {{ .Values.fleet.probes.readiness.failureThreshold }}
+          timeoutSeconds: {{ .Values.fleet.probes.readiness.timeoutSeconds }}
+        # <<< OPENFRAME(helm)
         # >>> OPENFRAME(hardening): unconditional — tmp is always mounted — openframe/docs/helm-chart.md
         volumeMounts:
           - name: tmp
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index 661684fccc2..a966efc6367 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -81,6 +81,25 @@ fleet:
     enabled: true
     image: busybox:1.36
   # <<< OPENFRAME(helm)
+  # >>> OPENFRAME(helm): startup waits, liveness ignores dependencies — openframe/docs/helm-chart.md
+  # Failure budget is initialDelaySeconds + failureThreshold × periodSeconds
+  probes:
+    # failure budget 40 × 15 = 10 min
+    startup:
+      periodSeconds: 15
+      failureThreshold: 40
+      timeoutSeconds: 10
+    # failure budget 10 × 15 = 2.5 min
+    liveness:
+      periodSeconds: 15
+      failureThreshold: 10
+      timeoutSeconds: 10
+    # failure budget 10 × 15 = 2.5 min
+    readiness:
+      periodSeconds: 15
+      failureThreshold: 10
+      timeoutSeconds: 10
+  # <<< OPENFRAME(helm)
   tls:
     enabled: true
     # Set to true if you need a separate secret for just TLS data.
diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index 9dfbe489f1f..ef49c94b898 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -170,6 +170,40 @@ exactly-once semantics.
 | Additional CA certs | `fleet.additionalCAs.*` | Init-container injection of CA bundles from named ConfigMaps/Secrets, for private PKI. |
 | Dedicated vuln processing | `vulnProcessing.dedicated`, `vulnProcessing.schedule` | When `true`, runs vulnerability processing as a separate CronJob ([vulnprocessing/cronjob.yaml](../../charts/fleet/templates/vulnprocessing/cronjob.yaml)) and disables it in the main deployment. |
 | Vuln feed persistence | `vulnProcessing.persistence.*`, `vulnProcessing.staggerSchedule` | See [below](#vulnerability-feed-persistence-vuln-persistence). |
+| Probe split | `fleet.probes.*` | See [below](#probes). |
+
+## Probes
+
+Upstream puts both liveness and readiness on `/healthz`. That endpoint checks MySQL
+and Redis (`healthCheckers` in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)), and
+that breaks in two ways:
+
+- **Slow start.** Fleet waits for the database on its own, about 105s
+  (`defaultMaxAttempts = 15`, see
+  [config.go](../../server/datastore/mysql/config.go)). While it waits it isn't
+  listening on `listenPort` yet, so the default liveness kills it after 30s and it
+  never gets to finish waiting. Happens every time MySQL and Fleet come up together
+- **Redis blip.** One Redis outage makes `/healthz` return 500 in every Fleet pod at
+  once. Restarting them doesn't bring Redis back, it just piles a restart storm on
+  top of the outage
+
+So the fork splits the probes:
+
+| Probe | Path | Why |
+|-------|------|-----|
+| `startupProbe` | `/healthz` | Keeps liveness and readiness quiet until the deps answer. 40 × 15s = 10 min |
+| `livenessProbe` | `/version` | Restart only if the process itself is stuck. [version.go](../../server/version/version.go) returns a static struct, no auth, no MySQL, no Redis |
+| `readinessProbe` | `/healthz` | A pod that can't reach its deps drops out of the Service instead of dying |
+
+Numbers live under `fleet.probes.{startup,liveness,readiness}`. The failure budget is
+`initialDelaySeconds + failureThreshold × periodSeconds`
+([k8s probe docs](https://kubernetes.io/docs/concepts/workloads/pods/probes/)), and we
+leave `initialDelaySeconds` at 0.
+
+`timeoutSeconds` is 10s on all three because the default is 1s, and `/healthz` does
+real MySQL and Redis round trips on a node that may still be busy starting everything
+else. It caps a single attempt and sits inside the period, it isn't added on top, so
+keep it under `periodSeconds` or the timeout becomes the real cadence
 
 ## Vulnerability feed persistence (`vuln-persistence`)
 
@@ -267,10 +301,10 @@ helm upgrade --install fleet oci://ghcr.io/flamingo-stack/fleetmdm/helm-charts/f
 
 | File | Purpose |
 |------|---------|
-| `charts/fleet/values.yaml` | OpenFrame mode, externalized DB/cache/setup config, `cache.keyPrefixKey`, `waitForMysql`, `additionalCAs`, `vulnProcessing`, `deploymentAnnotations` |
+| `charts/fleet/values.yaml` | OpenFrame mode, externalized DB/cache/setup config, `cache.keyPrefixKey`, `waitForMysql`, `probes`, `additionalCAs`, `vulnProcessing`, `deploymentAnnotations` |
 | `charts/fleet/templates/configmap.yaml` | **New** — generated DB/cache ConfigMaps |
 | `charts/fleet/templates/secret.yaml` | **New** — generated DB password / admin-setup Secrets |
-| `charts/fleet/templates/deployment.yaml` | `FLEET_OPENFRAME_MODE`, `FLEET_OPENFRAME_MULTI_TENANCY_ENABLED` / `FLEET_OPENFRAME_TENANT_UUID` / `FLEET_OPENFRAME_TEAM_ID`, `FLEET_REDIS_KEY_PREFIX`, ConfigMap/Secret refs, annotations, CA init container |
+| `charts/fleet/templates/deployment.yaml` | `FLEET_OPENFRAME_MODE`, `FLEET_OPENFRAME_MULTI_TENANCY_ENABLED` / `FLEET_OPENFRAME_TENANT_UUID` / `FLEET_OPENFRAME_TEAM_ID`, `FLEET_REDIS_KEY_PREFIX`, ConfigMap/Secret refs, annotations, CA init container, probe split |
 | `charts/fleet/templates/job-migration.yaml` | `waitForMysql` init container, hook removal, TTL removal |
 | `charts/fleet/templates/vulnprocessing/cronjob.yaml` | Dedicated vuln-processing cron + `FLEET_REDIS_KEY_PREFIX`, feed-cache PVC mount, fsGroup, schedule stagger (moved from `templates/cron-vulnprocessing.yaml`) |
 | `charts/fleet/templates/vulnprocessing/pvc.yaml` | **New** — PVC persisting the vulnerability feed cache across cron runs |
diff --git a/openframe/docs/upstream-sync-conflict-resolution.md b/openframe/docs/upstream-sync-conflict-resolution.md
index 7066749ea2f..8600868142d 100644
--- a/openframe/docs/upstream-sync-conflict-resolution.md
+++ b/openframe/docs/upstream-sync-conflict-resolution.md
@@ -71,7 +71,7 @@ top. Fork edits inside shared files are wrapped in `// OPENFRAME(<slug>)` marker
 | `server/datastore/redis/redis.go` | pool `keyPrefix` fields, `KeyPrefix()` accessors, `normalizeKeyPrefix`, `newPrefixedConn`/`unwrapConn` around `redisc` | **Critical** — see watchlist; verify every pool `Get()` returns `newPrefixedConn(...)` |
 | `orbit/cmd/orbit/orbit.go` | 4 `openframe-*` flags, custom osqueryd path, token-refresher startup, `uuid` cmd, osquery flag passthrough, `NewOrbitClient(..., openFrameMode, authManager)` args | Keep all; re-thread the two extra `NewOrbitClient` args at both call sites |
 | `server/service/orbit_client.go` | `openFrameMode`/`authManager` fields, bearer-header block, `/tools/agent/fleetmdm-server` url prefix, `NewOrbitClient` signature | Keep the two trailing constructor params and the header/prefix logic |
-| `charts/fleet/*` | externalized config, `FLEET_OPENFRAME_MODE`, `FLEET_REDIS_KEY_PREFIX`, migration job, waitForMysql, additionalCAs | The chart is fork-owned — prefer ours, cherry-pick upstream chart improvements deliberately |
+| `charts/fleet/*` | externalized config, `FLEET_OPENFRAME_MODE`, `FLEET_REDIS_KEY_PREFIX`, migration job, waitForMysql, probe split, additionalCAs | The chart is fork-owned — prefer ours, cherry-pick upstream chart improvements deliberately |
 | `go.mod` / `go.sum` | fork adds `github.com/robfig/cron/v3` (token refresher) | Keep the require line on conflict; run `go mod tidy` after |
 
 ## Semantic-conflict watchlist (no git conflict — the dangerous ones)

From 12ad73c4dc32124cb1fe03fbf2e896efaf765daf Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Sun, 16 Aug 2026 11:11:22 +0200
Subject: [PATCH 02/10] put the startup probe on /version too, so a redis
 outage cannot block boot

---
 charts/fleet/templates/deployment.yaml |  4 +--
 charts/fleet/values.yaml               | 18 +++++++------
 openframe/docs/helm-chart.md           | 35 ++++++++++++++++++++------
 3 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml
index 064e1415a05..0b3d4a15325 100644
--- a/charts/fleet/templates/deployment.yaml
+++ b/charts/fleet/templates/deployment.yaml
@@ -526,10 +526,10 @@ spec:
           {{- toYaml . | nindent 10 }}
         {{- end }}
         # <<< OPENFRAME(hardening)
-        # >>> OPENFRAME(helm): /healthz reports MySQL and Redis, so liveness uses /version instead — openframe/docs/helm-chart.md
+        # >>> OPENFRAME(helm): /healthz reports MySQL and Redis, so only readiness uses it — openframe/docs/helm-chart.md
         startupProbe:
           httpGet:
-            path: /healthz
+            path: /version
             port: {{ .Values.fleet.listenPort }}
             {{- if .Values.fleet.tls.enabled }}
             scheme: HTTPS
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index a966efc6367..eeaf10eeb5d 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -81,23 +81,25 @@ fleet:
     enabled: true
     image: busybox:1.36
   # <<< OPENFRAME(helm)
-  # >>> OPENFRAME(helm): startup waits, liveness ignores dependencies — openframe/docs/helm-chart.md
-  # Failure budget is initialDelaySeconds + failureThreshold × periodSeconds
+  # >>> OPENFRAME(helm): only readiness looks at dependencies — openframe/docs/helm-chart.md
+  # First probe runs at t=0, so the Nth failure lands at (N-1) × periodSeconds
   probes:
-    # failure budget 40 × 15 = 10 min
+    # Startup has to stay above the ~105s fleet spends retrying MySQL with the
+    # port still closed, or the probe cuts a legitimate wait short. Above that it only
+    # catches a hang, because a MySQL that never answers exits the process by itself
     startup:
       periodSeconds: 15
-      failureThreshold: 40
+      failureThreshold: 25
       timeoutSeconds: 10
-    # failure budget 10 × 15 = 2.5 min
+    # restarts after 10 × 15 = 2.5 min
     liveness:
       periodSeconds: 15
-      failureThreshold: 10
+      failureThreshold: 11
       timeoutSeconds: 10
-    # failure budget 10 × 15 = 2.5 min
+    # leaves the Service after 10 × 15 = 2.5 min
     readiness:
       periodSeconds: 15
-      failureThreshold: 10
+      failureThreshold: 11
       timeoutSeconds: 10
   # <<< OPENFRAME(helm)
   tls:
diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index ef49c94b898..b97e45cce09 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -178,8 +178,9 @@ Upstream puts both liveness and readiness on `/healthz`. That endpoint checks My
 and Redis (`healthCheckers` in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)), and
 that breaks in two ways:
 
-- **Slow start.** Fleet waits for the database on its own, about 105s
-  (`defaultMaxAttempts = 15`, see
+- **Slow start.** Fleet waits for the database on its own, about 105s: 15 attempts
+  sleeping 0,1,2,…,14 seconds ([common.go](../../server/platform/mysql/common.go),
+  count from `defaultMaxAttempts` in
   [config.go](../../server/datastore/mysql/config.go)). While it waits it isn't
   listening on `listenPort` yet, so the default liveness kills it after 30s and it
   never gets to finish waiting. Happens every time MySQL and Fleet come up together
@@ -187,18 +188,36 @@ that breaks in two ways:
   once. Restarting them doesn't bring Redis back, it just piles a restart storm on
   top of the outage
 
-So the fork splits the probes:
+So only readiness looks at the dependencies:
 
 | Probe | Path | Why |
 |-------|------|-----|
-| `startupProbe` | `/healthz` | Keeps liveness and readiness quiet until the deps answer. 40 × 15s = 10 min |
+| `startupProbe` | `/version` | An answer on `/version` already means the boot finished. Fleet opens the listener at [serve.go:1148](../../cmd/fleet/serve.go), long after `initDatastore` and the migration check, so nothing serves before MySQL is in |
 | `livenessProbe` | `/version` | Restart only if the process itself is stuck. [version.go](../../server/version/version.go) returns a static struct, no auth, no MySQL, no Redis |
 | `readinessProbe` | `/healthz` | A pod that can't reach its deps drops out of the Service instead of dying |
 
-Numbers live under `fleet.probes.{startup,liveness,readiness}`. The failure budget is
-`initialDelaySeconds + failureThreshold × periodSeconds`
-([k8s probe docs](https://kubernetes.io/docs/concepts/workloads/pods/probes/)), and we
-leave `initialDelaySeconds` at 0.
+Putting `/healthz` on startup would have kept the same bug at boot: with Redis down,
+a pod that gets rescheduled mid outage never starts, gets killed and lands in
+CrashLoopBackOff. Tenants run one replica, so that is full downtime that outlives the
+Redis outage by the backoff. On `/version` the pod comes up, stays out of the Service
+and joins it the moment Redis is back, with no restart.
+
+Numbers live under `fleet.probes.{startup,liveness,readiness}`. The first probe runs at
+t=0, so the Nth failure lands at `(N - 1) × periodSeconds`: startup gives up at 6 min,
+liveness and readiness at 2.5 min.
+
+The startup number has a floor and no real ceiling. The floor is the ~105s Fleet spends
+retrying MySQL with the port still closed: go under it and the probe cuts a legitimate
+wait short, which is the original bug with liveness playing that role at 30s. Above the
+floor it only catches a hang, because a MySQL that never answers ends the process
+anyway, through `initFatal` in `initDatastore` ([serve.go:256](../../cmd/fleet/serve.go))
+or `os.Exit(1)` on an unapplied migration ([serve.go:272](../../cmd/fleet/serve.go)).
+So 6 min reads as "how long we tolerate a stuck boot", not "how long we wait for the
+database". That wait is Fleet's own and a probe can only cut it short, never extend it.
+
+Readiness holds a broken pod in the Service for 2.5 min, which is deliberate. On one
+replica dropping it earlier changes nothing, the tenant is down regardless. Lower it
+if you ever run Fleet with more than one replica per tenant.
 
 `timeoutSeconds` is 10s on all three because the default is 1s, and `/healthz` does
 real MySQL and Redis round trips on a node that may still be busy starting everything

From 5e79df6e414fe0c5e4e61c6de5361142688ab4a0 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Sun, 16 Aug 2026 11:23:42 +0200
Subject: [PATCH 03/10] clean up the probe docs and comments

---
 charts/fleet/values.yaml     | 11 ++++++-----
 openframe/docs/helm-chart.md | 16 ++++++++++++----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index eeaf10eeb5d..b198cf54902 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -84,19 +84,20 @@ fleet:
   # >>> OPENFRAME(helm): only readiness looks at dependencies — openframe/docs/helm-chart.md
   # First probe runs at t=0, so the Nth failure lands at (N-1) × periodSeconds
   probes:
-    # Startup has to stay above the ~105s fleet spends retrying MySQL with the
-    # port still closed, or the probe cuts a legitimate wait short. Above that it only
-    # catches a hang, because a MySQL that never answers exits the process by itself
+    # gives up on the 25th failure, 24 × 15 = 6 min. Has to stay above the ~105s fleet
+    # spends retrying MySQL with the port still closed, or the probe cuts a legitimate
+    # wait short. Above that it only catches a hang, because a MySQL that never answers
+    # exits the process by itself
     startup:
       periodSeconds: 15
       failureThreshold: 25
       timeoutSeconds: 10
-    # restarts after 10 × 15 = 2.5 min
+    # restarts on the 11th failure, 10 × 15 = 2.5 min
     liveness:
       periodSeconds: 15
       failureThreshold: 11
       timeoutSeconds: 10
-    # leaves the Service after 10 × 15 = 2.5 min
+    # leaves the Service on the 11th failure, 10 × 15 = 2.5 min
     readiness:
       periodSeconds: 15
       failureThreshold: 11
diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index b97e45cce09..a8698a803b1 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -192,10 +192,18 @@ So only readiness looks at the dependencies:
 
 | Probe | Path | Why |
 |-------|------|-----|
-| `startupProbe` | `/version` | An answer on `/version` already means the boot finished. Fleet opens the listener at [serve.go:1148](../../cmd/fleet/serve.go), long after `initDatastore` and the migration check, so nothing serves before MySQL is in |
-| `livenessProbe` | `/version` | Restart only if the process itself is stuck. [version.go](../../server/version/version.go) returns a static struct, no auth, no MySQL, no Redis |
+| `startupProbe` | `/version` | An answer on `/version` already means the boot finished. Fleet reaches `srv.ListenAndServe` long after `initDatastore` and `evalMigrationStatus`, so nothing serves before MySQL is in |
+| `livenessProbe` | `/version` | Restart only if the listener stops answering. [version.go](../../server/version/version.go) returns a static struct, no auth, no MySQL, no Redis |
 | `readinessProbe` | `/healthz` | A pod that can't reach its deps drops out of the Service instead of dying |
 
+That liveness is deliberately narrow. A Fleet that still answers on `/version` but is
+wedged on an exhausted connection pool will not be restarted, because a restart is not
+what fixes that. Readiness is what takes it out of rotation.
+
+The startup probe is close to a duplicate of liveness, both on `/version`, and liveness
+alone already covers the 105s. It earns its place with two things: a wider window for a
+stuck boot, 6 min against 2.5, and keeping readiness quiet until the process is up.
+
 Putting `/healthz` on startup would have kept the same bug at boot: with Redis down,
 a pod that gets rescheduled mid outage never starts, gets killed and lands in
 CrashLoopBackOff. Tenants run one replica, so that is full downtime that outlives the
@@ -210,8 +218,8 @@ The startup number has a floor and no real ceiling. The floor is the ~105s Fleet
 retrying MySQL with the port still closed: go under it and the probe cuts a legitimate
 wait short, which is the original bug with liveness playing that role at 30s. Above the
 floor it only catches a hang, because a MySQL that never answers ends the process
-anyway, through `initFatal` in `initDatastore` ([serve.go:256](../../cmd/fleet/serve.go))
-or `os.Exit(1)` on an unapplied migration ([serve.go:272](../../cmd/fleet/serve.go)).
+anyway, through `initFatal` in `initDatastore` or `os.Exit(1)` on an unapplied
+migration in `evalMigrationStatus` (both in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)).
 So 6 min reads as "how long we tolerate a stuck boot", not "how long we wait for the
 database". That wait is Fleet's own and a probe can only cut it short, never extend it.
 

From e80d0f275d4a4c22a77b9218e22916e5b18a2e23 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Sun, 16 Aug 2026 11:40:37 +0200
Subject: [PATCH 04/10] doc improvements

---
 openframe/docs/helm-chart.md | 39 ++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index a8698a803b1..9c2a34449a9 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -200,6 +200,29 @@ That liveness is deliberately narrow. A Fleet that still answers on `/version` b
 wedged on an exhausted connection pool will not be restarted, because a restart is not
 what fixes that. Readiness is what takes it out of rotation.
 
+### What moving liveness off `/healthz` gives up
+
+The MySQL checker is not a ping. `Datastore.HealthCheck`
+([mysql.go](../../server/datastore/mysql/mysql.go)) runs `SELECT @@read_only` and
+returns an error when the answer is 1, with the comment saying so outright: fail the
+endpoint so the orchestrator restarts Fleet with fresh DB connections. Upstream added
+it for AWS Aurora, where a failover demotes the old writer to a reader behind the same
+endpoint. So upstream's liveness on `/healthz` was not only an oversight, it was also a
+self-heal after a database failover, and `connMaxLifetime: 0` means the pool never
+recycles those connections on its own.
+
+We give that up knowingly, because we cannot reach the state it repairs. Each tenant
+runs its own single MySQL StatefulSet in its own namespace
+(`fleetmdm-mysql-0.fleetmdm-mysql.<namespace>.svc.cluster.local`), with no replica, no
+reader endpoint and nothing that promotes or demotes. When that MySQL goes away the
+connections break with socket errors and `database/sql` opens new ones. `@@read_only`
+only turns 1 if somebody sets it by hand.
+
+If Fleet ever moves onto a database that can demote a writer behind a stable endpoint,
+put this back. `health.Handler` ([health.go](../../server/health/health.go)) supports a
+`?check=<name>` filter, so a narrow `/healthz?check=mysql` liveness is available without
+dragging Redis back into the restart decision.
+
 The startup probe is close to a duplicate of liveness, both on `/version`, and liveness
 alone already covers the 105s. It earns its place with two things: a wider window for a
 stuck boot, 6 min against 2.5, and keeping readiness quiet until the process is up.
@@ -218,8 +241,10 @@ The startup number has a floor and no real ceiling. The floor is the ~105s Fleet
 retrying MySQL with the port still closed: go under it and the probe cuts a legitimate
 wait short, which is the original bug with liveness playing that role at 30s. Above the
 floor it only catches a hang, because a MySQL that never answers ends the process
-anyway, through `initFatal` in `initDatastore` or `os.Exit(1)` on an unapplied
-migration in `evalMigrationStatus` (both in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)).
+anyway: `initFatal` inside `initDatastore`, or `evalMigrationStatus` reporting an
+unapplied migration and `serve` exiting on it (both in
+[cmd/fleet/datastore.go](../../cmd/fleet/datastore.go), called from
+[cmd/fleet/serve.go](../../cmd/fleet/serve.go)).
 So 6 min reads as "how long we tolerate a stuck boot", not "how long we wait for the
 database". That wait is Fleet's own and a probe can only cut it short, never extend it.
 
@@ -227,10 +252,12 @@ Readiness holds a broken pod in the Service for 2.5 min, which is deliberate. On
 replica dropping it earlier changes nothing, the tenant is down regardless. Lower it
 if you ever run Fleet with more than one replica per tenant.
 
-`timeoutSeconds` is 10s on all three because the default is 1s, and `/healthz` does
-real MySQL and Redis round trips on a node that may still be busy starting everything
-else. It caps a single attempt and sits inside the period, it isn't added on top, so
-keep it under `periodSeconds` or the timeout becomes the real cadence
+`timeoutSeconds` is 10s on all three. The point is getting away from the 1s default:
+readiness needs it because `/healthz` does real MySQL and Redis round trips, and even
+`/version` can miss a 1s deadline on a node still busy starting everything else. One
+number everywhere keeps it readable. It caps a single attempt and sits inside the
+period, it isn't added on top, so keep it under `periodSeconds` or the timeout becomes
+the real cadence
 
 ## Vulnerability feed persistence (`vuln-persistence`)
 

From 8f1cbccd25a3d8e7767949c1d681759f1f66f986 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Sun, 16 Aug 2026 12:39:40 +0200
Subject: [PATCH 05/10] doc improvements

---
 openframe/docs/helm-chart.md | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index 9c2a34449a9..52264d9893e 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -211,12 +211,24 @@ endpoint. So upstream's liveness on `/healthz` was not only an oversight, it was
 self-heal after a database failover, and `connMaxLifetime: 0` means the pool never
 recycles those connections on its own.
 
-We give that up knowingly, because we cannot reach the state it repairs. Each tenant
-runs its own single MySQL StatefulSet in its own namespace
-(`fleetmdm-mysql-0.fleetmdm-mysql.<namespace>.svc.cluster.local`), with no replica, no
-reader endpoint and nothing that promotes or demotes. When that MySQL goes away the
-connections break with socket errors and `database/sql` opens new ones. `@@read_only`
-only turns 1 if somebody sets it by hand.
+We give that up knowingly, because neither database we run gets into the state it
+repairs.
+
+Tenant Fleet talks to a single MySQL StatefulSet in its own namespace,
+`fleetmdm-mysql-0.fleetmdm-mysql.<namespace>.svc.cluster.local` out of the
+`fleetmdm-mysql` ConfigMap. No replica, no reader endpoint, nothing that promotes or
+demotes. When that MySQL goes away the connections break with socket errors and
+`database/sql` opens new ones. `@@read_only` only turns 1 if somebody sets it by hand.
+
+The chart can also be pointed at Cloud SQL, which is what the platform-level Fleet app
+does. A Cloud SQL HA failover doesn't leave a demoted writer behind a stable endpoint
+either: the standby serves on the same shared static IP, the old primary is destroyed
+and recreated as the new standby, and open connections are closed rather than turned
+read-only ([Cloud SQL HA](https://cloud.google.com/sql/docs/mysql/high-availability)).
+
+That failover takes about 60 seconds, and 60 seconds fits inside the 2.5 min readiness
+budget. So a Cloud SQL failover doesn't even push the pod out of the Service, and
+liveness on `/version` leaves it alone while `database/sql` reconnects.
 
 If Fleet ever moves onto a database that can demote a writer behind a stable endpoint,
 put this back. `health.Handler` ([health.go](../../server/health/health.go)) supports a

From 7a2821b3afa8ed1c17a539a786bc3e730650ceb9 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Sun, 16 Aug 2026 15:59:14 +0200
Subject: [PATCH 06/10] put the startup probe back on /healthz

---
 charts/fleet/templates/deployment.yaml |  4 +--
 charts/fleet/values.yaml               |  7 ++---
 openframe/docs/helm-chart.md           | 38 +++++++++++++-------------
 3 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml
index 0b3d4a15325..064e1415a05 100644
--- a/charts/fleet/templates/deployment.yaml
+++ b/charts/fleet/templates/deployment.yaml
@@ -526,10 +526,10 @@ spec:
           {{- toYaml . | nindent 10 }}
         {{- end }}
         # <<< OPENFRAME(hardening)
-        # >>> OPENFRAME(helm): /healthz reports MySQL and Redis, so only readiness uses it — openframe/docs/helm-chart.md
+        # >>> OPENFRAME(helm): /healthz reports MySQL and Redis, so liveness uses /version instead — openframe/docs/helm-chart.md
         startupProbe:
           httpGet:
-            path: /version
+            path: /healthz
             port: {{ .Values.fleet.listenPort }}
             {{- if .Values.fleet.tls.enabled }}
             scheme: HTTPS
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index b198cf54902..4905902e59d 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -81,13 +81,10 @@ fleet:
     enabled: true
     image: busybox:1.36
   # <<< OPENFRAME(helm)
-  # >>> OPENFRAME(helm): only readiness looks at dependencies — openframe/docs/helm-chart.md
+  # >>> OPENFRAME(helm): only liveness ignores dependencies — openframe/docs/helm-chart.md
   # First probe runs at t=0, so the Nth failure lands at (N-1) × periodSeconds
   probes:
-    # gives up on the 25th failure, 24 × 15 = 6 min. Has to stay above the ~105s fleet
-    # spends retrying MySQL with the port still closed, or the probe cuts a legitimate
-    # wait short. Above that it only catches a hang, because a MySQL that never answers
-    # exits the process by itself
+    # gives up on the 25th failure, 24 × 15 = 6 min
     startup:
       periodSeconds: 15
       failureThreshold: 25
diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index 52264d9893e..4e797659474 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -188,18 +188,31 @@ that breaks in two ways:
   once. Restarting them doesn't bring Redis back, it just piles a restart storm on
   top of the outage
 
-So only readiness looks at the dependencies:
+So liveness is the one that stops looking at the dependencies:
 
 | Probe | Path | Why |
 |-------|------|-----|
-| `startupProbe` | `/version` | An answer on `/version` already means the boot finished. Fleet reaches `srv.ListenAndServe` long after `initDatastore` and `evalMigrationStatus`, so nothing serves before MySQL is in |
+| `startupProbe` | `/healthz` | Holds liveness and readiness off until the process is up and its deps answer |
 | `livenessProbe` | `/version` | Restart only if the listener stops answering. [version.go](../../server/version/version.go) returns a static struct, no auth, no MySQL, no Redis |
 | `readinessProbe` | `/healthz` | A pod that can't reach its deps drops out of the Service instead of dying |
 
+Startup keeps `/healthz` because on that path the two endpoints are nearly the same
+thing. Fleet never reaches the listener without its dependencies: `initDatastore` gives
+up on MySQL and `redis.NewPool` fails its cluster refresh, both through `initFatal` and
+`os.Exit(1)`. By the time anything answers on `listenPort`, MySQL and Redis were both
+reachable anyway.
+
 That liveness is deliberately narrow. A Fleet that still answers on `/version` but is
 wedged on an exhausted connection pool will not be restarted, because a restart is not
 what fixes that. Readiness is what takes it out of rotation.
 
+The chain this breaks is the one that took every tenant down at once. Redis went away,
+running pods started failing `/healthz`, liveness killed them, and from then on they
+could not come back, because a fresh boot dies in `redis.NewPool` while Redis is still
+missing. Liveness on `/version` removes the first link: a running pod rides the outage
+out and rejoins the Service when Redis returns. A pod that restarts mid outage for some
+other reason still cannot start, and no probe setting changes that.
+
 ### What moving liveness off `/healthz` gives up
 
 The MySQL checker is not a ping. `Datastore.HealthCheck`
@@ -235,16 +248,6 @@ put this back. `health.Handler` ([health.go](../../server/health/health.go)) sup
 `?check=<name>` filter, so a narrow `/healthz?check=mysql` liveness is available without
 dragging Redis back into the restart decision.
 
-The startup probe is close to a duplicate of liveness, both on `/version`, and liveness
-alone already covers the 105s. It earns its place with two things: a wider window for a
-stuck boot, 6 min against 2.5, and keeping readiness quiet until the process is up.
-
-Putting `/healthz` on startup would have kept the same bug at boot: with Redis down,
-a pod that gets rescheduled mid outage never starts, gets killed and lands in
-CrashLoopBackOff. Tenants run one replica, so that is full downtime that outlives the
-Redis outage by the backoff. On `/version` the pod comes up, stays out of the Service
-and joins it the moment Redis is back, with no restart.
-
 Numbers live under `fleet.probes.{startup,liveness,readiness}`. The first probe runs at
 t=0, so the Nth failure lands at `(N - 1) × periodSeconds`: startup gives up at 6 min,
 liveness and readiness at 2.5 min.
@@ -252,13 +255,10 @@ liveness and readiness at 2.5 min.
 The startup number has a floor and no real ceiling. The floor is the ~105s Fleet spends
 retrying MySQL with the port still closed: go under it and the probe cuts a legitimate
 wait short, which is the original bug with liveness playing that role at 30s. Above the
-floor it only catches a hang, because a MySQL that never answers ends the process
-anyway: `initFatal` inside `initDatastore`, or `evalMigrationStatus` reporting an
-unapplied migration and `serve` exiting on it (both in
-[cmd/fleet/datastore.go](../../cmd/fleet/datastore.go), called from
-[cmd/fleet/serve.go](../../cmd/fleet/serve.go)).
-So 6 min reads as "how long we tolerate a stuck boot", not "how long we wait for the
-database". That wait is Fleet's own and a probe can only cut it short, never extend it.
+floor it only catches a hang, because a boot that cannot reach its dependencies ends the
+process anyway. So 6 min reads as "how long we tolerate a stuck boot", not "how long we
+wait for the database". That wait is Fleet's own and a probe can only cut it short,
+never extend it.
 
 Readiness holds a broken pod in the Service for 2.5 min, which is deliberate. On one
 replica dropping it earlier changes nothing, the tenant is down regardless. Lower it

From 28b80060389374000c5c289fd13e6b3c97d03531 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Mon, 17 Aug 2026 10:24:44 +0200
Subject: [PATCH 07/10] retry the first redis dial and rebalance the probe
 budgets

---
 charts/fleet/templates/deployment.yaml |   8 +-
 charts/fleet/values.yaml               |  16 +--
 openframe/docs/helm-chart.md           | 149 +++++++++++++------------
 3 files changed, 91 insertions(+), 82 deletions(-)

diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml
index 064e1415a05..1b2234d5d31 100644
--- a/charts/fleet/templates/deployment.yaml
+++ b/charts/fleet/templates/deployment.yaml
@@ -359,6 +359,10 @@ spec:
           # <<< OPENFRAME(helm)
           - name: FLEET_REDIS_DATABASE
             value: "{{ .Values.cache.database }}"
+          # >>> OPENFRAME(helm): retry the first Redis dial — openframe/docs/helm-chart.md
+          - name: FLEET_REDIS_CONNECT_RETRY_ATTEMPTS
+            value: "{{ .Values.cache.connectRetryAttempts }}"
+          # <<< OPENFRAME(helm)
           {{- if .Values.cache.usePassword }}
           - name: FLEET_REDIS_PASSWORD
             valueFrom:
@@ -526,7 +530,7 @@ spec:
           {{- toYaml . | nindent 10 }}
         {{- end }}
         # <<< OPENFRAME(hardening)
-        # >>> OPENFRAME(helm): /healthz reports MySQL and Redis, so liveness uses /version instead — openframe/docs/helm-chart.md
+        # >>> OPENFRAME(helm): probe timings, upstream sets none — openframe/docs/helm-chart.md
         startupProbe:
           httpGet:
             path: /healthz
@@ -539,7 +543,7 @@ spec:
           timeoutSeconds: {{ .Values.fleet.probes.startup.timeoutSeconds }}
         livenessProbe:
           httpGet:
-            path: /version
+            path: /healthz
             port: {{ .Values.fleet.listenPort }}
             {{- if .Values.fleet.tls.enabled }}
             scheme: HTTPS
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index 4905902e59d..f48c456caef 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -81,23 +81,22 @@ fleet:
     enabled: true
     image: busybox:1.36
   # <<< OPENFRAME(helm)
-  # >>> OPENFRAME(helm): only liveness ignores dependencies — openframe/docs/helm-chart.md
-  # First probe runs at t=0, so the Nth failure lands at (N-1) × periodSeconds
+  # >>> OPENFRAME(helm): upstream sets no probe timings and the defaults kill too early — openframe/docs/helm-chart.md
   probes:
-    # gives up on the 25th failure, 24 × 15 = 6 min
+    # 24 × 15 = 6 min
     startup:
       periodSeconds: 15
       failureThreshold: 25
       timeoutSeconds: 10
-    # restarts on the 11th failure, 10 × 15 = 2.5 min
+    # 16 × 15 = 4 min, keep it above readiness
     liveness:
       periodSeconds: 15
-      failureThreshold: 11
+      failureThreshold: 17
       timeoutSeconds: 10
-    # leaves the Service on the 11th failure, 10 × 15 = 2.5 min
+    # 8 × 15 = 2 min
     readiness:
       periodSeconds: 15
-      failureThreshold: 11
+      failureThreshold: 9
       timeoutSeconds: 10
   # <<< OPENFRAME(helm)
   tls:
@@ -378,6 +377,9 @@ cache:
   usePassword: false
   secretName: redis
   passwordKey: redis-password
+  # >>> OPENFRAME(helm): upstream defaults this to 0, so a boot without Redis dies at once — openframe/docs/helm-chart.md
+  connectRetryAttempts: 20
+  # <<< OPENFRAME(helm)
   # >>> OPENFRAME(redis-key-prefix): fork-added per-tenant Redis key prefix — openframe/docs/redis-key-prefix.md
   # When set, wires FLEET_REDIS_KEY_PREFIX from existingConfigMap[keyPrefixKey].
   # Required when sharing one Redis across multiple Fleet servers (multi-tenant).
diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index 4e797659474..b7350ca9354 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -174,83 +174,64 @@ exactly-once semantics.
 
 ## Probes
 
-Upstream puts both liveness and readiness on `/healthz`. That endpoint checks MySQL
-and Redis (`healthCheckers` in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)), and
-that breaks in two ways:
+All three probes use `/healthz`, the same endpoint upstream uses. It checks MySQL and
+Redis (`healthCheckers` in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)), and that is
+deliberate: Fleet cannot serve without either one, and `Datastore.HealthCheck`
+([mysql.go](../../server/datastore/mysql/mysql.go)) goes further than a ping, failing on
+`SELECT @@read_only = 1` specifically so the orchestrator restarts Fleet with fresh
+connections after a database failover.
+
+| Probe | Path | Budget |
+|-------|------|--------|
+| `startupProbe` | `/healthz` | 6 min |
+| `livenessProbe` | `/healthz` | 4 min |
+| `readinessProbe` | `/healthz` | 2 min |
+
+What the fork changes is the timing, not the endpoint. Upstream sets no timings at all,
+so every probe runs on the Kubernetes defaults: 1s timeout, 10s period, 3 failures. That
+kills a container 30s after the first failed probe, and Fleet needs far longer than that
+in two ordinary situations.
 
 - **Slow start.** Fleet waits for the database on its own, about 105s: 15 attempts
   sleeping 0,1,2,…,14 seconds ([common.go](../../server/platform/mysql/common.go),
   count from `defaultMaxAttempts` in
   [config.go](../../server/datastore/mysql/config.go)). While it waits it isn't
-  listening on `listenPort` yet, so the default liveness kills it after 30s and it
-  never gets to finish waiting. Happens every time MySQL and Fleet come up together
-- **Redis blip.** One Redis outage makes `/healthz` return 500 in every Fleet pod at
-  once. Restarting them doesn't bring Redis back, it just piles a restart storm on
-  top of the outage
-
-So liveness is the one that stops looking at the dependencies:
-
-| Probe | Path | Why |
-|-------|------|-----|
-| `startupProbe` | `/healthz` | Holds liveness and readiness off until the process is up and its deps answer |
-| `livenessProbe` | `/version` | Restart only if the listener stops answering. [version.go](../../server/version/version.go) returns a static struct, no auth, no MySQL, no Redis |
-| `readinessProbe` | `/healthz` | A pod that can't reach its deps drops out of the Service instead of dying |
-
-Startup keeps `/healthz` because on that path the two endpoints are nearly the same
-thing. Fleet never reaches the listener without its dependencies: `initDatastore` gives
-up on MySQL and `redis.NewPool` fails its cluster refresh, both through `initFatal` and
-`os.Exit(1)`. By the time anything answers on `listenPort`, MySQL and Redis were both
-reachable anyway.
-
-That liveness is deliberately narrow. A Fleet that still answers on `/version` but is
-wedged on an exhausted connection pool will not be restarted, because a restart is not
-what fixes that. Readiness is what takes it out of rotation.
-
-The chain this breaks is the one that took every tenant down at once. Redis went away,
-running pods started failing `/healthz`, liveness killed them, and from then on they
-could not come back, because a fresh boot dies in `redis.NewPool` while Redis is still
-missing. Liveness on `/version` removes the first link: a running pod rides the outage
-out and rejoins the Service when Redis returns. A pod that restarts mid outage for some
-other reason still cannot start, and no probe setting changes that.
-
-### What moving liveness off `/healthz` gives up
-
-The MySQL checker is not a ping. `Datastore.HealthCheck`
-([mysql.go](../../server/datastore/mysql/mysql.go)) runs `SELECT @@read_only` and
-returns an error when the answer is 1, with the comment saying so outright: fail the
-endpoint so the orchestrator restarts Fleet with fresh DB connections. Upstream added
-it for AWS Aurora, where a failover demotes the old writer to a reader behind the same
-endpoint. So upstream's liveness on `/healthz` was not only an oversight, it was also a
-self-heal after a database failover, and `connMaxLifetime: 0` means the pool never
-recycles those connections on its own.
-
-We give that up knowingly, because neither database we run gets into the state it
-repairs.
-
-Tenant Fleet talks to a single MySQL StatefulSet in its own namespace,
-`fleetmdm-mysql-0.fleetmdm-mysql.<namespace>.svc.cluster.local` out of the
-`fleetmdm-mysql` ConfigMap. No replica, no reader endpoint, nothing that promotes or
-demotes. When that MySQL goes away the connections break with socket errors and
-`database/sql` opens new ones. `@@read_only` only turns 1 if somebody sets it by hand.
-
-The chart can also be pointed at Cloud SQL, which is what the platform-level Fleet app
-does. A Cloud SQL HA failover doesn't leave a demoted writer behind a stable endpoint
-either: the standby serves on the same shared static IP, the old primary is destroyed
-and recreated as the new standby, and open connections are closed rather than turned
-read-only ([Cloud SQL HA](https://cloud.google.com/sql/docs/mysql/high-availability)).
-
-That failover takes about 60 seconds, and 60 seconds fits inside the 2.5 min readiness
-budget. So a Cloud SQL failover doesn't even push the pod out of the Service, and
-liveness on `/version` leaves it alone while `database/sql` reconnects.
-
-If Fleet ever moves onto a database that can demote a writer behind a stable endpoint,
-put this back. `health.Handler` ([health.go](../../server/health/health.go)) supports a
-`?check=<name>` filter, so a narrow `/healthz?check=mysql` liveness is available without
-dragging Redis back into the restart decision.
+  listening on `listenPort` yet, so the default liveness kills it at 30s and it never
+  gets to finish waiting. This happens every time MySQL and Fleet come up together,
+  which on a tenant cluster is every reschedule. The `startupProbe` covers it: liveness
+  and readiness do not start until the process is serving.
+- **Dependency outages.** A hiccup shorter than 4 min no longer restarts anything. At
+  30s it did, which is what turned a five minute Redis absence into a fleet-wide restart
+  storm on 2026-08-16.
+
+Past 4 min a pod does still get killed, and that is fine now only because
+`cache.connectRetryAttempts` changed what a restart costs. Before it, a killed pod could
+not come back while Redis was still missing and simply crashlooped for the length of the
+outage. With the retry it restarts once and then waits the outage out.
+
+The three budgets are ordered by how expensive the action is. Readiness is cheapest and
+shortest: a pod out of rotation walks back in on the first successful probe. Liveness
+throws away a warm process, so it waits longer, and the two minutes between them are the
+point: that is the window where a pod takes no traffic but still has a chance to recover
+on its own. Set them equal and readiness stops meaning anything, because the pod dies at
+the same moment it leaves the Service. Startup is longest because it supervises a boot
+that is legitimately slow.
+
+If that trade ever needs revisiting, the lever is the endpoint, not the timings.
+`health.Handler` ([health.go](../../server/health/health.go)) takes a `?check=<name>`
+filter, so `/healthz?check=mysql` gives a liveness that keeps the failover self-heal but
+drops Redis out of the restart decision. `/version`
+([version.go](../../server/version/version.go)) is the other end of the scale: a static
+struct with no auth and no dependencies, which restarts only on a wedged listener.
+
+One case the timings handle well: a Cloud SQL HA failover takes about 60 seconds, and
+60 seconds fits inside every budget here, so it passes without a restart and without the
+pod leaving the Service ([Cloud SQL HA](https://cloud.google.com/sql/docs/mysql/high-availability)).
+On upstream's 30s it would not have.
 
 Numbers live under `fleet.probes.{startup,liveness,readiness}`. The first probe runs at
 t=0, so the Nth failure lands at `(N - 1) × periodSeconds`: startup gives up at 6 min,
-liveness and readiness at 2.5 min.
+liveness at 4 min, readiness at 2 min.
 
 The startup number has a floor and no real ceiling. The floor is the ~105s Fleet spends
 retrying MySQL with the port still closed: go under it and the probe cuts a legitimate
@@ -260,9 +241,31 @@ process anyway. So 6 min reads as "how long we tolerate a stuck boot", not "how
 wait for the database". That wait is Fleet's own and a probe can only cut it short,
 never extend it.
 
-Readiness holds a broken pod in the Service for 2.5 min, which is deliberate. On one
-replica dropping it earlier changes nothing, the tenant is down regardless. Lower it
-if you ever run Fleet with more than one replica per tenant.
+Readiness is deliberately the shortest of the three. It is the reversible one: a pod that
+drops out of the Service walks back in on the first successful probe, so reacting early
+costs nothing.
+
+### Why the startup budget needs `cache.connectRetryAttempts`
+
+A probe budget only means something while the process is alive. Fleet waits for MySQL on
+its own, so the startup probe genuinely covers that. Redis was different: upstream
+defaults `redis.connect_retry_attempts` to 0
+([config.go](../../server/config/config.go)), which is a single dial. A boot during a
+Redis outage failed that dial, `redis.NewPool` returned the error, and `initFatal` ended
+the process in seconds. The container was gone before the startup probe asked anything,
+so its budget never applied and the pod just crashlooped for the length of the outage.
+
+The fork sets `cache.connectRetryAttempts: 20`, which turns that single dial into an
+exponential backoff (`backoff/v4` defaults: 500ms initial, ×1.5, capped at 60s per
+attempt). Now the process stays up and retrying, which is exactly the state a startup
+probe is meant to supervise, and the 6 min startup budget becomes the real ceiling: Redis
+back within it and the pod boots, still missing and the probe kills the container.
+
+One caveat on the retry. It only applies to errors Go reports as temporary or as
+timeouts; a plain `connection refused` is wrapped in `backoff.Permanent` and is not
+retried ([redis.go](../../server/datastore/redis/redis.go)). A vanished node pool leaves
+the DNS names resolving and the dials timing out, which is the retryable shape, but a
+Redis that is up and actively refusing connections still exits immediately.
 
 `timeoutSeconds` is 10s on all three. The point is getting away from the 1s default:
 readiness needs it because `/healthz` does real MySQL and Redis round trips, and even
@@ -367,7 +370,7 @@ helm upgrade --install fleet oci://ghcr.io/flamingo-stack/fleetmdm/helm-charts/f
 
 | File | Purpose |
 |------|---------|
-| `charts/fleet/values.yaml` | OpenFrame mode, externalized DB/cache/setup config, `cache.keyPrefixKey`, `waitForMysql`, `probes`, `additionalCAs`, `vulnProcessing`, `deploymentAnnotations` |
+| `charts/fleet/values.yaml` | OpenFrame mode, externalized DB/cache/setup config, `cache.keyPrefixKey`, `cache.connectRetryAttempts`, `waitForMysql`, `probes`, `additionalCAs`, `vulnProcessing`, `deploymentAnnotations` |
 | `charts/fleet/templates/configmap.yaml` | **New** — generated DB/cache ConfigMaps |
 | `charts/fleet/templates/secret.yaml` | **New** — generated DB password / admin-setup Secrets |
 | `charts/fleet/templates/deployment.yaml` | `FLEET_OPENFRAME_MODE`, `FLEET_OPENFRAME_MULTI_TENANCY_ENABLED` / `FLEET_OPENFRAME_TENANT_UUID` / `FLEET_OPENFRAME_TEAM_ID`, `FLEET_REDIS_KEY_PREFIX`, ConfigMap/Secret refs, annotations, CA init container, probe split |

From df0f092416e65fb35c0f91ae191ced0212b9cf31 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Mon, 17 Aug 2026 10:28:00 +0200
Subject: [PATCH 08/10] shorten liveness and readiness, the startup probe holds
 the wait now

---
 charts/fleet/values.yaml     |  8 ++++----
 openframe/docs/helm-chart.md | 20 ++++++++++----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index f48c456caef..70b55334146 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -88,15 +88,15 @@ fleet:
       periodSeconds: 15
       failureThreshold: 25
       timeoutSeconds: 10
-    # 16 × 15 = 4 min, keep it above readiness
+    # 8 × 15 = 2 min, keep it above readiness
     liveness:
       periodSeconds: 15
-      failureThreshold: 17
+      failureThreshold: 9
       timeoutSeconds: 10
-    # 8 × 15 = 2 min
+    # 4 × 15 = 1 min
     readiness:
       periodSeconds: 15
-      failureThreshold: 9
+      failureThreshold: 5
       timeoutSeconds: 10
   # <<< OPENFRAME(helm)
   tls:
diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index b7350ca9354..e8215579b87 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -184,8 +184,8 @@ connections after a database failover.
 | Probe | Path | Budget |
 |-------|------|--------|
 | `startupProbe` | `/healthz` | 6 min |
-| `livenessProbe` | `/healthz` | 4 min |
-| `readinessProbe` | `/healthz` | 2 min |
+| `livenessProbe` | `/healthz` | 2 min |
+| `readinessProbe` | `/healthz` | 1 min |
 
 What the fork changes is the timing, not the endpoint. Upstream sets no timings at all,
 so every probe runs on the Kubernetes defaults: 1s timeout, 10s period, 3 failures. That
@@ -200,22 +200,22 @@ in two ordinary situations.
   gets to finish waiting. This happens every time MySQL and Fleet come up together,
   which on a tenant cluster is every reschedule. The `startupProbe` covers it: liveness
   and readiness do not start until the process is serving.
-- **Dependency outages.** A hiccup shorter than 4 min no longer restarts anything. At
+- **Dependency outages.** A hiccup shorter than 2 min no longer restarts anything. At
   30s it did, which is what turned a five minute Redis absence into a fleet-wide restart
   storm on 2026-08-16.
 
-Past 4 min a pod does still get killed, and that is fine now only because
+Past 2 min a pod does still get killed, and that is fine now only because
 `cache.connectRetryAttempts` changed what a restart costs. Before it, a killed pod could
 not come back while Redis was still missing and simply crashlooped for the length of the
 outage. With the retry it restarts once and then waits the outage out.
 
 The three budgets are ordered by how expensive the action is. Readiness is cheapest and
 shortest: a pod out of rotation walks back in on the first successful probe. Liveness
-throws away a warm process, so it waits longer, and the two minutes between them are the
-point: that is the window where a pod takes no traffic but still has a chance to recover
-on its own. Set them equal and readiness stops meaning anything, because the pod dies at
-the same moment it leaves the Service. Startup is longest because it supervises a boot
-that is legitimately slow.
+throws away a warm process, so it waits longer, and the minute between them is the point:
+that is the window where a pod takes no traffic but still has a chance to recover on its
+own. Set them equal and readiness stops meaning anything, because the pod dies at the same
+moment it leaves the Service. Startup is by far the longest, because with the Redis retry
+in place it is the probe that actually holds a pod through an outage.
 
 If that trade ever needs revisiting, the lever is the endpoint, not the timings.
 `health.Handler` ([health.go](../../server/health/health.go)) takes a `?check=<name>`
@@ -231,7 +231,7 @@ On upstream's 30s it would not have.
 
 Numbers live under `fleet.probes.{startup,liveness,readiness}`. The first probe runs at
 t=0, so the Nth failure lands at `(N - 1) × periodSeconds`: startup gives up at 6 min,
-liveness at 4 min, readiness at 2 min.
+liveness at 2 min, readiness at 1 min.
 
 The startup number has a floor and no real ceiling. The floor is the ~105s Fleet spends
 retrying MySQL with the port still closed: go under it and the probe cuts a legitimate

From 940d7193164e6954651e91d385687cbf38e8e3c2 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Mon, 17 Aug 2026 11:32:35 +0200
Subject: [PATCH 09/10] cut the probe docs down

---
 openframe/docs/helm-chart.md | 154 +++++++++++++----------------------
 1 file changed, 55 insertions(+), 99 deletions(-)

diff --git a/openframe/docs/helm-chart.md b/openframe/docs/helm-chart.md
index e8215579b87..cf0ef3fc040 100644
--- a/openframe/docs/helm-chart.md
+++ b/openframe/docs/helm-chart.md
@@ -174,105 +174,61 @@ exactly-once semantics.
 
 ## Probes
 
-All three probes use `/healthz`, the same endpoint upstream uses. It checks MySQL and
-Redis (`healthCheckers` in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)), and that is
-deliberate: Fleet cannot serve without either one, and `Datastore.HealthCheck`
-([mysql.go](../../server/datastore/mysql/mysql.go)) goes further than a ping, failing on
-`SELECT @@read_only = 1` specifically so the orchestrator restarts Fleet with fresh
-connections after a database failover.
-
-| Probe | Path | Budget |
-|-------|------|--------|
-| `startupProbe` | `/healthz` | 6 min |
-| `livenessProbe` | `/healthz` | 2 min |
-| `readinessProbe` | `/healthz` | 1 min |
-
-What the fork changes is the timing, not the endpoint. Upstream sets no timings at all,
-so every probe runs on the Kubernetes defaults: 1s timeout, 10s period, 3 failures. That
-kills a container 30s after the first failed probe, and Fleet needs far longer than that
-in two ordinary situations.
-
-- **Slow start.** Fleet waits for the database on its own, about 105s: 15 attempts
-  sleeping 0,1,2,…,14 seconds ([common.go](../../server/platform/mysql/common.go),
-  count from `defaultMaxAttempts` in
-  [config.go](../../server/datastore/mysql/config.go)). While it waits it isn't
-  listening on `listenPort` yet, so the default liveness kills it at 30s and it never
-  gets to finish waiting. This happens every time MySQL and Fleet come up together,
-  which on a tenant cluster is every reschedule. The `startupProbe` covers it: liveness
-  and readiness do not start until the process is serving.
-- **Dependency outages.** A hiccup shorter than 2 min no longer restarts anything. At
-  30s it did, which is what turned a five minute Redis absence into a fleet-wide restart
-  storm on 2026-08-16.
-
-Past 2 min a pod does still get killed, and that is fine now only because
-`cache.connectRetryAttempts` changed what a restart costs. Before it, a killed pod could
-not come back while Redis was still missing and simply crashlooped for the length of the
-outage. With the retry it restarts once and then waits the outage out.
-
-The three budgets are ordered by how expensive the action is. Readiness is cheapest and
-shortest: a pod out of rotation walks back in on the first successful probe. Liveness
-throws away a warm process, so it waits longer, and the minute between them is the point:
-that is the window where a pod takes no traffic but still has a chance to recover on its
-own. Set them equal and readiness stops meaning anything, because the pod dies at the same
-moment it leaves the Service. Startup is by far the longest, because with the Redis retry
-in place it is the probe that actually holds a pod through an outage.
-
-If that trade ever needs revisiting, the lever is the endpoint, not the timings.
-`health.Handler` ([health.go](../../server/health/health.go)) takes a `?check=<name>`
-filter, so `/healthz?check=mysql` gives a liveness that keeps the failover self-heal but
-drops Redis out of the restart decision. `/version`
-([version.go](../../server/version/version.go)) is the other end of the scale: a static
-struct with no auth and no dependencies, which restarts only on a wedged listener.
-
-One case the timings handle well: a Cloud SQL HA failover takes about 60 seconds, and
-60 seconds fits inside every budget here, so it passes without a restart and without the
-pod leaving the Service ([Cloud SQL HA](https://cloud.google.com/sql/docs/mysql/high-availability)).
-On upstream's 30s it would not have.
-
-Numbers live under `fleet.probes.{startup,liveness,readiness}`. The first probe runs at
-t=0, so the Nth failure lands at `(N - 1) × periodSeconds`: startup gives up at 6 min,
-liveness at 2 min, readiness at 1 min.
-
-The startup number has a floor and no real ceiling. The floor is the ~105s Fleet spends
-retrying MySQL with the port still closed: go under it and the probe cuts a legitimate
-wait short, which is the original bug with liveness playing that role at 30s. Above the
-floor it only catches a hang, because a boot that cannot reach its dependencies ends the
-process anyway. So 6 min reads as "how long we tolerate a stuck boot", not "how long we
-wait for the database". That wait is Fleet's own and a probe can only cut it short,
-never extend it.
-
-Readiness is deliberately the shortest of the three. It is the reversible one: a pod that
-drops out of the Service walks back in on the first successful probe, so reacting early
-costs nothing.
-
-### Why the startup budget needs `cache.connectRetryAttempts`
-
-A probe budget only means something while the process is alive. Fleet waits for MySQL on
-its own, so the startup probe genuinely covers that. Redis was different: upstream
-defaults `redis.connect_retry_attempts` to 0
-([config.go](../../server/config/config.go)), which is a single dial. A boot during a
-Redis outage failed that dial, `redis.NewPool` returned the error, and `initFatal` ended
-the process in seconds. The container was gone before the startup probe asked anything,
-so its budget never applied and the pod just crashlooped for the length of the outage.
-
-The fork sets `cache.connectRetryAttempts: 20`, which turns that single dial into an
-exponential backoff (`backoff/v4` defaults: 500ms initial, ×1.5, capped at 60s per
-attempt). Now the process stays up and retrying, which is exactly the state a startup
-probe is meant to supervise, and the 6 min startup budget becomes the real ceiling: Redis
-back within it and the pod boots, still missing and the probe kills the container.
-
-One caveat on the retry. It only applies to errors Go reports as temporary or as
-timeouts; a plain `connection refused` is wrapped in `backoff.Permanent` and is not
-retried ([redis.go](../../server/datastore/redis/redis.go)). A vanished node pool leaves
-the DNS names resolving and the dials timing out, which is the retryable shape, but a
-Redis that is up and actively refusing connections still exits immediately.
-
-`timeoutSeconds` is 10s on all three. The point is getting away from the 1s default:
-readiness needs it because `/healthz` does real MySQL and Redis round trips, and even
-`/version` can miss a 1s deadline on a node still busy starting everything else. One
-number everywhere keeps it readable. It caps a single attempt and sits inside the
-period, it isn't added on top, so keep it under `periodSeconds` or the timeout becomes
-the real cadence
+All three probes keep upstream's endpoint, `/healthz`, which checks MySQL and Redis
+(`healthCheckers` in [cmd/fleet/serve.go](../../cmd/fleet/serve.go)). What the fork adds
+is timings, which upstream leaves unset, and a retry on the first Redis dial.
+
+| Probe | Budget | `fleet.probes.*` |
+|-------|--------|------------------|
+| `startupProbe` | 6 min | `25 × 15s` |
+| `livenessProbe` | 2 min | `9 × 15s` |
+| `readinessProbe` | 1 min | `5 × 15s` |
+
+Without timings every probe runs on the Kubernetes defaults, which kill a container 30s
+after the first failure. That is far too short for Fleet: it waits for MySQL on its own
+for about 105s (15 attempts sleeping 0,1,…,14s, see
+[common.go](../../server/platform/mysql/common.go) and `defaultMaxAttempts` in
+[config.go](../../server/datastore/mysql/config.go)) and does not listen on `listenPort`
+while it waits.
+
+That is what the `startupProbe` is for. Liveness and readiness do not run until it
+succeeds, so the boot gets its own budget and the other two can stay short.
+
+The three budgets are ordered by how expensive the action is. Readiness is reversible and
+so the shortest; liveness throws away a warm process; startup supervises a boot that is
+legitimately slow. The minute between readiness and liveness is deliberate: it is the
+window where a pod takes no traffic but can still recover on its own.
+
+### Why `cache.connectRetryAttempts` is set
+
+A probe budget only means something while the process is alive. Upstream defaults
+`redis.connect_retry_attempts` to 0 ([config.go](../../server/config/config.go)), a single
+dial, so a boot during a Redis outage ended in `initFatal` within seconds and the pod
+crashlooped for the length of the outage with no probe ever consulted.
+
+The fork sets it to 20, which turns that dial into an exponential backoff (`backoff/v4`
+defaults, capped at 60s per attempt). The process now stays up retrying and the 6 min
+startup budget becomes the real ceiling.
+
+One caveat: the retry only covers errors Go reports as temporary or as timeouts, while a
+plain `connection refused` is `backoff.Permanent` and still exits at once
+([redis.go](../../server/datastore/redis/redis.go)).
+
+### Notes
+
+`timeoutSeconds` is 10s everywhere, up from the 1s default, because `/healthz` does real
+MySQL and Redis round trips. It caps one attempt and sits inside the period rather than
+adding to it, so keep it under `periodSeconds`.
+
+The first probe runs at t=0, so the Nth failure lands at `(N - 1) × periodSeconds`. A
+Cloud SQL HA failover takes about 60s and therefore passes without a restart and without
+leaving the Service ([Cloud SQL HA](https://cloud.google.com/sql/docs/mysql/high-availability));
+on upstream's 30s it would not have.
+
+If the endpoint ever needs to change rather than the timings, `health.Handler`
+([health.go](../../server/health/health.go)) takes a `?check=<name>` filter, and `/version`
+([version.go](../../server/version/version.go)) is a static struct with no dependencies at
+all.
 
 ## Vulnerability feed persistence (`vuln-persistence`)
 

From 32e9b6d3e7308162463a8311a66d8d151fc40c98 Mon Sep 17 00:00:00 2001
From: Viktor Ishchenko <viktori@flamingo.cx>
Date: Mon, 17 Aug 2026 17:45:39 +0200
Subject: [PATCH 10/10] drop the redis retry count to 10

---
 charts/fleet/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
index 70b55334146..5ec1edffcad 100644
--- a/charts/fleet/values.yaml
+++ b/charts/fleet/values.yaml
@@ -378,7 +378,7 @@ cache:
   secretName: redis
   passwordKey: redis-password
   # >>> OPENFRAME(helm): upstream defaults this to 0, so a boot without Redis dies at once — openframe/docs/helm-chart.md
-  connectRetryAttempts: 20
+  connectRetryAttempts: 10
   # <<< OPENFRAME(helm)
   # >>> OPENFRAME(redis-key-prefix): fork-added per-tenant Redis key prefix — openframe/docs/redis-key-prefix.md
   # When set, wires FLEET_REDIS_KEY_PREFIX from existingConfigMap[keyPrefixKey].