From 1bcbf69d0c0e69b4d52dc0c1759ceecedaa14889 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 06:33:42 +0000 Subject: [PATCH 1/3] harden(k3s): add secrets-encryption at-rest playbook (Phase 4) etcd snapshots ship to Cloudflare R2 in plaintext today, so every Secret in etcd (Sealed-Secrets master key, hcloud token, Cloudflare tunnel token, SMTP/Grafana creds) is readable offline from a snapshot. Enable AES-CBC secrets encryption at rest to close that exposure in every future snapshot. This is the last outstanding item from the 2026-06-19 optimization sweep. New operator-supervised one-shot `ansible/playbooks/k3s-secrets-encryption.yml` encoding the doc-cited HA "enable on existing cluster" flow as five phased plays: precondition+enable -> flag rollout + rolling restart -> start-stage/ hash-match gate + rotate-keys -> post-rotate rolling restart -> verify Enabled. Safety: - Destructive CLI stages (enable, rotate-keys) are status-gated so an interrupted run resumes rather than double-applies. - Hash-match assert guards rotate-keys (k3s: mismatched-hash rotation can permanently corrupt the cluster). - Every rolling play is serial:1 + max_fail_percentage:0 (etcd quorum-safe); Ready-wait uses default([]) since the operator kubeconfig points at cp-1. - Requires -e snapshot_confirmed=true to force a pre-change etcd snapshot. Version gate: modern enable-on-existing-cluster flow needs v1.33.10+/v1.34.6+/v1.35.3+k3s1; cluster runs v1.36.2+k3s1 -> supported. Doc: https://docs.k3s.io/cli/secrets-encrypt Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DSsgEHGrf6oKAQoerbB5Lt --- ansible/playbooks/k3s-secrets-encryption.yml | 262 ++++++++++++++++++ docs/2026-06-19-unused-optimization-levers.md | 3 +- 2 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 ansible/playbooks/k3s-secrets-encryption.yml diff --git a/ansible/playbooks/k3s-secrets-encryption.yml b/ansible/playbooks/k3s-secrets-encryption.yml new file mode 100644 index 00000000..a65c86ed --- /dev/null +++ b/ansible/playbooks/k3s-secrets-encryption.yml @@ -0,0 +1,262 @@ +# One-shot, OPERATOR-SUPERVISED playbook — do NOT import into site.yml. +# Phase 4 (see docs/2026-06-19-unused-optimization-levers.md): enable k3s Secrets Encryption at rest. +# +# WHY. etcd snapshots ship to Cloudflare R2 (infra/cloudflare_storage.tf, bucket etcd_snapshots) in +# PLAINTEXT today — so every Secret in etcd is readable offline from a snapshot: the Sealed-Secrets +# master key, the hcloud API token, the Cloudflare tunnel token, SMTP/Grafana creds. Enabling +# `--secrets-encryption` AES-encrypts Secrets at rest in etcd, and therefore in every FUTURE snapshot, +# closing that offline-read exposure. This is the last outstanding item from the 2026-06-19 sweep. +# +# ┌── READ BEFORE RUNNING ─────────────────────────────────────────────────────────────────────────┐ +# │ k3s warns: "Failure to follow proper procedure for rotating encryption keys can leave your │ +# │ cluster permanently corrupted." This is the highest-care node-config change in this repo. │ +# │ │ +# │ 1. TAKE A FRESH etcd SNAPSHOT FIRST — it is the rollback anchor. (It is also the LAST plaintext │ +# │ snapshot; treat/expire it deliberately.) On any control plane: │ +# │ k3s etcd-snapshot save --name pre-secrets-encryption │ +# │ Then re-run this playbook passing -e snapshot_confirmed=true to acknowledge you did. │ +# │ 2. Dry-run first: just ansible-check k3s-secrets-encryption │ +# │ 3. Apply: just ansible-converge k3s-secrets-encryption -e snapshot_confirmed=true │ +# └──────────────────────────────────────────────────────────────────────────────────────────────────┘ +# +# DOC-BACKED SEQUENCE (https://docs.k3s.io/cli/secrets-encrypt — "Enable Secrets Encryption on an +# Existing Cluster", HA variant). Version-gated: the modern enable-on-existing-cluster flow is +# available as of the March 2026 releases (v1.33.10+/v1.34.6+/v1.35.3+k3s1). This cluster runs +# v1.36.2+k3s1 → supported. The exact ordered flow, encoded as the five plays below: +# 1. status == "Disabled, no configuration file found" (precondition) +# 2. `k3s secrets-encrypt enable` (ONE server) +# 3. add `secrets-encryption: true`, restart S1→S2→S3 (rolling, one at a time) +# 4. status == stage "start" + "All hashes match" (gate before rotate) +# 5. `k3s secrets-encrypt rotate-keys` (ONE server; reencrypts existing secrets) +# 6. restart S1→S2→S3 again (rolling) +# 7. status == "Enabled" + stage "reencrypt_finished" + "All hashes match" (verify) +# +# SAFETY. Every rolling play is serial:1 + max_fail_percentage:0 + order:sorted, so a bad restart on +# any node halts the play with etcd quorum (2/3) intact; and the Ready-wait uses default([]) because +# the operator kubeconfig points at cp-1, which is briefly unreachable while it restarts. A k3s +# restart is NON-disruptive to running pods (it re-attaches to existing containers via containerd). +# The destructive CLI stages (`enable`, `rotate-keys`) are STATUS-GATED, so an interrupted run resumes +# from where it stopped rather than double-applying. If a run halts mid-way, inspect state by hand +# (`k3s secrets-encrypt status` on each CP) before re-running. +# +# REVERSIBILITY (stateful — snapshot is the anchor): `k3s secrets-encrypt disable` on one server → +# restart all → `k3s secrets-encrypt reencrypt --force --skip`, then remove this drop-in. + +# ── Play 1 — precondition + enable (one server) ──────────────────────────────────────────────────── +- name: "Secrets encryption — precondition + enable" + hosts: control_planes + gather_facts: false + become: true + run_once: true # the `enable` subcommand is issued on exactly ONE server (k3s coordinates the rest) + vars: + k3s_bin: /usr/local/bin/k3s + tasks: + - name: Require an explicit pre-change etcd snapshot acknowledgement + ansible.builtin.assert: + that: + - snapshot_confirmed | default(false) | bool + fail_msg: >- + Refusing to proceed. Take a fresh etcd snapshot first + (`k3s etcd-snapshot save --name pre-secrets-encryption`), then re-run with + -e snapshot_confirmed=true. The snapshot is the only rollback anchor for this change. + success_msg: "Pre-change etcd snapshot acknowledged by operator." + + - name: Read current secrets-encryption status + ansible.builtin.command: + cmd: "{{ k3s_bin }} secrets-encrypt status" + register: se_status_pre + changed_when: false + failed_when: false + when: not ansible_check_mode + + - name: "Enable secrets encryption (only if currently Disabled with no config)" + # Gated on the exact pristine state so a resumed run never re-issues enable over an + # already-configured cluster. After this, encryption is not yet active — restart + rotate follow. + ansible.builtin.command: + cmd: "{{ k3s_bin }} secrets-encrypt enable" + register: se_enable + changed_when: se_enable.rc == 0 + when: + - not ansible_check_mode + - "'no configuration file found' in (se_status_pre.stdout | default(''))" + + - name: Report enable-stage decision + ansible.builtin.debug: + msg: >- + {{ 'enable issued on ' ~ inventory_hostname + if (se_enable is defined and se_enable is not skipped) + else 'enable skipped — cluster is past the pristine state (resuming later stage)' }} + when: not ansible_check_mode + +# ── Play 2 — add flag to every server + rolling restart (S1→S2→S3) ───────────────────────────────── +- name: "Secrets encryption — deploy flag + rolling restart" + hosts: control_planes + gather_facts: false + become: true + serial: 1 + max_fail_percentage: 0 + order: sorted + handlers: + - name: Restart k3s + ansible.builtin.systemd: + name: k3s + state: restarted + tasks: + - name: Create k3s config drop-in directory + ansible.builtin.file: + path: /etc/rancher/k3s/config.yaml.d + state: directory + owner: root + group: root + mode: "0755" + + - name: Deploy secrets-encryption drop-in + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml.d/secrets-encryption.yaml + owner: root + group: root + mode: "0644" + content: | + # AES-CBC encryption of Secret objects at rest in embedded etcd (and thus in every future + # R2 snapshot). Kept as a persistent config flag — not just the one-time `secrets-encrypt + # enable` — so the setting survives the frequent k3s restarts on this cluster (kured + # reboots, SUC upgrades, other ansible one-shots). Key material lives in + # /var/lib/rancher/k3s/server/cred/encryption-config.json (node-local, never in git). + secrets-encryption: true + notify: Restart k3s + + - name: Flush handlers (restart k3s before verifying) + ansible.builtin.meta: flush_handlers + + - name: Wait for this node to be Ready again + kubernetes.core.k8s_info: + kind: Node + name: "{{ kubernetes_node_name }}" + register: node_info + delegate_to: localhost + become: false + until: > + (node_info.resources | default([]) | length > 0) and (node_info.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') + | map(attribute='status') | list | first | default('')) == 'True' + retries: 30 + delay: 10 + when: not ansible_check_mode + +# ── Play 3 — gate on "start" stage + rotate keys (one server) ────────────────────────────────────── +- name: "Secrets encryption — verify start stage + rotate keys" + hosts: control_planes + gather_facts: false + become: true + run_once: true + vars: + k3s_bin: /usr/local/bin/k3s + tasks: + - name: Read status after the flag rollout + ansible.builtin.command: + cmd: "{{ k3s_bin }} secrets-encrypt status" + register: se_status_start + changed_when: false + failed_when: false + when: not ansible_check_mode + + - name: "Gate: all server hashes must match before rotating keys" + # Rotating with mismatched hashes across nodes is the documented path to corruption. Halt here + # unless every server agrees. (Skip the gate only once already Enabled — i.e. a completed re-run.) + ansible.builtin.assert: + that: + - "'All hashes match' in (se_status_start.stdout | default(''))" + fail_msg: >- + Server encryption hashes do not match across control planes — NOT rotating keys. + Ensure Play 2 restarted every server (status should show stage 'start', all hashes match) + before continuing. Inspect with `k3s secrets-encrypt status` on each CP. + when: + - not ansible_check_mode + - "'Encryption Status: Enabled' not in (se_status_start.stdout | default(''))" + + - name: "Rotate keys to activate encryption + reencrypt existing secrets (only at stage 'start')" + # rotate-keys is the modern (v1.30+) command that activates the new key AND reencrypts existing + # secrets (~5/sec). Gated on stage 'start' so a resumed/completed run does not re-rotate. + ansible.builtin.command: + cmd: "{{ k3s_bin }} secrets-encrypt rotate-keys" + register: se_rotate + changed_when: se_rotate.rc == 0 + when: + - not ansible_check_mode + - "'Current Rotation Stage: start' in (se_status_start.stdout | default(''))" + +# ── Play 4 — second rolling restart (S1→S2→S3) so every node reaches reencrypt_finished ──────────── +- name: "Secrets encryption — post-rotate rolling restart" + hosts: control_planes + gather_facts: false + become: true + serial: 1 + max_fail_percentage: 0 + order: sorted + vars: + k3s_bin: /usr/local/bin/k3s + tasks: + - name: Read this node's status + ansible.builtin.command: + cmd: "{{ k3s_bin }} secrets-encrypt status" + register: se_status_node + changed_when: false + failed_when: false + when: not ansible_check_mode + + - name: "Restart k3s to pick up rotated keys (only if this node isn't already finished)" + # Gated per-node on reencrypt_finished, so a completed re-run restarts nothing. + ansible.builtin.systemd: + name: k3s + state: restarted + when: + - not ansible_check_mode + - "'reencrypt_finished' not in (se_status_node.stdout | default(''))" + + - name: Wait for this node to be Ready again + kubernetes.core.k8s_info: + kind: Node + name: "{{ kubernetes_node_name }}" + register: node_info + delegate_to: localhost + become: false + until: > + (node_info.resources | default([]) | length > 0) and (node_info.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') + | map(attribute='status') | list | first | default('')) == 'True' + retries: 30 + delay: 10 + when: not ansible_check_mode + +# ── Play 5 — final verification ──────────────────────────────────────────────────────────────────── +- name: "Secrets encryption — verify Enabled" + hosts: control_planes + gather_facts: false + become: true + run_once: true + vars: + k3s_bin: /usr/local/bin/k3s + tasks: + - name: Read final status + ansible.builtin.command: + cmd: "{{ k3s_bin }} secrets-encrypt status" + register: se_status_final + changed_when: false + failed_when: false + when: not ansible_check_mode + + - name: "Assert encryption is Enabled and reencryption finished with matching hashes" + ansible.builtin.assert: + that: + - "'Encryption Status: Enabled' in (se_status_final.stdout | default(''))" + - "'reencrypt_finished' in (se_status_final.stdout | default(''))" + - "'All hashes match' in (se_status_final.stdout | default(''))" + fail_msg: "Secrets encryption did NOT reach Enabled/reencrypt_finished. Status:\n{{ se_status_final.stdout | default('(no output)') }}" + success_msg: "Secrets encryption Enabled on all control planes (reencrypt_finished, hashes match)." + when: not ansible_check_mode + + - name: "Next steps" + ansible.builtin.debug: + msg: >- + Encryption at rest is live. Take a NEW etcd snapshot so R2 holds an encrypted one + (`k3s etcd-snapshot save --name post-secrets-encryption`), and expire the pre-change + plaintext snapshot. Confirm `just verify-mtu` and cluster health are clean. + when: not ansible_check_mode diff --git a/docs/2026-06-19-unused-optimization-levers.md b/docs/2026-06-19-unused-optimization-levers.md index 9cadaee4..02144dfa 100644 --- a/docs/2026-06-19-unused-optimization-levers.md +++ b/docs/2026-06-19-unused-optimization-levers.md @@ -119,6 +119,7 @@ Revised phasing agreed 2026-06-19 — batch the cheap additive flags into ONE ro - **Status (2026-06-19): 2a DONE.** Applied via `ansible/playbooks/k3s-etcd-metrics.yml` (etcd-only; handler-based notify `Restart k3s` + `flush_handlers`, `serial:1`/`max_fail_percentage:0`, `ss` non-loopback :2381 verify). Rolling converge succeeded on all 3 CPs (`ok=7 changed=2 failed=0` each). **Verified:** all nodes Ready, `readyz` etcd ok, no firing alerts, `just verify-mtu` clean; and a throwaway pod scraped `http://10.0.0.{2,3,4}:2381/metrics` returning **636/611/609 `etcd_*` series** — proving both the flag took effect and the pod→node:2381 path (which 2b's vmagent will use) works. The `until` readiness wait was hardened with `default([])` (cp-1 is the kubeconfig endpoint and restarts first, so the API is briefly unreachable mid-restart). **2b (etcd scrape + rules) — ✅ DONE 2026-06-19 (commits cfe02db→adb21db).** Gotcha: ArgoCD's `resource.exclusions` drops `Endpoints`/`EndpointSlice`, so the chart's `kubeEtcd` Service-scrape (which needs a manually-populated Endpoints) was applied as Service+VMServiceScrape but the Endpoints was silently NOT applied → **0 targets**. Pivoted to a **`VMStaticScrape`** in `extraObjects` hitting `10.0.0.{2,3,4}:2381` over http (static targets need no Endpoints). Kept `kubeEtcd.enabled=true` ONLY for the etcd rule group (chart gates it on `kubeEtcd.enabled`) with its scrape objects suppressed (`endpoints:[]`, `service.enabled:false`, `vmScrape.enabled:false`) → no cruft. **Verified live:** `up{job=kube-etcd}` 3/3, `etcd_server_has_leader=1` ×3, etcd VMRule (15 alerts) loaded, no false alerts. **No duplicates:** the apiserver job carries etcd-CLIENT metrics (`etcd_request_*` — disjoint names); `:2381` carries the 127 etcd-SERVER names (`etcd_server_*`, `etcd_disk_*`, …). Lesson recorded in memory `argocd-excludes-endpoints-use-vmstaticscrape`. - **Phase 3 — k3s `embedded-registry: true` (Spegel P2P image mirror).** ✅ **DONE 2026-06-19 (commit 700e754)** via `ansible/playbooks/k3s-embedded-registry.yml` (rolling, serial:1, :5001 listener gate). Two drop-ins in one restart: `embedded-registry: true` + `registries.yaml` with the **`"*"` wildcard** (chosen over explicit: zero-maintenance, covers all 5 in-use registries incl `ecr-public.aws.com` that a 4-entry list missed; safe because NO private/authenticated pulls exist → cred-sharing moot, and v1.35.5 is past the v1.35.1 wildcard-mirror fix #13539). **No firewall change** (5001/6443 ride the Hetzner private net, unfiltered; same as :2381). Default-endpoint fallback ON → worst case = slower pull, never failed. **Verified live:** all 3 CPs enabled + P2P bootstrap-connectivity reached, cp-1/cp-3 advertising (61/66 digests, all 5 registries), and a **proven peer cache-hit** — cp-3 pulled `busybox:1.35.0` from a peer (`spegel_mirror_requests_total{cache="hit"}=5`). **Caveat (RESOLVED — was transient):** cp-2's spegel metrics initially didn't surface via the kubelet-proxy `/metrics` scrape, but this self-resolved — it was a metric-registration lag at startup (matches the `will retry in the background` warning). Re-checked: all 3 nodes now expose an identical 31-line spegel metric set. No action taken; no fix needed. Doc src https://docs.k3s.io/installation/registry-mirror. - **Phase 4 — k3s `secrets-encryption: true` (highest-value security item; highest care).** VERIFIED: NOT a simple flag on HA — ordered stateful sequence: `k3s secrets-encrypt enable` on one server → restart ALL servers with `secrets-encryption: true` → `k3s secrets-encrypt rotate-keys` → restart all again → verify `Enabled`. Only protects FUTURE etcd writes + FUTURE S3 snapshots (already-taken snapshots stay plaintext). Operator-in-the-loop with playbook assist, not fully autonomous. + - **Playbook authored 2026-07-14: `ansible/playbooks/k3s-secrets-encryption.yml`** (pending operator run). Encodes the exact doc-cited HA flow as five phased plays (precondition+enable → flag rollout+rolling restart → start-stage/hash-match gate+rotate-keys → post-rotate rolling restart → verify Enabled). The two destructive CLI stages (`enable`, `rotate-keys`) are **status-gated** so an interrupted run resumes rather than double-applies; a hash-match assert guards `rotate-keys` (k3s: mismatched-hash rotation can permanently corrupt); every rolling play is `serial:1`/`max_fail_percentage:0` (etcd quorum-safe); requires `-e snapshot_confirmed=true` to force a pre-change snapshot. Doc: https://docs.k3s.io/cli/secrets-encrypt. **Version gate:** the modern enable-on-existing-cluster flow needs v1.33.10+/v1.34.6+/v1.35.3+k3s1 (March 2026 releases) — cluster is on v1.36.2+k3s1 → supported. Run: `just ansible-check k3s-secrets-encryption`, then `just ansible-converge k3s-secrets-encryption -e snapshot_confirmed=true`. **Parked (do NOT do without the stated prerequisite):** - `kube/system-reserved` / `kube-reserved` — measure live node headroom on the 8GB CX33s first; mis-sizing on small nodes causes evictions. @@ -137,7 +138,7 @@ Revised phasing agreed 2026-06-19 — batch the cheap additive flags into ONE ro - **Phase 3 — Spegel embedded registry** ✅ **DONE** (ansible `k3s-embedded-registry.yml`, wildcard mirror, peer cache-hit proven). - **Phase 5 — Cilium BandwidthManager + BBR** ✅ **DONE** (done out of order, ahead of Phase 4, per operator choice). **Only remaining: Phase 4 — secrets-encryption** (the highest-care stateful one; hard to roll back — take a fresh etcd snapshot first, then the exact `enable→restart→rotate-keys→restart` sequence). - **Phase 3 — Spegel** (`embedded-registry` + `registries.yaml` + firewall verify). -- **Phase 4 — secrets-encryption** (stateful sequence, operator-in-the-loop). +- **Phase 4 — secrets-encryption** (stateful sequence, operator-in-the-loop). Playbook authored 2026-07-14 (`ansible/playbooks/k3s-secrets-encryption.yml`); **pending operator run** (`-e snapshot_confirmed=true`). - **Phase 5 (Tier 3) — Cilium BandwidthManager + BBR** ✅ **DONE** (canary per-node agent restart, verify-mtu PASS, host cc=bbr/qdisc=fq, internet-facing deploys flipped). **All planned phases complete.** Safety over speed: one rolling pass at a time, verify Ready + etcd quorum (2/3) between nodes, ship the gitops scrape only after the flag is confirmed live. The k3s items move the reboot invariant (parallel pulls) and security posture (at-rest encryption) — the most valuable findings, entirely missed by a CPU-usage lens. From b4d2abfbe655001495a4dd293f194928777ca3a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 09:07:17 +0000 Subject: [PATCH 2/3] harden(k3s): use secretbox provider for secrets-encryption Set secrets-encryption-provider: secretbox (XSalsa20-Poly1305, AEAD) instead of the k3s default aescbc (AES-CBC, unauthenticated). Chosen before the first run: the Play 3 rotate-keys step lands all data on a secretbox key (the documented aescbc->secretbox migration), so we never pay a later rotate+reencrypt to switch providers. Free now, expensive after go-live. Provider support: k3s >= v1.32.4+k3s1; cluster v1.36.2+k3s1. Doc: https://docs.k3s.io/security/secrets-encryption Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DSsgEHGrf6oKAQoerbB5Lt --- ansible/playbooks/k3s-secrets-encryption.yml | 26 +++++++++++++------ docs/2026-06-19-unused-optimization-levers.md | 1 + 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ansible/playbooks/k3s-secrets-encryption.yml b/ansible/playbooks/k3s-secrets-encryption.yml index a65c86ed..02a385cd 100644 --- a/ansible/playbooks/k3s-secrets-encryption.yml +++ b/ansible/playbooks/k3s-secrets-encryption.yml @@ -25,7 +25,7 @@ # v1.36.2+k3s1 → supported. The exact ordered flow, encoded as the five plays below: # 1. status == "Disabled, no configuration file found" (precondition) # 2. `k3s secrets-encrypt enable` (ONE server) -# 3. add `secrets-encryption: true`, restart S1→S2→S3 (rolling, one at a time) +# 3. add `secrets-encryption: true` + `-provider: secretbox`, restart S1→S2→S3 (rolling) # 4. status == stage "start" + "All hashes match" (gate before rotate) # 5. `k3s secrets-encrypt rotate-keys` (ONE server; reencrypts existing secrets) # 6. restart S1→S2→S3 again (rolling) @@ -117,12 +117,20 @@ group: root mode: "0644" content: | - # AES-CBC encryption of Secret objects at rest in embedded etcd (and thus in every future - # R2 snapshot). Kept as a persistent config flag — not just the one-time `secrets-encrypt - # enable` — so the setting survives the frequent k3s restarts on this cluster (kured - # reboots, SUC upgrades, other ansible one-shots). Key material lives in + # Encrypt Secret objects at rest in embedded etcd (and thus in every future R2 snapshot). + # Kept as a persistent config flag — not just the one-time `secrets-encrypt enable` — so the + # setting survives the frequent k3s restarts on this cluster (kured reboots, SUC upgrades, + # other ansible one-shots). Key material lives in # /var/lib/rancher/k3s/server/cred/encryption-config.json (node-local, never in git). secrets-encryption: true + # Use secretbox (XSalsa20-Poly1305, an AEAD/authenticated cipher) instead of the k3s default + # aescbc (AES-CBC, unauthenticated). With this provider present before the Play 3 + # `rotate-keys` step, the reencrypt lands ALL data on a secretbox key — the exact documented + # aescbc→secretbox migration — so we never pay a later rotate+reencrypt to switch. Choosing + # it now (before the first run) is free; switching after go-live is a full key cycle. + # Provider support: k3s ≥ v1.32.4+k3s1 (April 2025); cluster is v1.36.2+k3s1. + # Doc: https://docs.k3s.io/security/secrets-encryption (Choosing Encryption Provider). + secrets-encryption-provider: secretbox notify: Restart k3s - name: Flush handlers (restart k3s before verifying) @@ -256,7 +264,9 @@ - name: "Next steps" ansible.builtin.debug: msg: >- - Encryption at rest is live. Take a NEW etcd snapshot so R2 holds an encrypted one - (`k3s etcd-snapshot save --name post-secrets-encryption`), and expire the pre-change - plaintext snapshot. Confirm `just verify-mtu` and cluster health are clean. + Encryption at rest is live. Confirm the ACTIVE key in `k3s secrets-encrypt status` is a + secretbox key (Key Type is NOT AES-CBC) — the rotate-keys step should have made it active. + Then take a NEW etcd snapshot so R2 holds an encrypted one + (`k3s etcd-snapshot save --name post-secrets-encryption`), expire the pre-change plaintext + snapshot, and confirm `just verify-mtu` + cluster health are clean. when: not ansible_check_mode diff --git a/docs/2026-06-19-unused-optimization-levers.md b/docs/2026-06-19-unused-optimization-levers.md index 02144dfa..bedf59e3 100644 --- a/docs/2026-06-19-unused-optimization-levers.md +++ b/docs/2026-06-19-unused-optimization-levers.md @@ -120,6 +120,7 @@ Revised phasing agreed 2026-06-19 — batch the cheap additive flags into ONE ro - **Phase 3 — k3s `embedded-registry: true` (Spegel P2P image mirror).** ✅ **DONE 2026-06-19 (commit 700e754)** via `ansible/playbooks/k3s-embedded-registry.yml` (rolling, serial:1, :5001 listener gate). Two drop-ins in one restart: `embedded-registry: true` + `registries.yaml` with the **`"*"` wildcard** (chosen over explicit: zero-maintenance, covers all 5 in-use registries incl `ecr-public.aws.com` that a 4-entry list missed; safe because NO private/authenticated pulls exist → cred-sharing moot, and v1.35.5 is past the v1.35.1 wildcard-mirror fix #13539). **No firewall change** (5001/6443 ride the Hetzner private net, unfiltered; same as :2381). Default-endpoint fallback ON → worst case = slower pull, never failed. **Verified live:** all 3 CPs enabled + P2P bootstrap-connectivity reached, cp-1/cp-3 advertising (61/66 digests, all 5 registries), and a **proven peer cache-hit** — cp-3 pulled `busybox:1.35.0` from a peer (`spegel_mirror_requests_total{cache="hit"}=5`). **Caveat (RESOLVED — was transient):** cp-2's spegel metrics initially didn't surface via the kubelet-proxy `/metrics` scrape, but this self-resolved — it was a metric-registration lag at startup (matches the `will retry in the background` warning). Re-checked: all 3 nodes now expose an identical 31-line spegel metric set. No action taken; no fix needed. Doc src https://docs.k3s.io/installation/registry-mirror. - **Phase 4 — k3s `secrets-encryption: true` (highest-value security item; highest care).** VERIFIED: NOT a simple flag on HA — ordered stateful sequence: `k3s secrets-encrypt enable` on one server → restart ALL servers with `secrets-encryption: true` → `k3s secrets-encrypt rotate-keys` → restart all again → verify `Enabled`. Only protects FUTURE etcd writes + FUTURE S3 snapshots (already-taken snapshots stay plaintext). Operator-in-the-loop with playbook assist, not fully autonomous. - **Playbook authored 2026-07-14: `ansible/playbooks/k3s-secrets-encryption.yml`** (pending operator run). Encodes the exact doc-cited HA flow as five phased plays (precondition+enable → flag rollout+rolling restart → start-stage/hash-match gate+rotate-keys → post-rotate rolling restart → verify Enabled). The two destructive CLI stages (`enable`, `rotate-keys`) are **status-gated** so an interrupted run resumes rather than double-applies; a hash-match assert guards `rotate-keys` (k3s: mismatched-hash rotation can permanently corrupt); every rolling play is `serial:1`/`max_fail_percentage:0` (etcd quorum-safe); requires `-e snapshot_confirmed=true` to force a pre-change snapshot. Doc: https://docs.k3s.io/cli/secrets-encrypt. **Version gate:** the modern enable-on-existing-cluster flow needs v1.33.10+/v1.34.6+/v1.35.3+k3s1 (March 2026 releases) — cluster is on v1.36.2+k3s1 → supported. Run: `just ansible-check k3s-secrets-encryption`, then `just ansible-converge k3s-secrets-encryption -e snapshot_confirmed=true`. + - **Provider: `secretbox` (not the default `aescbc`).** The drop-in sets `secrets-encryption-provider: secretbox` (XSalsa20-Poly1305, AEAD/authenticated) over aescbc (AES-CBC, unauthenticated). Chosen *before first run* so the `rotate-keys` step lands all data on a secretbox key with zero extra migration — switching post-go-live would cost a full rotate+reencrypt. Provider support: k3s ≥ v1.32.4+k3s1. Doc: https://docs.k3s.io/security/secrets-encryption. **Parked (do NOT do without the stated prerequisite):** - `kube/system-reserved` / `kube-reserved` — measure live node headroom on the 8GB CX33s first; mis-sizing on small nodes causes evictions. From 59251e583c91ec644280f678306e5c12314f822c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 09:33:12 +0000 Subject: [PATCH 3/3] harden(k3s): CIS hardening flags (NodeRestriction + kubelet stream/cipher) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold k3s node-config hardening into the secrets-encryption PR. One drop-in, one rolling restart, delivered via a new ansible one-shot: - kube-apiserver: enable-admission-plugins=NodeRestriction (not in the default set; caps each kubelet to its own Node + bound pods — meaningful here since all 3 nodes are cp_worker so every kubelet holds node creds). - kubelet: streaming-connection-idle-timeout=5m + strong AEAD tls-cipher-suites (both real kubelet CLI flags per the k3s hardening guide). Safety: adds a LOCAL-apiserver /readyz gate per node, because a bad kube-apiserver-arg crash-loops the in-process apiserver while the kubelet still reports Ready via the LB — a plain node-Ready gate would mask it and march on. serial:1 + max_fail_percentage:0 halts before the next node. Doc: https://docs.k3s.io/security/hardening-guide Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DSsgEHGrf6oKAQoerbB5Lt --- ansible/playbooks/k3s-cis-hardening.yml | 127 ++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 ansible/playbooks/k3s-cis-hardening.yml diff --git a/ansible/playbooks/k3s-cis-hardening.yml b/ansible/playbooks/k3s-cis-hardening.yml new file mode 100644 index 00000000..09455413 --- /dev/null +++ b/ansible/playbooks/k3s-cis-hardening.yml @@ -0,0 +1,127 @@ +# One-shot playbook — do NOT import into site.yml. +# k3s CIS hardening flags folded into the secrets-encryption PR (all k3s node-config in one place). +# Two low-risk, doc-recommended additions from the k3s CIS Hardening Guide +# (https://docs.k3s.io/security/hardening-guide), both delivered as ONE config drop-in / ONE restart: +# +# 1. kube-apiserver `--enable-admission-plugins=NodeRestriction` +# NodeRestriction is NOT in the apiserver's default-enabled set and k3s does not add it. It +# limits each kubelet to mutating only its OWN Node object and the pods bound to it. On this +# cluster ALL three nodes are cp_worker, so every kubelet holds node credentials — this directly +# shrinks the blast radius of a single compromised node. hccm/Cilium/kured use their own +# ServiceAccounts (not kubelet identity), so they are unaffected. `--enable-admission-plugins` +# is ADDITIVE to the apiserver defaults (it does not disable the default plugins). +# +# 2. kubelet `--streaming-connection-idle-timeout=5m` + `--tls-cipher-suites=` +# Bounds idle exec/attach/port-forward streams and restricts the kubelet server to strong AEAD +# ciphers (CIS kubelet items). BOTH are genuine kubelet CLI flags (the k3s hardening guide +# recommends them verbatim AS kubelet-arg — so, unlike `--max-parallel-image-pulls`, they will +# not crash-loop k3s with "unknown flag"). Still validated one node at a time below. +# +# SAFETY — a bad `kube-apiserver-arg` crash-loops the IN-PROCESS apiserver, but the kubelet keeps +# reporting Ready (it reaches the API via the LB to a healthy peer), so a plain node-Ready gate would +# MASK a broken apiserver and let the rollout march on to break all three. This play adds a +# LOCAL-apiserver `/readyz` gate (on-node `k3s kubectl` uses 127.0.0.1:6443) so a bad flag fails the +# host → serial:1 + max_fail_percentage:0 halts before the next node, etcd quorum (2/3) intact. A k3s +# restart is non-disruptive to running pods (containerd re-attach). +# +# Reversibility: node-config — remove the drop-in + restart. Both changes are additive flags. +# +# Run: just ansible-check k3s-cis-hardening # dry-run (--check --diff), no changes +# just ansible-converge k3s-cis-hardening # apply (rolling, one node at a time) +- name: k3s CIS hardening (NodeRestriction + kubelet stream/cipher) + hosts: control_planes + gather_facts: false + become: true + serial: 1 + max_fail_percentage: 0 + order: sorted + handlers: + - name: Restart k3s + ansible.builtin.systemd: + name: k3s + state: restarted + tasks: + - name: Create k3s config drop-in directory + ansible.builtin.file: + path: /etc/rancher/k3s/config.yaml.d + state: directory + owner: root + group: root + mode: "0755" + + - name: Deploy CIS hardening drop-in + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml.d/cis-hardening.yaml + owner: root + group: root + mode: "0644" + content: | + # k3s CIS Hardening Guide flags. `+` appends to k3s's own apiserver/kubelet args (and to the + # kubelet-arg lists set by the eviction/resolver drop-ins) — it does not replace them. + # NodeRestriction: additive admission plugin, caps each kubelet to its own Node + bound pods. + kube-apiserver-arg+: + - "enable-admission-plugins=NodeRestriction" + # Bound idle streaming sessions; restrict kubelet TLS to strong AEAD ciphers. Both are real + # kubelet CLI flags per https://docs.k3s.io/security/hardening-guide. + kubelet-arg+: + - "streaming-connection-idle-timeout=5m" + - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" + notify: Restart k3s + + - name: Flush handlers (restart k3s before verifying) + ansible.builtin.meta: flush_handlers + + - name: Wait for this node to be Ready again + kubernetes.core.k8s_info: + kind: Node + name: "{{ kubernetes_node_name }}" + register: node_info + delegate_to: localhost + become: false + # default([]) tolerates the API endpoint (cp-1) being briefly unreachable while it restarts. + until: > + (node_info.resources | default([]) | length > 0) and (node_info.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') + | map(attribute='status') | list | first | default('')) == 'True' + retries: 30 + delay: 10 + when: not ansible_check_mode + + - name: Verify the LOCAL apiserver came back (catches a bad kube-apiserver-arg) + # On-node k3s.yaml points at 127.0.0.1:6443, so this probes THIS node's apiserver specifically — + # not a healthy peer via the LB. A rejected apiserver flag keeps it down → this fails → halt. + ansible.builtin.command: + cmd: k3s kubectl get --raw /readyz + register: local_readyz + changed_when: false + until: local_readyz.rc == 0 and (local_readyz.stdout | trim) == 'ok' + retries: 18 + delay: 10 + when: not ansible_check_mode + + - name: Verify kubelet picked up the streaming-connection-idle-timeout + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get --raw /api/v1/nodes/{{ kubernetes_node_name }}/proxy/configz | python3 -c " + import json, sys + cfg = json.load(sys.stdin)['kubeletconfig'] + v = cfg.get('streamingConnectionIdleTimeout') + assert v == '5m0s', f'streamingConnectionIdleTimeout not applied: {v!r}' + print('streamingConnectionIdleTimeout: ' + v) + " + executable: /bin/bash + delegate_to: localhost + become: false + register: kubelet_verify + changed_when: false + until: kubelet_verify.rc == 0 + retries: 12 + delay: 5 + when: not ansible_check_mode + + - name: Summary (per node) + ansible.builtin.debug: + msg: >- + {{ kubernetes_node_name }}: NodeRestriction enabled (local apiserver healthy), kubelet + streaming-idle-timeout=5m + strong TLS ciphers applied. + when: not ansible_check_mode