diff --git a/docs/HARDENING.md b/docs/HARDENING.md index d7e8970..08a8f57 100644 --- a/docs/HARDENING.md +++ b/docs/HARDENING.md @@ -324,6 +324,10 @@ sudo gpasswd -d "$USER" docker # if you don't need Docker in this distro ### Worked example: dedicated Linux server (maximum containment) +> **Working files:** [`examples/dedicated-server/`](../examples/dedicated-server/) carries the +> nftables fence, egress alarm, auditd rules, divergence tripwire and systemd drop-in from a real +> run of this section on bare-metal Ubuntu 26.04, with the reasoning behind each choice. + > **Guided:** run **`/orchestrator:provision`** on the new box to be walked through this section > interactively — an interview, phase-by-phase checkpoints with verification, and resumable progress in > `.claude/state/provision-progress.json`. This section stays the source of truth; the command executes it. @@ -465,13 +469,32 @@ loop needs — the Anthropic API, GitHub, your notifier — and **log the drops* # /etc/nftables.d/recode-agent.nft — adjust the resolver/set mechanics to taste; # domain-based filtering needs a resolving frontend (e.g. a local proxy) or periodic set refresh. table inet recode_agent { + set allowed_v4 { type ipv4_addr; } + set allowed_v6 { type ipv6_addr; } + chain output { - type filter hook output priority 0; - meta skuid != "recode-agent" accept + type filter hook output priority 0; policy accept; + # JUMP on a positive UID match — do NOT write `meta skuid != accept`. + # Kernel-generated packets (ICMPv6 MLD/ND, DHCP renewal) have NO owning + # socket, so `skuid != N` never matches them and they fall through to the + # drop below — silently filtering the host's own network stack. The symptom + # arrives days later as a lease that won't renew or IPv6 quietly degrading, + # with nothing to connect it back to this file. + meta skuid 1001 jump agent_out # numeric uid; `id -u ` + } + + chain agent_out { + oifname "lo" accept ct state established,related accept + # Link-local multicast is local discovery noise: deny WITHOUT the log + # prefix, or mDNS/MLD will page you forever and the alarm stops being read. + ip daddr 224.0.0.0/4 drop + ip6 daddr ff00::/8 drop udp dport 53 accept # DNS (or pin to a local resolver) - tcp dport 443 ip daddr @allowed_v4 accept # populate from api.anthropic.com, github.com, ntfy.sh - counter log prefix "recode-agent-egress-drop " drop + tcp dport 53 accept + ip daddr @allowed_v4 tcp dport 443 accept # refreshed from DNS on a timer + ip6 daddr @allowed_v6 tcp dport 443 accept + counter log prefix "recode-agent-egress-drop " level warn drop } } ``` @@ -496,6 +519,15 @@ Three things the sketch above glosses over, learned the hard way: would rather not run a resolver-refresh loop, the alternative is coarse — permit DNS and TCP 443 to *any* destination — but understand that this stops odd ports and non-HTTPS exfil only, and gives you no meaningful exfiltration alarm, which is most of the value. +- **Resolve the hosts the agent actually contacts, not the ones you assume.** `statsig.anthropic.com` + (in an earlier version of this list) has **no A record**; Claude Code's feature-flag traffic goes to + `api.statsig.com` / `statsigapi.net` / `events.statsigapi.net` / `featureassets.org`, on Google + Cloud. Leaving them out is defensible — the loop works without telemetry — but it pages you forever, + and an alarm that cries wolf gets ignored. Decide deliberately; don't discover it as noise. +- **Alerts must be readable and de-duplicated.** A raw kernel log line (`IN= OUT= SRC= DST= LEN=…`) + is unreadable on a phone. Parse `DST`/`DPT`/`PROTO`, reverse-resolve the address, and suppress + repeats per destination (15 minutes works) — otherwise one blocked endpoint produces dozens of + identical pushes and the channel becomes noise. **6. Detection.** Single-purpose boxes make auditing cheap: - `auditd` watches on `.claude/scripts/`, `.claude/self/`, `.env`, and the unit files — any write diff --git a/examples/dedicated-server/README.md b/examples/dedicated-server/README.md new file mode 100644 index 0000000..d28e229 --- /dev/null +++ b/examples/dedicated-server/README.md @@ -0,0 +1,109 @@ +# Dedicated-server provisioning artifacts + +Working files from a real `/orchestrator:provision` run on bare-metal **Ubuntu 26.04**, kept so the +next box doesn't rebuild them from the sketches in [`docs/HARDENING.md`](../../docs/HARDENING.md). +They implement the worked example's steps 5–6 (kernel egress fence, detection) plus the systemd +drop-in from step 4. + +Substitute `recode-agent` / uid `1001` / `recode-notifications` for your own values before installing. + +## What's here + +| File | Installs to | Purpose | +|---|---|---| +| `nftables/recode-agent.nft` | `/etc/nftables.d/` | UID-matched egress fence for the agent user | +| `systemd/recode-agent-nft.service` | `/etc/systemd/system/` | Loads the table at boot; deletes it on stop | +| `bin/egress-alarm.sh` | `/usr/local/sbin/` | Follows the kernel log, pushes blocked egress to ntfy | +| `systemd/egress-alarm.service` | `/etc/systemd/system/` | Supervises the follower (runs as root — see below) | +| `bin/divergence-check.sh` | `/usr/local/sbin/` | Alerts when the agent's checkout diverges from `origin/main` | +| `systemd/divergence-check.{service,timer}` | `/etc/systemd/system/` | Hourly tripwire | +| `audit/recode-agent.rules` | `/etc/audit/rules.d/` | auditd watches on scripts, `.env`, settings, unit files | +| `systemd/pr-loop-hardening.conf` | `~/.config/systemd/user/pr-loop-.service.d/` | User-unit-safe hardening drop-in | + +Install: + +```bash +sudo mkdir -p /etc/nftables.d +sudo install -m 644 nftables/recode-agent.nft /etc/nftables.d/ +sudo install -m 755 bin/*.sh /usr/local/sbin/ +sudo install -m 644 systemd/*.service systemd/*.timer /etc/systemd/system/ +sudo install -m 640 audit/recode-agent.rules /etc/audit/rules.d/ +sudo nft -c -f /etc/nftables.d/recode-agent.nft # syntax check BEFORE enabling +sudo systemctl daemon-reload +sudo systemctl enable --now recode-agent-nft.service egress-alarm.service divergence-check.timer +sudo augenrules --load +``` + +The user drop-in goes in as the agent user, not root: + +```bash +mkdir -p ~/.config/systemd/user/pr-loop-.service.d +cp pr-loop-hardening.conf ~/.config/systemd/user/pr-loop-.service.d/hardening.conf +systemctl --user daemon-reload && systemctl --user restart pr-loop-.service +``` + +## Decisions behind these files + +Each of these cost real debugging time; the rationale matters more than the syntax. + +**The fence restricts protocol and port, not destination.** Per-destination IP-set allowlisting was +built, tested, and abandoned: GitHub rotates within its fleet (the set held `140.82.121.3/.6/.10` +while traffic went to `.4/.5`), Google load balancers served different addresses than our resolver +for `downloads.claude.ai` and Datadog, and a DNS-refresh timer can't close that race. It dropped +real work. The decisive argument is in HARDENING.md already: **GitHub is a sanctioned exfiltration +channel** — once the agent can push branches it can move data through a permitted destination, so an +allowlist that must include GitHub was never going to close that path. What remains is worth having: +no outbound SSH, no arbitrary ports, no non-HTTPS protocols, and an alarm that only fires on +genuinely anomalous traffic. For true per-domain control, build an L7 proxy with a domain allowlist +and block direct 443 — that is the only version that actually works. + +**Jump on a positive UID match; never `skuid != accept`.** Kernel-generated packets (ICMPv6 +MLD/ND, DHCP renewal) have no owning socket, so a `!=` rule never matches them and they fall through +to the drop — silently filtering the host's own network stack. Observed live as dropped MLD listener +reports. The symptom arrives days later as a lease that won't renew, with nothing to connect it back. + +**Link-local multicast is dropped without the log prefix.** mDNS/MLD is local discovery noise; +alarming on it trains you to ignore the channel. + +**The alarm runs as root** — the nft chain matches the agent's UID, so root's `curl` to the notifier +is not subject to the allowlist it reports on. Same reason the divergence check runs as root: it must +sit outside the agent's trust zone to be meaningful. + +**`egress-alarm.sh` deliberately does not use `set -e` / `pipefail`.** Every field is a best-effort +parse or a reverse lookup that legitimately fails — an address with no PTR, a log line with no `DPT`. +Under `set -e` the first such failure killed the follower and events were lost *silently*, which is +worse than no alarm at all, because silence reads as safety. Only destinations with reverse DNS ever +alerted, and nothing indicated the rest were being swallowed. + +**The alarm cooldown keys on destination *and* port.** Keyed on IP alone, a different port to the +same host is suppressed — `example.com:80` vanished because `example.com:443` had alerted minutes +earlier. + +**`ufw` is left alone.** If ufw is active, do not enable `nftables.service`: it runs +`/etc/nftables.conf`, which conventionally begins with `flush ruleset` and would wipe ufw's rules at +boot. `recode-agent-nft.service` only ever adds its own table, and removes it on stop. + +**The systemd drop-in is the user-unit-safe subset.** `ProtectKernelModules`/`ProtectKernelTunables`/ +`ProtectControlGroups` imply `CapabilityBoundingSet` changes needing `CAP_SETPCAP`, which an +unprivileged `systemd --user` manager lacks — the unit then dies with `218/CAPABILITIES` and +restart-loops while `systemctl --user is-active` still reports `active`. `ProtectSystem=strict`, +`ReadWritePaths=` and `PrivateTmp=` need mount namespaces and may also fail under the Ubuntu ≥24.04 +userns restriction; test them one at a time. For the full directive set, promote the loop to a +**system** unit with `User=` — at the cost of `arm-loop.sh` recreating user units on every +re-arm. + +## Verifying, not assuming + +Three separate layers on this run looked correct in their configuration and did nothing in practice: +inert `Write(...)` deny rules, `is-active` on a crash-looping unit, and an alarm dying on a missing +PTR record. Test each one: + +```bash +sudo -u curl -sI -m 10 https://api.github.com | head -1 # expect HTTP/2 200 +sudo -u curl -sI -m 8 http://example.com | head -1 # expect nothing + one alert +sudo -u timeout 5 ssh -o BatchMode=yes 1.1.1.1 # expect nothing + one alert +journalctl --user -u pr-loop-.service -n 20 --no-pager # expect loop-daemon: starting +``` + +Two of those alerts should come from addresses with no reverse DNS and one from an address with it — +that exercises both paths through the lookup, which is where the silent failure lived. diff --git a/examples/dedicated-server/audit/recode-agent.rules b/examples/dedicated-server/audit/recode-agent.rules new file mode 100644 index 0000000..290388d --- /dev/null +++ b/examples/dedicated-server/audit/recode-agent.rules @@ -0,0 +1,9 @@ +# auditd watches on the loop's daemon-executed paths and credentials. +# Any write here outside an expected driver window is worth investigating. +-w /home/recode-agent/reCode/.claude/scripts/ -p wa -k recode_scripts +-w /home/recode-agent/reCode/self/ -p wa -k recode_self +-w /home/recode-agent/reCode/.env -p wa -k recode_env +-w /home/recode-agent/reCode/.claude/settings.local.json -p wa -k recode_settings +-w /home/recode-agent/.config/systemd/user/ -p wa -k recode_units +-w /etc/claude-code/managed-settings.json -p wa -k recode_managed +-w /etc/nftables.d/recode-agent.nft -p wa -k recode_nft diff --git a/examples/dedicated-server/bin/divergence-check.sh b/examples/dedicated-server/bin/divergence-check.sh new file mode 100755 index 0000000..7ab46d9 --- /dev/null +++ b/examples/dedicated-server/bin/divergence-check.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Root-owned tripwire: alert when the agent's daemon-executed paths diverge +# from origin/main. Enforced from OUTSIDE the agent's trust zone. +set -euo pipefail +R=/home/recode-agent/reCode +TOPIC="${NTFY_TOPIC:-recode-notifications}" +PATHS=".claude/scripts self" + +as_agent() { sudo -u recode-agent git -C "$R" "$@"; } + +as_agent fetch -q origin main 2>/dev/null || true +dirty=$(as_agent status --porcelain -- $PATHS 2>/dev/null || true) +drift=$(as_agent diff --stat origin/main -- $PATHS 2>/dev/null || true) + +if [ -n "$dirty" ] || [ -n "$drift" ]; then + body=$(printf 'uncommitted:\n%s\n\nvs origin/main:\n%s\n' "$dirty" "$drift") + curl -fsS -m 10 \ + -H "Title: BusyBee: agent checkout diverges from origin/main" \ + -H "Priority: high" \ + -d "$body" "https://ntfy.sh/$TOPIC" >/dev/null || true + echo "$body" +fi diff --git a/examples/dedicated-server/bin/egress-alarm.sh b/examples/dedicated-server/bin/egress-alarm.sh new file mode 100755 index 0000000..44d23c9 --- /dev/null +++ b/examples/dedicated-server/bin/egress-alarm.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Follow the kernel log and push blocked-egress events to ntfy. +# Blocked egress from this box is an intrusion signal, not noise -- so keep it +# readable and de-duplicated, or it becomes noise and gets ignored. +# Runs as root: the UID-matched nft rule does not apply to this curl. +# +# NOTE: deliberately NOT `set -e`/`pipefail`. Every field here comes from a +# best-effort parse or a reverse lookup that legitimately fails (an address +# with no PTR, a log line with no DPT). Under `set -e` the first such failure +# kills the follower and events are lost SILENTLY -- which is worse than no +# alarm at all, because silence reads as safety. +set -u +TOPIC="${NTFY_TOPIC:-recode-notifications}" +COOLDOWN="${COOLDOWN:-900}" # seconds before re-alerting on the same dst:port + +declare -A seen +journalctl -k -f -n0 -o cat | while IFS= read -r line; do + case "$line" in *recode-egress-drop*) ;; *) continue ;; esac + + dst=$(printf '%s' "$line" | grep -oE 'DST=[0-9a-fA-F.:]+' | head -1 | cut -d= -f2) + dpt=$(printf '%s' "$line" | grep -oE 'DPT=[0-9]+' | head -1 | cut -d= -f2) + proto=$(printf '%s' "$line" | grep -oE 'PROTO=[A-Z0-9]+' | head -1 | cut -d= -f2) + [ -n "${dst:-}" ] || continue + + # Key on destination AND port: the same host on a different port is a + # different event, and suppressing it hides real signal. + key="${dst}:${dpt:-none}/${proto:-?}" + now=$(date +%s) + prev=${seen[$key]:-0} + if [ $((now - prev)) -lt "$COOLDOWN" ]; then continue; fi + seen[$key]=$now + + host=$(getent hosts "$dst" 2>/dev/null | awk '{print $2}' | head -1) + [ -n "${host:-}" ] || host="$dst" + + curl -fsS -m 10 \ + -H "Title: BusyBee: agent egress BLOCKED" \ + -H "Priority: high" \ + -H "Tags: rotating_light" \ + -d "${proto:-?} -> ${host}:${dpt:-?} (raw dst ${dst})" \ + "https://ntfy.sh/$TOPIC" >/dev/null 2>&1 +done diff --git a/examples/dedicated-server/nftables/recode-agent.nft b/examples/dedicated-server/nftables/recode-agent.nft new file mode 100644 index 0000000..36f3d0f --- /dev/null +++ b/examples/dedicated-server/nftables/recode-agent.nft @@ -0,0 +1,43 @@ +#!/usr/sbin/nft -f +# Protocol/port egress fence for the loop's agent user (HARDENING.md step 5). +# Independent of ufw: adds only its own table, never flushes the ruleset. +# +# JUMP on a positive skuid match -- `skuid != N accept` never matches +# kernel-generated packets (ICMPv6 MLD/ND, DHCP renewal), which would then fall +# through to the drop and silently filter the host's own network stack. +# +# Scope, stated honestly: this restricts PROTOCOL and PORT, not destination. +# Per-destination filtering by IP set was tried and abandoned -- GitHub, +# Datadog, downloads.claude.ai and the remote-control endpoint all rotate +# addresses across fleets no DNS snapshot can track, so it dropped real work +# between refreshes. And since GitHub must be reachable for the loop to +# function, destination filtering could never have closed the exfiltration +# path anyway (HARDENING.md lists GitHub as a sanctioned exfil channel among +# the irreducible risks). What remains is worth having: no outbound SSH, no +# arbitrary ports, no non-HTTPS protocols -- and an alarm that only fires on +# genuinely anomalous traffic, so it stays worth reading. +# +# For true per-domain control, the correct build is an L7 proxy with a domain +# allowlist, with direct 443 blocked and the agent forced through it. +table inet recode_agent { + chain output { + type filter hook output priority 0; policy accept; + meta skuid 1001 jump agent_out + } + + chain agent_out { + oifname "lo" accept + ct state established,related accept + + # Link-local multicast: denied, never alarmed on (discovery noise). + ip daddr 224.0.0.0/4 drop + ip6 daddr ff00::/8 drop + + udp dport 53 accept + tcp dport 53 accept + tcp dport 443 accept + + # Anything else from this UID is anomalous: block it and page. + counter log prefix "recode-egress-drop " level warn drop + } +} diff --git a/examples/dedicated-server/systemd/divergence-check.service b/examples/dedicated-server/systemd/divergence-check.service new file mode 100644 index 0000000..cfedd88 --- /dev/null +++ b/examples/dedicated-server/systemd/divergence-check.service @@ -0,0 +1,6 @@ +[Unit] +Description=Check the agent checkout against origin/main + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/divergence-check.sh diff --git a/examples/dedicated-server/systemd/divergence-check.timer b/examples/dedicated-server/systemd/divergence-check.timer new file mode 100644 index 0000000..1b8a7cd --- /dev/null +++ b/examples/dedicated-server/systemd/divergence-check.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Hourly divergence check of the agent checkout + +[Timer] +OnBootSec=10min +OnUnitActiveSec=1h +AccuracySec=5min + +[Install] +WantedBy=timers.target diff --git a/examples/dedicated-server/systemd/egress-alarm.service b/examples/dedicated-server/systemd/egress-alarm.service new file mode 100644 index 0000000..fd5c08f --- /dev/null +++ b/examples/dedicated-server/systemd/egress-alarm.service @@ -0,0 +1,12 @@ +[Unit] +Description=Alert on blocked egress from the loop agent +After=recode-agent-nft.service network-online.target +Wants=network-online.target + +[Service] +ExecStart=/usr/local/sbin/egress-alarm.sh +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/examples/dedicated-server/systemd/pr-loop-hardening.conf b/examples/dedicated-server/systemd/pr-loop-hardening.conf new file mode 100644 index 0000000..e9c30c4 --- /dev/null +++ b/examples/dedicated-server/systemd/pr-loop-hardening.conf @@ -0,0 +1,7 @@ +[Service] +NoNewPrivileges=yes +RestrictSUIDSGID=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +UMask=0077 +MemoryMax=8G +CPUQuota=200% diff --git a/examples/dedicated-server/systemd/recode-agent-nft.service b/examples/dedicated-server/systemd/recode-agent-nft.service new file mode 100644 index 0000000..46d125c --- /dev/null +++ b/examples/dedicated-server/systemd/recode-agent-nft.service @@ -0,0 +1,13 @@ +[Unit] +Description=Load recode-agent egress fence (nftables) +After=network-pre.target ufw.service +Wants=network-pre.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/nft -f /etc/nftables.d/recode-agent.nft +ExecStop=/usr/sbin/nft delete table inet recode_agent + +[Install] +WantedBy=multi-user.target