fix(loop): systemd actually supervises claude-rc (no more fake-forever oneshot) (#124)

.claude/scripts/arm-loop.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# @orchestrator-managed arm-loop v5
+# @orchestrator-managed arm-loop v6
 # arm-loop.sh — installs the cron-less PR-loop as systemd (user) units
 # (issue #102). Templated + re-stamped by `/orchestrator:setup`/`sync`; do
 # not hand-edit the copy scaffold.sh wrote into this repo if you want future
@@ -223,10 +223,15 @@ systemctl --user daemon-reload
 # may have a driver in flight; a re-arm only rewrites its unit file, and the
 # owner restarts it explicitly when they want the new unit picked up.
 systemctl --user enable --now "pr-loop-$repo_slug.service"
-# claude-rc: enable + restart on purpose — Type=oneshot + RemainAfterExit
-# stays "active" forever, so `enable --now` would never re-run ExecStart and
-# a re-arm would silently keep serving the OLD unit. Restart is safe here
-# (independent of the loop daemon) and relaunches the tmux with the fresh unit.
+# claude-rc: enable + restart on purpose — even with Type=simple +
+# Restart=on-failure (issue #124), `systemctl --user enable --now` on an
+# ALREADY-enabled, already-running unit is a no-op: it does not re-run
+# ExecStart. So a re-arm's freshly-written unit file (new PATH, capacity,
+# permission-mode, spawn mode, etc.) would silently keep being ignored by the
+# still-running OLD supervisor process until something restarts it. `restart`
+# is what actually loads the new unit; it is safe here (independent of the
+# loop daemon) — the inline supervisor's ExecStop/kill-session step tears
+# down the old tmux session cleanly before the fresh ExecStart relaunches it.
 systemctl --user enable "claude-rc-$repo_slug.service"
 systemctl --user restart "claude-rc-$repo_slug.service"
 
@@ -236,11 +241,15 @@ cat <<EOF
 
 armed:
   pr-loop-$repo_slug.service       (the cron-less loop daemon; adaptive tick+sleep)
-  claude-rc-$repo_slug.service     (claude remote-control, in tmux session rc-$repo_slug)
+  claude-rc-$repo_slug.service     (claude remote-control, supervised tmux session
+                                     rc-$repo_slug; auto-restarts within RestartSec=10s
+                                     if the planner process dies)
 
 inspect:
   systemctl --user status pr-loop-$repo_slug.service
+  systemctl --user status claude-rc-$repo_slug.service
   journalctl --user -u pr-loop-$repo_slug.service -f
+  journalctl --user -u claude-rc-$repo_slug.service -f
   tail -f "$repo_root/.claude/state/loop-runs.log"
   tmux attach -t rc-$repo_slug
 

.claude/skills/setup/templates/arm-loop.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# @orchestrator-managed arm-loop v5
+# @orchestrator-managed arm-loop v6
 # arm-loop.sh — installs the cron-less PR-loop as systemd (user) units
 # (issue #102). Templated + re-stamped by `/orchestrator:setup`/`sync`; do
 # not hand-edit the copy scaffold.sh wrote into this repo if you want future
@@ -149,10 +149,15 @@ systemctl --user daemon-reload
 # may have a driver in flight; a re-arm only rewrites its unit file, and the
 # owner restarts it explicitly when they want the new unit picked up.
 systemctl --user enable --now "pr-loop-$repo_slug.service"
-# claude-rc: enable + restart on purpose — Type=oneshot + RemainAfterExit
-# stays "active" forever, so `enable --now` would never re-run ExecStart and
-# a re-arm would silently keep serving the OLD unit. Restart is safe here
-# (independent of the loop daemon) and relaunches the tmux with the fresh unit.
+# claude-rc: enable + restart on purpose — even with Type=simple +
+# Restart=on-failure (issue #124), `systemctl --user enable --now` on an
+# ALREADY-enabled, already-running unit is a no-op: it does not re-run
+# ExecStart. So a re-arm's freshly-written unit file (new PATH, capacity,
+# permission-mode, spawn mode, etc.) would silently keep being ignored by the
+# still-running OLD supervisor process until something restarts it. `restart`
+# is what actually loads the new unit; it is safe here (independent of the
+# loop daemon) — the inline supervisor's ExecStop/kill-session step tears
+# down the old tmux session cleanly before the fresh ExecStart relaunches it.
 systemctl --user enable "claude-rc-$repo_slug.service"
 systemctl --user restart "claude-rc-$repo_slug.service"
 
@@ -162,11 +167,15 @@ cat <<EOF
 
 armed:
   pr-loop-$repo_slug.service       (the cron-less loop daemon; adaptive tick+sleep)
-  claude-rc-$repo_slug.service     (claude remote-control, in tmux session rc-$repo_slug)
+  claude-rc-$repo_slug.service     (claude remote-control, supervised tmux session
+                                     rc-$repo_slug; auto-restarts within RestartSec=10s
+                                     if the planner process dies)
 
 inspect:
   systemctl --user status pr-loop-$repo_slug.service
+  systemctl --user status claude-rc-$repo_slug.service
   journalctl --user -u pr-loop-$repo_slug.service -f
+  journalctl --user -u claude-rc-$repo_slug.service -f
   tail -f "$repo_root/.claude/state/loop-runs.log"
   tmux attach -t rc-$repo_slug
 

.claude/skills/setup/templates/claude-rc.service

@@ -1,42 +1,46 @@
-# @orchestrator-managed claude-rc-service v5
-# systemd (user) unit TEMPLATE (issue #102). Installed into
-# ~/.config/systemd/user/claude-rc-<repo-slug>.service by
+# @orchestrator-managed claude-rc-service v6
+# systemd (user) unit TEMPLATE (issue #102; supervision fix issue #124).
+# Installed into ~/.config/systemd/user/claude-rc-<repo-slug>.service by
 # `.claude/scripts/arm-loop.sh`, which substitutes the __PLACEHOLDER__ tokens
-# below and runs `systemctl --user enable --now`.
+# below and runs `systemctl --user enable` + `restart`.
 #
-# `claude remote-control` has no documented headless mode, so this unit runs
+# `claude remote-control` has no documented headless mode and renders a
+# QR/status view that very likely needs a real TTY, so this unit still runs
 # it inside a DETACHED tmux session (`tmux new -d -s rc-<repo-slug>`): tmux
-# gives systemd a stable child to track AND gives a human a local attach point
-# (`tmux attach -t rc-<repo-slug>`) to see the QR/status or restart it by hand.
+# gives a human a local attach point (`tmux attach -t rc-<repo-slug>`) to see
+# the QR/status or restart it by hand, without gambling on TTY-independence.
 #
-# Type=oneshot + RemainAfterExit=yes (not Type=forking): `tmux new -d` talks
-# to the tmux SERVER over a socket and exits immediately once the detached
-# session exists — if a tmux server is already running, ExecStart's own PID
-# has no parent/child relationship to the long-lived process at all, so
-# systemd cannot reliably track it as a "forked" child. This is the standard
-# pattern for supervising a tmux/screen-managed daemon from systemd. Caveat:
-# systemd only observes ExecStart's (successful) exit, not the health of the
-# `claude remote-control` process running inside the tmux session — if THAT
-# process itself crashes, tmux keeps the (now-empty) session and systemd sees
-# nothing wrong. Inspect with `tmux attach -t rc-<repo-slug>`; restart with
-# `systemctl --user restart claude-rc-<repo-slug>.service`.
+# Type=simple + Restart=on-failure (issue #124, replacing the old
+# Type=oneshot + RemainAfterExit=yes): ExecStart is now an INLINE supervisor
+# that (1) starts the detached tmux session running `claude remote-control`,
+# then (2) BLOCKS, polling every 10s whether the tmux session still exists
+# and its pane is still alive (`tmux has-session` / `tmux list-panes ...
+# #{pane_dead}`), and (3) exits non-zero the instant the session or its pane
+# is gone (planner crash, `pkill -f remote-control`, a stray `tmux
+# kill-session`, etc). Because ExecStart itself is now the long-lived
+# process systemd actually tracks, a crash of the INNER `claude
+# remote-control` process is no longer invisible to systemd: the supervisor
+# loop notices within ~10s, ExecStart exits non-zero, the unit transitions to
+# failed, and Restart=on-failure relaunches it automatically within
+# RestartSec — no more permanently-"active" unit hiding a dead planner.
 #
 # DO NOT hand-edit the INSTALLED copy under ~/.config/systemd/user/ — it will
 # be silently overwritten the next time arm-loop.sh runs. Edit THIS checked-in
 # template instead (then re-run arm-loop.sh), or fork it under a different
 # name if you want a permanently custom copy. Re-run `/orchestrator:sync` to
 # pick up plugin updates to this template before re-arming.
 [Unit]
-Description=claude remote-control server for __REPO_SLUG__ (detached tmux)
+Description=claude remote-control server for __REPO_SLUG__ (supervised tmux)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
-Type=oneshot
-RemainAfterExit=yes
+Type=simple
+Restart=on-failure
+RestartSec=10s
 WorkingDirectory=__WORKDIR__
 ExecStartPre=-/usr/bin/tmux kill-session -t rc-__REPO_SLUG__
-ExecStart=/usr/bin/tmux new -d -s rc-__REPO_SLUG__ -c __WORKDIR__ /bin/bash -c 'export PATH=__CLAUDE_DIR__:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; exec __CLAUDE_BIN__ remote-control --spawn __SPAWN_MODE__ --name __RC_NAME__ --remote-control-session-name-prefix __REPO_SLUG__ --capacity __CAPACITY__ --permission-mode __PERMISSION_MODE__'
+ExecStart=/bin/bash -c '/usr/bin/tmux new -d -s rc-__REPO_SLUG__ -c __WORKDIR__ /bin/bash -c "export PATH=__CLAUDE_DIR__:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; exec __CLAUDE_BIN__ remote-control --spawn __SPAWN_MODE__ --name __RC_NAME__ --remote-control-session-name-prefix __REPO_SLUG__ --capacity __CAPACITY__ --permission-mode __PERMISSION_MODE__"; while /usr/bin/tmux has-session -t rc-__REPO_SLUG__ 2>/dev/null; do dead="$(/usr/bin/tmux list-panes -t rc-__REPO_SLUG__ -F "#{pane_dead}" 2>/dev/null | head -n1)"; if [ "$dead" = "1" ]; then exit 1; fi; sleep 10; done; exit 1'
 ExecStop=-/usr/bin/tmux kill-session -t rc-__REPO_SLUG__
 
 [Install]

.claude/systemd/claude-rc.service

@@ -1,42 +1,46 @@
-# @orchestrator-managed claude-rc-service v5
-# systemd (user) unit TEMPLATE (issue #102). Installed into
-# ~/.config/systemd/user/claude-rc-<repo-slug>.service by
+# @orchestrator-managed claude-rc-service v6
+# systemd (user) unit TEMPLATE (issue #102; supervision fix issue #124).
+# Installed into ~/.config/systemd/user/claude-rc-<repo-slug>.service by
 # `.claude/scripts/arm-loop.sh`, which substitutes the __PLACEHOLDER__ tokens
-# below and runs `systemctl --user enable --now`.
+# below and runs `systemctl --user enable` + `restart`.
 #
-# `claude remote-control` has no documented headless mode, so this unit runs
+# `claude remote-control` has no documented headless mode and renders a
+# QR/status view that very likely needs a real TTY, so this unit still runs
 # it inside a DETACHED tmux session (`tmux new -d -s rc-<repo-slug>`): tmux
-# gives systemd a stable child to track AND gives a human a local attach point
-# (`tmux attach -t rc-<repo-slug>`) to see the QR/status or restart it by hand.
+# gives a human a local attach point (`tmux attach -t rc-<repo-slug>`) to see
+# the QR/status or restart it by hand, without gambling on TTY-independence.
 #
-# Type=oneshot + RemainAfterExit=yes (not Type=forking): `tmux new -d` talks
-# to the tmux SERVER over a socket and exits immediately once the detached
-# session exists — if a tmux server is already running, ExecStart's own PID
-# has no parent/child relationship to the long-lived process at all, so
-# systemd cannot reliably track it as a "forked" child. This is the standard
-# pattern for supervising a tmux/screen-managed daemon from systemd. Caveat:
-# systemd only observes ExecStart's (successful) exit, not the health of the
-# `claude remote-control` process running inside the tmux session — if THAT
-# process itself crashes, tmux keeps the (now-empty) session and systemd sees
-# nothing wrong. Inspect with `tmux attach -t rc-<repo-slug>`; restart with
-# `systemctl --user restart claude-rc-<repo-slug>.service`.
+# Type=simple + Restart=on-failure (issue #124, replacing the old
+# Type=oneshot + RemainAfterExit=yes): ExecStart is now an INLINE supervisor
+# that (1) starts the detached tmux session running `claude remote-control`,
+# then (2) BLOCKS, polling every 10s whether the tmux session still exists
+# and its pane is still alive (`tmux has-session` / `tmux list-panes ...
+# #{pane_dead}`), and (3) exits non-zero the instant the session or its pane
+# is gone (planner crash, `pkill -f remote-control`, a stray `tmux
+# kill-session`, etc). Because ExecStart itself is now the long-lived
+# process systemd actually tracks, a crash of the INNER `claude
+# remote-control` process is no longer invisible to systemd: the supervisor
+# loop notices within ~10s, ExecStart exits non-zero, the unit transitions to
+# failed, and Restart=on-failure relaunches it automatically within
+# RestartSec — no more permanently-"active" unit hiding a dead planner.
 #
 # DO NOT hand-edit the INSTALLED copy under ~/.config/systemd/user/ — it will
 # be silently overwritten the next time arm-loop.sh runs. Edit THIS checked-in
 # template instead (then re-run arm-loop.sh), or fork it under a different
 # name if you want a permanently custom copy. Re-run `/orchestrator:sync` to
 # pick up plugin updates to this template before re-arming.
 [Unit]
-Description=claude remote-control server for __REPO_SLUG__ (detached tmux)
+Description=claude remote-control server for __REPO_SLUG__ (supervised tmux)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
-Type=oneshot
-RemainAfterExit=yes
+Type=simple
+Restart=on-failure
+RestartSec=10s
 WorkingDirectory=__WORKDIR__
 ExecStartPre=-/usr/bin/tmux kill-session -t rc-__REPO_SLUG__
-ExecStart=/usr/bin/tmux new -d -s rc-__REPO_SLUG__ -c __WORKDIR__ /bin/bash -c 'export PATH=__CLAUDE_DIR__:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; exec __CLAUDE_BIN__ remote-control --spawn __SPAWN_MODE__ --name __RC_NAME__ --remote-control-session-name-prefix __REPO_SLUG__ --capacity __CAPACITY__ --permission-mode __PERMISSION_MODE__'
+ExecStart=/bin/bash -c '/usr/bin/tmux new -d -s rc-__REPO_SLUG__ -c __WORKDIR__ /bin/bash -c "export PATH=__CLAUDE_DIR__:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; exec __CLAUDE_BIN__ remote-control --spawn __SPAWN_MODE__ --name __RC_NAME__ --remote-control-session-name-prefix __REPO_SLUG__ --capacity __CAPACITY__ --permission-mode __PERMISSION_MODE__"; while /usr/bin/tmux has-session -t rc-__REPO_SLUG__ 2>/dev/null; do dead="$(/usr/bin/tmux list-panes -t rc-__REPO_SLUG__ -F "#{pane_dead}" 2>/dev/null | head -n1)"; if [ "$dead" = "1" ]; then exit 1; fi; sleep 10; done; exit 1'
 ExecStop=-/usr/bin/tmux kill-session -t rc-__REPO_SLUG__
 
 [Install]