From f9bca35847956b47aa6f0c9e03369566d32bbe31 Mon Sep 17 00:00:00 2001
From: Pete Cornish <outofcoffee@gmail.com>
Date: Tue, 1 Sep 2026 21:17:42 +0100
Subject: [PATCH 1/2] test: wait for the engine counters before asserting on
 them
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A node reads `running` as soon as the engine process is alive, but the
token counters come from the daemon's background sampler, which retries
about once a second until its first reading lands — and the fake engine
needs a moment to answer at all. The metrics assertions ran inside that
window, so CI saw resource bars (taken from the node) with no counters
beside them. Poll for the counters instead of the state.
---
 examples/fleet-docker/run-tests.sh | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/examples/fleet-docker/run-tests.sh b/examples/fleet-docker/run-tests.sh
index 44593f9d..9e1720e8 100755
--- a/examples/fleet-docker/run-tests.sh
+++ b/examples/fleet-docker/run-tests.sh
@@ -171,6 +171,31 @@ wait_for_state() {
   return 1
 }
 
+#######################################
+# Wait until the engine's token counters reach the fleet view. A node reads
+# `running` as soon as the engine process is alive, which is earlier than the
+# counters exist: the engine needs a moment to answer at all, and the daemon
+# reports the reading its background sampler took rather than scraping when
+# asked. The sampler retries about once a second until the first reading
+# lands, so the counters trail the state by a second or two and a state check
+# returns inside that window.
+# Arguments:
+#   Timeout in seconds.
+# Returns:
+#   0 once the counters appear, 1 on timeout.
+#######################################
+wait_for_tokens() {
+  local timeout="$1"
+  local deadline=$((SECONDS + timeout))
+  while (( SECONDS < deadline )); do
+    if [[ "$(fleet metrics)" == *"prompt tokens"* ]]; then
+      return 0
+    fi
+    sleep 1
+  done
+  return 1
+}
+
 #######################################
 # Tear the stack down unless --keep was given. Registered as an EXIT trap so a
 # failure part-way through still cleans up.
@@ -362,6 +387,10 @@ test_metrics() {
   # config, and only a client can supply one.
   fleet start studio >/dev/null
   wait_for_state studio running 30 || true
+  # Running is the process, not a sample; let the counters land before reading
+  # them. A timeout is not fatal here — the assertions below say what was
+  # missing, which is more use than an abort.
+  wait_for_tokens 30 || true
 
   local out
   out="$(fleet metrics)"

From 16960b6c890c2ed355ec287ba15cd80cd4850573 Mon Sep 17 00:00:00 2001
From: Pete Cornish <outofcoffee@gmail.com>
Date: Tue, 1 Sep 2026 21:27:47 +0100
Subject: [PATCH 2/2] test: surface compose failures and retry a node restart
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Every docker compose call sent its output and exit status to /dev/null,
so a failure explained nothing. A bring-up that could not build left only
"Tearing down..." and an exit code behind, and a failed `start laptop`
became ninety seconds of connection-refused with no clue why — one
attempt was made and never retried, so a node that lost the race for its
published host port stayed down for the rest of the run.

Route the lifecycle commands through a helper that stays quiet on success
and reports the captured output on failure, retry the restart while
waiting for it, and dump container state and logs when a wait times out.

Also read `fleet status` into a variable rather than piping it to grep:
under pipefail a matching `grep -q` can exit first and leave the pipeline
reporting the writer's SIGPIPE, which the caller reads as "nothing
unreachable" — the opposite of what was found.
---
 examples/fleet-docker/run-tests.sh | 95 ++++++++++++++++++++++++++++--
 1 file changed, 90 insertions(+), 5 deletions(-)

diff --git a/examples/fleet-docker/run-tests.sh b/examples/fleet-docker/run-tests.sh
index 9e1720e8..cafa26b2 100755
--- a/examples/fleet-docker/run-tests.sh
+++ b/examples/fleet-docker/run-tests.sh
@@ -92,6 +92,55 @@ assert_equals() {
   fi
 }
 
+#######################################
+# Run a docker compose command, showing its output only when it fails. These
+# commands are noisy on success and the test's own output is the point, but a
+# silent failure is worse than noise: a `compose up` that cannot pull leaves
+# nothing behind but "Tearing down..." and an exit code.
+# Globals:
+#   HERE
+# Arguments:
+#   Arguments to pass to docker compose.
+# Returns:
+#   The command's exit status.
+#######################################
+compose() {
+  local out rc=0
+  out="$(docker compose -f "${HERE}/compose.yaml" "$@" 2>&1)" || rc=$?
+  if (( rc != 0 )); then
+    echo "Error: docker compose $* failed (exit ${rc}):" >&2
+    echo "${out}" >&2
+  fi
+  return "${rc}"
+}
+
+#######################################
+# The container state for one service, or "" when docker cannot say.
+# Globals:
+#   HERE
+# Arguments:
+#   Service name.
+# Outputs:
+#   Writes the state to stdout.
+#######################################
+container_state() {
+  docker compose -f "${HERE}/compose.yaml" ps --format '{{.State}}' "$1" 2>/dev/null
+}
+
+#######################################
+# Dump what the containers are doing, for a wait that timed out. `fleet status`
+# only reports that a port refused the connection; whether the container is
+# even up, and what its daemon said on the way down, is the part worth having.
+# Globals:
+#   HERE
+# Outputs:
+#   Writes container state and recent logs to stderr.
+#######################################
+diagnose_fleet() {
+  docker compose -f "${HERE}/compose.yaml" ps >&2 2>&1 || true
+  docker compose -f "${HERE}/compose.yaml" logs --tail 20 >&2 2>&1 || true
+}
+
 #######################################
 # Run `spinloop fleet` against the example's fleet.yaml.
 # Globals:
@@ -142,13 +191,18 @@ node_state() {
 wait_for_fleet() {
   local deadline=$((SECONDS + READY_TIMEOUT_SECS))
   while (( SECONDS < deadline )); do
-    if ! fleet status | grep -q "unreachable"; then
+    # Read the table into a variable rather than piping it: under `pipefail`
+    # a `grep -q` that matches and exits first can leave the pipeline
+    # reporting the writer's SIGPIPE, which reads here as "nothing
+    # unreachable" — the opposite of what was found.
+    if [[ "$(fleet status)" != *unreachable* ]]; then
       return 0
     fi
     sleep 2
   done
   echo "Error: the fleet did not become reachable in ${READY_TIMEOUT_SECS}s" >&2
   fleet status >&2 || true
+  diagnose_fleet
   return 1
 }
 
@@ -171,6 +225,38 @@ wait_for_state() {
   return 1
 }
 
+#######################################
+# Bring a stopped node back and wait for its daemon to answer again. The start
+# is retried rather than issued once: a container that has just gone away can
+# still hold its published host port, and a start that loses that race leaves
+# the node refusing connections for the rest of the run — which is a 90-second
+# wait on a fleet that was never coming back.
+# Globals:
+#   READY_TIMEOUT_SECS
+# Arguments:
+#   Node name.
+# Returns:
+#   0 once the node answers, 1 on timeout.
+#######################################
+restart_node() {
+  local name="$1"
+  local deadline=$((SECONDS + READY_TIMEOUT_SECS))
+  while (( SECONDS < deadline )); do
+    local state
+    state="$(node_state "${name}")"
+    if [[ -n "${state}" && "${state}" != "unreachable" ]]; then
+      return 0
+    fi
+    if [[ "$(container_state "${name}")" != "running" ]]; then
+      compose start "${name}" || true
+    fi
+    sleep 2
+  done
+  echo "Error: ${name} did not come back in ${READY_TIMEOUT_SECS}s" >&2
+  diagnose_fleet
+  return 1
+}
+
 #######################################
 # Wait until the engine's token counters reach the fleet view. A node reads
 # `running` as soon as the engine process is alive, which is earlier than the
@@ -409,7 +495,7 @@ test_metrics() {
 #######################################
 test_unreachable_node() {
   echo "A stopped node degrades, the rest keep reporting"
-  docker compose -f "${HERE}/compose.yaml" stop laptop >/dev/null 2>&1
+  compose stop laptop
 
   local out
   out="$(fleet status)"
@@ -420,8 +506,7 @@ test_unreachable_node() {
   fleet status >/dev/null
   assert_equals "status still succeeds" "$?" "0"
 
-  docker compose -f "${HERE}/compose.yaml" start laptop >/dev/null 2>&1
-  wait_for_fleet || true
+  restart_node laptop || true
 }
 
 #######################################
@@ -496,7 +581,7 @@ main() {
   (cd "${REPO_ROOT}" && go build -o "${SPINLOOP_BIN}" ./cmd/spinloop)
 
   echo "Bringing the fleet up..."
-  docker compose -f "${HERE}/compose.yaml" up -d --build >/dev/null 2>&1
+  compose up -d --build
   wait_for_fleet
 
   echo