fix(tests): the RSS gate could report unfinished warmup as a leak (#765)

tests/test_http_rss_growth.sh

@@ -69,6 +69,38 @@ THRESHOLD_KB=64
 
 rss_of() { awk '/^VmRSS/{print $2}' "/proc/$1/status" 2>/dev/null; }
 
+# Drive a globbed batch and return the number of requests that answered 200.
+# The count is load-bearing, not diagnostics: a batch that ends early (connection
+# refused under the per-IP cap, dropped socket) silently shortens the interval a
+# checkpoint bounds, and every number downstream becomes meaningless.
+batch_200s() { # $1 = url with glob
+    curl -s -o /dev/null -w '%{http_code}\n' "$1" 2>/dev/null | grep -c '^200$'
+}
+
+# TWO consecutive measured batches, and the verdict is the SECOND one (#765).
+#
+# A single warmup-then-measure is only valid if the warmup actually reached
+# steady state, and it cannot tell whether it did. When it doesn't, B-A
+# re-absorbs the ~1.4 MB of one-time arena growth and reports it as a leak —
+# forcing WARMUP=5 on a leak-free binary reproduces CI's exact figures to within
+# ~1% on both checks (RSS1 2880 kB/1474 B-per-req vs 2852/1460; RSS2 1532/784 vs
+# 1516/776). Whether the warmup was *truncated* or merely *too short for this
+# machine* produces the identical signature, so counting requests is not enough:
+# WARMUP=5 completes 5/5 and is still wrong.
+#
+# The discriminator is physical rather than statistical: **one-time warmup
+# decays, a per-request leak does not.** A leak of N bytes/req leaks the same N
+# in every subsequent batch, forever; arena warmup is spent and does not recur.
+# So run two identical measured batches and judge the second. By then WARMUP +
+# MEASURE requests have run, so any warmup tail is behind us — and a real leak is
+# entirely unaffected by which batch you measure it in.
+#
+# This is why a slow or contended runner can no longer produce a phantom leak,
+# and it does not loosen the threshold by a single byte. That matters: this is
+# the only instrument that sees per-request leaks at all (LSan never runs in a
+# signal-killed server), so a gate that cries wolf gets quietened, and then the
+# class goes unwatched.
+
 # $1 = label, $2 = route path, $3 = server script body
 run_growth_check() {
     local label="$1" route="$2" body="$3"
@@ -93,25 +125,42 @@ run_growth_check() {
         sleep 0.1
     done
 
-    curl -s -o /dev/null "http://127.0.0.1:$port$route?[1-$WARMUP]"
+    local warm m1 m2 c growth1
+    warm=$(batch_200s "http://127.0.0.1:$port$route?[1-$WARMUP]")
     a=$(rss_of "$srv_pid")
-    curl -s -o /dev/null "http://127.0.0.1:$port$route?[1-$MEASURE]"
+    m1=$(batch_200s "http://127.0.0.1:$port$route?[1-$MEASURE]")
     b=$(rss_of "$srv_pid")
+    m2=$(batch_200s "http://127.0.0.1:$port$route?[1-$MEASURE]")
+    c=$(rss_of "$srv_pid")
 
     kill "$srv_pid" 2>/dev/null || true
     wait "$srv_pid" 2>/dev/null || true
     rm -f "$srv_file" "/tmp/eigs_rss_srv_$$.log"
 
-    if [ -z "$a" ] || [ -z "$b" ]; then
-        fail "$label could not read VmRSS" "a='$a' b='$b'"
+    if [ "$warm" -ne "$WARMUP" ] || [ "$m1" -ne "$MEASURE" ] || [ "$m2" -ne "$MEASURE" ]; then
+        fail "$label INSTRUMENT: batch incomplete, RSS deltas are meaningless" \
+             "warmup $warm/$WARMUP, batches $m1/$MEASURE and $m2/$MEASURE — not a leak measurement"
         return
     fi
-    growth=$((b - a))
+
+    if [ -z "$a" ] || [ -z "$b" ] || [ -z "$c" ]; then
+        fail "$label could not read VmRSS" "a='$a' b='$b' c='$c'"
+        return
+    fi
+    growth1=$((b - a))
+    growth=$((c - b))
     if [ "$growth" -le "$THRESHOLD_KB" ]; then
-        ok "$label steady-state growth ${growth} kB over $MEASURE reqs (<= ${THRESHOLD_KB} kB)"
+        # growth1 is reported only when it disagrees — that is the fingerprint of
+        # a warmup that had not finished, and it is worth seeing in the log
+        # before someone re-diagnoses it as an intermittent leak (#765).
+        if [ "$growth1" -gt "$THRESHOLD_KB" ]; then
+            ok "$label steady-state growth ${growth} kB over $MEASURE reqs (<= ${THRESHOLD_KB} kB; first batch ${growth1} kB = unfinished warmup, decayed as expected)"
+        else
+            ok "$label steady-state growth ${growth} kB over $MEASURE reqs (<= ${THRESHOLD_KB} kB)"
+        fi
     else
         fail "$label leaked ${growth} kB over $MEASURE reqs" \
-             "$(( growth * 1024 / MEASURE )) B/req; threshold ${THRESHOLD_KB} kB"
+             "$(( growth * 1024 / MEASURE )) B/req; threshold ${THRESHOLD_KB} kB; first batch ${growth1} kB — growth did NOT decay, so this is a leak, not warmup"
     fi
 }
 
@@ -139,16 +188,26 @@ done
 
 if curl -s "http://127.0.0.1:$PORT_A/asetup" | grep -q "ok" \
    && curl -s "http://127.0.0.1:$PORT_A/secret" | grep -q "top secret"; then
-    curl -s -o /dev/null "http://127.0.0.1:$PORT_A/secret?[1-$WARMUP]"
+    WARM=$(batch_200s "http://127.0.0.1:$PORT_A/secret?[1-$WARMUP]")
     A=$(rss_of "$AUTH_PID")
-    curl -s -o /dev/null "http://127.0.0.1:$PORT_A/secret?[1-$MEASURE]"
+    M1=$(batch_200s "http://127.0.0.1:$PORT_A/secret?[1-$MEASURE]")
     B=$(rss_of "$AUTH_PID")
-    GROWTH=$((B - A))
-    if [ "$GROWTH" -le "$THRESHOLD_KB" ]; then
-        ok "RSS2 authed route steady-state growth ${GROWTH} kB over $MEASURE reqs (<= ${THRESHOLD_KB} kB)"
+    M2=$(batch_200s "http://127.0.0.1:$PORT_A/secret?[1-$MEASURE]")
+    C=$(rss_of "$AUTH_PID")
+    GROWTH1=$((B - A))
+    GROWTH=$((C - B))
+    if [ "$WARM" -ne "$WARMUP" ] || [ "$M1" -ne "$MEASURE" ] || [ "$M2" -ne "$MEASURE" ]; then
+        fail "RSS2 authed route INSTRUMENT: batch incomplete, RSS deltas are meaningless" \
+             "warmup $WARM/$WARMUP, batches $M1/$MEASURE and $M2/$MEASURE — not a leak measurement"
+    elif [ "$GROWTH" -le "$THRESHOLD_KB" ]; then
+        if [ "$GROWTH1" -gt "$THRESHOLD_KB" ]; then
+            ok "RSS2 authed route steady-state growth ${GROWTH} kB over $MEASURE reqs (<= ${THRESHOLD_KB} kB; first batch ${GROWTH1} kB = unfinished warmup, decayed as expected)"
+        else
+            ok "RSS2 authed route steady-state growth ${GROWTH} kB over $MEASURE reqs (<= ${THRESHOLD_KB} kB)"
+        fi
     else
         fail "RSS2 authed route leaked ${GROWTH} kB over $MEASURE reqs" \
-             "$(( GROWTH * 1024 / MEASURE )) B/req"
+             "$(( GROWTH * 1024 / MEASURE )) B/req; first batch ${GROWTH1} kB — growth did NOT decay, so this is a leak, not warmup"
     fi
 else
     fail "RSS2 authed route did not come up" "port $PORT_A"