From 779fd5244f3cced40d67d7a71df862b06287cf9f Mon Sep 17 00:00:00 2001 From: Pete Cornish Date: Sun, 6 Sep 2026 02:03:59 +0100 Subject: [PATCH 1/3] feat(fleet): set and show a keep deadline from the dashboard --- cmd/spinloop/dashboard_detail.go | 11 +- cmd/spinloop/dashboard_keep_test.go | 443 ++++++++++++++++++ cmd/spinloop/dashboard_model.go | 175 ++++++- cmd/spinloop/dashboard_render.go | 5 + cmd/spinloop/fleet.go | 4 +- cmd/spinloop/fleet_dashboard.go | 14 +- cmd/spinloop/fleet_dashboard_test.go | 4 + cmd/spinloop/metrics_render.go | 21 + cmd/spinloop/remote.go | 24 +- cmd/spinloop/retain_render_test.go | 167 +++++++ docs/commands/fleet.md | 38 +- docs/openapi.yaml | 10 + internal/fleet/node.go | 14 + internal/fleet/remote_node.go | 22 + internal/fleet/remote_node_test.go | 82 +++- internal/metrics/metrics.go | 7 + internal/remote/remote.go | 5 + internal/remote/remote_test.go | 10 +- .../changes/dashboard-keep/.openspec.yaml | 2 + openspec/changes/dashboard-keep/design.md | 183 ++++++++ openspec/changes/dashboard-keep/proposal.md | 66 +++ .../dashboard-keep/specs/fleet-client/spec.md | 170 +++++++ .../dashboard-keep/specs/remote-stats/spec.md | 58 +++ openspec/changes/dashboard-keep/tasks.md | 98 ++++ remote/lambda/shared/stats.ts | 9 + remote/lambda/stats/index.ts | 24 + remote/test/stats.test.ts | 118 ++++- 27 files changed, 1754 insertions(+), 30 deletions(-) create mode 100644 cmd/spinloop/dashboard_keep_test.go create mode 100644 cmd/spinloop/retain_render_test.go create mode 100644 openspec/changes/dashboard-keep/.openspec.yaml create mode 100644 openspec/changes/dashboard-keep/design.md create mode 100644 openspec/changes/dashboard-keep/proposal.md create mode 100644 openspec/changes/dashboard-keep/specs/fleet-client/spec.md create mode 100644 openspec/changes/dashboard-keep/specs/remote-stats/spec.md create mode 100644 openspec/changes/dashboard-keep/tasks.md diff --git a/cmd/spinloop/dashboard_detail.go b/cmd/spinloop/dashboard_detail.go index fb4c62b0..10d8b54b 100644 --- a/cmd/spinloop/dashboard_detail.go +++ b/cmd/spinloop/dashboard_detail.go @@ -92,6 +92,15 @@ func (m *dashModel) updateDetailKey(msg tea.KeyMsg) tea.Cmd { if len(m.entries) > 0 && m.actions[m.cursor].verb == "" { return m.beginAction("start") } + case "k": + // Open the duration prompt on the node in view, the same way the grid + // does — kept in the model, so it is checked before this view's own + // keys once it is open. + if m.keepOffered() { + m.keepPrompt = true + m.keepBuf = "4h" + m.keepErr = "" + } case "a": if len(m.entries) > 0 { m.abortAction() @@ -244,6 +253,6 @@ func (m dashModel) detailView() string { parts = append(parts, dashClip(line, w)) } parts = append(parts, divider) - parts = append(parts, m.footerLine(w, dashFooterHints(dashDetailKeys, m.canAbort()))) + parts = append(parts, m.footerLine(w, dashFooterHints(m.detailKeys(), m.canAbort()))) return strings.Join(parts, "\n") } diff --git a/cmd/spinloop/dashboard_keep_test.go b/cmd/spinloop/dashboard_keep_test.go new file mode 100644 index 00000000..471b5033 --- /dev/null +++ b/cmd/spinloop/dashboard_keep_test.go @@ -0,0 +1,443 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/spinloop-ai/spinloop/internal/daemon" + "github.com/spinloop-ai/spinloop/internal/fleet" + "github.com/spinloop-ai/spinloop/internal/metrics" + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// keeperDashNode wraps the fake with the keep capability: a node the dashboard's +// type assertion finds to be a Keeper, so the keep key and prompt are offered +// for it. Keep records the duration and returns a deadline; keepErr makes it +// refuse, to exercise a failed keep, and deadline fixes the control plane's +// reply when a test wants a specific one. +type keeperDashNode struct { + f *fakeDashNode + keeps int + keepDur time.Duration + keepErr error + deadline string +} + +var ( + _ fleet.Node = (*keeperDashNode)(nil) + _ fleet.Keeper = (*keeperDashNode)(nil) +) + +func (n *keeperDashNode) Name() string { return n.f.Name() } + +func (n *keeperDashNode) Status(ctx context.Context) (daemon.StatusResponse, error) { + return n.f.Status(ctx) +} + +func (n *keeperDashNode) Metrics(ctx context.Context) (metrics.Stats, error) { return n.f.Metrics(ctx) } + +func (n *keeperDashNode) Start(ctx context.Context) (daemon.StatusResponse, error) { + return n.f.Start(ctx) +} + +func (n *keeperDashNode) StartWith(ctx context.Context, dc *remote.DeployConfig, engineKey string) (daemon.StatusResponse, error) { + return n.f.StartWith(ctx, dc, engineKey) +} + +func (n *keeperDashNode) Stop(ctx context.Context) (daemon.StatusResponse, error) { + return n.f.Stop(ctx) +} + +func (n *keeperDashNode) Logs(ctx context.Context, offset int64, limit int) (daemon.LogsResponse, error) { + return n.f.Logs(ctx, offset, limit) +} + +func (n *keeperDashNode) Keep(ctx context.Context, d time.Duration) (string, error) { + if n.keepErr != nil { + return "", n.keepErr + } + n.keeps++ + n.keepDur = d + if n.deadline != "" { + return n.deadline, nil + } + return time.Now().Add(d).UTC().Format(time.RFC3339), nil +} + +// keeperModel is a one-node board over a keeperDashNode, the shape every keep +// test drives. +func keeperModel(node *keeperDashNode) *dashModel { + return &dashModel{ + entries: []dashEntry{{name: node.f.Name(), kind: fleet.KindRemote, node: node}}, + results: []fleet.NodeResult{{Name: node.f.Name()}}, + actions: make([]dashAction, 1), + width: 120, height: 40, + } +} + +// openKeepPrompt presses k on the model and returns the model with the prompt +// open, failing the test if k did not open it. +func openKeepPrompt(t *testing.T, m *dashModel) *dashModel { + t.Helper() + next, cmd := m.Update(dashKey("k")) + if cmd != nil { + t.Fatal("opening the prompt set off an action") + } + m = next.(*dashModel) + if !m.keepPrompt { + t.Fatal("k did not open the keep prompt") + } + return m +} + +// typeKey sends one key to the model and hands back the model, dropping the +// command — what the prompt's typing and backspace never produce. +func typeKey(m *dashModel, s string) *dashModel { + next, _ := m.Update(dashKey(s)) + return next.(*dashModel) +} + +// TestDashKeepPromptOpensOnTheKeyForARemoteNode: k opens the prompt, pre-filled +// with 4h, for a node that can be kept. +func TestDashKeepPromptOpensOnTheKeyForARemoteNode(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + if m.keepBuf != "4h" { + t.Errorf("prompt not pre-filled with 4h, got %q", m.keepBuf) + } +} + +// A local daemon node has no retention tag to set, so k drives nothing for it: +// the prompt does not open and no action is set off. +func TestDashKeepDrivesNothingForALocalNode(t *testing.T) { + f := newFakeDashNode("stopped") + m := &dashModel{ + entries: []dashEntry{{name: "box", kind: fleet.KindDaemon, node: f}}, + results: []fleet.NodeResult{{Name: "box"}}, + actions: make([]dashAction, 1), + width: 120, height: 40, + } + next, cmd := m.Update(dashKey("k")) + if cmd != nil { + t.Fatal("k on a local node set off an action") + } + if next.(*dashModel).keepPrompt { + t.Fatal("k opened the keep prompt for a local node") + } +} + +// While the prompt is open the board stands still: navigation keys go to the +// prompt (and change nothing), not to the grid. +func TestDashKeepPromptStandsStillOverNavigation(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + before := m.cursor + for _, key := range []string{"down", "up", "left", "right", "r"} { + next, _ := m.Update(dashKey(key)) + m = next.(*dashModel) + } + if m.cursor != before { + t.Errorf("navigation moved the selection while the prompt was open: %d -> %d", before, m.cursor) + } + if !m.keepPrompt { + t.Fatal("the prompt closed on a navigation key") + } +} + +// A confirmed entry that is a positive duration sets off a keep of that length; +// the pre-filled 4h is the one-Enter common case. +func TestDashKeepConfirmsADuration(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped"), deadline: "2030-01-02T04:00:00Z"} + m := keeperModel(node) + m = openKeepPrompt(t, m) + _, cmd := m.Update(dashKey("enter")) + if cmd == nil { + t.Fatal("enter did not set off the keep") + } + if m.keepPrompt { + t.Fatal("the prompt stayed open after a valid confirm") + } + if m.actions[0].verb != "keep" { + t.Fatalf("no keep recorded on the node: %+v", m.actions[0]) + } + runAction(t, cmd) + if node.keeps != 1 || node.keepDur != 4*time.Hour { + t.Errorf("keep not sent with the pre-filled 4h: keeps=%d dur=%s", node.keeps, node.keepDur) + } +} + +// A typed duration replaces the pre-fill and is what the keep is sent with. +func TestDashKeepTypesADuration(t *testing.T) { + for _, tc := range []struct { + entry string + want time.Duration + }{ + {"90m", 90 * time.Minute}, + {"1h30m", 90 * time.Minute}, + } { + t.Run(tc.entry, func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + // Clear the pre-filled 4h, then type the entry. + m = typeKey(m, "backspace") + m = typeKey(m, "backspace") + m = typeKey(m, tc.entry) + next, cmd := m.Update(dashKey("enter")) + m = next.(*dashModel) + if cmd == nil { + t.Fatalf("enter did not set off the keep for %q", tc.entry) + } + runAction(t, cmd) + if node.keepDur != tc.want { + t.Errorf("keep sent with %s, want %s", node.keepDur, tc.want) + } + }) + } +} + +// A confirmed entry that is not a positive duration leaves the prompt open with +// the reason at the foot, and sends nothing: 4hours is not a duration, and an +// empty entry is not one either. +func TestDashKeepLeavesThePromptOpenOnAnInvalidEntry(t *testing.T) { + for _, entry := range []string{"4hours", ""} { + t.Run(entry, func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + // Clear the pre-filled 4h; type the entry when there is one. + m = typeKey(m, "backspace") + m = typeKey(m, "backspace") + if entry != "" { + m = typeKey(m, entry) + } + next, cmd := m.Update(dashKey("enter")) + if cmd != nil { + t.Fatal("an invalid entry set off a keep") + } + m = next.(*dashModel) + if !m.keepPrompt { + t.Fatal("the prompt closed on an invalid entry") + } + if m.keepErr == "" { + t.Fatal("no reason shown for the invalid entry") + } + if node.keeps != 0 { + t.Fatal("a keep was sent for an invalid entry") + } + }) + } +} + +// esc cancels the prompt and sends nothing; q cancels and quits. +func TestDashKeepCancels(t *testing.T) { + t.Run("esc", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := openKeepPrompt(t, keeperModel(node)) + next, cmd := m.Update(dashKey("esc")) + if cmd != nil { + t.Fatal("esc set off a command") + } + if next.(*dashModel).keepPrompt { + t.Fatal("esc did not cancel the prompt") + } + if node.keeps != 0 { + t.Fatal("esc sent a keep") + } + }) + t.Run("q", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := openKeepPrompt(t, keeperModel(node)) + _, cmd := m.Update(dashKey("q")) + if cmd == nil { + t.Fatal("q did not quit") + } + if node.keeps != 0 { + t.Fatal("q sent a keep") + } + }) +} + +// A keep in flight rides the node's action: the tile carries the spinner and +// the keeping verb, a busy node takes no second keep, and completion clears the +// action, brings the node forward for one more read, and names the deadline. +func TestDashKeepInFlightAndCompletion(t *testing.T) { + dashFixNow(t, dashTestClock) + node := &keeperDashNode{f: newFakeDashNode("stopped"), deadline: "2030-01-02T04:00:00Z"} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, cmd := m.Update(dashKey("enter")) + m = next.(*dashModel) + + // In flight: the tile shows the keep, and the node takes no second keep. + if m.actions[0].verb != "keep" { + t.Fatalf("no keep recorded: %+v", m.actions[0]) + } + tile := dashTestTile(node.f.Name(), m.results[0], true, m.actions[0]) + if !strings.Contains(tile, "keeping") { + t.Errorf("in-flight tile does not show the keep:\n%s", tile) + } + knext, _ := m.Update(dashKey("k")) + if knext.(*dashModel).keepPrompt { + t.Fatal("a busy node opened a second keep prompt") + } + + // Completion: the action clears, the node is read again now, and the footer + // names the deadline the control plane set. + smsg, _ := runAction(t, cmd).(dashActionMsg) + if smsg.retainUntil != "2030-01-02T04:00:00Z" { + t.Errorf("completion message did not carry the deadline: %q", smsg.retainUntil) + } + m2, _ := m.Update(smsg) + mm := m2.(*dashModel) + if mm.actions[0].verb != "" { + t.Fatalf("the finished keep was not cleared: %+v", mm.actions[0]) + } + if mm.dueAt(0).After(time.Now()) { + t.Error("the kept node was not brought forward for one more read") + } + if !strings.Contains(mm.statusLine, "keep — retain until 2030-01-02T04:00:00Z") { + t.Errorf("completion line: %q", mm.statusLine) + } +} + +// A keep that the control plane refuses is reported as a failure with its +// reason, and the board stays open — including the named no-update-URL case. +func TestDashKeepFailureShowsItsReason(t *testing.T) { + for _, tc := range []struct { + err error + want string + }{ + {errors.New("no update_url configured: the remote deployment needs to be updated for keep support"), "no update_url"}, + {errors.New("keep returned HTTP 404: no running instance"), "no running instance"}, + } { + t.Run(tc.want, func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped"), keepErr: tc.err} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, cmd := m.Update(dashKey("enter")) + m = next.(*dashModel) + fmsg, _ := runAction(t, cmd).(dashActionMsg) + m2, _ := m.Update(fmsg) + mm := m2.(*dashModel) + if mm.actions[0].verb != "" { + t.Fatalf("the failed keep was not cleared: %+v", mm.actions[0]) + } + if !strings.Contains(mm.statusLine, "keep failed —") { + t.Errorf("failure line does not say it failed: %q", mm.statusLine) + } + if !strings.Contains(mm.statusLine, tc.want) { + t.Errorf("failure line missing the reason %q: %q", tc.want, mm.statusLine) + } + }) + } +} + +// The abort key drives nothing on a keep in flight: only a start is abortable, +// so a keep's wait runs to its own end. +func TestDashAbortDrivesNothingOnAKeep(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, _ := m.Update(dashKey("enter")) + m = next.(*dashModel) + if m.actions[0].verb != "keep" { + t.Fatal("no keep in flight") + } + anext, _ := m.Update(dashKey("a")) + mm := anext.(*dashModel) + if mm.actions[0].verb != "keep" { + t.Fatalf("the abort ended the keep in flight: %+v", mm.actions[0]) + } + if mm.actions[0].aborted { + t.Fatal("the abort marked the keep as abandoned") + } +} + +// The keep hint shows only where the key would drive something: an idle remote +// node shows it, a local node hides it, and a busy remote node hides it. +func TestDashKeepHintOnlyWhereItDrivesSomething(t *testing.T) { + t.Run("idle remote shows it", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + if !strings.Contains(m.gridKeys(), "k keep") { + t.Errorf("grid hint missing the keep key: %q", m.gridKeys()) + } + if !strings.Contains(m.detailKeys(), "k keep") { + t.Errorf("detail hint missing the keep key: %q", m.detailKeys()) + } + }) + t.Run("local node hides it", func(t *testing.T) { + f := newFakeDashNode("stopped") + m := &dashModel{ + entries: []dashEntry{{name: "box", kind: fleet.KindDaemon, node: f}}, + results: []fleet.NodeResult{{Name: "box"}}, + actions: make([]dashAction, 1), + width: 120, height: 40, + } + if strings.Contains(m.gridKeys(), "k keep") { + t.Errorf("grid hint offers a keep a local node cannot take: %q", m.gridKeys()) + } + }) + t.Run("busy remote hides it", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, _ := m.Update(dashKey("enter")) + m = next.(*dashModel) + if strings.Contains(m.gridKeys(), "k keep") { + t.Errorf("grid hint offers a second keep while one is in flight: %q", m.gridKeys()) + } + }) +} + +// The deadline rides the node's read onto the tile and the detail screen, +// beside the last-active line, and a read without one draws no line. +func TestDashTileAndDetailShowRetainUntil(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + r := fleet.NodeResult{ + Name: "env", + Outcome: fleet.OutcomeOK, + Metrics: metrics.Stats{ + State: "stopped", Runner: "llamacpp", ModelID: "org/m", + RetainUntil: "2030-01-02T04:00:00Z", + }, + } + m.results[0] = r + + tile := dashTestTile("env", r, true, dashAction{}) + if !strings.Contains(tile, "retain until 2030-01-02T04:00:00Z") { + t.Errorf("tile missing the retain-until line:\n%s", tile) + } + lines := m.detailNodeLines() + found := false + for _, l := range lines { + if strings.Contains(l, "retain until 2030-01-02T04:00:00Z") { + found = true + } + } + if !found { + t.Errorf("detail view missing the retain-until line:\n%s", strings.Join(lines, "\n")) + } +} + +// A read without a deadline — a local node, or an unkept or lapsed environment — +// draws no line on the tile. +func TestDashTileOmitsRetainUntilWhenAbsent(t *testing.T) { + r := fleet.NodeResult{ + Name: "box", + Outcome: fleet.OutcomeOK, + Metrics: metrics.Stats{State: "running", Runner: "llamacpp", ModelID: "org/m", + CPU: &metrics.CpuStat{Utilization: 30}}, + } + tile := dashTestTile("box", r, true, dashAction{}) + if strings.Contains(tile, "retain until") { + t.Errorf("tile invented a deadline:\n%s", tile) + } +} diff --git a/cmd/spinloop/dashboard_model.go b/cmd/spinloop/dashboard_model.go index 851eaecc..424eb751 100644 --- a/cmd/spinloop/dashboard_model.go +++ b/cmd/spinloop/dashboard_model.go @@ -10,6 +10,7 @@ import ( "fmt" "strings" "time" + "unicode/utf8" tea "github.com/charmbracelet/bubbletea" "github.com/spinloop-ai/spinloop/internal/daemon" @@ -68,7 +69,13 @@ type dashModel struct { cursor int // the selected node scrollRow int // the first grid row on screen - confirm bool // a stop is waiting on its confirmation + confirm bool // a stop is waiting on its confirmation + // A keep is waiting on its duration: the prompt is open, the duration typed + // so far (pre-filled so the common case is one Enter), and the parse reason + // shown at the foot when a confirmed entry is not a positive duration. + keepPrompt bool + keepBuf string + keepErr string statusLine string // gauge is the board's resource-series format: false draws the bar @@ -159,12 +166,16 @@ type dashActionProgressMsg struct { phase fleet.StartPhase } -// dashActionMsg is one completed start or stop. +// dashActionMsg is one completed start, stop, or keep. type dashActionMsg struct { node string - verb string // "start" or "stop" + verb string // "start", "stop", or "keep" status daemon.StatusResponse err error + // retainUntil is the deadline a keep set, RFC 3339, carried so the footer's + // outcome can name it. Empty for a start or stop, and for a keep that failed + // before the control plane set anything. + retainUntil string } // dashTickCmd schedules the next fast tick. The tick is one-shot, so every @@ -281,6 +292,14 @@ func (m *dashModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } m.applyDetailLog(msg.result) case tea.KeyMsg: + // The keep prompt stands in front of everything: while it is open, + // every key goes to it — navigation, selection, refresh, even the stop + // confirmation — so the board holds still until the duration is entered + // or cancelled. It is checked before confirm and detail so it works + // whether it was opened from the grid or the detail view. + if m.keepPrompt { + return m, m.updateKeepPromptKey(msg) + } if m.confirm { switch msg.String() { case "y": @@ -347,6 +366,15 @@ func (m *dashModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if len(m.entries) > 0 && m.actions[m.cursor].verb == "" { cmd = m.beginAction("start") } + case "k": + // Open the duration prompt, pre-filled so the common case is one + // Enter. keepOffered gates it to a node that can be kept and has + // nothing in flight; elsewhere the key drives nothing. + if m.keepOffered() { + m.keepPrompt = true + m.keepBuf = "4h" + m.keepErr = "" + } case "a": // End the wait on the node's in-flight action — the one-shot // equivalent of Ctrl+C. A node with nothing in flight is driven @@ -515,6 +543,11 @@ func dashActionLine(msg dashActionMsg, aborted bool) string { } return msg.node + ": " + msg.verb + " failed — " + r.Detail() } + if msg.verb == "keep" { + // A keep has no engine state to report — its result is the deadline the + // control plane set, which is what the operator came to set. + return msg.node + ": keep — retain until " + msg.retainUntil + } state := msg.status.State if state == "" { state = "done" @@ -583,6 +616,94 @@ func (m *dashModel) beginAction(verb string) tea.Cmd { return run } +// beginKeep sets off a keep of the selected node for the confirmed duration. +// It reuses beginAction's scaffolding — one action per node, the tile's +// spinner, the short read interval for the duration of the call — but the call +// itself is a keep, not a start or stop: the node must be a Keeper (a remote +// environment), and the call returns the deadline it set rather than an engine +// state. A node that is not a Keeper, or that already has an action in flight, +// is driven by nothing, and its reason lands on the status line. +func (m *dashModel) beginKeep(d time.Duration) tea.Cmd { + e := m.entries[m.cursor] + m.keepPrompt = false + m.keepBuf = "" + m.keepErr = "" + if e.node == nil { + m.statusLine = e.name + ": " + e.standing.Detail() + return nil + } + if m.actions[m.cursor].verb != "" { + m.statusLine = e.name + ": still " + dashVerbProgress(m.actions[m.cursor].verb) + return nil + } + keeper, ok := e.node.(fleet.Keeper) + if !ok { + m.statusLine = e.name + ": keep needs a remote environment" + return nil + } + spin := !m.actionInFlight() + ctx, cancel := context.WithCancel(context.Background()) + m.actions[m.cursor] = dashAction{verb: "keep", since: dashNow(), cancel: cancel} + // The node is read on the short interval for the duration of the action, + // starting now rather than at its kind's next due time: what a keep changed + // is its own read's retainUntil. + m.scheduleRead(m.cursor, time.Time{}) + m.statusLine = dashVerbProgress("keep") + " " + e.name + "…" + run := func() tea.Msg { + deadline, err := keeper.Keep(ctx, d) + cancel() + return dashActionMsg{node: e.name, verb: "keep", retainUntil: deadline, err: err} + } + if spin { + return tea.Batch(run, dashSpinTickCmd()) + } + return run +} + +// updateKeepPromptKey answers the keys the keep prompt reads: printable runes +// append to the duration, backspace removes the last, Enter confirms, esc +// cancels, and q/Ctrl+C cancels and quits (mirroring the stop confirmation's +// own quit). A confirmed entry that is not a positive duration leaves the +// prompt open and names why at the foot, so a mistyped entry is corrected in +// place rather than lost. +func (m *dashModel) updateKeepPromptKey(msg tea.KeyMsg) tea.Cmd { + switch s := msg.String(); { + case s == "enter": + d, err := time.ParseDuration(strings.TrimSpace(m.keepBuf)) + if err != nil || d <= 0 { + m.keepErr = "enter a duration like 4h" + return nil + } + return m.beginKeep(d) + case s == "backspace": + m.keepBuf = chopLastRune(m.keepBuf) + m.keepErr = "" + case s == "esc": + m.keepPrompt = false + m.keepBuf = "" + m.keepErr = "" + case s == "ctrl+c", s == "q": + m.keepPrompt = false + m.keepBuf = "" + m.keepErr = "" + return tea.Quit + case msg.Type == tea.KeyRunes: + m.keepBuf += s + m.keepErr = "" + } + return nil +} + +// chopLastRune drops the final rune from a string, counting a wide rune as the +// bytes that make it up rather than cutting a UTF-8 sequence mid-way. +func chopLastRune(s string) string { + if s == "" { + return s + } + _, size := utf8.DecodeLastRuneInString(s) + return s[:len(s)-size] +} + // abortAction ends the wait on the selected node's in-flight start. Only a // start is abortable: it is the one action with no deadline of its own — a // cold cloud wake takes minutes, so a wait the operator no longer wants to @@ -617,6 +738,41 @@ func (m dashModel) canAbort() bool { return len(m.entries) > 0 && m.actions[m.cursor].verb == "start" } +// keepOffered reports whether the keep key would do anything for the node under +// the cursor: the node must support a keep (a remote environment) and have +// nothing in flight. A local daemon node has no retention tag to set, and a +// node already acting takes no second action. The footer uses this to include +// the keep hint only where it would drive something, the same way canAbort +// gates the abort hint. +func (m dashModel) keepOffered() bool { + if len(m.entries) == 0 { + return false + } + if m.actions[m.cursor].verb != "" { + return false + } + _, ok := m.entries[m.cursor].node.(fleet.Keeper) + return ok +} + +// gridKeys and detailKeys are the two footers' key help with the keep entry +// included only where keepOffered says the node under the cursor can be kept — +// the same gate the k key itself answers to, so a hint is never shown for a key +// that would drive nothing on that node. +func (m dashModel) gridKeys() string { + if m.keepOffered() { + return "↑↓←→ move s start k keep a abort x stop r refresh q quit" + } + return dashGridKeys +} + +func (m dashModel) detailKeys() string { + if m.keepOffered() { + return "esc back s start k keep x stop a abort f follow" + } + return dashDetailKeys +} + // indexOf finds an entry by name. Fleet-file names are unique — the fleet // file itself refuses a repeat — so a find is a position, not a set. func (m *dashModel) indexOf(name string) int { @@ -684,7 +840,7 @@ func (m dashModel) View() string { if hi > lo { parts = append(parts, strings.Join(rows[lo:hi], "\n")) } - parts = append(parts, m.footerLine(w, dashFooterHints(dashGridKeys, m.canAbort()))) + parts = append(parts, m.footerLine(w, dashFooterHints(m.gridKeys(), m.canAbort()))) return strings.Join(parts, "\n") } @@ -711,7 +867,16 @@ const dashGridKeys = "↑↓←→ move s start a abort x stop g format // splitting them on a space would draw their first word as though it were one. func (m dashModel) footerLine(w int, keys string) string { line := dashKeyHints(keys) - if m.confirm && len(m.entries) > 0 { + if m.keepPrompt && len(m.entries) > 0 { + // The prompt replaces the key help for its life: what is being typed, + // and the two keys that resolve it. The parse reason, when a confirmed + // entry was not a duration, rides at the foot beside the status line. + line = "keep " + m.entries[m.cursor].name + " for: " + m.keepBuf + "_ " + dashHintGap + + dashKeyHints("enter keep"+dashHintGap+"esc cancel") + if m.keepErr != "" { + line += " " + m.keepErr + } + } else if m.confirm && len(m.entries) > 0 { line = "stop " + m.entries[m.cursor].name + "?" + dashHintGap + dashKeyHints("y yes"+dashHintGap+"n no") } diff --git a/cmd/spinloop/dashboard_render.go b/cmd/spinloop/dashboard_render.go index df734cce..2e4e9de5 100644 --- a/cmd/spinloop/dashboard_render.go +++ b/cmd/spinloop/dashboard_render.go @@ -438,6 +438,11 @@ func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool fmt.Fprintln(w, line) } renderLastActiveIndented(w, m.LastActiveAt, m.IdleSeconds) + // The retention deadline, beside last-active and from the same read: a + // remote environment the operator has kept draws it whatever the engine's + // state, and a read without one — a local node, an unkept or lapsed + // environment — draws nothing. + renderRetainIndented(w, m.RetainUntil) if resources { if gauge { renderStatGauges(w, m.CPU, m.Memory, m.GPUs) diff --git a/cmd/spinloop/fleet.go b/cmd/spinloop/fleet.go index c1c4de9a..f464a615 100644 --- a/cmd/spinloop/fleet.go +++ b/cmd/spinloop/fleet.go @@ -203,8 +203,10 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string) fmt.Fprintln(w) // Before the continue, for the same reason the remote formats show it // before theirs: a node whose engine has stopped still has a useful - // answer to "when did this last do anything?". + // answer to "when did this last do anything?" — and, for a retained + // remote environment, "until is it kept?". renderLastActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds) + renderRetainIndented(w, stats.RetainUntil) switch format { case "bar": // No state gate, for the same reason the remote bar format has diff --git a/cmd/spinloop/fleet_dashboard.go b/cmd/spinloop/fleet_dashboard.go index 45cfef62..baca0a1a 100644 --- a/cmd/spinloop/fleet_dashboard.go +++ b/cmd/spinloop/fleet_dashboard.go @@ -26,11 +26,15 @@ func fleetDashboardCmd() *cobra.Command { what the bar format of fleet metrics prints — state, what it serves, the resource bars, the token counters — repainted on an interval. -The view is read-only apart from three keys: s starts the selected node, a -abandons a start or stop still in flight on it (the wait ends, the node is -free again — a wake the cloud is carrying goes on), x stops it after a -confirmation. The arrow keys move the selection, r forces a refresh, q or -Ctrl+C leaves. +The view is read-only apart from four keys: s starts the selected node, k +keeps a remote environment for a duration you type — it asks how long, +pre-filled with 4h, and reports the deadline the control plane set when the +keep is done — a abandons a start still in flight on it (the wait ends, the +node is free again — a wake the cloud is carrying goes on), x stops it after +a confirmation. The arrow keys move the selection, r forces a refresh, q or +Ctrl+C leaves. The keep key shows only for a node that can be kept — a remote +environment — and a kept environment's tile and detail view carry its +deadline beside the last-active line, whatever the engine's state. A node that cannot be reached is still a tile, showing why, and a node whose token reference is unresolvable holds its reason for the life of the view. diff --git a/cmd/spinloop/fleet_dashboard_test.go b/cmd/spinloop/fleet_dashboard_test.go index 06b24f60..8e951fec 100644 --- a/cmd/spinloop/fleet_dashboard_test.go +++ b/cmd/spinloop/fleet_dashboard_test.go @@ -248,6 +248,10 @@ func dashKey(s string) tea.KeyMsg { return tea.KeyMsg{Type: tea.KeyEsc} case "ctrl+c": return tea.KeyMsg{Type: tea.KeyCtrlC} + case "enter": + return tea.KeyMsg{Type: tea.KeyEnter} + case "backspace": + return tea.KeyMsg{Type: tea.KeyBackspace} default: return tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)} } diff --git a/cmd/spinloop/metrics_render.go b/cmd/spinloop/metrics_render.go index fee496e3..c4b4c925 100644 --- a/cmd/spinloop/metrics_render.go +++ b/cmd/spinloop/metrics_render.go @@ -50,6 +50,27 @@ func renderLastActiveKeyValue(w io.Writer, lastActiveAt string, idleSeconds int) } } +// renderRetainIndented draws the retention-deadline line in the indented block +// the bar format and both fleet formats use, aligned to the bar-label column +// and beside the last-active line. The deadline is an absolute instant the +// control plane set, relayed verbatim — it is not re-checked here, so a line is +// drawn exactly when the read carries one (the stats reply drops it once it has +// passed). Local daemon nodes carry no deadline, so their line is absent. +func renderRetainIndented(w io.Writer, retainUntil string) { + if retainUntil != "" { + fmt.Fprintf(w, " %-9s %s\n", "retain until", retainUntil) + } +} + +// renderRetainKeyValue draws the retention deadline as a row of the table +// format, padded to the key column its neighbours use, beside the last-active +// row. +func renderRetainKeyValue(w io.Writer, retainUntil string) { + if retainUntil != "" { + fmt.Fprintf(w, "retain until: %s\n", retainUntil) + } +} + // validateMetricsFormat rejects a --format value the metrics commands do not // understand, naming the ones they do. Both `remote metrics` and // `fleet metrics` run it before doing any work. diff --git a/cmd/spinloop/remote.go b/cmd/spinloop/remote.go index 804c6fae..cda3f774 100644 --- a/cmd/spinloop/remote.go +++ b/cmd/spinloop/remote.go @@ -883,6 +883,9 @@ func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCos fmt.Fprintf(w, "model: %s\n", resp.ModelID) } renderLastActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds) + // A stopped environment can still be retained: its deadline is the + // control plane's, so it rides this branch too. + renderRetainKeyValue(w, resp.RetainUntil) return nil } @@ -905,6 +908,7 @@ func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCos fmt.Fprintf(w, "uptime: %s\n", formatDuration(resp.UptimeSeconds)) } renderLastActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds) + renderRetainKeyValue(w, resp.RetainUntil) renderTokenLines(w, resp.Tokens) renderGPUTable(w, resp.GPUs) @@ -978,13 +982,14 @@ func formatMetricsHeader(resp *remote.StatsResponse, w io.Writer) { func formatMetricsBar(resp *remote.StatsResponse, cfg remote.Config, w io.Writer) error { formatMetricsHeader(resp, w) - // Before the series: when the endpoint last did work is worth showing for - // whatever state it is in, and the retained history is too — a stopped - // endpoint's readings up to the stop answer what it was doing until it - // stopped. A stopped endpoint's current reading carries no resource - // figures, so the series it draws come from the history alone, or not at - // all where the daemon predates it. + // Before the series: when the endpoint last did work — and, for a retained + // endpoint, until it is kept — is worth showing in whatever state it is in, + // and the retained history is too: a stopped endpoint's readings up to the + // stop answer what it was doing until it stopped. A stopped endpoint's + // current reading carries no resource figures, so the series it draws come + // from the history alone, or not at all where the daemon predates it. renderLastActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds) + renderRetainIndented(w, resp.RetainUntil) renderStatBars(w, resp.CPU, resp.Memory, resp.GPUs, resp.History, barLineW) renderTokenLines(w, resp.Tokens) @@ -996,10 +1001,11 @@ func formatMetricsBar(resp *remote.StatsResponse, cfg remote.Config, w io.Writer func formatMetricsGauge(resp *remote.StatsResponse, cfg remote.Config, w io.Writer) error { formatMetricsHeader(resp, w) - // Before the early return: a stopped endpoint draws no gauges, but when - // it last did work is exactly what a stopped endpoint is worth asking - // about. + // Before the early return: a stopped endpoint draws no gauges, but when it + // last did work — and until it is retained — is exactly what a stopped + // endpoint is worth asking about. renderLastActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds) + renderRetainIndented(w, resp.RetainUntil) if resp.State != "running" { return nil diff --git a/cmd/spinloop/retain_render_test.go b/cmd/spinloop/retain_render_test.go new file mode 100644 index 00000000..8da9476e --- /dev/null +++ b/cmd/spinloop/retain_render_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "strings" + "testing" +) + +// The retention deadline rides the stats read and is drawn beside the +// last-active line in every surface that draws that line. The control plane is +// the single home of the "is it still in the future?" judgement, so these +// renderers only ever ask "does the read carry a deadline?" and draw it when +// present, and nothing when it is empty. + +const retainedDeadline = "2030-01-02T04:00:00Z" + +func TestRemoteMetricsBarShowsRetainUntil(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "running", + "instanceType": "g6e.xlarge", + "modelId": "unsloth/Qwen3.6-27B", + "cpu": {"utilization": 24}, + "retainUntil": "`+retainedDeadline+`" + }`) + + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=bar"}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if !strings.Contains(out, "retain until "+retainedDeadline) { + t.Errorf("bar format missing the retain-until line:\n%s", out) + } + // Beside last-active, before the bars: a fact about the endpoint, not a + // utilisation reading. + active := strings.Index(out, "last active") + retain := strings.Index(out, "retain until") + bars := strings.Index(out, "CPU") + if !(retain >= 0 && bars >= 0 && retain < bars) { + t.Errorf("retain-until line is not before the bars:\n%s", out) + } + if active >= 0 && !(retain > active) { + t.Errorf("retain-until line is not beside (after) last-active:\n%s", out) + } +} + +func TestRemoteMetricsTableShowsRetainUntil(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "running", + "instanceType": "g6e.xlarge", + "modelId": "unsloth/Qwen3.6-27B", + "cpu": {"utilization": 24}, + "retainUntil": "`+retainedDeadline+`" + }`) + + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=table"}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + // A key-value row, padded to the key column its neighbours use. + if !strings.Contains(out, "retain until: "+retainedDeadline) { + t.Errorf("table format missing the retain-until row:\n%s", out) + } +} + +// A stopped environment can still be retained: the deadline is the control +// plane's, not the engine's, so the line survives the non-running short-circuit +// in both formats. +func TestRemoteMetricsStoppedRetainedStillShowsRetainUntil(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "stopped", + "runner": "llamacpp", + "modelId": "unsloth/Qwen3.6-27B", + "retainUntil": "`+retainedDeadline+`" + }`) + + for format, want := range map[string]string{ + "bar": "retain until " + retainedDeadline, + "table": "retain until: " + retainedDeadline, + } { + t.Run(format, func(t *testing.T) { + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if !strings.Contains(out, want) { + t.Errorf("%s format dropped the deadline for a stopped, retained endpoint:\n%s", format, out) + } + }) + } +} + +// No deadline on the read, no line: the renderer does not invent one, and it +// does not disturb the rest of the report. +func TestRemoteMetricsOmitsRetainUntilWhenAbsent(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "running", + "cpu": {"utilization": 24}, + "lastActiveAt": "2026-08-10T10:00:00Z", + "idleSeconds": 125 + }`) + + for _, format := range []string{"bar", "table"} { + t.Run(format, func(t *testing.T) { + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if strings.Contains(out, "retain until") { + t.Errorf("%s format claimed a deadline the read does not carry:\n%s", format, out) + } + if !strings.Contains(out, "last active") { + t.Errorf("%s format lost the rest of the report:\n%s", format, out) + } + }) + } +} + +// The deadline rides the node's stats, so `fleet metrics` draws it beside +// last-active whatever the node's state. A faked daemon carrying the field +// stands in for a retained remote environment on the same render path. +func TestFleetMetricsShowsRetainUntil(t *testing.T) { + fleetNodeWithMetrics(t, map[string]any{ + "state": "running", + "runner": "llamacpp", + "modelId": "org/qwen", + "cpu": map[string]any{"utilization": 30.0}, + "retainUntil": retainedDeadline, + }) + + out := captureStdout(t, func() { + if err := cmdFleet([]string{"metrics"}); err != nil { + t.Fatalf("cmdFleet metrics: %v", err) + } + }) + if !strings.Contains(out, "retain until "+retainedDeadline) { + t.Errorf("fleet metrics missing the retain-until line:\n%s", out) + } +} + +// A node whose read carries no deadline draws no line, so a local daemon node — +// which never has one — is simply quieter on that line. +func TestFleetMetricsOmitsRetainUntilWhenAbsent(t *testing.T) { + fleetNodeWithMetrics(t, map[string]any{ + "state": "running", + "runner": "llamacpp", + "modelId": "org/qwen", + "cpu": map[string]any{"utilization": 30.0}, + "lastActiveAt": "2026-08-10T10:00:00Z", + "idleSeconds": 125, + }) + + out := captureStdout(t, func() { + if err := cmdFleet([]string{"metrics"}); err != nil { + t.Fatalf("cmdFleet metrics: %v", err) + } + }) + if strings.Contains(out, "retain until") { + t.Errorf("fleet metrics claimed a deadline a local node never carries:\n%s", out) + } +} diff --git a/docs/commands/fleet.md b/docs/commands/fleet.md index bb3626eb..18104753 100644 --- a/docs/commands/fleet.md +++ b/docs/commands/fleet.md @@ -262,6 +262,12 @@ node's engine has done some work. A node whose engine has *stopped* still shows it — the daemon keeps the record across a stop, and "how long since this did anything?" is worth more about a stopped engine than about a busy one. +A `kind: remote` environment carries a `retain until` figure beside the +last-active one, on the same omitted-when-absent terms: it shows the +deadline the idle sweep will not pass while it is in the future, and is gone +once it has passed or was never set. It is the same line the dashboard draws +on a kept environment's tile and detail screen, from the same read. + `--watch`/`-w` redraws the whole fleet on an interval, clearing the screen in place with no scrollback. Each refresh is rendered into a buffer first, so a slow node delays the refresh but never tears the display. Ctrl+C exits @@ -302,6 +308,7 @@ spinloop fleet dashboard --fleet f.yaml # another fleet file | `r` | Force a refresh of every node, now | | `g` | Toggle every tile's resource series between bar (sparklines of the retained history) and gauge (the current reading) | | `s` | Start the selected node — without confirmation | +| `k` | Keep a remote environment for a duration you type — shown only for a node that can be kept, and only while it has no action in flight | | `a` | Abandon a start in flight on the selected node — the wait ends, the node is free again (a stop in flight is not abortable) | | `x` | Stop the selected node — it asks first (`y` sends, `n` or `esc` cancel) | | `q` or `Ctrl+C` | Leave | @@ -330,8 +337,25 @@ refresh. A stop in flight is not abortable: it targets an engine already running rather than a cold wake with no deadline of its own, and `a` drives nothing while one is in progress. +`keep` is a remote-environment action: a local daemon has no idle sweep, so +there is no deadline to set, and the key does not show for one. Pressing it +opens a prompt at the foot of the view, pre-filled with `4h`, asking how long +the environment should be retained. The prompt is the confirmation — there is +no second one — so the operator sees the duration it will set before choosing +to send it: a keep overwrites the deadline and ends nothing, where a stop +ends something and so asks. Type the duration and press `enter` to send it; +`esc` cancels; `q` or `Ctrl+C` cancel the prompt and leave the dashboard, as +the stop confirmation does. An entry that does not parse as a positive +duration leaves the prompt open and shows the parse reason in the footer's +hint slot, so the entry is kept and corrected in place. While the keep runs +its tile carries it, and it is not abortable — one fast signed call, so `a` +drives nothing on it. When it finishes, the status line reports the deadline +the control plane set and the node is re-read at once, which is what brings +the `retain until` figure onto the tile and detail screen at the node's next +round rather than waiting out its full cadence. + Everything else in the view is `fleet status`/`metrics`/`logs` in place — it -is read-only apart from those three action keys. It needs a real terminal: a +is read-only apart from those four action keys. It needs a real terminal: a piped run is refused, and it says so by way of `fleet metrics --watch`, which is the streamable surface. @@ -348,11 +372,13 @@ spinloop fleet dashboard # select a node, press Enter for its full metrics and log, Esc to go back ``` -`s`, `x` and `a` drive the node shown exactly as they drive the selected node -on the grid — the same no-confirmation start, the same stop confirmation, the -same abandon. `q`/`Ctrl+C` are grid keys only and do nothing here — `Esc` back -to the grid first, then quit from there — so a stray quit keystroke while -looking at a node can't end the session out from under you. The rest of the +`s`, `k`, `x` and `a` drive the node shown exactly as they drive the selected +node on the grid — the same no-confirmation start, the same keep prompt, the +same stop confirmation, the same abandon. `q`/`Ctrl+C` are grid keys only and +do nothing here — `Esc` back to the grid first, then quit from there — so a +stray quit keystroke while looking at a node can't end the session out from +under you. The one exception is the keep prompt: while it is open it answers +to `q`/`Ctrl+C` the way the stop confirmation does, cancelling and leaving. The rest of the fleet keeps refreshing behind the view, and any action already in flight on another node keeps running. A node whose engine has never run shows the same explanation `fleet logs` gives for it, not an empty pane. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index d15b2725..fde05cee 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -433,6 +433,16 @@ components: Absent, not `not-ready`, when it does not apply: no engine is running, its runner has no known health-check convention, or this daemon predates the check. + retainUntil: + type: string + format: date-time + description: | + The environment's retention deadline, RFC 3339: the idle sweep + will not terminate the instance before it. A property of the cloud + instance, not the engine, so it is absent for local daemon nodes + and for remote environments without an update URL. The stats reply + carries it only while it is a time in the future — a passed + deadline keeps nothing, so it is dropped there and on this read. TokenStats: type: object diff --git a/internal/fleet/node.go b/internal/fleet/node.go index 1be6af2a..040619a5 100644 --- a/internal/fleet/node.go +++ b/internal/fleet/node.go @@ -54,6 +54,20 @@ type ProgressStarter interface { StartWithProgress(ctx context.Context, report func(StartPhase)) (daemon.StatusResponse, error) } +// Keeper is an optional node capability: a node whose instance can be pinned so +// the idle sweep does not terminate it before a stated deadline. Only remote +// environments have it — the retention tag lives on a cloud instance — so a +// local daemon node does not implement it. A caller that can offer a keep +// (the dashboard) asserts for it and hides the key when it is absent. +// +// d is how long, from now, to keep the instance. The return is the control +// plane's own deadline, RFC 3339: the absolute instant the instance is kept +// until, as the control plane recorded it — not the caller's clock plus d — so +// what a caller shows matches what the control plane holds. +type Keeper interface { + Keep(ctx context.Context, d time.Duration) (string, error) +} + // Node is one member of the fleet. Only daemonNode implements it today; the // interface exists so a remote-environment kind (an `spinloop remote` // environment read through its stats Lambda, which already yields diff --git a/internal/fleet/remote_node.go b/internal/fleet/remote_node.go index 7501c3aa..5425fbc2 100644 --- a/internal/fleet/remote_node.go +++ b/internal/fleet/remote_node.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/spinloop-ai/spinloop/internal/daemon" "github.com/spinloop-ai/spinloop/internal/metrics" @@ -105,6 +106,26 @@ func (n *remoteNode) Stop(ctx context.Context) (daemon.StatusResponse, error) { return statusFromRemote(*resp), nil } +// Keep pins this environment's instance so the idle sweep does not terminate it +// before now plus d. The deadline is computed here and passed as an absolute +// time, the way the CLI computes it; a keep is what keeps the instance, not +// what the engine is doing, so the call needs no engine or deploy context. If +// the control plane's reply names the deadline it recorded, that is returned +// verbatim; when the reply omits it — a control plane that predates keep +// echoing the value back — the requested deadline is returned instead, so a +// caller always holds something to show. +func (n *remoteNode) Keep(ctx context.Context, d time.Duration) (string, error) { + deadline := time.Now().Add(d) + resp, err := remote.Keep(ctx, n.cfg, deadline) + if err != nil { + return "", err + } + if resp.RetainUntil != "" { + return resp.RetainUntil, nil + } + return deadline.UTC().Format(time.RFC3339), nil +} + // remoteEngineTail caps how many engine log events a node read pulls. Remote logs // are a bounded tail pulled from the log store, not a byte cursor, so a follow of // a chatty engine must not page through an unbounded window. @@ -165,6 +186,7 @@ func statsFromRemote(resp remote.StatsResponse) metrics.Stats { Errors: resp.Errors, LastActiveAt: resp.LastActiveAt, IdleSeconds: resp.IdleSeconds, + RetainUntil: resp.RetainUntil, } } diff --git a/internal/fleet/remote_node_test.go b/internal/fleet/remote_node_test.go index 6d8ec0fe..a617891f 100644 --- a/internal/fleet/remote_node_test.go +++ b/internal/fleet/remote_node_test.go @@ -82,7 +82,8 @@ func TestStatsFromRemote(t *testing.T) { got := statsFromRemote(remote.StatsResponse{ State: "running", Runner: "llamacpp", ModelID: "org/m", UptimeSeconds: 10, Tokens: tokens, LastActiveAt: "2026-01-02T00:00:00Z", IdleSeconds: 5, Version: "1.2.3", - History: history, + History: history, + RetainUntil: "2026-01-02T04:00:00Z", }) if got.State != "running" || got.Runner != "llamacpp" || got.ModelID != "org/m" || got.UptimeSeconds != 10 { t.Errorf("statsFromRemote = %+v", got) @@ -103,6 +104,14 @@ func TestStatsFromRemote(t *testing.T) { if got := statsFromRemote(remote.StatsResponse{State: "running"}); got.History != nil { t.Errorf("an absent history became present: %+v", got.History) } + if got.RetainUntil != "2026-01-02T04:00:00Z" { + t.Errorf("retainUntil not carried over: %q", got.RetainUntil) + } + // An environment with no live retention carries nothing: the field is + // absent, not a zero time a formatter would have to special-case. + if got := statsFromRemote(remote.StatsResponse{State: "running"}); got.RetainUntil != "" { + t.Errorf("no retention should map to an empty retainUntil, got %q", got.RetainUntil) + } } func TestLogsFromRemote(t *testing.T) { @@ -234,6 +243,77 @@ func TestRemoteNodeStatusOverTheControlPlane(t *testing.T) { } } +// A remote node's keep pins the instance over its control plane: the deadline +// is computed from the duration and sent as an absolute time, and the value +// returned to a caller is the control plane's own deadline, not the caller's +// clock plus the duration. +func TestRemoteNodeKeepOverTheControlPlane(t *testing.T) { + stubAWSCreds(t) + srv := remoteControlServer(t, + `{"retainUntil":"2026-01-02T04:00:00Z"}`, http.StatusOK) + cfg := remote.Config{StartURL: srv.URL, StopURL: srv.URL, UpdateURL: srv.URL, Region: "us-east-1"} + node, err := NewRemoteNode("env", cfg) + if err != nil { + t.Fatal(err) + } + got, err := node.(Keeper).Keep(context.Background(), 4*time.Hour) + if err != nil { + t.Fatal(err) + } + if got != "2026-01-02T04:00:00Z" { + t.Errorf("keep returned %q, want the control plane's deadline", got) + } +} + +// When the control plane's reply omits the deadline — one that predates keep +// echoing the value back — the keep still succeeds and returns the requested +// deadline, so a caller always has something to show. +func TestRemoteNodeKeepFallsBackToTheRequestedDeadline(t *testing.T) { + stubAWSCreds(t) + srv := remoteControlServer(t, `{}`, http.StatusOK) + cfg := remote.Config{StartURL: srv.URL, StopURL: srv.URL, UpdateURL: srv.URL, Region: "us-east-1"} + node, _ := NewRemoteNode("env", cfg) + before := time.Now() + got, err := node.(Keeper).Keep(context.Background(), 4*time.Hour) + if err != nil { + t.Fatal(err) + } + deadline, perr := time.Parse(time.RFC3339, got) + if perr != nil { + t.Fatalf("fallback deadline is not RFC 3339: %q", got) + } + want := before.Add(4 * time.Hour) + if d := deadline.Sub(want); d < -5*time.Second || d > 5*time.Second { + t.Errorf("fallback deadline = %s, want the requested now+4h (%s)", deadline, want) + } +} + +// A keep on an environment whose config has no update URL is a configuration +// error before any call, naming the fix — it is not a call that fails part-way. +func TestRemoteNodeKeepWithoutAnUpdateURL(t *testing.T) { + stubAWSCreds(t) + cfg := remote.Config{StartURL: "http://x", StopURL: "http://x", Region: "us-east-1"} + node, _ := NewRemoteNode("env", cfg) + if _, err := node.(Keeper).Keep(context.Background(), time.Hour); err == nil || + !strings.Contains(err.Error(), "no update_url") { + t.Errorf("expected a no-update-url error, got %v", err) + } +} + +// The keep capability is exactly one of the remote node's: a local daemon node +// has no retention tag to set, so it does not implement Keeper. A caller (the +// dashboard) relies on this boundary to decide whether to offer a keep at all. +func TestKeeperIsRemoteOnly(t *testing.T) { + rn, _ := NewRemoteNode("env", remote.Config{StartURL: "http://x", StopURL: "http://x", Region: "r"}) + if _, ok := rn.(Keeper); !ok { + t.Error("a remote node should implement Keeper") + } + dn := &daemonNode{name: "dev", client: &Client{BaseURL: "http://127.0.0.1:1", Token: "t"}} + if _, ok := any(dn).(Keeper); ok { + t.Error("a local daemon node should not implement Keeper") + } +} + // A remote node drives start, stop and metrics over its control plane exactly // like a node would, mapping each reply onto the node's types. func TestRemoteNodeStartStopMetricsOverTheControlPlane(t *testing.T) { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index be14a88e..860c0bfc 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -52,6 +52,13 @@ type Stats struct { // no engine is running, its runner has no known health-check convention, // or this daemon predates the check. Ready string `json:"ready,omitempty"` + // RetainUntil is the environment's retention deadline, RFC 3339: the idle + // sweep will not terminate the instance before it. It is a property of the + // cloud instance, not the engine, so it is empty for local daemon nodes and + // for remote environments without an update URL. Empty once the deadline + // has passed, because the stats reply drops it there — a past tag keeps + // nothing. Formatters omit the line when it is empty. + RetainUntil string `json:"retainUntil,omitempty"` } // TokenStats holds per-engine token/request counters from /metrics. diff --git a/internal/remote/remote.go b/internal/remote/remote.go index 1da21cb0..189150a4 100644 --- a/internal/remote/remote.go +++ b/internal/remote/remote.go @@ -745,6 +745,11 @@ type StatsResponse struct { // the daemon's /v1/status by the stats Lambda. Empty when the daemon was // unreachable or the control plane predates this. Version string `json:"version"` + // RetainUntil is the instance's retention deadline, RFC 3339, read from the + // Retain-Until tag by the stats Lambda. Empty when the instance has no tag + // or its deadline has already passed — the reply carries it only while it + // still keeps the instance alive — and for control planes that predate keep. + RetainUntil string `json:"retainUntil"` } // The stat sub-types are aliases into internal/metrics, their canonical home diff --git a/internal/remote/remote_test.go b/internal/remote/remote_test.go index 964675ed..c896f6b3 100644 --- a/internal/remote/remote_test.go +++ b/internal/remote/remote_test.go @@ -966,7 +966,8 @@ func TestStats_Success(t *testing.T) { "temperature": 72 }], "cpu": {"utilization": 23.5}, - "memory": {"total": 17179869184, "used": 4294967296} + "memory": {"total": 17179869184, "used": 4294967296}, + "retainUntil": "2026-01-02T04:00:00Z" }`)) })) defer server.Close() @@ -979,6 +980,9 @@ func TestStats_Success(t *testing.T) { if resp.Environment != "dev" || resp.State != "running" { t.Errorf("unexpected response: %+v", resp) } + if resp.RetainUntil != "2026-01-02T04:00:00Z" { + t.Errorf("retainUntil not decoded: %q", resp.RetainUntil) + } if resp.Tokens == nil || resp.Tokens.Requests != 342 { t.Errorf("unexpected tokens: %+v", resp.Tokens) } @@ -1014,6 +1018,10 @@ func TestStats_Stopped(t *testing.T) { if resp.State != "stopped" || resp.Tokens != nil || len(resp.GPUs) != 0 { t.Errorf("stopped instance should have no metrics: %+v", resp) } + // A reply without the field leaves it empty, not a zero time. + if resp.RetainUntil != "" { + t.Errorf("no retainUntil in the reply should decode empty, got %q", resp.RetainUntil) + } } func TestStats_WithEnvironment(t *testing.T) { diff --git a/openspec/changes/dashboard-keep/.openspec.yaml b/openspec/changes/dashboard-keep/.openspec.yaml new file mode 100644 index 00000000..1a62d62b --- /dev/null +++ b/openspec/changes/dashboard-keep/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-06 diff --git a/openspec/changes/dashboard-keep/design.md b/openspec/changes/dashboard-keep/design.md new file mode 100644 index 00000000..4c935e4d --- /dev/null +++ b/openspec/changes/dashboard-keep/design.md @@ -0,0 +1,183 @@ +# Design: dashboard keep + +## Context + +See proposal.md for the motivation (issues #158 and #157). The state of the +code the change lands in: + +- The dashboard (`cmd/spinloop`, roughly 1,500 lines of model and render) is + Bubble Tea with every widget hand-rolled — the spinner, the key-help footer, + the grid scrolling — and no Bubbles component. Start and stop run through a + per-node action value (`dashAction`): the call runs in its own goroutine, + reports progress onto the node's tile, and on completion the footer shows + the one-shot outcome and the node is re-read immediately. +- Keep already works end to end for the one-shot CLI: `remote.Keep` sends + `set-keep` to the update Lambda, which sets the `Retain-Until` EC2 tag, and + the idle sweep honours it. The fleet's `remoteNode` wraps the same + `remote.Config` and already answers status, metrics, start, stop and logs + through it. +- The dashboard refreshes remote environments through the stats Lambda + (the metrics call), whose reply does not today carry the deadline. The + Lambda already parses the tag into its instance info; it does not send it. +- Nodes advertise partial capabilities through optional interfaces that the + dashboard type-asserts (a start that reports progress), rather than through + methods every node must answer. + +## Goals / Non-Goals + +**Goals:** + +- Keep settable from the grid and the detail screen for remote environments, + with a duration the operator types. +- The deadline visible on the tile and the detail screen, from the normal + refresh, worded the same as every one-shot surface that reads the same + stats reply. +- No new dependencies; the dashboard's hand-rolled, terminal-free test style + unchanged. +- A control plane that predates the reply field reads back cleanly and shows + no line. + +**Non-Goals:** + +- Clearing or shortening a keep from the dashboard: setting a new deadline + overwrites, and the sweep takes over once a deadline passes. +- Keep for local daemon nodes: there is no idle sweep on a daemon, so no + deadline to set. +- Changing how `remote status` reports the tag: it keeps reporting the tag's + value as it does today, including a passed one. +- Any interactive account of the sweep itself (when it next runs, why it + skipped an instance). + +## Decisions + +### 1. A hand-rolled duration field, no Bubbles + +The prompt is one state flag and one buffer string on the dashboard model, +driven from the model's existing key switch: printable runes append, backspace +deletes, enter confirms, escape cancels. A duration is a handful of characters +from a fixed set (digits and `h`, `m`, `s`), typed left to right, so a cursor +model, selection, and paste handling would be machinery for input that never +arrives. This matches how the dashboard already owns its spinner, key help and +scrolling, and the state tests without a terminal exactly like the rest of the +model. + +Alternative considered: `bubbles/textinput`. It is the first Bubbles use in a +deliberately hand-rolled TUI, it is a general text editor where a duration +needs four keys, and styling it into the one-accent-chrome footer would cost +most of writing the field. + +### 2. A capability interface, not a Node method + +Keep rides a new optional capability interface in the style of the existing +progress-reporting start, implemented only by the remote node. The dashboard +type-asserts for it, as it already does for the progress start, and the key +help and the key handler gate on the same assertion. + +Alternatives considered: adding `Keep` to the node interface would force the +local daemon node to answer a method it can never honestly answer; gating on +the fleet file's node kind in the dashboard would move a node-kind decision +into the screen layer and around the node abstraction. + +### 3. Keep takes a duration and returns the deadline + +The capability method takes a duration and returns the deadline the control +plane set. Computing now-plus-duration inside the node keeps "now" in one +place for every keeper, and the reply's value — not the caller's arithmetic — +is what the footer line and the panel show. + +### 4. The deadline rides the stats reply, only while active + +The stats Lambda includes the parsed tag in its reply only while it is a time +in the future, and the client maps it across the existing path: the stats +reply type, the node's stats mapping, and the shared stats shape, as an +omitted-when-absent field. The "is it still active" judgement sits in the +control plane, once, instead of in every client that reads the reply; a passed +tag is no active retention, so nothing downstream can draw a deadline the +sweep has already moved past. + +Alternatives considered: the dashboard issuing a status read alongside its +metrics read for remote environments doubles the signed control-plane calls +per environment per minute for one line; filtering passed deadlines on the +client side leaves a stale deadline on the wire for every other reader and +puts a clock comparison in each of them. + +The field is additive: an environment whose stats Lambda has not been +redeployed simply omits it, and the dashboard shows no line there. + +### 5. One shared line beside the last-active figure + +The deadline renders as a line in the shared bar-format body, next to the +last-active line that body already prints: the dashboard tile, the one-shot +fleet metrics, and the one-shot remote metrics all call that body, so they +cannot word the same read differently. The table format prints the line when +the read carries it, and the json format carries the field on its own. The +line is omitted wherever the read has no deadline, the same rule the +last-active figure follows for a figure it lacks. + +### 6. No second confirmation; not abortable + +The prompt is the confirmation: the operator sees the duration it will set, +and choosing to send it is the commitment. A stop asks because it ends +something; a keep overwrites a deadline and ends nothing. The abort key exists +to end a wait that has no deadline of its own — a cold cloud wake; a keep is +one fast signed call, so abort drives nothing on it, as it does on a stop. + +### 7. The prompt state mirrors the stop confirmation + +The prompt is handled in the model's update before the grid/detail dispatch, +the same position the stop confirmation occupies, so it works from both +screens and navigation, selection and refreshes all stand still while it is +open. Confirming an entry that does not parse as a positive duration leaves +the prompt open and shows the parse reason in the footer's hint slot — the +operator keeps their entry and corrects it in place. Escape cancels; quit and +interrupt cancel the prompt and leave the dashboard, exactly as the stop +confirmation does. + +The prompt opens pre-filled with `4h`, the duration the one-shot command's +help already advertises as its example. The common overnight keep is then the +key and a confirm, and a mistaken confirm still does what the footer showed. + +### 8. The key help names keep only where it does something + +The footer's key help drops the keep entry for a node that does not support +retention or has an action in flight — the same mechanism that already drops +the abort entry where nothing is abortable. + +### 9. The action reuses the existing in-flight machinery + +Keep runs through the per-node action value with its own verb: the spinner and +elapsed time on the tile, one action per node, the node re-read immediately on +completion — which is what brings the new deadline onto the panel at the +node's next round rather than waiting out its full cadence. The action's +completion message carries the deadline so the footer line can report the +value the control plane set; start and stop leave it empty. + +## Risks / Trade-offs + +- [A pre-filled `4h` means a stray confirm sets four hours of retention] → + the buffer is visible in the footer before the confirm; a keep overwrites + and the sweep resumes when the deadline passes, so a mistaken one is + correctable and bounded; the default is the CLI's own example value, not a + larger one. +- [The Lambda change reaches environments only as each deployment updates] → + the field is additive and the keep action itself reports its deadline on + the status line from its own reply, so the feature is complete before the + panel line appears; the line then lands per environment as deployments + update. +- [A stopped instance keeps its tag across the stop] → intended: the sweep + must not terminate a retained stopped instance, and the line on a stopped + tile is exactly the protection the operator set. +- [The hand-rolled field has no cursor motion] → a duration is typed left to + right and is at most a handful of characters; backspace and retype is + cheaper than the cursor keys an editor would bring. + +## Migration Plan + +- The Go side (client, capability, dashboard, shared line) and the stats + Lambda change ship together; nothing on the Go side requires the Lambda + change to function — it only makes the panel line appear. +- The Lambda change is a routine deployment of the remote project; no data + migration, no client gating: older clients ignore the extra field and newer + clients omit the line where the field is absent. +- Rolling back the CLI/dashboard removes the key; the extra reply field is + inert to older clients. diff --git a/openspec/changes/dashboard-keep/proposal.md b/openspec/changes/dashboard-keep/proposal.md new file mode 100644 index 00000000..5a27b62b --- /dev/null +++ b/openspec/changes/dashboard-keep/proposal.md @@ -0,0 +1,66 @@ +# Dashboard keep + +## Why + +The dashboard can start and stop a fleet's nodes, but the one fact an operator +needs while watching a remote environment — until when the cloud's idle sweep +will leave it alone — is neither settable nor visible there. Keeping a node +alive (overnight debugging, a long-running task) means leaving the dashboard +and running `spinloop remote keep` one-shot, and the deadline it sets then +appears nowhere in the view the operator is actually watching. Two open issues +ask for the two halves: #158, set a user-specified keep period from the +dashboard, and #157, show the keep-until time on the node's detail screen. + +## What Changes + +- The dashboard gains a keep action on the selected node: a key opens a + duration prompt (a hand-rolled single-line field in the existing model — no + new dependencies), and confirming sends the node's retention deadline to now + plus the entered duration, through the same control-plane path + `spinloop remote keep` already uses. +- Keep is offered only where it can work: `kind: remote` environments, through + a node capability in the style of the existing progress-reporting start. The + key help hides the key for local daemon nodes, exactly as it already hides + the abort key where it would do nothing. +- The fleet's read of a remote environment carries the retention deadline: the + stats Lambda already parses the instance's Retain-Until tag and now includes + it in its reply, and the client maps it onto the shared stats shape. Older + control planes omit the field, and the deadline line simply does not appear + — the same graceful degradation the stats reply's other relayed facts use. +- The dashboard's node panel shows the deadline — the tile and the full-screen + detail screen draw the same lines, so both show it — whenever a read carries + one, and omits it when a node has no active retention. The line is the + shared one the bar format prints, so the one-shot `fleet metrics` and + `remote metrics` render it the same way. +- Keep proceeds without confirmation — it overwrites the deadline and ends + nothing — and is not abortable: it is a single fast call, like a stop. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `fleet-client`: the dashboard gains a keep action (the duration prompt, its + validation, its in-flight and outcome handling, and the key help), and the + node panel — tile and detail screen — shows the retention deadline a read + carries. +- `remote-stats`: the stats Lambda's reply carries the instance's retention + deadline when the Retain-Until tag is present, so every surface that reads + the environment's stats can show it. + +## Impact + +- `internal/fleet`: a new capability interface, implemented by the remote node + over the existing `remote.Keep` call; local daemon nodes do not implement it. +- `internal/remote`: the stats reply type gains the deadline field; the node's + stats mapping carries it across. +- `internal/metrics`: the shared stats shape gains the deadline field. +- `remote/lambda/stats`: the reply includes the tag the Lambda already parses. +- `cmd/spinloop`: the dashboard model (prompt state and keys, the action), the + shared line renderer, and the command's help text. +- `docs/commands/fleet.md`: the dashboard's keys. +- No new dependencies. The control-plane change is additive: an environment on + an older deployment reads back without the field and shows no line. diff --git a/openspec/changes/dashboard-keep/specs/fleet-client/spec.md b/openspec/changes/dashboard-keep/specs/fleet-client/spec.md new file mode 100644 index 00000000..15458f77 --- /dev/null +++ b/openspec/changes/dashboard-keep/specs/fleet-client/spec.md @@ -0,0 +1,170 @@ +## ADDED Requirements + +### Requirement: The dashboard keeps the selected node + +The dashboard SHALL let the operator set the retention deadline of the node +currently selected, from the keyboard, through the same node operation the +one-shot keep command uses. Keep applies to a node's instance — the time until +which the cloud's idle sweep leaves it alone — so it SHALL be offered only for +nodes that support retention, namely the fleet's remote environments; a node +without retention support SHALL take no keep. + +The keep key SHALL open a duration prompt rather than send anything. The +prompt SHALL name the node it will apply to, show the duration as the operator +has it, and name its confirm and cancel keys. It SHALL open pre-filled with a +default duration, so that the common case is the key and a confirm, and the +confirm SHALL send the duration as shown, whatever the operator has changed it +to. The duration SHALL be expressed in the same syntax the one-shot keep +command accepts. + +A confirm of an entry that is not a positive duration SHALL send nothing: the +prompt SHALL stay open and show why the entry was rejected, so the operator can +correct it and confirm again. A cancel SHALL close the prompt and send nothing. +Once the prompt is open, the navigation and selection keys SHALL drive +nothing: the prompt takes the keyboard until it is confirmed or cancelled. + +Keep SHALL not ask for a second confirmation after the prompt: the operator +has typed, or deliberately kept, the duration and chosen to send it, and a +keep overwrites a deadline rather than ending anything. A keep in flight SHALL +be shown on the node's panel like any action — its verb and how long it has +been running — and SHALL NOT be abortable: it is one fast call with no +open-ended wait, and abandoning the wait on it would leave the operator unsure +whether the deadline was set. + +A node that already has an action in flight SHALL take no keep; the dashboard +SHALL say it is still working and drive nothing. The key help SHALL name the +keep key only for the node it describes when that node supports retention and +has no action in flight, so the operator is never invited to press a key that +would do nothing there. + +The outcome of a keep SHALL be shown on the status line: on success the +deadline the control plane set, on failure the control plane's own reason — a +deployment that predates keep support, an environment with no instance to +retain — and a failed keep SHALL NOT close the dashboard. What the keep +changes — the deadline the node now carries — SHALL appear on the node's panel +through the normal refresh, without the operator asking for it. + +The node's detail view SHALL offer keep on the node it shows, through the same +prompt and the same rules the grid applies to the selected node. + +#### Scenario: Keep opens a prompt, it does not send + +- **WHEN** the operator selects a remote environment with no action in flight + and issues keep +- **THEN** a duration prompt opens naming the node, pre-filled with a default + duration, and nothing has been sent + +#### Scenario: A confirmed keep sets the deadline + +- **WHEN** the operator changes the prompt's duration and confirms it +- **THEN** the node is kept until now plus the confirmed duration and the + status line shows the deadline the control plane set + +#### Scenario: A pre-filled keep is one key and a confirm + +- **WHEN** the operator issues keep on a remote environment and confirms the + prompt without changing its duration +- **THEN** the node is kept until now plus the default duration + +#### Scenario: A bad duration sends nothing + +- **WHEN** the operator confirms an entry that is not a positive duration +- **THEN** nothing is sent, the prompt stays open, and it shows why the entry + was rejected + +#### Scenario: Cancelling sends nothing + +- **WHEN** the operator opens the keep prompt and cancels it +- **THEN** the prompt closes and the node's deadline is unchanged + +#### Scenario: The prompt takes the keyboard + +- **WHEN** the keep prompt is open and the operator presses a navigation key +- **THEN** the selection does not move and the prompt stays open + +#### Scenario: A local node takes no keep + +- **WHEN** the operator issues keep on a local daemon node +- **THEN** the dashboard drives nothing: a local daemon has no idle sweep to + hold off, so there is no deadline to set + +#### Scenario: The key help hides keep where it would do nothing + +- **WHEN** the node under the cursor is a local daemon node, or has an action + in flight +- **THEN** the key help does not name the keep key +- **WHEN** that node is a remote environment with no action in flight +- **THEN** the key help names the keep key + +#### Scenario: A busy node is not kept again + +- **WHEN** the operator issues keep on a node whose start, stop, or keep is + still in flight +- **THEN** the dashboard drives nothing and says the node is still working + +#### Scenario: A keep in flight cannot be aborted + +- **WHEN** the operator issues the abort on a node whose keep is in flight +- **THEN** the abort drives nothing: the keep keeps running, the panel keeps + showing it, and its outcome lands on the status line when the call returns + +#### Scenario: A keep that fails keeps the dashboard open + +- **WHEN** a keep is sent to an environment whose deployment predates keep + support, or to one with no instance to retain +- **THEN** the failure is shown on the status line with the control plane's + own reason and the dashboard keeps running with its refreshes + +#### Scenario: Keep from the detail view + +- **WHEN** the operator issues keep from the detail view of a remote + environment +- **THEN** the same prompt opens with the same rules, and its outcome is + shown as the grid would show it + +### Requirement: The dashboard panels show the retention deadline + +A node's panel SHALL show the node's retention deadline — the time until which +the idle sweep leaves the node alone — whenever the node's last answered +refresh carries one, worded the same way the one-shot surfaces report it, and +SHALL omit the line when the answer carries none: a node with no active +retention, a node without retention at all, or a read from a control plane +that predates the field. The deadline SHALL appear on the tile and on the +node's detail screen, which draw the same lines, so the two cannot show the +same read differently, and it SHALL sit among the panel's other time facts +rather than displacing the node's state or what it serves. + +A deadline that has passed carries no active retention: a refresh taken after +the deadline SHALL not show the line, and a panel SHALL NOT draw a deadline +the sweep has already moved past as though the node were still held. + +#### Scenario: A retained node shows its deadline + +- **WHEN** a remote environment's refresh answer carries a retention deadline + in the future +- **THEN** its tile shows the deadline, and its detail screen shows the same + line + +#### Scenario: A node with no retention shows no line + +- **WHEN** a node's refresh answer carries no retention deadline +- **THEN** its panel shows no deadline line and renders the rest exactly as + before + +#### Scenario: An older control plane degrades to no line + +- **WHEN** a remote environment's control plane predates the deadline in its + stats reply +- **THEN** its panel shows no deadline line and no error, and the rest of the + panel is unaffected + +#### Scenario: A passed deadline disappears + +- **WHEN** a node's retention deadline has passed and a later refresh answers +- **THEN** the panel no longer shows the deadline + +#### Scenario: Tile and detail agree + +- **WHEN** the operator opens the detail screen of a retained node +- **THEN** the detail screen's deadline line is the same line, worded the + same, the tile draws for the same read diff --git a/openspec/changes/dashboard-keep/specs/remote-stats/spec.md b/openspec/changes/dashboard-keep/specs/remote-stats/spec.md new file mode 100644 index 00000000..c87dbcba --- /dev/null +++ b/openspec/changes/dashboard-keep/specs/remote-stats/spec.md @@ -0,0 +1,58 @@ +## ADDED Requirements + +### Requirement: The stats reply carries the retention deadline + +The stats Lambda's reply SHALL carry the instance's retention deadline — the +time until which the idle sweep leaves the instance alone, parsed from the +instance's Retain-Until tag, which the Lambda already reads — when that tag is +a time in the future, and SHALL omit it when the tag is absent or a time that +has passed. The deadline is a fact the control plane holds about the instance, +not one the on-instance daemon reports: it SHALL be present for a stopped +instance the sweep would otherwise terminate, and absent for an environment +with no instance at all. + +A control plane that predates the field SHALL simply omit it, with no error: +every reader of the reply treats an absent deadline as "no active retention" +rather than as a failure. + +The client SHALL map the deadline onto the shared stats shape every fleet and +remote stats surface reads from, so a dashboard panel, a one-shot fleet report, +and a one-shot remote report cannot word the same read differently. + +The deadline SHALL be reported beside the report's other time facts — the +last-active figure — in every format the command supports, and omitted in +every format when the read carries none, following the same omission rule the +last-active figure uses for a figure it does not have. + +#### Scenario: A retained instance's stats carry the deadline + +- **WHEN** the user reads the stats of an environment whose instance carries a + Retain-Until tag at a time in the future +- **THEN** the reply carries that deadline + +#### Scenario: A passed tag carries no deadline + +- **WHEN** the user reads the stats of an instance whose Retain-Until tag is a + time that has passed +- **THEN** the reply carries no deadline: a passed tag holds no active + retention + +#### Scenario: An untagged or instance-less environment carries no deadline + +- **WHEN** the user reads the stats of an instance with no Retain-Until tag, + or of an environment with no instance +- **THEN** the reply carries no deadline + +#### Scenario: An older control plane is read without error + +- **WHEN** the user reads the stats of an environment whose control plane + predates the field +- **THEN** the read succeeds, the deadline is simply absent, and the rest of + the report renders as before + +#### Scenario: Every format carries the deadline + +- **WHEN** the user reads the stats of a retained environment with `bar`, + `table`, or `json` output +- **THEN** each output carries the deadline in its own idiom, beside the + last-active figure, and each omits it when the read carries none diff --git a/openspec/changes/dashboard-keep/tasks.md b/openspec/changes/dashboard-keep/tasks.md new file mode 100644 index 00000000..994b7324 --- /dev/null +++ b/openspec/changes/dashboard-keep/tasks.md @@ -0,0 +1,98 @@ +# Tasks: dashboard keep + +## 1. The deadline rides the stats read (Go) + +- [x] 1.1 Add a `RetainUntil` field to `metrics.Stats`, omitted when empty, + beside `LastActiveAt` and `Ready` — verify with `go build ./...` and the + mapping test in 1.3. +- [x] 1.2 Add a `RetainUntil` field to `remote.StatsResponse`, matching the + Lambda's JSON — verify with a decode test in `internal/remote` that a + reply carrying the field lands on it and one without leaves it empty. +- [x] 1.3 Carry the deadline across the node's stats mapping + (`statsFromRemote`) — verify with `internal/fleet` tests: a stats reply + with a deadline ends up on `metrics.Stats`, and one without leaves it + empty. + +## 2. The stats Lambda reports the deadline while it is active + +- [x] 2.1 Include the instance's parsed Retain-Until tag in the stats reply + only while it is a time in the future — verify with vitest cases in + `remote/test` (a future tag is present, a passed tag is absent, an + untagged instance is absent) and `pnpm test` passing in `remote/`. + +## 3. A node capability for keep + +- [x] 3.1 Add the keep capability interface — take a duration, return the + deadline the control plane set — beside the progress-reporting start + interface in `internal/fleet` — verify with `go build ./...`. +- [x] 3.2 Implement the capability on the remote node over the existing + `remote.Keep` call — verify with `internal/fleet` tests: a keep sends + the set-keep command with the now-plus-duration deadline and returns the + control plane's value, and a config without the update URL surfaces the + control plane's named error. +- [x] 3.3 Confirm the local daemon node does not implement the capability — + verify by a test asserting the dashboard's type assertion fails for it + (the same test file that covers the progress-start assertion). + +## 4. The shared deadline line + +- [x] 4.1 Add the retain-until line to the shared bar-format body, beside the + last-active line, omitted when the read carries none — verify with + render tests: the line's wording when present, its omission when the + field is empty. +- [x] 4.2 Wire the line into the one-shot `fleet metrics` bar format, the + `remote metrics` bar format, and the `remote metrics` table format when + present — verify with the existing render tests for those surfaces + extended for the line. + +## 5. The dashboard keep action + +- [x] 5.1 Add the prompt state to the dashboard model — a flag and a duration + buffer — opened by the keep key from the grid and the detail view, + pre-filled with `4h`, handled before the grid/detail dispatch so + navigation, selection and refreshes stand still while it is open — + verify with model tests: the prompt opens on the key for a remote node, + drives nothing for a local node, and navigation keys do nothing while it + is open. +- [x] 5.2 Handle the prompt's keys — append, backspace, confirm, cancel — and + leave the prompt open with the parse reason in the footer when a confirm + is not a positive duration — verify with model tests over valid entries + (`4h`, `90m`, `1h30m`) and invalid ones (`4hours`, empty). +- [x] 5.3 Run a confirmed keep through the per-node action machinery — verb + `keep`, the tile's spinner and elapsed time, one action per node, the + node re-read immediately on completion — and carry the deadline on the + action's completion message — verify with model tests: a keep in flight + shows on the tile, a busy node takes no second keep, and completion + clears the action and schedules the re-read. +- [x] 5.4 Render the footer outcome for a finished keep — success shows the + deadline the control plane set, failure shows its reason, the dashboard + stays open — verify with tests over the line's wording, including the + no-update-URL failure. +- [x] 5.5 Make the abort key drive nothing on a keep in flight — verify with a + model test alongside the existing stop-in-flight case. +- [x] 5.6 Show the keep entry in the key help only for a node that supports + keep and has no action in flight, on both the grid and the detail + footer — verify with tests: a local node hides it, a busy remote node + hides it, an idle remote node shows it. +- [x] 5.7 Draw the deadline line on the tile and the detail screen from the + read, omitted when the read carries none — verify with tile render + tests: a read with a deadline shows the line on the tile, a read without + one shows no line, and the detail screen draws the same line. +- [x] 5.8 Update the `fleet dashboard` command's long help to name the keep + key — verify with `go run ./cmd/spinloop fleet dashboard --help` + printing the key alongside the existing ones. + +## 6. Docs + +- [x] 6.1 Update `docs/commands/fleet.md` for the keep key, the duration + prompt, and the deadline line on panels — verify by reading the section + against the implemented keys and lines. + +## 7. Final checks + +- [x] 7.1 `go test ./... -cover` at 80% total or above, `go vet ./...`, and + `gofmt -l` clean — verify all three pass. +- [x] 7.2 The `remote/` suite (`pnpm test`) passes with the new stats reply + field — verify the run is green. +- [x] 7.3 `openspec validate dashboard-keep` passes — verify the command + reports the change valid. diff --git a/remote/lambda/shared/stats.ts b/remote/lambda/shared/stats.ts index 2b6de34f..75232e29 100644 --- a/remote/lambda/shared/stats.ts +++ b/remote/lambda/shared/stats.ts @@ -109,4 +109,13 @@ export interface StatsResult { * the field — the formatters simply omit the line. */ version?: string; + /** + * The instance's retention deadline, RFC 3339: the idle sweep will not + * terminate it before this. Carried only while the instance's Retain-Until + * tag is still a time in the future — a passed deadline keeps nothing, so it + * is dropped here, and an untagged instance has none. It is a property of the + * cloud instance, not the engine, so it is present whatever the engine's + * state — a stopped, retained instance still reports it. + */ + retainUntil?: string; } diff --git a/remote/lambda/stats/index.ts b/remote/lambda/stats/index.ts index 15f7eeee..64c0c943 100644 --- a/remote/lambda/stats/index.ts +++ b/remote/lambda/stats/index.ts @@ -2,6 +2,7 @@ import type { LambdaFunctionURLEvent, LambdaFunctionURLResult } from 'aws-lambda import { errorName, findManagedInstance, + type InstanceInfo, readDeployConfig, requireEnv, runShellCommand, @@ -29,6 +30,23 @@ function envFilter(env: string) { return [{ Name: `tag:${ENV_TAG_KEY}`, Values: [env] }]; } +/** + * The retention deadline to carry in a reply, or nothing: present only while + * the instance's Retain-Until tag is still a time in the future. A passed + * deadline keeps nothing, so the reply drops it there; an untagged instance and + * a not-yet-deployed environment have none. The judgement lives here, not in the + * formatters, so a reader of the reply need not re-check the clock — and it + * applies to every reply branch, since a stopped, retained instance still + * reports its deadline. + */ +function retainUntilIfActive(instance: InstanceInfo | null): string | undefined { + const until = instance?.retainUntil; + if (until && until.getTime() > Date.now()) { + return until.toISOString(); + } + return undefined; +} + /** * The stats Lambda called by `spinloop remote metrics`. The control plane * contributes what only it knows — environment, instance id/type, uptime @@ -65,6 +83,9 @@ export async function handler(event: LambdaFunctionURLEvent): Promise { expect(DAEMON_STATUS_CMD).toContain(DAEMON_UNREACHABLE); }); }); + +// The stats Lambda: reports an environment's instance and engine metrics, and — +// while its Retain-Until tag is still a time in the future — the retention +// deadline itself, in every reply branch. + +const LAMBDA_ENV = { + TAG_KEY: 'cloud-vm-llm:managed', + TAG_VALUE: 'true', +}; + +const findManagedInstance = vi.fn(); +const readDeployConfig = vi.fn(); +const runShellCommand = vi.fn(); + +vi.mock('../lambda/shared/aws', async (importOriginal) => ({ + ...(await importOriginal()), + findManagedInstance: (...args: unknown[]) => findManagedInstance(...args), + readDeployConfig: (...args: unknown[]) => readDeployConfig(...args), + runShellCommand: (...args: unknown[]) => runShellCommand(...args), +})); + +let handler: (event: LambdaFunctionURLEvent) => Promise; + +beforeAll(async () => { + Object.assign(process.env, LAMBDA_ENV); + ({ handler } = await import('../lambda/stats/index')); +}); + +function bodyOf(result: unknown): Record { + return JSON.parse((result as { statusCode: number; body: string }).body); +} + +function statusOf(result: unknown): number { + return (result as { statusCode: number }).statusCode; +} + +function statsEvent(query: Record) { + return { + queryStringParameters: query, + } as unknown as LambdaFunctionURLEvent; +} + +// The engine scrape is not what these cases assert on: the daemon answers +// nothing, so the reply carries no engine figures — only the control plane's +// own, which is where retainUntil lives. +beforeEach(() => { + vi.clearAllMocks(); + readDeployConfig.mockResolvedValue({ runner: 'llamacpp', modelId: 'org/m' }); + runShellCommand.mockResolvedValue({ status: 'Failed', stdout: '' }); +}); + +const futureTag = '2030-01-02T04:00:00.000Z'; +const pastTag = '2020-01-02T04:00:00.000Z'; + +describe('retainUntil', () => { + it('is present on a running instance whose tag is in the future', async () => { + findManagedInstance.mockResolvedValue({ + instanceId: 'i-run', + state: 'running', + retainUntil: new Date(futureTag), + }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body.state).toBe('running'); + expect(body.retainUntil).toBe(futureTag); + }); + + it('is present on a stopped instance whose tag is in the future', async () => { + findManagedInstance.mockResolvedValue({ + instanceId: 'i-stopped', + state: 'stopped', + retainUntil: new Date(futureTag), + }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body.state).toBe('stopped'); + expect(body.retainUntil).toBe(futureTag); + }); + + it('is absent when the tag has already passed', async () => { + findManagedInstance.mockResolvedValue({ + instanceId: 'i-run', + state: 'running', + retainUntil: new Date(pastTag), + }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body).not.toHaveProperty('retainUntil'); + }); + + it('is absent for an untagged instance', async () => { + findManagedInstance.mockResolvedValue({ instanceId: 'i-run', state: 'running' }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body).not.toHaveProperty('retainUntil'); + }); + + it('is absent for an undeployed environment (no instance at all)', async () => { + findManagedInstance.mockResolvedValue(null); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body.state).toBe('undeployed'); + expect(body).not.toHaveProperty('retainUntil'); + }); +}); From bda52190a0f7983405e78bee773066cf8243fa9a Mon Sep 17 00:00:00 2001 From: Pete Cornish Date: Sun, 6 Sep 2026 14:31:36 +0100 Subject: [PATCH 2/3] refactor(fleet): render the keep relatively on the active line Word the since-last-work figure as "active" rather than "last active" across the tile, detail, one-shot metrics, status views, and routing, and show a kept environment's deadline as a relative figure after it on the same line ("active 2m 5s ago keep for 2h") instead of an absolute "retain until" timestamp. The json format keeps the absolute field for consumers. --- cmd/spinloop/dashboard_keep_test.go | 28 ++- cmd/spinloop/dashboard_render.go | 17 +- cmd/spinloop/fleet.go | 8 +- cmd/spinloop/fleet_dashboard_test.go | 2 +- cmd/spinloop/fleet_test.go | 6 +- cmd/spinloop/last_active_test.go | 42 ++-- cmd/spinloop/metrics_render.go | 103 ++++++--- cmd/spinloop/metrics_render_test.go | 10 +- cmd/spinloop/remote.go | 36 +-- cmd/spinloop/retain_render_test.go | 211 ++++++++++++------ cmd/spinloop/status_render.go | 6 +- cmd/spinloop/status_render_test.go | 2 +- docs/commands/fleet.md | 23 +- docs/commands/remote.md | 2 +- internal/fleet/select.go | 2 +- openspec/changes/dashboard-keep/design.md | 22 +- .../dashboard-keep/specs/remote-stats/spec.md | 13 +- openspec/changes/dashboard-keep/tasks.md | 8 +- 18 files changed, 333 insertions(+), 208 deletions(-) diff --git a/cmd/spinloop/dashboard_keep_test.go b/cmd/spinloop/dashboard_keep_test.go index 471b5033..8d1c7ab5 100644 --- a/cmd/spinloop/dashboard_keep_test.go +++ b/cmd/spinloop/dashboard_keep_test.go @@ -396,9 +396,12 @@ func TestDashKeepHintOnlyWhereItDrivesSomething(t *testing.T) { }) } -// The deadline rides the node's read onto the tile and the detail screen, -// beside the last-active line, and a read without one draws no line. -func TestDashTileAndDetailShowRetainUntil(t *testing.T) { +// The deadline rides the node's read onto the tile and the detail screen, as a +// relative keep after the active figure on the same line, and a read without one +// draws no keep. +func TestDashTileAndDetailShowKeep(t *testing.T) { + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + dashFixNow(t, at) node := &keeperDashNode{f: newFakeDashNode("stopped")} m := keeperModel(node) r := fleet.NodeResult{ @@ -406,30 +409,31 @@ func TestDashTileAndDetailShowRetainUntil(t *testing.T) { Outcome: fleet.OutcomeOK, Metrics: metrics.Stats{ State: "stopped", Runner: "llamacpp", ModelID: "org/m", - RetainUntil: "2030-01-02T04:00:00Z", + LastActiveAt: "2025-12-31T23:58:15Z", IdleSeconds: 125, + RetainUntil: at.Add(2 * time.Hour).UTC().Format(time.RFC3339), }, } m.results[0] = r tile := dashTestTile("env", r, true, dashAction{}) - if !strings.Contains(tile, "retain until 2030-01-02T04:00:00Z") { - t.Errorf("tile missing the retain-until line:\n%s", tile) + if line := aLineContaining(tile, "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("tile did not put the keep on the active line:\n%s", tile) } lines := m.detailNodeLines() found := false for _, l := range lines { - if strings.Contains(l, "retain until 2030-01-02T04:00:00Z") { + if strings.Contains(l, "keep for 2h") { found = true } } if !found { - t.Errorf("detail view missing the retain-until line:\n%s", strings.Join(lines, "\n")) + t.Errorf("detail view missing the keep:\n%s", strings.Join(lines, "\n")) } } // A read without a deadline — a local node, or an unkept or lapsed environment — -// draws no line on the tile. -func TestDashTileOmitsRetainUntilWhenAbsent(t *testing.T) { +// draws no keep on the tile. +func TestDashTileOmitsKeepWhenAbsent(t *testing.T) { r := fleet.NodeResult{ Name: "box", Outcome: fleet.OutcomeOK, @@ -437,7 +441,7 @@ func TestDashTileOmitsRetainUntilWhenAbsent(t *testing.T) { CPU: &metrics.CpuStat{Utilization: 30}}, } tile := dashTestTile("box", r, true, dashAction{}) - if strings.Contains(tile, "retain until") { - t.Errorf("tile invented a deadline:\n%s", tile) + if strings.Contains(tile, "keep for") { + t.Errorf("tile invented a keep the read does not carry:\n%s", tile) } } diff --git a/cmd/spinloop/dashboard_render.go b/cmd/spinloop/dashboard_render.go index 2e4e9de5..da6226c3 100644 --- a/cmd/spinloop/dashboard_render.go +++ b/cmd/spinloop/dashboard_render.go @@ -297,7 +297,7 @@ func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time, if s := r.Metrics.State; s != "" { fmt.Fprintln(&b, dashStateLine(s, r.Metrics)+age) } - dashTileReportBody(&b, r.Metrics, true, gauge, lineW) + dashTileReportBody(&b, r.Metrics, true, gauge, lineW, now) } case r.Outcome == "": fmt.Fprintf(&b, "%s\nwaiting for first refresh…\n", name) @@ -315,7 +315,7 @@ func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time, if !gauge && len(r.Metrics.History) > 0 { resources = true } - dashTileReportBody(&b, r.Metrics, resources, gauge, lineW) + dashTileReportBody(&b, r.Metrics, resources, gauge, lineW, now) } lines := strings.Split(b.String(), "\n") lines = lines[:len(lines)-1] // the trailing newline splits an extra empty piece @@ -433,16 +433,15 @@ func dashStateLine(state string, m metrics.Stats) string { // answer has it. A settled tile gates the resources block on the node being // running; the in-flight tile draws whatever there is, because a boot half // done has some of the facts and not the rest. -func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool, lineW int) { +func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool, lineW int, now time.Time) { if line := dashTileServingLine(m); line != "" { fmt.Fprintln(w, line) } - renderLastActiveIndented(w, m.LastActiveAt, m.IdleSeconds) - // The retention deadline, beside last-active and from the same read: a - // remote environment the operator has kept draws it whatever the engine's - // state, and a read without one — a local node, an unkept or lapsed - // environment — draws nothing. - renderRetainIndented(w, m.RetainUntil) + // The active figure and, for a kept remote environment, the relative keep + // after it — one line, from the same read, whatever the engine's state. A + // read without either — a local node, an unkept or lapsed environment — + // draws nothing. + renderActiveIndented(w, m.LastActiveAt, m.IdleSeconds, m.RetainUntil, now) if resources { if gauge { renderStatGauges(w, m.CPU, m.Memory, m.GPUs) diff --git a/cmd/spinloop/fleet.go b/cmd/spinloop/fleet.go index f464a615..bf65c0ca 100644 --- a/cmd/spinloop/fleet.go +++ b/cmd/spinloop/fleet.go @@ -187,6 +187,7 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string) if format == "json" { return renderFleetMetricsJSON(w, results) } + now := metricsNow() for i, r := range results { if i > 0 { fmt.Fprintln(w) @@ -203,10 +204,9 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string) fmt.Fprintln(w) // Before the continue, for the same reason the remote formats show it // before theirs: a node whose engine has stopped still has a useful - // answer to "when did this last do anything?" — and, for a retained - // remote environment, "until is it kept?". - renderLastActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds) - renderRetainIndented(w, stats.RetainUntil) + // answer to "when did it last do anything?" — and, for a retained + // remote environment, "how long is it kept?". + renderActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds, stats.RetainUntil, now) switch format { case "bar": // No state gate, for the same reason the remote bar format has diff --git a/cmd/spinloop/fleet_dashboard_test.go b/cmd/spinloop/fleet_dashboard_test.go index 8e951fec..70dd868e 100644 --- a/cmd/spinloop/fleet_dashboard_test.go +++ b/cmd/spinloop/fleet_dashboard_test.go @@ -355,7 +355,7 @@ func TestDashTileRunningByteStable(t *testing.T) { want := dashTileExpected([]string{ dashExpectedHeader("up running (up 2h 0m 0s)", dashHealthy), "llamacpp org/qwen:q4", - " last active 12s ago", + " active 12s ago", dashBar("CPU", 42), dashBar("RAM", 30), dashBar("GPU util", 61), diff --git a/cmd/spinloop/fleet_test.go b/cmd/spinloop/fleet_test.go index 45a44a80..52441e73 100644 --- a/cmd/spinloop/fleet_test.go +++ b/cmd/spinloop/fleet_test.go @@ -771,8 +771,8 @@ func TestCmdFleetStatusShowsIdleTime(t *testing.T) { } }) // 125s formats the same way the uptime column does, so the two read alike. - if !strings.Contains(out, "last active 2m 5s ago") { - t.Errorf("last-active time missing or misformatted:\n%s", out) + if !strings.Contains(out, "active 2m 5s ago") { + t.Errorf("active time missing or misformatted:\n%s", out) } } @@ -794,7 +794,7 @@ func TestCmdFleetStatusOmitsIdleWithoutActivity(t *testing.T) { t.Error(err) } }) - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("activity claimed when the daemon recorded none:\n%s", out) } // The rest of the row is unaffected. diff --git a/cmd/spinloop/last_active_test.go b/cmd/spinloop/last_active_test.go index 24def7fe..5838ff47 100644 --- a/cmd/spinloop/last_active_test.go +++ b/cmd/spinloop/last_active_test.go @@ -52,13 +52,13 @@ func TestRemoteMetricsBarShowsLastActive(t *testing.T) { t.Fatalf("cmdRemoteMetrics: %v", err) } }) - if !strings.Contains(out, "last active 2m 5s ago") { - t.Errorf("bar format missing the last-active line:\n%s", out) + if !strings.Contains(out, "active 2m 5s ago") { + t.Errorf("bar format missing the active line:\n%s", out) } // It belongs between the header and the bars: a fact about the endpoint, // read before the utilisation readings rather than among them. header := strings.Index(out, "g6e.xlarge") - active := strings.Index(out, "last active") + active := strings.Index(out, "active") bars := strings.Index(out, "CPU") if !(header < active && active < bars) { t.Errorf("last-active line is not between the header and the bars:\n%s", out) @@ -74,11 +74,11 @@ func TestRemoteMetricsTableShowsLastActive(t *testing.T) { } }) // Padded to the same key column as its neighbours, and beside uptime. - if !strings.Contains(out, "last active: 2m 5s ago") { - t.Errorf("table format missing the last-active row:\n%s", out) + if !strings.Contains(out, "active: 2m 5s ago") { + t.Errorf("table format missing the active row:\n%s", out) } - if strings.Index(out, "uptime:") > strings.Index(out, "last active:") { - t.Errorf("last active should follow uptime:\n%s", out) + if strings.Index(out, "uptime:") > strings.Index(out, "active:") { + t.Errorf("active should follow uptime:\n%s", out) } } @@ -119,8 +119,8 @@ func TestRemoteMetricsStoppedStillShowsLastActive(t *testing.T) { // The bar format indents the line to the bar-label column; the table // format makes it a key-value row. Same fact, each format's own idiom. for format, want := range map[string]string{ - "bar": "last active 1h 0m 0s ago", - "table": "last active: 1h 0m 0s ago", + "bar": "active 1h 0m 0s ago", + "table": "active: 1h 0m 0s ago", } { t.Run(format, func(t *testing.T) { out := captureStdout(t, func() { @@ -149,8 +149,8 @@ func TestLastActiveZeroIdleStillRenders(t *testing.T) { }`) for format, want := range map[string]string{ - "bar": "last active 0s ago", - "table": "last active: 0s ago", + "bar": "active 0s ago", + "table": "active: 0s ago", } { t.Run(format, func(t *testing.T) { out := captureStdout(t, func() { @@ -181,7 +181,7 @@ func TestLastActiveOmittedWithoutATimestamp(t *testing.T) { } }) // No line at all, rather than one implying it has sat unused. - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("%s format invented activity from nothing:\n%s", format, out) } if !strings.Contains(out, "CPU") { @@ -219,8 +219,8 @@ func TestRemoteStatusShowsLastActive(t *testing.T) { t.Fatalf("cmdRemoteStatus: %v", err) } }) - if !strings.Contains(out, "last active: 2m 5s ago") { - t.Errorf("status missing the last-active line:\n%s", out) + if !strings.Contains(out, "active: 2m 5s ago") { + t.Errorf("status missing the active line:\n%s", out) } // The lines it already printed are untouched. for _, want := range []string{"state: running", "healthy: true", "base_url: http://198.51.100.7:8000"} { @@ -238,7 +238,7 @@ func TestRemoteStatusZeroIdleStillRenders(t *testing.T) { t.Fatalf("cmdRemoteStatus: %v", err) } }) - if !strings.Contains(out, "last active: 0s ago") { + if !strings.Contains(out, "active: 0s ago") { t.Errorf("status hid an endpoint that is working right now:\n%s", out) } } @@ -253,7 +253,7 @@ func TestRemoteStatusOmitsLastActiveWhenAbsent(t *testing.T) { t.Fatalf("cmdRemoteStatus: %v", err) } }) - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("status invented activity for a stopped instance:\n%s", out) } if !strings.Contains(out, "state: stopped") { @@ -290,8 +290,8 @@ func TestFleetMetricsShowsLastActive(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if !strings.Contains(out, "last active 2m 5s ago") { - t.Errorf("fleet %s metrics missing the last-active line:\n%s", format, out) + if !strings.Contains(out, "active 2m 5s ago") { + t.Errorf("fleet %s metrics missing the active line:\n%s", format, out) } }) } @@ -341,7 +341,7 @@ func TestFleetMetricsOmitsLastActiveWithoutActivity(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("fleet metrics claimed activity a node never reported:\n%s", out) } } @@ -360,7 +360,7 @@ func TestFleetMetricsStoppedNodeStillShowsLastActive(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if !strings.Contains(out, "last active 10m 0s ago") { - t.Errorf("a stopped node dropped its last-active figure:\n%s", out) + if !strings.Contains(out, "active 10m 0s ago") { + t.Errorf("a stopped node dropped its active figure:\n%s", out) } } diff --git a/cmd/spinloop/metrics_render.go b/cmd/spinloop/metrics_render.go index c4b4c925..b373561e 100644 --- a/cmd/spinloop/metrics_render.go +++ b/cmd/spinloop/metrics_render.go @@ -9,6 +9,7 @@ package main import ( "fmt" "io" + "time" "github.com/spinloop-ai/spinloop/internal/metrics" ) @@ -31,43 +32,89 @@ func lastActiveText(lastActiveAt string, idleSeconds int) string { return formatDuration(idleSeconds) + " ago" } -// renderLastActiveIndented draws the last-active line in the indented block -// the bar format and both fleet formats use, aligned to the bar-label column. -// -// Not a bar itself: an elapsed time has no ceiling to fill against, and a bar -// would imply one. -func renderLastActiveIndented(w io.Writer, lastActiveAt string, idleSeconds int) { - if text := lastActiveText(lastActiveAt, idleSeconds); text != "" { - fmt.Fprintf(w, " %-9s %s\n", "last active", text) +// metricsNow returns the current time for the one-shot metrics renderers. A +// variable so a test can fix the keep duration a read renders. The dashboard +// keeps its own clock (dashNow) and passes its now down instead. +var metricsNow = time.Now + +// keepText is the relative retention figure — "keep for 2h" — drawn after the +// active figure on the same line. It is "" when the read carries no deadline or +// the deadline is not in the future: the control plane is the home of the "is +// it still active?" judgement (it drops the field there), so this only formats +// the value it is given, never re-checks the clock against a tag. +func keepText(retainUntil string, now time.Time) string { + if retainUntil == "" { + return "" + } + deadline, err := time.Parse(time.RFC3339, retainUntil) + if err != nil { + return "" } + d := deadline.Sub(now) + if d <= 0 { + return "" + } + return "keep for " + formatKeepDuration(d) } -// renderLastActiveKeyValue draws the same fact as a row of the table format, -// padded to the key column its neighbours use. -func renderLastActiveKeyValue(w io.Writer, lastActiveAt string, idleSeconds int) { - if text := lastActiveText(lastActiveAt, idleSeconds); text != "" { - fmt.Fprintf(w, "last active: %s\n", text) +// formatKeepDuration renders a keep's remaining time dropping zero units — +// "2h", "24m", "1h 30m" — so the figure stays short enough to share the active +// line in a 42-column tile. +func formatKeepDuration(d time.Duration) string { + d = d.Round(time.Second) + h := int(d.Hours()) + m := int(d.Minutes()) % 60 + s := int(d.Seconds()) % 60 + switch { + case h > 0: + if m > 0 { + if s > 0 { + return fmt.Sprintf("%dh %dm %ds", h, m, s) + } + return fmt.Sprintf("%dh %dm", h, m) + } + return fmt.Sprintf("%dh", h) + case m > 0: + if s > 0 { + return fmt.Sprintf("%dm %ds", m, s) + } + return fmt.Sprintf("%dm", m) + default: + return fmt.Sprintf("%ds", s) } } -// renderRetainIndented draws the retention-deadline line in the indented block -// the bar format and both fleet formats use, aligned to the bar-label column -// and beside the last-active line. The deadline is an absolute instant the -// control plane set, relayed verbatim — it is not re-checked here, so a line is -// drawn exactly when the read carries one (the stats reply drops it once it has -// passed). Local daemon nodes carry no deadline, so their line is absent. -func renderRetainIndented(w io.Writer, retainUntil string) { - if retainUntil != "" { - fmt.Fprintf(w, " %-9s %s\n", "retain until", retainUntil) +// renderActiveIndented draws the combined active-and-keep line in the indented +// block the bar format and the tile use: the "ago" figure, and, when the read +// carries a retention deadline, the relative keep after it — "active 2m 5s +// ago keep for 2h". The line is present whenever either fact is, and absent +// when neither is. It is not a bar: an elapsed time has no ceiling to fill +// against, and a bar would imply one. +func renderActiveIndented(w io.Writer, lastActiveAt string, idleSeconds int, retainUntil string, now time.Time) { + active := lastActiveText(lastActiveAt, idleSeconds) + keep := keepText(retainUntil, now) + switch { + case active != "" && keep != "": + fmt.Fprintf(w, " %-9s %s %s\n", "active", active, keep) + case active != "": + fmt.Fprintf(w, " %-9s %s\n", "active", active) + case keep != "": + fmt.Fprintf(w, " %s\n", keep) } } -// renderRetainKeyValue draws the retention deadline as a row of the table -// format, padded to the key column its neighbours use, beside the last-active -// row. -func renderRetainKeyValue(w io.Writer, retainUntil string) { - if retainUntil != "" { - fmt.Fprintf(w, "retain until: %s\n", retainUntil) +// renderActiveKeyValue draws the same line as a row of the table format, the +// key padded to the column its neighbours use. +func renderActiveKeyValue(w io.Writer, lastActiveAt string, idleSeconds int, retainUntil string, now time.Time) { + active := lastActiveText(lastActiveAt, idleSeconds) + keep := keepText(retainUntil, now) + switch { + case active != "" && keep != "": + fmt.Fprintf(w, "%-13s %s %s\n", "active:", active, keep) + case active != "": + fmt.Fprintf(w, "%-13s %s\n", "active:", active) + case keep != "": + fmt.Fprintf(w, "%-13s %s\n", "active:", keep) } } diff --git a/cmd/spinloop/metrics_render_test.go b/cmd/spinloop/metrics_render_test.go index ecf2a97a..5d569aa9 100644 --- a/cmd/spinloop/metrics_render_test.go +++ b/cmd/spinloop/metrics_render_test.go @@ -285,20 +285,20 @@ func TestFormatMetricsBarStoppedWithHistory(t *testing.T) { t.Fatal(err) } want := "prod stopped org/qwen:q4\n" + - " last active 12s ago\n" + + " active 12s ago\n" + " CPU " + strings.Repeat(" ", 38) + "▁" + ansiGreen + "▂" + ansiReset + " 20%\n" if got := b.String(); got != want { t.Errorf("stopped bar = %q, want %q", got, want) } // The gauge format draws no series for a stopped endpoint: the header and - // the last-active line, and nothing after. + // the active line, and nothing after. b.Reset() if err := formatMetricsGauge(resp, remote.Config{}, &b); err != nil { t.Fatal(err) } want = "prod stopped org/qwen:q4\n" + - " last active 12s ago\n" + " active 12s ago\n" if got := b.String(); got != want { t.Errorf("stopped gauge = %q, want %q", got, want) } @@ -326,8 +326,8 @@ func TestFormatMetricsBarRunning(t *testing.T) { if !strings.HasPrefix(got, "prod running g5.xlarge org/qwen:q4 0.4.3\n") { t.Errorf("header: %q", got) } - if !strings.Contains(got, " last active 3s ago\n") { - t.Errorf("last active line missing: %q", got) + if !strings.Contains(got, " active 3s ago\n") { + t.Errorf("active line missing: %q", got) } // Every series drew a sparkline from the history: no gauge in the output. if strings.Contains(got, "░") { diff --git a/cmd/spinloop/remote.go b/cmd/spinloop/remote.go index cda3f774..d712e2bf 100644 --- a/cmd/spinloop/remote.go +++ b/cmd/spinloop/remote.go @@ -762,11 +762,18 @@ func runRemoteStatus(args []string) error { if resp.BaseURL != "" { fmt.Printf("base_url: %s\n", resp.BaseURL) } - if text := lastActiveText(fact.LastActiveAt, fact.IdleSeconds); text != "" { - fmt.Printf("last active: %s\n", text) - } - if resp.RetainUntil != "" { - fmt.Printf("retain until: %s\n", resp.RetainUntil) + // The active figure and, for a retained instance, the relative keep after + // it — one line, like the metrics view, rather than an absolute deadline on + // its own. + active := lastActiveText(fact.LastActiveAt, fact.IdleSeconds) + keep := keepText(resp.RetainUntil, metricsNow()) + switch { + case active != "" && keep != "": + fmt.Printf("active: %s %s\n", active, keep) + case active != "": + fmt.Printf("active: %s\n", active) + case keep != "": + fmt.Printf("active: %s\n", keep) } return nil } @@ -872,6 +879,7 @@ func runMetricsWatch(cfg remote.Config, format string, withCost bool) error { } func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCost bool, cfg remote.Config, w io.Writer) error { + now := metricsNow() fmt.Fprintf(w, "environment: %s\n", resp.Environment) fmt.Fprintf(w, "state: %s\n", resp.State) @@ -882,10 +890,9 @@ func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCos if resp.ModelID != "" { fmt.Fprintf(w, "model: %s\n", resp.ModelID) } - renderLastActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds) // A stopped environment can still be retained: its deadline is the // control plane's, so it rides this branch too. - renderRetainKeyValue(w, resp.RetainUntil) + renderActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) return nil } @@ -907,8 +914,7 @@ func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCos if resp.UptimeSeconds > 0 { fmt.Fprintf(w, "uptime: %s\n", formatDuration(resp.UptimeSeconds)) } - renderLastActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds) - renderRetainKeyValue(w, resp.RetainUntil) + renderActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) renderTokenLines(w, resp.Tokens) renderGPUTable(w, resp.GPUs) @@ -980,16 +986,16 @@ func formatMetricsHeader(resp *remote.StatsResponse, w io.Writer) { } func formatMetricsBar(resp *remote.StatsResponse, cfg remote.Config, w io.Writer) error { + now := metricsNow() formatMetricsHeader(resp, w) // Before the series: when the endpoint last did work — and, for a retained - // endpoint, until it is kept — is worth showing in whatever state it is in, + // endpoint, how long it is kept — is worth showing in whatever state it is in, // and the retained history is too: a stopped endpoint's readings up to the // stop answer what it was doing until it stopped. A stopped endpoint's // current reading carries no resource figures, so the series it draws come // from the history alone, or not at all where the daemon predates it. - renderLastActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds) - renderRetainIndented(w, resp.RetainUntil) + renderActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) renderStatBars(w, resp.CPU, resp.Memory, resp.GPUs, resp.History, barLineW) renderTokenLines(w, resp.Tokens) @@ -999,13 +1005,13 @@ func formatMetricsBar(resp *remote.StatsResponse, cfg remote.Config, w io.Writer } func formatMetricsGauge(resp *remote.StatsResponse, cfg remote.Config, w io.Writer) error { + now := metricsNow() formatMetricsHeader(resp, w) // Before the early return: a stopped endpoint draws no gauges, but when it - // last did work — and until it is retained — is exactly what a stopped + // last did work — and how long it is kept — is exactly what a stopped // endpoint is worth asking about. - renderLastActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds) - renderRetainIndented(w, resp.RetainUntil) + renderActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) if resp.State != "running" { return nil diff --git a/cmd/spinloop/retain_render_test.go b/cmd/spinloop/retain_render_test.go index 8da9476e..8626c3d5 100644 --- a/cmd/spinloop/retain_render_test.go +++ b/cmd/spinloop/retain_render_test.go @@ -3,24 +3,58 @@ package main import ( "strings" "testing" + "time" ) -// The retention deadline rides the stats read and is drawn beside the -// last-active line in every surface that draws that line. The control plane is -// the single home of the "is it still in the future?" judgement, so these -// renderers only ever ask "does the read carry a deadline?" and draw it when -// present, and nothing when it is empty. +// The keep rides the stats read as an absolute deadline, and every surface that +// draws the active figure draws the keep as a relative figure after it — "active +// 2m 5s ago keep for 2h" — on the same line. The control plane is the single +// home of the "is it still in the future?" judgement (it drops the field there), +// so these renderers only ever ask "does the read carry a deadline that is still +// ahead?" and draw it when it is, and nothing when it is not. -const retainedDeadline = "2030-01-02T04:00:00Z" +// keepNow pins the one-shot metrics clock and returns an RFC3339 deadline d +// after it, so a test can assert the exact relative figure that deadline +// renders. +func keepNow(t *testing.T, d time.Duration) string { + t.Helper() + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + metricsNow = func() time.Time { return at } + t.Cleanup(func() { metricsNow = time.Now }) + return at.Add(d).UTC().Format(time.RFC3339) +} -func TestRemoteMetricsBarShowsRetainUntil(t *testing.T) { +// aLineContaining returns the first output line carrying every phrase, or "". +func aLineContaining(out string, phrases ...string) string { + for _, line := range strings.Split(out, "\n") { + ok := true + for _, p := range phrases { + if !strings.Contains(line, p) { + ok = false + break + } + } + if ok { + return line + } + } + return "" +} + +// A kept, active endpoint draws the keep after the active figure, on the same +// line — the point of sharing the line is that "when did it last do anything" +// and "how long is it kept" are read at a glance, not on two rows. +func TestRemoteMetricsBarKeepsOnTheActiveLine(t *testing.T) { + deadline := keepNow(t, 2*time.Hour) statsServer(t, `{ "environment": "dev", "state": "running", "instanceType": "g6e.xlarge", "modelId": "unsloth/Qwen3.6-27B", "cpu": {"utilization": 24}, - "retainUntil": "`+retainedDeadline+`" + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + "retainUntil": "`+deadline+`" }`) out := captureStdout(t, func() { @@ -28,30 +62,30 @@ func TestRemoteMetricsBarShowsRetainUntil(t *testing.T) { t.Fatalf("cmdRemoteMetrics: %v", err) } }) - if !strings.Contains(out, "retain until "+retainedDeadline) { - t.Errorf("bar format missing the retain-until line:\n%s", out) + if line := aLineContaining(out, "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("bar format did not put the keep on the active line:\n%s", out) } - // Beside last-active, before the bars: a fact about the endpoint, not a - // utilisation reading. - active := strings.Index(out, "last active") - retain := strings.Index(out, "retain until") - bars := strings.Index(out, "CPU") - if !(retain >= 0 && bars >= 0 && retain < bars) { - t.Errorf("retain-until line is not before the bars:\n%s", out) - } - if active >= 0 && !(retain > active) { - t.Errorf("retain-until line is not beside (after) last-active:\n%s", out) + // Beside the active figure, before the bars: a fact about the endpoint, not + // a utilisation reading. + if line := aLineContaining(out, "keep for 2h"); line != "" { + if strings.Index(out, "CPU") < strings.Index(line, "keep for 2h") { + t.Errorf("keep is not before the bars:\n%s", out) + } } } -func TestRemoteMetricsTableShowsRetainUntil(t *testing.T) { +// The table format draws the same combined line as a key-value row. +func TestRemoteMetricsTableKeepsOnTheActiveRow(t *testing.T) { + deadline := keepNow(t, 2*time.Hour) statsServer(t, `{ "environment": "dev", "state": "running", "instanceType": "g6e.xlarge", "modelId": "unsloth/Qwen3.6-27B", "cpu": {"utilization": 24}, - "retainUntil": "`+retainedDeadline+`" + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + "retainUntil": "`+deadline+`" }`) out := captureStdout(t, func() { @@ -59,79 +93,108 @@ func TestRemoteMetricsTableShowsRetainUntil(t *testing.T) { t.Fatalf("cmdRemoteMetrics: %v", err) } }) - // A key-value row, padded to the key column its neighbours use. - if !strings.Contains(out, "retain until: "+retainedDeadline) { - t.Errorf("table format missing the retain-until row:\n%s", out) + if line := aLineContaining(out, "active:", "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("table format did not put the keep on the active row:\n%s", out) } } -// A stopped environment can still be retained: the deadline is the control -// plane's, not the engine's, so the line survives the non-running short-circuit -// in both formats. -func TestRemoteMetricsStoppedRetainedStillShowsRetainUntil(t *testing.T) { - statsServer(t, `{ - "environment": "dev", - "state": "stopped", - "runner": "llamacpp", - "modelId": "unsloth/Qwen3.6-27B", - "retainUntil": "`+retainedDeadline+`" - }`) - - for format, want := range map[string]string{ - "bar": "retain until " + retainedDeadline, - "table": "retain until: " + retainedDeadline, +// The relative figure drops zero units: hours, minutes, and a mix each render +// their own short form. +func TestKeepDurationRendersRelatively(t *testing.T) { + // A stopped, unactive endpoint shows the keep alone, so each duration is + // read straight off the bar. + for d, want := range map[time.Duration]string{ + 2 * time.Hour: "keep for 2h", + 24 * time.Minute: "keep for 24m", + 90 * time.Minute: "keep for 1h 30m", } { - t.Run(format, func(t *testing.T) { + t.Run(want, func(t *testing.T) { + deadline := keepNow(t, d) + statsServer(t, `{ + "environment": "dev", + "state": "stopped", + "runner": "llamacpp", + "modelId": "unsloth/Qwen3.6-27B", + "retainUntil": "`+deadline+`" + }`) out := captureStdout(t, func() { - if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { + if err := cmdRemoteMetrics([]string{"--format=bar"}); err != nil { t.Fatalf("cmdRemoteMetrics: %v", err) } }) if !strings.Contains(out, want) { - t.Errorf("%s format dropped the deadline for a stopped, retained endpoint:\n%s", format, out) + t.Errorf("%s: %s format missing %q:\n%s", d, "bar", want, out) } }) } } -// No deadline on the read, no line: the renderer does not invent one, and it -// does not disturb the rest of the report. -func TestRemoteMetricsOmitsRetainUntilWhenAbsent(t *testing.T) { - statsServer(t, `{ - "environment": "dev", - "state": "running", - "cpu": {"utilization": 24}, - "lastActiveAt": "2026-08-10T10:00:00Z", - "idleSeconds": 125 - }`) +// A stopped environment can still be kept: the deadline is the control plane's, +// not the engine's, so the keep survives the non-running short-circuit in both +// formats. +func TestRemoteMetricsStoppedKeptStillShowsKeep(t *testing.T) { + deadline := keepNow(t, 4*time.Hour) + for format := range map[string]bool{"bar": true, "table": true} { + t.Run(format, func(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "stopped", + "runner": "llamacpp", + "modelId": "unsloth/Qwen3.6-27B", + "retainUntil": "`+deadline+`" + }`) + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if !strings.Contains(out, "keep for 4h") { + t.Errorf("%s format dropped the keep for a stopped, kept endpoint:\n%s", format, out) + } + }) + } +} +// No deadline on the read, no keep: the renderer does not invent one, and it +// leaves the active figure (now just "active") in place. +func TestRemoteMetricsOmitsKeepWhenAbsent(t *testing.T) { for _, format := range []string{"bar", "table"} { t.Run(format, func(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "running", + "cpu": {"utilization": 24}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125 + }`) out := captureStdout(t, func() { if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { t.Fatalf("cmdRemoteMetrics: %v", err) } }) - if strings.Contains(out, "retain until") { - t.Errorf("%s format claimed a deadline the read does not carry:\n%s", format, out) + if strings.Contains(out, "keep for") { + t.Errorf("%s format claimed a keep the read does not carry:\n%s", format, out) } - if !strings.Contains(out, "last active") { - t.Errorf("%s format lost the rest of the report:\n%s", format, out) + if !strings.Contains(out, "2m 5s ago") { + t.Errorf("%s format lost the active figure:\n%s", format, out) } }) } } -// The deadline rides the node's stats, so `fleet metrics` draws it beside -// last-active whatever the node's state. A faked daemon carrying the field -// stands in for a retained remote environment on the same render path. -func TestFleetMetricsShowsRetainUntil(t *testing.T) { +// The deadline rides the node's stats, so `fleet metrics` draws the keep after +// the active figure whatever the node's state. A faked daemon carrying the field +// stands in for a kept remote environment on the same render path. +func TestFleetMetricsShowsKeep(t *testing.T) { + deadline := keepNow(t, 2*time.Hour) fleetNodeWithMetrics(t, map[string]any{ - "state": "running", - "runner": "llamacpp", - "modelId": "org/qwen", - "cpu": map[string]any{"utilization": 30.0}, - "retainUntil": retainedDeadline, + "state": "running", + "runner": "llamacpp", + "modelId": "org/qwen", + "cpu": map[string]any{"utilization": 30.0}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + "retainUntil": deadline, }) out := captureStdout(t, func() { @@ -139,20 +202,20 @@ func TestFleetMetricsShowsRetainUntil(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if !strings.Contains(out, "retain until "+retainedDeadline) { - t.Errorf("fleet metrics missing the retain-until line:\n%s", out) + if line := aLineContaining(out, "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("fleet metrics did not put the keep on the active line:\n%s", out) } } -// A node whose read carries no deadline draws no line, so a local daemon node — -// which never has one — is simply quieter on that line. -func TestFleetMetricsOmitsRetainUntilWhenAbsent(t *testing.T) { +// A node whose read carries no deadline draws no keep, so a local daemon node — +// which never has one — is simply quieter on that figure. +func TestFleetMetricsOmitsKeepWhenAbsent(t *testing.T) { fleetNodeWithMetrics(t, map[string]any{ "state": "running", "runner": "llamacpp", "modelId": "org/qwen", "cpu": map[string]any{"utilization": 30.0}, - "lastActiveAt": "2026-08-10T10:00:00Z", + "lastActiveAt": "2025-12-31T23:58:15Z", "idleSeconds": 125, }) @@ -161,7 +224,7 @@ func TestFleetMetricsOmitsRetainUntilWhenAbsent(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if strings.Contains(out, "retain until") { - t.Errorf("fleet metrics claimed a deadline a local node never carries:\n%s", out) + if strings.Contains(out, "keep for") { + t.Errorf("fleet metrics claimed a keep a local node never carries:\n%s", out) } } diff --git a/cmd/spinloop/status_render.go b/cmd/spinloop/status_render.go index fc8ec172..04864805 100644 --- a/cmd/spinloop/status_render.go +++ b/cmd/spinloop/status_render.go @@ -45,10 +45,10 @@ func (f statusFact) servingText() string { } // How long since it last did work — deliberately not labelled "idle": that // word is already an engine state meaning nothing has been started. Shown - // only when there is a recorded last-active time; without one there is - // nothing to measure from. + // only when there is a recorded active time; without one there is nothing + // to measure from. if f.LastActiveAt != "" { - serving += fmt.Sprintf(" (last active %s ago)", formatDuration(f.IdleSeconds)) + serving += fmt.Sprintf(" (active %s ago)", formatDuration(f.IdleSeconds)) } if f.Version != "" { serving += fmt.Sprintf(" (%s)", f.Version) diff --git a/cmd/spinloop/status_render_test.go b/cmd/spinloop/status_render_test.go index 48383552..9ca27e8d 100644 --- a/cmd/spinloop/status_render_test.go +++ b/cmd/spinloop/status_render_test.go @@ -13,7 +13,7 @@ func TestStatusFactServingText(t *testing.T) { UptimeSeconds: 30, LastActiveAt: "2026-01-02T00:00:00Z", IdleSeconds: 30, } got := f.servingText() - for _, want := range []string{"llamacpp qwen", "(up 30s)", "(last active 30s ago)", "(1.2.0)"} { + for _, want := range []string{"llamacpp qwen", "(up 30s)", "(active 30s ago)", "(1.2.0)"} { if !strings.Contains(got, want) { t.Errorf("servingText %q missing %q", got, want) } diff --git a/docs/commands/fleet.md b/docs/commands/fleet.md index 18104753..2d26513e 100644 --- a/docs/commands/fleet.md +++ b/docs/commands/fleet.md @@ -225,12 +225,12 @@ failure — the rest of the fleet still renders and the command still exits 0: ``` NODE STATE SERVING -studio running llamacpp org/qwen (up 1h 2m 5s) (last active 12s ago) +studio running llamacpp org/qwen (up 1h 2m 5s) (active 12s ago) gpu-box idle llamacpp org/qwen offline unreachable dial tcp 10.0.0.9:4242: connect: connection refused ``` -"last active" comes from the activity each daemon tracks, so a glance answers +"active" comes from the activity each daemon tracks, so a glance answers "which of my nodes is doing nothing?". It is absent until a node's engine has actually done some work — a daemon that has served nothing reports no activity rather than claiming it has been quiet since it started. The wording avoids @@ -256,17 +256,18 @@ no history falls back to the gauge drawing of its current reading, so a fleet mixed with older daemons renders each node the best way it can. A stopped node keeps its readings, so its sparkline runs to the stop. -Each node's block carries the same `last active` figure the status table +Each node's block carries the same `active` figure the status table shows, for the reasons given above, and on the same terms: absent until the node's engine has done some work. A node whose engine has *stopped* still shows it — the daemon keeps the record across a stop, and "how long since this did anything?" is worth more about a stopped engine than about a busy one. -A `kind: remote` environment carries a `retain until` figure beside the -last-active one, on the same omitted-when-absent terms: it shows the -deadline the idle sweep will not pass while it is in the future, and is gone -once it has passed or was never set. It is the same line the dashboard draws -on a kept environment's tile and detail screen, from the same read. +A `kind: remote` environment carries a relative keep after that figure, on the +same line — `active 2m 5s ago keep for 2h` — on the same omitted-when-absent +terms: it shows how long the idle sweep will hold the box while the deadline is +in the future, and is gone once it has passed or was never set. It is the same +line the dashboard draws on a kept environment's tile and detail screen, from +the same read. `--watch`/`-w` redraws the whole fleet on an interval, clearing the screen in place with no scrollback. Each refresh is rendered into a buffer first, so a @@ -351,8 +352,8 @@ hint slot, so the entry is kept and corrected in place. While the keep runs its tile carries it, and it is not abortable — one fast signed call, so `a` drives nothing on it. When it finishes, the status line reports the deadline the control plane set and the node is re-read at once, which is what brings -the `retain until` figure onto the tile and detail screen at the node's next -round rather than waiting out its full cadence. +the relative `keep for …` figure onto the tile and detail screen at the node's +next round rather than waiting out its full cadence. Everything else in the view is `fleet status`/`metrics`/`logs` in place — it is read-only apart from those four action keys. It needs a real terminal: a @@ -453,7 +454,7 @@ Fleet: ./fleet.yaml Prefer: idle Would use gpu-box at http://gpu-box:8080/v1 - serving qwen3-27b, last active 312s ago (prefer idle) + serving qwen3-27b, active 312s ago (prefer idle) ``` When nothing is serving that model it shows the whole fleet's state and names diff --git a/docs/commands/remote.md b/docs/commands/remote.md index 0527ba30..86d4b9b2 100644 --- a/docs/commands/remote.md +++ b/docs/commands/remote.md @@ -169,7 +169,7 @@ of the current reading, so the default output degrades rather than goes blank. A stopped engine keeps its readings: the sparkline runs to the stop, ending at it. -Both report **`last active`** — how long since the endpoint's engine last did +Both report **`active`** — how long since the endpoint's engine last did any work. It comes from the activity the on-instance daemon tracks, so it is one answer decided on the box rather than something each command re-derives from raw counters. `status` asks the daemon alongside the health check it diff --git a/internal/fleet/select.go b/internal/fleet/select.go index d93574cc..bf0c5f2f 100644 --- a/internal/fleet/select.go +++ b/internal/fleet/select.go @@ -286,7 +286,7 @@ func reasonFor(c candidate, w Want, woken bool) string { case c.result.Status.LastActiveAt == "": return fmt.Sprintf("serving %s, no work yet (prefer %s)", serving, w.prefer()) default: - return fmt.Sprintf("serving %s, last active %ds ago (prefer %s)", + return fmt.Sprintf("serving %s, active %ds ago (prefer %s)", serving, c.result.Status.IdleSeconds, w.prefer()) } } diff --git a/openspec/changes/dashboard-keep/design.md b/openspec/changes/dashboard-keep/design.md index 4c935e4d..02576cb9 100644 --- a/openspec/changes/dashboard-keep/design.md +++ b/openspec/changes/dashboard-keep/design.md @@ -104,15 +104,19 @@ puts a clock comparison in each of them. The field is additive: an environment whose stats Lambda has not been redeployed simply omits it, and the dashboard shows no line there. -### 5. One shared line beside the last-active figure - -The deadline renders as a line in the shared bar-format body, next to the -last-active line that body already prints: the dashboard tile, the one-shot -fleet metrics, and the one-shot remote metrics all call that body, so they -cannot word the same read differently. The table format prints the line when -the read carries it, and the json format carries the field on its own. The -line is omitted wherever the read has no deadline, the same rule the -last-active figure follows for a figure it lacks. +### 5. One shared line: the active figure, and the keep after it + +The active figure — how long since the engine last did work — and, for a kept +environment, the retention keep render as a single line in the shared body the +tile and both one-shot metrics reports call, so they cannot word the same read +differently: `active 2m 5s ago keep for 2h`. The keep is the deadline's +remaining time, rendered relatively by the client rather than the absolute +timestamp the control plane holds — short enough to share the line in a +42-column tile, and it reads at a glance without the operator subtracting. The +table format prints the same line as its key-value row; the json format carries +the absolute field on its own, for a consumer that wants to do its own math. +The line is omitted where the read has neither an active time nor a deadline, +and shows whichever it has when it has one. ### 6. No second confirmation; not abortable diff --git a/openspec/changes/dashboard-keep/specs/remote-stats/spec.md b/openspec/changes/dashboard-keep/specs/remote-stats/spec.md index c87dbcba..20ce9b06 100644 --- a/openspec/changes/dashboard-keep/specs/remote-stats/spec.md +++ b/openspec/changes/dashboard-keep/specs/remote-stats/spec.md @@ -19,10 +19,11 @@ The client SHALL map the deadline onto the shared stats shape every fleet and remote stats surface reads from, so a dashboard panel, a one-shot fleet report, and a one-shot remote report cannot word the same read differently. -The deadline SHALL be reported beside the report's other time facts — the -last-active figure — in every format the command supports, and omitted in -every format when the read carries none, following the same omission rule the -last-active figure uses for a figure it does not have. +The deadline SHALL be reported on the report's active-figure line — rendered by +the client as a relative remaining time, e.g. `keep for 2h`, not the absolute +timestamp — in every format the command supports, and omitted in every format +when the read carries none, following the same omission rule the active figure +uses for a figure it does not have. #### Scenario: A retained instance's stats carry the deadline @@ -54,5 +55,5 @@ last-active figure uses for a figure it does not have. - **WHEN** the user reads the stats of a retained environment with `bar`, `table`, or `json` output -- **THEN** each output carries the deadline in its own idiom, beside the - last-active figure, and each omits it when the read carries none +- **THEN** each output carries the deadline in its own idiom on the active + figure's line, and each omits it when the read carries none diff --git a/openspec/changes/dashboard-keep/tasks.md b/openspec/changes/dashboard-keep/tasks.md index 994b7324..95bec4a8 100644 --- a/openspec/changes/dashboard-keep/tasks.md +++ b/openspec/changes/dashboard-keep/tasks.md @@ -36,10 +36,10 @@ ## 4. The shared deadline line -- [x] 4.1 Add the retain-until line to the shared bar-format body, beside the - last-active line, omitted when the read carries none — verify with - render tests: the line's wording when present, its omission when the - field is empty. +- [x] 4.1 Merge the relative keep onto the shared active-figure line in the + bar-format body — `active … ago keep for …` — omitted when the read + carries neither — verify with render tests: the line's wording when a + keep is present, and its omission when the field is empty. - [x] 4.2 Wire the line into the one-shot `fleet metrics` bar format, the `remote metrics` bar format, and the `remote metrics` table format when present — verify with the existing render tests for those surfaces From f9ddbce7b0f24bfbc1bd790ac1f07f4ad86620df Mon Sep 17 00:00:00 2001 From: Pete Cornish Date: Sun, 6 Sep 2026 14:41:30 +0100 Subject: [PATCH 3/3] docs(openspec): archive dashboard-keep and sync its specs to main --- .../2026-09-06-dashboard-keep}/.openspec.yaml | 0 .../2026-09-06-dashboard-keep}/design.md | 0 .../2026-09-06-dashboard-keep}/proposal.md | 0 .../specs/fleet-client/spec.md | 0 .../specs/remote-stats/spec.md | 0 .../2026-09-06-dashboard-keep}/tasks.md | 0 openspec/specs/fleet-client/spec.md | 168 ++++++++++++++++++ openspec/specs/remote-stats/spec.md | 59 +++++- 8 files changed, 226 insertions(+), 1 deletion(-) rename openspec/changes/{dashboard-keep => archive/2026-09-06-dashboard-keep}/.openspec.yaml (100%) rename openspec/changes/{dashboard-keep => archive/2026-09-06-dashboard-keep}/design.md (100%) rename openspec/changes/{dashboard-keep => archive/2026-09-06-dashboard-keep}/proposal.md (100%) rename openspec/changes/{dashboard-keep => archive/2026-09-06-dashboard-keep}/specs/fleet-client/spec.md (100%) rename openspec/changes/{dashboard-keep => archive/2026-09-06-dashboard-keep}/specs/remote-stats/spec.md (100%) rename openspec/changes/{dashboard-keep => archive/2026-09-06-dashboard-keep}/tasks.md (100%) diff --git a/openspec/changes/dashboard-keep/.openspec.yaml b/openspec/changes/archive/2026-09-06-dashboard-keep/.openspec.yaml similarity index 100% rename from openspec/changes/dashboard-keep/.openspec.yaml rename to openspec/changes/archive/2026-09-06-dashboard-keep/.openspec.yaml diff --git a/openspec/changes/dashboard-keep/design.md b/openspec/changes/archive/2026-09-06-dashboard-keep/design.md similarity index 100% rename from openspec/changes/dashboard-keep/design.md rename to openspec/changes/archive/2026-09-06-dashboard-keep/design.md diff --git a/openspec/changes/dashboard-keep/proposal.md b/openspec/changes/archive/2026-09-06-dashboard-keep/proposal.md similarity index 100% rename from openspec/changes/dashboard-keep/proposal.md rename to openspec/changes/archive/2026-09-06-dashboard-keep/proposal.md diff --git a/openspec/changes/dashboard-keep/specs/fleet-client/spec.md b/openspec/changes/archive/2026-09-06-dashboard-keep/specs/fleet-client/spec.md similarity index 100% rename from openspec/changes/dashboard-keep/specs/fleet-client/spec.md rename to openspec/changes/archive/2026-09-06-dashboard-keep/specs/fleet-client/spec.md diff --git a/openspec/changes/dashboard-keep/specs/remote-stats/spec.md b/openspec/changes/archive/2026-09-06-dashboard-keep/specs/remote-stats/spec.md similarity index 100% rename from openspec/changes/dashboard-keep/specs/remote-stats/spec.md rename to openspec/changes/archive/2026-09-06-dashboard-keep/specs/remote-stats/spec.md diff --git a/openspec/changes/dashboard-keep/tasks.md b/openspec/changes/archive/2026-09-06-dashboard-keep/tasks.md similarity index 100% rename from openspec/changes/dashboard-keep/tasks.md rename to openspec/changes/archive/2026-09-06-dashboard-keep/tasks.md diff --git a/openspec/specs/fleet-client/spec.md b/openspec/specs/fleet-client/spec.md index d6759b67..d9b13536 100644 --- a/openspec/specs/fleet-client/spec.md +++ b/openspec/specs/fleet-client/spec.md @@ -1317,3 +1317,171 @@ the key help line SHALL name it. - **WHEN** the dashboard draws its key help line - **THEN** it names `g` as the format toggle +### Requirement: The dashboard keeps the selected node + +The dashboard SHALL let the operator set the retention deadline of the node +currently selected, from the keyboard, through the same node operation the +one-shot keep command uses. Keep applies to a node's instance — the time until +which the cloud's idle sweep leaves it alone — so it SHALL be offered only for +nodes that support retention, namely the fleet's remote environments; a node +without retention support SHALL take no keep. + +The keep key SHALL open a duration prompt rather than send anything. The +prompt SHALL name the node it will apply to, show the duration as the operator +has it, and name its confirm and cancel keys. It SHALL open pre-filled with a +default duration, so that the common case is the key and a confirm, and the +confirm SHALL send the duration as shown, whatever the operator has changed it +to. The duration SHALL be expressed in the same syntax the one-shot keep +command accepts. + +A confirm of an entry that is not a positive duration SHALL send nothing: the +prompt SHALL stay open and show why the entry was rejected, so the operator can +correct it and confirm again. A cancel SHALL close the prompt and send nothing. +Once the prompt is open, the navigation and selection keys SHALL drive +nothing: the prompt takes the keyboard until it is confirmed or cancelled. + +Keep SHALL not ask for a second confirmation after the prompt: the operator +has typed, or deliberately kept, the duration and chosen to send it, and a +keep overwrites a deadline rather than ending anything. A keep in flight SHALL +be shown on the node's panel like any action — its verb and how long it has +been running — and SHALL NOT be abortable: it is one fast call with no +open-ended wait, and abandoning the wait on it would leave the operator unsure +whether the deadline was set. + +A node that already has an action in flight SHALL take no keep; the dashboard +SHALL say it is still working and drive nothing. The key help SHALL name the +keep key only for the node it describes when that node supports retention and +has no action in flight, so the operator is never invited to press a key that +would do nothing there. + +The outcome of a keep SHALL be shown on the status line: on success the +deadline the control plane set, on failure the control plane's own reason — a +deployment that predates keep support, an environment with no instance to +retain — and a failed keep SHALL NOT close the dashboard. What the keep +changes — the deadline the node now carries — SHALL appear on the node's panel +through the normal refresh, without the operator asking for it. + +The node's detail view SHALL offer keep on the node it shows, through the same +prompt and the same rules the grid applies to the selected node. + +#### Scenario: Keep opens a prompt, it does not send + +- **WHEN** the operator selects a remote environment with no action in flight + and issues keep +- **THEN** a duration prompt opens naming the node, pre-filled with a default + duration, and nothing has been sent + +#### Scenario: A confirmed keep sets the deadline + +- **WHEN** the operator changes the prompt's duration and confirms it +- **THEN** the node is kept until now plus the confirmed duration and the + status line shows the deadline the control plane set + +#### Scenario: A pre-filled keep is one key and a confirm + +- **WHEN** the operator issues keep on a remote environment and confirms the + prompt without changing its duration +- **THEN** the node is kept until now plus the default duration + +#### Scenario: A bad duration sends nothing + +- **WHEN** the operator confirms an entry that is not a positive duration +- **THEN** nothing is sent, the prompt stays open, and it shows why the entry + was rejected + +#### Scenario: Cancelling sends nothing + +- **WHEN** the operator opens the keep prompt and cancels it +- **THEN** the prompt closes and the node's deadline is unchanged + +#### Scenario: The prompt takes the keyboard + +- **WHEN** the keep prompt is open and the operator presses a navigation key +- **THEN** the selection does not move and the prompt stays open + +#### Scenario: A local node takes no keep + +- **WHEN** the operator issues keep on a local daemon node +- **THEN** the dashboard drives nothing: a local daemon has no idle sweep to + hold off, so there is no deadline to set + +#### Scenario: The key help hides keep where it would do nothing + +- **WHEN** the node under the cursor is a local daemon node, or has an action + in flight +- **THEN** the key help does not name the keep key +- **WHEN** that node is a remote environment with no action in flight +- **THEN** the key help names the keep key + +#### Scenario: A busy node is not kept again + +- **WHEN** the operator issues keep on a node whose start, stop, or keep is + still in flight +- **THEN** the dashboard drives nothing and says the node is still working + +#### Scenario: A keep in flight cannot be aborted + +- **WHEN** the operator issues the abort on a node whose keep is in flight +- **THEN** the abort drives nothing: the keep keeps running, the panel keeps + showing it, and its outcome lands on the status line when the call returns + +#### Scenario: A keep that fails keeps the dashboard open + +- **WHEN** a keep is sent to an environment whose deployment predates keep + support, or to one with no instance to retain +- **THEN** the failure is shown on the status line with the control plane's + own reason and the dashboard keeps running with its refreshes + +#### Scenario: Keep from the detail view + +- **WHEN** the operator issues keep from the detail view of a remote + environment +- **THEN** the same prompt opens with the same rules, and its outcome is + shown as the grid would show it + +### Requirement: The dashboard panels show the retention deadline + +A node's panel SHALL show the node's retention deadline — the time until which +the idle sweep leaves the node alone — whenever the node's last answered +refresh carries one, worded the same way the one-shot surfaces report it, and +SHALL omit the line when the answer carries none: a node with no active +retention, a node without retention at all, or a read from a control plane +that predates the field. The deadline SHALL appear on the tile and on the +node's detail screen, which draw the same lines, so the two cannot show the +same read differently, and it SHALL sit among the panel's other time facts +rather than displacing the node's state or what it serves. + +A deadline that has passed carries no active retention: a refresh taken after +the deadline SHALL not show the line, and a panel SHALL NOT draw a deadline +the sweep has already moved past as though the node were still held. + +#### Scenario: A retained node shows its deadline + +- **WHEN** a remote environment's refresh answer carries a retention deadline + in the future +- **THEN** its tile shows the deadline, and its detail screen shows the same + line + +#### Scenario: A node with no retention shows no line + +- **WHEN** a node's refresh answer carries no retention deadline +- **THEN** its panel shows no deadline line and renders the rest exactly as + before + +#### Scenario: An older control plane degrades to no line + +- **WHEN** a remote environment's control plane predates the deadline in its + stats reply +- **THEN** its panel shows no deadline line and no error, and the rest of the + panel is unaffected + +#### Scenario: A passed deadline disappears + +- **WHEN** a node's retention deadline has passed and a later refresh answers +- **THEN** the panel no longer shows the deadline + +#### Scenario: Tile and detail agree + +- **WHEN** the operator opens the detail screen of a retained node +- **THEN** the detail screen's deadline line is the same line, worded the + same, the tile draws for the same read diff --git a/openspec/specs/remote-stats/spec.md b/openspec/specs/remote-stats/spec.md index eb5d33de..187ff6b2 100644 --- a/openspec/specs/remote-stats/spec.md +++ b/openspec/specs/remote-stats/spec.md @@ -181,5 +181,62 @@ When the on-instance daemon's metrics reply carries a history of system readings #### Scenario: A daemon without history degrades - **WHEN** the instance runs a daemon whose reply carries no history and the user runs `spinloop remote metrics` -- **THEN** the report omits the history field and bar format draws the current reading in the gauge's filled style +- **THEN** the report omits the history field and bar format draws the current reading in the gauge's filled style +### Requirement: The stats reply carries the retention deadline + +The stats Lambda's reply SHALL carry the instance's retention deadline — the +time until which the idle sweep leaves the instance alone, parsed from the +instance's Retain-Until tag, which the Lambda already reads — when that tag is +a time in the future, and SHALL omit it when the tag is absent or a time that +has passed. The deadline is a fact the control plane holds about the instance, +not one the on-instance daemon reports: it SHALL be present for a stopped +instance the sweep would otherwise terminate, and absent for an environment +with no instance at all. + +A control plane that predates the field SHALL simply omit it, with no error: +every reader of the reply treats an absent deadline as "no active retention" +rather than as a failure. + +The client SHALL map the deadline onto the shared stats shape every fleet and +remote stats surface reads from, so a dashboard panel, a one-shot fleet report, +and a one-shot remote report cannot word the same read differently. + +The deadline SHALL be reported on the report's active-figure line — rendered by +the client as a relative remaining time, e.g. `keep for 2h`, not the absolute +timestamp — in every format the command supports, and omitted in every format +when the read carries none, following the same omission rule the active figure +uses for a figure it does not have. + +#### Scenario: A retained instance's stats carry the deadline + +- **WHEN** the user reads the stats of an environment whose instance carries a + Retain-Until tag at a time in the future +- **THEN** the reply carries that deadline + +#### Scenario: A passed tag carries no deadline + +- **WHEN** the user reads the stats of an instance whose Retain-Until tag is a + time that has passed +- **THEN** the reply carries no deadline: a passed tag holds no active + retention + +#### Scenario: An untagged or instance-less environment carries no deadline + +- **WHEN** the user reads the stats of an instance with no Retain-Until tag, + or of an environment with no instance +- **THEN** the reply carries no deadline + +#### Scenario: An older control plane is read without error + +- **WHEN** the user reads the stats of an environment whose control plane + predates the field +- **THEN** the read succeeds, the deadline is simply absent, and the rest of + the report renders as before + +#### Scenario: Every format carries the deadline + +- **WHEN** the user reads the stats of a retained environment with `bar`, + `table`, or `json` output +- **THEN** each output carries the deadline in its own idiom on the active + figure's line, and each omits it when the read carries none