diff --git a/cmd/spinloop/dashboard_detail.go b/cmd/spinloop/dashboard_detail.go index fb4c62b..10d8b54 100644 --- a/cmd/spinloop/dashboard_detail.go +++ b/cmd/spinloop/dashboard_detail.go @@ -92,6 +92,15 @@ func (m *dashModel) updateDetailKey(msg tea.KeyMsg) tea.Cmd { if len(m.entries) > 0 && m.actions[m.cursor].verb == "" { return m.beginAction("start") } + case "k": + // Open the duration prompt on the node in view, the same way the grid + // does — kept in the model, so it is checked before this view's own + // keys once it is open. + if m.keepOffered() { + m.keepPrompt = true + m.keepBuf = "4h" + m.keepErr = "" + } case "a": if len(m.entries) > 0 { m.abortAction() @@ -244,6 +253,6 @@ func (m dashModel) detailView() string { parts = append(parts, dashClip(line, w)) } parts = append(parts, divider) - parts = append(parts, m.footerLine(w, dashFooterHints(dashDetailKeys, m.canAbort()))) + parts = append(parts, m.footerLine(w, dashFooterHints(m.detailKeys(), m.canAbort()))) return strings.Join(parts, "\n") } diff --git a/cmd/spinloop/dashboard_keep_test.go b/cmd/spinloop/dashboard_keep_test.go new file mode 100644 index 0000000..8d1c7ab --- /dev/null +++ b/cmd/spinloop/dashboard_keep_test.go @@ -0,0 +1,447 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/spinloop-ai/spinloop/internal/daemon" + "github.com/spinloop-ai/spinloop/internal/fleet" + "github.com/spinloop-ai/spinloop/internal/metrics" + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// keeperDashNode wraps the fake with the keep capability: a node the dashboard's +// type assertion finds to be a Keeper, so the keep key and prompt are offered +// for it. Keep records the duration and returns a deadline; keepErr makes it +// refuse, to exercise a failed keep, and deadline fixes the control plane's +// reply when a test wants a specific one. +type keeperDashNode struct { + f *fakeDashNode + keeps int + keepDur time.Duration + keepErr error + deadline string +} + +var ( + _ fleet.Node = (*keeperDashNode)(nil) + _ fleet.Keeper = (*keeperDashNode)(nil) +) + +func (n *keeperDashNode) Name() string { return n.f.Name() } + +func (n *keeperDashNode) Status(ctx context.Context) (daemon.StatusResponse, error) { + return n.f.Status(ctx) +} + +func (n *keeperDashNode) Metrics(ctx context.Context) (metrics.Stats, error) { return n.f.Metrics(ctx) } + +func (n *keeperDashNode) Start(ctx context.Context) (daemon.StatusResponse, error) { + return n.f.Start(ctx) +} + +func (n *keeperDashNode) StartWith(ctx context.Context, dc *remote.DeployConfig, engineKey string) (daemon.StatusResponse, error) { + return n.f.StartWith(ctx, dc, engineKey) +} + +func (n *keeperDashNode) Stop(ctx context.Context) (daemon.StatusResponse, error) { + return n.f.Stop(ctx) +} + +func (n *keeperDashNode) Logs(ctx context.Context, offset int64, limit int) (daemon.LogsResponse, error) { + return n.f.Logs(ctx, offset, limit) +} + +func (n *keeperDashNode) Keep(ctx context.Context, d time.Duration) (string, error) { + if n.keepErr != nil { + return "", n.keepErr + } + n.keeps++ + n.keepDur = d + if n.deadline != "" { + return n.deadline, nil + } + return time.Now().Add(d).UTC().Format(time.RFC3339), nil +} + +// keeperModel is a one-node board over a keeperDashNode, the shape every keep +// test drives. +func keeperModel(node *keeperDashNode) *dashModel { + return &dashModel{ + entries: []dashEntry{{name: node.f.Name(), kind: fleet.KindRemote, node: node}}, + results: []fleet.NodeResult{{Name: node.f.Name()}}, + actions: make([]dashAction, 1), + width: 120, height: 40, + } +} + +// openKeepPrompt presses k on the model and returns the model with the prompt +// open, failing the test if k did not open it. +func openKeepPrompt(t *testing.T, m *dashModel) *dashModel { + t.Helper() + next, cmd := m.Update(dashKey("k")) + if cmd != nil { + t.Fatal("opening the prompt set off an action") + } + m = next.(*dashModel) + if !m.keepPrompt { + t.Fatal("k did not open the keep prompt") + } + return m +} + +// typeKey sends one key to the model and hands back the model, dropping the +// command — what the prompt's typing and backspace never produce. +func typeKey(m *dashModel, s string) *dashModel { + next, _ := m.Update(dashKey(s)) + return next.(*dashModel) +} + +// TestDashKeepPromptOpensOnTheKeyForARemoteNode: k opens the prompt, pre-filled +// with 4h, for a node that can be kept. +func TestDashKeepPromptOpensOnTheKeyForARemoteNode(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + if m.keepBuf != "4h" { + t.Errorf("prompt not pre-filled with 4h, got %q", m.keepBuf) + } +} + +// A local daemon node has no retention tag to set, so k drives nothing for it: +// the prompt does not open and no action is set off. +func TestDashKeepDrivesNothingForALocalNode(t *testing.T) { + f := newFakeDashNode("stopped") + m := &dashModel{ + entries: []dashEntry{{name: "box", kind: fleet.KindDaemon, node: f}}, + results: []fleet.NodeResult{{Name: "box"}}, + actions: make([]dashAction, 1), + width: 120, height: 40, + } + next, cmd := m.Update(dashKey("k")) + if cmd != nil { + t.Fatal("k on a local node set off an action") + } + if next.(*dashModel).keepPrompt { + t.Fatal("k opened the keep prompt for a local node") + } +} + +// While the prompt is open the board stands still: navigation keys go to the +// prompt (and change nothing), not to the grid. +func TestDashKeepPromptStandsStillOverNavigation(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + before := m.cursor + for _, key := range []string{"down", "up", "left", "right", "r"} { + next, _ := m.Update(dashKey(key)) + m = next.(*dashModel) + } + if m.cursor != before { + t.Errorf("navigation moved the selection while the prompt was open: %d -> %d", before, m.cursor) + } + if !m.keepPrompt { + t.Fatal("the prompt closed on a navigation key") + } +} + +// A confirmed entry that is a positive duration sets off a keep of that length; +// the pre-filled 4h is the one-Enter common case. +func TestDashKeepConfirmsADuration(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped"), deadline: "2030-01-02T04:00:00Z"} + m := keeperModel(node) + m = openKeepPrompt(t, m) + _, cmd := m.Update(dashKey("enter")) + if cmd == nil { + t.Fatal("enter did not set off the keep") + } + if m.keepPrompt { + t.Fatal("the prompt stayed open after a valid confirm") + } + if m.actions[0].verb != "keep" { + t.Fatalf("no keep recorded on the node: %+v", m.actions[0]) + } + runAction(t, cmd) + if node.keeps != 1 || node.keepDur != 4*time.Hour { + t.Errorf("keep not sent with the pre-filled 4h: keeps=%d dur=%s", node.keeps, node.keepDur) + } +} + +// A typed duration replaces the pre-fill and is what the keep is sent with. +func TestDashKeepTypesADuration(t *testing.T) { + for _, tc := range []struct { + entry string + want time.Duration + }{ + {"90m", 90 * time.Minute}, + {"1h30m", 90 * time.Minute}, + } { + t.Run(tc.entry, func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + // Clear the pre-filled 4h, then type the entry. + m = typeKey(m, "backspace") + m = typeKey(m, "backspace") + m = typeKey(m, tc.entry) + next, cmd := m.Update(dashKey("enter")) + m = next.(*dashModel) + if cmd == nil { + t.Fatalf("enter did not set off the keep for %q", tc.entry) + } + runAction(t, cmd) + if node.keepDur != tc.want { + t.Errorf("keep sent with %s, want %s", node.keepDur, tc.want) + } + }) + } +} + +// A confirmed entry that is not a positive duration leaves the prompt open with +// the reason at the foot, and sends nothing: 4hours is not a duration, and an +// empty entry is not one either. +func TestDashKeepLeavesThePromptOpenOnAnInvalidEntry(t *testing.T) { + for _, entry := range []string{"4hours", ""} { + t.Run(entry, func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + // Clear the pre-filled 4h; type the entry when there is one. + m = typeKey(m, "backspace") + m = typeKey(m, "backspace") + if entry != "" { + m = typeKey(m, entry) + } + next, cmd := m.Update(dashKey("enter")) + if cmd != nil { + t.Fatal("an invalid entry set off a keep") + } + m = next.(*dashModel) + if !m.keepPrompt { + t.Fatal("the prompt closed on an invalid entry") + } + if m.keepErr == "" { + t.Fatal("no reason shown for the invalid entry") + } + if node.keeps != 0 { + t.Fatal("a keep was sent for an invalid entry") + } + }) + } +} + +// esc cancels the prompt and sends nothing; q cancels and quits. +func TestDashKeepCancels(t *testing.T) { + t.Run("esc", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := openKeepPrompt(t, keeperModel(node)) + next, cmd := m.Update(dashKey("esc")) + if cmd != nil { + t.Fatal("esc set off a command") + } + if next.(*dashModel).keepPrompt { + t.Fatal("esc did not cancel the prompt") + } + if node.keeps != 0 { + t.Fatal("esc sent a keep") + } + }) + t.Run("q", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := openKeepPrompt(t, keeperModel(node)) + _, cmd := m.Update(dashKey("q")) + if cmd == nil { + t.Fatal("q did not quit") + } + if node.keeps != 0 { + t.Fatal("q sent a keep") + } + }) +} + +// A keep in flight rides the node's action: the tile carries the spinner and +// the keeping verb, a busy node takes no second keep, and completion clears the +// action, brings the node forward for one more read, and names the deadline. +func TestDashKeepInFlightAndCompletion(t *testing.T) { + dashFixNow(t, dashTestClock) + node := &keeperDashNode{f: newFakeDashNode("stopped"), deadline: "2030-01-02T04:00:00Z"} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, cmd := m.Update(dashKey("enter")) + m = next.(*dashModel) + + // In flight: the tile shows the keep, and the node takes no second keep. + if m.actions[0].verb != "keep" { + t.Fatalf("no keep recorded: %+v", m.actions[0]) + } + tile := dashTestTile(node.f.Name(), m.results[0], true, m.actions[0]) + if !strings.Contains(tile, "keeping") { + t.Errorf("in-flight tile does not show the keep:\n%s", tile) + } + knext, _ := m.Update(dashKey("k")) + if knext.(*dashModel).keepPrompt { + t.Fatal("a busy node opened a second keep prompt") + } + + // Completion: the action clears, the node is read again now, and the footer + // names the deadline the control plane set. + smsg, _ := runAction(t, cmd).(dashActionMsg) + if smsg.retainUntil != "2030-01-02T04:00:00Z" { + t.Errorf("completion message did not carry the deadline: %q", smsg.retainUntil) + } + m2, _ := m.Update(smsg) + mm := m2.(*dashModel) + if mm.actions[0].verb != "" { + t.Fatalf("the finished keep was not cleared: %+v", mm.actions[0]) + } + if mm.dueAt(0).After(time.Now()) { + t.Error("the kept node was not brought forward for one more read") + } + if !strings.Contains(mm.statusLine, "keep — retain until 2030-01-02T04:00:00Z") { + t.Errorf("completion line: %q", mm.statusLine) + } +} + +// A keep that the control plane refuses is reported as a failure with its +// reason, and the board stays open — including the named no-update-URL case. +func TestDashKeepFailureShowsItsReason(t *testing.T) { + for _, tc := range []struct { + err error + want string + }{ + {errors.New("no update_url configured: the remote deployment needs to be updated for keep support"), "no update_url"}, + {errors.New("keep returned HTTP 404: no running instance"), "no running instance"}, + } { + t.Run(tc.want, func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped"), keepErr: tc.err} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, cmd := m.Update(dashKey("enter")) + m = next.(*dashModel) + fmsg, _ := runAction(t, cmd).(dashActionMsg) + m2, _ := m.Update(fmsg) + mm := m2.(*dashModel) + if mm.actions[0].verb != "" { + t.Fatalf("the failed keep was not cleared: %+v", mm.actions[0]) + } + if !strings.Contains(mm.statusLine, "keep failed —") { + t.Errorf("failure line does not say it failed: %q", mm.statusLine) + } + if !strings.Contains(mm.statusLine, tc.want) { + t.Errorf("failure line missing the reason %q: %q", tc.want, mm.statusLine) + } + }) + } +} + +// The abort key drives nothing on a keep in flight: only a start is abortable, +// so a keep's wait runs to its own end. +func TestDashAbortDrivesNothingOnAKeep(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, _ := m.Update(dashKey("enter")) + m = next.(*dashModel) + if m.actions[0].verb != "keep" { + t.Fatal("no keep in flight") + } + anext, _ := m.Update(dashKey("a")) + mm := anext.(*dashModel) + if mm.actions[0].verb != "keep" { + t.Fatalf("the abort ended the keep in flight: %+v", mm.actions[0]) + } + if mm.actions[0].aborted { + t.Fatal("the abort marked the keep as abandoned") + } +} + +// The keep hint shows only where the key would drive something: an idle remote +// node shows it, a local node hides it, and a busy remote node hides it. +func TestDashKeepHintOnlyWhereItDrivesSomething(t *testing.T) { + t.Run("idle remote shows it", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + if !strings.Contains(m.gridKeys(), "k keep") { + t.Errorf("grid hint missing the keep key: %q", m.gridKeys()) + } + if !strings.Contains(m.detailKeys(), "k keep") { + t.Errorf("detail hint missing the keep key: %q", m.detailKeys()) + } + }) + t.Run("local node hides it", func(t *testing.T) { + f := newFakeDashNode("stopped") + m := &dashModel{ + entries: []dashEntry{{name: "box", kind: fleet.KindDaemon, node: f}}, + results: []fleet.NodeResult{{Name: "box"}}, + actions: make([]dashAction, 1), + width: 120, height: 40, + } + if strings.Contains(m.gridKeys(), "k keep") { + t.Errorf("grid hint offers a keep a local node cannot take: %q", m.gridKeys()) + } + }) + t.Run("busy remote hides it", func(t *testing.T) { + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + m = openKeepPrompt(t, m) + next, _ := m.Update(dashKey("enter")) + m = next.(*dashModel) + if strings.Contains(m.gridKeys(), "k keep") { + t.Errorf("grid hint offers a second keep while one is in flight: %q", m.gridKeys()) + } + }) +} + +// The deadline rides the node's read onto the tile and the detail screen, as a +// relative keep after the active figure on the same line, and a read without one +// draws no keep. +func TestDashTileAndDetailShowKeep(t *testing.T) { + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + dashFixNow(t, at) + node := &keeperDashNode{f: newFakeDashNode("stopped")} + m := keeperModel(node) + r := fleet.NodeResult{ + Name: "env", + Outcome: fleet.OutcomeOK, + Metrics: metrics.Stats{ + State: "stopped", Runner: "llamacpp", ModelID: "org/m", + LastActiveAt: "2025-12-31T23:58:15Z", IdleSeconds: 125, + RetainUntil: at.Add(2 * time.Hour).UTC().Format(time.RFC3339), + }, + } + m.results[0] = r + + tile := dashTestTile("env", r, true, dashAction{}) + if line := aLineContaining(tile, "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("tile did not put the keep on the active line:\n%s", tile) + } + lines := m.detailNodeLines() + found := false + for _, l := range lines { + if strings.Contains(l, "keep for 2h") { + found = true + } + } + if !found { + t.Errorf("detail view missing the keep:\n%s", strings.Join(lines, "\n")) + } +} + +// A read without a deadline — a local node, or an unkept or lapsed environment — +// draws no keep on the tile. +func TestDashTileOmitsKeepWhenAbsent(t *testing.T) { + r := fleet.NodeResult{ + Name: "box", + Outcome: fleet.OutcomeOK, + Metrics: metrics.Stats{State: "running", Runner: "llamacpp", ModelID: "org/m", + CPU: &metrics.CpuStat{Utilization: 30}}, + } + tile := dashTestTile("box", r, true, dashAction{}) + if strings.Contains(tile, "keep for") { + t.Errorf("tile invented a keep the read does not carry:\n%s", tile) + } +} diff --git a/cmd/spinloop/dashboard_model.go b/cmd/spinloop/dashboard_model.go index 851eaec..424eb75 100644 --- a/cmd/spinloop/dashboard_model.go +++ b/cmd/spinloop/dashboard_model.go @@ -10,6 +10,7 @@ import ( "fmt" "strings" "time" + "unicode/utf8" tea "github.com/charmbracelet/bubbletea" "github.com/spinloop-ai/spinloop/internal/daemon" @@ -68,7 +69,13 @@ type dashModel struct { cursor int // the selected node scrollRow int // the first grid row on screen - confirm bool // a stop is waiting on its confirmation + confirm bool // a stop is waiting on its confirmation + // A keep is waiting on its duration: the prompt is open, the duration typed + // so far (pre-filled so the common case is one Enter), and the parse reason + // shown at the foot when a confirmed entry is not a positive duration. + keepPrompt bool + keepBuf string + keepErr string statusLine string // gauge is the board's resource-series format: false draws the bar @@ -159,12 +166,16 @@ type dashActionProgressMsg struct { phase fleet.StartPhase } -// dashActionMsg is one completed start or stop. +// dashActionMsg is one completed start, stop, or keep. type dashActionMsg struct { node string - verb string // "start" or "stop" + verb string // "start", "stop", or "keep" status daemon.StatusResponse err error + // retainUntil is the deadline a keep set, RFC 3339, carried so the footer's + // outcome can name it. Empty for a start or stop, and for a keep that failed + // before the control plane set anything. + retainUntil string } // dashTickCmd schedules the next fast tick. The tick is one-shot, so every @@ -281,6 +292,14 @@ func (m *dashModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } m.applyDetailLog(msg.result) case tea.KeyMsg: + // The keep prompt stands in front of everything: while it is open, + // every key goes to it — navigation, selection, refresh, even the stop + // confirmation — so the board holds still until the duration is entered + // or cancelled. It is checked before confirm and detail so it works + // whether it was opened from the grid or the detail view. + if m.keepPrompt { + return m, m.updateKeepPromptKey(msg) + } if m.confirm { switch msg.String() { case "y": @@ -347,6 +366,15 @@ func (m *dashModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if len(m.entries) > 0 && m.actions[m.cursor].verb == "" { cmd = m.beginAction("start") } + case "k": + // Open the duration prompt, pre-filled so the common case is one + // Enter. keepOffered gates it to a node that can be kept and has + // nothing in flight; elsewhere the key drives nothing. + if m.keepOffered() { + m.keepPrompt = true + m.keepBuf = "4h" + m.keepErr = "" + } case "a": // End the wait on the node's in-flight action — the one-shot // equivalent of Ctrl+C. A node with nothing in flight is driven @@ -515,6 +543,11 @@ func dashActionLine(msg dashActionMsg, aborted bool) string { } return msg.node + ": " + msg.verb + " failed — " + r.Detail() } + if msg.verb == "keep" { + // A keep has no engine state to report — its result is the deadline the + // control plane set, which is what the operator came to set. + return msg.node + ": keep — retain until " + msg.retainUntil + } state := msg.status.State if state == "" { state = "done" @@ -583,6 +616,94 @@ func (m *dashModel) beginAction(verb string) tea.Cmd { return run } +// beginKeep sets off a keep of the selected node for the confirmed duration. +// It reuses beginAction's scaffolding — one action per node, the tile's +// spinner, the short read interval for the duration of the call — but the call +// itself is a keep, not a start or stop: the node must be a Keeper (a remote +// environment), and the call returns the deadline it set rather than an engine +// state. A node that is not a Keeper, or that already has an action in flight, +// is driven by nothing, and its reason lands on the status line. +func (m *dashModel) beginKeep(d time.Duration) tea.Cmd { + e := m.entries[m.cursor] + m.keepPrompt = false + m.keepBuf = "" + m.keepErr = "" + if e.node == nil { + m.statusLine = e.name + ": " + e.standing.Detail() + return nil + } + if m.actions[m.cursor].verb != "" { + m.statusLine = e.name + ": still " + dashVerbProgress(m.actions[m.cursor].verb) + return nil + } + keeper, ok := e.node.(fleet.Keeper) + if !ok { + m.statusLine = e.name + ": keep needs a remote environment" + return nil + } + spin := !m.actionInFlight() + ctx, cancel := context.WithCancel(context.Background()) + m.actions[m.cursor] = dashAction{verb: "keep", since: dashNow(), cancel: cancel} + // The node is read on the short interval for the duration of the action, + // starting now rather than at its kind's next due time: what a keep changed + // is its own read's retainUntil. + m.scheduleRead(m.cursor, time.Time{}) + m.statusLine = dashVerbProgress("keep") + " " + e.name + "…" + run := func() tea.Msg { + deadline, err := keeper.Keep(ctx, d) + cancel() + return dashActionMsg{node: e.name, verb: "keep", retainUntil: deadline, err: err} + } + if spin { + return tea.Batch(run, dashSpinTickCmd()) + } + return run +} + +// updateKeepPromptKey answers the keys the keep prompt reads: printable runes +// append to the duration, backspace removes the last, Enter confirms, esc +// cancels, and q/Ctrl+C cancels and quits (mirroring the stop confirmation's +// own quit). A confirmed entry that is not a positive duration leaves the +// prompt open and names why at the foot, so a mistyped entry is corrected in +// place rather than lost. +func (m *dashModel) updateKeepPromptKey(msg tea.KeyMsg) tea.Cmd { + switch s := msg.String(); { + case s == "enter": + d, err := time.ParseDuration(strings.TrimSpace(m.keepBuf)) + if err != nil || d <= 0 { + m.keepErr = "enter a duration like 4h" + return nil + } + return m.beginKeep(d) + case s == "backspace": + m.keepBuf = chopLastRune(m.keepBuf) + m.keepErr = "" + case s == "esc": + m.keepPrompt = false + m.keepBuf = "" + m.keepErr = "" + case s == "ctrl+c", s == "q": + m.keepPrompt = false + m.keepBuf = "" + m.keepErr = "" + return tea.Quit + case msg.Type == tea.KeyRunes: + m.keepBuf += s + m.keepErr = "" + } + return nil +} + +// chopLastRune drops the final rune from a string, counting a wide rune as the +// bytes that make it up rather than cutting a UTF-8 sequence mid-way. +func chopLastRune(s string) string { + if s == "" { + return s + } + _, size := utf8.DecodeLastRuneInString(s) + return s[:len(s)-size] +} + // abortAction ends the wait on the selected node's in-flight start. Only a // start is abortable: it is the one action with no deadline of its own — a // cold cloud wake takes minutes, so a wait the operator no longer wants to @@ -617,6 +738,41 @@ func (m dashModel) canAbort() bool { return len(m.entries) > 0 && m.actions[m.cursor].verb == "start" } +// keepOffered reports whether the keep key would do anything for the node under +// the cursor: the node must support a keep (a remote environment) and have +// nothing in flight. A local daemon node has no retention tag to set, and a +// node already acting takes no second action. The footer uses this to include +// the keep hint only where it would drive something, the same way canAbort +// gates the abort hint. +func (m dashModel) keepOffered() bool { + if len(m.entries) == 0 { + return false + } + if m.actions[m.cursor].verb != "" { + return false + } + _, ok := m.entries[m.cursor].node.(fleet.Keeper) + return ok +} + +// gridKeys and detailKeys are the two footers' key help with the keep entry +// included only where keepOffered says the node under the cursor can be kept — +// the same gate the k key itself answers to, so a hint is never shown for a key +// that would drive nothing on that node. +func (m dashModel) gridKeys() string { + if m.keepOffered() { + return "↑↓←→ move s start k keep a abort x stop r refresh q quit" + } + return dashGridKeys +} + +func (m dashModel) detailKeys() string { + if m.keepOffered() { + return "esc back s start k keep x stop a abort f follow" + } + return dashDetailKeys +} + // indexOf finds an entry by name. Fleet-file names are unique — the fleet // file itself refuses a repeat — so a find is a position, not a set. func (m *dashModel) indexOf(name string) int { @@ -684,7 +840,7 @@ func (m dashModel) View() string { if hi > lo { parts = append(parts, strings.Join(rows[lo:hi], "\n")) } - parts = append(parts, m.footerLine(w, dashFooterHints(dashGridKeys, m.canAbort()))) + parts = append(parts, m.footerLine(w, dashFooterHints(m.gridKeys(), m.canAbort()))) return strings.Join(parts, "\n") } @@ -711,7 +867,16 @@ const dashGridKeys = "↑↓←→ move s start a abort x stop g format // splitting them on a space would draw their first word as though it were one. func (m dashModel) footerLine(w int, keys string) string { line := dashKeyHints(keys) - if m.confirm && len(m.entries) > 0 { + if m.keepPrompt && len(m.entries) > 0 { + // The prompt replaces the key help for its life: what is being typed, + // and the two keys that resolve it. The parse reason, when a confirmed + // entry was not a duration, rides at the foot beside the status line. + line = "keep " + m.entries[m.cursor].name + " for: " + m.keepBuf + "_ " + dashHintGap + + dashKeyHints("enter keep"+dashHintGap+"esc cancel") + if m.keepErr != "" { + line += " " + m.keepErr + } + } else if m.confirm && len(m.entries) > 0 { line = "stop " + m.entries[m.cursor].name + "?" + dashHintGap + dashKeyHints("y yes"+dashHintGap+"n no") } diff --git a/cmd/spinloop/dashboard_render.go b/cmd/spinloop/dashboard_render.go index df734cc..da6226c 100644 --- a/cmd/spinloop/dashboard_render.go +++ b/cmd/spinloop/dashboard_render.go @@ -297,7 +297,7 @@ func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time, if s := r.Metrics.State; s != "" { fmt.Fprintln(&b, dashStateLine(s, r.Metrics)+age) } - dashTileReportBody(&b, r.Metrics, true, gauge, lineW) + dashTileReportBody(&b, r.Metrics, true, gauge, lineW, now) } case r.Outcome == "": fmt.Fprintf(&b, "%s\nwaiting for first refresh…\n", name) @@ -315,7 +315,7 @@ func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time, if !gauge && len(r.Metrics.History) > 0 { resources = true } - dashTileReportBody(&b, r.Metrics, resources, gauge, lineW) + dashTileReportBody(&b, r.Metrics, resources, gauge, lineW, now) } lines := strings.Split(b.String(), "\n") lines = lines[:len(lines)-1] // the trailing newline splits an extra empty piece @@ -433,11 +433,15 @@ func dashStateLine(state string, m metrics.Stats) string { // answer has it. A settled tile gates the resources block on the node being // running; the in-flight tile draws whatever there is, because a boot half // done has some of the facts and not the rest. -func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool, lineW int) { +func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool, lineW int, now time.Time) { if line := dashTileServingLine(m); line != "" { fmt.Fprintln(w, line) } - renderLastActiveIndented(w, m.LastActiveAt, m.IdleSeconds) + // The active figure and, for a kept remote environment, the relative keep + // after it — one line, from the same read, whatever the engine's state. A + // read without either — a local node, an unkept or lapsed environment — + // draws nothing. + renderActiveIndented(w, m.LastActiveAt, m.IdleSeconds, m.RetainUntil, now) if resources { if gauge { renderStatGauges(w, m.CPU, m.Memory, m.GPUs) diff --git a/cmd/spinloop/fleet.go b/cmd/spinloop/fleet.go index c1c4de9..bf65c0c 100644 --- a/cmd/spinloop/fleet.go +++ b/cmd/spinloop/fleet.go @@ -187,6 +187,7 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string) if format == "json" { return renderFleetMetricsJSON(w, results) } + now := metricsNow() for i, r := range results { if i > 0 { fmt.Fprintln(w) @@ -203,8 +204,9 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string) fmt.Fprintln(w) // Before the continue, for the same reason the remote formats show it // before theirs: a node whose engine has stopped still has a useful - // answer to "when did this last do anything?". - renderLastActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds) + // answer to "when did it last do anything?" — and, for a retained + // remote environment, "how long is it kept?". + renderActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds, stats.RetainUntil, now) switch format { case "bar": // No state gate, for the same reason the remote bar format has diff --git a/cmd/spinloop/fleet_dashboard.go b/cmd/spinloop/fleet_dashboard.go index 45cfef6..baca0a1 100644 --- a/cmd/spinloop/fleet_dashboard.go +++ b/cmd/spinloop/fleet_dashboard.go @@ -26,11 +26,15 @@ func fleetDashboardCmd() *cobra.Command { what the bar format of fleet metrics prints — state, what it serves, the resource bars, the token counters — repainted on an interval. -The view is read-only apart from three keys: s starts the selected node, a -abandons a start or stop still in flight on it (the wait ends, the node is -free again — a wake the cloud is carrying goes on), x stops it after a -confirmation. The arrow keys move the selection, r forces a refresh, q or -Ctrl+C leaves. +The view is read-only apart from four keys: s starts the selected node, k +keeps a remote environment for a duration you type — it asks how long, +pre-filled with 4h, and reports the deadline the control plane set when the +keep is done — a abandons a start still in flight on it (the wait ends, the +node is free again — a wake the cloud is carrying goes on), x stops it after +a confirmation. The arrow keys move the selection, r forces a refresh, q or +Ctrl+C leaves. The keep key shows only for a node that can be kept — a remote +environment — and a kept environment's tile and detail view carry its +deadline beside the last-active line, whatever the engine's state. A node that cannot be reached is still a tile, showing why, and a node whose token reference is unresolvable holds its reason for the life of the view. diff --git a/cmd/spinloop/fleet_dashboard_test.go b/cmd/spinloop/fleet_dashboard_test.go index 06b24f6..70dd868 100644 --- a/cmd/spinloop/fleet_dashboard_test.go +++ b/cmd/spinloop/fleet_dashboard_test.go @@ -248,6 +248,10 @@ func dashKey(s string) tea.KeyMsg { return tea.KeyMsg{Type: tea.KeyEsc} case "ctrl+c": return tea.KeyMsg{Type: tea.KeyCtrlC} + case "enter": + return tea.KeyMsg{Type: tea.KeyEnter} + case "backspace": + return tea.KeyMsg{Type: tea.KeyBackspace} default: return tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)} } @@ -351,7 +355,7 @@ func TestDashTileRunningByteStable(t *testing.T) { want := dashTileExpected([]string{ dashExpectedHeader("up running (up 2h 0m 0s)", dashHealthy), "llamacpp org/qwen:q4", - " last active 12s ago", + " active 12s ago", dashBar("CPU", 42), dashBar("RAM", 30), dashBar("GPU util", 61), diff --git a/cmd/spinloop/fleet_test.go b/cmd/spinloop/fleet_test.go index 45a44a8..52441e7 100644 --- a/cmd/spinloop/fleet_test.go +++ b/cmd/spinloop/fleet_test.go @@ -771,8 +771,8 @@ func TestCmdFleetStatusShowsIdleTime(t *testing.T) { } }) // 125s formats the same way the uptime column does, so the two read alike. - if !strings.Contains(out, "last active 2m 5s ago") { - t.Errorf("last-active time missing or misformatted:\n%s", out) + if !strings.Contains(out, "active 2m 5s ago") { + t.Errorf("active time missing or misformatted:\n%s", out) } } @@ -794,7 +794,7 @@ func TestCmdFleetStatusOmitsIdleWithoutActivity(t *testing.T) { t.Error(err) } }) - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("activity claimed when the daemon recorded none:\n%s", out) } // The rest of the row is unaffected. diff --git a/cmd/spinloop/last_active_test.go b/cmd/spinloop/last_active_test.go index 24def7f..5838ff4 100644 --- a/cmd/spinloop/last_active_test.go +++ b/cmd/spinloop/last_active_test.go @@ -52,13 +52,13 @@ func TestRemoteMetricsBarShowsLastActive(t *testing.T) { t.Fatalf("cmdRemoteMetrics: %v", err) } }) - if !strings.Contains(out, "last active 2m 5s ago") { - t.Errorf("bar format missing the last-active line:\n%s", out) + if !strings.Contains(out, "active 2m 5s ago") { + t.Errorf("bar format missing the active line:\n%s", out) } // It belongs between the header and the bars: a fact about the endpoint, // read before the utilisation readings rather than among them. header := strings.Index(out, "g6e.xlarge") - active := strings.Index(out, "last active") + active := strings.Index(out, "active") bars := strings.Index(out, "CPU") if !(header < active && active < bars) { t.Errorf("last-active line is not between the header and the bars:\n%s", out) @@ -74,11 +74,11 @@ func TestRemoteMetricsTableShowsLastActive(t *testing.T) { } }) // Padded to the same key column as its neighbours, and beside uptime. - if !strings.Contains(out, "last active: 2m 5s ago") { - t.Errorf("table format missing the last-active row:\n%s", out) + if !strings.Contains(out, "active: 2m 5s ago") { + t.Errorf("table format missing the active row:\n%s", out) } - if strings.Index(out, "uptime:") > strings.Index(out, "last active:") { - t.Errorf("last active should follow uptime:\n%s", out) + if strings.Index(out, "uptime:") > strings.Index(out, "active:") { + t.Errorf("active should follow uptime:\n%s", out) } } @@ -119,8 +119,8 @@ func TestRemoteMetricsStoppedStillShowsLastActive(t *testing.T) { // The bar format indents the line to the bar-label column; the table // format makes it a key-value row. Same fact, each format's own idiom. for format, want := range map[string]string{ - "bar": "last active 1h 0m 0s ago", - "table": "last active: 1h 0m 0s ago", + "bar": "active 1h 0m 0s ago", + "table": "active: 1h 0m 0s ago", } { t.Run(format, func(t *testing.T) { out := captureStdout(t, func() { @@ -149,8 +149,8 @@ func TestLastActiveZeroIdleStillRenders(t *testing.T) { }`) for format, want := range map[string]string{ - "bar": "last active 0s ago", - "table": "last active: 0s ago", + "bar": "active 0s ago", + "table": "active: 0s ago", } { t.Run(format, func(t *testing.T) { out := captureStdout(t, func() { @@ -181,7 +181,7 @@ func TestLastActiveOmittedWithoutATimestamp(t *testing.T) { } }) // No line at all, rather than one implying it has sat unused. - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("%s format invented activity from nothing:\n%s", format, out) } if !strings.Contains(out, "CPU") { @@ -219,8 +219,8 @@ func TestRemoteStatusShowsLastActive(t *testing.T) { t.Fatalf("cmdRemoteStatus: %v", err) } }) - if !strings.Contains(out, "last active: 2m 5s ago") { - t.Errorf("status missing the last-active line:\n%s", out) + if !strings.Contains(out, "active: 2m 5s ago") { + t.Errorf("status missing the active line:\n%s", out) } // The lines it already printed are untouched. for _, want := range []string{"state: running", "healthy: true", "base_url: http://198.51.100.7:8000"} { @@ -238,7 +238,7 @@ func TestRemoteStatusZeroIdleStillRenders(t *testing.T) { t.Fatalf("cmdRemoteStatus: %v", err) } }) - if !strings.Contains(out, "last active: 0s ago") { + if !strings.Contains(out, "active: 0s ago") { t.Errorf("status hid an endpoint that is working right now:\n%s", out) } } @@ -253,7 +253,7 @@ func TestRemoteStatusOmitsLastActiveWhenAbsent(t *testing.T) { t.Fatalf("cmdRemoteStatus: %v", err) } }) - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("status invented activity for a stopped instance:\n%s", out) } if !strings.Contains(out, "state: stopped") { @@ -290,8 +290,8 @@ func TestFleetMetricsShowsLastActive(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if !strings.Contains(out, "last active 2m 5s ago") { - t.Errorf("fleet %s metrics missing the last-active line:\n%s", format, out) + if !strings.Contains(out, "active 2m 5s ago") { + t.Errorf("fleet %s metrics missing the active line:\n%s", format, out) } }) } @@ -341,7 +341,7 @@ func TestFleetMetricsOmitsLastActiveWithoutActivity(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if strings.Contains(out, "last active") { + if strings.Contains(out, "active") { t.Errorf("fleet metrics claimed activity a node never reported:\n%s", out) } } @@ -360,7 +360,7 @@ func TestFleetMetricsStoppedNodeStillShowsLastActive(t *testing.T) { t.Fatalf("cmdFleet metrics: %v", err) } }) - if !strings.Contains(out, "last active 10m 0s ago") { - t.Errorf("a stopped node dropped its last-active figure:\n%s", out) + if !strings.Contains(out, "active 10m 0s ago") { + t.Errorf("a stopped node dropped its active figure:\n%s", out) } } diff --git a/cmd/spinloop/metrics_render.go b/cmd/spinloop/metrics_render.go index fee496e..b373561 100644 --- a/cmd/spinloop/metrics_render.go +++ b/cmd/spinloop/metrics_render.go @@ -9,6 +9,7 @@ package main import ( "fmt" "io" + "time" "github.com/spinloop-ai/spinloop/internal/metrics" ) @@ -31,22 +32,89 @@ func lastActiveText(lastActiveAt string, idleSeconds int) string { return formatDuration(idleSeconds) + " ago" } -// renderLastActiveIndented draws the last-active line in the indented block -// the bar format and both fleet formats use, aligned to the bar-label column. -// -// Not a bar itself: an elapsed time has no ceiling to fill against, and a bar -// would imply one. -func renderLastActiveIndented(w io.Writer, lastActiveAt string, idleSeconds int) { - if text := lastActiveText(lastActiveAt, idleSeconds); text != "" { - fmt.Fprintf(w, " %-9s %s\n", "last active", text) +// metricsNow returns the current time for the one-shot metrics renderers. A +// variable so a test can fix the keep duration a read renders. The dashboard +// keeps its own clock (dashNow) and passes its now down instead. +var metricsNow = time.Now + +// keepText is the relative retention figure — "keep for 2h" — drawn after the +// active figure on the same line. It is "" when the read carries no deadline or +// the deadline is not in the future: the control plane is the home of the "is +// it still active?" judgement (it drops the field there), so this only formats +// the value it is given, never re-checks the clock against a tag. +func keepText(retainUntil string, now time.Time) string { + if retainUntil == "" { + return "" + } + deadline, err := time.Parse(time.RFC3339, retainUntil) + if err != nil { + return "" + } + d := deadline.Sub(now) + if d <= 0 { + return "" + } + return "keep for " + formatKeepDuration(d) +} + +// formatKeepDuration renders a keep's remaining time dropping zero units — +// "2h", "24m", "1h 30m" — so the figure stays short enough to share the active +// line in a 42-column tile. +func formatKeepDuration(d time.Duration) string { + d = d.Round(time.Second) + h := int(d.Hours()) + m := int(d.Minutes()) % 60 + s := int(d.Seconds()) % 60 + switch { + case h > 0: + if m > 0 { + if s > 0 { + return fmt.Sprintf("%dh %dm %ds", h, m, s) + } + return fmt.Sprintf("%dh %dm", h, m) + } + return fmt.Sprintf("%dh", h) + case m > 0: + if s > 0 { + return fmt.Sprintf("%dm %ds", m, s) + } + return fmt.Sprintf("%dm", m) + default: + return fmt.Sprintf("%ds", s) + } +} + +// renderActiveIndented draws the combined active-and-keep line in the indented +// block the bar format and the tile use: the "ago" figure, and, when the read +// carries a retention deadline, the relative keep after it — "active 2m 5s +// ago keep for 2h". The line is present whenever either fact is, and absent +// when neither is. It is not a bar: an elapsed time has no ceiling to fill +// against, and a bar would imply one. +func renderActiveIndented(w io.Writer, lastActiveAt string, idleSeconds int, retainUntil string, now time.Time) { + active := lastActiveText(lastActiveAt, idleSeconds) + keep := keepText(retainUntil, now) + switch { + case active != "" && keep != "": + fmt.Fprintf(w, " %-9s %s %s\n", "active", active, keep) + case active != "": + fmt.Fprintf(w, " %-9s %s\n", "active", active) + case keep != "": + fmt.Fprintf(w, " %s\n", keep) } } -// renderLastActiveKeyValue draws the same fact as a row of the table format, -// padded to the key column its neighbours use. -func renderLastActiveKeyValue(w io.Writer, lastActiveAt string, idleSeconds int) { - if text := lastActiveText(lastActiveAt, idleSeconds); text != "" { - fmt.Fprintf(w, "last active: %s\n", text) +// renderActiveKeyValue draws the same line as a row of the table format, the +// key padded to the column its neighbours use. +func renderActiveKeyValue(w io.Writer, lastActiveAt string, idleSeconds int, retainUntil string, now time.Time) { + active := lastActiveText(lastActiveAt, idleSeconds) + keep := keepText(retainUntil, now) + switch { + case active != "" && keep != "": + fmt.Fprintf(w, "%-13s %s %s\n", "active:", active, keep) + case active != "": + fmt.Fprintf(w, "%-13s %s\n", "active:", active) + case keep != "": + fmt.Fprintf(w, "%-13s %s\n", "active:", keep) } } diff --git a/cmd/spinloop/metrics_render_test.go b/cmd/spinloop/metrics_render_test.go index ecf2a97..5d569aa 100644 --- a/cmd/spinloop/metrics_render_test.go +++ b/cmd/spinloop/metrics_render_test.go @@ -285,20 +285,20 @@ func TestFormatMetricsBarStoppedWithHistory(t *testing.T) { t.Fatal(err) } want := "prod stopped org/qwen:q4\n" + - " last active 12s ago\n" + + " active 12s ago\n" + " CPU " + strings.Repeat(" ", 38) + "▁" + ansiGreen + "▂" + ansiReset + " 20%\n" if got := b.String(); got != want { t.Errorf("stopped bar = %q, want %q", got, want) } // The gauge format draws no series for a stopped endpoint: the header and - // the last-active line, and nothing after. + // the active line, and nothing after. b.Reset() if err := formatMetricsGauge(resp, remote.Config{}, &b); err != nil { t.Fatal(err) } want = "prod stopped org/qwen:q4\n" + - " last active 12s ago\n" + " active 12s ago\n" if got := b.String(); got != want { t.Errorf("stopped gauge = %q, want %q", got, want) } @@ -326,8 +326,8 @@ func TestFormatMetricsBarRunning(t *testing.T) { if !strings.HasPrefix(got, "prod running g5.xlarge org/qwen:q4 0.4.3\n") { t.Errorf("header: %q", got) } - if !strings.Contains(got, " last active 3s ago\n") { - t.Errorf("last active line missing: %q", got) + if !strings.Contains(got, " active 3s ago\n") { + t.Errorf("active line missing: %q", got) } // Every series drew a sparkline from the history: no gauge in the output. if strings.Contains(got, "░") { diff --git a/cmd/spinloop/remote.go b/cmd/spinloop/remote.go index 804c6fa..d712e2b 100644 --- a/cmd/spinloop/remote.go +++ b/cmd/spinloop/remote.go @@ -762,11 +762,18 @@ func runRemoteStatus(args []string) error { if resp.BaseURL != "" { fmt.Printf("base_url: %s\n", resp.BaseURL) } - if text := lastActiveText(fact.LastActiveAt, fact.IdleSeconds); text != "" { - fmt.Printf("last active: %s\n", text) - } - if resp.RetainUntil != "" { - fmt.Printf("retain until: %s\n", resp.RetainUntil) + // The active figure and, for a retained instance, the relative keep after + // it — one line, like the metrics view, rather than an absolute deadline on + // its own. + active := lastActiveText(fact.LastActiveAt, fact.IdleSeconds) + keep := keepText(resp.RetainUntil, metricsNow()) + switch { + case active != "" && keep != "": + fmt.Printf("active: %s %s\n", active, keep) + case active != "": + fmt.Printf("active: %s\n", active) + case keep != "": + fmt.Printf("active: %s\n", keep) } return nil } @@ -872,6 +879,7 @@ func runMetricsWatch(cfg remote.Config, format string, withCost bool) error { } func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCost bool, cfg remote.Config, w io.Writer) error { + now := metricsNow() fmt.Fprintf(w, "environment: %s\n", resp.Environment) fmt.Fprintf(w, "state: %s\n", resp.State) @@ -882,7 +890,9 @@ func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCos if resp.ModelID != "" { fmt.Fprintf(w, "model: %s\n", resp.ModelID) } - renderLastActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds) + // A stopped environment can still be retained: its deadline is the + // control plane's, so it rides this branch too. + renderActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) return nil } @@ -904,7 +914,7 @@ func formatMetricsTable(ctx context.Context, resp *remote.StatsResponse, withCos if resp.UptimeSeconds > 0 { fmt.Fprintf(w, "uptime: %s\n", formatDuration(resp.UptimeSeconds)) } - renderLastActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds) + renderActiveKeyValue(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) renderTokenLines(w, resp.Tokens) renderGPUTable(w, resp.GPUs) @@ -976,15 +986,16 @@ func formatMetricsHeader(resp *remote.StatsResponse, w io.Writer) { } func formatMetricsBar(resp *remote.StatsResponse, cfg remote.Config, w io.Writer) error { + now := metricsNow() formatMetricsHeader(resp, w) - // Before the series: when the endpoint last did work is worth showing for - // whatever state it is in, and the retained history is too — a stopped - // endpoint's readings up to the stop answer what it was doing until it - // stopped. A stopped endpoint's current reading carries no resource - // figures, so the series it draws come from the history alone, or not at - // all where the daemon predates it. - renderLastActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds) + // Before the series: when the endpoint last did work — and, for a retained + // endpoint, how long it is kept — is worth showing in whatever state it is in, + // and the retained history is too: a stopped endpoint's readings up to the + // stop answer what it was doing until it stopped. A stopped endpoint's + // current reading carries no resource figures, so the series it draws come + // from the history alone, or not at all where the daemon predates it. + renderActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) renderStatBars(w, resp.CPU, resp.Memory, resp.GPUs, resp.History, barLineW) renderTokenLines(w, resp.Tokens) @@ -994,12 +1005,13 @@ func formatMetricsBar(resp *remote.StatsResponse, cfg remote.Config, w io.Writer } func formatMetricsGauge(resp *remote.StatsResponse, cfg remote.Config, w io.Writer) error { + now := metricsNow() formatMetricsHeader(resp, w) - // Before the early return: a stopped endpoint draws no gauges, but when - // it last did work is exactly what a stopped endpoint is worth asking - // about. - renderLastActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds) + // Before the early return: a stopped endpoint draws no gauges, but when it + // last did work — and how long it is kept — is exactly what a stopped + // endpoint is worth asking about. + renderActiveIndented(w, resp.LastActiveAt, resp.IdleSeconds, resp.RetainUntil, now) if resp.State != "running" { return nil diff --git a/cmd/spinloop/retain_render_test.go b/cmd/spinloop/retain_render_test.go new file mode 100644 index 0000000..8626c3d --- /dev/null +++ b/cmd/spinloop/retain_render_test.go @@ -0,0 +1,230 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// The keep rides the stats read as an absolute deadline, and every surface that +// draws the active figure draws the keep as a relative figure after it — "active +// 2m 5s ago keep for 2h" — on the same line. The control plane is the single +// home of the "is it still in the future?" judgement (it drops the field there), +// so these renderers only ever ask "does the read carry a deadline that is still +// ahead?" and draw it when it is, and nothing when it is not. + +// keepNow pins the one-shot metrics clock and returns an RFC3339 deadline d +// after it, so a test can assert the exact relative figure that deadline +// renders. +func keepNow(t *testing.T, d time.Duration) string { + t.Helper() + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + metricsNow = func() time.Time { return at } + t.Cleanup(func() { metricsNow = time.Now }) + return at.Add(d).UTC().Format(time.RFC3339) +} + +// aLineContaining returns the first output line carrying every phrase, or "". +func aLineContaining(out string, phrases ...string) string { + for _, line := range strings.Split(out, "\n") { + ok := true + for _, p := range phrases { + if !strings.Contains(line, p) { + ok = false + break + } + } + if ok { + return line + } + } + return "" +} + +// A kept, active endpoint draws the keep after the active figure, on the same +// line — the point of sharing the line is that "when did it last do anything" +// and "how long is it kept" are read at a glance, not on two rows. +func TestRemoteMetricsBarKeepsOnTheActiveLine(t *testing.T) { + deadline := keepNow(t, 2*time.Hour) + statsServer(t, `{ + "environment": "dev", + "state": "running", + "instanceType": "g6e.xlarge", + "modelId": "unsloth/Qwen3.6-27B", + "cpu": {"utilization": 24}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + "retainUntil": "`+deadline+`" + }`) + + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=bar"}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if line := aLineContaining(out, "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("bar format did not put the keep on the active line:\n%s", out) + } + // Beside the active figure, before the bars: a fact about the endpoint, not + // a utilisation reading. + if line := aLineContaining(out, "keep for 2h"); line != "" { + if strings.Index(out, "CPU") < strings.Index(line, "keep for 2h") { + t.Errorf("keep is not before the bars:\n%s", out) + } + } +} + +// The table format draws the same combined line as a key-value row. +func TestRemoteMetricsTableKeepsOnTheActiveRow(t *testing.T) { + deadline := keepNow(t, 2*time.Hour) + statsServer(t, `{ + "environment": "dev", + "state": "running", + "instanceType": "g6e.xlarge", + "modelId": "unsloth/Qwen3.6-27B", + "cpu": {"utilization": 24}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + "retainUntil": "`+deadline+`" + }`) + + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=table"}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if line := aLineContaining(out, "active:", "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("table format did not put the keep on the active row:\n%s", out) + } +} + +// The relative figure drops zero units: hours, minutes, and a mix each render +// their own short form. +func TestKeepDurationRendersRelatively(t *testing.T) { + // A stopped, unactive endpoint shows the keep alone, so each duration is + // read straight off the bar. + for d, want := range map[time.Duration]string{ + 2 * time.Hour: "keep for 2h", + 24 * time.Minute: "keep for 24m", + 90 * time.Minute: "keep for 1h 30m", + } { + t.Run(want, func(t *testing.T) { + deadline := keepNow(t, d) + statsServer(t, `{ + "environment": "dev", + "state": "stopped", + "runner": "llamacpp", + "modelId": "unsloth/Qwen3.6-27B", + "retainUntil": "`+deadline+`" + }`) + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=bar"}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if !strings.Contains(out, want) { + t.Errorf("%s: %s format missing %q:\n%s", d, "bar", want, out) + } + }) + } +} + +// A stopped environment can still be kept: the deadline is the control plane's, +// not the engine's, so the keep survives the non-running short-circuit in both +// formats. +func TestRemoteMetricsStoppedKeptStillShowsKeep(t *testing.T) { + deadline := keepNow(t, 4*time.Hour) + for format := range map[string]bool{"bar": true, "table": true} { + t.Run(format, func(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "stopped", + "runner": "llamacpp", + "modelId": "unsloth/Qwen3.6-27B", + "retainUntil": "`+deadline+`" + }`) + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if !strings.Contains(out, "keep for 4h") { + t.Errorf("%s format dropped the keep for a stopped, kept endpoint:\n%s", format, out) + } + }) + } +} + +// No deadline on the read, no keep: the renderer does not invent one, and it +// leaves the active figure (now just "active") in place. +func TestRemoteMetricsOmitsKeepWhenAbsent(t *testing.T) { + for _, format := range []string{"bar", "table"} { + t.Run(format, func(t *testing.T) { + statsServer(t, `{ + "environment": "dev", + "state": "running", + "cpu": {"utilization": 24}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125 + }`) + out := captureStdout(t, func() { + if err := cmdRemoteMetrics([]string{"--format=" + format}); err != nil { + t.Fatalf("cmdRemoteMetrics: %v", err) + } + }) + if strings.Contains(out, "keep for") { + t.Errorf("%s format claimed a keep the read does not carry:\n%s", format, out) + } + if !strings.Contains(out, "2m 5s ago") { + t.Errorf("%s format lost the active figure:\n%s", format, out) + } + }) + } +} + +// The deadline rides the node's stats, so `fleet metrics` draws the keep after +// the active figure whatever the node's state. A faked daemon carrying the field +// stands in for a kept remote environment on the same render path. +func TestFleetMetricsShowsKeep(t *testing.T) { + deadline := keepNow(t, 2*time.Hour) + fleetNodeWithMetrics(t, map[string]any{ + "state": "running", + "runner": "llamacpp", + "modelId": "org/qwen", + "cpu": map[string]any{"utilization": 30.0}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + "retainUntil": deadline, + }) + + out := captureStdout(t, func() { + if err := cmdFleet([]string{"metrics"}); err != nil { + t.Fatalf("cmdFleet metrics: %v", err) + } + }) + if line := aLineContaining(out, "2m 5s ago", "keep for 2h"); line == "" { + t.Errorf("fleet metrics did not put the keep on the active line:\n%s", out) + } +} + +// A node whose read carries no deadline draws no keep, so a local daemon node — +// which never has one — is simply quieter on that figure. +func TestFleetMetricsOmitsKeepWhenAbsent(t *testing.T) { + fleetNodeWithMetrics(t, map[string]any{ + "state": "running", + "runner": "llamacpp", + "modelId": "org/qwen", + "cpu": map[string]any{"utilization": 30.0}, + "lastActiveAt": "2025-12-31T23:58:15Z", + "idleSeconds": 125, + }) + + out := captureStdout(t, func() { + if err := cmdFleet([]string{"metrics"}); err != nil { + t.Fatalf("cmdFleet metrics: %v", err) + } + }) + if strings.Contains(out, "keep for") { + t.Errorf("fleet metrics claimed a keep a local node never carries:\n%s", out) + } +} diff --git a/cmd/spinloop/status_render.go b/cmd/spinloop/status_render.go index fc8ec17..0486480 100644 --- a/cmd/spinloop/status_render.go +++ b/cmd/spinloop/status_render.go @@ -45,10 +45,10 @@ func (f statusFact) servingText() string { } // How long since it last did work — deliberately not labelled "idle": that // word is already an engine state meaning nothing has been started. Shown - // only when there is a recorded last-active time; without one there is - // nothing to measure from. + // only when there is a recorded active time; without one there is nothing + // to measure from. if f.LastActiveAt != "" { - serving += fmt.Sprintf(" (last active %s ago)", formatDuration(f.IdleSeconds)) + serving += fmt.Sprintf(" (active %s ago)", formatDuration(f.IdleSeconds)) } if f.Version != "" { serving += fmt.Sprintf(" (%s)", f.Version) diff --git a/cmd/spinloop/status_render_test.go b/cmd/spinloop/status_render_test.go index 4838355..9ca27e8 100644 --- a/cmd/spinloop/status_render_test.go +++ b/cmd/spinloop/status_render_test.go @@ -13,7 +13,7 @@ func TestStatusFactServingText(t *testing.T) { UptimeSeconds: 30, LastActiveAt: "2026-01-02T00:00:00Z", IdleSeconds: 30, } got := f.servingText() - for _, want := range []string{"llamacpp qwen", "(up 30s)", "(last active 30s ago)", "(1.2.0)"} { + for _, want := range []string{"llamacpp qwen", "(up 30s)", "(active 30s ago)", "(1.2.0)"} { if !strings.Contains(got, want) { t.Errorf("servingText %q missing %q", got, want) } diff --git a/docs/commands/fleet.md b/docs/commands/fleet.md index bb3626e..2d26513 100644 --- a/docs/commands/fleet.md +++ b/docs/commands/fleet.md @@ -225,12 +225,12 @@ failure — the rest of the fleet still renders and the command still exits 0: ``` NODE STATE SERVING -studio running llamacpp org/qwen (up 1h 2m 5s) (last active 12s ago) +studio running llamacpp org/qwen (up 1h 2m 5s) (active 12s ago) gpu-box idle llamacpp org/qwen offline unreachable dial tcp 10.0.0.9:4242: connect: connection refused ``` -"last active" comes from the activity each daemon tracks, so a glance answers +"active" comes from the activity each daemon tracks, so a glance answers "which of my nodes is doing nothing?". It is absent until a node's engine has actually done some work — a daemon that has served nothing reports no activity rather than claiming it has been quiet since it started. The wording avoids @@ -256,12 +256,19 @@ no history falls back to the gauge drawing of its current reading, so a fleet mixed with older daemons renders each node the best way it can. A stopped node keeps its readings, so its sparkline runs to the stop. -Each node's block carries the same `last active` figure the status table +Each node's block carries the same `active` figure the status table shows, for the reasons given above, and on the same terms: absent until the node's engine has done some work. A node whose engine has *stopped* still shows it — the daemon keeps the record across a stop, and "how long since this did anything?" is worth more about a stopped engine than about a busy one. +A `kind: remote` environment carries a relative keep after that figure, on the +same line — `active 2m 5s ago keep for 2h` — on the same omitted-when-absent +terms: it shows how long the idle sweep will hold the box while the deadline is +in the future, and is gone once it has passed or was never set. It is the same +line the dashboard draws on a kept environment's tile and detail screen, from +the same read. + `--watch`/`-w` redraws the whole fleet on an interval, clearing the screen in place with no scrollback. Each refresh is rendered into a buffer first, so a slow node delays the refresh but never tears the display. Ctrl+C exits @@ -302,6 +309,7 @@ spinloop fleet dashboard --fleet f.yaml # another fleet file | `r` | Force a refresh of every node, now | | `g` | Toggle every tile's resource series between bar (sparklines of the retained history) and gauge (the current reading) | | `s` | Start the selected node — without confirmation | +| `k` | Keep a remote environment for a duration you type — shown only for a node that can be kept, and only while it has no action in flight | | `a` | Abandon a start in flight on the selected node — the wait ends, the node is free again (a stop in flight is not abortable) | | `x` | Stop the selected node — it asks first (`y` sends, `n` or `esc` cancel) | | `q` or `Ctrl+C` | Leave | @@ -330,8 +338,25 @@ refresh. A stop in flight is not abortable: it targets an engine already running rather than a cold wake with no deadline of its own, and `a` drives nothing while one is in progress. +`keep` is a remote-environment action: a local daemon has no idle sweep, so +there is no deadline to set, and the key does not show for one. Pressing it +opens a prompt at the foot of the view, pre-filled with `4h`, asking how long +the environment should be retained. The prompt is the confirmation — there is +no second one — so the operator sees the duration it will set before choosing +to send it: a keep overwrites the deadline and ends nothing, where a stop +ends something and so asks. Type the duration and press `enter` to send it; +`esc` cancels; `q` or `Ctrl+C` cancel the prompt and leave the dashboard, as +the stop confirmation does. An entry that does not parse as a positive +duration leaves the prompt open and shows the parse reason in the footer's +hint slot, so the entry is kept and corrected in place. While the keep runs +its tile carries it, and it is not abortable — one fast signed call, so `a` +drives nothing on it. When it finishes, the status line reports the deadline +the control plane set and the node is re-read at once, which is what brings +the relative `keep for …` figure onto the tile and detail screen at the node's +next round rather than waiting out its full cadence. + Everything else in the view is `fleet status`/`metrics`/`logs` in place — it -is read-only apart from those three action keys. It needs a real terminal: a +is read-only apart from those four action keys. It needs a real terminal: a piped run is refused, and it says so by way of `fleet metrics --watch`, which is the streamable surface. @@ -348,11 +373,13 @@ spinloop fleet dashboard # select a node, press Enter for its full metrics and log, Esc to go back ``` -`s`, `x` and `a` drive the node shown exactly as they drive the selected node -on the grid — the same no-confirmation start, the same stop confirmation, the -same abandon. `q`/`Ctrl+C` are grid keys only and do nothing here — `Esc` back -to the grid first, then quit from there — so a stray quit keystroke while -looking at a node can't end the session out from under you. The rest of the +`s`, `k`, `x` and `a` drive the node shown exactly as they drive the selected +node on the grid — the same no-confirmation start, the same keep prompt, the +same stop confirmation, the same abandon. `q`/`Ctrl+C` are grid keys only and +do nothing here — `Esc` back to the grid first, then quit from there — so a +stray quit keystroke while looking at a node can't end the session out from +under you. The one exception is the keep prompt: while it is open it answers +to `q`/`Ctrl+C` the way the stop confirmation does, cancelling and leaving. The rest of the fleet keeps refreshing behind the view, and any action already in flight on another node keeps running. A node whose engine has never run shows the same explanation `fleet logs` gives for it, not an empty pane. @@ -427,7 +454,7 @@ Fleet: ./fleet.yaml Prefer: idle Would use gpu-box at http://gpu-box:8080/v1 - serving qwen3-27b, last active 312s ago (prefer idle) + serving qwen3-27b, active 312s ago (prefer idle) ``` When nothing is serving that model it shows the whole fleet's state and names diff --git a/docs/commands/remote.md b/docs/commands/remote.md index 0527ba3..86d4b9b 100644 --- a/docs/commands/remote.md +++ b/docs/commands/remote.md @@ -169,7 +169,7 @@ of the current reading, so the default output degrades rather than goes blank. A stopped engine keeps its readings: the sparkline runs to the stop, ending at it. -Both report **`last active`** — how long since the endpoint's engine last did +Both report **`active`** — how long since the endpoint's engine last did any work. It comes from the activity the on-instance daemon tracks, so it is one answer decided on the box rather than something each command re-derives from raw counters. `status` asks the daemon alongside the health check it diff --git a/docs/openapi.yaml b/docs/openapi.yaml index d15b272..fde05ce 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -433,6 +433,16 @@ components: Absent, not `not-ready`, when it does not apply: no engine is running, its runner has no known health-check convention, or this daemon predates the check. + retainUntil: + type: string + format: date-time + description: | + The environment's retention deadline, RFC 3339: the idle sweep + will not terminate the instance before it. A property of the cloud + instance, not the engine, so it is absent for local daemon nodes + and for remote environments without an update URL. The stats reply + carries it only while it is a time in the future — a passed + deadline keeps nothing, so it is dropped there and on this read. TokenStats: type: object diff --git a/internal/fleet/node.go b/internal/fleet/node.go index 1be6af2..040619a 100644 --- a/internal/fleet/node.go +++ b/internal/fleet/node.go @@ -54,6 +54,20 @@ type ProgressStarter interface { StartWithProgress(ctx context.Context, report func(StartPhase)) (daemon.StatusResponse, error) } +// Keeper is an optional node capability: a node whose instance can be pinned so +// the idle sweep does not terminate it before a stated deadline. Only remote +// environments have it — the retention tag lives on a cloud instance — so a +// local daemon node does not implement it. A caller that can offer a keep +// (the dashboard) asserts for it and hides the key when it is absent. +// +// d is how long, from now, to keep the instance. The return is the control +// plane's own deadline, RFC 3339: the absolute instant the instance is kept +// until, as the control plane recorded it — not the caller's clock plus d — so +// what a caller shows matches what the control plane holds. +type Keeper interface { + Keep(ctx context.Context, d time.Duration) (string, error) +} + // Node is one member of the fleet. Only daemonNode implements it today; the // interface exists so a remote-environment kind (an `spinloop remote` // environment read through its stats Lambda, which already yields diff --git a/internal/fleet/remote_node.go b/internal/fleet/remote_node.go index 7501c3a..5425fbc 100644 --- a/internal/fleet/remote_node.go +++ b/internal/fleet/remote_node.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/spinloop-ai/spinloop/internal/daemon" "github.com/spinloop-ai/spinloop/internal/metrics" @@ -105,6 +106,26 @@ func (n *remoteNode) Stop(ctx context.Context) (daemon.StatusResponse, error) { return statusFromRemote(*resp), nil } +// Keep pins this environment's instance so the idle sweep does not terminate it +// before now plus d. The deadline is computed here and passed as an absolute +// time, the way the CLI computes it; a keep is what keeps the instance, not +// what the engine is doing, so the call needs no engine or deploy context. If +// the control plane's reply names the deadline it recorded, that is returned +// verbatim; when the reply omits it — a control plane that predates keep +// echoing the value back — the requested deadline is returned instead, so a +// caller always holds something to show. +func (n *remoteNode) Keep(ctx context.Context, d time.Duration) (string, error) { + deadline := time.Now().Add(d) + resp, err := remote.Keep(ctx, n.cfg, deadline) + if err != nil { + return "", err + } + if resp.RetainUntil != "" { + return resp.RetainUntil, nil + } + return deadline.UTC().Format(time.RFC3339), nil +} + // remoteEngineTail caps how many engine log events a node read pulls. Remote logs // are a bounded tail pulled from the log store, not a byte cursor, so a follow of // a chatty engine must not page through an unbounded window. @@ -165,6 +186,7 @@ func statsFromRemote(resp remote.StatsResponse) metrics.Stats { Errors: resp.Errors, LastActiveAt: resp.LastActiveAt, IdleSeconds: resp.IdleSeconds, + RetainUntil: resp.RetainUntil, } } diff --git a/internal/fleet/remote_node_test.go b/internal/fleet/remote_node_test.go index 6d8ec0f..a617891 100644 --- a/internal/fleet/remote_node_test.go +++ b/internal/fleet/remote_node_test.go @@ -82,7 +82,8 @@ func TestStatsFromRemote(t *testing.T) { got := statsFromRemote(remote.StatsResponse{ State: "running", Runner: "llamacpp", ModelID: "org/m", UptimeSeconds: 10, Tokens: tokens, LastActiveAt: "2026-01-02T00:00:00Z", IdleSeconds: 5, Version: "1.2.3", - History: history, + History: history, + RetainUntil: "2026-01-02T04:00:00Z", }) if got.State != "running" || got.Runner != "llamacpp" || got.ModelID != "org/m" || got.UptimeSeconds != 10 { t.Errorf("statsFromRemote = %+v", got) @@ -103,6 +104,14 @@ func TestStatsFromRemote(t *testing.T) { if got := statsFromRemote(remote.StatsResponse{State: "running"}); got.History != nil { t.Errorf("an absent history became present: %+v", got.History) } + if got.RetainUntil != "2026-01-02T04:00:00Z" { + t.Errorf("retainUntil not carried over: %q", got.RetainUntil) + } + // An environment with no live retention carries nothing: the field is + // absent, not a zero time a formatter would have to special-case. + if got := statsFromRemote(remote.StatsResponse{State: "running"}); got.RetainUntil != "" { + t.Errorf("no retention should map to an empty retainUntil, got %q", got.RetainUntil) + } } func TestLogsFromRemote(t *testing.T) { @@ -234,6 +243,77 @@ func TestRemoteNodeStatusOverTheControlPlane(t *testing.T) { } } +// A remote node's keep pins the instance over its control plane: the deadline +// is computed from the duration and sent as an absolute time, and the value +// returned to a caller is the control plane's own deadline, not the caller's +// clock plus the duration. +func TestRemoteNodeKeepOverTheControlPlane(t *testing.T) { + stubAWSCreds(t) + srv := remoteControlServer(t, + `{"retainUntil":"2026-01-02T04:00:00Z"}`, http.StatusOK) + cfg := remote.Config{StartURL: srv.URL, StopURL: srv.URL, UpdateURL: srv.URL, Region: "us-east-1"} + node, err := NewRemoteNode("env", cfg) + if err != nil { + t.Fatal(err) + } + got, err := node.(Keeper).Keep(context.Background(), 4*time.Hour) + if err != nil { + t.Fatal(err) + } + if got != "2026-01-02T04:00:00Z" { + t.Errorf("keep returned %q, want the control plane's deadline", got) + } +} + +// When the control plane's reply omits the deadline — one that predates keep +// echoing the value back — the keep still succeeds and returns the requested +// deadline, so a caller always has something to show. +func TestRemoteNodeKeepFallsBackToTheRequestedDeadline(t *testing.T) { + stubAWSCreds(t) + srv := remoteControlServer(t, `{}`, http.StatusOK) + cfg := remote.Config{StartURL: srv.URL, StopURL: srv.URL, UpdateURL: srv.URL, Region: "us-east-1"} + node, _ := NewRemoteNode("env", cfg) + before := time.Now() + got, err := node.(Keeper).Keep(context.Background(), 4*time.Hour) + if err != nil { + t.Fatal(err) + } + deadline, perr := time.Parse(time.RFC3339, got) + if perr != nil { + t.Fatalf("fallback deadline is not RFC 3339: %q", got) + } + want := before.Add(4 * time.Hour) + if d := deadline.Sub(want); d < -5*time.Second || d > 5*time.Second { + t.Errorf("fallback deadline = %s, want the requested now+4h (%s)", deadline, want) + } +} + +// A keep on an environment whose config has no update URL is a configuration +// error before any call, naming the fix — it is not a call that fails part-way. +func TestRemoteNodeKeepWithoutAnUpdateURL(t *testing.T) { + stubAWSCreds(t) + cfg := remote.Config{StartURL: "http://x", StopURL: "http://x", Region: "us-east-1"} + node, _ := NewRemoteNode("env", cfg) + if _, err := node.(Keeper).Keep(context.Background(), time.Hour); err == nil || + !strings.Contains(err.Error(), "no update_url") { + t.Errorf("expected a no-update-url error, got %v", err) + } +} + +// The keep capability is exactly one of the remote node's: a local daemon node +// has no retention tag to set, so it does not implement Keeper. A caller (the +// dashboard) relies on this boundary to decide whether to offer a keep at all. +func TestKeeperIsRemoteOnly(t *testing.T) { + rn, _ := NewRemoteNode("env", remote.Config{StartURL: "http://x", StopURL: "http://x", Region: "r"}) + if _, ok := rn.(Keeper); !ok { + t.Error("a remote node should implement Keeper") + } + dn := &daemonNode{name: "dev", client: &Client{BaseURL: "http://127.0.0.1:1", Token: "t"}} + if _, ok := any(dn).(Keeper); ok { + t.Error("a local daemon node should not implement Keeper") + } +} + // A remote node drives start, stop and metrics over its control plane exactly // like a node would, mapping each reply onto the node's types. func TestRemoteNodeStartStopMetricsOverTheControlPlane(t *testing.T) { diff --git a/internal/fleet/select.go b/internal/fleet/select.go index d93574c..bf0c5f2 100644 --- a/internal/fleet/select.go +++ b/internal/fleet/select.go @@ -286,7 +286,7 @@ func reasonFor(c candidate, w Want, woken bool) string { case c.result.Status.LastActiveAt == "": return fmt.Sprintf("serving %s, no work yet (prefer %s)", serving, w.prefer()) default: - return fmt.Sprintf("serving %s, last active %ds ago (prefer %s)", + return fmt.Sprintf("serving %s, active %ds ago (prefer %s)", serving, c.result.Status.IdleSeconds, w.prefer()) } } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index be14a88..860c0bf 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -52,6 +52,13 @@ type Stats struct { // no engine is running, its runner has no known health-check convention, // or this daemon predates the check. Ready string `json:"ready,omitempty"` + // RetainUntil is the environment's retention deadline, RFC 3339: the idle + // sweep will not terminate the instance before it. It is a property of the + // cloud instance, not the engine, so it is empty for local daemon nodes and + // for remote environments without an update URL. Empty once the deadline + // has passed, because the stats reply drops it there — a past tag keeps + // nothing. Formatters omit the line when it is empty. + RetainUntil string `json:"retainUntil,omitempty"` } // TokenStats holds per-engine token/request counters from /metrics. diff --git a/internal/remote/remote.go b/internal/remote/remote.go index 1da21cb..189150a 100644 --- a/internal/remote/remote.go +++ b/internal/remote/remote.go @@ -745,6 +745,11 @@ type StatsResponse struct { // the daemon's /v1/status by the stats Lambda. Empty when the daemon was // unreachable or the control plane predates this. Version string `json:"version"` + // RetainUntil is the instance's retention deadline, RFC 3339, read from the + // Retain-Until tag by the stats Lambda. Empty when the instance has no tag + // or its deadline has already passed — the reply carries it only while it + // still keeps the instance alive — and for control planes that predate keep. + RetainUntil string `json:"retainUntil"` } // The stat sub-types are aliases into internal/metrics, their canonical home diff --git a/internal/remote/remote_test.go b/internal/remote/remote_test.go index 964675e..c896f6b 100644 --- a/internal/remote/remote_test.go +++ b/internal/remote/remote_test.go @@ -966,7 +966,8 @@ func TestStats_Success(t *testing.T) { "temperature": 72 }], "cpu": {"utilization": 23.5}, - "memory": {"total": 17179869184, "used": 4294967296} + "memory": {"total": 17179869184, "used": 4294967296}, + "retainUntil": "2026-01-02T04:00:00Z" }`)) })) defer server.Close() @@ -979,6 +980,9 @@ func TestStats_Success(t *testing.T) { if resp.Environment != "dev" || resp.State != "running" { t.Errorf("unexpected response: %+v", resp) } + if resp.RetainUntil != "2026-01-02T04:00:00Z" { + t.Errorf("retainUntil not decoded: %q", resp.RetainUntil) + } if resp.Tokens == nil || resp.Tokens.Requests != 342 { t.Errorf("unexpected tokens: %+v", resp.Tokens) } @@ -1014,6 +1018,10 @@ func TestStats_Stopped(t *testing.T) { if resp.State != "stopped" || resp.Tokens != nil || len(resp.GPUs) != 0 { t.Errorf("stopped instance should have no metrics: %+v", resp) } + // A reply without the field leaves it empty, not a zero time. + if resp.RetainUntil != "" { + t.Errorf("no retainUntil in the reply should decode empty, got %q", resp.RetainUntil) + } } func TestStats_WithEnvironment(t *testing.T) { diff --git a/openspec/changes/archive/2026-09-06-dashboard-keep/.openspec.yaml b/openspec/changes/archive/2026-09-06-dashboard-keep/.openspec.yaml new file mode 100644 index 0000000..1a62d62 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-dashboard-keep/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-06 diff --git a/openspec/changes/archive/2026-09-06-dashboard-keep/design.md b/openspec/changes/archive/2026-09-06-dashboard-keep/design.md new file mode 100644 index 0000000..02576cb --- /dev/null +++ b/openspec/changes/archive/2026-09-06-dashboard-keep/design.md @@ -0,0 +1,187 @@ +# Design: dashboard keep + +## Context + +See proposal.md for the motivation (issues #158 and #157). The state of the +code the change lands in: + +- The dashboard (`cmd/spinloop`, roughly 1,500 lines of model and render) is + Bubble Tea with every widget hand-rolled — the spinner, the key-help footer, + the grid scrolling — and no Bubbles component. Start and stop run through a + per-node action value (`dashAction`): the call runs in its own goroutine, + reports progress onto the node's tile, and on completion the footer shows + the one-shot outcome and the node is re-read immediately. +- Keep already works end to end for the one-shot CLI: `remote.Keep` sends + `set-keep` to the update Lambda, which sets the `Retain-Until` EC2 tag, and + the idle sweep honours it. The fleet's `remoteNode` wraps the same + `remote.Config` and already answers status, metrics, start, stop and logs + through it. +- The dashboard refreshes remote environments through the stats Lambda + (the metrics call), whose reply does not today carry the deadline. The + Lambda already parses the tag into its instance info; it does not send it. +- Nodes advertise partial capabilities through optional interfaces that the + dashboard type-asserts (a start that reports progress), rather than through + methods every node must answer. + +## Goals / Non-Goals + +**Goals:** + +- Keep settable from the grid and the detail screen for remote environments, + with a duration the operator types. +- The deadline visible on the tile and the detail screen, from the normal + refresh, worded the same as every one-shot surface that reads the same + stats reply. +- No new dependencies; the dashboard's hand-rolled, terminal-free test style + unchanged. +- A control plane that predates the reply field reads back cleanly and shows + no line. + +**Non-Goals:** + +- Clearing or shortening a keep from the dashboard: setting a new deadline + overwrites, and the sweep takes over once a deadline passes. +- Keep for local daemon nodes: there is no idle sweep on a daemon, so no + deadline to set. +- Changing how `remote status` reports the tag: it keeps reporting the tag's + value as it does today, including a passed one. +- Any interactive account of the sweep itself (when it next runs, why it + skipped an instance). + +## Decisions + +### 1. A hand-rolled duration field, no Bubbles + +The prompt is one state flag and one buffer string on the dashboard model, +driven from the model's existing key switch: printable runes append, backspace +deletes, enter confirms, escape cancels. A duration is a handful of characters +from a fixed set (digits and `h`, `m`, `s`), typed left to right, so a cursor +model, selection, and paste handling would be machinery for input that never +arrives. This matches how the dashboard already owns its spinner, key help and +scrolling, and the state tests without a terminal exactly like the rest of the +model. + +Alternative considered: `bubbles/textinput`. It is the first Bubbles use in a +deliberately hand-rolled TUI, it is a general text editor where a duration +needs four keys, and styling it into the one-accent-chrome footer would cost +most of writing the field. + +### 2. A capability interface, not a Node method + +Keep rides a new optional capability interface in the style of the existing +progress-reporting start, implemented only by the remote node. The dashboard +type-asserts for it, as it already does for the progress start, and the key +help and the key handler gate on the same assertion. + +Alternatives considered: adding `Keep` to the node interface would force the +local daemon node to answer a method it can never honestly answer; gating on +the fleet file's node kind in the dashboard would move a node-kind decision +into the screen layer and around the node abstraction. + +### 3. Keep takes a duration and returns the deadline + +The capability method takes a duration and returns the deadline the control +plane set. Computing now-plus-duration inside the node keeps "now" in one +place for every keeper, and the reply's value — not the caller's arithmetic — +is what the footer line and the panel show. + +### 4. The deadline rides the stats reply, only while active + +The stats Lambda includes the parsed tag in its reply only while it is a time +in the future, and the client maps it across the existing path: the stats +reply type, the node's stats mapping, and the shared stats shape, as an +omitted-when-absent field. The "is it still active" judgement sits in the +control plane, once, instead of in every client that reads the reply; a passed +tag is no active retention, so nothing downstream can draw a deadline the +sweep has already moved past. + +Alternatives considered: the dashboard issuing a status read alongside its +metrics read for remote environments doubles the signed control-plane calls +per environment per minute for one line; filtering passed deadlines on the +client side leaves a stale deadline on the wire for every other reader and +puts a clock comparison in each of them. + +The field is additive: an environment whose stats Lambda has not been +redeployed simply omits it, and the dashboard shows no line there. + +### 5. One shared line: the active figure, and the keep after it + +The active figure — how long since the engine last did work — and, for a kept +environment, the retention keep render as a single line in the shared body the +tile and both one-shot metrics reports call, so they cannot word the same read +differently: `active 2m 5s ago keep for 2h`. The keep is the deadline's +remaining time, rendered relatively by the client rather than the absolute +timestamp the control plane holds — short enough to share the line in a +42-column tile, and it reads at a glance without the operator subtracting. The +table format prints the same line as its key-value row; the json format carries +the absolute field on its own, for a consumer that wants to do its own math. +The line is omitted where the read has neither an active time nor a deadline, +and shows whichever it has when it has one. + +### 6. No second confirmation; not abortable + +The prompt is the confirmation: the operator sees the duration it will set, +and choosing to send it is the commitment. A stop asks because it ends +something; a keep overwrites a deadline and ends nothing. The abort key exists +to end a wait that has no deadline of its own — a cold cloud wake; a keep is +one fast signed call, so abort drives nothing on it, as it does on a stop. + +### 7. The prompt state mirrors the stop confirmation + +The prompt is handled in the model's update before the grid/detail dispatch, +the same position the stop confirmation occupies, so it works from both +screens and navigation, selection and refreshes all stand still while it is +open. Confirming an entry that does not parse as a positive duration leaves +the prompt open and shows the parse reason in the footer's hint slot — the +operator keeps their entry and corrects it in place. Escape cancels; quit and +interrupt cancel the prompt and leave the dashboard, exactly as the stop +confirmation does. + +The prompt opens pre-filled with `4h`, the duration the one-shot command's +help already advertises as its example. The common overnight keep is then the +key and a confirm, and a mistaken confirm still does what the footer showed. + +### 8. The key help names keep only where it does something + +The footer's key help drops the keep entry for a node that does not support +retention or has an action in flight — the same mechanism that already drops +the abort entry where nothing is abortable. + +### 9. The action reuses the existing in-flight machinery + +Keep runs through the per-node action value with its own verb: the spinner and +elapsed time on the tile, one action per node, the node re-read immediately on +completion — which is what brings the new deadline onto the panel at the +node's next round rather than waiting out its full cadence. The action's +completion message carries the deadline so the footer line can report the +value the control plane set; start and stop leave it empty. + +## Risks / Trade-offs + +- [A pre-filled `4h` means a stray confirm sets four hours of retention] → + the buffer is visible in the footer before the confirm; a keep overwrites + and the sweep resumes when the deadline passes, so a mistaken one is + correctable and bounded; the default is the CLI's own example value, not a + larger one. +- [The Lambda change reaches environments only as each deployment updates] → + the field is additive and the keep action itself reports its deadline on + the status line from its own reply, so the feature is complete before the + panel line appears; the line then lands per environment as deployments + update. +- [A stopped instance keeps its tag across the stop] → intended: the sweep + must not terminate a retained stopped instance, and the line on a stopped + tile is exactly the protection the operator set. +- [The hand-rolled field has no cursor motion] → a duration is typed left to + right and is at most a handful of characters; backspace and retype is + cheaper than the cursor keys an editor would bring. + +## Migration Plan + +- The Go side (client, capability, dashboard, shared line) and the stats + Lambda change ship together; nothing on the Go side requires the Lambda + change to function — it only makes the panel line appear. +- The Lambda change is a routine deployment of the remote project; no data + migration, no client gating: older clients ignore the extra field and newer + clients omit the line where the field is absent. +- Rolling back the CLI/dashboard removes the key; the extra reply field is + inert to older clients. diff --git a/openspec/changes/archive/2026-09-06-dashboard-keep/proposal.md b/openspec/changes/archive/2026-09-06-dashboard-keep/proposal.md new file mode 100644 index 0000000..5a27b62 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-dashboard-keep/proposal.md @@ -0,0 +1,66 @@ +# Dashboard keep + +## Why + +The dashboard can start and stop a fleet's nodes, but the one fact an operator +needs while watching a remote environment — until when the cloud's idle sweep +will leave it alone — is neither settable nor visible there. Keeping a node +alive (overnight debugging, a long-running task) means leaving the dashboard +and running `spinloop remote keep` one-shot, and the deadline it sets then +appears nowhere in the view the operator is actually watching. Two open issues +ask for the two halves: #158, set a user-specified keep period from the +dashboard, and #157, show the keep-until time on the node's detail screen. + +## What Changes + +- The dashboard gains a keep action on the selected node: a key opens a + duration prompt (a hand-rolled single-line field in the existing model — no + new dependencies), and confirming sends the node's retention deadline to now + plus the entered duration, through the same control-plane path + `spinloop remote keep` already uses. +- Keep is offered only where it can work: `kind: remote` environments, through + a node capability in the style of the existing progress-reporting start. The + key help hides the key for local daemon nodes, exactly as it already hides + the abort key where it would do nothing. +- The fleet's read of a remote environment carries the retention deadline: the + stats Lambda already parses the instance's Retain-Until tag and now includes + it in its reply, and the client maps it onto the shared stats shape. Older + control planes omit the field, and the deadline line simply does not appear + — the same graceful degradation the stats reply's other relayed facts use. +- The dashboard's node panel shows the deadline — the tile and the full-screen + detail screen draw the same lines, so both show it — whenever a read carries + one, and omits it when a node has no active retention. The line is the + shared one the bar format prints, so the one-shot `fleet metrics` and + `remote metrics` render it the same way. +- Keep proceeds without confirmation — it overwrites the deadline and ends + nothing — and is not abortable: it is a single fast call, like a stop. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `fleet-client`: the dashboard gains a keep action (the duration prompt, its + validation, its in-flight and outcome handling, and the key help), and the + node panel — tile and detail screen — shows the retention deadline a read + carries. +- `remote-stats`: the stats Lambda's reply carries the instance's retention + deadline when the Retain-Until tag is present, so every surface that reads + the environment's stats can show it. + +## Impact + +- `internal/fleet`: a new capability interface, implemented by the remote node + over the existing `remote.Keep` call; local daemon nodes do not implement it. +- `internal/remote`: the stats reply type gains the deadline field; the node's + stats mapping carries it across. +- `internal/metrics`: the shared stats shape gains the deadline field. +- `remote/lambda/stats`: the reply includes the tag the Lambda already parses. +- `cmd/spinloop`: the dashboard model (prompt state and keys, the action), the + shared line renderer, and the command's help text. +- `docs/commands/fleet.md`: the dashboard's keys. +- No new dependencies. The control-plane change is additive: an environment on + an older deployment reads back without the field and shows no line. diff --git a/openspec/changes/archive/2026-09-06-dashboard-keep/specs/fleet-client/spec.md b/openspec/changes/archive/2026-09-06-dashboard-keep/specs/fleet-client/spec.md new file mode 100644 index 0000000..15458f7 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-dashboard-keep/specs/fleet-client/spec.md @@ -0,0 +1,170 @@ +## ADDED Requirements + +### Requirement: The dashboard keeps the selected node + +The dashboard SHALL let the operator set the retention deadline of the node +currently selected, from the keyboard, through the same node operation the +one-shot keep command uses. Keep applies to a node's instance — the time until +which the cloud's idle sweep leaves it alone — so it SHALL be offered only for +nodes that support retention, namely the fleet's remote environments; a node +without retention support SHALL take no keep. + +The keep key SHALL open a duration prompt rather than send anything. The +prompt SHALL name the node it will apply to, show the duration as the operator +has it, and name its confirm and cancel keys. It SHALL open pre-filled with a +default duration, so that the common case is the key and a confirm, and the +confirm SHALL send the duration as shown, whatever the operator has changed it +to. The duration SHALL be expressed in the same syntax the one-shot keep +command accepts. + +A confirm of an entry that is not a positive duration SHALL send nothing: the +prompt SHALL stay open and show why the entry was rejected, so the operator can +correct it and confirm again. A cancel SHALL close the prompt and send nothing. +Once the prompt is open, the navigation and selection keys SHALL drive +nothing: the prompt takes the keyboard until it is confirmed or cancelled. + +Keep SHALL not ask for a second confirmation after the prompt: the operator +has typed, or deliberately kept, the duration and chosen to send it, and a +keep overwrites a deadline rather than ending anything. A keep in flight SHALL +be shown on the node's panel like any action — its verb and how long it has +been running — and SHALL NOT be abortable: it is one fast call with no +open-ended wait, and abandoning the wait on it would leave the operator unsure +whether the deadline was set. + +A node that already has an action in flight SHALL take no keep; the dashboard +SHALL say it is still working and drive nothing. The key help SHALL name the +keep key only for the node it describes when that node supports retention and +has no action in flight, so the operator is never invited to press a key that +would do nothing there. + +The outcome of a keep SHALL be shown on the status line: on success the +deadline the control plane set, on failure the control plane's own reason — a +deployment that predates keep support, an environment with no instance to +retain — and a failed keep SHALL NOT close the dashboard. What the keep +changes — the deadline the node now carries — SHALL appear on the node's panel +through the normal refresh, without the operator asking for it. + +The node's detail view SHALL offer keep on the node it shows, through the same +prompt and the same rules the grid applies to the selected node. + +#### Scenario: Keep opens a prompt, it does not send + +- **WHEN** the operator selects a remote environment with no action in flight + and issues keep +- **THEN** a duration prompt opens naming the node, pre-filled with a default + duration, and nothing has been sent + +#### Scenario: A confirmed keep sets the deadline + +- **WHEN** the operator changes the prompt's duration and confirms it +- **THEN** the node is kept until now plus the confirmed duration and the + status line shows the deadline the control plane set + +#### Scenario: A pre-filled keep is one key and a confirm + +- **WHEN** the operator issues keep on a remote environment and confirms the + prompt without changing its duration +- **THEN** the node is kept until now plus the default duration + +#### Scenario: A bad duration sends nothing + +- **WHEN** the operator confirms an entry that is not a positive duration +- **THEN** nothing is sent, the prompt stays open, and it shows why the entry + was rejected + +#### Scenario: Cancelling sends nothing + +- **WHEN** the operator opens the keep prompt and cancels it +- **THEN** the prompt closes and the node's deadline is unchanged + +#### Scenario: The prompt takes the keyboard + +- **WHEN** the keep prompt is open and the operator presses a navigation key +- **THEN** the selection does not move and the prompt stays open + +#### Scenario: A local node takes no keep + +- **WHEN** the operator issues keep on a local daemon node +- **THEN** the dashboard drives nothing: a local daemon has no idle sweep to + hold off, so there is no deadline to set + +#### Scenario: The key help hides keep where it would do nothing + +- **WHEN** the node under the cursor is a local daemon node, or has an action + in flight +- **THEN** the key help does not name the keep key +- **WHEN** that node is a remote environment with no action in flight +- **THEN** the key help names the keep key + +#### Scenario: A busy node is not kept again + +- **WHEN** the operator issues keep on a node whose start, stop, or keep is + still in flight +- **THEN** the dashboard drives nothing and says the node is still working + +#### Scenario: A keep in flight cannot be aborted + +- **WHEN** the operator issues the abort on a node whose keep is in flight +- **THEN** the abort drives nothing: the keep keeps running, the panel keeps + showing it, and its outcome lands on the status line when the call returns + +#### Scenario: A keep that fails keeps the dashboard open + +- **WHEN** a keep is sent to an environment whose deployment predates keep + support, or to one with no instance to retain +- **THEN** the failure is shown on the status line with the control plane's + own reason and the dashboard keeps running with its refreshes + +#### Scenario: Keep from the detail view + +- **WHEN** the operator issues keep from the detail view of a remote + environment +- **THEN** the same prompt opens with the same rules, and its outcome is + shown as the grid would show it + +### Requirement: The dashboard panels show the retention deadline + +A node's panel SHALL show the node's retention deadline — the time until which +the idle sweep leaves the node alone — whenever the node's last answered +refresh carries one, worded the same way the one-shot surfaces report it, and +SHALL omit the line when the answer carries none: a node with no active +retention, a node without retention at all, or a read from a control plane +that predates the field. The deadline SHALL appear on the tile and on the +node's detail screen, which draw the same lines, so the two cannot show the +same read differently, and it SHALL sit among the panel's other time facts +rather than displacing the node's state or what it serves. + +A deadline that has passed carries no active retention: a refresh taken after +the deadline SHALL not show the line, and a panel SHALL NOT draw a deadline +the sweep has already moved past as though the node were still held. + +#### Scenario: A retained node shows its deadline + +- **WHEN** a remote environment's refresh answer carries a retention deadline + in the future +- **THEN** its tile shows the deadline, and its detail screen shows the same + line + +#### Scenario: A node with no retention shows no line + +- **WHEN** a node's refresh answer carries no retention deadline +- **THEN** its panel shows no deadline line and renders the rest exactly as + before + +#### Scenario: An older control plane degrades to no line + +- **WHEN** a remote environment's control plane predates the deadline in its + stats reply +- **THEN** its panel shows no deadline line and no error, and the rest of the + panel is unaffected + +#### Scenario: A passed deadline disappears + +- **WHEN** a node's retention deadline has passed and a later refresh answers +- **THEN** the panel no longer shows the deadline + +#### Scenario: Tile and detail agree + +- **WHEN** the operator opens the detail screen of a retained node +- **THEN** the detail screen's deadline line is the same line, worded the + same, the tile draws for the same read diff --git a/openspec/changes/archive/2026-09-06-dashboard-keep/specs/remote-stats/spec.md b/openspec/changes/archive/2026-09-06-dashboard-keep/specs/remote-stats/spec.md new file mode 100644 index 0000000..20ce9b0 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-dashboard-keep/specs/remote-stats/spec.md @@ -0,0 +1,59 @@ +## ADDED Requirements + +### Requirement: The stats reply carries the retention deadline + +The stats Lambda's reply SHALL carry the instance's retention deadline — the +time until which the idle sweep leaves the instance alone, parsed from the +instance's Retain-Until tag, which the Lambda already reads — when that tag is +a time in the future, and SHALL omit it when the tag is absent or a time that +has passed. The deadline is a fact the control plane holds about the instance, +not one the on-instance daemon reports: it SHALL be present for a stopped +instance the sweep would otherwise terminate, and absent for an environment +with no instance at all. + +A control plane that predates the field SHALL simply omit it, with no error: +every reader of the reply treats an absent deadline as "no active retention" +rather than as a failure. + +The client SHALL map the deadline onto the shared stats shape every fleet and +remote stats surface reads from, so a dashboard panel, a one-shot fleet report, +and a one-shot remote report cannot word the same read differently. + +The deadline SHALL be reported on the report's active-figure line — rendered by +the client as a relative remaining time, e.g. `keep for 2h`, not the absolute +timestamp — in every format the command supports, and omitted in every format +when the read carries none, following the same omission rule the active figure +uses for a figure it does not have. + +#### Scenario: A retained instance's stats carry the deadline + +- **WHEN** the user reads the stats of an environment whose instance carries a + Retain-Until tag at a time in the future +- **THEN** the reply carries that deadline + +#### Scenario: A passed tag carries no deadline + +- **WHEN** the user reads the stats of an instance whose Retain-Until tag is a + time that has passed +- **THEN** the reply carries no deadline: a passed tag holds no active + retention + +#### Scenario: An untagged or instance-less environment carries no deadline + +- **WHEN** the user reads the stats of an instance with no Retain-Until tag, + or of an environment with no instance +- **THEN** the reply carries no deadline + +#### Scenario: An older control plane is read without error + +- **WHEN** the user reads the stats of an environment whose control plane + predates the field +- **THEN** the read succeeds, the deadline is simply absent, and the rest of + the report renders as before + +#### Scenario: Every format carries the deadline + +- **WHEN** the user reads the stats of a retained environment with `bar`, + `table`, or `json` output +- **THEN** each output carries the deadline in its own idiom on the active + figure's line, and each omits it when the read carries none diff --git a/openspec/changes/archive/2026-09-06-dashboard-keep/tasks.md b/openspec/changes/archive/2026-09-06-dashboard-keep/tasks.md new file mode 100644 index 0000000..95bec4a --- /dev/null +++ b/openspec/changes/archive/2026-09-06-dashboard-keep/tasks.md @@ -0,0 +1,98 @@ +# Tasks: dashboard keep + +## 1. The deadline rides the stats read (Go) + +- [x] 1.1 Add a `RetainUntil` field to `metrics.Stats`, omitted when empty, + beside `LastActiveAt` and `Ready` — verify with `go build ./...` and the + mapping test in 1.3. +- [x] 1.2 Add a `RetainUntil` field to `remote.StatsResponse`, matching the + Lambda's JSON — verify with a decode test in `internal/remote` that a + reply carrying the field lands on it and one without leaves it empty. +- [x] 1.3 Carry the deadline across the node's stats mapping + (`statsFromRemote`) — verify with `internal/fleet` tests: a stats reply + with a deadline ends up on `metrics.Stats`, and one without leaves it + empty. + +## 2. The stats Lambda reports the deadline while it is active + +- [x] 2.1 Include the instance's parsed Retain-Until tag in the stats reply + only while it is a time in the future — verify with vitest cases in + `remote/test` (a future tag is present, a passed tag is absent, an + untagged instance is absent) and `pnpm test` passing in `remote/`. + +## 3. A node capability for keep + +- [x] 3.1 Add the keep capability interface — take a duration, return the + deadline the control plane set — beside the progress-reporting start + interface in `internal/fleet` — verify with `go build ./...`. +- [x] 3.2 Implement the capability on the remote node over the existing + `remote.Keep` call — verify with `internal/fleet` tests: a keep sends + the set-keep command with the now-plus-duration deadline and returns the + control plane's value, and a config without the update URL surfaces the + control plane's named error. +- [x] 3.3 Confirm the local daemon node does not implement the capability — + verify by a test asserting the dashboard's type assertion fails for it + (the same test file that covers the progress-start assertion). + +## 4. The shared deadline line + +- [x] 4.1 Merge the relative keep onto the shared active-figure line in the + bar-format body — `active … ago keep for …` — omitted when the read + carries neither — verify with render tests: the line's wording when a + keep is present, and its omission when the field is empty. +- [x] 4.2 Wire the line into the one-shot `fleet metrics` bar format, the + `remote metrics` bar format, and the `remote metrics` table format when + present — verify with the existing render tests for those surfaces + extended for the line. + +## 5. The dashboard keep action + +- [x] 5.1 Add the prompt state to the dashboard model — a flag and a duration + buffer — opened by the keep key from the grid and the detail view, + pre-filled with `4h`, handled before the grid/detail dispatch so + navigation, selection and refreshes stand still while it is open — + verify with model tests: the prompt opens on the key for a remote node, + drives nothing for a local node, and navigation keys do nothing while it + is open. +- [x] 5.2 Handle the prompt's keys — append, backspace, confirm, cancel — and + leave the prompt open with the parse reason in the footer when a confirm + is not a positive duration — verify with model tests over valid entries + (`4h`, `90m`, `1h30m`) and invalid ones (`4hours`, empty). +- [x] 5.3 Run a confirmed keep through the per-node action machinery — verb + `keep`, the tile's spinner and elapsed time, one action per node, the + node re-read immediately on completion — and carry the deadline on the + action's completion message — verify with model tests: a keep in flight + shows on the tile, a busy node takes no second keep, and completion + clears the action and schedules the re-read. +- [x] 5.4 Render the footer outcome for a finished keep — success shows the + deadline the control plane set, failure shows its reason, the dashboard + stays open — verify with tests over the line's wording, including the + no-update-URL failure. +- [x] 5.5 Make the abort key drive nothing on a keep in flight — verify with a + model test alongside the existing stop-in-flight case. +- [x] 5.6 Show the keep entry in the key help only for a node that supports + keep and has no action in flight, on both the grid and the detail + footer — verify with tests: a local node hides it, a busy remote node + hides it, an idle remote node shows it. +- [x] 5.7 Draw the deadline line on the tile and the detail screen from the + read, omitted when the read carries none — verify with tile render + tests: a read with a deadline shows the line on the tile, a read without + one shows no line, and the detail screen draws the same line. +- [x] 5.8 Update the `fleet dashboard` command's long help to name the keep + key — verify with `go run ./cmd/spinloop fleet dashboard --help` + printing the key alongside the existing ones. + +## 6. Docs + +- [x] 6.1 Update `docs/commands/fleet.md` for the keep key, the duration + prompt, and the deadline line on panels — verify by reading the section + against the implemented keys and lines. + +## 7. Final checks + +- [x] 7.1 `go test ./... -cover` at 80% total or above, `go vet ./...`, and + `gofmt -l` clean — verify all three pass. +- [x] 7.2 The `remote/` suite (`pnpm test`) passes with the new stats reply + field — verify the run is green. +- [x] 7.3 `openspec validate dashboard-keep` passes — verify the command + reports the change valid. diff --git a/openspec/specs/fleet-client/spec.md b/openspec/specs/fleet-client/spec.md index d6759b6..d9b1353 100644 --- a/openspec/specs/fleet-client/spec.md +++ b/openspec/specs/fleet-client/spec.md @@ -1317,3 +1317,171 @@ the key help line SHALL name it. - **WHEN** the dashboard draws its key help line - **THEN** it names `g` as the format toggle +### Requirement: The dashboard keeps the selected node + +The dashboard SHALL let the operator set the retention deadline of the node +currently selected, from the keyboard, through the same node operation the +one-shot keep command uses. Keep applies to a node's instance — the time until +which the cloud's idle sweep leaves it alone — so it SHALL be offered only for +nodes that support retention, namely the fleet's remote environments; a node +without retention support SHALL take no keep. + +The keep key SHALL open a duration prompt rather than send anything. The +prompt SHALL name the node it will apply to, show the duration as the operator +has it, and name its confirm and cancel keys. It SHALL open pre-filled with a +default duration, so that the common case is the key and a confirm, and the +confirm SHALL send the duration as shown, whatever the operator has changed it +to. The duration SHALL be expressed in the same syntax the one-shot keep +command accepts. + +A confirm of an entry that is not a positive duration SHALL send nothing: the +prompt SHALL stay open and show why the entry was rejected, so the operator can +correct it and confirm again. A cancel SHALL close the prompt and send nothing. +Once the prompt is open, the navigation and selection keys SHALL drive +nothing: the prompt takes the keyboard until it is confirmed or cancelled. + +Keep SHALL not ask for a second confirmation after the prompt: the operator +has typed, or deliberately kept, the duration and chosen to send it, and a +keep overwrites a deadline rather than ending anything. A keep in flight SHALL +be shown on the node's panel like any action — its verb and how long it has +been running — and SHALL NOT be abortable: it is one fast call with no +open-ended wait, and abandoning the wait on it would leave the operator unsure +whether the deadline was set. + +A node that already has an action in flight SHALL take no keep; the dashboard +SHALL say it is still working and drive nothing. The key help SHALL name the +keep key only for the node it describes when that node supports retention and +has no action in flight, so the operator is never invited to press a key that +would do nothing there. + +The outcome of a keep SHALL be shown on the status line: on success the +deadline the control plane set, on failure the control plane's own reason — a +deployment that predates keep support, an environment with no instance to +retain — and a failed keep SHALL NOT close the dashboard. What the keep +changes — the deadline the node now carries — SHALL appear on the node's panel +through the normal refresh, without the operator asking for it. + +The node's detail view SHALL offer keep on the node it shows, through the same +prompt and the same rules the grid applies to the selected node. + +#### Scenario: Keep opens a prompt, it does not send + +- **WHEN** the operator selects a remote environment with no action in flight + and issues keep +- **THEN** a duration prompt opens naming the node, pre-filled with a default + duration, and nothing has been sent + +#### Scenario: A confirmed keep sets the deadline + +- **WHEN** the operator changes the prompt's duration and confirms it +- **THEN** the node is kept until now plus the confirmed duration and the + status line shows the deadline the control plane set + +#### Scenario: A pre-filled keep is one key and a confirm + +- **WHEN** the operator issues keep on a remote environment and confirms the + prompt without changing its duration +- **THEN** the node is kept until now plus the default duration + +#### Scenario: A bad duration sends nothing + +- **WHEN** the operator confirms an entry that is not a positive duration +- **THEN** nothing is sent, the prompt stays open, and it shows why the entry + was rejected + +#### Scenario: Cancelling sends nothing + +- **WHEN** the operator opens the keep prompt and cancels it +- **THEN** the prompt closes and the node's deadline is unchanged + +#### Scenario: The prompt takes the keyboard + +- **WHEN** the keep prompt is open and the operator presses a navigation key +- **THEN** the selection does not move and the prompt stays open + +#### Scenario: A local node takes no keep + +- **WHEN** the operator issues keep on a local daemon node +- **THEN** the dashboard drives nothing: a local daemon has no idle sweep to + hold off, so there is no deadline to set + +#### Scenario: The key help hides keep where it would do nothing + +- **WHEN** the node under the cursor is a local daemon node, or has an action + in flight +- **THEN** the key help does not name the keep key +- **WHEN** that node is a remote environment with no action in flight +- **THEN** the key help names the keep key + +#### Scenario: A busy node is not kept again + +- **WHEN** the operator issues keep on a node whose start, stop, or keep is + still in flight +- **THEN** the dashboard drives nothing and says the node is still working + +#### Scenario: A keep in flight cannot be aborted + +- **WHEN** the operator issues the abort on a node whose keep is in flight +- **THEN** the abort drives nothing: the keep keeps running, the panel keeps + showing it, and its outcome lands on the status line when the call returns + +#### Scenario: A keep that fails keeps the dashboard open + +- **WHEN** a keep is sent to an environment whose deployment predates keep + support, or to one with no instance to retain +- **THEN** the failure is shown on the status line with the control plane's + own reason and the dashboard keeps running with its refreshes + +#### Scenario: Keep from the detail view + +- **WHEN** the operator issues keep from the detail view of a remote + environment +- **THEN** the same prompt opens with the same rules, and its outcome is + shown as the grid would show it + +### Requirement: The dashboard panels show the retention deadline + +A node's panel SHALL show the node's retention deadline — the time until which +the idle sweep leaves the node alone — whenever the node's last answered +refresh carries one, worded the same way the one-shot surfaces report it, and +SHALL omit the line when the answer carries none: a node with no active +retention, a node without retention at all, or a read from a control plane +that predates the field. The deadline SHALL appear on the tile and on the +node's detail screen, which draw the same lines, so the two cannot show the +same read differently, and it SHALL sit among the panel's other time facts +rather than displacing the node's state or what it serves. + +A deadline that has passed carries no active retention: a refresh taken after +the deadline SHALL not show the line, and a panel SHALL NOT draw a deadline +the sweep has already moved past as though the node were still held. + +#### Scenario: A retained node shows its deadline + +- **WHEN** a remote environment's refresh answer carries a retention deadline + in the future +- **THEN** its tile shows the deadline, and its detail screen shows the same + line + +#### Scenario: A node with no retention shows no line + +- **WHEN** a node's refresh answer carries no retention deadline +- **THEN** its panel shows no deadline line and renders the rest exactly as + before + +#### Scenario: An older control plane degrades to no line + +- **WHEN** a remote environment's control plane predates the deadline in its + stats reply +- **THEN** its panel shows no deadline line and no error, and the rest of the + panel is unaffected + +#### Scenario: A passed deadline disappears + +- **WHEN** a node's retention deadline has passed and a later refresh answers +- **THEN** the panel no longer shows the deadline + +#### Scenario: Tile and detail agree + +- **WHEN** the operator opens the detail screen of a retained node +- **THEN** the detail screen's deadline line is the same line, worded the + same, the tile draws for the same read diff --git a/openspec/specs/remote-stats/spec.md b/openspec/specs/remote-stats/spec.md index eb5d33d..187ff6b 100644 --- a/openspec/specs/remote-stats/spec.md +++ b/openspec/specs/remote-stats/spec.md @@ -181,5 +181,62 @@ When the on-instance daemon's metrics reply carries a history of system readings #### Scenario: A daemon without history degrades - **WHEN** the instance runs a daemon whose reply carries no history and the user runs `spinloop remote metrics` -- **THEN** the report omits the history field and bar format draws the current reading in the gauge's filled style +- **THEN** the report omits the history field and bar format draws the current reading in the gauge's filled style +### Requirement: The stats reply carries the retention deadline + +The stats Lambda's reply SHALL carry the instance's retention deadline — the +time until which the idle sweep leaves the instance alone, parsed from the +instance's Retain-Until tag, which the Lambda already reads — when that tag is +a time in the future, and SHALL omit it when the tag is absent or a time that +has passed. The deadline is a fact the control plane holds about the instance, +not one the on-instance daemon reports: it SHALL be present for a stopped +instance the sweep would otherwise terminate, and absent for an environment +with no instance at all. + +A control plane that predates the field SHALL simply omit it, with no error: +every reader of the reply treats an absent deadline as "no active retention" +rather than as a failure. + +The client SHALL map the deadline onto the shared stats shape every fleet and +remote stats surface reads from, so a dashboard panel, a one-shot fleet report, +and a one-shot remote report cannot word the same read differently. + +The deadline SHALL be reported on the report's active-figure line — rendered by +the client as a relative remaining time, e.g. `keep for 2h`, not the absolute +timestamp — in every format the command supports, and omitted in every format +when the read carries none, following the same omission rule the active figure +uses for a figure it does not have. + +#### Scenario: A retained instance's stats carry the deadline + +- **WHEN** the user reads the stats of an environment whose instance carries a + Retain-Until tag at a time in the future +- **THEN** the reply carries that deadline + +#### Scenario: A passed tag carries no deadline + +- **WHEN** the user reads the stats of an instance whose Retain-Until tag is a + time that has passed +- **THEN** the reply carries no deadline: a passed tag holds no active + retention + +#### Scenario: An untagged or instance-less environment carries no deadline + +- **WHEN** the user reads the stats of an instance with no Retain-Until tag, + or of an environment with no instance +- **THEN** the reply carries no deadline + +#### Scenario: An older control plane is read without error + +- **WHEN** the user reads the stats of an environment whose control plane + predates the field +- **THEN** the read succeeds, the deadline is simply absent, and the rest of + the report renders as before + +#### Scenario: Every format carries the deadline + +- **WHEN** the user reads the stats of a retained environment with `bar`, + `table`, or `json` output +- **THEN** each output carries the deadline in its own idiom on the active + figure's line, and each omits it when the read carries none diff --git a/remote/lambda/shared/stats.ts b/remote/lambda/shared/stats.ts index 2b6de34..75232e2 100644 --- a/remote/lambda/shared/stats.ts +++ b/remote/lambda/shared/stats.ts @@ -109,4 +109,13 @@ export interface StatsResult { * the field — the formatters simply omit the line. */ version?: string; + /** + * The instance's retention deadline, RFC 3339: the idle sweep will not + * terminate it before this. Carried only while the instance's Retain-Until + * tag is still a time in the future — a passed deadline keeps nothing, so it + * is dropped here, and an untagged instance has none. It is a property of the + * cloud instance, not the engine, so it is present whatever the engine's + * state — a stopped, retained instance still reports it. + */ + retainUntil?: string; } diff --git a/remote/lambda/stats/index.ts b/remote/lambda/stats/index.ts index 15f7eee..64c0c94 100644 --- a/remote/lambda/stats/index.ts +++ b/remote/lambda/stats/index.ts @@ -2,6 +2,7 @@ import type { LambdaFunctionURLEvent, LambdaFunctionURLResult } from 'aws-lambda import { errorName, findManagedInstance, + type InstanceInfo, readDeployConfig, requireEnv, runShellCommand, @@ -29,6 +30,23 @@ function envFilter(env: string) { return [{ Name: `tag:${ENV_TAG_KEY}`, Values: [env] }]; } +/** + * The retention deadline to carry in a reply, or nothing: present only while + * the instance's Retain-Until tag is still a time in the future. A passed + * deadline keeps nothing, so the reply drops it there; an untagged instance and + * a not-yet-deployed environment have none. The judgement lives here, not in the + * formatters, so a reader of the reply need not re-check the clock — and it + * applies to every reply branch, since a stopped, retained instance still + * reports its deadline. + */ +function retainUntilIfActive(instance: InstanceInfo | null): string | undefined { + const until = instance?.retainUntil; + if (until && until.getTime() > Date.now()) { + return until.toISOString(); + } + return undefined; +} + /** * The stats Lambda called by `spinloop remote metrics`. The control plane * contributes what only it knows — environment, instance id/type, uptime @@ -65,6 +83,9 @@ export async function handler(event: LambdaFunctionURLEvent): Promise { expect(DAEMON_STATUS_CMD).toContain(DAEMON_UNREACHABLE); }); }); + +// The stats Lambda: reports an environment's instance and engine metrics, and — +// while its Retain-Until tag is still a time in the future — the retention +// deadline itself, in every reply branch. + +const LAMBDA_ENV = { + TAG_KEY: 'cloud-vm-llm:managed', + TAG_VALUE: 'true', +}; + +const findManagedInstance = vi.fn(); +const readDeployConfig = vi.fn(); +const runShellCommand = vi.fn(); + +vi.mock('../lambda/shared/aws', async (importOriginal) => ({ + ...(await importOriginal()), + findManagedInstance: (...args: unknown[]) => findManagedInstance(...args), + readDeployConfig: (...args: unknown[]) => readDeployConfig(...args), + runShellCommand: (...args: unknown[]) => runShellCommand(...args), +})); + +let handler: (event: LambdaFunctionURLEvent) => Promise; + +beforeAll(async () => { + Object.assign(process.env, LAMBDA_ENV); + ({ handler } = await import('../lambda/stats/index')); +}); + +function bodyOf(result: unknown): Record { + return JSON.parse((result as { statusCode: number; body: string }).body); +} + +function statusOf(result: unknown): number { + return (result as { statusCode: number }).statusCode; +} + +function statsEvent(query: Record) { + return { + queryStringParameters: query, + } as unknown as LambdaFunctionURLEvent; +} + +// The engine scrape is not what these cases assert on: the daemon answers +// nothing, so the reply carries no engine figures — only the control plane's +// own, which is where retainUntil lives. +beforeEach(() => { + vi.clearAllMocks(); + readDeployConfig.mockResolvedValue({ runner: 'llamacpp', modelId: 'org/m' }); + runShellCommand.mockResolvedValue({ status: 'Failed', stdout: '' }); +}); + +const futureTag = '2030-01-02T04:00:00.000Z'; +const pastTag = '2020-01-02T04:00:00.000Z'; + +describe('retainUntil', () => { + it('is present on a running instance whose tag is in the future', async () => { + findManagedInstance.mockResolvedValue({ + instanceId: 'i-run', + state: 'running', + retainUntil: new Date(futureTag), + }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body.state).toBe('running'); + expect(body.retainUntil).toBe(futureTag); + }); + + it('is present on a stopped instance whose tag is in the future', async () => { + findManagedInstance.mockResolvedValue({ + instanceId: 'i-stopped', + state: 'stopped', + retainUntil: new Date(futureTag), + }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body.state).toBe('stopped'); + expect(body.retainUntil).toBe(futureTag); + }); + + it('is absent when the tag has already passed', async () => { + findManagedInstance.mockResolvedValue({ + instanceId: 'i-run', + state: 'running', + retainUntil: new Date(pastTag), + }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body).not.toHaveProperty('retainUntil'); + }); + + it('is absent for an untagged instance', async () => { + findManagedInstance.mockResolvedValue({ instanceId: 'i-run', state: 'running' }); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body).not.toHaveProperty('retainUntil'); + }); + + it('is absent for an undeployed environment (no instance at all)', async () => { + findManagedInstance.mockResolvedValue(null); + + const result = await handler(statsEvent({ env: 'dev' })); + const body = bodyOf(result); + expect(statusOf(result)).toBe(200); + expect(body.state).toBe('undeployed'); + expect(body).not.toHaveProperty('retainUntil'); + }); +});