diff --git a/cmd/spinloop/commands.go b/cmd/spinloop/commands.go index aeb111a9..56bd9e6a 100644 --- a/cmd/spinloop/commands.go +++ b/cmd/spinloop/commands.go @@ -313,10 +313,11 @@ func fleetCmd() *cobra.Command { Short: "observe and drive the engines in a fleet file", Long: `observes and drives the engines named in a fleet file (fleet.yaml by default; --fleet names another). Observation is fleet-wide (status, metrics, -logs, and dashboard — the live tiled view); start, stop and route act on a -single node, and with no node they list the fleet and touch nothing. A node -that fails is a rendered row, never an error — only a problem with the fleet -file itself fails a command.`, +logs, and dashboard — the live tiled view); start and stop take one or more +node names, or --all for the whole fleet, and with neither they list the +fleet and touch nothing; deploy provisions kind: remote nodes' AWS +environments the same way. A node that fails is a rendered row, never an +error — only a problem with the fleet file itself fails a command.`, SilenceErrors: true, SilenceUsage: true, RunE: groupFallback, @@ -329,6 +330,7 @@ file itself fails a command.`, fleetRouteCmd(), fleetStartCmd(), fleetStopCmd(), + fleetDeployCmd(), ) return fleet } diff --git a/cmd/spinloop/fleet.go b/cmd/spinloop/fleet.go index afd44f01..1a05aeb6 100644 --- a/cmd/spinloop/fleet.go +++ b/cmd/spinloop/fleet.go @@ -1,7 +1,8 @@ // The `fleet` command group: one spinloop observing every engine you run. // It reads a fleet.yaml naming the machines, fans out over their daemon -// control APIs, and renders the cluster. Observation is fleet-wide; starting -// and stopping an engine is deliberately one node at a time. +// control APIs, and renders the cluster. Observation is fleet-wide; starting, +// stopping and deploying take one or more named nodes, or --all — never the +// whole fleet by default. package main @@ -13,11 +14,17 @@ import ( "io" "os" "os/signal" + "path/filepath" "strings" + "sync" "syscall" "time" "github.com/spf13/cobra" + "golang.org/x/term" + + "github.com/spinloop-ai/spinloop/internal/config" + "github.com/spinloop-ai/spinloop/internal/daemon" "github.com/spinloop-ai/spinloop/internal/fleet" ) @@ -242,85 +249,480 @@ func renderFleetMetricsJSON(w io.Writer, results []fleet.NodeResult) error { return nil } -// fleetStartCmd starts one named node's engine. +// fleetStartCmd starts one or more nodes' engines, or every node with --all. +// A kind: daemon node whose Spinloop source resolves is started with the +// deploy config that source derives (StartWith) — telling the daemon what to +// run, exactly as a routed wake already does for the Spinloop being +// launched. A kind: daemon node with no resolvable source, and a kind: +// remote node regardless, get a plain start: a remote environment's +// StartWith always refuses a config, since what it serves is fixed at +// deploy time. func fleetStartCmd() *cobra.Command { - var path string + var ( + path string + all bool + ) c := &cobra.Command{ Use: "start", - Short: "start an engine on one node", + Short: "start one or more nodes' engines", Args: cobra.ArbitraryArgs, SilenceErrors: true, SilenceUsage: true, RunE: func(c *cobra.Command, args []string) error { resolve(c) - return driveOneNode("start", path, args, func(ctx context.Context, n fleet.Node) fleet.NodeResult { - status, err := n.Start(ctx) - return fleet.Result(n.Name(), err, status) - }) + cfg, err := fleet.Resolve(path) + if err != nil { + return err + } + return runFleetDrive("start", cfg, all, args, fleetStartCall(cfg)) }, } - c.Flags().StringVar(&path, "fleet", "", fleetFileUsage) + fs := c.Flags() + fs.StringVar(&path, "fleet", "", fleetFileUsage) + fs.BoolVar(&all, "all", false, "start every node in the fleet") c.ValidArgsFunction = noPositionals compRegister(c, "fleet", compFiles) return c } -// fleetStopCmd stops one named node's engine. +// fleetStopCmd stops one or more nodes' engines, or every node with --all. +// Stopping takes no config, so unlike start it has nothing to resolve — only +// its target selection is shared with start. func fleetStopCmd() *cobra.Command { - var path string + var ( + path string + all bool + ) c := &cobra.Command{ Use: "stop", - Short: "stop an engine on one node", + Short: "stop one or more nodes' engines", Args: cobra.ArbitraryArgs, SilenceErrors: true, SilenceUsage: true, RunE: func(c *cobra.Command, args []string) error { resolve(c) - return driveOneNode("stop", path, args, func(ctx context.Context, n fleet.Node) fleet.NodeResult { + cfg, err := fleet.Resolve(path) + if err != nil { + return err + } + call := func(ctx context.Context, n fleet.Node) fleet.NodeResult { status, err := n.Stop(ctx) return fleet.Result(n.Name(), err, status) + } + return runFleetDrive("stop", cfg, all, args, call) + }, + } + fs := c.Flags() + fs.StringVar(&path, "fleet", "", fleetFileUsage) + fs.BoolVar(&all, "all", false, "stop every node in the fleet") + c.ValidArgsFunction = noPositionals + compRegister(c, "fleet", compFiles) + return c +} + +// fleetStartCall builds fleet start's per-node call, closing over cfg so it +// can recover each targeted node's NodeConfig (kind, File) from the bare +// Node fleet.Call is handed — Call's signature carries no NodeConfig, but +// every node it is called with came from cfg in the first place, so the +// lookup by name always succeeds. +func fleetStartCall(cfg *fleet.Config) fleet.Call { + return func(ctx context.Context, n fleet.Node) fleet.NodeResult { + entry, _ := cfg.Node(n.Name()) + if entry.Kind != fleet.KindDaemon { + status, err := n.Start(ctx) + return fleet.Result(n.Name(), err, status) + } + arg, source, err := resolveNodeSpinloop(entry, cfg.Dir) + if err != nil { + return fleet.Result(n.Name(), err, daemon.StatusResponse{}) + } + sel, spinloopPath, err := readSpinloop(fmt.Sprintf("spinloop fleet start %s", n.Name()), arg) + if err != nil { + return fleet.Result(n.Name(), err, daemon.StatusResponse{}) + } + if err := applySpinloopEnv(sel, spinloopPath); err != nil { + return fleet.Result(n.Name(), err, daemon.StatusResponse{}) + } + dc, err := deployConfigForNode(sel, spinloopPath) + if err != nil { + return fleet.Result(n.Name(), err, daemon.StatusResponse{}) + } + engineKey, err := cfg.EngineToken(entry) + if err != nil { + return fleet.Result(n.Name(), err, daemon.StatusResponse{}) + } + fmt.Printf("%s: using %s (%s)\n", n.Name(), spinloopPath, source) + status, err := n.StartWith(ctx, &dc, engineKey) + return fleet.Result(n.Name(), err, status) + } +} + +// runFleetDrive selects the nodes a mutating fleet command targets and fans +// call out over them: no names and no --all fails, listing the fleet's +// nodes; --all and names together fails as ambiguous; named nodes fail +// before anything is touched if any name is unknown. Replaces the old +// driveOneNode now that start and stop both take several nodes or --all, +// not just one. +func runFleetDrive(verb string, cfg *fleet.Config, all bool, names []string, call fleet.Call) error { + if all && len(names) > 0 { + return fmt.Errorf("spinloop fleet %s: --all is ambiguous with node names", verb) + } + var target *fleet.Config + if all { + target = cfg + } else { + if len(names) == 0 { + return fmt.Errorf( + "spinloop fleet %s needs a node, or --all: %s", + verb, strings.Join(cfg.Names(), ", ")) + } + narrowed, err := cfg.OnlyNames(names) + if err != nil { + return err + } + target = narrowed + } + results := target.FanOut(context.Background(), call) + var bad []string + for _, r := range results { + if !r.OK() { + bad = append(bad, r.Name) + fmt.Printf("%s %s: %s\n", r.Name, r.Outcome, r.Detail()) + continue + } + fmt.Printf("%s %s\n", r.Name, r.Status.State) + } + if len(bad) > 0 { + return fmt.Errorf("%s: failed: %s", verb, strings.Join(bad, ", ")) + } + return nil +} + +// fleetDeployCmd creates the AWS environment for one or more kind: remote +// nodes, or every kind: remote node with --all, deriving what each serves +// from its resolved Spinloop source — the same derivation and registration +// a standalone `spinloop remote deploy` performs for one file, so the two +// can never disagree about what a given Spinloop deploys. +func fleetDeployCmd() *cobra.Command { + var ( + path string + all bool + dryRun bool + overwrite bool + reseed bool + allowedCidr string + region string + spinloopVersion string + apiKeyEnv string + ) + c := &cobra.Command{ + Use: "deploy", + Short: "create the AWS environment for one or more remote nodes", + Long: `deploys the AWS environment for each named kind: remote node, or +every kind: remote node with --all, deriving what to serve from each node's +own Spinloop source: its file field, or its name resolved as a registered +alias or a same-named subdirectory beside the fleet file. Reuses the same +derivation, consent, and registration behavior as "spinloop remote deploy".`, + Args: cobra.ArbitraryArgs, + SilenceErrors: true, + SilenceUsage: true, + RunE: func(c *cobra.Command, args []string) error { + resolve(c) + return runFleetDeploy(path, all, args, deployOpts{ + dryRun: dryRun, + overwrite: overwrite, + reseed: reseed, + allowedCidr: allowedCidr, + region: region, + spinloopVersion: spinloopVersion, + apiKeyEnv: apiKeyEnv, }) }, } - c.Flags().StringVar(&path, "fleet", "", fleetFileUsage) + fs := c.Flags() + fs.StringVar(&path, "fleet", "", fleetFileUsage) + fs.BoolVar(&all, "all", false, "deploy every kind: remote node in the fleet") + fs.BoolVarP(&dryRun, "dry-run", "n", false, "print the config that would be deployed, without sending it") + fs.BoolVar(&overwrite, "overwrite", false, "proceed against an already-registered or live environment") + fs.BoolVar(&reseed, "reseed", false, "re-fetch the weights even if they are already in S3 (starts a ~20-minute seed)") + fs.StringVar(&allowedCidr, "allowed-cidr", "", "who may reach each environment's instance (default: your public IP as a /32, on first deploy)") + fs.StringVar(®ion, "region", "", "AWS region of the control plane (default: AWS_REGION or us-east-1)") + fs.StringVar(&spinloopVersion, "spinloop-version", "", "spinloop release each environment's instances install at boot (default: latest)") + fs.StringVar(&apiKeyEnv, "api-key-env", "", "name the environment variable holding the engine key to create or rotate, applied to every targeted node; with no flag each environment keeps its stored key") c.ValidArgsFunction = noPositionals compRegister(c, "fleet", compFiles) return c } -// driveOneNode runs a mutating call against exactly one node. Fan-out is for -// observation: starting or stopping every engine at once is a footgun, so -// these demand a node name and otherwise list the fleet without touching -// anything. -func driveOneNode(verb, path string, args []string, call fleet.Call) error { +// runFleetDeploy is the body of `spinloop fleet deploy`. +func runFleetDeploy(path string, all bool, names []string, opts deployOpts) error { cfg, err := fleet.Resolve(path) if err != nil { return err } - rest := args - if len(rest) == 0 { + if all && len(names) > 0 { + return fmt.Errorf("spinloop fleet deploy: --all is ambiguous with node names") + } + + remoteNames := make([]string, 0, len(cfg.Nodes)) + for _, n := range cfg.Nodes { + if n.Kind == fleet.KindRemote { + remoteNames = append(remoteNames, n.Name) + } + } + + var targets []string + switch { + case all: + targets = remoteNames + case len(names) == 0: return fmt.Errorf( - "spinloop fleet %s needs a node: %s\n(%s acts on one node at a time, never the whole fleet)", - verb, strings.Join(cfg.Names(), ", "), verb) + "spinloop fleet deploy needs a node, or --all: %s", + strings.Join(remoteNames, ", ")) + default: + for _, name := range names { + entry, ok := cfg.Node(name) + if !ok { + return fmt.Errorf("no node %q in %s (known nodes: %s)", + name, cfg.Path, strings.Join(cfg.Names(), ", ")) + } + if entry.Kind != fleet.KindRemote { + return fmt.Errorf( + "node %q is kind %q: fleet deploy provisions cloud environments, and %[1]s is not one", + name, entry.Kind) + } + } + targets = names } - name := rest[0] - entry, ok := cfg.Node(name) - if !ok { - return fmt.Errorf("no node %q in %s (known nodes: %s)", - name, cfg.Path, strings.Join(cfg.Names(), ", ")) + + results := make([]fleetDeployResult, len(targets)) + done := make([]bool, len(targets)) + var mu sync.Mutex + var wg sync.WaitGroup + + // A live spinner only makes sense where the previous frame can be + // erased — skip it entirely for a piped or redirected run (a log file, + // CI) rather than spamming it with escape codes. + var stop, spinnerDone chan struct{} + if term.IsTerminal(int(os.Stdout.Fd())) { + stop = make(chan struct{}) + spinnerDone = make(chan struct{}) + go renderDeploySpinner(targets, results, done, &mu, stop, spinnerDone) } - node, err := cfg.NewNode(entry) - if err != nil { - return err + + for i, name := range targets { + wg.Add(1) + go func(i int, name string) { + defer wg.Done() + r := deployOneNode(cfg, name, opts) + mu.Lock() + results[i] = r + done[i] = true + mu.Unlock() + }(i, name) + } + wg.Wait() + if stop != nil { + // Stop and wait for the spinner's own erase before printing the + // report below it — otherwise the two interleave. + close(stop) + <-spinnerDone } - r := call(context.Background(), node) - if !r.OK() { - return fmt.Errorf("%s %s: %s", verb, name, r.Detail()) + + var bad []string + for i, r := range results { + if i > 0 { + fmt.Println() + } + fmt.Println(deployHeader(r.node, r.outcome)) + fmt.Print(r.text()) + if r.outcome != deployRowOK { + bad = append(bad, r.node) + } + } + fmt.Println() + fmt.Println(deploySummary(len(targets), len(bad))) + if len(bad) > 0 { + return fmt.Errorf("fleet deploy: failed or guarded: %s", strings.Join(bad, ", ")) } - fmt.Printf("%s %s\n", name, r.Status.State) return nil } +// deploySpinnerFrames are the classic Braille dots, cycled while a node's +// deploy is still in flight. +var deploySpinnerFrames = [...]string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} + +const ( + ansiGreen = "\033[92m" + ansiRed = "\033[31m" + ansiYellow = "\033[33m" + ansiGrey = "\033[90m" + ansiReset = "\033[0m" +) + +// renderDeploySpinner redraws one line per target in place — a spinner +// beside whichever nodes are still deploying, a coloured mark beside +// whichever have finished — until stop is closed, then erases its own +// lines and closes done. A `fleet deploy --all` can be several concurrent +// AWS calls running for minutes; this is what keeps it from looking hung. +func renderDeploySpinner(targets []string, results []fleetDeployResult, done []bool, mu *sync.Mutex, stop <-chan struct{}, closeWhenDone chan<- struct{}) { + defer close(closeWhenDone) + ticker := time.NewTicker(120 * time.Millisecond) + defer ticker.Stop() + frame := 0 + drawn := 0 + redraw := func() { + if drawn > 0 { + fmt.Printf("\033[%dA\033[J", drawn) + } + mu.Lock() + for i, name := range targets { + if done[i] { + fmt.Printf("%s %s\n", nodeGlyph(results[i].outcome), name) + } else { + fmt.Printf("%s%s%s %s deploying...\n", ansiGrey, deploySpinnerFrames[frame%len(deploySpinnerFrames)], ansiReset, name) + } + } + mu.Unlock() + drawn = len(targets) + frame++ + } + for { + select { + case <-stop: + if drawn > 0 { + fmt.Printf("\033[%dA\033[J", drawn) + } + return + case <-ticker.C: + redraw() + } + } +} + +// nodeGlyph is the coloured mark a finished node's spinner line, and its +// report header, both show for the same outcome. +func nodeGlyph(outcome deployRowOutcome) string { + switch outcome { + case deployRowOK: + return ansiGreen + "✓" + ansiReset + case deployRowGuarded: + return ansiYellow + "⚠" + ansiReset + default: + return ansiRed + "✗" + ansiReset + } +} + +// deployHeader is the line that separates one node's report from the next — +// the gap `fleet deploy --all`'s output used to lack entirely. +func deployHeader(node string, outcome deployRowOutcome) string { + return nodeGlyph(outcome) + " " + node +} + +// deploySummary is the final line naming how many of the targeted nodes +// deployed clean, so a large --all run's result is legible at a glance +// without counting rows. +func deploySummary(total, bad int) string { + ok := total - bad + if bad == 0 { + return fmt.Sprintf("%s%d/%d deployed%s", ansiGreen, ok, total, ansiReset) + } + return fmt.Sprintf("%s%d/%d deployed%s, %s%d failed or guarded%s", ansiGreen, ok, total, ansiReset, ansiRed, bad, ansiReset) +} + +// deployRowOutcome is one node's fleet-deploy outcome — a row, not an abort: +// one node's guard or failure never stops the others. +type deployRowOutcome int + +const ( + deployRowOK deployRowOutcome = iota + deployRowGuarded + deployRowFailed +) + +// fleetDeployResult is one targeted node's fleet-deploy outcome. +type fleetDeployResult struct { + node string + outcome deployRowOutcome + detail string // guard/failure message, or the deploy's plan/result text on success +} + +// text renders one node's result: the deploy's own plan/result text on +// success, or a labelled one-liner on guard or failure. +func (r fleetDeployResult) text() string { + switch r.outcome { + case deployRowGuarded: + return fmt.Sprintf(" guarded: %s\n", r.detail) + case deployRowFailed: + return fmt.Sprintf(" failed: %s\n", r.detail) + default: + return r.detail + } +} + +// deployOneNode resolves and deploys a single targeted node. It never +// returns an error itself — a bad node becomes a fleetDeployResult, so the +// caller's fan-out can label it without aborting the others. +func deployOneNode(cfg *fleet.Config, name string, opts deployOpts) fleetDeployResult { + entry, _ := cfg.Node(name) + arg, source, err := resolveNodeSpinloop(entry, cfg.Dir) + if err != nil { + return fleetDeployResult{node: name, outcome: deployRowFailed, detail: err.Error()} + } + _, spinloopPath, dc, env, err := deriveDeployTarget(fmt.Sprintf("spinloop fleet deploy %s", name), arg) + if err != nil { + return fleetDeployResult{node: name, outcome: deployRowFailed, detail: err.Error()} + } + outcome, err := runDeploy(spinloopPath, env, dc, opts) + if err != nil { + var guarded *errDeployGuarded + if errors.As(err, &guarded) { + return fleetDeployResult{node: name, outcome: deployRowGuarded, detail: err.Error()} + } + return fleetDeployResult{node: name, outcome: deployRowFailed, detail: err.Error()} + } + text := fmt.Sprintf("using %s (%s)\n%s", spinloopPath, source, outcome.Text) + return fleetDeployResult{node: name, outcome: deployRowOK, detail: text} +} + +// resolveNodeSpinloop resolves the argument to hand readSpinloop for one +// node's declared Spinloop source, trying in order and stopping at the +// first that resolves: node.File (relative to fleetDir), node.Name as a +// registered `spinloop alias`, node.Name as a subdirectory beside the fleet +// file containing a Spinloop file. source labels which one supplied it, for +// reporting. Neither `fleet deploy` nor `fleet start` falls back to acting +// without one — a node for which none resolves is a per-node failure naming +// all three. +// +// The alias tier resolves to the alias's own target path directly, rather +// than handing node.Name to readSpinloop and letting it resolve the alias +// itself: readSpinloop's resolveAlias deliberately lets a same-named path on +// disk beat a registered alias (so an existing invocation never changes +// meaning just because an alias gets registered later) — the opposite of +// this function's own precedence, alias before subdirectory. A node named +// the same as its own subdirectory would otherwise silently resolve to the +// subdirectory even with an alias registered, contradicting the order this +// function documents. +func resolveNodeSpinloop(node fleet.NodeConfig, fleetDir string) (arg, source string, err error) { + if node.File != "" { + return filepath.Join(fleetDir, node.File), fmt.Sprintf("file %s", node.File), nil + } + cfgFile, err := config.Load() + if err != nil { + return "", "", err + } + if aliasPath, ok := cfgFile.Alias(node.Name); ok { + return aliasPath, fmt.Sprintf("alias %q", node.Name), nil + } + subdir := filepath.Join(fleetDir, node.Name) + if info, statErr := os.Stat(subdir); statErr == nil && info.IsDir() { + return subdir, fmt.Sprintf("subdirectory %s", subdir), nil + } + return "", "", fmt.Errorf( + "node %q names no Spinloop source: no `file` field, no `spinloop alias` named %q, and no %s subdirectory beside the fleet file", + node.Name, node.Name, node.Name) +} + // fleetRouteCmd reports the node a harness launch would choose for a Spinloop, // and changes nothing: no config is pushed, no engine started, no harness // config written. It is how a routing decision is checked before an agent diff --git a/cmd/spinloop/fleet_deploy_test.go b/cmd/spinloop/fleet_deploy_test.go new file mode 100644 index 00000000..84327252 --- /dev/null +++ b/cmd/spinloop/fleet_deploy_test.go @@ -0,0 +1,340 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/spinloop-ai/spinloop/internal/config" + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// fleetDeployServer answers every environment's deploy call with a +// deterministic base URL, so a fan-out over several nodes can be told apart +// by the environment each request named. +func fleetDeployServer(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + env := r.URL.Query().Get("env") + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"deployed":true,"environment":%q,"base_url":"http://198.51.100.9:8000/v1"}`, env) + })) + t.Cleanup(srv.Close) + return srv +} + +// stubFleetDeploySeams points the deploy seams at server for every +// environment, reporting none as registered or live (so no node needs +// --overwrite) unless overridden by the caller after this returns. +func stubFleetDeploySeams(t *testing.T, server *httptest.Server) { + t.Helper() + origDiscover, origStatus, origDetect := deployDiscoverFn, remoteStatusFn, detectPublicCIDRFn + t.Cleanup(func() { deployDiscoverFn, remoteStatusFn, detectPublicCIDRFn = origDiscover, origStatus, origDetect }) + deployDiscoverFn = func(context.Context, aws.Config, string) (remote.ControlPlane, error) { + return remote.ControlPlane{Config: remote.Config{ + StartURL: server.URL, StopURL: server.URL, DeployURL: server.URL, Region: "us-east-1", + }}, nil + } + remoteStatusFn = func(context.Context, remote.Config) (*remote.Response, error) { + return &remote.Response{StatusCode: 200, State: "undeployed"}, nil + } + detectPublicCIDRFn = func(context.Context) (string, error) { return "203.0.113.7/32", nil } +} + +// writeFleetDeploySetup lays out a fleet file exercising every resolution +// tier: gpu-a and gpu-b via an explicit file field, aliased via a +// registered alias, subdir-env via a same-named subdirectory, no-source via +// none of the three, plus a kind: daemon node (studio) fleet deploy must +// never touch. Returns the fleet directory (already the working directory). +func writeFleetDeploySetup(t *testing.T) string { + t.Helper() + isolateConfig(t) + dir := writeFleetFile(t, ` +nodes: + - name: gpu-a + kind: remote + file: ./gpu-a.Spinloop + - name: gpu-b + kind: remote + file: ./gpu-b.Spinloop + - name: aliased + kind: remote + - name: subdir-env + kind: remote + - name: no-source + kind: remote + - name: studio + host: studio.local +`) + write := func(name, env string) { + t.Helper() + body := fmt.Sprintf("PROVIDER llamacpp\nMODEL org/m:Q4\nCONTEXT 8192\nREMOTE %s\n", env) + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + write("gpu-a.Spinloop", "gpu-a") + write("gpu-b.Spinloop", "gpu-b") + + aliasedPath := filepath.Join(dir, "aliased.Spinloop") + write("aliased.Spinloop", "aliased") + if err := config.Update(func(f *config.File) error { + f.SetAlias("aliased", aliasedPath) + return nil + }); err != nil { + t.Fatal(err) + } + + if err := os.Mkdir(filepath.Join(dir, "subdir-env"), 0o700); err != nil { + t.Fatal(err) + } + subdirBody := "PROVIDER llamacpp\nMODEL org/m:Q4\nCONTEXT 8192\nREMOTE subdir-env\n" + if err := os.WriteFile(filepath.Join(dir, "subdir-env", "Spinloop"), []byte(subdirBody), 0o600); err != nil { + t.Fatal(err) + } + return dir +} + +func TestCmdFleetDeployAll(t *testing.T) { + writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + // no-source can never resolve, so --all still fails overall — but must + // still deploy every node that *does* resolve. + err := cmdFleet([]string{"deploy", "--all"}) + if err == nil { + t.Fatal("--all should fail overall because no-source cannot resolve") + } + + for _, env := range []string{"gpu-a", "gpu-b", "aliased", "subdir-env"} { + if _, statErr := os.Stat(mustEnvConfigPath(t, env)); statErr != nil { + t.Errorf("environment %q was not registered: %v", env, statErr) + } + } + if !strings.Contains(err.Error(), "no-source") { + t.Errorf("error should mention the unresolved node, got %v", err) + } +} + +func TestCmdFleetDeployNamedNodes(t *testing.T) { + writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + if err := cmdFleet([]string{"deploy", "gpu-a", "gpu-b"}); err != nil { + t.Fatalf("deploy gpu-a gpu-b: %v", err) + } + for _, env := range []string{"gpu-a", "gpu-b"} { + if _, statErr := os.Stat(mustEnvConfigPath(t, env)); statErr != nil { + t.Errorf("environment %q was not registered: %v", env, statErr) + } + } + // Untargeted nodes must be left alone. + if _, statErr := os.Stat(mustEnvConfigPath(t, "aliased")); statErr == nil { + t.Error("aliased was deployed despite not being named") + } +} + +func TestCmdFleetDeployNoTargetIsAnError(t *testing.T) { + writeFleetDeploySetup(t) + err := cmdFleet([]string{"deploy"}) + if err == nil { + t.Fatal("fleet deploy with no node and no --all was accepted") + } + for _, want := range []string{"gpu-a", "gpu-b", "aliased"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q should list the remote nodes, missing %q", err, want) + } + } + // studio (kind: daemon) must not be offered as a deploy target. + if strings.Contains(err.Error(), "studio") { + t.Errorf("error %q should not list the daemon node", err) + } +} + +func TestCmdFleetDeployAllPlusNamesIsAmbiguous(t *testing.T) { + writeFleetDeploySetup(t) + err := cmdFleet([]string{"deploy", "--all", "gpu-a"}) + if err == nil || !strings.Contains(err.Error(), "ambiguous") { + t.Fatalf("want an ambiguous-target error, got %v", err) + } +} + +func TestCmdFleetDeployUnknownNodeFailsBeforeDeploying(t *testing.T) { + writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + err := cmdFleet([]string{"deploy", "gpu-a", "nope"}) + if err == nil || !strings.Contains(err.Error(), "nope") { + t.Fatalf("want an unknown-node error naming it, got %v", err) + } + if _, statErr := os.Stat(mustEnvConfigPath(t, "gpu-a")); statErr == nil { + t.Error("gpu-a was deployed even though the command should have failed before touching anything") + } +} + +func TestCmdFleetDeployNamingADaemonNodeFails(t *testing.T) { + writeFleetDeploySetup(t) + err := cmdFleet([]string{"deploy", "studio"}) + if err == nil || !strings.Contains(err.Error(), "studio") { + t.Fatalf("want an error naming studio, got %v", err) + } +} + +func TestCmdFleetDeployUnresolvedNodeFailsOnlyThatNode(t *testing.T) { + writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + out := captureStdout(t, func() { + err := cmdFleet([]string{"deploy", "gpu-a", "no-source"}) + if err == nil { + t.Fatal("want a failure because no-source cannot resolve") + } + if !strings.Contains(err.Error(), "no-source") { + t.Errorf("error should name the failed node, got %v", err) + } + }) + if !strings.Contains(out, "gpu-a") { + t.Errorf("output should still show gpu-a's deploy: %s", out) + } + if _, statErr := os.Stat(mustEnvConfigPath(t, "gpu-a")); statErr != nil { + t.Errorf("gpu-a should still have deployed despite no-source failing: %v", statErr) + } +} + +// A node whose source *resolves* but whose Spinloop is itself undeployable +// (missing REMOTE, here) must fail only that node — resolveNodeSpinloop +// succeeding is not the same as deriveDeployTarget succeeding, and the two +// failure sites must not be conflated. +func TestCmdFleetDeployResolvedButUndeployableSpinloopFailsOnlyThatNode(t *testing.T) { + dir := writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + // gpu-a's own file, minus REMOTE — deriveDeployTarget refuses this, but + // resolveNodeSpinloop has already succeeded by the time it does. + noRemote := filepath.Join(dir, "gpu-a.Spinloop") + if err := os.WriteFile(noRemote, []byte("PROVIDER llamacpp\nMODEL org/m:Q4\nCONTEXT 8192\n"), 0o600); err != nil { + t.Fatal(err) + } + + out := captureStdout(t, func() { + err := cmdFleet([]string{"deploy", "gpu-a", "gpu-b"}) + if err == nil { + t.Fatal("want a failure because gpu-a's Spinloop names no REMOTE") + } + if !strings.Contains(err.Error(), "gpu-a") { + t.Errorf("error should name gpu-a, got %v", err) + } + }) + if !strings.Contains(out, "REMOTE") { + t.Errorf("output should explain the missing REMOTE: %s", out) + } + if _, statErr := os.Stat(mustEnvConfigPath(t, "gpu-b")); statErr != nil { + t.Errorf("gpu-b should still have deployed despite gpu-a's Spinloop being undeployable: %v", statErr) + } +} + +func TestCmdFleetDeployAliasWinsOverSubdirectory(t *testing.T) { + dir := writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + // Register an alias under the subdir-env node's own name too, pointing + // at a *different* Spinloop (a different REMOTE), and confirm the alias + // wins. + altPath := filepath.Join(dir, "alt.Spinloop") + if err := os.WriteFile(altPath, []byte("PROVIDER llamacpp\nMODEL org/m:Q4\nCONTEXT 8192\nREMOTE alt-env\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := config.Update(func(f *config.File) error { + f.SetAlias("subdir-env", altPath) + return nil + }); err != nil { + t.Fatal(err) + } + + if err := cmdFleet([]string{"deploy", "subdir-env"}); err != nil { + t.Fatalf("deploy subdir-env: %v", err) + } + if _, statErr := os.Stat(mustEnvConfigPath(t, "alt-env")); statErr != nil { + t.Errorf("the alias's environment (alt-env) should have been used: %v", statErr) + } + if _, statErr := os.Stat(mustEnvConfigPath(t, "subdir-env")); statErr == nil { + t.Error("the subdirectory's environment should not have been used once an alias exists") + } +} + +func TestCmdFleetDeployGuardDoesNotBlockSiblings(t *testing.T) { + writeFleetDeploySetup(t) + stubAWSEnv(t) + server := fleetDeployServer(t) + stubFleetDeploySeams(t, server) + + if err := remote.SaveEnvironment("gpu-a", remote.Config{StartURL: "https://s", StopURL: "https://x", Region: "us-east-1"}); err != nil { + t.Fatal(err) + } + + err := cmdFleet([]string{"deploy", "gpu-a", "gpu-b"}) + if err == nil { + t.Fatal("want a failure because gpu-a is guarded") + } + if !strings.Contains(err.Error(), "gpu-a") { + t.Errorf("error should name the guarded node, got %v", err) + } + if _, statErr := os.Stat(mustEnvConfigPath(t, "gpu-b")); statErr != nil { + t.Errorf("gpu-b should still have deployed despite gpu-a being guarded: %v", statErr) + } +} + +func TestCmdFleetDeployDryRunTouchesNothing(t *testing.T) { + writeFleetDeploySetup(t) + + called := false + origDiscover := deployDiscoverFn + t.Cleanup(func() { deployDiscoverFn = origDiscover }) + deployDiscoverFn = func(context.Context, aws.Config, string) (remote.ControlPlane, error) { + called = true + return remote.ControlPlane{}, fmt.Errorf("must not be called") + } + + out := captureStdout(t, func() { + if err := cmdFleet([]string{"deploy", "gpu-a", "gpu-b", "--dry-run"}); err != nil { + t.Errorf("deploy --dry-run: %v", err) + } + }) + if called { + t.Error("--dry-run must touch nothing — not even discovery") + } + for _, want := range []string{"gpu-a", "gpu-b", "environment: gpu-a", "environment: gpu-b"} { + if !strings.Contains(out, want) { + t.Errorf("dry-run output missing %q:\n%s", want, out) + } + } +} + +// mustEnvConfigPath resolves where a deployed environment would be +// registered, under the isolated config dir this test's HOME points at. +func mustEnvConfigPath(t *testing.T, env string) string { + t.Helper() + path, err := remote.EnvConfigPath(env) + if err != nil { + t.Fatal(err) + } + return path +} diff --git a/cmd/spinloop/fleet_test.go b/cmd/spinloop/fleet_test.go index 69cdfa63..c54546ab 100644 --- a/cmd/spinloop/fleet_test.go +++ b/cmd/spinloop/fleet_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "net/http" @@ -13,6 +14,11 @@ import ( "syscall" "testing" "time" + + "github.com/spinloop-ai/spinloop/internal/daemon" + "github.com/spinloop-ai/spinloop/internal/fleet" + "github.com/spinloop-ai/spinloop/internal/metrics" + "github.com/spinloop-ai/spinloop/internal/remote" ) // stubNode serves a daemon control API for one fleet node. @@ -79,13 +85,20 @@ func writeFleetFile(t *testing.T, body string) string { } // twoNodeFleet writes a fleet of one reachable node and one that is down. +// "up" declares a file field naming a minimal Spinloop, so `fleet start` +// resolves a source for it — required for any kind: daemon node since +// resolution became mandatory (see resolveNodeSpinloop). func twoNodeFleet(t *testing.T, state string) *httptest.Server { t.Helper() + t.Setenv("SPINLOOP_CONFIG_DIR", t.TempDir()) up := stubNode(t, state) host, port := hostPort(t, up) - writeFleetFile(t, fmt.Sprintf( - "nodes:\n - name: up\n host: %s\n port: %d\n - name: down\n host: 127.0.0.1\n port: 1\n", + dir := writeFleetFile(t, fmt.Sprintf( + "nodes:\n - name: up\n host: %s\n port: %d\n file: ./up.Spinloop\n - name: down\n host: 127.0.0.1\n port: 1\n", host, port)) + if err := os.WriteFile(filepath.Join(dir, "up.Spinloop"), []byte("PROVIDER llamacpp\nMODEL org/m:Q4\n"), 0o600); err != nil { + t.Fatal(err) + } return up } @@ -217,8 +230,9 @@ func TestCmdFleetStartStopDriveOneNode(t *testing.T) { } } -// Mutating verbs are single-node by contract: with no node they list the -// fleet and touch nothing. +// Mutating verbs demand an explicit target: with no node and no --all they +// list the fleet and touch nothing, rather than acting on the whole fleet +// by accident. func TestCmdFleetStartStopRequireANode(t *testing.T) { twoNodeFleet(t, "idle") for _, verb := range []string{"start", "stop"} { @@ -235,6 +249,273 @@ func TestCmdFleetStartStopRequireANode(t *testing.T) { } } +// threeStubNodesFleet writes a fleet of three reachable daemon nodes, each +// declaring a file field naming a minimal Spinloop, so `fleet start` (which +// now requires a resolvable source for every kind: daemon node) can start +// any of them. +func threeStubNodesFleet(t *testing.T, state string) { + t.Helper() + t.Setenv("SPINLOOP_CONFIG_DIR", t.TempDir()) + var lines strings.Builder + for _, name := range []string{"a", "b", "c"} { + srv := stubNode(t, state) + host, port := hostPort(t, srv) + fmt.Fprintf(&lines, " - name: %s\n host: %s\n port: %d\n file: ./%s.Spinloop\n", name, host, port, name) + } + dir := writeFleetFile(t, "nodes:\n"+lines.String()) + for _, name := range []string{"a", "b", "c"} { + path := filepath.Join(dir, name+".Spinloop") + if err := os.WriteFile(path, []byte("PROVIDER llamacpp\nMODEL org/m:Q4\n"), 0o600); err != nil { + t.Fatal(err) + } + } +} + +func TestCmdFleetStartSeveralNamedNodes(t *testing.T) { + threeStubNodesFleet(t, "idle") + out := captureStdout(t, func() { + if err := cmdFleet([]string{"start", "a", "b"}); err != nil { + t.Error(err) + } + }) + for _, want := range []string{"a running", "b running"} { + if !strings.Contains(out, want) { + t.Errorf("start a b output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "c") { + t.Errorf("start a b should not touch c:\n%s", out) + } +} + +func TestCmdFleetStartAll(t *testing.T) { + threeStubNodesFleet(t, "idle") + out := captureStdout(t, func() { + if err := cmdFleet([]string{"start", "--all"}); err != nil { + t.Error(err) + } + }) + for _, want := range []string{"a running", "b running", "c running"} { + if !strings.Contains(out, want) { + t.Errorf("start --all output missing %q:\n%s", want, out) + } + } +} + +func TestCmdFleetStopSeveralNamedNodesAndAll(t *testing.T) { + threeStubNodesFleet(t, "running") + out := captureStdout(t, func() { + if err := cmdFleet([]string{"stop", "a", "b"}); err != nil { + t.Error(err) + } + }) + for _, want := range []string{"a stopped", "b stopped"} { + if !strings.Contains(out, want) { + t.Errorf("stop a b output missing %q:\n%s", want, out) + } + } + + out = captureStdout(t, func() { + if err := cmdFleet([]string{"stop", "--all"}); err != nil { + t.Error(err) + } + }) + for _, want := range []string{"a stopped", "b stopped", "c stopped"} { + if !strings.Contains(out, want) { + t.Errorf("stop --all output missing %q:\n%s", want, out) + } + } +} + +func TestCmdFleetStartStopAllPlusNamesIsAmbiguous(t *testing.T) { + threeStubNodesFleet(t, "idle") + for _, verb := range []string{"start", "stop"} { + err := cmdFleet([]string{verb, "--all", "a"}) + if err == nil || !strings.Contains(err.Error(), "ambiguous") { + t.Errorf("fleet %s --all a: want an ambiguous-target error, got %v", verb, err) + } + } +} + +func TestCmdFleetStartUnknownNameAmongSeveralFailsBeforeStartingAny(t *testing.T) { + threeStubNodesFleet(t, "idle") + out := captureStdout(t, func() { + err := cmdFleet([]string{"start", "a", "nope"}) + if err == nil || !strings.Contains(err.Error(), "nope") { + t.Errorf("want an unknown-node error naming it, got %v", err) + } + }) + if out != "" { + t.Errorf("nothing should have started, got output:\n%s", out) + } +} + +// A kind: daemon node with no file field, no matching alias, and no +// matching subdirectory cannot resolve a Spinloop source, so fleet start +// fails for it — no fallback to a plain, config-less start. +func TestCmdFleetStartDaemonNodeWithNoResolvableSourceFails(t *testing.T) { + twoNodeFleet(t, "idle") // "down" declares no file field + var err error + out := captureStdout(t, func() { + err = cmdFleet([]string{"start", "down"}) + }) + if err == nil { + t.Fatal("start on a node with no resolvable Spinloop source was accepted") + } + if !strings.Contains(err.Error(), "down") { + t.Errorf("error %q should name the failed node", err) + } + for _, want := range []string{"file", "alias", "subdirectory"} { + if !strings.Contains(out, want) { + t.Errorf("output %q should mention %q", out, want) + } + } +} + +// fakeFleetNode is a minimal fleet.Node for exercising fleetStartCall's +// dispatch directly, without a real daemon or control plane — it just +// counts which of Start/StartWith was called. +type fakeFleetNode struct { + name string + startCalls, startWithCalls int +} + +func (f *fakeFleetNode) Name() string { return f.name } +func (f *fakeFleetNode) Status(context.Context) (daemon.StatusResponse, error) { + return daemon.StatusResponse{}, nil +} +func (f *fakeFleetNode) Metrics(context.Context) (metrics.Stats, error) { return metrics.Stats{}, nil } +func (f *fakeFleetNode) Start(context.Context) (daemon.StatusResponse, error) { + f.startCalls++ + return daemon.StatusResponse{State: "running"}, nil +} +func (f *fakeFleetNode) StartWith(context.Context, *remote.DeployConfig, string) (daemon.StatusResponse, error) { + f.startWithCalls++ + return daemon.StatusResponse{State: "running"}, nil +} +func (f *fakeFleetNode) Stop(context.Context) (daemon.StatusResponse, error) { + return daemon.StatusResponse{}, nil +} +func (f *fakeFleetNode) Logs(context.Context, int64, int) (daemon.LogsResponse, error) { + return daemon.LogsResponse{}, nil +} + +// A kind: remote node's start always uses a plain start, never StartWith — +// StartWith refuses a config for that kind unconditionally (see +// remoteNode.StartWith), so fleetStartCall must not even attempt it. +func TestFleetStartCallRemoteNodeUsesPlainStart(t *testing.T) { + t.Setenv("SPINLOOP_CONFIG_DIR", t.TempDir()) + dir := writeFleetFile(t, "nodes:\n - name: gpu-env\n kind: remote\n file: ./gpu-env.Spinloop\n") + if err := os.WriteFile(filepath.Join(dir, "gpu-env.Spinloop"), []byte("PROVIDER llamacpp\nMODEL org/m:Q4\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := fleet.Resolve(filepath.Join(dir, "fleet.yaml")) + if err != nil { + t.Fatal(err) + } + node := &fakeFleetNode{name: "gpu-env"} + r := fleetStartCall(cfg)(context.Background(), node) + if !r.OK() { + t.Fatalf("fleetStartCall on a remote node = %+v", r) + } + if node.startWithCalls != 0 { + t.Errorf("StartWith was called %d times for a remote node, want 0", node.startWithCalls) + } + if node.startCalls != 1 { + t.Errorf("Start was called %d times, want 1", node.startCalls) + } +} + +// A kind: daemon node with a resolvable source is started via StartWith, +// never a plain Start — the whole point of the resolved config. +func TestFleetStartCallDaemonNodeUsesStartWith(t *testing.T) { + t.Setenv("SPINLOOP_CONFIG_DIR", t.TempDir()) + dir := writeFleetFile(t, "nodes:\n - name: dev-1\n host: dev1.local\n file: ./dev-1.Spinloop\n") + if err := os.WriteFile(filepath.Join(dir, "dev-1.Spinloop"), []byte("PROVIDER llamacpp\nMODEL org/m:Q4\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := fleet.Resolve(filepath.Join(dir, "fleet.yaml")) + if err != nil { + t.Fatal(err) + } + node := &fakeFleetNode{name: "dev-1"} + r := fleetStartCall(cfg)(context.Background(), node) + if !r.OK() { + t.Fatalf("fleetStartCall on a daemon node = %+v", r) + } + if node.startCalls != 0 { + t.Errorf("Start was called %d times for a daemon node with a resolved source, want 0", node.startCalls) + } + if node.startWithCalls != 1 { + t.Errorf("StartWith was called %d times, want 1", node.startWithCalls) + } +} + +// A resolved source that is itself broken — unparseable, or naming a +// provider StartWith's derivation cannot serve — must fail without ever +// calling Start or StartWith. This is the guard coverage of a bare 0%/100% +// count misses: resolveNodeSpinloop succeeding is not the same as the node +// being safe to start. +func TestFleetStartCallResolvedButBrokenSourceNeverStarts(t *testing.T) { + cases := map[string]string{ + "unparseable Spinloop": "this is not a Spinloop\x00\x01", + // deployConfigForNode shares remote deploy's runnerFor, which only + // accepts llamacpp/vllm — the same limit that already applies to a + // routed wake. An MLX (or any other) provider is resolved but + // cannot be turned into a deploy config. + "unsupported provider": "PROVIDER mlx\nMODEL org/m\n", + "no model at all": "PROVIDER llamacpp\n", + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + t.Setenv("SPINLOOP_CONFIG_DIR", t.TempDir()) + dir := writeFleetFile(t, "nodes:\n - name: dev-1\n host: dev1.local\n file: ./dev-1.Spinloop\n") + if err := os.WriteFile(filepath.Join(dir, "dev-1.Spinloop"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := fleet.Resolve(filepath.Join(dir, "fleet.yaml")) + if err != nil { + t.Fatal(err) + } + node := &fakeFleetNode{name: "dev-1"} + r := fleetStartCall(cfg)(context.Background(), node) + if r.OK() { + t.Fatalf("fleetStartCall on a broken source = %+v, want a failure", r) + } + if node.startCalls != 0 || node.startWithCalls != 0 { + t.Errorf("Start/StartWith were called (%d/%d) for a broken source, want neither", + node.startCalls, node.startWithCalls) + } + }) + } +} + +// An engine token that resolves to nothing must fail the node before +// StartWith is ever attempted, exactly as an unset tokenEnv does for the +// daemon's own bearer token. +func TestFleetStartCallUnresolvedEngineTokenNeverStarts(t *testing.T) { + t.Setenv("SPINLOOP_CONFIG_DIR", t.TempDir()) + dir := writeFleetFile(t, "nodes:\n - name: dev-1\n host: dev1.local\n file: ./dev-1.Spinloop\n engineTokenEnv: NOWHERE_ENGINE_KEY\n") + if err := os.WriteFile(filepath.Join(dir, "dev-1.Spinloop"), []byte("PROVIDER llamacpp\nMODEL org/m:Q4\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := fleet.Resolve(filepath.Join(dir, "fleet.yaml")) + if err != nil { + t.Fatal(err) + } + node := &fakeFleetNode{name: "dev-1"} + r := fleetStartCall(cfg)(context.Background(), node) + if r.OK() { + t.Fatalf("fleetStartCall with an unresolved engine token = %+v, want a failure", r) + } + if !strings.Contains(r.Detail(), "NOWHERE_ENGINE_KEY") { + t.Errorf("failure %q should name the unresolved variable", r.Detail()) + } + if node.startWithCalls != 0 { + t.Errorf("StartWith was called %d times, want 0", node.startWithCalls) + } +} + func TestCmdFleetUnknownNodeNamesTheKnownOnes(t *testing.T) { twoNodeFleet(t, "idle") err := cmdFleet([]string{"stop", "nope"}) diff --git a/cmd/spinloop/remote.go b/cmd/spinloop/remote.go index 32f1cb13..0d9887bd 100644 --- a/cmd/spinloop/remote.go +++ b/cmd/spinloop/remote.go @@ -1378,87 +1378,174 @@ installs the latest published release.`, // runRemoteDeploy is the body of `spinloop remote deploy`. func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr, region, spinloopVersion, apiKeyEnv string) error { - // deploy reads the Spinloop for what to serve, so unlike the other - // subcommands it always needs one — the per-user remote config alone is not - // enough. - sel, spinloopPath, err := readSpinloop("spinloop remote deploy ", spinloopArg(args)) + _, spinloopPath, dc, env, err := deriveDeployTarget("spinloop remote deploy ", spinloopArg(args)) if err != nil { return err } + outcome, err := runDeploy(spinloopPath, env, dc, deployOpts{ + dryRun: dryRun, + overwrite: overwrite, + reseed: reseed, + allowedCidr: allowedCidr, + region: region, + spinloopVersion: spinloopVersion, + apiKeyEnv: apiKeyEnv, + }) + if err != nil { + return err + } + fmt.Print(outcome.Text) + return nil +} + +// deriveDeployTarget turns a raw Spinloop argument (a bare alias name, a +// path, or a URL — whatever readSpinloop accepts) into everything a deploy +// needs: the Spinloop it read, the deploy config it derives, and the +// environment name it registers under. usage is readSpinloop's error-message +// context, so a caller other than `remote deploy` (namely `fleet deploy`) +// gets a message naming itself rather than a hard-coded command line. +// +// This is deliberately the same derivation `remote deploy` has always done — +// readSpinloop's alias-or-path resolution, the Spinloop's local environment, +// deployConfigFor, then the REMOTE name — so a node's resolved Spinloop +// source and a standalone `remote deploy` of the same file can never +// disagree about what they deploy. +func deriveDeployTarget(usage, spinloopArg string) (sel spinloop.Selection, spinloopPath string, dc remote.DeployConfig, env string, err error) { + sel, spinloopPath, err = readSpinloop(usage, spinloopArg) + if err != nil { + return + } // Respect the Spinloop's local environment (.env beside it, then its ENV // lines) before any AWS work, so the credentials the deploy signs with, the // region, and the SPINLOOP_REMOTE_* overrides all see it. ENV stays local — it // never enters dc, so nothing here reaches the deployed instance. - if err := applySpinloopEnv(sel, spinloopPath); err != nil { - return err + if err = applySpinloopEnv(sel, spinloopPath); err != nil { + return } - // A supplied key arrives the way every other secret does: as a reference - // to an environment variable, never a literal on the command line. The - // Spinloop's local environment has just been applied, so the variable is - // resolvable from the process environment; one set nowhere fails before - // anything is sent. - var apiKey string - if apiKeyEnv != "" { - apiKey = os.Getenv(apiKeyEnv) - if apiKey == "" { - return fmt.Errorf( - "--api-key-env: %s is not set: export it, or put it in the .env beside the Spinloop", - apiKeyEnv) - } - } - dc, err := deployConfigFor(sel, spinloopPath) + dc, err = deployConfigFor(sel, spinloopPath) if err != nil { - return err + return } // The environment name is the Spinloop's REMOTE — the committed link between // the Spinloop and its deployment. One source of truth: deploy registers the // environment under exactly the name the same Spinloop's REMOTE resolves to. - env := sel.Remote + env = sel.Remote if env == "" || !remote.IsEnvName(env) { - return fmt.Errorf( + err = fmt.Errorf( "%s must name its environment with `REMOTE ` (e.g. REMOTE %s) — deploy creates and registers that environment", spinloopPath, dc.ServedModelName) + return } - if allowedCidr != "" && !cidrPattern.MatchString(allowedCidr) { - return fmt.Errorf("--allowed-cidr must be an IPv4 CIDR (e.g. 203.0.113.7/32), got %q", allowedCidr) + return +} + +// deployOpts are the flags a deploy takes, independent of the Spinloop file +// itself — shared by `remote deploy` and `fleet deploy`, which each collect +// them from their own flag set. +type deployOpts struct { + dryRun bool + overwrite bool + reseed bool + allowedCidr string + region string + spinloopVersion string + // apiKeyEnv names the environment variable holding an externally + // supplied key to store as the environment's engine key, never a + // literal on the command line. Empty means the control plane manages + // its own key as it always has. + apiKeyEnv string +} + +// deployOutcome is a successful (or dry-run) deploy's result: the plan/result +// text a standalone `remote deploy` prints verbatim, plus the values a +// caller driving several nodes wants without re-parsing that text. A failed +// or guarded deploy is reported through the error runDeploy returns instead +// — an errDeployGuarded distinguishes "needs --overwrite" from any other +// failure. +type deployOutcome struct { + Text string + DryRun bool + BaseURL string + Seeding bool + SeedID string + EnvConfigPath string +} + +// errDeployGuarded means the named environment is already registered or +// live, and --overwrite was not given. Distinct from any other deploy +// failure so a caller driving several nodes (fleet deploy) can label this +// one "guarded" rather than "failed". +type errDeployGuarded struct { + env, what string +} + +func (e *errDeployGuarded) Error() string { + return fmt.Sprintf("environment %q %s — pass --overwrite to redeploy over it", e.env, e.what) +} + +// runDeploy is everything a deploy does once its target is known: validate +// the deploy-only flags, print the plan, and — unless --dry-run — clobber- +// guard, discover the control plane, deploy, and register the environment. +// It writes nothing to stdout itself; the caller decides what to do with +// deployOutcome.Text, which is how `fleet deploy` labels several nodes' +// outcomes instead of interleaving raw prints from concurrent goroutines. +func runDeploy(spinloopPath, env string, dc remote.DeployConfig, opts deployOpts) (deployOutcome, error) { + if opts.allowedCidr != "" && !cidrPattern.MatchString(opts.allowedCidr) { + return deployOutcome{}, fmt.Errorf("--allowed-cidr must be an IPv4 CIDR (e.g. 203.0.113.7/32), got %q", opts.allowedCidr) } // The spinloop release a fresh boot installs: empty (or `latest`) means the // boot's own default, a pin means exactly that release. Normalised the way // the control plane is — the v a tag carries is not part of the version — // and checked here, so a typo is named now rather than as a 404 inside a // boot nobody is watching. - if pin := strings.TrimPrefix(strings.TrimSpace(spinloopVersion), "v"); pin != "" && pin != "latest" { + if pin := strings.TrimPrefix(strings.TrimSpace(opts.spinloopVersion), "v"); pin != "" && pin != "latest" { if !spinloopVersionPattern.MatchString(pin) { - return fmt.Errorf("--spinloop-version must be a release version (e.g. 1.26.1) or latest, got %q", spinloopVersion) + return deployOutcome{}, fmt.Errorf("--spinloop-version must be a release version (e.g. 1.26.1) or latest, got %q", opts.spinloopVersion) } dc.SpinloopVersion = pin } + // A supplied key arrives the way every other secret does: as a reference + // to an environment variable, never a literal on the command line. By + // this point deriveDeployTarget has already applied the Spinloop's local + // environment (a process-wide side effect), so the variable is + // resolvable from here whichever caller is deploying; one set nowhere + // fails before anything is sent. + var apiKey string + if opts.apiKeyEnv != "" { + apiKey = os.Getenv(opts.apiKeyEnv) + if apiKey == "" { + return deployOutcome{}, fmt.Errorf( + "--api-key-env: %s is not set: export it, or put it in the .env beside the Spinloop", + opts.apiKeyEnv) + } + } - fmt.Printf("Deploying from %s\n", spinloopPath) - fmt.Printf(" environment: %s\n", env) - fmt.Printf(" runner: %s\n", dc.Runner) - fmt.Printf(" model: %s", dc.ModelID) + var buf strings.Builder + fmt.Fprintf(&buf, "Deploying from %s\n", spinloopPath) + fmt.Fprintf(&buf, " environment: %s\n", env) + fmt.Fprintf(&buf, " runner: %s\n", dc.Runner) + fmt.Fprintf(&buf, " model: %s", dc.ModelID) if dc.Quant != "" { - fmt.Printf(" (%s)", dc.Quant) + fmt.Fprintf(&buf, " (%s)", dc.Quant) } - fmt.Println() - fmt.Printf(" context: %d\n", dc.ContextSize) + buf.WriteByte('\n') + fmt.Fprintf(&buf, " context: %d\n", dc.ContextSize) if dc.Parallel > 0 { - fmt.Printf(" parallel: %d\n", dc.Parallel) + fmt.Fprintf(&buf, " parallel: %d\n", dc.Parallel) } - fmt.Printf(" served: %s\n", dc.ServedModelName) + fmt.Fprintf(&buf, " served: %s\n", dc.ServedModelName) // Companions are easy to get wrong quietly — a renamed file yields no // drafter and a slower endpoint with no error — so show what was picked up. for _, role := range slices.Sorted(maps.Keys(dc.Companions)) { - fmt.Printf(" %-8s %s\n", role+":", dc.Companions[role]) + fmt.Fprintf(&buf, " %-8s %s\n", role+":", dc.Companions[role]) } if len(dc.ServeArgs) > 0 { - fmt.Printf(" args: %s\n", strings.Join(dc.ServeArgs, " ")) + fmt.Fprintf(&buf, " args: %s\n", strings.Join(dc.ServeArgs, " ")) } // Worth stating: a re-seed costs a ~20-minute instance and re-downloads the // weights, so --reseed --dry-run must not look like a plain deploy. - if reseed { - fmt.Println(" reseed: yes — the weights will be re-fetched even if already in S3") + if opts.reseed { + buf.WriteString(" reseed: yes — the weights will be re-fetched even if already in S3\n") } // A fresh boot's spinloop: latest is a promise, not an absence, so the plan // always says which release a boot will install. @@ -1466,26 +1553,26 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr, if spinloopVer == "" { spinloopVer = "latest" } - fmt.Printf(" spinloop: %s\n", spinloopVer) + fmt.Fprintf(&buf, " spinloop: %s\n", spinloopVer) // A key is worth stating too — it rotates — but the value is never // printed, in the dry run or the report. - if apiKeyEnv != "" { - fmt.Printf(" api key: stored from %s\n", apiKeyEnv) + if opts.apiKeyEnv != "" { + fmt.Fprintf(&buf, " api key: stored from %s\n", opts.apiKeyEnv) } - if dryRun { - return nil + if opts.dryRun { + return deployOutcome{Text: buf.String(), DryRun: true}, nil } // The control URLs come from the control plane's stack outputs — the // environment may not exist yet, so there is nothing local to resolve. ctx := context.Background() - awsCfg, err := remote.LoadAWSConfig(ctx, resolveRegion(region)) + awsCfg, err := remote.LoadAWSConfig(ctx, resolveRegion(opts.region)) if err != nil { - return err + return deployOutcome{}, err } layer, err := deployDiscoverFn(ctx, awsCfg, controlPlaneStackName) if err != nil { - return err + return deployOutcome{}, err } cfg := layer.Config cfg.Environment = env @@ -1494,7 +1581,7 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr, // whose instance is live, needs explicit consent to redeploy over. envConfigPath, err := remote.EnvConfigPath(env) if err != nil { - return err + return deployOutcome{}, err } registered := false if _, err := os.Stat(envConfigPath); err == nil { @@ -1504,57 +1591,64 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr, if status, err := remoteStatusFn(ctx, cfg); err == nil { live = status.State == "running" || status.State == "pending" || status.State == "starting" } - if (registered || live) && !overwrite { + if (registered || live) && !opts.overwrite { what := "is already registered" if live { what = "has a live instance" } - return fmt.Errorf( - "environment %q %s — pass --overwrite to redeploy over it", env, what) + return deployOutcome{}, &errDeployGuarded{env: env, what: what} } // Ingress is per environment. A fresh environment needs a CIDR (default: // the caller's public address); an existing one keeps its ingress unless a // CIDR is given explicitly. + allowedCidr := opts.allowedCidr if allowedCidr == "" && !registered { allowedCidr, err = detectPublicCIDRFn(ctx) if err != nil { - return fmt.Errorf("detecting your public IP for the allowed CIDR: %w (pass --allowed-cidr)", err) + return deployOutcome{}, fmt.Errorf("detecting your public IP for the allowed CIDR: %w (pass --allowed-cidr)", err) } - fmt.Printf(" ingress: %s (your public IP; override with --allowed-cidr)\n", allowedCidr) + fmt.Fprintf(&buf, " ingress: %s (your public IP; override with --allowed-cidr)\n", allowedCidr) } - resp, err := remoteDeployFn(ctx, cfg, dc, allowedCidr, reseed, apiKey) + resp, err := remoteDeployFn(ctx, cfg, dc, allowedCidr, opts.reseed, apiKey) if err != nil { - return err + return deployOutcome{}, err } // Register the environment so REMOTE (and the other remote // subcommands) resolve to it from now on. cfg.BaseURL = resp.BaseURL if err := remote.SaveEnvironment(env, cfg); err != nil { - return err + return deployOutcome{}, err } - fmt.Println() - fmt.Printf("deployed: environment %s at %s\n", env, resp.BaseURL) - fmt.Printf("registered: %s\n", envConfigPath) + buf.WriteByte('\n') + fmt.Fprintf(&buf, "deployed: environment %s at %s\n", env, resp.BaseURL) + fmt.Fprintf(&buf, "registered: %s\n", envConfigPath) // A rotation is worth stating out loud: it invalidates the key a live // agent may still be holding, and the action — never the value — is all // the reply carries. if resp.APIKeyAction == "rotated" { - fmt.Println("api key: rotated — the previous key no longer works") + buf.WriteString("api key: rotated — the previous key no longer works\n") } else if resp.APIKeyAction != "" { - fmt.Println("api key: created") + buf.WriteString("api key: created\n") } if resp.Seeding { - fmt.Printf("seeding the weights — follow it with `spinloop remote seed status %s`.\n", resp.SeedID) - fmt.Println("Wait for it to finish before `spinloop remote start`, or the instance will") - fmt.Println("start against an incomplete download.") + fmt.Fprintf(&buf, "seeding the weights — follow it with `spinloop remote seed status %s`.\n", resp.SeedID) + buf.WriteString("Wait for it to finish before `spinloop remote start`, or the instance will\n") + buf.WriteString("start against an incomplete download.\n") } else { - fmt.Println("weights already in place — `spinloop remote start` will serve this.") + buf.WriteString("weights already in place — `spinloop remote start` will serve this.\n") } - return nil + + return deployOutcome{ + Text: buf.String(), + BaseURL: resp.BaseURL, + Seeding: resp.Seeding, + SeedID: resp.SeedID, + EnvConfigPath: envConfigPath, + }, nil } // cidrPattern matches an IPv4 CIDR, the same shape the deploy Lambda accepts. diff --git a/docs/commands/fleet.md b/docs/commands/fleet.md index fbea6cdc..6877bf2b 100644 --- a/docs/commands/fleet.md +++ b/docs/commands/fleet.md @@ -10,8 +10,10 @@ spinloop fleet metrics # each node's engine + system metrics spinloop fleet metrics -w # the same, redrawn in place until interrupted spinloop fleet dashboard # the interactive tiled view — watch it, drive it spinloop fleet route my-spinloop # which node a harness launch would pick -spinloop fleet start gpu-box # start one node's engine -spinloop fleet stop gpu-box # stop it +spinloop fleet start gpu-box # start one or more nodes' engines +spinloop fleet start --all # start every node in the fleet +spinloop fleet stop gpu-box # stop one or more nodes' engines +spinloop fleet deploy --all # create every kind: remote node's AWS environment ``` A fleet is also where [`spinloop harness`](harness.md#launching-against-your-fleet) @@ -94,13 +96,60 @@ nodes: ``` The environment's control URLs live in its `remote.json` (under -`~/.config/spinloop/remotes//`), written by `spinloop remote deploy` and never -stored in the fleet file. So a daemon and an environment sit side by side as the -same kind of row, and an environment that has not been deployed yet shows as -`config-error` on its row rather than blanking the fleet. See +`~/.config/spinloop/remotes//`), written by `spinloop remote deploy` — or by +[`spinloop fleet deploy`](#deploying-remote-nodes), which creates it from the +fleet file itself — and never stored in the fleet file. So a daemon and an +environment sit side by side as the same kind of row, and an environment that +has not been deployed yet shows as `config-error` on its row rather than +blanking the fleet. See [`examples/fleet-remote`](../../examples/fleet-remote/README.md) and [`examples/fleet-mixed`](../../examples/fleet-mixed/README.md). +### A node's Spinloop source + +Both `fleet deploy` (for a `kind: remote` node's environment) and `fleet +start` (for a `kind: daemon` node's engine) need to know what Spinloop file +describes what a node runs. A node names it with `file`, resolved relative to +the fleet file: + +```yaml +nodes: + - name: qwen + kind: remote + file: ./envs/qwen.Spinloop +``` + +`file` is optional, because the node's own `name` already doubles as a lookup +key. When it is absent, resolution tries, in order: + +1. `name` registered as a `spinloop alias` (`spinloop alias add qwen + ./envs/qwen.Spinloop`) — the same lookup a bare `spinloop remote deploy + qwen` already performs; +2. a subdirectory named after the node, beside the fleet file — `qwen/Spinloop` + next to `fleet.yaml` for a node named `qwen`, no fields needed on either + side. + +A fleet laid out as one subdirectory per node therefore needs nothing beyond +each node's own `name`: + +``` +fleet.yaml +qwen/Spinloop +llama/Spinloop +``` + +Nothing resolving is a per-node error naming all three ways a source could +have been given. For `fleet deploy` that always fails the node (there is +nothing to create an environment from); for `fleet start` on a `kind: daemon` +node it likewise fails that node's start — there is no fallback to a plain, +config-less start once this field exists. A `kind: remote` node's `start` is +unaffected by any of this: what it serves is fixed at deploy time, not pushed +at start time. + +This does not apply to `spinloop fleet dashboard`'s `s` key, which still +starts the selected node with a plain start, whatever the CLI's `fleet start` +would resolve for it. + ### Spreading or consolidating `prefer` decides which node wins when several could all serve you: @@ -392,29 +441,88 @@ Use it to check a route before an agent depends on it, to see what the other ## Starting and stopping -`fleet start` and `fleet stop` take **one node**: +`fleet start` and `fleet stop` take one or more node names, or `--all` for +the whole fleet: + +```sh +spinloop fleet start gpu-box # one node +spinloop fleet start gpu-box gpu-box-2 # several +spinloop fleet start --all # every node in the file +spinloop fleet stop --all +``` + +With neither a node nor `--all` they list the fleet and do nothing, rather +than acting on the whole fleet by accident; `--all` together with node names +is refused as ambiguous. An unknown name fails before anything is touched, +naming the nodes you could have meant. Several targeted nodes are driven +independently — one node's failure is reported against it alone and does not +stop the others, and the command exits non-zero if any of them failed. The +daemon's own rules still hold: starting a node whose engine is already +running reports its conflict, and stopping one that is not running succeeds +quietly. + +**Starting a `kind: daemon` node now requires its [Spinloop +source](#a-nodes-spinloop-source) to resolve.** When it does, `fleet start` +derives a deploy config from it and pushes it with the start (`StartWith`) — +telling the daemon what to run, the same way a routed `harness` launch +already tells a node what to run when it wakes one. When it does not resolve, +`fleet start` fails that node rather than starting it with whatever the +daemon already happens to have configured. This is a breaking change: every +fleet file with a `kind: daemon` node needs a `file` field, a matching alias, +or a matching subdirectory added, or `fleet start` fails for that node. A +`kind: remote` node's `start` is unaffected either way. + +## Deploying remote nodes + +`fleet deploy` creates the AWS environment for one or more `kind: remote` +nodes — the step that otherwise has to happen outside the fleet file +entirely, one `spinloop remote deploy ` at a time: + +```sh +spinloop fleet deploy qwen # one node +spinloop fleet deploy qwen llama # several +spinloop fleet deploy --all # every kind: remote node in the file +``` + +Each node deploys from its own resolved [Spinloop +source](#a-nodes-spinloop-source), reusing the exact derivation, consent, and +registration `spinloop remote deploy` uses for the same file — the two can +never disagree about what a given Spinloop deploys. A `kind: daemon` node +named explicitly fails the command, explaining that `deploy` provisions cloud +environments and that node is not one; `--all` only ever selects `kind: +remote` nodes, so a daemon node is never swept in by it. As with +`start`/`stop`, no node and no `--all` lists the fleet's `kind: remote` nodes +and deploys nothing, `--all` plus node names is refused as ambiguous, and +several targeted nodes deploy independently — one node's guard or failure is +reported against it alone. ```sh -spinloop fleet start gpu-box +spinloop fleet deploy --all --dry-run # print every plan, deploy nothing +spinloop fleet deploy qwen --overwrite # redeploy over a registered environment ``` -They deliberately refuse to act on the whole fleet — mutating every engine at -once is a footgun — so with no node they list the fleet and do nothing. An -unknown name fails, naming the nodes you could have meant. The daemon's own -rules still hold: starting a node whose engine is already running reports its -conflict, and stopping one that is not running succeeds quietly. +`--dry-run`, `--overwrite`, `--reseed`, `--allowed-cidr`, `--region`, and +`--spinloop-version` mean exactly what they mean on [`spinloop remote +deploy`](remote.md), applied per node. ## Flags | Flag | Meaning | | ---- | ------- | | `--fleet ` | The fleet file (default `./fleet.yaml`) | +| `--all` | `start`/`stop`/`deploy`: act on every node (or every `kind: remote` node, for `deploy`) instead of named ones | | `--node ` | `route` only: report this node rather than choosing one | | `--prefer` | `route` only: rank by `idle` or `active`, overriding the file | | `--format` | `metrics`: `bar` (default), `table`, or `json`; `logs`: `text` (default) or `json` | | `-w`, `--watch` | `metrics` only: redraw on an interval until interrupted | | `-f`, `--follow` | `logs` only: keep printing new output until interrupted | | `--limit` | `logs` only: lines of backlog per node (default 200) | +| `-n`, `--dry-run` | `deploy` only: print the plan for each targeted node without deploying | +| `--overwrite` | `deploy` only: proceed against an already-registered or live environment | +| `--reseed` | `deploy` only: re-fetch the weights even if already in S3 | +| `--allowed-cidr` | `deploy` only: who may reach each environment's instance | +| `--region` | `deploy` only: AWS region of the control plane | +| `--spinloop-version` | `deploy` only: spinloop release each environment installs at boot | ## See also diff --git a/docs/commands/remote.md b/docs/commands/remote.md index 2a97d5ee..cc21ca91 100644 --- a/docs/commands/remote.md +++ b/docs/commands/remote.md @@ -336,6 +336,12 @@ beside the Spinloop) and sends the value: the deploy creates or **rotates** the environment's key, so the old value stops working — and the reply says which happened, never the value itself. +Deploying several environments this way means running `remote deploy` once +per Spinloop file. [`spinloop fleet deploy`](fleet.md#deploying-remote-nodes) +does the same derivation, consent, and registration (`--api-key-env` +included) for every `kind: remote` node a fleet file names — or a chosen few +— in one command, each from its own resolved Spinloop source. + ## Flags | Flag | Meaning | diff --git a/examples/fleet-docker/fleet.yaml b/examples/fleet-docker/fleet.yaml index 5d90ca48..b3f8eab7 100644 --- a/examples/fleet-docker/fleet.yaml +++ b/examples/fleet-docker/fleet.yaml @@ -8,6 +8,13 @@ # daemon's; the engine is a different port, and normally the daemon reports it # — but here it binds 8080 inside the container and is published on another # port outside, which the daemon cannot know. That is what `engine:` is for. +# +# studio and gpu-box declare a `file`, so `spinloop fleet start` knows what to +# run on them — it points at the same Spinloop client/Spinloop wears for a +# routed launch, so a plain `fleet start` and a routed wake agree on what +# "this fleet's fake model" means. laptop deliberately declares none: it is +# what `fleet start laptop` (and `--all`) demonstrate failing on, rather than +# falling back to a plain, config-less start. prefer: idle nodes: @@ -19,6 +26,7 @@ nodes: port: 14242 tokenEnv: STUDIO_TOKEN engineTokenEnv: STUDIO_ENGINE_KEY + file: ./client/Spinloop engine: port: 18080 @@ -26,6 +34,7 @@ nodes: host: 127.0.0.1 port: 14243 tokenEnv: GPU_BOX_TOKEN + file: ./client/Spinloop engine: port: 18081 diff --git a/examples/fleet-docker/run-tests.sh b/examples/fleet-docker/run-tests.sh index cafa26b2..79851157 100755 --- a/examples/fleet-docker/run-tests.sh +++ b/examples/fleet-docker/run-tests.sh @@ -304,16 +304,18 @@ cleanup() { } ####################################### -# Assert a node that has never been told anything cannot be started. The -# daemon reads no Spinloop, so until a client sends a config there is nothing -# for `fleet start` to run — and it says so rather than guessing. +# Assert a node with no declared Spinloop source cannot be started. laptop +# names no `file`, no matching `spinloop alias`, and has no same-named +# subdirectory beside fleet.yaml — so `fleet start` refuses it client-side, +# before the daemon is ever contacted, rather than falling back to a plain +# start with nothing to run. ####################################### test_untold_node_cannot_start() { - echo "A node that has been told nothing" + echo "A node with no Spinloop source" local out out="$(fleet_with_stderr start laptop || true)" - assert_contains "starting an untold node says there is nothing to serve" \ - "${out}" "nothing to serve" + assert_contains "starting a sourceless node names the three ways one could resolve" \ + "${out}" "no Spinloop source" assert_equals "and nothing started" "$(node_state laptop)" "idle" } @@ -339,8 +341,9 @@ test_cold_start() { ####################################### test_start_stop_one_node() { echo "Driving one node" - # By now routing has woken studio once, so it has a config stored. Before - # that it had nothing: a node is told what to run, it does not know. + # studio declares a file field, so fleet start resolves what to run on it + # and pushes that config — the same one client/Spinloop names, which + # routing (test_routing, run before this) already woke it with once. fleet start studio >/dev/null if wait_for_state studio running 30; then pass "fleet start studio brings it up" @@ -365,6 +368,35 @@ test_start_stop_one_node() { fi } +####################################### +# Assert --all drives every node at once, and one node's failure to resolve +# a Spinloop source (laptop, still sourceless) does not stop the others from +# starting. +####################################### +test_start_all() { + echo "Starting the whole fleet with --all" + local out + out="$(fleet_with_stderr start --all || true)" + if wait_for_state studio running 30 && wait_for_state gpu-box running 30; then + pass "fleet start --all brings up studio and gpu-box" + else + fail "fleet start --all brings up studio and gpu-box" "running" \ + "studio=$(node_state studio) gpu-box=$(node_state gpu-box)" + fi + assert_equals "laptop is left idle -- it has no Spinloop source" \ + "$(node_state laptop)" "idle" + assert_contains "the summary names laptop as the one that failed" \ + "${out}" "laptop" + + fleet stop --all >/dev/null + if wait_for_state studio stopped 30 && wait_for_state gpu-box stopped 30; then + pass "fleet stop --all stops studio and gpu-box" + else + fail "fleet stop --all stops studio and gpu-box" "stopped" \ + "studio=$(node_state studio) gpu-box=$(node_state gpu-box)" + fi +} + ####################################### # Assert routing picks a node and wakes one when nothing is serving. This is # the published-port case: the engine binds 8080 inside each container and is @@ -375,7 +407,10 @@ test_start_stop_one_node() { ####################################### test_routing() { echo "Routing a launch at a node" - # The only Spinloop here: the nodes hold none. + # client/Spinloop is also what studio and gpu-box's fleet.yaml file field + # points fleet start at; a routed wake derives its config the same way, + # independently, from whatever Spinloop is being launched rather than from + # the node's own declared source. local spinloop_file="${HERE}/client/Spinloop" local out @@ -593,6 +628,8 @@ main() { echo test_start_stop_one_node echo + test_start_all + echo test_metrics echo test_unreachable_node diff --git a/examples/fleet-local/README.md b/examples/fleet-local/README.md index b54b7b9c..f1591d26 100644 --- a/examples/fleet-local/README.md +++ b/examples/fleet-local/README.md @@ -39,10 +39,13 @@ them. Nothing about the Spinloop or the command changes. nodes: - name: local host: 127.0.0.1 + file: ./Spinloop ``` -Everything else is a default worth knowing about, because each becomes a -decision on a real network: +`file` is the one line worth pausing on: it is what lets `fleet start local` +resolve what to run without a prior launch (more on that below). Everything +else is a default worth knowing about, because each becomes a decision on a +real network: - **No token.** A daemon on loopback needs none. Any node reachable across a network does — the daemon refuses to listen on a non-loopback address without @@ -94,9 +97,11 @@ spinloop fleet route # which node a launch would pick, changing nothing spinloop harness -O # wear ./Spinloop, route, wake if needed, launch ``` -The first launch is what tells the node anything at all: until then it has no -config and a bare `spinloop fleet start local` would say so. After one launch it -has the config stored, so `fleet start` restarts the same thing. +`fleet.yaml`'s `file: ./Spinloop` means `spinloop fleet start local` works from +cold, before any launch — it resolves the same Spinloop a routed launch would, +and pushes it. A launch still does more (waits for the engine to load, then +launches your agent against it), but starting the node no longer needs one to +have happened first. `spinloop fleet route` before your first launch: diff --git a/examples/fleet-local/fleet.yaml b/examples/fleet-local/fleet.yaml index 28df26e4..60b0d2e2 100644 --- a/examples/fleet-local/fleet.yaml +++ b/examples/fleet-local/fleet.yaml @@ -9,6 +9,12 @@ # token; the engine is on loopback too, which is fine because the node is # reached over loopback; and llama.cpp's port is the one the daemon reports, so # there is no `engine:` block to write. Everything in this file is the default. +# +# `file` points `spinloop fleet start local` at the same Spinloop `spinloop +# harness -O` wears — the one Spinloop file describes what this node runs +# either way, so `fleet start` works from cold, before any launch has told +# the daemon anything. nodes: - name: local host: 127.0.0.1 + file: ./Spinloop diff --git a/examples/fleet-mixed/README.md b/examples/fleet-mixed/README.md index 9772f624..d9cfd2b9 100644 --- a/examples/fleet-mixed/README.md +++ b/examples/fleet-mixed/README.md @@ -9,22 +9,40 @@ single table. ### 1. Bring up a daemon (a machine node) -On the box you want in the fleet, run the daemon: +On the box you want in the fleet, run the daemon — it takes no Spinloop of +its own, just its flags: ```sh -SPINLOOP_API_TOKEN=… spinloop daemon ./Spinloop +SPINLOOP_API_TOKEN=… spinloop daemon ``` Put that token in a `.env` beside `fleet.yaml` (copy [`.env.example`](.env.example)). This is the same as [`examples/fleet`](../fleet/README.md); for daemons in containers rather than real machines, see -[`examples/fleet-docker`](../fleet-docker/README.md). +[`examples/fleet-docker`](../fleet-docker/README.md). What it runs is decided +by whoever starts it — see the next section. -### 2. Register the environments (the cloud nodes) +### 2. Give each node a Spinloop source, then bring them up -Each remote environment is created and registered by a `spinloop remote deploy` -of a `Spinloop` that says `REMOTE ` — see [`spinloop -remote`](../../docs/commands/remote.md). +Every node here finds its Spinloop through the subdirectory convention — +[`gpu-box/Spinloop`](gpu-box/Spinloop) for the machine, +[`qwen/Spinloop`](qwen/Spinloop) and [`llama/Spinloop`](llama/Spinloop) for +the two environments — so none of them declares a `file` field in +`fleet.yaml`. A registered `spinloop alias` named after a node would resolve +the same way, and win over the subdirectory; an explicit `file` field can +point anywhere else again, which is what +[`examples/fleet-docker`](../fleet-docker/) uses instead. See +[`spinloop fleet`](../../docs/commands/fleet.md#a-nodes-spinloop-source) for +the full resolution order. + +```sh +spinloop fleet start gpu-box # tell the daemon what to run, and start it +spinloop fleet deploy --all # create both environments from this file +``` + +Creating the environments this way is the same as running `spinloop remote +deploy` once per `Spinloop` that says `REMOTE ` — see [`spinloop +remote`](../../docs/commands/remote.md) — just one command for both. ### 3. Observe the whole fleet diff --git a/examples/fleet-mixed/fleet.yaml b/examples/fleet-mixed/fleet.yaml index ccd79384..d530c5c0 100644 --- a/examples/fleet-mixed/fleet.yaml +++ b/examples/fleet-mixed/fleet.yaml @@ -9,6 +9,11 @@ # env. The token and the control URLs stay out of this file — in the .env and # in each environment's remote.json — so it names machines and environments, # never secrets or accounts. +# +# None of the three nodes below declares a `file` field: each one's own name +# is already a subdirectory beside this file (gpu-box/Spinloop, qwen/Spinloop, +# llama/Spinloop) — what `fleet start`/`fleet deploy` read to know what a node +# runs, with nothing to declare beyond the node's own name. nodes: # A GPU box on the network, reached over the daemon's control API. - name: gpu-box diff --git a/examples/fleet-mixed/gpu-box/Spinloop b/examples/fleet-mixed/gpu-box/Spinloop new file mode 100644 index 00000000..3b3b6118 --- /dev/null +++ b/examples/fleet-mixed/gpu-box/Spinloop @@ -0,0 +1,6 @@ +# What gpu-box runs — read by `spinloop fleet start gpu-box`, which derives +# a deploy config from it and pushes it to the daemon. +PROVIDER llamacpp +ALIAS gemma-4-12b-it +MODEL unsloth/gemma-4-12b-it-GGUF:Q6_K +CONTEXT 32768 diff --git a/examples/fleet-mixed/llama/Spinloop b/examples/fleet-mixed/llama/Spinloop new file mode 100644 index 00000000..49e4ceff --- /dev/null +++ b/examples/fleet-mixed/llama/Spinloop @@ -0,0 +1,8 @@ +# What the "llama" environment serves. Found by fleet deploy through the +# subdirectory convention — llama/Spinloop, matching the node's own name in +# ../fleet.yaml, so no `file` field is needed there. +PROVIDER llamacpp +ALIAS llama-3-70b +MODEL meta-llama/Llama-3.3-70B-Instruct-GGUF:Q4_K_M +CONTEXT 65536 +REMOTE llama diff --git a/examples/fleet-mixed/qwen/Spinloop b/examples/fleet-mixed/qwen/Spinloop new file mode 100644 index 00000000..2d33d3ec --- /dev/null +++ b/examples/fleet-mixed/qwen/Spinloop @@ -0,0 +1,6 @@ +# What the "qwen" environment serves — read by `spinloop fleet deploy qwen`. +PROVIDER llamacpp +ALIAS qwen3.6-27b +MODEL unsloth/Qwen3.6-27B-MTP-GGUF:UD-Q6_K_XL +CONTEXT 131072 +REMOTE qwen diff --git a/examples/fleet-remote/README.md b/examples/fleet-remote/README.md index fb7bc6ce..3907a907 100644 --- a/examples/fleet-remote/README.md +++ b/examples/fleet-remote/README.md @@ -20,6 +20,17 @@ remote`](../../docs/commands/remote.md). List what you already have: spinloop remote ls ``` +Or create both of this fleet's environments straight from the file: + +```sh +spinloop fleet deploy --all --dry-run # see the plan for qwen and llama first +spinloop fleet deploy --all # then create them +``` + +That reads [`qwen/Spinloop`](qwen/Spinloop) and +[`llama/Spinloop`](llama/Spinloop) — see the next section for how a node finds +its own Spinloop file. + ### 2. Name them as a fleet [fleet.yaml](fleet.yaml) lists the environments — the node's name is the environment: @@ -27,9 +38,22 @@ spinloop remote ls ```yaml nodes: - name: qwen # the registered environment, and what you type at `fleet start qwen` - kind: remote + kind: remote # resolved from qwen/Spinloop — see fleet.yaml + + - name: llama + kind: remote # resolved from llama/Spinloop the same way ``` +Neither node declares a `file` field: each one's own name is already a +subdirectory beside this file (`qwen/Spinloop`, `llama/Spinloop`), so there is +nothing more to declare. A registered `spinloop alias` named after a node +would resolve the same way, and win over the subdirectory if both existed — +or a `file` field can point anywhere else entirely, which is what +[`examples/fleet-docker`](../fleet-docker/) uses to reuse one Spinloop +(`client/Spinloop`) whose name matches neither node that runs it. See +[`spinloop fleet`](../../docs/commands/fleet.md#a-nodes-spinloop-source) for +the full resolution order. + ### 3. Observe from anywhere From any machine your AWS credentials reach: diff --git a/examples/fleet-remote/fleet.yaml b/examples/fleet-remote/fleet.yaml index c032a7dc..084d75e2 100644 --- a/examples/fleet-remote/fleet.yaml +++ b/examples/fleet-remote/fleet.yaml @@ -2,13 +2,25 @@ # # spinloop fleet status # one row per environment: state and what it serves # spinloop fleet metrics -w # a live dashboard +# spinloop fleet deploy --all # create both environments from this file # # Each node's `name` is the registered environment it drives — one per -# `spinloop remote deploy` — and what you type at `fleet start `. `kind: -# remote` marks it as a cloud environment rather than a host. No bearer tokens -# here: the control plane signs each call, and the environment's URLs live in -# its remote.json, not in this file. So, like every fleet file, this one names -# environments, never an account. +# `spinloop remote deploy`, or per `spinloop fleet deploy` — and what you type +# at `fleet start `. `kind: remote` marks it as a cloud environment +# rather than a host. No bearer tokens here: the control plane signs each +# call, and the environment's URLs live in its remote.json, not in this file. +# So, like every fleet file, this one names environments, never an account. +# +# `fleet deploy` needs to know what each environment serves — the same thing +# `spinloop remote deploy ` needs, just resolved from the node rather +# than typed on the command line. Neither node here declares a `file` field: +# each one's own name is a subdirectory beside this file (qwen/Spinloop, +# llama/Spinloop) — the convention for a fleet laid out as one subdirectory +# per node, with nothing to declare beyond the node's name. (A registered +# `spinloop alias` named after a node would resolve the same way, and win +# over the subdirectory if both existed.) See +# [`examples/fleet-docker`](../fleet-docker/) for a `file` field pointing +# somewhere that does *not* match the node's own name. nodes: - name: qwen kind: remote diff --git a/examples/fleet-remote/llama/Spinloop b/examples/fleet-remote/llama/Spinloop new file mode 100644 index 00000000..c734a4be --- /dev/null +++ b/examples/fleet-remote/llama/Spinloop @@ -0,0 +1,8 @@ +# What the "llama" environment serves. Found by fleet deploy through the +# subdirectory convention — this file sits at llama/Spinloop, matching the +# node's own name in ../fleet.yaml, so no `file` field is needed there. +PROVIDER llamacpp +ALIAS llama-3-70b +MODEL meta-llama/Llama-3.3-70B-Instruct-GGUF:Q4_K_M +CONTEXT 65536 +REMOTE llama diff --git a/examples/fleet-remote/qwen/Spinloop b/examples/fleet-remote/qwen/Spinloop new file mode 100644 index 00000000..167465f2 --- /dev/null +++ b/examples/fleet-remote/qwen/Spinloop @@ -0,0 +1,8 @@ +# What the "qwen" environment serves — read by `spinloop fleet deploy qwen` +# the same way `spinloop remote deploy ./qwen.Spinloop` would read it directly. +# See docs/commands/remote.md for what each instruction does. +PROVIDER llamacpp +ALIAS qwen3.6-27b +MODEL unsloth/Qwen3.6-27B-MTP-GGUF:UD-Q6_K_XL +CONTEXT 131072 +REMOTE qwen diff --git a/examples/fleet/README.md b/examples/fleet/README.md index e3cbc3df..0cf59758 100644 --- a/examples/fleet/README.md +++ b/examples/fleet/README.md @@ -2,17 +2,20 @@ Observing several machines' engines from one place. -Each machine runs the daemon: +Each machine runs the daemon — it takes no Spinloop of its own, just its flags: ```sh # on studio.local and gpu-box, with a token since they are network-reachable -SPINLOOP_API_TOKEN=… spinloop daemon ./Spinloop +SPINLOOP_API_TOKEN=… spinloop daemon # on this machine, loopback-only needs no token -spinloop daemon --api-addr 127.0.0.1:4242 ./Spinloop +spinloop daemon --api-addr 127.0.0.1:4242 ``` -Then from anywhere that can reach them: +What a node runs is decided by whoever starts it — `fleet.yaml` names the +Spinloop that describes it (see `studio`'s entry: `studio/Spinloop`, found +automatically since the subdirectory's name matches the node's), so +`fleet start` knows what to push. Then from anywhere that can reach them: ```sh cp .env.example .env # fill in each node's token diff --git a/examples/fleet/fleet.yaml b/examples/fleet/fleet.yaml index 2e663264..e424e935 100644 --- a/examples/fleet/fleet.yaml +++ b/examples/fleet/fleet.yaml @@ -17,6 +17,11 @@ prefer: idle nodes: # A Mac on the LAN serving MLX models. Its daemon listens on the default # port, and it is reachable over the network — so it needs a token. + # `fleet start studio` needs to know what to push — without a source, a + # node has nothing to start with until some other client tells it. Here + # that source is `studio/Spinloop`, resolved automatically because the + # subdirectory's name matches the node's; a `file` field or a registered + # `spinloop alias` named "studio" are the other two ways to give it. - name: studio host: studio.local tokenEnv: STUDIO_TOKEN diff --git a/examples/fleet/studio/Spinloop b/examples/fleet/studio/Spinloop new file mode 100644 index 00000000..4b565820 --- /dev/null +++ b/examples/fleet/studio/Spinloop @@ -0,0 +1,10 @@ +# What studio runs — read by `spinloop fleet start studio`, which derives a +# deploy config from it and pushes it to the daemon. PROVIDER llamacpp here, +# not mlx: pushing a config to a node — from `fleet start` or from a routed +# wake — only supports the runners remote deploy also supports, llamacpp or +# vllm. That is a pre-existing limit of the derivation this reuses, not +# something new here. +PROVIDER llamacpp +ALIAS qwen3-27b +MODEL unsloth/Qwen3-27B-GGUF:Q4_K_M +CONTEXT 32768 diff --git a/internal/fleet/config.go b/internal/fleet/config.go index 1433bc48..1b702ea8 100644 --- a/internal/fleet/config.go +++ b/internal/fleet/config.go @@ -118,6 +118,15 @@ type NodeConfig struct { // publishing it on a different port than it binds inside, a node // reached through a tunnel. Engine *EngineOverride `yaml:"engine"` + // File names the Spinloop file that describes what this node runs — + // what `spinloop fleet deploy` reads to create a kind: remote node's + // environment, and what `spinloop fleet start` reads to tell a kind: + // daemon node's engine what to run. Resolved relative to the fleet + // file's directory. Optional: a node's own Name is tried as a + // registered `spinloop alias`, then as a same-named subdirectory + // beside the fleet file, before either command gives up on it. Not + // read by any other fleet command. + File string `yaml:"file"` } // EngineOverride is a node's declared engine endpoint. Each field is optional @@ -237,13 +246,25 @@ func (c *Config) Node(name string) (NodeConfig, bool) { // fan-out still runs, over a fleet of one. An unknown name fails here, naming // what could have been typed, rather than at the socket. func (c *Config) Only(name string) (*Config, error) { - entry, ok := c.Node(name) - if !ok { - return nil, fmt.Errorf("no node %q in %s (known nodes: %s)", - name, c.Path, strings.Join(c.Names(), ", ")) + return c.OnlyNames([]string{name}) +} + +// OnlyNames narrows the config to several named nodes, in the order given, +// so a command that fans out by default can be pointed at exactly the nodes +// named rather than the whole fleet. An unknown name fails here, before any +// node is touched, naming what could have been typed. +func (c *Config) OnlyNames(names []string) (*Config, error) { + nodes := make([]NodeConfig, 0, len(names)) + for _, name := range names { + entry, ok := c.Node(name) + if !ok { + return nil, fmt.Errorf("no node %q in %s (known nodes: %s)", + name, c.Path, strings.Join(c.Names(), ", ")) + } + nodes = append(nodes, entry) } narrowed := *c - narrowed.Nodes = []NodeConfig{entry} + narrowed.Nodes = nodes return &narrowed, nil } diff --git a/internal/fleet/config_test.go b/internal/fleet/config_test.go index b86f91fa..d9f220fc 100644 --- a/internal/fleet/config_test.go +++ b/internal/fleet/config_test.go @@ -487,6 +487,78 @@ func TestPreferSetting(t *testing.T) { } } +// The file field names a node's Spinloop source; it is stored as declared +// (resolution relative to the fleet directory is resolveNodeSpinloop's job, +// not parsing's), needs no particular kind, and is optional. +func TestFileField(t *testing.T) { + path := writeFleet(t, ` +nodes: + - name: gpu-env + kind: remote + file: ./envs/gpu.Spinloop + - name: dev-1 + host: dev1.local + file: ../shared/dev.Spinloop + - name: plain + host: plain.local +`, "") + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + remote, _ := cfg.Node("gpu-env") + if remote.File != "./envs/gpu.Spinloop" { + t.Errorf("remote node File = %q", remote.File) + } + daemonNode, _ := cfg.Node("dev-1") + if daemonNode.File != "../shared/dev.Spinloop" { + t.Errorf("daemon node File = %q, want it to parse the same as any other kind", daemonNode.File) + } + plain, _ := cfg.Node("plain") + if plain.File != "" { + t.Errorf("plain node File = %q, want empty", plain.File) + } +} + +func TestOnlyNames(t *testing.T) { + path := writeFleet(t, ` +nodes: + - name: a + host: a.local + - name: b + host: b.local + - name: c + host: c.local +`, "") + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + + narrowed, err := cfg.OnlyNames([]string{"c", "a"}) + if err != nil { + t.Fatal(err) + } + if got := narrowed.Names(); len(got) != 2 || got[0] != "c" || got[1] != "a" { + t.Errorf("OnlyNames order = %v, want [c a] (the order given, not file order)", got) + } + + if _, err := cfg.OnlyNames([]string{"a", "nope"}); err == nil { + t.Fatal("an unknown name among several should fail") + } else if !strings.Contains(err.Error(), "nope") { + t.Errorf("error %q does not name the unknown node", err) + } + + // Only(name) is OnlyNames([]string{name}), unchanged for its own callers. + one, err := cfg.Only("b") + if err != nil { + t.Fatal(err) + } + if got := one.Names(); len(got) != 1 || got[0] != "b" { + t.Errorf("Only(b).Names() = %v, want [b]", got) + } +} + func TestPreferRejectsUnknownValue(t *testing.T) { _, err := Load(writeFleet(t, "prefer: whatever\nnodes:\n - name: a\n host: a.local\n", "")) if err == nil { diff --git a/openspec/changes/archive/2026-09-03-fleet-deploy/.openspec.yaml b/openspec/changes/archive/2026-09-03-fleet-deploy/.openspec.yaml new file mode 100644 index 00000000..9696e00f --- /dev/null +++ b/openspec/changes/archive/2026-09-03-fleet-deploy/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-03 diff --git a/openspec/changes/archive/2026-09-03-fleet-deploy/design.md b/openspec/changes/archive/2026-09-03-fleet-deploy/design.md new file mode 100644 index 00000000..013cf78f --- /dev/null +++ b/openspec/changes/archive/2026-09-03-fleet-deploy/design.md @@ -0,0 +1,344 @@ +## Context + +`spinloop remote deploy ` already does everything a +`kind: remote` node's deploy needs: it derives a `remote.DeployConfig` from a +Spinloop file (`deployConfigFor`), resolves the target environment name from +the Spinloop's `REMOTE` instruction, guards against clobbering a +registered-or-live environment unless `--overwrite`, prints a plan (or stops +there on `--dry-run`), calls the control plane, and registers the result +under `~/.config/spinloop/remotes//remote.json`. That whole body lives +in one function, `runRemoteDeploy` in `cmd/spinloop/remote.go`, driven by a +single Spinloop path. + +Separately, `internal/fleet/wake.go`'s `Config.Wake` already derives a +`remote.DeployConfig` too (via `deployConfigForNode`, the node-owned variant: +no forced context size, the preset's bind survives) and pushes it to a node +with `Node.StartWith` — but only when a routed launch wakes an idle node to +serve *that launch's* Spinloop. Nothing is stored against the node; the +config is recomputed fresh every time from whatever is being launched. A +`daemonNode.StartWith` forwards it to the daemon; a `remoteNode.StartWith` +always refuses — a remote environment's model is fixed for good at deploy +time, not pushed at wake time (`internal/fleet/remote_node.go:77-83`). + +`fleet.yaml` (`internal/fleet/config.go`) has no field connecting a node to a +Spinloop file at all today. That is the actual gap two different fleet +commands hit: + +- A `kind: remote` node's environment can only be created outside the fleet + file, one `remote deploy ` at a time. +- A `kind: daemon` node's `fleet start ` has no way to say what that + node should run — it can only start whatever the daemon is already + configured with, unlike a routed wake, which always knows because it is + driven by the Spinloop being launched, not by the node. + +Both gaps are closed by the same fix: give a node a resolvable link to a +Spinloop file, then let each command use it the way it already knows how to +use a `DeployConfig` — `fleet deploy` via `deployConfigFor` (cloud-owned, +persistent), `fleet start` via `deployConfigForNode` + `StartWith` (node-owned, +one wake), exactly `Config.Wake` already does per launch. See `proposal.md` +for why this gap matters and `specs/fleet-config` and `specs/fleet-client` for +the resulting behavior. + +## Goals / Non-Goals + +**Goals:** +- One command deploys every named remote node a fleet file names, without + giving up any behavior a standalone `remote deploy` already provides for + one. +- A node's resolved Spinloop source and a standalone `remote deploy + `/`` can never disagree, because both run through + the same resolution and the same derivation code. +- `fleet start` on a `kind: daemon` node uses the same resolved source to + tell the daemon what to run, the same way a routed wake already does for + the Spinloop being launched — and requires it, exactly as `fleet deploy` + requires one for a `kind: remote` node. No unreleased tool has existing + users to protect, so there is no fallback path to design around: one + resolution mechanism, required everywhere it is the only source of truth. +- `fleet deploy`, `fleet start`, and `fleet stop` all take one or more + explicit node names or `--all`, so bringing up or down several nodes at + once is one command rather than one invocation per node. +- One node's deploy, start, or stop failure does not stop the others + targeted in the same command. + +**Non-Goals:** +- Provisioning `kind: daemon` nodes (installing the daemon on a bare + machine). Out of scope per the proposal — this only tells an + already-running daemon what to run. +- Changing how a routed launch (`spinloop harness`) wakes a node. That path + keeps deriving its `DeployConfig` from the Spinloop being launched, exactly + as today; a node's own resolved source is a separate, independent input + used only by `fleet start` run directly. +- Changing anything about how an already-deployed remote node's environment + itself answers a stop or a status call, or how a `kind: remote` node's + `start` behaves — it always uses a plain start, resolved source or not. +- A new deploy Lambda contract or control-plane change. `fleet deploy` is a + client-side batching of the same calls `remote deploy` already makes. + +## Decisions + +### Extract `runRemoteDeploy`'s body into reusable functions + +`runRemoteDeploy(args, dryRun, overwrite, reseed, allowedCidr, region, +spinloopVersion)` currently resolves its Spinloop argument via `readSpinloop` +(which itself tries `resolveAlias` before treating the argument as a literal +path or URL) and prints/returns directly. Split it into: + +- `deriveDeployTarget(spinloopArg string) (sel spinloop.Selection, + spinloopPath string, dc remote.DeployConfig, env string, err error)` — the + existing `readSpinloop` (alias-or-path resolution) + `applySpinloopEnv` + + `deployConfigFor` + `REMOTE`-name resolution, unchanged in behavior. Taking + the raw, unresolved argument (rather than an already-resolved path) is what + lets `fleet deploy` hand it a node's bare name and get the same alias + resolution a standalone `remote deploy ` gets. +- `runDeploy(env string, dc remote.DeployConfig, opts deployOpts) deployOutcome` + — everything from the plan print onward (the existing body from `fmt.Printf("Deploying from ...")` + through registration), taking the already-derived `dc` and `env` rather than + re-deriving them, and returning a value instead of writing straight to + stdout/returning an error, so a fleet-wide caller can label each node's + outcome instead of interleaving raw prints. + + `spinloop remote deploy` becomes a thin wrapper: derive, then call + `runDeploy` once and print its outcome exactly as today. + +This is the same shape the codebase already uses for `deployConfigFor` / +`deployConfigForNode` sharing one `deployConfig` body — a derivation function +plus a target-specific wrapper — so it is consistent with the existing +pattern rather than a new one. + +**Alternative considered**: have `fleet deploy` shell out to `spinloop remote +deploy` as a subprocess per node. Rejected — it would need to reconstruct +flags as argv, lose typed error handling (the per-node "guard vs. failure" +distinction the spec requires), and complicate testing. + +### A node's Spinloop source resolves the same way for every consumer + +Add `File string \`yaml:"file"\`` to `NodeConfig`, available on either node +kind. This is deliberately *not* a new resolution mechanism: a node's `name` +is already a key `spinloop alias` can map to a Spinloop file, and +`readSpinloop` already turns a directory argument into `/Spinloop` +(`cmd/spinloop/main.go`'s `os.Stat` + `IsDir` check ahead of `os.ReadFile`, +the same join `spinloop apply ` relies on today) — so a node whose name +matches an existing alias, or that simply has a same-named subdirectory +beside the fleet file, needs no `file` field at all. + +A shared helper resolves one node to one argument, tried in order and +stopping at the first that resolves: + +```go +func resolveNodeSpinloop(node fleet.NodeConfig, fleetDir string) (arg, source string, err error) +``` + +1. `file` set → resolve it relative to `fleetDir` into a path, and use that. + A real path never matches an alias name, so `readSpinloop` (called next, + inside `deriveDeployTarget`) treats it as the literal Spinloop file (or + URL) to read, exactly as an explicit argument to `remote deploy ` + would. +2. `file` unset → check the node's own `Name` against the alias registry + (`config.Load().Alias(name)`, the same lookup `resolveAlias` makes) — a + hit means the alias's own target path becomes the argument, resolved + here rather than by handing `Name` to `readSpinloop` and letting *its* + `resolveAlias` step resolve it. +3. No alias named after the node → check whether `/` exists + as a directory; a hit means that directory becomes the argument, and + `readSpinloop`'s own directory join finds `//Spinloop` + inside `deriveDeployTarget`. +4. None of the three resolve → `resolveNodeSpinloop` returns an error naming + all three: no `file` field, no alias named ``, no `/` + subdirectory beside the fleet file. + +Step 2 resolves the alias's path itself rather than reusing `readSpinloop`'s +own `resolveAlias` on the bare `Name` — an earlier version of this design did +the latter, and it is wrong: `resolveAlias` deliberately lets a same-named +path on disk beat a registered alias (documented at its definition: "every +existing invocation passes a path, so registering an alias must never change +what an already-working command does"). A node named the same as its own +subdirectory (the common case for the layout step 3 exists to support) would +then resolve to the subdirectory even with an alias registered — the +opposite of this function's own precedence, alias before subdirectory. This +was caught by `TestCmdFleetDeployAliasWinsOverSubdirectory` failing against +the original implementation. Steps 2 and 4 need one read of the alias +registry to decide *whether* to try it — that read is unavoidable because +the caller needs to know whether to fall through to the subdirectory check, +not just call `deriveDeployTarget` once and inspect the error. + +`resolveNodeSpinloop` is shared by both consumers, and both treat step 4 the +same way: a hard per-node failure (see the next two decisions). Neither +falls back to acting without a resolved source. + +**Alternative considered**: make `file` required whenever no alias named +after the node exists, dropping the subdirectory convention. Rejected per +the request to support a fleet laid out as one subdirectory per node +(`fleet.yaml` beside `dev-1/Spinloop`, `dev-2/Spinloop`, …) with zero +per-node configuration beyond the node's own name. + +### `fleet deploy` requires an explicit target + +``` +spinloop fleet deploy +spinloop fleet deploy --all +``` + +No node and no `--all` fails, listing the fleet's `kind: remote` nodes, +deploying nothing — the same rule `driveOneNode` already enforces today for +`start`/`stop` (and that `start`/`stop` keep after their own rewrite below): +a command that creates or mutates cloud resources for however many nodes +are listed must never do so by accident because the operator forgot an +argument. `--all` and explicit node names together is +rejected as ambiguous. Named args resolve to exactly those nodes, in the +order given; an unknown name fails before anything is deployed. A named +`kind: daemon` node fails the command outright. `--all` selects every `kind: +remote` node and nothing else — a `kind: daemon` node is never a candidate +for it, so there is nothing to skip or report for that case. + +For each targeted node, `resolveNodeSpinloop` runs; a node for which nothing +resolves fails for that node alone, without touching the other targeted +nodes (see fleet-client's "derives and applies each node's config" +requirement). + +Deploys run concurrently, mirroring `Config.FanOut`'s shape +(`internal/fleet/fanout.go`) but calling `runDeploy` per node instead of a +daemon HTTP call. Reusing `FanOut` itself is not a fit: it is built around +`Node`/`Call` (a live daemon or remote-node handle and a read/write against +it), while a deploy has no `Node` yet — deploying *creates* what a `Node` +would later address. `fleetDeployCmd` therefore builds its own small +concurrent loop, keyed by node name, collecting one outcome per node the +same shape `NodeResult` already gives fan-out callers (ok / guarded / +failed), rendered as one line per node plus a final non-zero exit when any +node failed. + +### `fleet start` and `fleet stop` take multiple nodes or `--all`, reusing `FanOut` + +``` +spinloop fleet start | spinloop fleet start --all +spinloop fleet stop | spinloop fleet stop --all +``` + +Same target-selection rule as `fleet deploy`, shared by both commands: no +node and no `--all` fails, listing the fleet's nodes; `--all` plus names is +ambiguous; an unknown name fails before anything starts or stops. Neither is +restricted to one kind — both a `kind: daemon` and a `kind: remote` name are +valid targets for either, since starting and stopping (unlike creating a +cloud environment) are meaningful for both. `--all` therefore selects every +node in the file, not just the remote ones. + +Unlike a deploy target, every node `start`/`stop` targets already exists as +a `fleet.Node` — a `kind: daemon` node's daemon is already reachable, a +`kind: remote` node's environment is already registered (`fleet deploy`, or +a standalone `remote deploy`, already ran). So both reuse `Config.FanOut` +directly instead of a bespoke loop, unlike `fleet deploy` (see that +decision's reasoning about a deploy having no `Node` yet): + +1. Add `func (c *Config) OnlyNames(names []string) (*Config, error)` to + `internal/fleet/config.go`, narrowing to several named nodes in the order + given — an unknown name fails immediately, naming the known nodes, before + any node is touched. `Only(name string)` becomes `OnlyNames([]string{name})`, + unchanged for its existing callers (`fleet logs `, + `select.go`'s `--node` pin). +2. Add a shared `runFleetDrive(cfg *Config, all bool, names []string, call + fleet.Call) ([]fleet.NodeResult, error)` in `cmd/spinloop/fleet.go`: applies + the target-selection rule above (delegating to `OnlyNames` or `FanOut` + over the whole `cfg`) and returns the fanned-out results, or an error for + a selection problem (no target, ambiguous target, unknown name) caught + before any node is touched. This replaces `driveOneNode`, which is + deleted — both `fleetStartCmd` and `fleetStopCmd` call `runFleetDrive` + with their own `call`, then render the results and pick an exit code + through one shared renderer, the way `fleet deploy` already labels + deployed/guarded/failed per node. +3. `fleetStartCmd` builds its `call` closing over `cfg`, looking up + `cfg.Node(n.Name())` to recover the targeted node's `NodeConfig` (kind, + `File`) — `Call`'s signature (`func(ctx, Node) NodeResult`) does not + carry it, but every node the closure is called with came from `cfg` in + the first place, so the lookup by name always succeeds; no signature + change to `fleet.Call`/`FanOut` is needed. For a `kind: daemon` entry, the + closure resolves a source (`resolveNodeSpinloop`), derives a `dc` + (`deployConfigForNode`), and calls `n.StartWith`; for `kind: remote`, it + calls `n.Start` unchanged. Failure to resolve is a `NodeResult` like any + other — `FanOut` already treats a bad node as a row, not an abort, so a + mix of resolved and unresolved daemon nodes in the same `--all`/ + multi-name run behaves the same way `fleet deploy` already does per node. +4. `fleetStopCmd`'s `call` is exactly today's `n.Stop(ctx)` closure, needing + no node lookup at all — stopping takes no config, so it has nothing new + to resolve. It changes only in how its target is selected (`runFleetDrive` + instead of `driveOneNode`'s single name), not in what it does to a node. + +**Alternative considered**: fall back to a plain, config-less `Start` when +nothing resolves for a `kind: daemon` node, so a fleet file with no +`file`/alias/subdirectory for it keeps working. Rejected: that fallback +exists only to protect a user of today's `fleet start` who has not adopted +this field, and there is no such user yet — carrying it would mean +permanently maintaining two start paths (config-less and config-driven) for +a distinction that only matters during a migration nobody needs to make. A +`kind: daemon` node without a resolvable source is a fleet-file omission to +fix, the same as an undeployed `kind: remote` node is. + +**Alternative considered**: leave `fleet stop` on `driveOneNode`, single-node +only, while only `start` moves to `runFleetDrive`. This was the original +shape of this decision, on the reasoning that stopping several engines at +once is a different risk than starting them. Superseded per the request for +symmetry — `stop` needs no config resolution, so giving it the same +target-selection surface as `start` costs nothing beyond the shared +`runFleetDrive` plumbing both already need, and a fleet operator does not +have to remember which of the two mutating commands takes `--all`. + +### Command placement + +`fleetDeployCmd` lives in `cmd/spinloop/fleet.go` beside the other fleet +subcommands, calling into `cmd/spinloop/remote.go`'s new +`deriveDeployTarget` / `runDeploy` and the new `resolveNodeSpinloop` helper +(same package, so no export needed); `fleetStartCmd`'s new implementation +calls the same `resolveNodeSpinloop` and `deployConfigForNode`, plus the new +`internal/fleet` `OnlyNames` and the new shared `runFleetDrive`, which +`fleetStopCmd` also calls. No new `internal/fleet` dependency on +`internal/remote`'s deploy internals beyond what `NewNode` already imports. + +## Risks / Trade-offs + +- **Concurrent AWS calls per fleet deploy** → each node deploys a distinct + environment (distinct Lambda invocation, distinct S3/EC2 resources), so + there is no shared mutable state to race on; this mirrors `FanOut` already + running concurrent calls against distinct nodes. +- **Partial success is easy to misread as full success** → both commands + print one outcome line per targeted node (deployed/started, guarded, + failed) and exit non-zero on any failure, the same "row, not a silent + gap" convention `fleet status` and `fleet metrics` already use for + unreachable nodes. `fleet deploy` specifically: real usage surfaced that + several nodes' full plan/result text run together with nothing marking + where one ends and the next begins, and that a `--all` run gives no + feedback while AWS calls are still in flight (task 9) — fixed with a + live per-node spinner while deploying, a coloured ✓/⚠/✗ header per node's + report, and a closing summary line, all skipped for a non-TTY run. +- **`fleet start --all`/`fleet stop --all` act on every node in the fleet at + once, daemon and remote alike** → each start or stop is still gated by + the daemon's or the control plane's own rules (an already-running engine + reports its conflict, per node; a stop is idempotent), and a `kind: + remote` wake or stop is the same call `spinloop remote start`/`stop` + already makes for one environment; `--all` costs no more than running the + command against each node in turn, just concurrently and in one command. +- **A node's resolved Spinloop file drifts from its fleet-file entry + unnoticed** → out of scope here; `fleet deploy`'s job is to run the deploy + that file describes, not to detect drift. `spinloop fleet route` already + gives an operator a way to check what a node is actually serving. +- **Three fallback tiers make it non-obvious which Spinloop file a node will + actually use** → both `fleet deploy` and `fleet start` state the resolved + source (the path used, or the alias name) before acting, the same way + `remote deploy` already announces "Using alias …"; nothing happens + silently from an unexpected source. +- **`fleet start` on a `kind: daemon` node with no resolvable source now + fails instead of starting** → deliberate (see the "requires a daemon + node's resolved source" decision); every fleet file with a `kind: daemon` + node needs a `file` field, a matching alias, or a matching subdirectory + before this ships, including the example fleets (task 7.3). +- **An alias or subdirectory coincidentally named after a node resolves to + the wrong Spinloop** → mitigated by always announcing the resolved source + before acting (previous bullet); an operator who wants a specific source + can always pin it with an explicit `file` field, which wins over both + fallbacks. + +## Migration Plan + +A new field and a new subcommand, plus a breaking change to `fleet start` +for any `kind: daemon` node with no resolvable Spinloop source (see +proposal.md, marked **BREAKING**) — every fleet file needs a `file` field, +alias, or subdirectory added for each `kind: daemon` node it lists, +including this repo's own example fleets (task 7.3). `remote deploy` itself +is unchanged. No data migration, no flag renames. diff --git a/openspec/changes/archive/2026-09-03-fleet-deploy/proposal.md b/openspec/changes/archive/2026-09-03-fleet-deploy/proposal.md new file mode 100644 index 00000000..887dc522 --- /dev/null +++ b/openspec/changes/archive/2026-09-03-fleet-deploy/proposal.md @@ -0,0 +1,110 @@ +## Why + +A fleet-file node has no declared link to the Spinloop file that says what it +runs. For a `kind: remote` node this means its AWS environment can only be +brought into existence outside `fleet.yaml` entirely, one environment at a +time, via `spinloop remote deploy ` run by hand. For a `kind: +daemon` node it means `spinloop fleet start ` can only start whatever +the daemon already happens to be configured to run — it has no way to say +"start this node serving what this Spinloop names," the way a routed launch +already can via `Config.Wake`/`StartWith` for the Spinloop it happens to be +launching. Standing up a multi-node remote fleet today means deploying each +environment separately and then, separately again, listing them in +`fleet.yaml`; naming what a daemon node should run means editing that +node's own local configuration rather than the fleet file. + +## What Changes + +- Add an optional `file:` field to a fleet-file node entry (either kind), + naming the Spinloop file that describes what the node runs. The path + resolves relative to the fleet file, the way other Spinloop-relative paths + already resolve. It is optional because a node's `name` already doubles as + a lookup key, resolved in order when `file` is absent: + 1. the node's own `name` resolved through the existing `spinloop alias` + registry, exactly as a bare argument to `spinloop remote deploy ` + already resolves today; + 2. a subdirectory named after the node, beside the fleet file (e.g. + `dev-1/Spinloop` beside a `fleet.yaml` naming node `dev-1`) — the same + "a name is also a directory to look in" convention a bare `spinloop + apply ` already follows for a local Spinloop. + + A node registered with `spinloop alias add `, or simply + laid out as `/Spinloop` beside the fleet file, therefore needs + no `file` field at all. +- Add `spinloop fleet deploy ` (or `--all`): deploys the AWS + environment for each named `kind: remote` node, or every `kind: remote` + node with `--all`, reusing the same derivation, consent, and registration + behavior as `spinloop remote deploy` — one node's deploy config comes from + its resolved Spinloop source. Naming no node and passing no `--all` fails, + listing the fleet's `kind: remote` nodes, rather than silently deploying + the whole fleet — the same "an explicit target is required" rule + `start`/`stop` already enforce for mutating fleet commands. Naming a + `kind: daemon` node fails, explaining that `fleet deploy` provisions cloud + environments and that node is not one; `--all` only ever selects `kind: + remote` nodes, so a daemon node is never swept in by it. +- Node deploys run independently and concurrently; one node's failure or a + registered/live guard on it is reported against that node and does not + stop the others. `--dry-run` and `--overwrite` carry the same meaning as + on `spinloop remote deploy`, applied per node. +- `spinloop fleet start ` and `spinloop fleet stop ` now + take one or more node names, or `--all` for every node in the file — the + same target-selection rule `fleet deploy` uses (no target is an error; + `--all` plus names is ambiguous; an unknown name fails before anything + starts or stops). Neither is restricted to one kind: a `kind: remote` name + is as valid a target as a `kind: daemon` one, for either command. Targeted + nodes are driven concurrently and independently — one node's failure is + reported against it alone and does not stop the others. +- **BREAKING**: on a `kind: daemon` node, `fleet start` now requires that + node's Spinloop source to resolve, the same way `fleet deploy` requires + one for a `kind: remote` node. The client derives a deploy config from it + (`deployConfigForNode`, the same derivation a routed wake already uses) + and starts the node's engine with it via `StartWith`, exactly as a routed + launch wakes a node — telling the daemon what to run rather than trusting + it already knows. A `kind: daemon` node with no resolvable source fails + for that node alone, naming the three ways one could have been given, + rather than falling back to a plain, config-less start. Every fleet file + with a `kind: daemon` node needs a `file` field, a matching alias, or a + matching subdirectory added before `fleet start` works on it again. A + `kind: remote` node's start is unaffected — what it serves is fixed at + deploy time, and its `StartWith` already refuses a deploy config for that + reason. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `fleet-config`: a fleet-file node (either kind) gains an optional `file` + path field naming the Spinloop file that describes what it runs, falling + back to resolving the node's own name as a registered alias, then to a + `/Spinloop` subdirectory beside the fleet file, when absent. +- `fleet-client`: add the `spinloop fleet deploy` command (node selection, + per-node deploy behavior, concurrency, reporting); modify `spinloop fleet + start` and `spinloop fleet stop` to both take multiple node names or + `--all`; and modify `start` to require and use a `kind: daemon` node's + resolved Spinloop source (**BREAKING** for a node with none — `stop` needs + no such source and is unaffected by that part). + +## Impact + +- `internal/fleet/config.go`: `NodeConfig` gains a `File` field + (`yaml:"file"`), resolved relative to the fleet file's directory when set; + `Config` gains `OnlyNames([]string)`, narrowing to several named nodes the + way `Only` already narrows to one (`Only` becomes a one-name call to it). +- `cmd/spinloop/fleet.go`: new `fleetDeployCmd`; `fleetStartCmd` and + `fleetStopCmd` are rewritten on a shared node-selection helper + (`OnlyNames`/`FanOut`, multiple names or `--all`) replacing `driveOneNode`, + which is deleted; `fleetStartCmd`'s call additionally requires and uses + the resolved source for daemon nodes via `StartWith`. All reuse + `readSpinloop`'s alias-then-path resolution, `deployConfigFor`/ + `deployConfigForNode`, `applySpinloopEnv`, and (for deploy) the + registration/consent logic factored out of `runRemoteDeploy` in + `cmd/spinloop/remote.go`. +- `docs/commands/fleet.md` and `docs/commands/remote.md`: document the new + field, its fallbacks, the new command, and `start`'s new requirement. +- `examples/fleet-remote/`, `examples/fleet-local/`, `examples/fleet-docker/`, + `examples/fleet-mixed/`: every example with a `kind: daemon` node needs a + `file` field, alias, or subdirectory added, or `fleet start` breaks for it. diff --git a/openspec/changes/archive/2026-09-03-fleet-deploy/specs/fleet-client/spec.md b/openspec/changes/archive/2026-09-03-fleet-deploy/specs/fleet-client/spec.md new file mode 100644 index 00000000..626b9853 --- /dev/null +++ b/openspec/changes/archive/2026-09-03-fleet-deploy/specs/fleet-client/spec.md @@ -0,0 +1,285 @@ +## ADDED Requirements + +### Requirement: Fleet deploy targets remote nodes + +`spinloop fleet deploy ` SHALL deploy the AWS environment for one or +more `kind: remote` nodes in the fleet file, named explicitly. +`spinloop fleet deploy --all` SHALL target every `kind: remote` node in the +file instead. Invoked with neither a node name nor `--all`, it SHALL fail, +listing the fleet's `kind: remote` nodes, and deploy nothing — mutating +however many cloud environments a fleet file lists SHALL NOT happen by +default. `--all` combined with one or more node names SHALL fail as +ambiguous. An unknown node name SHALL fail the command, naming the known +nodes, without deploying anything. A named `kind: daemon` node SHALL fail +the command, explaining that `fleet deploy` provisions cloud environments +and that node is not one; `--all` SHALL only ever select `kind: remote` +nodes, so a `kind: daemon` node is never targeted by it and is not reported +at all. + +#### Scenario: Deploy every remote node + +- **WHEN** `spinloop fleet deploy --all` runs against a file mixing `kind: + remote` and `kind: daemon` nodes +- **THEN** every `kind: remote` node is deployed and no `kind: daemon` node + is touched or mentioned + +#### Scenario: Deploy named nodes + +- **WHEN** `spinloop fleet deploy gpu-a gpu-b` runs and both are `kind: + remote` nodes in the file +- **THEN** only those two are deployed, whatever else the file lists + +#### Scenario: No target is an error + +- **WHEN** `spinloop fleet deploy` runs with no node arguments and no `--all` +- **THEN** it fails, listing the fleet's `kind: remote` nodes, and deploys + nothing + +#### Scenario: Combining --all with node names is an error + +- **WHEN** `spinloop fleet deploy --all gpu-a` runs +- **THEN** it fails as ambiguous and deploys nothing + +#### Scenario: An unknown node name fails the command + +- **WHEN** `spinloop fleet deploy nope` runs and no node is named `nope` +- **THEN** the command fails, naming the known nodes, and deploys nothing + +#### Scenario: Naming a daemon node explicitly fails + +- **WHEN** `spinloop fleet deploy studio` runs and `studio` is a `kind: + daemon` node +- **THEN** the command fails, explaining that `fleet deploy` provisions cloud + environments and `studio` is not one + +### Requirement: Fleet deploy derives and applies each node's config + +Each targeted node SHALL be deployed from the Spinloop file its deploy +source resolves to (see fleet-config's "Node Spinloop source" and "...falls +back to name-based lookup" requirements: its `file` field, else an alias +registered under its name, else a `/` subdirectory beside the fleet +file), deriving the deploy config and registering the resulting environment +exactly as `spinloop remote deploy ` does for that same file — the two +SHALL NOT be able to disagree about what a given Spinloop file deploys. A +targeted node for which no source resolves SHALL fail for that node alone, +naming all three ways one could have been given, without touching the other +targeted nodes. The resolved source (the path used, or the alias name when +one was used) SHALL be reported alongside that node's plan, so which of the +three supplied it is never left to be inferred. + +Nodes SHALL be deployed independently: one node already registered or live +SHALL require `--overwrite` for that node exactly as a standalone `remote +deploy` does, and refusing it SHALL NOT stop the other targeted nodes from +deploying. A node whose deploy fails for any other reason SHALL likewise be +reported against that node without aborting the rest. The command SHALL exit +non-zero when any targeted node failed to deploy, having still attempted +every other targeted node. + +`--dry-run` SHALL print the plan for every targeted node without deploying +any of them, exactly as a standalone `remote deploy --dry-run` does for one. +`--overwrite` SHALL apply to every targeted node that needs it. + +#### Scenario: A node deploys from its own Spinloop file + +- **WHEN** `fleet deploy` targets a node declaring `file: + ./envs/gpu.Spinloop` +- **THEN** that node's environment is created and registered from that file, + the same as `spinloop remote deploy ./envs/gpu.Spinloop` would produce, and + the resolved path is reported against that node + +#### Scenario: A node with no resolvable source fails only that node + +- **WHEN** `fleet deploy` targets two remote nodes and one declares no `file` + field, has no alias registered under its name, and has no same-named + subdirectory beside the fleet file +- **THEN** the other node still deploys, and the command reports against the + unresolved node that none of the `file` field, a matching alias, or a + matching subdirectory was found + +#### Scenario: One node's guard does not block the others + +- **WHEN** `fleet deploy` targets two remote nodes and one is already + registered while the other is not, and `--overwrite` is not given +- **THEN** the unregistered node deploys, the registered node is refused with + the same message a standalone `remote deploy` gives, and the command exits + non-zero + +#### Scenario: Dry run previews every targeted node + +- **WHEN** `spinloop fleet deploy --dry-run --all` runs +- **THEN** the plan for every `kind: remote` node in the file is printed and + no environment is created or registered + +### Requirement: Fleet deploy reports progress and results legibly + +`fleet deploy` SHALL indicate that a targeted node's deploy is still in +progress for as long as it is running, on an output capable of an in-place +update, so a run against several nodes — which can take AWS-call minutes per +node — is never silently unresponsive. Each targeted node's final report +SHALL be clearly delimited from every other targeted node's, identifying +which node it describes, so one node's report cannot be mistaken for +bleeding into the next. The command SHALL close with a summary stating how +many of the targeted nodes succeeded. + +On an output that is not an interactive terminal (piped, redirected, or +otherwise non-interactive), the command SHALL NOT emit an in-place progress +indicator or other terminal-control escape sequences — a downstream consumer +of that output (a log file, a script, CI) SHALL see only the per-node +reports and the summary, in the order the nodes were targeted. + +#### Scenario: Progress is shown while nodes are still deploying + +- **WHEN** `fleet deploy --all` targets several nodes on an interactive + terminal, and their deploys are still in progress +- **THEN** each still-deploying node is shown as in progress until it + finishes + +#### Scenario: Node reports are clearly separated + +- **WHEN** `fleet deploy` targets two or more nodes +- **THEN** each node's report is headed by something identifying that node, + so the boundary between one node's report and the next is unambiguous + +#### Scenario: A summary closes the report + +- **WHEN** `fleet deploy` finishes against several targeted nodes +- **THEN** the command's output ends with a line stating how many of the + targeted nodes deployed successfully + +#### Scenario: Piped output carries no escape sequences + +- **WHEN** `fleet deploy`'s output is piped or redirected rather than an + interactive terminal +- **THEN** the output contains no in-place progress indicator and no + terminal-control escape sequences, only the per-node reports and the + summary + +## MODIFIED Requirements + +### Requirement: Driving one node + +`spinloop fleet start ` SHALL call each named node's daemon start +endpoint (or push a resolved deploy config, for a `kind: daemon` node — see +below); `spinloop fleet stop ` SHALL call each named node's daemon +stop endpoint. `spinloop fleet start --all`/`spinloop fleet stop --all` +SHALL target every node in the file instead, of either kind. Either command +invoked with neither a node name nor `--all` SHALL fail and list the +available nodes, rather than acting on the whole fleet by default. `--all` +combined with one or more node names SHALL fail as ambiguous, for either +command. An unknown node name SHALL fail the command, naming the known +nodes, before anything is started or stopped. The daemon's own rules still +hold — a start while that node's engine is running is reported as the +daemon's conflict for that node, and a stop is idempotent. Multiple targeted +nodes SHALL be driven independently, for either command: one node's failure +(including, for start, an unresolved Spinloop source, see below) SHALL be +reported against that node alone and SHALL NOT stop the others; the command +SHALL exit non-zero when any targeted node failed. + +For a `kind: daemon` node, `fleet start` SHALL first resolve that node's +Spinloop source (see fleet-config's "Node Spinloop source" and "...falls +back to name-based lookup" requirements), and SHALL fail that node's start, +naming all three ways a source could have been given, when none resolves. +When one resolves, the client SHALL derive a deploy config from it — the +same node-owned derivation a routed wake already uses +(`deployConfigForNode`) — report the resolved source and derived config +alongside the node's name, and start the node's engine with that config +(`StartWith`) rather than a plain start, exactly as a routed wake tells a +node what to serve. A `kind: remote` node's start is unaffected regardless +of whether a source resolves for it: what it serves is fixed at deploy time, +not pushed at start time, so it always uses a plain start. + +#### Scenario: Start a named node + +- **WHEN** `spinloop fleet start gpu-box` runs, that node is idle, and its + Spinloop source resolves +- **THEN** the client derives a deploy config from the resolved source and + calls that node's daemon start endpoint with it, reporting the resulting + state + +#### Scenario: Start several named nodes + +- **WHEN** `spinloop fleet start gpu-a gpu-b` runs and both nodes' Spinloop + sources resolve +- **THEN** both nodes start, independently, whatever else the file lists + +#### Scenario: Start every node + +- **WHEN** `spinloop fleet start --all` runs against a file mixing `kind: + remote` and `kind: daemon` nodes, and every `kind: daemon` node's Spinloop + source resolves +- **THEN** every node in the file starts — the daemon nodes with their + resolved config, the remote nodes with a plain start + +#### Scenario: Start with no node names the fleet + +- **WHEN** `spinloop fleet start` runs with no node argument and no `--all` +- **THEN** it fails, listing the nodes, and starts nothing + +#### Scenario: Combining --all with node names is an error + +- **WHEN** `spinloop fleet start --all gpu-a` or `spinloop fleet stop --all + gpu-a` runs +- **THEN** it fails as ambiguous and neither starts nor stops anything + +#### Scenario: Unknown node + +- **WHEN** `spinloop fleet stop nope` runs and no node is named `nope` +- **THEN** it fails, naming the known nodes, and stops nothing + +#### Scenario: An unknown name among several fails before starting any + +- **WHEN** `spinloop fleet start gpu-a nope` runs and no node is named `nope` +- **THEN** the command fails, naming the known nodes, and starts neither node + +#### Scenario: Stop several named nodes + +- **WHEN** `spinloop fleet stop gpu-a gpu-b` runs +- **THEN** both nodes are stopped, independently, whatever else the file + lists + +#### Scenario: Stop every node + +- **WHEN** `spinloop fleet stop --all` runs against a file mixing `kind: + remote` and `kind: daemon` nodes +- **THEN** every node in the file is stopped + +#### Scenario: Stop with no node names the fleet + +- **WHEN** `spinloop fleet stop` runs with no node argument and no `--all` +- **THEN** it fails, listing the nodes, and stops nothing + +#### Scenario: An unknown name among several fails before stopping any + +- **WHEN** `spinloop fleet stop gpu-a nope` runs and no node is named `nope` +- **THEN** the command fails, naming the known nodes, and stops neither node + +#### Scenario: Starting a daemon node with a resolved source pushes it + +- **WHEN** `spinloop fleet start dev-1` runs, `dev-1` is a `kind: daemon` + node, and its Spinloop source resolves (by `file`, alias, or subdirectory) +- **THEN** the client derives a deploy config from the resolved Spinloop, + reports the resolved source, and starts `dev-1`'s engine with that config + +#### Scenario: Starting a daemon node with no resolvable source fails + +- **WHEN** `spinloop fleet start studio` runs, `studio` is a `kind: daemon` + node, and no `file` field, alias, or subdirectory resolves for it +- **THEN** the command fails for `studio`, naming the `file` field, the + alias registry, and the subdirectory convention as the three ways a source + could have been given, and nothing is started + +#### Scenario: One unresolved node among several fails only that node + +- **WHEN** `spinloop fleet start --all` runs, and one `kind: daemon` node in + the file has no resolvable Spinloop source while the rest do +- **THEN** every other targeted node starts, the unresolved node is reported + as failed naming the three ways a source could have been given, and the + command exits non-zero + +#### Scenario: Starting a remote node is unaffected by a resolved source + +- **WHEN** `spinloop fleet start gpu-env` runs, `gpu-env` is a `kind: remote` + node, and a Spinloop source resolves for it +- **THEN** the client starts it with a plain start; the resolved source is + not used, since a `kind: remote` node's `StartWith` always refuses a + deploy config diff --git a/openspec/changes/archive/2026-09-03-fleet-deploy/specs/fleet-config/spec.md b/openspec/changes/archive/2026-09-03-fleet-deploy/specs/fleet-config/spec.md new file mode 100644 index 00000000..f0b71d40 --- /dev/null +++ b/openspec/changes/archive/2026-09-03-fleet-deploy/specs/fleet-config/spec.md @@ -0,0 +1,92 @@ +## ADDED Requirements + +### Requirement: Node Spinloop source + +A fleet-file node, of either kind, MAY declare a `file` field naming the +Spinloop file that describes what it runs — the same file `spinloop fleet +deploy` reads to create a `kind: remote` node's environment, and the same +file `spinloop fleet start` reads to tell a `kind: daemon` node's engine what +to run. The path SHALL resolve relative to the fleet file's directory, the +same way other Spinloop-relative paths in the project resolve. The field +SHALL NOT be required to parse a fleet file — every fleet command other than +`deploy` and `start` is unaffected by it — but `deploy` and `start` each +SHALL require it (directly or via the fallbacks below) for the nodes they +act on; see fleet-client's "Driving one node" requirement. + +#### Scenario: A remote node names its Spinloop file + +- **WHEN** a `kind: remote` node declares `file: ./envs/gpu.Spinloop` +- **THEN** `spinloop fleet deploy` for that node reads the Spinloop at that + path, resolved relative to the fleet file's directory, to derive what to + deploy + +#### Scenario: A daemon node names its Spinloop file + +- **WHEN** a `kind: daemon` node declares `file: ./envs/gpu.Spinloop` +- **THEN** `spinloop fleet start` for that node reads the Spinloop at that + path, resolved relative to the fleet file's directory, to derive what to + start it with + +#### Scenario: The field is inert outside deploy and start + +- **WHEN** a node declares a `file` field +- **THEN** `fleet status`, `metrics`, `stop`, `route`, and `dashboard` behave + exactly as they do without it + +### Requirement: Node Spinloop source falls back to name-based lookup + +A node declaring no `file` field SHALL have its Spinloop source resolved +from its own `name`, tried in order: + +1. `name` resolved as a registered `spinloop alias` — the same lookup a bare + argument to `spinloop remote deploy ` already performs. +2. Failing that, a subdirectory named `` beside the fleet file, + containing a Spinloop file — the same directory-to-default-file + resolution an ordinary Spinloop path argument already gets when it names + a directory. + +A node for which neither resolves SHALL fail the command acting on it — +`fleet deploy` for a `kind: remote` node, `fleet start` for a `kind: daemon` +node — for that node alone, naming all three ways a source could have been +given: the `file` field, a `spinloop alias` named after the node, or a +`/` subdirectory beside the fleet file. + +#### Scenario: Resolved through a registered alias + +- **WHEN** a node named `gpu-env` declares no `file` field, and `spinloop + alias` has `gpu-env` registered to a Spinloop path +- **THEN** `fleet deploy` (if `gpu-env` is `kind: remote`) or `fleet start` + (if `kind: daemon`) reads the Spinloop the alias names + +#### Scenario: Resolved through a named subdirectory + +- **WHEN** a node named `dev-1` declares no `file` field, no alias named + `dev-1` is registered, and a `dev-1/` directory containing a Spinloop file + sits beside the fleet file +- **THEN** `fleet deploy` (if `dev-1` is `kind: remote`) or `fleet start` (if + `kind: daemon`) reads the Spinloop from that subdirectory + +#### Scenario: An alias wins over a same-named subdirectory + +- **WHEN** a node named `dev-1` declares no `file` field, an alias named + `dev-1` is registered, and a `dev-1/` subdirectory containing a Spinloop + file also sits beside the fleet file +- **THEN** the alias is used, not the subdirectory + +#### Scenario: None of the three resolve for a remote node + +- **WHEN** a `kind: remote` node declares no `file` field, no alias is + registered under its name, and no same-named subdirectory sits beside the + fleet file +- **THEN** `fleet deploy` fails for that node, naming the `file` field, the + alias registry, and the subdirectory convention as the three ways a source + could have been given + +#### Scenario: None of the three resolve for a daemon node + +- **WHEN** a `kind: daemon` node declares no `file` field, no alias is + registered under its name, and no same-named subdirectory sits beside the + fleet file +- **THEN** `fleet start` fails for that node, naming the `file` field, the + alias registry, and the subdirectory convention as the three ways a source + could have been given diff --git a/openspec/changes/archive/2026-09-03-fleet-deploy/tasks.md b/openspec/changes/archive/2026-09-03-fleet-deploy/tasks.md new file mode 100644 index 00000000..86963776 --- /dev/null +++ b/openspec/changes/archive/2026-09-03-fleet-deploy/tasks.md @@ -0,0 +1,249 @@ +## 1. Fleet file: `file` field on any node + +- [x] 1.1 Add `File string \`yaml:"file"\`` to `NodeConfig` in + `internal/fleet/config.go`, available on either `kind`, resolved + relative to `Config.Dir` when read (a helper alongside the existing + path handling, not at parse time — no fleet command other than + `deploy`/`start` needs it, and `start` must not require it). +- [x] 1.2 Confirm `validate()` does not require the field for any kind. +- [x] 1.3 Unit tests in `internal/fleet/config_test.go`: a `file` path + resolves relative to the fleet file's directory on either node kind; a + node without it parses fine (only `deploy`/`start` should care). + +## 2. Extract the reusable deploy body from `remote deploy` + +- [x] 2.1 In `cmd/spinloop/remote.go`, split `runRemoteDeploy` into + `deriveDeployTarget(spinloopArg string) (spinloop.Selection, string, + remote.DeployConfig, env string, error)` (the existing `readSpinloop` + alias-or-path resolution + Spinloop env application + `deployConfigFor` + + `REMOTE` name resolution + `--allowed-cidr`/`--spinloop-version` + validation) and `runDeploy(env string, dc remote.DeployConfig, opts + deployOpts) (deployOutcome, error)` (plan print through registration). +- [x] 2.2 Define `deployOpts` (dryRun, overwrite, reseed, allowedCidr, + region) and `deployOutcome` (what to print, or a guard/failure reason) + so a caller can render one node's result without interleaving raw + stdout writes from concurrent goroutines. +- [x] 2.3 Rewire `runRemoteDeploy` to call the two new functions and print + `deployOutcome` exactly as it prints today — no behavior change for + `spinloop remote deploy`. +- [x] 2.4 Run the existing `cmd/spinloop/remote_deploy_test.go` suite + unchanged and confirm it still passes against the refactor. + +## 3. Resolving a node's Spinloop source + +- [x] 3.1 Add `resolveNodeSpinloop(node fleet.NodeConfig, fleetDir string) + (arg string, source string, err error)` that tries, in order: (a) + `node.File` resolved relative to `fleetDir`; (b) an alias registered + under `node.Name` (`config.Load().Alias(node.Name)` — the same lookup + `resolveAlias` makes, checked here only to decide whether to fall + through, not to pre-resolve the path); (c) `filepath.Join(fleetDir, + node.Name)` when that path exists as a directory. Returns the argument + to hand `deriveDeployTarget` and a label for what resolved it (for + reporting), or an error naming all three when none resolve. +- [x] 3.2 Unit tests: each tier resolves independently; the alias tier wins + over a same-named subdirectory when both exist; the error names all + three when none resolve. + +## 4. `spinloop fleet deploy` command + +- [x] 4.1 Add `fleetDeployCmd` in `cmd/spinloop/fleet.go`: `Use: "deploy"`, + requires at least one node arg or `--all` (mutually exclusive), flags + `--fleet`, `--all`, `--dry-run`/`-n`, `--overwrite`, `--reseed`, + `--allowed-cidr`, `--region`, `--spinloop-version` (same deploy flags + and help text as `remote deploy`). +- [x] 4.2 Implement node selection: no node args and no `--all` → fail, + listing the fleet's `kind: remote` nodes; `--all` → every `kind: + remote` node in file order, `kind: daemon` nodes never selected and + never mentioned; named args → exactly those, failing before any + deploy runs if a name is unknown or names a `kind: daemon` node; + `--all` plus named args → fail as ambiguous. +- [x] 4.3 For each targeted node, call `resolveNodeSpinloop` (task 3.1); a + node for which nothing resolves yields a per-node failure rather than + aborting the others. +- [x] 4.4 Run `deriveDeployTarget` + `runDeploy` per targeted node + concurrently (bounded, e.g. `errgroup` or a simple worker loop keyed + by node name — see design.md's "`fleet deploy` requires an explicit + target"), reporting the resolved source (path or alias name) + alongside each node's plan. +- [x] 4.5 Render one line per targeted node (deployed / guarded / failed), + and a summary; exit non-zero if any targeted node failed or was + guarded without `--overwrite`. +- [x] 4.6 Register the command in the fleet command tree and shell + completion (`compRegister(c, "fleet", compFiles)`, node-name + completion for positional args as `start`/`stop` already do). + +## 5. `spinloop fleet start`/`stop` take multiple nodes or `--all` + +- [x] 5.1 Add `func (c *Config) OnlyNames(names []string) (*Config, error)` + to `internal/fleet/config.go`, narrowing to several named nodes in the + order given (unknown name fails immediately, naming the known nodes). + Reimplement `Only(name string)` as `OnlyNames([]string{name})`; confirm + its existing callers (`cmd/spinloop/fleet_logs.go`, + `internal/fleet/select.go`'s `--node` pin) and tests + (`internal/fleet/logs_test.go`) are unaffected. +- [x] 5.2 Add a shared `runFleetDrive(cfg *fleet.Config, all bool, names + []string, call fleet.Call) ([]fleet.NodeResult, error)` in + `cmd/spinloop/fleet.go`, replacing `driveOneNode` (deleted): no names + and no `--all` fails, listing the fleet's nodes; `--all` plus names + fails as ambiguous; `--all` runs `cfg.FanOut(ctx, call)`; named nodes + run `cfg.OnlyNames(names)` then `.FanOut(ctx, call)` (an unknown name + fails before anything is touched). +- [x] 5.3 Rewrite `fleetStartCmd` and `fleetStopCmd` in `cmd/spinloop/fleet.go` + on `runFleetDrive`: `Args: cobra.ArbitraryArgs`, add `--all` to both. + `fleetStartCmd`'s `call` closes over `cfg`, looks up `entry, _ := + cfg.Node(n.Name())` to recover the targeted node's `NodeConfig`; for a + `kind: daemon` entry, calls `resolveNodeSpinloop` (on failure, returns + a failed `NodeResult` naming all three ways a source could have been + given — no fallback to a plain start; on success, `readSpinloop` + + `applySpinloopEnv` + `deployConfigForNode` to derive a `dc`, then + `n.StartWith(ctx, &dc, engineKey)`, reporting the resolved source and + derived config); for `kind: remote`, always plain `n.Start(ctx)`. + `fleetStopCmd`'s `call` is unchanged from today — `n.Stop(ctx)` — it + needs no node lookup, only the new target-selection wrapper. +- [x] 5.4 Render one line per targeted node (started/stopped, guarded, + failed) through a shared renderer both commands call; exit non-zero + if any targeted node failed. +- [x] 5.5 Confirm `remoteNode.StartWith`'s existing refusal + (`internal/fleet/remote_node.go:77-83`) means a `kind: remote` node + is never sent a resolved config by `fleet start` — resolution is only + ever attempted for `kind: daemon` entries. + +## 6. Tests + +- [x] 6.1 `cmd/spinloop/fleet_test.go` (or a new `fleet_deploy_test.go`): + `--all` deploys every remote node and never mentions daemon nodes; + named args narrow the set; no target (no args, no `--all`) fails + listing remote nodes; `--all` plus named args fails as ambiguous; an + unknown name fails before deploying; naming a daemon node explicitly + fails. +- [x] 6.2 A node with no `file` field, no matching alias, and no matching + subdirectory fails only that node in `fleet deploy`; the rest still + deploy. +- [x] 6.3 A node resolved via alias and a node resolved via subdirectory + both deploy correctly in the same run; a node with both an alias and a + same-named subdirectory uses the alias. +- [x] 6.4 One node already registered/live is guarded without `--overwrite` + while a sibling node still deploys; the command exits non-zero. +- [x] 6.5 `--dry-run` prints every targeted node's plan and performs no AWS + calls (assert via the existing seams: `deployDiscoverFn`, + `remoteDeployFn`, etc. left uncalled). +- [x] 6.6 A node deployed via `fleet deploy` and the same Spinloop file + deployed via standalone `remote deploy` produce identical + `remote.DeployConfig` and registration output (parity test using + `deriveDeployTarget` directly). +- [x] 6.7 `fleet start` on a `kind: daemon` node with a resolved `file` + field, a resolved alias, and a resolved subdirectory each derive and + push the expected `StartWith` config; report includes the resolved + source. +- [x] 6.8 `fleet start` on a `kind: daemon` node with no resolvable source + fails, naming all three ways a source could have been given — assert + `Start` and `StartWith` are both never invoked. +- [x] 6.9 `fleet start` on a `kind: remote` node with a resolvable source + still calls plain `Start`, never `StartWith`. +- [x] 6.10 `internal/fleet/config_test.go`: `OnlyNames` narrows to several + named nodes in the order given; an unknown name among several fails + immediately, naming the known nodes; `Only`'s existing behavior and + tests (`internal/fleet/logs_test.go`) are unaffected. +- [x] 6.11 `cmd/spinloop/fleet_test.go`: `fleet start gpu-a gpu-b` starts + both (independently — one succeeding while the other fails does not + abort the first); `fleet start --all` starts every node in the file, + daemon and remote alike; `fleet start` with no args and no `--all` + fails listing the nodes; `--all` plus node args fails as ambiguous; an + unknown name among several fails before starting any. +- [x] 6.12 `cmd/spinloop/fleet_test.go`: the same set, mirrored for `fleet + stop` (`stop gpu-a gpu-b`, `stop --all`, no-target failure, `--all` + plus names ambiguous, unknown name among several) — `stop`'s `call` + needs no Spinloop-resolution coverage since it takes no config. +- [x] 6.13 `go test ./... -cover` stays at or above the project's 80% floor. + +## 7. Docs and examples + +- [x] 7.1 `docs/commands/fleet.md`: document the `file` field and the + alias/subdirectory fallbacks (generalized beyond "remote environments" + to any node) in a new "A node's Spinloop source" section, add a + `## Deploying remote nodes` section (command, flags, `--all`/named-arg + requirement, guard/failure reporting), and rewrite the "Starting and + stopping" section: both `start` and `stop` now take one or more node + names or `--all` (no more "one node at a time"), and `start` on a + `kind: daemon` node now needs a resolvable Spinloop source or fails for + it — **BREAKING**, called out as such. Flags table updated with `--all` + and the deploy-only flags. +- [x] 7.2 `docs/commands/remote.md`: cross-reference `fleet deploy` as the + batch alternative to running `remote deploy` once per environment. +- [x] 7.3 Every existing example with a `kind: daemon` node + (`examples/fleet-local/`, `examples/fleet-docker/`, + `examples/fleet-mixed/`, plus the reference-only `examples/fleet/`) now + has a `file` field (or, for `fleet-remote`'s `llama` node, the + subdirectory convention) for each such node — required for `spinloop + fleet start` to keep working, since resolution is now mandatory. + `examples/fleet-remote/` and `examples/fleet-mixed/` each demonstrate + both non-`file` resolution tiers (an explicit `file` on `qwen`, a + `llama/Spinloop` subdirectory for `llama`), so both are deployable via + `fleet deploy --all`. Each affected README updated; `fleet-docker`'s + `run-tests.sh` updated and re-run in full (Docker was available in this + environment) — every assertion passes, including new `test_start_all` + coverage of `--all` with one node (`laptop`, deliberately left + sourceless) failing without blocking the others. Incidentally fixed a + pre-existing doc bug in `examples/fleet/README.md` and + `examples/fleet-mixed/README.md`: `spinloop daemon` takes no Spinloop + path argument (passing one is an error), but both showed one. + +## 8. Validation + +- [x] 8.1 `gofmt -l .` clean. +- [x] 8.2 `go build ./...` and `go vet ./...` clean. +- [x] 8.3 Manually exercise `spinloop fleet deploy --dry-run --all` against + `examples/fleet-remote/` and `examples/fleet-mixed/` — printed plans + match what standalone `remote deploy --dry-run` prints for the same + Spinloop files (verified directly), correctly resolving both the + `file` tier and the subdirectory tier. +- [x] 8.4 Exercised `spinloop fleet start ` end-to-end against + `examples/fleet-docker/` via its real Docker Compose stack (not just a + unit test): the engine starts with the resolved config, and a node + with no resolvable source (`laptop`) fails naming the three ways one + could have been given, rather than starting. + `examples/fleet-local/`'s equivalent needs real `llama-server` and + weights this environment doesn't have, so its `file` field was + structurally verified (same shape as the working `fleet-docker`/ + `fleet-mixed` cases) rather than run end-to-end. +- [x] 8.5 Exercised `spinloop fleet start --all` and `spinloop fleet stop + --all` against `examples/fleet-docker/`'s real stack (three nodes, + mixed resolvable/unresolvable) via the new `test_start_all` — every + resolvable node starts/stops in one command each, the unresolvable one + is reported without blocking the others. + +## 9. `fleet deploy` progress and readability (post-review UX fix) + +Raised against real usage of `fleet deploy --all`: output for each node only +appeared once every node had finished (no feedback during what can be a +multi-minute AWS call), and successive nodes' output ran together with +nothing marking where one ended and the next began. + +- [x] 9.1 `runFleetDeploy` now shows a live per-node status line while + targeted nodes are still deploying — a grey Braille spinner beside a + pending node's name, redrawn in place (cursor-up + clear, matching the + codebase's existing `fleet metrics --watch` redraw convention) roughly + every 120ms — replaced by a coloured mark for that node the moment it + finishes, while the others keep spinning. Gated on + `golang.org/x/term.IsTerminal(os.Stdout.Fd())`, matching the existing + TTY check `fleet dashboard` already uses: a piped or redirected run + (a log file, CI) gets no spinner and no mid-flight escape codes. +- [x] 9.2 Every node's final report is now headed by a coloured mark plus + its name (`✓`/`⚠`/`✗`, green/yellow/red — the same association + `deployRowOK`/`Guarded`/`Failed` already carried) and followed by a + blank line before the next node's block, so one node's output can no + longer be mistaken for bleeding into the next. A final coloured + summary line (`N/M deployed`, or `N/M deployed, K failed or guarded`) + closes the report — legible at a glance for a large `--all` run + without counting rows. +- [x] 9.3 Verified by hand under a real TTY (`script`) against + `examples/fleet-remote/`: `--dry-run` shows the coloured headers and + summary; a real (credential-less, harmless) deploy attempt shows the + spinner genuinely cycling frames for both nodes concurrently before + timing out on AWS's own credential lookup. Verified again with stdout + piped to confirm the spinner is skipped and no escape codes leak into + redirected output. +- [x] 9.4 `go build`/`go vet`/`gofmt` clean; full `go test ./... -cover` + passing (existing tests run non-TTY, so they exercise the no-spinner + path — the coloured headers/summary still show and are covered by the + existing substring assertions). diff --git a/openspec/specs/fleet-client/spec.md b/openspec/specs/fleet-client/spec.md index 9ebe7730..0bacbc0a 100644 --- a/openspec/specs/fleet-client/spec.md +++ b/openspec/specs/fleet-client/spec.md @@ -100,30 +100,286 @@ cleanly on interrupt. ### Requirement: Driving one node -`spinloop fleet start ` and `spinloop fleet stop ` SHALL call the named -node's daemon start and stop endpoints. Start and stop SHALL require a node -name: invoked without one they SHALL fail and list the available nodes, rather -than acting on the whole fleet. An unknown node name SHALL fail, naming the -known nodes. The daemon's own rules still hold — a start while that node's -engine is running is reported as the daemon's conflict, and a stop is -idempotent. +`spinloop fleet start ` SHALL call each named node's daemon start +endpoint (or push a resolved deploy config, for a `kind: daemon` node — see +below); `spinloop fleet stop ` SHALL call each named node's daemon +stop endpoint. `spinloop fleet start --all`/`spinloop fleet stop --all` +SHALL target every node in the file instead, of either kind. Either command +invoked with neither a node name nor `--all` SHALL fail and list the +available nodes, rather than acting on the whole fleet by default. `--all` +combined with one or more node names SHALL fail as ambiguous, for either +command. An unknown node name SHALL fail the command, naming the known +nodes, before anything is started or stopped. The daemon's own rules still +hold — a start while that node's engine is running is reported as the +daemon's conflict for that node, and a stop is idempotent. Multiple targeted +nodes SHALL be driven independently, for either command: one node's failure +(including, for start, an unresolved Spinloop source, see below) SHALL be +reported against that node alone and SHALL NOT stop the others; the command +SHALL exit non-zero when any targeted node failed. + +For a `kind: daemon` node, `fleet start` SHALL first resolve that node's +Spinloop source (see fleet-config's "Node Spinloop source" and "...falls +back to name-based lookup" requirements), and SHALL fail that node's start, +naming all three ways a source could have been given, when none resolves. +When one resolves, the client SHALL derive a deploy config from it — the +same node-owned derivation a routed wake already uses +(`deployConfigForNode`) — report the resolved source and derived config +alongside the node's name, and start the node's engine with that config +(`StartWith`) rather than a plain start, exactly as a routed wake tells a +node what to serve. A `kind: remote` node's start is unaffected regardless +of whether a source resolves for it: what it serves is fixed at deploy time, +not pushed at start time, so it always uses a plain start. #### Scenario: Start a named node -- **WHEN** `spinloop fleet start gpu-box` runs and that node is idle -- **THEN** the client calls that node's daemon start endpoint and reports the - resulting state +- **WHEN** `spinloop fleet start gpu-box` runs, that node is idle, and its + Spinloop source resolves +- **THEN** the client derives a deploy config from the resolved source and + calls that node's daemon start endpoint with it, reporting the resulting + state + +#### Scenario: Start several named nodes + +- **WHEN** `spinloop fleet start gpu-a gpu-b` runs and both nodes' Spinloop + sources resolve +- **THEN** both nodes start, independently, whatever else the file lists + +#### Scenario: Start every node + +- **WHEN** `spinloop fleet start --all` runs against a file mixing `kind: + remote` and `kind: daemon` nodes, and every `kind: daemon` node's Spinloop + source resolves +- **THEN** every node in the file starts — the daemon nodes with their + resolved config, the remote nodes with a plain start #### Scenario: Start with no node names the fleet -- **WHEN** `spinloop fleet start` runs with no node argument +- **WHEN** `spinloop fleet start` runs with no node argument and no `--all` - **THEN** it fails, listing the nodes, and starts nothing +#### Scenario: Combining --all with node names is an error + +- **WHEN** `spinloop fleet start --all gpu-a` or `spinloop fleet stop --all + gpu-a` runs +- **THEN** it fails as ambiguous and neither starts nor stops anything + #### Scenario: Unknown node - **WHEN** `spinloop fleet stop nope` runs and no node is named `nope` - **THEN** it fails, naming the known nodes, and stops nothing +#### Scenario: An unknown name among several fails before starting any + +- **WHEN** `spinloop fleet start gpu-a nope` runs and no node is named `nope` +- **THEN** the command fails, naming the known nodes, and starts neither node + +#### Scenario: Stop several named nodes + +- **WHEN** `spinloop fleet stop gpu-a gpu-b` runs +- **THEN** both nodes are stopped, independently, whatever else the file + lists + +#### Scenario: Stop every node + +- **WHEN** `spinloop fleet stop --all` runs against a file mixing `kind: + remote` and `kind: daemon` nodes +- **THEN** every node in the file is stopped + +#### Scenario: Stop with no node names the fleet + +- **WHEN** `spinloop fleet stop` runs with no node argument and no `--all` +- **THEN** it fails, listing the nodes, and stops nothing + +#### Scenario: An unknown name among several fails before stopping any + +- **WHEN** `spinloop fleet stop gpu-a nope` runs and no node is named `nope` +- **THEN** the command fails, naming the known nodes, and stops neither node + +#### Scenario: Starting a daemon node with a resolved source pushes it + +- **WHEN** `spinloop fleet start dev-1` runs, `dev-1` is a `kind: daemon` + node, and its Spinloop source resolves (by `file`, alias, or subdirectory) +- **THEN** the client derives a deploy config from the resolved Spinloop, + reports the resolved source, and starts `dev-1`'s engine with that config + +#### Scenario: Starting a daemon node with no resolvable source fails + +- **WHEN** `spinloop fleet start studio` runs, `studio` is a `kind: daemon` + node, and no `file` field, alias, or subdirectory resolves for it +- **THEN** the command fails for `studio`, naming the `file` field, the + alias registry, and the subdirectory convention as the three ways a source + could have been given, and nothing is started + +#### Scenario: One unresolved node among several fails only that node + +- **WHEN** `spinloop fleet start --all` runs, and one `kind: daemon` node in + the file has no resolvable Spinloop source while the rest do +- **THEN** every other targeted node starts, the unresolved node is reported + as failed naming the three ways a source could have been given, and the + command exits non-zero + +#### Scenario: Starting a remote node is unaffected by a resolved source + +- **WHEN** `spinloop fleet start gpu-env` runs, `gpu-env` is a `kind: remote` + node, and a Spinloop source resolves for it +- **THEN** the client starts it with a plain start; the resolved source is + not used, since a `kind: remote` node's `StartWith` always refuses a + deploy config + +### Requirement: Fleet deploy targets remote nodes + +`spinloop fleet deploy ` SHALL deploy the AWS environment for one or +more `kind: remote` nodes in the fleet file, named explicitly. +`spinloop fleet deploy --all` SHALL target every `kind: remote` node in the +file instead. Invoked with neither a node name nor `--all`, it SHALL fail, +listing the fleet's `kind: remote` nodes, and deploy nothing — mutating +however many cloud environments a fleet file lists SHALL NOT happen by +default. `--all` combined with one or more node names SHALL fail as +ambiguous. An unknown node name SHALL fail the command, naming the known +nodes, without deploying anything. A named `kind: daemon` node SHALL fail +the command, explaining that `fleet deploy` provisions cloud environments +and that node is not one; `--all` SHALL only ever select `kind: remote` +nodes, so a `kind: daemon` node is never targeted by it and is not reported +at all. + +#### Scenario: Deploy every remote node + +- **WHEN** `spinloop fleet deploy --all` runs against a file mixing `kind: + remote` and `kind: daemon` nodes +- **THEN** every `kind: remote` node is deployed and no `kind: daemon` node + is touched or mentioned + +#### Scenario: Deploy named nodes + +- **WHEN** `spinloop fleet deploy gpu-a gpu-b` runs and both are `kind: + remote` nodes in the file +- **THEN** only those two are deployed, whatever else the file lists + +#### Scenario: No target is an error + +- **WHEN** `spinloop fleet deploy` runs with no node arguments and no `--all` +- **THEN** it fails, listing the fleet's `kind: remote` nodes, and deploys + nothing + +#### Scenario: Combining --all with node names is an error + +- **WHEN** `spinloop fleet deploy --all gpu-a` runs +- **THEN** it fails as ambiguous and deploys nothing + +#### Scenario: An unknown node name fails the command + +- **WHEN** `spinloop fleet deploy nope` runs and no node is named `nope` +- **THEN** the command fails, naming the known nodes, and deploys nothing + +#### Scenario: Naming a daemon node explicitly fails + +- **WHEN** `spinloop fleet deploy studio` runs and `studio` is a `kind: + daemon` node +- **THEN** the command fails, explaining that `fleet deploy` provisions cloud + environments and `studio` is not one + +### Requirement: Fleet deploy derives and applies each node's config + +Each targeted node SHALL be deployed from the Spinloop file its deploy +source resolves to (see fleet-config's "Node Spinloop source" and "...falls +back to name-based lookup" requirements: its `file` field, else an alias +registered under its name, else a `/` subdirectory beside the fleet +file), deriving the deploy config and registering the resulting environment +exactly as `spinloop remote deploy ` does for that same file — the two +SHALL NOT be able to disagree about what a given Spinloop file deploys. A +targeted node for which no source resolves SHALL fail for that node alone, +naming all three ways one could have been given, without touching the other +targeted nodes. The resolved source (the path used, or the alias name when +one was used) SHALL be reported alongside that node's plan, so which of the +three supplied it is never left to be inferred. + +Nodes SHALL be deployed independently: one node already registered or live +SHALL require `--overwrite` for that node exactly as a standalone `remote +deploy` does, and refusing it SHALL NOT stop the other targeted nodes from +deploying. A node whose deploy fails for any other reason SHALL likewise be +reported against that node without aborting the rest. The command SHALL exit +non-zero when any targeted node failed to deploy, having still attempted +every other targeted node. + +`--dry-run` SHALL print the plan for every targeted node without deploying +any of them, exactly as a standalone `remote deploy --dry-run` does for one. +`--overwrite` SHALL apply to every targeted node that needs it. + +#### Scenario: A node deploys from its own Spinloop file + +- **WHEN** `fleet deploy` targets a node declaring `file: + ./envs/gpu.Spinloop` +- **THEN** that node's environment is created and registered from that file, + the same as `spinloop remote deploy ./envs/gpu.Spinloop` would produce, and + the resolved path is reported against that node + +#### Scenario: A node with no resolvable source fails only that node + +- **WHEN** `fleet deploy` targets two remote nodes and one declares no `file` + field, has no alias registered under its name, and has no same-named + subdirectory beside the fleet file +- **THEN** the other node still deploys, and the command reports against the + unresolved node that none of the `file` field, a matching alias, or a + matching subdirectory was found + +#### Scenario: One node's guard does not block the others + +- **WHEN** `fleet deploy` targets two remote nodes and one is already + registered while the other is not, and `--overwrite` is not given +- **THEN** the unregistered node deploys, the registered node is refused with + the same message a standalone `remote deploy` gives, and the command exits + non-zero + +#### Scenario: Dry run previews every targeted node + +- **WHEN** `spinloop fleet deploy --dry-run --all` runs +- **THEN** the plan for every `kind: remote` node in the file is printed and + no environment is created or registered + +### Requirement: Fleet deploy reports progress and results legibly + +`fleet deploy` SHALL indicate that a targeted node's deploy is still in +progress for as long as it is running, on an output capable of an in-place +update, so a run against several nodes — which can take AWS-call minutes per +node — is never silently unresponsive. Each targeted node's final report +SHALL be clearly delimited from every other targeted node's, identifying +which node it describes, so one node's report cannot be mistaken for +bleeding into the next. The command SHALL close with a summary stating how +many of the targeted nodes succeeded. + +On an output that is not an interactive terminal (piped, redirected, or +otherwise non-interactive), the command SHALL NOT emit an in-place progress +indicator or other terminal-control escape sequences — a downstream consumer +of that output (a log file, a script, CI) SHALL see only the per-node +reports and the summary, in the order the nodes were targeted. + +#### Scenario: Progress is shown while nodes are still deploying + +- **WHEN** `fleet deploy --all` targets several nodes on an interactive + terminal, and their deploys are still in progress +- **THEN** each still-deploying node is shown as in progress until it + finishes + +#### Scenario: Node reports are clearly separated + +- **WHEN** `fleet deploy` targets two or more nodes +- **THEN** each node's report is headed by something identifying that node, + so the boundary between one node's report and the next is unambiguous + +#### Scenario: A summary closes the report + +- **WHEN** `fleet deploy` finishes against several targeted nodes +- **THEN** the command's output ends with a line stating how many of the + targeted nodes deployed successfully + +#### Scenario: Piped output carries no escape sequences + +- **WHEN** `fleet deploy`'s output is piped or redirected rather than an + interactive terminal +- **THEN** the output contains no in-place progress indicator and no + terminal-control escape sequences, only the per-node reports and the + summary + ### Requirement: Authenticated fan-out Every request the client makes to a node SHALL carry that node's resolved diff --git a/openspec/specs/fleet-config/spec.md b/openspec/specs/fleet-config/spec.md index ea445178..c12551ea 100644 --- a/openspec/specs/fleet-config/spec.md +++ b/openspec/specs/fleet-config/spec.md @@ -213,3 +213,93 @@ naming the variable, in the same way a missing engine-token variable is. `engineTokenEnv` - **THEN** the daemon node is started ungated, as it is today +### Requirement: Node Spinloop source + +A fleet-file node, of either kind, MAY declare a `file` field naming the +Spinloop file that describes what it runs — the same file `spinloop fleet +deploy` reads to create a `kind: remote` node's environment, and the same +file `spinloop fleet start` reads to tell a `kind: daemon` node's engine what +to run. The path SHALL resolve relative to the fleet file's directory, the +same way other Spinloop-relative paths in the project resolve. The field +SHALL NOT be required to parse a fleet file — every fleet command other than +`deploy` and `start` is unaffected by it — but `deploy` and `start` each +SHALL require it (directly or via the fallbacks below) for the nodes they +act on; see fleet-client's "Driving one node" requirement. + +#### Scenario: A remote node names its Spinloop file + +- **WHEN** a `kind: remote` node declares `file: ./envs/gpu.Spinloop` +- **THEN** `spinloop fleet deploy` for that node reads the Spinloop at that + path, resolved relative to the fleet file's directory, to derive what to + deploy + +#### Scenario: A daemon node names its Spinloop file + +- **WHEN** a `kind: daemon` node declares `file: ./envs/gpu.Spinloop` +- **THEN** `spinloop fleet start` for that node reads the Spinloop at that + path, resolved relative to the fleet file's directory, to derive what to + start it with + +#### Scenario: The field is inert outside deploy and start + +- **WHEN** a node declares a `file` field +- **THEN** `fleet status`, `metrics`, `stop`, `route`, and `dashboard` behave + exactly as they do without it + +### Requirement: Node Spinloop source falls back to name-based lookup + +A node declaring no `file` field SHALL have its Spinloop source resolved +from its own `name`, tried in order: + +1. `name` resolved as a registered `spinloop alias` — the same lookup a bare + argument to `spinloop remote deploy ` already performs. +2. Failing that, a subdirectory named `` beside the fleet file, + containing a Spinloop file — the same directory-to-default-file + resolution an ordinary Spinloop path argument already gets when it names + a directory. + +A node for which neither resolves SHALL fail the command acting on it — +`fleet deploy` for a `kind: remote` node, `fleet start` for a `kind: daemon` +node — for that node alone, naming all three ways a source could have been +given: the `file` field, a `spinloop alias` named after the node, or a +`/` subdirectory beside the fleet file. + +#### Scenario: Resolved through a registered alias + +- **WHEN** a node named `gpu-env` declares no `file` field, and `spinloop + alias` has `gpu-env` registered to a Spinloop path +- **THEN** `fleet deploy` (if `gpu-env` is `kind: remote`) or `fleet start` + (if `kind: daemon`) reads the Spinloop the alias names + +#### Scenario: Resolved through a named subdirectory + +- **WHEN** a node named `dev-1` declares no `file` field, no alias named + `dev-1` is registered, and a `dev-1/` directory containing a Spinloop file + sits beside the fleet file +- **THEN** `fleet deploy` (if `dev-1` is `kind: remote`) or `fleet start` (if + `kind: daemon`) reads the Spinloop from that subdirectory + +#### Scenario: An alias wins over a same-named subdirectory + +- **WHEN** a node named `dev-1` declares no `file` field, an alias named + `dev-1` is registered, and a `dev-1/` subdirectory containing a Spinloop + file also sits beside the fleet file +- **THEN** the alias is used, not the subdirectory + +#### Scenario: None of the three resolve for a remote node + +- **WHEN** a `kind: remote` node declares no `file` field, no alias is + registered under its name, and no same-named subdirectory sits beside the + fleet file +- **THEN** `fleet deploy` fails for that node, naming the `file` field, the + alias registry, and the subdirectory convention as the three ways a source + could have been given + +#### Scenario: None of the three resolve for a daemon node + +- **WHEN** a `kind: daemon` node declares no `file` field, no alias is + registered under its name, and no same-named subdirectory sits beside the + fleet file +- **THEN** `fleet start` fails for that node, naming the `file` field, the + alias registry, and the subdirectory convention as the three ways a source + could have been given