diff --git a/README.md b/README.md index 50764875..fd115f6e 100644 --- a/README.md +++ b/README.md @@ -208,9 +208,10 @@ spinloop harness [] [-H ] [--spinloop[=]] [args...] # launch the harness (a leading Spinloop or alias is # applied first; --get shows it; --set stores it) spinloop completion # tab completion (bash, zsh, powershell) -spinloop remote [path] +spinloop remote [path] # control the remote GPU inference instance # (bootstrap does the once-per-account setup; + # bake bakes the runner AMI(s) it launches from; # deploy sets what it serves, from the Spinloop; # pause stops it while keeping it re-wakeable; # restart gives a fresh engine at the same address; diff --git a/cmd/spinloop/commands.go b/cmd/spinloop/commands.go index 4dace5e6..aeb111a9 100644 --- a/cmd/spinloop/commands.go +++ b/cmd/spinloop/commands.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spinloop-ai/spinloop/internal/fleet" "github.com/spinloop-ai/spinloop/internal/harness" "github.com/spinloop-ai/spinloop/internal/opencode" @@ -290,8 +291,22 @@ func versionCmd() *cobra.Command { } } -// fleetCmd builds the fleet parent and its subcommands. The parent runs when -// no subcommand is named and reports the usage, as the old dispatch did. +// groupFallback is the RunE a command group (fleet, remote, seed) gets: bare, +// it shows the group's own help — the one cobra generates from the tree, so +// its subcommand list cannot drift from the tree — and a word that is not a +// subcommand is cobra's own unknown-command error. The help sentinel is +// pflag's, not the stdlib one: cobra's ExecuteC checks pflag.ErrHelp (cobra +// imports pflag as its flag package), and the stdlib twin would surface as a +// bare "flag: help requested" error instead of the help. +func groupFallback(c *cobra.Command, args []string) error { + if len(args) == 0 { + return pflag.ErrHelp + } + return cobra.NoArgs(c, args) +} + +// fleetCmd builds the fleet parent and its subcommands. The parent does +// nothing itself — see groupFallback. func fleetCmd() *cobra.Command { fleet := &cobra.Command{ Use: "fleet", @@ -302,14 +317,9 @@ logs, and dashboard — the live tiled view); start, stop and route act on a single node, and with no node they list the fleet and touch nothing. A node that fails is a rendered row, never an error — only a problem with the fleet file itself fails a command.`, - Args: cobra.ArbitraryArgs, - DisableFlagParsing: true, - SilenceErrors: true, - SilenceUsage: true, - RunE: func(c *cobra.Command, args []string) error { - resolve(c) - return fleetParentFallback(args) - }, + SilenceErrors: true, + SilenceUsage: true, + RunE: groupFallback, } fleet.AddCommand( fleetStatusCmd(), @@ -323,8 +333,8 @@ file itself fails a command.`, return fleet } -// remoteCmd builds the remote parent and its subcommands. The parent runs -// when no subcommand is named and reports the usage, as the old dispatch did. +// remoteCmd builds the remote parent and its subcommands. The parent does +// nothing itself — see groupFallback. func remoteCmd() *cobra.Command { remote := &cobra.Command{ Use: "remote", @@ -334,17 +344,13 @@ the same Spinloop. The endpoint's URLs come from the Spinloop's REMOTE — a bar name selects an environment under ~/.config/spinloop/remotes//, a path names a file — falling back to the default environment. Each subcommand's --help says what that step does.`, - Args: cobra.ArbitraryArgs, - DisableFlagParsing: true, - SilenceErrors: true, - SilenceUsage: true, - RunE: func(c *cobra.Command, args []string) error { - resolve(c) - return remoteParentFallback(args) - }, + SilenceErrors: true, + SilenceUsage: true, + RunE: groupFallback, } remote.AddCommand( remoteBootstrapCmd(), + remoteBakeCmd(), remoteStartCmd(), remotePauseCmd(), remoteRestartCmd(), diff --git a/cmd/spinloop/fleet.go b/cmd/spinloop/fleet.go index c0933d1b..afd44f01 100644 --- a/cmd/spinloop/fleet.go +++ b/cmd/spinloop/fleet.go @@ -21,18 +21,6 @@ import ( "github.com/spinloop-ai/spinloop/internal/fleet" ) -// fleetParentFallback reports the usage when no (named) subcommand is given, -// and rejects the ones that are not named — the subcommands themselves are real -// commands in the tree, so only the unknown and the bare cases reach here. -func fleetParentFallback(args []string) error { - if len(args) == 0 { - return fmt.Errorf("usage: spinloop fleet [node] [--fleet ]") - } - return fmt.Errorf( - "unknown fleet subcommand %q (expected status, metrics, logs, dashboard, route, start or stop)", - args[0]) -} - // cmdFleet runs the fleet subcommands through the tree — the seam the suite // calls directly. func cmdFleet(args []string) error { diff --git a/cmd/spinloop/fleet_test.go b/cmd/spinloop/fleet_test.go index be54f0f3..69cdfa63 100644 --- a/cmd/spinloop/fleet_test.go +++ b/cmd/spinloop/fleet_test.go @@ -279,11 +279,18 @@ func TestCmdFleetExplicitPath(t *testing.T) { } func TestCmdFleetUnknownSubcommand(t *testing.T) { - if err := cmdFleet([]string{"wat"}); err == nil { - t.Fatal("unknown subcommand accepted") + if err := cmdFleet([]string{"wat"}); err == nil || !strings.Contains(err.Error(), "unknown command") { + t.Fatalf("unknown subcommand should error, got %v", err) } - if err := cmdFleet(nil); err == nil { - t.Fatal("no subcommand accepted") + // Bare: no error — cobra shows the group's own help, with the + // subcommand list generated from the tree. + out := captureStdout(t, func() { + if err := cmdFleet([]string{}); err != nil { + t.Fatalf("bare fleet should show its help, got %v", err) + } + }) + if !strings.Contains(out, "Available Commands") { + t.Errorf("bare fleet should list its subcommands:\n%s", out) } } diff --git a/cmd/spinloop/remote.go b/cmd/spinloop/remote.go index dcce084e..c0bad9da 100644 --- a/cmd/spinloop/remote.go +++ b/cmd/spinloop/remote.go @@ -38,20 +38,8 @@ import ( // retention deadline to prevent the sweep from terminating the instance // early, and deploy sets what the instance will serve from the Spinloop itself. // Each subcommand takes an optional Spinloop path; see resolveRemoteConfig for -// how the remote config is found. -// cmdRemote reports the usage when no (named) subcommand is given, and rejects -// the ones that are not named — the subcommands themselves are real commands -// in the tree, so only the unknown and the bare cases reach here. -// remoteParentFallback reports the usage when no (named) subcommand is given, -// and rejects the ones that are not named — the subcommands themselves are real -// commands in the tree, so only the unknown and the bare cases reach here. -func remoteParentFallback(args []string) error { - if len(args) == 0 { - return fmt.Errorf("usage: spinloop remote [path]") - } - return fmt.Errorf( - "unknown remote subcommand %q (expected bootstrap, start, pause, restart, stop, status, metrics, logs, deploy, seed, env, ls or keep)", args[0]) -} +// how the remote config is found. The group itself does nothing — see +// groupFallback for what a bare or mistyped invocation gets. // cmdRemote runs the remote subcommands through the tree — the seam the suite // calls directly. diff --git a/cmd/spinloop/remote_bake.go b/cmd/spinloop/remote_bake.go new file mode 100644 index 00000000..9156edfe --- /dev/null +++ b/cmd/spinloop/remote_bake.go @@ -0,0 +1,161 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + + "github.com/spf13/cobra" +) + +// defaultBakeRunners is what `spinloop remote bake` bakes when no runners are +// named, and the full accepted set — the two engines an environment can run. +var defaultBakeRunners = []string{"llamacpp", "vllm"} + +// isBakeRunner reports whether r is a runner bake accepts. +func isBakeRunner(r string) bool { + for _, v := range defaultBakeRunners { + if r == v { + return true + } + } + return false +} + +// bakeRunnerSlot is the `bake [runner...]` completion: the accepted runners +// not already on the line. +func bakeRunnerSlot(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { + taken := map[string]bool{} + for _, a := range args { + taken[a] = true + } + var out []string + for _, r := range defaultBakeRunners { + if !taken[r] { + out = append(out, r) + } + } + return out, cobra.ShellCompDirectiveNoFileComp +} + +// remoteBakeCmd starts an AMI bake for each named runner. It drives the same +// CDK project bootstrap deploys but deploys nothing — the control plane (and +// its Image Builder pipelines) must already exist, so a missing one fails fast +// naming bootstrap rather than deploying implicitly. The wait is the default: +// the step after a bake is `spinloop remote deploy`, which needs the AMI. +func remoteBakeCmd() *cobra.Command { + var ( + noWait bool + ref string + dir string + region string + pkgMgr string + ) + c := &cobra.Command{ + Use: "bake [runner...]", + Short: "bake the runner AMIs an environment runs from", + Long: `starts an AMI bake for each named runner (llamacpp and vllm when +none are named) and waits until the AMI(s) are available (~20-40 min). It +drives the same CDK project bootstrap deploys, deploys nothing — the control +plane must already exist — and --no-wait returns as soon as the bakes are +queued, reporting how to check on them.`, + Args: cobra.ArbitraryArgs, + SilenceErrors: true, + SilenceUsage: true, + ValidArgsFunction: bakeRunnerSlot, + RunE: func(c *cobra.Command, args []string) error { + resolve(c) + return runRemoteBake(args, noWait, ref, dir, region, pkgMgr) + }, + } + fs := c.Flags() + fs.BoolVar(&noWait, "no-wait", false, "return as soon as the bakes are queued, rather than waiting for the AMI(s)") + fs.StringVar(&ref, "ref", "", "git ref of remote/ to download (default: matches this binary)") + fs.StringVar(&dir, "dir", "", "where to find the downloaded remote/ sources") + fs.StringVar(®ion, "region", "", "AWS region of the control plane (default: AWS_REGION or us-east-1)") + fs.StringVar(&pkgMgr, "package-manager", "", "package manager to use: pnpm or npm (default: auto-detect, preferring pnpm)") + compRegister(c, "dir", compFiles) + return c +} + +// runRemoteBake is the body of `spinloop remote bake`. +func runRemoteBake(args []string, noWait bool, ref, dir, region string, pkgMgr string) error { + runners := defaultBakeRunners + if len(args) > 0 { + runners = args + for _, r := range args { + if !isBakeRunner(r) { + return fmt.Errorf("unknown runner %q — bake accepts llamacpp and vllm", r) + } + } + } + + pmName, pmPinned, err := resolvePackageManagerName(pkgMgr) + if err != nil { + return err + } + + loc, err := resolveSourceLocation(ref, dir) + if err != nil { + return err + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + pm, err := preflightFn(pmName, pmPinned) + if err != nil { + return err + } + + resolvedRegion := resolveRegion(region) + cfg, err := loadCreds(ctx, resolvedRegion) + if err != nil { + return fmt.Errorf( + "resolving AWS credentials: %w (configure env credentials, a profile or an SSO session)", err) + } + + deployed, err := stackDeployedFn(ctx, cfg, controlPlaneStackName) + if err != nil { + return err + } + if !deployed { + return fmt.Errorf( + "the control plane is not deployed in %s — run `spinloop remote bootstrap` first", resolvedRegion) + } + + if err := downloadFn(ctx, loc.ref, loc.dir); err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "\nUsing %s to run the CDK project.\n", pm.name) + run := func(name string, argv ...string) error { + return runStep(ctx, name, argv, loc.dir) + } + if !dirExists(filepath.Join(loc.dir, "node_modules")) { + if err := run("install", pm.install...); err != nil { + return err + } + } + for _, r := range runners { + if err := run("bake "+r, pm.script("bake", r)...); err != nil { + return err + } + } + + if noWait { + fmt.Fprintln(os.Stderr, "\nThe AMI bake(s) run in the background (~20-40 min) — the commands above say how to check on them.") + } else { + fmt.Fprintln(os.Stderr, "\nWaiting for the AMI bake(s) to finish (this can take 20-40 minutes)...") + if err := waitForBake(ctx, cfg, runners); err != nil { + return err + } + fmt.Fprintln(os.Stderr, "AMI(s) available.") + } + + return pruneSourceCaches(loc) +} + +func cmdRemoteBake(args []string) error { return execCmd(remoteBakeCmd(), args) } diff --git a/cmd/spinloop/remote_bake_test.go b/cmd/spinloop/remote_bake_test.go new file mode 100644 index 00000000..07a8e141 --- /dev/null +++ b/cmd/spinloop/remote_bake_test.go @@ -0,0 +1,229 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// stubBakeSeams wires the shared seams to hermetic fakes for the bake flow: +// no AWS, no network, no pnpm. It returns a pointer to the recorded command +// list and to the bake-poll call count. +func stubBakeSeams(t *testing.T, deployed bool) (*[]recordedStep, *int) { + t.Helper() + var steps []recordedStep + bakedCalls := 0 + + origRun, origDl := runStep, downloadFn + origStack, origBaked, origPre := stackDeployedFn, bakedFn, preflightFn + t.Cleanup(func() { + runStep, downloadFn = origRun, origDl + stackDeployedFn, bakedFn, preflightFn = origStack, origBaked, origPre + }) + + runStep = func(_ context.Context, _ string, argv []string, workDir string) error { + steps = append(steps, recordedStep{argv: argv, dir: workDir}) + return nil + } + downloadFn = func(_ context.Context, _, destDir string) error { + if err := os.MkdirAll(destDir, 0o755); err != nil { + return err + } + return os.WriteFile(filepath.Join(destDir, "package.json"), []byte(`{"name":"cloud-vm-llm"}`), 0o644) + } + stackDeployedFn = func(context.Context, aws.Config, string) (bool, error) { return deployed, nil } + bakedFn = func(context.Context, aws.Config) (map[string]bool, error) { + bakedCalls++ + return map[string]bool{}, nil + } + preflightFn = func(name string, _ bool) (packageManager, error) { return managerByName(name), nil } + return &steps, &bakedCalls +} + +func TestBake_DefaultRunners(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + steps, bakedCalls := stubBakeSeams(t, true) + if err := cmdRemoteBake([]string{"--region", "us-east-1", "--no-wait"}); err != nil { + t.Fatal(err) + } + var got []string + for _, s := range *steps { + got = append(got, strings.Join(s.argv, " ")) + } + want := []string{"pnpm install", "pnpm run bake llamacpp", "pnpm run bake vllm"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("commands = %v, want %v", got, want) + } + if *bakedCalls != 0 { + t.Errorf("--no-wait should not poll the bake, polled %d times", *bakedCalls) + } +} + +func TestBake_SingleRunner(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + steps, _ := stubBakeSeams(t, true) + if err := cmdRemoteBake([]string{"llamacpp", "--region", "us-east-1", "--no-wait"}); err != nil { + t.Fatal(err) + } + var got []string + for _, s := range *steps { + got = append(got, strings.Join(s.argv, " ")) + } + want := []string{"pnpm install", "pnpm run bake llamacpp"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("commands = %v, want %v", got, want) + } +} + +func TestBake_UnknownRunnerRejected(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + steps, _ := stubBakeSeams(t, true) + err := cmdRemoteBake([]string{"bogus", "--region", "us-east-1"}) + if err == nil { + t.Fatal("expected an error for a bad runner") + } + if !strings.Contains(err.Error(), "llamacpp and vllm") { + t.Errorf("error should name the accepted runners, got %v", err) + } + if len(*steps) != 0 { + t.Errorf("bad runner should run nothing, got %v", *steps) + } +} + +func TestBake_NoControlPlane(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + steps, _ := stubBakeSeams(t, false) + err := cmdRemoteBake([]string{"--region", "us-east-1"}) + if err == nil { + t.Fatal("expected an error when the control plane is missing") + } + if !strings.Contains(err.Error(), "spinloop remote bootstrap") { + t.Errorf("error should point at bootstrap, got %v", err) + } + if len(*steps) != 0 { + t.Errorf("missing control plane should run nothing, got %v", *steps) + } +} + +func TestBake_WaitsByDefault(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + steps, _ := stubBakeSeams(t, true) + + origPoll := bakePollInterval + bakePollInterval = time.Millisecond + t.Cleanup(func() { bakePollInterval = origPoll }) + + origBaked := bakedFn + calls := 0 + bakedFn = func(context.Context, aws.Config) (map[string]bool, error) { + calls++ + if calls == 1 { + return map[string]bool{"llamacpp": true}, nil + } + return map[string]bool{"llamacpp": true, "vllm": true}, nil + } + t.Cleanup(func() { bakedFn = origBaked }) + + if err := cmdRemoteBake([]string{"--region", "us-east-1"}); err != nil { + t.Fatal(err) + } + if calls < 2 { + t.Errorf("default run should poll until both AMIs are available, polled %d times", calls) + } + if len(*steps) != 3 { + t.Errorf("bake should run install plus one bake per runner, got %v", *steps) + } +} + +func TestBake_SkipsInstallWhenPresent(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + steps, _ := stubBakeSeams(t, true) + cdkDir := must1(remote.SourceDir(remote.ResolveRef(version, ""))) + if err := os.MkdirAll(filepath.Join(cdkDir, "node_modules"), 0o755); err != nil { + t.Fatal(err) + } + if err := cmdRemoteBake([]string{"--region", "us-east-1", "--no-wait"}); err != nil { + t.Fatal(err) + } + for _, s := range *steps { + if strings.Contains(strings.Join(s.argv, " "), "install") { + t.Errorf("node_modules present should skip install, got %v", s.argv) + } + } +} + +func TestWaitForBake_Timeout(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + stubBakeSeams(t, true) + cfg, err := loadCreds(context.Background(), "us-east-1") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := waitForBake(ctx, cfg, []string{"llamacpp"}); err == nil { + t.Fatal("expected a timeout error for a cancelled context") + } +} + +func TestBake_PrunesOtherRefsByDefault(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + stubBakeSeams(t, true) + root := must1(remote.SourceRoot()) + stale := filepath.Join(root, "v9.9.9") + if err := os.MkdirAll(stale, 0o755); err != nil { + t.Fatal(err) + } + if err := cmdRemoteBake([]string{"--region", "us-east-1", "--no-wait"}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("stale ref cache should be pruned on success, err=%v", err) + } +} + +func TestBake_ExplicitDirNotPruned(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + stubBakeSeams(t, true) + root := must1(remote.SourceRoot()) + stale := filepath.Join(root, "v9.9.9") + if err := os.MkdirAll(stale, 0o755); err != nil { + t.Fatal(err) + } + if err := cmdRemoteBake([]string{"--region", "us-east-1", "--no-wait", "--dir", t.TempDir()}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err != nil { + t.Errorf("an explicit --dir run should not prune other refs, err=%v", err) + } +} + +func TestBakeRunnerSlot(t *testing.T) { + got, _ := bakeRunnerSlot(nil, nil, "") + if strings.Join(got, ",") != "llamacpp,vllm" { + t.Errorf("empty line should offer both runners, got %v", got) + } + got, _ = bakeRunnerSlot(nil, []string{"llamacpp"}, "") + if strings.Join(got, ",") != "vllm" { + t.Errorf("a typed runner should be dropped from the candidates, got %v", got) + } + got, _ = bakeRunnerSlot(nil, []string{"llamacpp", "vllm"}, "") + if len(got) != 0 { + t.Errorf("both runners typed should leave no candidates, got %v", got) + } +} diff --git a/cmd/spinloop/remote_bootstrap.go b/cmd/spinloop/remote_bootstrap.go index dad40755..f83d8fcb 100644 --- a/cmd/spinloop/remote_bootstrap.go +++ b/cmd/spinloop/remote_bootstrap.go @@ -3,7 +3,6 @@ package main import ( "bufio" "context" - "encoding/json" "errors" "fmt" "os" @@ -26,16 +25,16 @@ import ( const controlPlaneStackName = remote.ControlPlaneStackName // Seams: package variables so tests drive the flow without AWS, a network, or -// spawning a package manager/cdk. -type bootstrapStep func(ctx context.Context, name string, argv []string, workDir string) error +// spawning a package manager/cdk. Shared by bootstrap and bake. +type stepRunner func(ctx context.Context, name string, argv []string, workDir string) error var ( - bootstrapRunStep bootstrapStep = runBootstrapStep - bootstrapDownloadFn = remote.DownloadRemote - bootstrapAccountFn = remote.CallerIdentity - bootstrapStackDeployedFn = remote.ControlPlaneStackDeployed - bootstrapBakedFn = remote.BakedRunners - bootstrapPreflightFn = checkNodeAndPackageManager + runStep stepRunner = execStep + downloadFn = remote.DownloadRemote + accountFn = remote.CallerIdentity + stackDeployedFn = remote.ControlPlaneStackDeployed + bakedFn = remote.BakedRunners + preflightFn = checkNodeAndPackageManager ) // packageManagerEnv pins the Node package manager bootstrap drives the CDK @@ -134,47 +133,43 @@ func resolvePackageManagerName(flagVal string) (name string, pinned bool, err er return "", false, nil } -// cmdRemoteBootstrap deploys the account-level control plane once — +// remoteBootstrapCmd deploys the account-level control plane once — // analogous to `cdk bootstrap` — by downloading the remote/ CDK project and // driving its control-plane deploy. It creates no EIP, instance, or environment; -// those come from `spinloop remote deploy`. +// those come from `spinloop remote deploy`. It starts no AMI bake either — +// `spinloop remote bake` is the separate step after it. func remoteBootstrapCmd() *cobra.Command { var ( - runnersFlag string - hfToken string - ref string - dir string - region string - dryRun bool - assumeYes bool - wait bool - forceBake bool - pkgMgr string + hfToken string + ref string + dir string + region string + dryRun bool + assumeYes bool + pkgMgr string ) c := &cobra.Command{ Use: "bootstrap", Short: "set up the once-per-account control plane", Long: `does the once-per-account control-plane setup (Image Builder, the -lifecycle Lambdas, shared bucket/roles/VPC) with a consent gate.`, +lifecycle Lambdas, shared bucket/roles/VPC) with a consent gate. It bakes no +AMIs — spinloop remote bake is the next step after it.`, Args: cobra.ArbitraryArgs, SilenceErrors: true, SilenceUsage: true, ValidArgsFunction: noPositionals, RunE: func(c *cobra.Command, _ []string) error { resolve(c) - return runRemoteBootstrap(runnersFlag, hfToken, ref, dir, region, dryRun, assumeYes, wait, forceBake, pkgMgr) + return runRemoteBootstrap(hfToken, ref, dir, region, dryRun, assumeYes, pkgMgr) }, } fs := c.Flags() - fs.StringVar(&runnersFlag, "runners", "llamacpp,vllm", "comma-separated runner AMIs to bake") fs.StringVar(&hfToken, "hf-token", "", "Hugging Face token for the shared secret (optional)") fs.StringVar(&ref, "ref", "", "git ref of remote/ to download (default: matches this binary)") fs.StringVar(&dir, "dir", "", "where to place the downloaded remote/ sources") fs.StringVar(®ion, "region", "", "AWS region (default: AWS_REGION or us-east-1)") fs.BoolVarP(&dryRun, "dry-run", "n", false, "print the plan and exit without doing anything") fs.BoolVarP(&assumeYes, "yes", "y", false, "skip the confirmation prompt") - fs.BoolVar(&wait, "wait", false, "block until the AMI bake(s) finish") - fs.BoolVar(&forceBake, "force-bake", false, "re-bake the AMIs even if already bootstrapped") fs.StringVar(&pkgMgr, "package-manager", "", "package manager to use: pnpm or npm (default: auto-detect, preferring pnpm)") fs.SetInterspersed(false) compRegister(c, "dir", compFiles) @@ -182,27 +177,16 @@ lifecycle Lambdas, shared bucket/roles/VPC) with a consent gate.`, } // runRemoteBootstrap is the body of `spinloop remote bootstrap`. -func runRemoteBootstrap(runnersFlag, hfToken, ref, dir, region string, dryRun, assumeYes, wait, forceBake bool, pkgMgr string) error { - runners, err := parseRunners(runnersFlag) - if err != nil { - return err - } - +func runRemoteBootstrap(hfToken, ref, dir, region string, dryRun, assumeYes bool, pkgMgr string) error { pmName, pmPinned, err := resolvePackageManagerName(pkgMgr) if err != nil { return err } - resolvedRef := remote.ResolveRef(version, ref) - cdkDir, err := remote.SourceDir(resolvedRef) + loc, err := resolveSourceLocation(ref, dir) if err != nil { return err } - pruneAfter := true - if dir != "" { - cdkDir = dir - pruneAfter = false // an explicit --dir is the user's own; leave it alone - } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() @@ -212,21 +196,19 @@ func runRemoteBootstrap(runnersFlag, hfToken, ref, dir, region string, dryRun, a // AWS is best-effort for the plan; it is hard-required for a real run. account := "unknown" alreadyBootstrapped := false - var awsCfg aws.Config cfg, credsErr := loadCreds(ctx, resolvedRegion) if credsErr == nil { - awsCfg = cfg - if acct, err := bootstrapAccountFn(ctx, cfg); err == nil { + if acct, err := accountFn(ctx, cfg); err == nil { account = acct } - if dep, err := bootstrapStackDeployedFn(ctx, cfg, controlPlaneStackName); err == nil { + if dep, err := stackDeployedFn(ctx, cfg, controlPlaneStackName); err == nil { alreadyBootstrapped = dep } } var pm packageManager if !dryRun { - selected, err := bootstrapPreflightFn(pmName, pmPinned) + selected, err := preflightFn(pmName, pmPinned) if err != nil { return err } @@ -239,7 +221,7 @@ func runRemoteBootstrap(runnersFlag, hfToken, ref, dir, region string, dryRun, a pm, _ = selectPackageManager(pmName) } - renderBootstrapPlan(account, resolvedRegion, runners, resolvedRef, cdkDir, alreadyBootstrapped, pm) + renderBootstrapPlan(account, resolvedRegion, loc.ref, loc.dir, alreadyBootstrapped, pm) if dryRun { return nil @@ -249,56 +231,32 @@ func runRemoteBootstrap(runnersFlag, hfToken, ref, dir, region string, dryRun, a return nil } - if err := bootstrapDownloadFn(ctx, resolvedRef, cdkDir); err != nil { + if err := downloadFn(ctx, loc.ref, loc.dir); err != nil { return err } if hfToken != "" { - if err := upsertEnvVar(filepath.Join(cdkDir, ".env"), "HF_TOKEN", hfToken); err != nil { + if err := upsertEnvVar(filepath.Join(loc.dir, ".env"), "HF_TOKEN", hfToken); err != nil { return err } } - if err := setCdkContext(cdkDir, "runners", strings.Join(runners, ",")); err != nil { - return err - } fmt.Fprintf(os.Stderr, "\nUsing %s to run the CDK project.\n", pm.name) - if err := runBootstrapSequence(ctx, cdkDir, runners, alreadyBootstrapped, forceBake, pm); err != nil { + if err := runBootstrapSequence(ctx, loc.dir, pm); err != nil { return err } - fmt.Println("\nThe account is bootstrapped. Create an endpoint with:") - fmt.Println(" spinloop remote deploy # names an environment; discovers this control plane") - - if wait { - if credsErr != nil { - return fmt.Errorf("--wait needs AWS credentials to poll the bake") - } - fmt.Fprintln(os.Stderr, "\nWaiting for the AMI bake(s) to finish (this can take 20-40 minutes)...") - if err := waitForBake(ctx, awsCfg, runners); err != nil { - return err - } - fmt.Fprintln(os.Stderr, "AMI(s) available.") - } else { - fmt.Fprintln(os.Stderr, "\nThe AMI bake(s) run in the background (~20-40 min). Re-run with --wait to block, or check the Image Builder console.") - } + fmt.Println("\nThe account is bootstrapped. Before an environment can start, its AMI needs baking:") + fmt.Println(" spinloop remote bake # bakes the AMI(s) an environment runs from; waits until available") + fmt.Println(" spinloop remote deploy # names an environment; discovers this control plane") - if pruneAfter { - sourceRoot, err := remote.SourceRoot() - if err != nil { - return err - } - if err := remote.PruneSources(sourceRoot, resolvedRef); err != nil { - return err - } - } - return nil + return pruneSourceCaches(loc) } // runBootstrapSequence runs the package-manager/cdk steps in the sources // directory with the resolved manager. -func runBootstrapSequence(ctx context.Context, cdkDir string, runners []string, alreadyBootstrapped, forceBake bool, pm packageManager) error { +func runBootstrapSequence(ctx context.Context, cdkDir string, pm packageManager) error { run := func(name string, argv ...string) error { - return bootstrapRunStep(ctx, name, argv, cdkDir) + return runStep(ctx, name, argv, cdkDir) } if !dirExists(filepath.Join(cdkDir, "node_modules")) { if err := run("install", pm.install...); err != nil { @@ -311,19 +269,12 @@ func runBootstrapSequence(ctx context.Context, cdkDir string, runners []string, if err := run("deploy:image", pm.script("deploy:image")...); err != nil { return err } - if !alreadyBootstrapped || forceBake { - for _, r := range runners { - if err := run("bake "+r, pm.script("bake", r)...); err != nil { - return err - } - } - } return run("deploy", pm.script("deploy")...) } -// runBootstrapStep runs one external command in workDir, streaming its stdio, +// execStep runs one external command in workDir, streaming its stdio, // mirroring serve.go's exec pattern. Ctrl-C propagates via the context. -func runBootstrapStep(ctx context.Context, name string, argv []string, workDir string) error { +func execStep(ctx context.Context, name string, argv []string, workDir string) error { fmt.Fprintf(os.Stderr, "\n$ %s\n", strings.Join(argv, " ")) cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) cmd.Dir = workDir @@ -341,25 +292,6 @@ func runBootstrapStep(ctx context.Context, name string, argv []string, workDir s return nil } -// parseRunners splits and validates the --runners list. -func parseRunners(csv string) ([]string, error) { - var runners []string - for _, part := range strings.Split(csv, ",") { - r := strings.TrimSpace(part) - if r == "" { - continue - } - if _, err := runnerFor(r); err != nil { - return nil, fmt.Errorf("--runners: %w", err) - } - runners = append(runners, r) - } - if len(runners) == 0 { - return nil, fmt.Errorf("--runners is empty") - } - return runners, nil -} - func resolveRegion(flagVal string) string { if flagVal != "" { return flagVal @@ -451,49 +383,23 @@ func upsertEnvVar(path, key, value string) error { return os.WriteFile(path, []byte(strings.TrimLeft(out, "\n")), 0o600) } -// setCdkContext sets a context key in cdk.json (an additive JSON edit), so the -// value reaches every cdk invocation without a -c flag. -func setCdkContext(cdkDir, key, value string) error { - path := filepath.Join(cdkDir, "cdk.json") - data, err := os.ReadFile(path) - if err != nil { - return err - } - var doc map[string]any - if err := json.Unmarshal(data, &doc); err != nil { - return fmt.Errorf("parsing %s: %w", path, err) - } - ctxObj, _ := doc["context"].(map[string]any) - if ctxObj == nil { - ctxObj = map[string]any{} - doc["context"] = ctxObj - } - ctxObj[key] = value - out, err := json.MarshalIndent(doc, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, append(out, '\n'), 0o644) -} - -func renderBootstrapPlan(account, region string, runners []string, ref, cdkDir string, alreadyBootstrapped bool, pm packageManager) { +func renderBootstrapPlan(account, region string, ref, cdkDir string, alreadyBootstrapped bool, pm packageManager) { w := os.Stderr fmt.Fprintln(w, "spinloop remote bootstrap — account-level control-plane setup (once per account)") fmt.Fprintln(w) fmt.Fprintf(w, "AWS account: %s\n", account) fmt.Fprintf(w, "Region: %s\n", region) - fmt.Fprintf(w, "Runners: %s\n", strings.Join(runners, ", ")) fmt.Fprintf(w, "Sources: github.com/spinloop-ai/spinloop @ %s -> %s\n", ref, cdkDir) if alreadyBootstrapped { fmt.Fprintln(w, "Note: the account is already bootstrapped; this will update the control plane.") } fmt.Fprintln(w) fmt.Fprintln(w, "This deploys the control plane every environment reuses:") - fmt.Fprintln(w, " • EC2 Image Builder pipelines and the baked AMIs") + fmt.Fprintln(w, " • EC2 Image Builder pipelines (the AMI bakes are a separate step: spinloop remote bake)") fmt.Fprintln(w, " • the lifecycle Lambdas (start/stop/monitor/deploy) and their IAM") fmt.Fprintln(w, " • the shared S3 weights bucket, IAM roles, and VPC") fmt.Fprintln(w) - fmt.Fprintln(w, "Cost: an ongoing at-rest cost (bucket, AMIs) plus a per-hour GPU cost only while") + fmt.Fprintln(w, "Cost: an ongoing at-rest cost (bucket, and AMIs once baked) plus a per-hour GPU cost only while") fmt.Fprintln(w, "an environment is running. See remote/docs/costs.md for the breakdown.") fmt.Fprintln(w, "Note: the GPU vCPU quota must be > 0 in this region or a later launch fails.") fmt.Fprintln(w) @@ -501,9 +407,6 @@ func renderBootstrapPlan(account, region string, runners []string, ref, cdkDir s fmt.Fprintf(w, " %s\n", strings.Join(pm.install, " ")) fmt.Fprintf(w, " %s\n", strings.Join(pm.script("cdk", "bootstrap"), " ")) fmt.Fprintf(w, " %s\n", strings.Join(pm.script("deploy:image"), " ")) - for _, r := range runners { - fmt.Fprintf(w, " %s\n", strings.Join(pm.script("bake", r), " ")) - } fmt.Fprintf(w, " %s\n", strings.Join(pm.script("deploy"), " ")) } @@ -518,6 +421,10 @@ func confirmProceed() bool { } } +// bakePollInterval is how often waitForBake checks; a variable so tests can +// poll fast, like the dashboard's interval hooks. +var bakePollInterval = 60 * time.Second + // waitForBake polls until every requested runner has an available AMI, or the // context is cancelled. It is bounded by a generous timeout so it cannot hang // forever if a bake fails. @@ -525,7 +432,7 @@ func waitForBake(ctx context.Context, cfg aws.Config, runners []string) error { ctx, cancel := context.WithTimeout(ctx, 60*time.Minute) defer cancel() for { - baked, err := bootstrapBakedFn(ctx, cfg) + baked, err := bakedFn(ctx, cfg) if err != nil { return err } @@ -542,7 +449,7 @@ func waitForBake(ctx context.Context, cfg aws.Config, runners []string) error { select { case <-ctx.Done(): return fmt.Errorf("timed out waiting for the AMI bake(s): %w", ctx.Err()) - case <-time.After(60 * time.Second): + case <-time.After(bakePollInterval): } } } diff --git a/cmd/spinloop/remote_bootstrap_test.go b/cmd/spinloop/remote_bootstrap_test.go index ca8cdf18..dca36274 100644 --- a/cmd/spinloop/remote_bootstrap_test.go +++ b/cmd/spinloop/remote_bootstrap_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "os" "path/filepath" "strings" @@ -17,37 +16,35 @@ type recordedStep struct { dir string } -// stubBootstrapSeams wires the bootstrap package seams to hermetic fakes: no -// AWS, no network, no pnpm. It returns a pointer to the recorded command list. +// stubBootstrapSeams wires the shared bootstrap/bake seams to hermetic fakes: +// no AWS, no network, no pnpm. It returns a pointer to the recorded command +// list. func stubBootstrapSeams(t *testing.T, alreadyDeployed bool) *[]recordedStep { t.Helper() var steps []recordedStep - origRun, origDl := bootstrapRunStep, bootstrapDownloadFn - origAcct, origStack := bootstrapAccountFn, bootstrapStackDeployedFn - origPre := bootstrapPreflightFn + origRun, origDl := runStep, downloadFn + origAcct, origStack := accountFn, stackDeployedFn + origPre := preflightFn t.Cleanup(func() { - bootstrapRunStep, bootstrapDownloadFn = origRun, origDl - bootstrapAccountFn, bootstrapStackDeployedFn = origAcct, origStack - bootstrapPreflightFn = origPre + runStep, downloadFn = origRun, origDl + accountFn, stackDeployedFn = origAcct, origStack + preflightFn = origPre }) - bootstrapRunStep = func(_ context.Context, _ string, argv []string, workDir string) error { + runStep = func(_ context.Context, _ string, argv []string, workDir string) error { steps = append(steps, recordedStep{argv: argv, dir: workDir}) return nil } - bootstrapDownloadFn = func(_ context.Context, _, destDir string) error { + downloadFn = func(_ context.Context, _, destDir string) error { if err := os.MkdirAll(destDir, 0o755); err != nil { return err } - if err := os.WriteFile(filepath.Join(destDir, "package.json"), []byte(`{"name":"cloud-vm-llm"}`), 0o644); err != nil { - return err - } - return os.WriteFile(filepath.Join(destDir, "cdk.json"), []byte(`{"context":{}}`), 0o644) + return os.WriteFile(filepath.Join(destDir, "package.json"), []byte(`{"name":"cloud-vm-llm"}`), 0o644) } - bootstrapAccountFn = func(context.Context, aws.Config) (string, error) { return "1", nil } - bootstrapStackDeployedFn = func(context.Context, aws.Config, string) (bool, error) { return alreadyDeployed, nil } - bootstrapPreflightFn = func(name string, _ bool) (packageManager, error) { return managerByName(name), nil } + accountFn = func(context.Context, aws.Config) (string, error) { return "1", nil } + stackDeployedFn = func(context.Context, aws.Config, string) (bool, error) { return alreadyDeployed, nil } + preflightFn = func(name string, _ bool) (packageManager, error) { return managerByName(name), nil } return &steps } @@ -66,7 +63,7 @@ func withStdin(t *testing.T, input string) { t.Cleanup(func() { os.Stdin = orig; fh.Close() }) } -func TestBootstrap_EnvAndCdkWrites(t *testing.T) { +func TestBootstrap_EnvWrites(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("ALLOWED_CIDR=old\n"), 0o600); err != nil { t.Fatal(err) @@ -83,32 +80,20 @@ func TestBootstrap_EnvAndCdkWrites(t *testing.T) { t.Errorf(".env mode = %v, want 0600", fi.Mode().Perm()) } - if err := os.WriteFile(filepath.Join(dir, "cdk.json"), []byte(`{"app":"x","context":{"//":"note"}}`), 0o644); err != nil { - t.Fatal(err) - } - if err := setCdkContext(dir, "runners", "llamacpp,vllm"); err != nil { - t.Fatal(err) - } - var doc map[string]any - raw, _ := os.ReadFile(filepath.Join(dir, "cdk.json")) - if err := json.Unmarshal(raw, &doc); err != nil { - t.Fatal(err) - } - ctxObj := doc["context"].(map[string]any) - if ctxObj["runners"] != "llamacpp,vllm" || ctxObj["//"] != "note" || doc["app"] != "x" { - t.Errorf("cdk.json not merged as expected: %v", doc) - } } func TestBootstrap_PlanOutput(t *testing.T) { out := captureStderr(t, func() { - renderBootstrapPlan("1", "us-east-1", []string{"llamacpp", "vllm"}, "v1.10.0", "/tmp/cdk/v1.10.0", false, pnpmManager) + renderBootstrapPlan("1", "us-east-1", "v1.10.0", "/tmp/cdk/v1.10.0", false, pnpmManager) }) - for _, want := range []string{"AWS account: 1\n", "us-east-1", "llamacpp, vllm", "Image Builder", "Cost:", "pnpm run deploy\n"} { + for _, want := range []string{"AWS account: 1\n", "us-east-1", "Image Builder", "spinloop remote bake", "Cost:", "pnpm run deploy\n"} { if !strings.Contains(out, want) { t.Errorf("plan missing %q:\n%s", want, out) } } + if strings.Contains(out, "bake llamacpp") { + t.Errorf("plan should list no bake commands:\n%s", out) + } if strings.Contains(out, "$") { t.Errorf("plan should carry no dollar figures:\n%s", out) } @@ -156,42 +141,34 @@ func TestBootstrap_ConfirmGate(t *testing.T) { } } want := []string{ - "pnpm install", "pnpm run cdk bootstrap", "pnpm run deploy:image", - "pnpm run bake llamacpp", "pnpm run bake vllm", "pnpm run deploy", + "pnpm install", "pnpm run cdk bootstrap", "pnpm run deploy:image", "pnpm run deploy", } if strings.Join(got, "|") != strings.Join(want, "|") { t.Errorf("commands = %v, want %v", got, want) } }) +} - t.Run("already bootstrapped skips the bake", func(t *testing.T) { - isolateConfig(t) - stubAWSEnv(t) - steps := stubBootstrapSeams(t, true) // stack already deployed +func TestBootstrap_SignpostsTheBake(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + stubBootstrapSeams(t, false) + out := captureStdout(t, func() { if err := cmdRemoteBootstrap([]string{"--region", "us-east-1", "--yes"}); err != nil { t.Fatal(err) } - for _, s := range *steps { - if strings.Contains(strings.Join(s.argv, " "), "bake") { - t.Errorf("re-run should not bake without --force-bake, got %v", s.argv) - } - } }) + bakeIdx := strings.Index(out, "spinloop remote bake") + deployIdx := strings.Index(out, "spinloop remote deploy") + if bakeIdx < 0 || deployIdx < 0 { + t.Fatalf("success output should signpost bake then deploy:\n%s", out) + } + if bakeIdx > deployIdx { + t.Errorf("bake should be signposted before deploy:\n%s", out) + } } -func TestBootstrap_PreflightAndRunners(t *testing.T) { - t.Run("bad runner rejected before anything", func(t *testing.T) { - isolateConfig(t) - stubAWSEnv(t) - steps := stubBootstrapSeams(t, false) - if err := cmdRemoteBootstrap([]string{"--runners", "bogus", "--yes"}); err == nil { - t.Fatal("expected an error for a bad runner") - } - if len(*steps) != 0 { - t.Errorf("bad runner should run nothing, got %v", *steps) - } - }) - +func TestBootstrap_Preflight(t *testing.T) { t.Run("missing tooling fails naming both managers", func(t *testing.T) { t.Setenv("PATH", t.TempDir()) // no node/pnpm/npm _, err := checkNodeAndPackageManager("", false) @@ -263,9 +240,9 @@ func TestCheckNodeAndPackageManager_NodeVersion(t *testing.T) { func TestRunBootstrapSequence_SkipsSatisfiedSteps(t *testing.T) { var steps []recordedStep - orig := bootstrapRunStep - t.Cleanup(func() { bootstrapRunStep = orig }) - bootstrapRunStep = func(_ context.Context, _ string, argv []string, _ string) error { + orig := runStep + t.Cleanup(func() { runStep = orig }) + runStep = func(_ context.Context, _ string, argv []string, _ string) error { steps = append(steps, recordedStep{argv: argv}) return nil } @@ -273,18 +250,20 @@ func TestRunBootstrapSequence_SkipsSatisfiedSteps(t *testing.T) { if err := os.MkdirAll(filepath.Join(dir, "node_modules"), 0o755); err != nil { t.Fatal(err) } - // node_modules present + already bootstrapped, no force-bake: install and bake skip. - if err := runBootstrapSequence(context.Background(), dir, []string{"llamacpp"}, true, false, npmManager); err != nil { + // node_modules present: install skips, and no bake step exists at all. + if err := runBootstrapSequence(context.Background(), dir, npmManager); err != nil { t.Fatal(err) } + want := []string{"npm run cdk -- bootstrap", "npm run deploy:image", "npm run deploy"} + var got []string for _, s := range steps { - joined := strings.Join(s.argv, " ") - if strings.Contains(joined, "install") { + got = append(got, strings.Join(s.argv, " ")) + if strings.Contains(strings.Join(s.argv, " "), "install") { t.Errorf("node_modules present should skip install, got %v", s.argv) } - if strings.Contains(joined, "bake") { - t.Errorf("alreadyBootstrapped should skip bake, got %v", s.argv) - } + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("commands = %v, want %v", got, want) } } @@ -377,8 +356,7 @@ func TestBootstrap_NpmOverrideDrivesNpmCommands(t *testing.T) { got = append(got, strings.Join(s.argv, " ")) } want := []string{ - "npm install", "npm run cdk -- bootstrap", "npm run deploy:image", - "npm run bake -- llamacpp", "npm run bake -- vllm", "npm run deploy", + "npm install", "npm run cdk -- bootstrap", "npm run deploy:image", "npm run deploy", } if strings.Join(got, "|") != strings.Join(want, "|") { t.Errorf("commands = %v, want %v", got, want) diff --git a/cmd/spinloop/remote_seed.go b/cmd/spinloop/remote_seed.go index 8358d990..a163fc88 100644 --- a/cmd/spinloop/remote_seed.go +++ b/cmd/spinloop/remote_seed.go @@ -18,19 +18,11 @@ import ( // about the same model. func remoteSeedCmd() *cobra.Command { seed := &cobra.Command{ - Use: "seed", - Short: "start, watch and stop model weight seeds", - Args: cobra.ArbitraryArgs, - DisableFlagParsing: true, - SilenceErrors: true, - SilenceUsage: true, - RunE: func(c *cobra.Command, args []string) error { - resolve(c) - if len(args) == 0 { - return fmt.Errorf("usage: spinloop remote seed [args]") - } - return fmt.Errorf("unknown seed subcommand %q (expected start, status, ls or stop)", args[0]) - }, + Use: "seed", + Short: "start, watch and stop model weight seeds", + SilenceErrors: true, + SilenceUsage: true, + RunE: groupFallback, } seed.AddCommand( remoteSeedStartCmd(), diff --git a/cmd/spinloop/remote_seed_test.go b/cmd/spinloop/remote_seed_test.go index e265f35c..441e78e4 100644 --- a/cmd/spinloop/remote_seed_test.go +++ b/cmd/spinloop/remote_seed_test.go @@ -296,15 +296,21 @@ func TestRemoteSeed_NoSeedURLNamesTheValueToAdd(t *testing.T) { func TestRemoteSeed_UnknownSubcommand(t *testing.T) { err := cmdRemoteSeed([]string{"frobnicate"}) - if err == nil || !strings.Contains(err.Error(), "unknown seed subcommand") { + if err == nil || !strings.Contains(err.Error(), "unknown command") { t.Errorf("want an unknown-subcommand error, got %v", err) } } -func TestRemoteSeed_NoSubcommandShowsUsage(t *testing.T) { - err := cmdRemoteSeed([]string{}) - if err == nil || !strings.Contains(err.Error(), "start|status|ls|stop") { - t.Errorf("want usage, got %v", err) +func TestRemoteSeed_NoSubcommandShowsHelp(t *testing.T) { + // Bare: no error — cobra shows the group's own help, with the + // subcommand list generated from the tree. + out := captureStdout(t, func() { + if err := cmdRemoteSeed([]string{}); err != nil { + t.Fatalf("bare seed should show its help, got %v", err) + } + }) + if !strings.Contains(out, "Available Commands") { + t.Errorf("bare seed should list its subcommands:\n%s", out) } } diff --git a/cmd/spinloop/remote_sources.go b/cmd/spinloop/remote_sources.go new file mode 100644 index 00000000..201a2b85 --- /dev/null +++ b/cmd/spinloop/remote_sources.go @@ -0,0 +1,46 @@ +package main + +import ( + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// sourceLocation is where the CDK project's sources live for a run: the +// version-matched ref, the directory they are downloaded to, and whether +// sources from other refs get pruned once the run succeeds. +type sourceLocation struct { + ref string + dir string + prune bool +} + +// resolveSourceLocation applies the ref and --dir rules bootstrap and bake +// share: a ref matched to the running binary (an explicit --ref wins), a +// ref-keyed default directory, and pruning of other refs — except that an +// explicit --dir is the user's own location, neither keyed by ref nor pruned. +func resolveSourceLocation(ref, dir string) (sourceLocation, error) { + resolvedRef := remote.ResolveRef(version, ref) + locDir, err := remote.SourceDir(resolvedRef) + if err != nil { + return sourceLocation{}, err + } + loc := sourceLocation{ref: resolvedRef, dir: locDir, prune: true} + if dir != "" { + loc.dir = dir + loc.prune = false + } + return loc, nil +} + +// pruneSourceCaches drops every ref-keyed source cache except the one this run +// used, so stale-version checkouts do not accumulate. An explicit --dir is +// never pruned. +func pruneSourceCaches(loc sourceLocation) error { + if !loc.prune { + return nil + } + sourceRoot, err := remote.SourceRoot() + if err != nil { + return err + } + return remote.PruneSources(sourceRoot, loc.ref) +} diff --git a/cmd/spinloop/remote_test.go b/cmd/spinloop/remote_test.go index f48766fb..7bf6f414 100644 --- a/cmd/spinloop/remote_test.go +++ b/cmd/spinloop/remote_test.go @@ -101,8 +101,15 @@ func TestRemoteEnvName_Malformed(t *testing.T) { } func TestRemoteDispatch(t *testing.T) { - if err := run([]string{"remote"}); err == nil || !strings.Contains(err.Error(), "usage") { - t.Errorf("bare remote should error with usage, got %v", err) + // Bare remote shows the group's own help — generated from the tree, so + // its subcommand list cannot drift — rather than an error. + out := captureStdout(t, func() { + if err := run([]string{"remote"}); err != nil { + t.Fatalf("bare remote should show its help, got %v", err) + } + }) + if !strings.Contains(out, "bootstrap") || !strings.Contains(out, "bake") { + t.Errorf("bare remote help should name its subcommands, got:\n%s", out) } if err := run([]string{"remote", "bogus"}); err == nil || !strings.Contains(err.Error(), "bogus") { t.Errorf("unknown subcommand should error, got %v", err) @@ -527,13 +534,17 @@ func TestRemoteRestart_WakeFailureReportsRecovery(t *testing.T) { // The parent fallback names restart in both its usage line and its // unknown-subcommand list, so a mistyped or bare `remote` points to it. -func TestRemote_RestartInUsageAndUnknownList(t *testing.T) { +func TestRemote_RestartInGeneratedHelp(t *testing.T) { isolateConfig(t) - if err := run([]string{"remote"}); err == nil || !strings.Contains(err.Error(), "restart") { - t.Errorf("bare remote usage should name restart, got %v", err) - } - if err := run([]string{"remote", "bogus"}); err == nil || !strings.Contains(err.Error(), "restart") { - t.Errorf("unknown-subcommand list should name restart, got %v", err) + // A regression from when the usage was a hand-rolled list: restart had + // to be added to two places by hand. The help now comes from the tree. + out := captureStdout(t, func() { + if err := run([]string{"remote"}); err != nil { + t.Fatalf("bare remote should show its help, got %v", err) + } + }) + if !strings.Contains(out, "restart") { + t.Errorf("bare remote help should name restart, got:\n%s", out) } } diff --git a/docs/commands/remote.md b/docs/commands/remote.md index 72c7ccce..c1cc4cdb 100644 --- a/docs/commands/remote.md +++ b/docs/commands/remote.md @@ -6,6 +6,7 @@ while you're using it. ```sh spinloop remote bootstrap # once per account: deploy the control plane +spinloop remote bake # bake the runner AMI(s) an environment runs from spinloop remote deploy # create an endpoint (environment) and tell it what to serve spinloop remote start # boot it; prints the exports your agent needs (progress on stderr) spinloop remote status # is it up? is it healthy? @@ -26,15 +27,14 @@ enough that the pause is over. Before any endpoint can run, the account-level control plane has to exist — much like `cdk bootstrap`. `spinloop remote bootstrap` does it once per account: it downloads the `remote/` CDK project (version-matched to your binary) -and deploys the control plane — the EC2 Image Builder pipelines and baked AMIs, -the lifecycle Lambdas, and the shared weights bucket, roles and VPC — publishing -them as CloudFormation outputs that `spinloop remote deploy` discovers later. +and deploys the control plane — the EC2 Image Builder pipelines, the lifecycle +Lambdas, and the shared weights bucket, roles and VPC — publishing them as +CloudFormation outputs that `spinloop remote deploy` discovers later. It bakes +**no** AMIs — that is the separate `spinloop remote bake` step below. ```sh spinloop remote bootstrap # shows a consent plan, then deploys spinloop remote bootstrap --dry-run # print the plan and do nothing -spinloop remote bootstrap --runners llamacpp # bake only one engine's AMI -spinloop remote bootstrap --wait # block until the AMI bake(s) finish spinloop remote bootstrap --package-manager npm # use npm instead of pnpm ``` @@ -50,7 +50,28 @@ By default bootstrap uses `pnpm` and falls back to `npm` when `pnpm` isn't on th path, logging which one it picked. To pin the choice, pass `--package-manager` (`pnpm` or `npm`) or set `SPINLOOP_REMOTE_PACKAGE_MANAGER`; the flag wins over the env var. A pinned manager that isn't installed fails the preflight rather than -falling back. +falling back. `spinloop remote bake` honours the same flags. + +## Baking the AMIs + +Each engine runs from a baked AMI (driver + engine, no model). +`spinloop remote bake` starts a bake for each runner you name — both `llamacpp` +and `vllm` when you name none — and **waits** until the AMI(s) are available, so +the command returns at the point `spinloop remote deploy` can go: + +```sh +spinloop remote bake # bake both engines' AMIs; waits (~20-40 min) +spinloop remote bake llamacpp # bake one engine's AMI +spinloop remote bake --no-wait # return once the bakes are queued +``` + +Bakes are slow (a builder instance runs for 20–40 minutes) and independent of +the weight seed, so `--no-wait` lets them run in parallel — the command prints +how to check on them. A bake deploys nothing: it needs the control plane's +Image Builder pipelines, so if the control plane isn't deployed it fails telling +you to run `spinloop remote bootstrap` first. Re-bake only when the engine +version or the driver changes; the model is **not** baked in, and a new AMI is +picked up automatically once it is available. ## The usual flow @@ -104,7 +125,7 @@ needs it, never merely because the Spinloop was read. A bare name (no slash, no per-user registry at `~/.config/spinloop/remotes//remote.json`. This keeps deployment state per-user and per-instance: two projects name two environments without clobbering, and only the name — not the URLs — lives in the committed -Spinloop. `spinloop remote bootstrap` registers an environment for you; you can also +Spinloop. `spinloop remote deploy` registers an environment for you; you can also create one by hand. Given no path, `spinloop remote` reads the Spinloop `SPINLOOP_ALIAS` names, or @@ -319,15 +340,18 @@ something to reach for by habit. | `--reseed` | `deploy` only: re-fetch the weights even if they are already in S3 | | `--spinloop-version` | `deploy` only: the spinloop release fresh boots install (default: the latest published release) | +`bootstrap` and `bake` have their own too (`--ref`, `--dir`, `--region`, +`--package-manager`, and `--no-wait` on bake) — see their sections above. `logs` has its own set — see [reading the logs](#reading-the-logs). ## Notes -- Every subcommand takes an optional Spinloop path, a - [registered alias](alias.md), or a URL. Given none, it uses the alias - `SPINLOOP_ALIAS` names, and failing that `./Spinloop`. +- `bootstrap` and `bake` are account-level and take no Spinloop: the control + plane and the AMIs are shared by every environment. - `deploy` always needs a Spinloop — it's the thing being deployed. The others - fall back to your per-user config. + take an optional Spinloop path, a [registered alias](alias.md), or a URL. + Given none, they use the alias `SPINLOOP_ALIAS` names, and failing that + `./Spinloop`, and failing that your per-user config. - `deploy_url` is optional: a config written before `deploy` existed still works for `start`, `stop`, and `status`. - Only a self-hosted engine can be deployed (`llamacpp` or `vllm`). A hosted diff --git a/docs/env-vars.md b/docs/env-vars.md index ce507f8e..ea44c196 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -29,7 +29,7 @@ from the environment or a `.env` beside the Spinloop — never written into an | `SPINLOOP_REMOTE_ENV_URL` | Override the env Lambda Function URL. | | `SPINLOOP_REMOTE_UPDATE_URL` | Override the update Lambda Function URL (drives `keep`). | | `SPINLOOP_REMOTE_REGION` | Override the AWS region (else `AWS_REGION`, else the region in the Function URL host). | -| `SPINLOOP_REMOTE_PACKAGE_MANAGER` | Pin the package manager (`pnpm`/`npm`) `spinloop remote bootstrap` uses. | +| `SPINLOOP_REMOTE_PACKAGE_MANAGER` | Pin the package manager (`pnpm`/`npm`) `spinloop remote bootstrap` and `bake` use. | These let the remote commands run without a `remote.json` on disk — the config can come entirely from the environment. `spinloop remote logs` is the diff --git a/internal/remote/bake.go b/internal/remote/bake.go index 4e3afb79..1e5c7a73 100644 --- a/internal/remote/bake.go +++ b/internal/remote/bake.go @@ -11,7 +11,7 @@ import ( // BakedRunners reports, for the account's own images, which runners already // have a runtime AMI baked — read from the tags the Image Builder distribution // applies (`cloud-vm-llm:role=runtime-ami`, `cloud-vm-llm:runner=`). It -// lets `spinloop remote bootstrap --wait` tell when a bake has finished. +// lets `spinloop remote bake` tell when a bake has finished. func BakedRunners(ctx context.Context, cfg aws.Config) (map[string]bool, error) { out, err := ec2.NewFromConfig(cfg).DescribeImages(ctx, &ec2.DescribeImagesInput{ Owners: []string{"self"}, diff --git a/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/.openspec.yaml b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/.openspec.yaml new file mode 100644 index 00000000..b4b3ece7 --- /dev/null +++ b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-01 diff --git a/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/design.md b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/design.md new file mode 100644 index 00000000..19ffd3b8 --- /dev/null +++ b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/design.md @@ -0,0 +1,169 @@ +## Context + +`spinloop remote bootstrap` (cmd/spinloop/remote_bootstrap.go) downloads the +version-matched `remote/` CDK sources into a ref-keyed cache under the user +config dir (`internal/remote/source.go`), picks a package manager, prints a +consent plan, and runs: install → `cdk bootstrap` → `deploy:image` → +`bake ` (per `--runners`, default both) → `deploy`, optionally +blocking on the bake with `--wait`. The bake is the slow, asynchronous, +re-runnable part (a 20–40 min Image Builder build), and it is orthogonal to +the control plane it currently rides on. `pnpm bake ` +(remote/scripts/bake) already exists in the CDK project and only needs the +pipeline `deploy:image` created — no deploy of its own. + +## Goals / Non-Goals + +**Goals:** + +- Bootstrap becomes a clean, fast, consent-gated control-plane step with no + bake on its success path, and its success output signposts the bake. +- The bake is a first-class `spinloop remote bake` command that reuses + bootstrap's existing source/package-manager machinery rather than + reimplementing it. +- No change to the TypeScript CDK project. + +**Non-Goals:** + +- Watching the bake from `deploy`/`start` (they keep failing today when no AMI + exists; nothing changes there). +- Any new CDK context, pipeline, or stack behaviour. +- Changing how the runtime Lambda picks up a new AMI (tags, unchanged). + +## Decisions + +### A new `spinloop remote bake [runner...]` command, not a printed pnpm recipe + +The signpost is a command the user can run as-is, not instructions to find the +sources cache and run a package script. Bootstrap's sources land in +`/cdk//` — a path users do not know, which bootstrap prunes of +other refs after success, and which needs AWS credentials arranged for the +`aws` CLI anyway. A CLI command keeps the whole journey (`bootstrap` → `bake` +→ `deploy`) inside spinloop and makes the signpost one line. +Alternatives rejected: printing manual `pnpm bake` steps (burdens the user with +the cache path, the package manager, and credentials); keeping the bake on +bootstrap behind an opt-in flag (the issue asks for it not to trigger at all). + +### Bake reuses bootstrap's source machinery and cache policy + +Bake resolves the same version-matched ref (`remote.ResolveRef`), downloads +into the same ref-keyed default location (a present checkout is a no-op in +`DownloadRemote`), honours the same `--ref`/`--dir` and the same +`--package-manager`/`SPINLOOP_REMOTE_PACKAGE_MANAGER` resolution, and prunes +other refs after success when it used the default location — so both commands +share one cache policy instead of two that could drift. It then runs `install` +(only when `node_modules` is absent) and one `pnpm run bake ` per +runner. +Alternative rejected: requiring a prior bootstrap to have left sources in +place — a newer spinloop release between the two commands would break bake, +while the download path already costs nothing when the checkout exists. + +### Bake deploys nothing, and fails early when the control plane is absent + +`pnpm bake` needs the Image Builder pipeline `deploy:image` created. Without a +control plane, the script's own error ("Run 'pnpm deploy:image' first") points +at a manual CDK step CLI users should not run. Bake therefore resolves AWS +credentials and checks the control-plane stack (the same +`ControlPlaneStackDeployed` call bootstrap's plan uses) before starting any +bake, and fails naming `spinloop remote bootstrap` as the first step. AWS +credentials are a hard requirement for bake in all forms: the check needs +them, and the `aws` CLI inside the bake script does. Preflight is the same +shape as bootstrap's: Node 22+, a package manager on PATH, resolvable +credentials. + +### No consent gate on bake; bootstrap keeps its gate + +Bootstrap's plan-and-confirm exists because it creates shared, account-level +resources with ongoing cost. Bake is the user's explicit, narrow action — they +named the runners — and touches only an Image Builder build (a builder +instance for 20–40 min). A one-line note (runners, region, expected +duration) precedes the steps; no prompt. +Alternative rejected: a confirm prompt — it would gate the very command the +issue asks to make the obvious next step. + +### Runners are positional on bake, defaulting to both + +`spinloop remote bake llamacpp` reads naturally in the signpost and matches +how the group's other commands take their primary noun as a positional (the +environment on `deploy`, `start`, …). Absent arguments, both runners bake — +the same default bootstrap had, so a user who wants the old end state runs one +extra command with no arguments. Validation reuses the provider→runner mapping +(`runnerFor`'s accepted set). +Alternative rejected: carrying over bootstrap's `--runners llamacpp,vllm` +flag — a flag for the command's primary input is clunkier and would leave two +spellings for the same list. + +### Bootstrap loses `--runners`, `--wait`, `--force-bake` + +- `--runners`: its only real effect was the bake loop. The `cdk.json` + `context.runners` write it also performed is dead — the CDK reads no such + key, and the image stack always creates both runners' pipelines — so the + write goes too. +- `--wait`: nothing is left to wait for. The waiting moves to bake — and + becomes its default behaviour, with a `--no-wait` hand-off (below); + `waitForBake` itself is reused unchanged (polling `BakedRunners` every + 60 s under a 60-minute bound). +- `--force-bake`: with no automatic bake, "re-bake even if already + bootstrapped" has no referent — running `bake` is already the re-bake. + +The success message becomes the signpost: the account is bootstrapped; next, +`spinloop remote bake ` (the bake is what an environment needs before +its first start), then `spinloop remote deploy `. The plan's command list +and resource bullet drop the bakes ("Image Builder pipelines" — no "and baked +AMIs"). + +### Bake blocks until the AMI is available, with a `--no-wait` hand-off + +The step after bake is `deploy` — and a first start needs the AMI — so the +default flow should finish at the point the user can go: bake queues the +build(s), then blocks on the same bounded `waitForBake` poll bootstrap's +`--wait` used. `--no-wait` returns as soon as the bakes are queued, reporting +how to check on them (the `pnpm bake` script already prints the build ARN and +the progress command), which keeps the documented parallelism of a bake and a +weight seed. +Alternative rejected: keeping bootstrap's opt-in `--wait` shape — it would +leave the common path handing off to the Image Builder console for 20–40 +minutes. + +### Shared helpers stay in package main, seams stay per-concern + +`parseRunners`, `waitForBake`, the package-manager machinery +(`packageManager`, `selectPackageManager`, `resolvePackageManagerName`, +`checkNodeAndPackageManager`), the step runner, and the ref/dir/download/prune +sequence are shared between the two commands (the bootstrap-prefixed names lose +their prefix where they become shared). The test seams (the `*Fn`/`Step` +package variables) are shared too, so `remote_bake_test.go` can drive the bake +flow hermetically exactly the way `remote_bootstrap_test.go` drives bootstrap +today. `BakedRunners` stays in `internal/remote` (comment updated); the CDK +project is untouched. + +## Risks / Trade-offs + +- **Scripts calling `bootstrap --runners/--wait/--force-bake` break** → + BREAKING by intent (issue #139). pflag rejects the removed flags with an + unknown-flag error naming nothing; release notes and the updated docs carry + the `bake` replacement, and bootstrap's own success output now says what to + run next. +- **A new-account journey is one step longer** → the point of the change. + `deploy`/`start` with no AMI present fail as they do today; the signpost + puts `bake` in front of `deploy` so the order is discoverable. +- **Bake's stack check is one extra AWS call per run** → a single + `DescribeStacks`, negligible against a 20–40-minute build. +- **Bake pruning other ref caches could surprise a user juggling binary + versions** → same policy bootstrap already applies on success; an explicit + `--dir` opts out, and a re-download of a pruned ref is cheap. +- **A failed bake only surfaces as the 60-minute wait timeout** — + `BakedRunners` sees available AMIs, not failed builds, so a broken driver + install waits out the bound before reporting. → The bake script's streamed + output (build ARN, progress command) is on the terminal the whole time, so + the operator can see the failure in parallel; failing fast on the build + state is a follow-up, not part of this split. + +## Migration Plan + +No data or infrastructure migration — pure CLI surface. Rollback is reverting +the change; nothing on AWS is affected by either direction (accounts that +already bootstrapped keep their pipelines and AMIs). + +## Open Questions + +None — the spec-level behaviour is settled by the proposal and deltas. diff --git a/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/proposal.md b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/proposal.md new file mode 100644 index 00000000..2b71e76f --- /dev/null +++ b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/proposal.md @@ -0,0 +1,57 @@ +## Why + +`spinloop remote bootstrap` currently kicks off the AMI bakes (both runners by +default) as part of the once-per-account control-plane setup. A bake is slow +(~20–40 min), asynchronous, re-runnable, and sometimes unnecessary — yet it is +wired into bootstrap's success path, its consent plan, and its flags +(`--runners`, `--wait`, `--force-bake`). The control plane should be a clean, +fast, self-contained step; the bake is a separate concern that the user should +choose to run (issue #139). + +## What Changes + +- New `spinloop remote bake [runner...]` command: starts an AMI bake for each + requested runner (default: both `llamacpp` and `vllm`). It reuses bootstrap's + CDK-source machinery — the same version-matched download, ref-keyed cache, + package-manager selection, `--ref`/`--dir`/`--package-manager` flags — and + runs `pnpm bake ` per runner. Bake blocks until the baked AMI(s) are + available; a `--no-wait` flag returns as soon as the bakes are queued, + reporting how to check on them. +- **BREAKING** `spinloop remote bootstrap` no longer starts any AMI bake: the + `--runners`, `--wait`, and `--force-bake` flags are removed, the sequence is + just install → `cdk bootstrap` → `deploy:image` → `deploy`, and the consent + plan no longer lists bakes. Bootstrap's success output signposts + `spinloop remote bake` as the next step, ahead of `spinloop remote deploy`. +- Docs follow: the bootstrap section of `docs/commands/remote.md` and + `remote/README.md` describe the split flow. + +## Capabilities + +### New Capabilities + +(none — the bake lives under the existing provisioning capability) + +### Modified Capabilities + +- `endpoint-provisioning`: bootstrap stops baking AMIs (and loses + `--runners`/`--wait`/`--force-bake`); new requirement defining + `spinloop remote bake` as the separate, signposted bake command with its own + `--wait` and the shared source/package-manager machinery. +- `remote-endpoint`: the remote command group's subcommand list gains `bake`. + +## Impact + +- `cmd/spinloop/remote_bootstrap.go` — flag and sequence removal, plan and + success-message changes; `cmd/spinloop/remote_bake.go` (new) — the bake + command; `cmd/spinloop/remote_sources.go` (new) — the shared source + resolution; `waitForBake` and the step seams are used by both. +- `internal/remote/bake.go` — `BakedRunners` now serves bake's default wait + instead of `bootstrap --wait` (comment change only). +- Tests: `cmd/spinloop/remote_bootstrap_test.go` updated, `remote_bake_test.go` + new; coverage stays ≥ 80%. +- Docs: `docs/commands/remote.md`, `remote/README.md`, `docs/env-vars.md`. +- No change to the TypeScript CDK project in `remote/` — `pnpm bake` already + exists and is what the new command drives. +- User journey for a new account: bootstrap → bake → deploy (previously + bootstrap → deploy, with the bake a side effect of bootstrap). Accounts that + already bootstrapped with an older binary have their AMIs and are unaffected. diff --git a/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/specs/endpoint-provisioning/spec.md b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/specs/endpoint-provisioning/spec.md new file mode 100644 index 00000000..52df410b --- /dev/null +++ b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/specs/endpoint-provisioning/spec.md @@ -0,0 +1,219 @@ +## ADDED Requirements + +### Requirement: AMI bake is a separate command + +The system SHALL provide `spinloop remote bake`, which starts an AMI bake for +each runner named as a positional argument — `llamacpp` and `vllm` — defaulting +to both when none are named. It SHALL drive the same CDK project that +bootstrap orchestrates, with the same version-matched source download into the +same ref-keyed default location, the same package-manager selection and +override, and `--ref` and `--dir` flags matching bootstrap's. Bake SHALL NOT +deploy any stack; when the control-plane stack is not deployed, it SHALL fail +before starting any bake, naming `spinloop remote bootstrap` as the step to run +first. Bake SHALL block until every requested runner's AMI is available; a +`--no-wait` flag SHALL return as soon as the bakes are queued, reporting how to +check on them, rather than blocking for the bake duration. + +#### Scenario: Default bake covers both runners + +- **WHEN** the user runs `spinloop remote bake` with no arguments +- **THEN** a bake is started for both `llamacpp` and `vllm` + +#### Scenario: A single runner is baked + +- **WHEN** the user runs `spinloop remote bake llamacpp` +- **THEN** only the `llamacpp` AMI bake is started + +#### Scenario: An unknown runner is rejected + +- **WHEN** the user names a runner that is neither `llamacpp` nor `vllm` +- **THEN** bake fails before starting any bake, naming the accepted runners + +#### Scenario: No control plane + +- **WHEN** the control-plane stack is not deployed and bake runs +- **THEN** it fails before starting any bake, saying to run + `spinloop remote bootstrap` first + +#### Scenario: Bake waits by default + +- **WHEN** the user runs `spinloop remote bake` without `--no-wait` +- **THEN** the command blocks until the requested runners' AMIs are available + before finishing + +#### Scenario: Handing off with --no-wait + +- **WHEN** the user passes `--no-wait` +- **THEN** the command returns as soon as the bakes are queued, reporting how + to check on them, rather than blocking for the bake duration + +### Requirement: Idempotent bootstrap + +Bootstrap SHALL be safe to re-run: it SHALL skip the package-manager install when +dependencies are present and `cdk bootstrap` when the account and region are already +bootstrapped, and SHALL not redeploy a control-plane stack that is unchanged. Because it +touches only control plane and never a live instance, re-running SHALL NOT +require any override. + +#### Scenario: Re-running skips satisfied steps + +- **WHEN** bootstrap is re-run +- **THEN** it skips installation and CDK bootstrap that are already done and + no-ops the unchanged control-plane stack, without requiring an override + +### Requirement: Bootstrap collects only the shared-secret token + +Bootstrap SHALL collect the one control-plane setting the CDK has no default +for and write it where the CDK reads it: an optional Hugging Face token for +the shared secret used when seeding gated weights. Which runner AMIs to bake +is not a bootstrap setting — the engine is a per-environment choice made at +`deploy`, and the runners are named by `spinloop remote bake` itself. The +allowed ingress CIDR is also per-environment and belongs to `deploy`, not here. + +#### Scenario: Runners are not a bootstrap setting + +- **WHEN** the user runs bootstrap +- **THEN** no runner selection is requested or written, since the runners are + named at `spinloop remote bake` + +#### Scenario: The allowed CIDR is not a bootstrap setting + +- **WHEN** the user runs bootstrap +- **THEN** no ingress CIDR is requested or written, since it is scoped per + environment at `spinloop remote deploy` + +## MODIFIED Requirements + +### Requirement: Bootstrap deploys the control plane + +The system SHALL provide `spinloop remote bootstrap`, which deploys the +account-level control plane that every remote environment reuses — the EC2 +Image Builder pipelines, the environment-aware lifecycle Lambdas and their IAM, +and the shared S3 weights bucket, IAM roles and VPC — by obtaining the CDK +project shipped in `remote/` and driving its deploy of the control-plane stack. +Bootstrap SHALL NOT start any AMI bake; the bake is a separate +`spinloop remote bake` step. Bootstrap SHALL NOT create any Elastic IP or EC2 +instance, and SHALL NOT register an environment; those belong to +`spinloop remote deploy`. Bootstrap SHALL NOT reimplement the infrastructure; +it SHALL orchestrate the existing CDK project. On success, bootstrap SHALL +signpost `spinloop remote bake` as the next step, ahead of +`spinloop remote deploy`. + +#### Scenario: A successful bootstrap yields the control plane + +- **WHEN** `spinloop remote bootstrap` completes +- **THEN** the control-plane stack is deployed — Image Builder pipelines, the + lifecycle Lambdas, and the shared bucket/roles/VPC — with no Elastic IP or + instance created and no AMI bake started + +#### Scenario: Bootstrap signposts the bake + +- **WHEN** `spinloop remote bootstrap` completes +- **THEN** its output names `spinloop remote bake` as the next step, ahead of + `spinloop remote deploy` + +#### Scenario: Orchestration stops on a failed step + +- **WHEN** any step in the sequence fails +- **THEN** bootstrap stops and reports which step failed rather than continuing + +### Requirement: Version-matched CDK sources + +Bootstrap and `spinloop remote bake` SHALL obtain the CDK project by +downloading the `remote/` tree from the project repository at a reference +matching the running binary's version, so the infrastructure matches the CLI +driving it. A `--ref` flag SHALL override the reference, and a `--dir` flag +SHALL override where the sources are placed (defaulting under the user config +directory). For a development build with no release version, bootstrap and +bake SHALL fall back to a documented default reference. The CDK sources SHALL +NOT be embedded in the binary, since a package-manager install is required at +runtime regardless. + +The default source location SHALL be keyed by the resolved reference, so a re-run +at the same version reuses its sources while a different binary version downloads +fresh. On a successful bootstrap or bake using the default location, sources +from other references SHALL be pruned. An explicit `--dir` SHALL be treated as +the user's own location: neither keyed by reference nor pruned. + +#### Scenario: Sources match the binary version + +- **WHEN** a released `spinloop` runs bootstrap with no `--ref` +- **THEN** it downloads the `remote/` sources at the tag matching its version + +#### Scenario: Development build falls back + +- **WHEN** a `dev` build runs bootstrap with no `--ref` +- **THEN** it uses the documented fallback reference rather than guessing + +#### Scenario: A new version does not reuse stale sources + +- **WHEN** bootstrap or bake runs from a binary whose resolved reference + differs from a previously downloaded one in the default location +- **THEN** it downloads sources for the new reference, and on success the + superseded reference's sources are pruned + +### Requirement: A Node package manager is selected, overridable, and logged + +Bootstrap and `spinloop remote bake` SHALL select the Node package manager they +drive the CDK project with. Absent an explicit choice, they SHALL auto-detect by +PATH lookup, preferring `pnpm` and falling back to `npm` when `pnpm` is not on +the path. The user MAY override +the selection with a `--package-manager` flag or an `SPINLOOP_REMOTE_PACKAGE_MANAGER` +environment variable, whose only accepted values are `pnpm` and `npm`; the flag +SHALL take precedence over the environment variable, which SHALL take precedence +over auto-detection. An unrecognised override value SHALL be rejected with an +error naming the accepted values. The selected manager SHALL be used consistently +for every Node step (install, `cdk`, `deploy:image`, `deploy`) and +reflected in the printed plan. Before the steps run, bootstrap SHALL log which +package manager it selected, so the run is self-explanatory. When auto-detecting +and both managers are present, `pnpm` SHALL win; the choice SHALL NOT depend on +which lockfiles are present, since the `remote/` project ships a `pnpm` lockfile +yet runs correctly under either manager. + +#### Scenario: pnpm is preferred when auto-detecting + +- **WHEN** no override is given and both `pnpm` and `npm` are on the path +- **THEN** bootstrap selects `pnpm`, logs that it is using `pnpm`, and runs every + step with `pnpm` + +#### Scenario: npm is used when pnpm is absent + +- **WHEN** no override is given, `pnpm` is not on the path, but `npm` is +- **THEN** bootstrap selects `npm`, logs that it is using `npm`, and runs every + step with `npm` using npm's argument conventions + +#### Scenario: An explicit override is honoured + +- **WHEN** the user passes `--package-manager npm` (or sets + `SPINLOOP_REMOTE_PACKAGE_MANAGER=npm`) while `pnpm` is also present +- **THEN** bootstrap uses `npm` regardless of auto-detection, and the flag wins + if both the flag and the environment variable are set + +#### Scenario: An unrecognised override is rejected + +- **WHEN** the override value is neither `pnpm` nor `npm` +- **THEN** bootstrap fails with an error naming the accepted values, before + deploying anything + +## REMOVED Requirements + +### Requirement: Control-plane settings are collected + +**Reason**: The per-runner setting it collected (`--runners`) existed to feed +the bake, which is now a separate command that names its own runners; what +remains is only the Hugging Face token. + +**Migration**: Runner selection moves to `spinloop remote bake`'s positional +arguments (defaulting to both runners); what bootstrap still collects is +described by the re-added "Bootstrap collects only the shared-secret token" +requirement. + +### Requirement: Idempotent bootstrap with an asynchronous bake + +**Reason**: Bootstrap no longer starts the AMI bake, so there is no +asynchronous bake to hand off or wait for; its re-run idempotency remains, +described without the bake. + +**Migration**: Re-run behaviour is carried by the "Idempotent bootstrap" +requirement; baking and waiting for it is `spinloop remote bake` (which waits +by default), per the "AMI bake is a separate command" requirement. diff --git a/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/specs/remote-endpoint/spec.md b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/specs/remote-endpoint/spec.md new file mode 100644 index 00000000..0a638628 --- /dev/null +++ b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/specs/remote-endpoint/spec.md @@ -0,0 +1,123 @@ +## MODIFIED Requirements + +### Requirement: Remote command group + +The system SHALL provide a `remote` command group with the subcommands +`bootstrap`, `bake`, `start`, `stop`, `restart`, `status`, `deploy`, `ls`, +`metrics`, and `keep`. `start`, `stop`, `restart`, `status`, `metrics` and +`deploy` each +take an optional Spinloop path: +`start` SHALL boot the endpoint and block until it is serving, then perform a +quick TCP probe of the inference endpoint — if the probe fails, a warning is +printed to stderr explaining the network mismatch (see the Remote Start Probe +specification) — and finally print the base URL and API key as shell exports; +`start` SHALL also accept a `--keep DURATION` flag that sets the instance +retention deadline to `now + DURATION`, preventing the idle sweep from +terminating it before that time (see the Remote Keep specification); +`stop` SHALL stop it immediately rather than waiting for its idle timer; +`restart` SHALL stop the endpoint in the manner of a pause — without +terminating it, so its boot disk, its weights and its stable address are +preserved — and SHALL immediately start it again, blocking until it is serving +and reporting progress as `start` does (see the Reporting a start in progress +specification); `restart` SHALL accept a `--force` flag with a `-F` short form +that, when set, performs the stop without first asking the engine to shut down +(see the Endpoint Lifecycle specification for forced stops); +`status` SHALL report instance state and endpoint health without side effects +and SHALL NOT perform any TCP probe, and SHALL include the `Retain-Until` +deadline when the instance has an active retention tag; +`keep` SHALL set the `Retain-Until` tag on the environment's instance for the +given duration, without starting or stopping the instance (see the Remote Keep +specification); `metrics` SHALL report instance state, token usage, resource +consumption, and GPU information for a running instance; `deploy` SHALL set +what the endpoint serves. `ls` SHALL list the registered remote environments +(see the Remote Environments specification). `bootstrap` SHALL stand up the +account-level AWS control plane (once per account) by obtaining and driving the +CDK project, and takes its own flags rather than a Spinloop path (see the +Endpoint Provisioning specification). `bake` SHALL start an AMI bake for each +runner named, and takes runner names rather than a Spinloop path (see the +Endpoint Provisioning specification). An unrecognised subcommand SHALL fail +naming the accepted ones. + +#### Scenario: Starting the endpoint + +- **WHEN** the user runs `spinloop remote start` and the endpoint reports ready +- **THEN** the base URL and API key are printed as `export` lines + +#### Scenario: Starting warns when the network is not admitted + +- **WHEN** the user runs `spinloop remote start` and the endpoint reports ready + but the TCP probe to the inference port fails +- **THEN** a warning is printed to stderr with a remediation command, and the + command still exits 0 + +#### Scenario: Starting with a keep flag + +- **WHEN** the user runs `spinloop remote start --keep 4h` and the endpoint reports ready +- **THEN** the base URL and API key are printed as `export` lines, and the + instance retention deadline is set to 4 hours from now + +#### Scenario: Waiting through a cold start + +- **WHEN** the endpoint reports that it is still starting +- **THEN** the command waits and retries until it is ready or the timeout + passes, rather than failing on the first attempt + +#### Scenario: Restarting the endpoint + +- **WHEN** the user runs `spinloop remote restart` for a running environment and + the endpoint reports ready again +- **THEN** the instance was stopped and re-woken without being terminated, the + command blocked until the model was serving again, and the environment's + address is the one its configuration records + +#### Scenario: Forcing a restart skips the engine stop + +- **WHEN** the user runs `spinloop remote restart --force` (or `-F`) +- **THEN** the instance is stopped without the engine being asked to shut down + first, and the command then blocks until the model is serving again + +#### Scenario: Restarting a stopped endpoint starts it + +- **WHEN** the user runs `spinloop remote restart` for an environment whose instance is already stopped +- **THEN** the instance is re-woken rather than replaced, and the command blocks + until the model is serving again, as with a plain start + +#### Scenario: A failed re-wake says how to recover + +- **WHEN** the stop half of a restart has taken effect but the wake fails +- **THEN** the command fails saying the instance is stopped and that + `spinloop remote start` will bring it back + +#### Scenario: Listing environments + +- **WHEN** the user runs `spinloop remote ls` +- **THEN** the registered environments are listed rather than any endpoint being + contacted + +#### Scenario: Setting a keep deadline + +- **WHEN** the user runs `spinloop remote keep 2h` +- **THEN** the instance retention tag is set and the deadline is reported + +#### Scenario: Metrics reports instance figures + +- **WHEN** the user runs `spinloop remote metrics` with a running instance +- **THEN** token counts, resource usage, and GPU information are displayed + +#### Scenario: Bootstrap is a recognised subcommand + +- **WHEN** the user runs `spinloop remote bootstrap` +- **THEN** the command is dispatched to the provisioning flow rather than + reported as unknown + +#### Scenario: Bake is a recognised subcommand + +- **WHEN** the user runs `spinloop remote bake llamacpp` +- **THEN** the command is dispatched to the bake flow rather than reported as + unknown + +#### Scenario: Unknown subcommand + +- **WHEN** the user runs `spinloop remote frobnicate` +- **THEN** the command fails listing the accepted subcommands, which include + `bootstrap`, `bake`, `metrics`, and `keep` diff --git a/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/tasks.md b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/tasks.md new file mode 100644 index 00000000..7dd465e1 --- /dev/null +++ b/openspec/changes/archive/2026-09-02-split-bake-from-bootstrap/tasks.md @@ -0,0 +1,33 @@ +## 1. Share the bootstrap machinery + +- [x] 1.1 Rename the generic bootstrap-prefixed seams and helpers in `cmd/spinloop/remote_bootstrap.go` to shared names both commands use (the step runner and its `*Fn` test seams, `waitForBake`), keeping them as package variables so tests stay hermetic +- [x] 1.2 Extract the source-resolution sequence (resolve ref, default/explicit dir, download, prune-other-refs-on-success) into a shared helper that bootstrap and bake both call + +## 2. Strip the bake from bootstrap + +- [x] 2.1 Remove the `--runners`, `--wait`, and `--force-bake` flags and the bake loop from `runBootstrapSequence` (sequence becomes install → `cdk bootstrap` → `deploy:image` → `deploy`), and drop the now-dead `cdk.json` `context.runners` write +- [x] 2.2 Update the plan rendering (no bake lines; the Image Builder bullet no longer claims baked AMIs) and the success output to signpost `spinloop remote bake` as the next step ahead of `spinloop remote deploy`; update the command's `Short`/`Long` + +## 3. Add `spinloop remote bake` + +- [x] 3.1 New `cmd/spinloop/remote_bake.go`: `bake [runner...]` with positional runners (default both, validated against the accepted runner set), a `--no-wait` flag (the wait is the default), and `--ref`/`--dir`/`--region`/`--package-manager` flags matching bootstrap's; register it on the remote command group and give the positional argument runner-name completion +- [x] 3.2 Bake body: preflight (Node 22+, a package manager, resolvable AWS credentials), fail early naming `spinloop remote bootstrap` when the control-plane stack is not deployed, shared source resolution, run install-if-needed plus one bake step per runner, `waitForBake` polling by default (skipped with `--no-wait`, which reports how to check on the builds), and prune other refs on success when the default location was used + +## 4. internal/remote + +- [x] 4.1 Update the `BakedRunners` comment: it now serves bake's default wait rather than `bootstrap --wait` + +## 5. Tests + +- [x] 5.1 Update `cmd/spinloop/remote_bootstrap_test.go`: expected sequence without bake steps, plan output without bakes, the success signpost, drop the `--runners`/`--force-bake`/`--wait` tests and the `runners` context-write assertion +- [x] 5.2 New `cmd/spinloop/remote_bake_test.go` driving the shared seams hermetically: default both runners, a single runner, an unknown runner rejected before anything runs, no control plane failing early naming bootstrap, the default wait polling to available (and the timeout path), `--no-wait` returning once the bakes are queued, and default-location prune vs an explicit `--dir` + +## 6. Docs + +- [x] 6.1 `docs/commands/remote.md`: rewrite the bootstrap section (drop the `--runners`/`--wait` examples) and add a `bake` section (usage, runner default, `--no-wait`, the fail-early when the control plane is missing) +- [x] 6.2 `remote/README.md`: the Deploy section's journey becomes bootstrap → bake → deploy; fix the inline comment that says bootstrap does "control-plane stack + pipelines + bakes" +- [x] 6.3 `docs/env-vars.md`: note that `SPINLOOP_REMOTE_PACKAGE_MANAGER` applies to `bootstrap` and `bake` alike + +## 7. Verification + +- [x] 7.1 `gofmt -w ./...`, `go vet ./...`, and `go test ./... -cover` green with total coverage ≥ 80% diff --git a/openspec/specs/endpoint-provisioning/spec.md b/openspec/specs/endpoint-provisioning/spec.md index e191915f..cccac352 100644 --- a/openspec/specs/endpoint-provisioning/spec.md +++ b/openspec/specs/endpoint-provisioning/spec.md @@ -11,18 +11,29 @@ endpoints is provisioned through `spinloop remote bootstrap`. The system SHALL provide `spinloop remote bootstrap`, which deploys the account-level control plane that every remote environment reuses — the EC2 -Image Builder pipelines and baked AMIs, the environment-aware lifecycle Lambdas -and their IAM, and the shared S3 weights bucket, IAM roles and VPC — by obtaining -the CDK project shipped in `remote/` and driving its deploy of the control-plane stack. -Bootstrap SHALL NOT create any Elastic IP or EC2 instance, and SHALL NOT register -an environment; those belong to `spinloop remote deploy`. Bootstrap SHALL NOT -reimplement the infrastructure; it SHALL orchestrate the existing CDK project. +Image Builder pipelines, the environment-aware lifecycle Lambdas and their IAM, +and the shared S3 weights bucket, IAM roles and VPC — by obtaining the CDK +project shipped in `remote/` and driving its deploy of the control-plane stack. +Bootstrap SHALL NOT start any AMI bake; the bake is a separate +`spinloop remote bake` step. Bootstrap SHALL NOT create any Elastic IP or EC2 +instance, and SHALL NOT register an environment; those belong to +`spinloop remote deploy`. Bootstrap SHALL NOT reimplement the infrastructure; +it SHALL orchestrate the existing CDK project. On success, bootstrap SHALL +signpost `spinloop remote bake` as the next step, ahead of +`spinloop remote deploy`. #### Scenario: A successful bootstrap yields the control plane - **WHEN** `spinloop remote bootstrap` completes -- **THEN** the control-plane stack is deployed — Image Builder, the lifecycle Lambdas, - and the shared bucket/roles/VPC — with no Elastic IP or instance created +- **THEN** the control-plane stack is deployed — Image Builder pipelines, the + lifecycle Lambdas, and the shared bucket/roles/VPC — with no Elastic IP or + instance created and no AMI bake started + +#### Scenario: Bootstrap signposts the bake + +- **WHEN** `spinloop remote bootstrap` completes +- **THEN** its output names `spinloop remote bake` as the next step, ahead of + `spinloop remote deploy` #### Scenario: Orchestration stops on a failed step @@ -73,20 +84,21 @@ other than an explicit yes as a decline that makes no changes. ### Requirement: Version-matched CDK sources -Bootstrap SHALL obtain the CDK project by downloading the `remote/` tree from the -project repository at a reference matching the running binary's version, so the -infrastructure matches the CLI driving it. A `--ref` flag SHALL override the -reference, and a `--dir` flag SHALL override where the sources are placed -(defaulting under the user config directory). For a development build with no -release version, bootstrap SHALL fall back to a documented default reference. The -CDK sources SHALL NOT be embedded in the binary, since a package-manager -install is required at runtime regardless. +Bootstrap and `spinloop remote bake` SHALL obtain the CDK project by +downloading the `remote/` tree from the project repository at a reference +matching the running binary's version, so the infrastructure matches the CLI +driving it. A `--ref` flag SHALL override the reference, and a `--dir` flag +SHALL override where the sources are placed (defaulting under the user config +directory). For a development build with no release version, bootstrap and +bake SHALL fall back to a documented default reference. The CDK sources SHALL +NOT be embedded in the binary, since a package-manager install is required at +runtime regardless. The default source location SHALL be keyed by the resolved reference, so a re-run at the same version reuses its sources while a different binary version downloads -fresh. On a successful bootstrap using the default location, sources from other -references SHALL be pruned. An explicit `--dir` SHALL be treated as the user's own -location: neither keyed by reference nor pruned. +fresh. On a successful bootstrap or bake using the default location, sources +from other references SHALL be pruned. An explicit `--dir` SHALL be treated as +the user's own location: neither keyed by reference nor pruned. #### Scenario: Sources match the binary version @@ -100,8 +112,8 @@ location: neither keyed by reference nor pruned. #### Scenario: A new version does not reuse stale sources -- **WHEN** bootstrap runs from a binary whose resolved reference differs from a - previously downloaded one in the default location +- **WHEN** bootstrap or bake runs from a binary whose resolved reference + differs from a previously downloaded one in the default location - **THEN** it downloads sources for the new reference, and on success the superseded reference's sources are pruned @@ -142,15 +154,16 @@ be confirmed, without attempting to raise it. ### Requirement: A Node package manager is selected, overridable, and logged -Bootstrap SHALL select the Node package manager it drives the CDK project with. -Absent an explicit choice, it SHALL auto-detect by PATH lookup, preferring `pnpm` -and falling back to `npm` when `pnpm` is not on the path. The user MAY override +Bootstrap and `spinloop remote bake` SHALL select the Node package manager they +drive the CDK project with. Absent an explicit choice, they SHALL auto-detect by +PATH lookup, preferring `pnpm` and falling back to `npm` when `pnpm` is not on +the path. The user MAY override the selection with a `--package-manager` flag or an `SPINLOOP_REMOTE_PACKAGE_MANAGER` environment variable, whose only accepted values are `pnpm` and `npm`; the flag SHALL take precedence over the environment variable, which SHALL take precedence over auto-detection. An unrecognised override value SHALL be rejected with an error naming the accepted values. The selected manager SHALL be used consistently -for every Node step (install, `cdk`, `deploy:image`, `bake`, `deploy`) and +for every Node step (install, `cdk`, `deploy:image`, `deploy`) and reflected in the printed plan. Before the steps run, bootstrap SHALL log which package manager it selected, so the run is self-explanatory. When auto-detecting and both managers are present, `pnpm` SHALL win; the choice SHALL NOT depend on @@ -182,36 +195,60 @@ yet runs correctly under either manager. - **THEN** bootstrap fails with an error naming the accepted values, before deploying anything -### Requirement: Control-plane settings are collected +### Requirement: AMI bake is a separate command -Bootstrap SHALL collect the control-plane settings the CDK has no default for and write -them where the CDK reads them: which runner AMIs to bake (`--runners`, defaulting -to both `llamacpp` and `vllm`) so any environment can pick its engine at deploy -time, and an optional Hugging Face token for the shared secret used when seeding -gated weights. The engine is a per-environment choice made at `deploy`, so a -single runner is not a bootstrap setting; the allowed ingress CIDR is also -per-environment and belongs to `deploy`, not here. +The system SHALL provide `spinloop remote bake`, which starts an AMI bake for +each runner named as a positional argument — `llamacpp` and `vllm` — defaulting +to both when none are named. It SHALL drive the same CDK project that +bootstrap orchestrates, with the same version-matched source download into the +same ref-keyed default location, the same package-manager selection and +override, and `--ref` and `--dir` flags matching bootstrap's. Bake SHALL NOT +deploy any stack; when the control-plane stack is not deployed, it SHALL fail +before starting any bake, naming `spinloop remote bootstrap` as the step to run +first. Bake SHALL block until every requested runner's AMI is available; a +`--no-wait` flag SHALL return as soon as the bakes are queued, reporting how to +check on them, rather than blocking for the bake duration. -#### Scenario: Both runner AMIs are baked by default +#### Scenario: Default bake covers both runners -- **WHEN** the user runs bootstrap without selecting runners -- **THEN** AMIs for both `llamacpp` and `vllm` are baked +- **WHEN** the user runs `spinloop remote bake` with no arguments +- **THEN** a bake is started for both `llamacpp` and `vllm` -#### Scenario: The allowed CIDR is not a bootstrap setting +#### Scenario: A single runner is baked -- **WHEN** the user runs bootstrap -- **THEN** no ingress CIDR is requested or written, since it is scoped per - environment at `spinloop remote deploy` +- **WHEN** the user runs `spinloop remote bake llamacpp` +- **THEN** only the `llamacpp` AMI bake is started + +#### Scenario: An unknown runner is rejected + +- **WHEN** the user names a runner that is neither `llamacpp` nor `vllm` +- **THEN** bake fails before starting any bake, naming the accepted runners + +#### Scenario: No control plane + +- **WHEN** the control-plane stack is not deployed and bake runs +- **THEN** it fails before starting any bake, saying to run + `spinloop remote bootstrap` first -### Requirement: Idempotent bootstrap with an asynchronous bake +#### Scenario: Bake waits by default + +- **WHEN** the user runs `spinloop remote bake` without `--no-wait` +- **THEN** the command blocks until the requested runners' AMIs are available + before finishing + +#### Scenario: Handing off with --no-wait + +- **WHEN** the user passes `--no-wait` +- **THEN** the command returns as soon as the bakes are queued, reporting how + to check on them, rather than blocking for the bake duration + +### Requirement: Idempotent bootstrap Bootstrap SHALL be safe to re-run: it SHALL skip the package-manager install when dependencies are present and `cdk bootstrap` when the account and region are already bootstrapped, and SHALL not redeploy a control-plane stack that is unchanged. Because it touches only control plane and never a live instance, re-running SHALL NOT -require any override. Because the AMI bake is slow, by default bootstrap SHALL -start the bake and hand off, telling the user how to wait, rather than blocking. A -`--wait` flag SHALL block until the bake completes. +require any override. #### Scenario: Re-running skips satisfied steps @@ -219,13 +256,23 @@ start the bake and hand off, telling the user how to wait, rather than blocking. - **THEN** it skips installation and CDK bootstrap that are already done and no-ops the unchanged control-plane stack, without requiring an override -#### Scenario: The slow bake does not block by default +### Requirement: Bootstrap collects only the shared-secret token + +Bootstrap SHALL collect the one control-plane setting the CDK has no default +for and write it where the CDK reads it: an optional Hugging Face token for +the shared secret used when seeding gated weights. Which runner AMIs to bake +is not a bootstrap setting — the engine is a per-environment choice made at +`deploy`, and the runners are named by `spinloop remote bake` itself. The +allowed ingress CIDR is also per-environment and belongs to `deploy`, not here. -- **WHEN** bootstrap reaches the AMI bake without `--wait` -- **THEN** it starts the bake and reports how to wait for it, rather than - blocking for the full bake duration +#### Scenario: Runners are not a bootstrap setting -#### Scenario: Waiting on request +- **WHEN** the user runs bootstrap +- **THEN** no runner selection is requested or written, since the runners are + named at `spinloop remote bake` -- **WHEN** the user passes `--wait` -- **THEN** bootstrap blocks until the bake completes before finishing +#### Scenario: The allowed CIDR is not a bootstrap setting + +- **WHEN** the user runs bootstrap +- **THEN** no ingress CIDR is requested or written, since it is scoped per + environment at `spinloop remote deploy` diff --git a/openspec/specs/remote-endpoint/spec.md b/openspec/specs/remote-endpoint/spec.md index bd19b4c4..944c5950 100644 --- a/openspec/specs/remote-endpoint/spec.md +++ b/openspec/specs/remote-endpoint/spec.md @@ -8,9 +8,9 @@ what to serve from a Spinloop: the `spinloop remote` command group. ### Requirement: Remote command group The system SHALL provide a `remote` command group with the subcommands -`bootstrap`, `start`, `stop`, `restart`, `status`, `deploy`, `ls`, `metrics`, -and `keep`. `start`, `stop`, `restart`, `status`, `metrics` and `deploy` each -take an optional Spinloop path: +`bootstrap`, `bake`, `start`, `stop`, `restart`, `status`, `deploy`, `ls`, +`metrics`, and `keep`. `start`, `stop`, `restart`, `status`, `metrics` and +`deploy` each take an optional Spinloop path: `start` SHALL boot the endpoint and block until it is serving, then perform a quick TCP probe of the inference endpoint — if the probe fails, a warning is printed to stderr explaining the network mismatch (see the Remote Start Probe @@ -37,6 +37,8 @@ what the endpoint serves. `ls` SHALL list the registered remote environments (see the Remote Environments specification). `bootstrap` SHALL stand up the account-level AWS control plane (once per account) by obtaining and driving the CDK project, and takes its own flags rather than a Spinloop path (see the +Endpoint Provisioning specification). `bake` SHALL start an AMI bake for each +runner named, and takes runner names rather than a Spinloop path (see the Endpoint Provisioning specification). An unrecognised subcommand SHALL fail naming the accepted ones. @@ -112,11 +114,17 @@ naming the accepted ones. - **THEN** the command is dispatched to the provisioning flow rather than reported as unknown +#### Scenario: Bake is a recognised subcommand + +- **WHEN** the user runs `spinloop remote bake llamacpp` +- **THEN** the command is dispatched to the bake flow rather than reported as + unknown + #### Scenario: Unknown subcommand - **WHEN** the user runs `spinloop remote frobnicate` - **THEN** the command fails listing the accepted subcommands, which include - `bootstrap`, `metrics`, and `keep` + `bootstrap`, `bake`, `metrics`, and `keep` ### Requirement: Reporting a start in progress diff --git a/remote/README.md b/remote/README.md index 1a3bcd4d..2e71fc8d 100644 --- a/remote/README.md +++ b/remote/README.md @@ -45,9 +45,10 @@ availability zone — it tries each g6e zone in turn until one has capacity. The image stack defines an Image Builder **pipeline**, not a build, so deploying it never runs (or fails on) a bake. You trigger bakes out-of-band -with `pnpm bake `; each successful bake **tags** its AMI with its -engine, and the start Lambda launches the **newest AMI matching the engine it -was told to run**. A failed bake produces no new AMI and changes nothing. +with `spinloop remote bake ` (or `pnpm bake ` by hand); each +successful bake **tags** its AMI with its engine, and the start Lambda launches +the **newest AMI matching the engine it was told to run**. A failed bake +produces no new AMI and changes nothing. The control plane **renders** the boot script and the daemon's service unit, and the boot script **installs** the spinloop binary that runs them — the @@ -58,7 +59,7 @@ order launches instances whose daemon never starts. ``` spinloop remote bootstrap ─▶ control-plane stack (Lambdas, S3, VPC, roles) + bake pipelines - pnpm bake llamacpp ─▶ Image Builder pipeline ─(async)─▶ AMI (driver + engine), tagged +spinloop remote bake llamacpp ─▶ Image Builder pipeline ─(async)─▶ AMI (driver + engine), tagged spinloop remote deploy ─▶ deploy Lambda ─▶ creates env : EIP, SG (your CIDR), │ API key, deploy-config (what to serve) └─ seeds weights ─▶ S3 weights bucket (shared) @@ -121,24 +122,26 @@ aws ec2 describe-instance-type-offerings --location-type availability-zone \ The one-time account setup is [`spinloop remote bootstrap`](../docs/commands/remote.md#bootstrapping-the-account), which drives this directory for you — download, consent plan, then the shared -deploy and the AMI bakes. Endpoints come after it, one `spinloop remote deploy` -per environment: +deploy. Baking the AMIs is the separate +[`spinloop remote bake`](../docs/commands/remote.md#baking-the-amis) step after +it; endpoints come after that, one `spinloop remote deploy` per environment: ```sh -spinloop remote bootstrap # once per account: control-plane stack + pipelines + bakes +spinloop remote bootstrap # once per account: control-plane stack + pipelines +spinloop remote bake # bakes the runner AMI(s); waits until they are available spinloop remote deploy # creates the Spinloop's REMOTE environment and says # what it serves; seeds the weights if missing ``` -Under the hood, bootstrap runs this directory's own commands — usable by hand -too: +Under the hood, bootstrap and bake run this directory's own commands — usable +by hand too: ```sh pnpm install pnpm cdk bootstrap # once per account/region pnpm deploy:image # creates the bake pipelines — instant, no build yet -pnpm bake llamacpp # bakes that engine's AMI — ~15-25 min, in the background pnpm run deploy # deploys the control-plane stack (Lambdas, VPC, S3 bucket) +pnpm bake llamacpp # bakes that engine's AMI — ~15-25 min, in the background ``` - `pnpm deploy:image` only creates the pipelines, so it deploys in seconds and