feat(remote): create or rotate an environment's engine key

cmd/spinloop/remote.go

@@ -1346,6 +1346,7 @@ func remoteDeployCmd() *cobra.Command {
 		allowedCidr     string
 		region          string
 		spinloopVersion string
+		apiKeyEnv       string
 	)
 	c := &cobra.Command{
 		Use:   "deploy",
@@ -1361,7 +1362,7 @@ installs the latest published release.`,
 		ValidArgsFunction: aliasSlot,
 		RunE: func(c *cobra.Command, args []string) error {
 			resolve(c)
-			return runRemoteDeploy(args, dryRun, overwrite, reseed, allowedCidr, region, spinloopVersion)
+			return runRemoteDeploy(args, dryRun, overwrite, reseed, allowedCidr, region, spinloopVersion, apiKeyEnv)
 		},
 	}
 	fs := c.Flags()
@@ -1371,11 +1372,12 @@ installs the latest published release.`,
 	fs.StringVar(&allowedCidr, "allowed-cidr", "", "who may reach this environment's instance (default: your public IP as a /32, on first deploy)")
 	fs.StringVar(&region, "region", "", "AWS region of the control plane (default: AWS_REGION or us-east-1)")
 	fs.StringVar(&spinloopVersion, "spinloop-version", "", "spinloop release the environment's instances install at boot (default: latest)")
+	fs.StringVar(&apiKeyEnv, "api-key-env", "", "the environment variable holding the API key to store for this environment (a variable name, never the key itself)")
 	return c
 }
 
 // runRemoteDeploy is the body of `spinloop remote deploy`.
-func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr, region, spinloopVersion string) error {
+func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr, region, spinloopVersion, apiKeyEnv string) error {
 	// deploy reads the Spinloop for what to serve, so unlike the other
 	// subcommands it always needs one — the per-user remote config alone is not
 	// enough.
@@ -1390,6 +1392,20 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr,
 	if err := applySpinloopEnv(sel, spinloopPath); err != nil {
 		return err
 	}
+	// A supplied key arrives the way every other secret does: as a reference
+	// to an environment variable, never a literal on the command line. The
+	// Spinloop's local environment has just been applied, so the variable is
+	// resolvable from the process environment; one set nowhere fails before
+	// anything is sent.
+	var apiKey string
+	if apiKeyEnv != "" {
+		apiKey = os.Getenv(apiKeyEnv)
+		if apiKey == "" {
+			return fmt.Errorf(
+				"--api-key-env: %s is not set: export it, or put it in the .env beside the Spinloop",
+				apiKeyEnv)
+		}
+	}
 	dc, err := deployConfigFor(sel, spinloopPath)
 	if err != nil {
 		return err
@@ -1451,6 +1467,11 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr,
 		spinloopVer = "latest"
 	}
 	fmt.Printf("  spinloop:  %s\n", spinloopVer)
+	// A key is worth stating too — it rotates — but the value is never
+	// printed, in the dry run or the report.
+	if apiKeyEnv != "" {
+		fmt.Printf("  api key:  stored from %s\n", apiKeyEnv)
+	}
 	if dryRun {
 		return nil
 	}
@@ -1503,7 +1524,7 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr,
 		fmt.Printf("  ingress: %s (your public IP; override with --allowed-cidr)\n", allowedCidr)
 	}
 
-	resp, err := remoteDeployFn(ctx, cfg, dc, allowedCidr, reseed)
+	resp, err := remoteDeployFn(ctx, cfg, dc, allowedCidr, reseed, apiKey)
 	if err != nil {
 		return err
 	}
@@ -1518,6 +1539,14 @@ func runRemoteDeploy(args []string, dryRun, overwrite, reseed bool, allowedCidr,
 	fmt.Println()
 	fmt.Printf("deployed: environment %s at %s\n", env, resp.BaseURL)
 	fmt.Printf("registered: %s\n", envConfigPath)
+	// A rotation is worth stating out loud: it invalidates the key a live
+	// agent may still be holding, and the action — never the value — is all
+	// the reply carries.
+	if resp.APIKeyAction == "rotated" {
+		fmt.Println("api key: rotated — the previous key no longer works")
+	} else if resp.APIKeyAction != "" {
+		fmt.Println("api key: created")
+	}
 	if resp.Seeding {
 		fmt.Printf("seeding the weights — follow it with `spinloop remote seed status %s`.\n", resp.SeedID)
 		fmt.Println("Wait for it to finish before `spinloop remote start`, or the instance will")

cmd/spinloop/remote_deploy_test.go

@@ -617,6 +617,101 @@ func TestRemoteDeploy_SpinloopVersion(t *testing.T) {
 	})
 }
 
+// A supplied key is resolved from the environment the Spinloop's local
+// environment populated, reaches the signed body as a request-scoped field,
+// and the report says what happened to it — the action, never the value.
+func TestRemoteDeploy_APIKeyEnvReachesTheRequest(t *testing.T) {
+	isolateConfig(t)
+	stubAWSEnv(t)
+
+	var gotAPIKey string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		var got struct {
+			APIKey string `json:"apiKey"`
+		}
+		_ = json.Unmarshal(body, &got)
+		gotAPIKey = got.APIKey
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"deployed":true,"environment":"testenv","base_url":"http://198.51.100.9:8000/v1","apiKeyAction":"rotated"}`))
+	}))
+	defer server.Close()
+	stubDeploySeams(t, server.URL, "undeployed")
+	writeDeployEnvSpinloop(t, "testenv")
+	t.Setenv("SHARED_KEY", "sk-supplied")
+
+	out := captureStdout(t, func() {
+		if err := cmdRemoteDeploy([]string{"--api-key-env", "SHARED_KEY"}); err != nil {
+			t.Fatalf("cmdRemoteDeploy: %v", err)
+		}
+	})
+
+	if gotAPIKey != "sk-supplied" {
+		t.Errorf("the request carried apiKey %q, want the resolved value", gotAPIKey)
+	}
+	// The report names the action and never the value.
+	if !strings.Contains(out, "api key: rotated") {
+		t.Errorf("the report should say the key was rotated, got:\n%s", out)
+	}
+	if strings.Contains(out, "sk-supplied") {
+		t.Errorf("the report printed the key value:\n%s", out)
+	}
+}
+
+// A named variable that is set nowhere fails before anything is sent.
+func TestRemoteDeploy_APIKeyEnvUnsetFailsBeforeSending(t *testing.T) {
+	isolateConfig(t)
+	stubAWSEnv(t)
+
+	sent := false
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		sent = true
+		w.Write([]byte(`{"deployed":true}`))
+	}))
+	defer server.Close()
+	stubDeploySeams(t, server.URL, "undeployed")
+	writeDeployEnvSpinloop(t, "testenv")
+	os.Unsetenv("NOWHERE_DEPLOY_KEY")
+
+	err := cmdRemoteDeploy([]string{"--api-key-env", "NOWHERE_DEPLOY_KEY"})
+	if err == nil || !strings.Contains(err.Error(), "NOWHERE_DEPLOY_KEY") {
+		t.Errorf("want an error naming the variable, got %v", err)
+	}
+	if sent {
+		t.Error("the deploy was sent despite the key failing to resolve")
+	}
+}
+
+// A dry run with a key names the variable it will store from — never the
+// value — and still touches nothing.
+func TestRemoteDeploy_APIKeyEnvDryRunNamesTheVariable(t *testing.T) {
+	isolateConfig(t)
+	called := false
+	origDiscover := deployDiscoverFn
+	t.Cleanup(func() { deployDiscoverFn = origDiscover })
+	deployDiscoverFn = func(context.Context, aws.Config, string) (remote.ControlPlane, error) {
+		called = true
+		return remote.ControlPlane{}, fmt.Errorf("must not be called")
+	}
+	writeDeployEnvSpinloop(t, "testenv")
+	t.Setenv("SHARED_KEY", "sk-supplied")
+
+	out := captureStdout(t, func() {
+		if err := cmdRemoteDeploy([]string{"--dry-run", "--api-key-env", "SHARED_KEY"}); err != nil {
+			t.Fatalf("cmdRemoteDeploy --dry-run: %v", err)
+		}
+	})
+	if called {
+		t.Error("--dry-run must touch nothing — not even discovery")
+	}
+	if !strings.Contains(out, "api key:  stored from SHARED_KEY") {
+		t.Errorf("--dry-run should name the key's variable, got:\n%s", out)
+	}
+	if strings.Contains(out, "sk-supplied") {
+		t.Errorf("--dry-run printed the key value:\n%s", out)
+	}
+}
+
 // Guard the assumption deployConfigFor relies on: PROVIDER names the engine.
 func TestRunnerFor(t *testing.T) {
 	for _, provider := range []string{"llamacpp", "vllm"} {

docs/commands/fleet.md

@@ -155,6 +155,20 @@ its engine's key out: it says only that one is required.
     engineTokenEnv: GATED_ENGINE_KEY  # to talk to its engine
 ```
 
+A `kind: remote` environment is always keyed, so it needs an engine key too —
+its `engineTokenEnv` works as above, and a fleet-wide `apiKeyEnv` is the
+default for every remote node that does not name one of its own. Either way the
+launch fails before it starts the agent rather than pointing it at a gate it
+cannot pass:
+
+```yaml
+apiKeyEnv: REMOTE_ENGINE_KEY   # the default for every kind: remote node
+nodes:
+  - name: qwen
+    kind: remote
+    engineTokenEnv: OTHER_KEY  # overrides it for this node
+```
+
 ## A node that is down never blanks the view
 
 Fan-out is for observing, so a node that cannot be reached is a **row**, not a

docs/commands/remote.md

@@ -329,6 +329,13 @@ or a seed you want to run again. It starts the same ~20-minute seed instance a
 first deploy does, and re-downloads the weights, so it is opt-in rather than
 something to reach for by habit.
 
+By default deploy stores the engine key the environment was first deployed
+with — one generated per environment, held in its secret. To choose or change
+it, `--api-key-env VAR` resolves `VAR` from your environment (or the `.env`
+beside the Spinloop) and sends the value: the deploy creates or **rotates** the
+environment's key, so the old value stops working — and the reply says which
+happened, never the value itself.
+
 ## Flags
 
 | Flag | Meaning |
@@ -339,6 +346,7 @@ something to reach for by habit.
 | `-n`, `--dry-run` | `deploy` only: print what would be sent, without sending it |
 | `--reseed` | `deploy` only: re-fetch the weights even if they are already in S3 |
 | `--spinloop-version` | `deploy` only: the spinloop release fresh boots install (default: the latest published release) |
+| `--api-key-env` | `deploy` only: name the environment variable holding the engine key to create or rotate; with no flag the stored key is kept |
 
 `bootstrap` and `bake` have their own too (`--ref`, `--dir`, `--region`,
 `--package-manager`, and `--no-wait` on bake) — see their sections above.

docs/env-vars.md

@@ -17,6 +17,7 @@ from the environment or a `.env` beside the Spinloop — never written into an
 | `SPINLOOP_LOG_LEVEL` | `spinloop daemon`, `spinloop serve` | How much spinloop records about the control API and the supervised engine: `debug`, `info` (default), `warn` or `error`. Precedence: `--log-level` flag > `SPINLOOP_LOG_LEVEL` > `info`. An unrecognised value refuses to start rather than falling back to the default. Under `spinloop serve` the `.env` beside the Spinloop can set it; the daemon reads no Spinloop, so there it comes from the environment its service manager gives it. Records go to stderr; see [what gets logged](commands/serve.md#what-gets-logged). |
 | *(per-node, named by `tokenEnv`)* | `spinloop fleet` | A fleet node's bearer token. `fleet.yaml` names the variable rather than holding the value; it resolves from the environment, then the `.env` beside the fleet file. See [fleet](commands/fleet.md). |
 | *(per-node, named by `engineTokenEnv`)* | `spinloop fleet`, `spinloop harness` | The key a fleet node's **engine** is gated with. Resolved the same way, and supplied by the client when it starts that engine — so the node holds no key of its own and the two ends cannot disagree. See [fleet](commands/fleet.md). |
+| *(fleet-wide, named by `apiKeyEnv`)* | `spinloop fleet`, `spinloop harness` | The default key for a `kind: remote` environment's engine, for every remote node that does not name its own `engineTokenEnv`. Resolved the same way. See [fleet](commands/fleet.md). |
 
 ## Remote (`spinloop remote`)
 

internal/fleet/config.go

@@ -71,6 +71,15 @@ type Config struct {
 	// should be used — spread the work, or consolidate it. Empty means
 	// PreferIdle.
 	Prefer Prefer `yaml:"prefer"`
+	// APIKeyEnv names the environment variable holding the key this fleet's
+	// remote nodes require, shared by every one of them: a remote's engine is
+	// always gated by its key, so a fleet of remotes can name the variable
+	// once rather than on each node — a node's own EngineTokenEnv overrides
+	// it. It is a remote-only default: a daemon gates on its own
+	// EngineTokenEnv, and a fleet-wide key must not start gating an engine
+	// that was never set up to accept one. As with every other secret in
+	// this file, the value is never written here.
+	APIKeyEnv string `yaml:"apiKeyEnv"`
 
 	// Path is the file this was read from, and Dir its directory — the .env
 	// beside it fills token references.
@@ -267,6 +276,24 @@ func (c *Config) EngineToken(n NodeConfig) (string, error) {
 	return c.resolveTokenEnv(n, n.EngineTokenEnv)
 }
 
+// RemoteEngineToken resolves the key a remote node's engine requires: the
+// variable the node names when it names one, else the fleet-wide APIKeyEnv.
+// A remote's engine is always gated by its key, so a node that names no
+// resolvable key fails here, before a launch depends on it — the way every
+// other missing secret in this file is named, the node and the fix.
+func (c *Config) RemoteEngineToken(n NodeConfig) (string, error) {
+	name := n.EngineTokenEnv
+	if name == "" {
+		name = c.APIKeyEnv
+	}
+	if name == "" {
+		return "", fmt.Errorf(
+			"node %q is a remote environment, so its engine key must be set: name the variable holding it, in this node's `engineTokenEnv` or the file's fleet-wide `apiKeyEnv` (%s)",
+			n.Name, c.Path)
+	}
+	return c.resolveTokenEnv(n, name)
+}
+
 // resolveTokenEnv reads one of a node's token references: the process
 // environment first, then the .env beside the fleet file — the precedence
 // spinloop uses everywhere, so an exported value wins and the .env only fills a