diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d87f3d5f..4a1c25f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,3 +90,19 @@ jobs: - name: Fleet integration test run: examples/fleet-docker/run-tests.sh + + # The dockerised gateway example, driven end to end: real daemons, a real + # gateway process selecting, waking and keying over the network. Same + # double duty as the fleet job — coverage and example at once. + gateway-integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Gateway integration test + run: examples/gateway-docker/run-tests.sh diff --git a/cmd/spinloop/commands.go b/cmd/spinloop/commands.go index e54217d8..3977ff1b 100644 --- a/cmd/spinloop/commands.go +++ b/cmd/spinloop/commands.go @@ -66,6 +66,7 @@ harness could be configured with, spinloop show what it has been.`, serveCmd(), upCmd(), daemonCmd(), + gatewayCmd(), exportCmd(), initProvidersCmd(), harnessCmd(), diff --git a/cmd/spinloop/fleet.go b/cmd/spinloop/fleet.go index bf65c0ca..736cd6fb 100644 --- a/cmd/spinloop/fleet.go +++ b/cmd/spinloop/fleet.go @@ -771,9 +771,17 @@ func runFleetRoute(path, node, prefer string, args []string) error { resolvedPath) } if isEndpoint(target) { - return fmt.Errorf( - "FLEET %s names an endpoint, and gateway routing is not implemented yet: "+ - "name a fleet file to choose a node from", target) + // The endpoint has already done the choosing: no fleet file to read, + // no node to query, nothing to start. Say where a launch would point + // the agent. + if sel.BaseURL != "" { + fmt.Printf("This Spinloop pins BASEURL %s, so a launch would not route at all.\n", sel.BaseURL) + return nil + } + fmt.Printf("Spinloop: %s\nFleet: %s (an endpoint, not a fleet file)\n\n", resolvedPath, target) + fmt.Printf("The endpoint has already chosen: a launch would point the agent at %s.\n", endpointBaseURL(target)) + fmt.Println("No node is queried, and nothing is started.") + return nil } cfg, err := fleet.Resolve(resolveFleetPath(target, fromFlag, resolvedPath)) if err != nil { @@ -814,8 +822,13 @@ func runFleetRoute(path, node, prefer string, args []string) error { fmt.Printf("\nA launch could not start one either: %v\n", dcErr) return nil } - if wake, ok := cfg.WouldWake(none.Results, dc); ok { - fmt.Printf("\nA launch would wake %s and wait for its engine. Nothing has been started.\n", wake.Name) + if wake, ok := cfg.WouldWake(none.Results, fleet.ConstantConfig(dc, dcErr)); ok { + if cfg.Wakes() { + fmt.Printf("\nA launch would wake %s and wait for its engine. Nothing has been started.\n", wake.Name) + } else { + fmt.Printf("\nA launch would refuse: wake is off in %s. Start %s with `spinloop fleet start %s`. Nothing has been started.\n", + cfg.Path, wake.Name, wake.Name) + } return nil } fmt.Println("\nNo node could be woken for it either. Nothing has been started.") diff --git a/cmd/spinloop/gateway.go b/cmd/spinloop/gateway.go new file mode 100644 index 00000000..8a1b551d --- /dev/null +++ b/cmd/spinloop/gateway.go @@ -0,0 +1,202 @@ +// spinloop gateway: the fleet's OpenAI-compatible front door. It is the fleet +// client wearing a server — selection, waking, endpoint resolution, and key +// handling all come from internal/fleet, and the HTTP surface lives in +// internal/gateway, so this command resolves the fleet file, resolves its own +// token the way the daemon's is resolved, and serves. + +package main + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spinloop-ai/spinloop/internal/fleet" + "github.com/spinloop-ai/spinloop/internal/gateway" + "github.com/spinloop-ai/spinloop/internal/remote" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +func gatewayCmd() *cobra.Command { + var fleetPath, listen, apiToken, apiTokenFile string + var wakeTimeout time.Duration + var loopback bool + c := &cobra.Command{ + Use: "gateway", + Short: "serve the fleet under one OpenAI-compatible endpoint", + Long: `runs in the foreground, the way spinloop serve does: it holds the +fleet file it serves (a --fleet path, or ./fleet.yaml), answers +/v1/models and completion requests by choosing a node with the fleet's +own selector, and wakes a node when nothing is serving what a request +asks for, holding the request until the engine answers. It needs the +same environment a machine running spinloop fleet start would: the +tokens the fleet file names, set here or in the .env beside it. + +A Spinloop points an agent at it with a FLEET that names its address: + + FLEET http://gateway.internal:4000 + +The agent then needs only the gateway's token, as OPENAI_API_KEY.`, + Args: cobra.NoArgs, + SilenceErrors: true, + SilenceUsage: true, + RunE: func(c *cobra.Command, args []string) error { + resolve(c) + return runGatewayCommand(fleetPath, listen, apiToken, apiTokenFile, wakeTimeout, loopback, c.Flags()) + }, + } + fs := c.Flags() + fs.StringVarP(&fleetPath, "fleet", "f", "", fleetFileUsage) + fs.StringVar(&listen, "listen", gateway.DefaultListen, "the address to listen on") + fs.BoolVarP(&loopback, "loopback", "l", false, "bind the gateway to loopback on the default port ("+gateway.LoopbackListen+"); needs no token") + fs.StringVar(&apiTokenFile, "api-token-file", "", "read the gateway's bearer token from this file") + fs.StringVar(&apiToken, "api-token", "", "the gateway's bearer token") + fs.DurationVar(&wakeTimeout, "wake-timeout", 0, "how long to wait for a woken engine to answer") + compRegister(c, "fleet", compFiles) + return c +} + +// cmdGateway runs the command through the tree — the seam the suite calls. +func cmdGateway(args []string) error { return execCmd(gatewayCmd(), args) } + +// runGatewayCommand is the body of `spinloop gateway`: the server, and the +// signal handling that shuts it down cleanly. +func runGatewayCommand(fleetPath, listen, apiToken, apiTokenFile string, wakeTimeout time.Duration, loopback bool, flags *pflag.FlagSet) error { + // Whether --listen was typed at all, not whether it differs from the + // default: --listen :4000 --loopback is still a conflict, and a + // compare-against-default check would let it pass. + listenExplicit := flags.Changed("listen") + listen, err := gatewayListenAddr(listen, listenExplicit, loopback) + if err != nil { + return err + } + // The override lives as long as the server serves, not as long as the + // setup takes. + var restore func() + if wakeTimeout > 0 { + prev := fleet.WakeTimeout + fleet.WakeTimeout = wakeTimeout + restore = func() { fleet.WakeTimeout = prev } + defer restore() + } + srv, ln, err := newGatewayServer(fleetPath, listen, apiToken, apiTokenFile) + if err != nil { + return err + } + defer ln.Close() + + // The handler goes in before a signal can arrive, so a signal at any point + // from here on shuts the server down rather than killing the process. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sigCh) + go srv.Serve(ln) + + // Foreground until signalled, the way the daemon waits: the signal is the + // only exit, so the command returns nil when a clean shutdown ends Serve's + // http.ErrServerClosed with it. + <-sigCh + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + srv.Shutdown(ctx) + cancel() + return nil +} + +// gatewayListenAddr resolves the address the gateway listens on: --loopback +// takes the default, and an explicit address and --loopback are two answers +// to one question. +func gatewayListenAddr(listen string, listenExplicit, loopback bool) (string, error) { + if loopback && listenExplicit { + return "", fmt.Errorf("--loopback and --listen both given: pass one") + } + if loopback { + return gateway.LoopbackListen, nil + } + return listen, nil +} + +// newGatewayServer resolves the fleet file and the gateway's token, checks the +// file's token references the way a startup must, opens the listener, and +// prints the address a Spinloop names in its FLEET. Everything that can fail +// without serving fails here, before a listener exists. +func newGatewayServer(fleetPath, listen, apiToken, apiTokenFile string) (*http.Server, net.Listener, error) { + cfg, err := fleet.Resolve(fleetPath) + if err != nil { + return nil, nil, err + } + token, err := daemonToken(apiToken, apiTokenFile) + if err != nil { + return nil, nil, err + } + logger, err := commandLogger("") + if err != nil { + return nil, nil, err + } + + // The gateway's own startup failures are the fleet file's: a token variable + // it names that is set nowhere fails here, naming the node, rather than + // surfacing later as a per-request authentication failure. + for _, entry := range cfg.Nodes { + if _, err := cfg.Token(entry); err != nil { + return nil, nil, err + } + if entry.Kind == fleet.KindRemote { + if _, err := cfg.RemoteEngineToken(entry); err != nil { + return nil, nil, err + } + } else if _, err := cfg.EngineToken(entry); err != nil { + return nil, nil, err + } + } + + // The gateway is the client that wakes a node, so it resolves what each + // node runs the way `spinloop fleet start` does: the node's own source, + // never a config invented for the request. + cfgFor := func(entry fleet.NodeConfig) (remote.DeployConfig, error) { + arg, _, err := resolveNodeSpinloop(entry, cfg.Dir) + if err != nil { + return remote.DeployConfig{}, err + } + sel, path, err := readSpinloop("the Spinloop of node "+entry.Name, arg) + if err != nil { + return remote.DeployConfig{}, err + } + if err := applySpinloopEnv(sel, path); err != nil { + return remote.DeployConfig{}, err + } + return deployConfigForNode(sel, path) + } + + h := gateway.New(cfg, token, gateway.Options{ConfigFor: cfgFor, Log: logger}) + ln, err := gateway.Listen(listen, token) + if err != nil { + return nil, nil, err + } + + fmt.Printf("Gateway for %s is listening on %s\n", cfg.Path, ln.Addr().String()) + fmt.Printf("Name %s in a Spinloop's FLEET\n\n", fleetURL(ln.Addr().String())) + return &http.Server{Handler: h}, ln, nil +} + +// fleetURL turns the address the gateway listens on into the value a Spinloop +// names in its FLEET: an http URL the agent's machine can reach. The host it +// can know is the one it was told to bind; for a wildcard bind the host is +// whatever this machine is called from the other side, which only the operator +// knows. +func fleetURL(listenAddr string) string { + host, port, err := net.SplitHostPort(listenAddr) + if err != nil { + return listenAddr + } + if host == "" || host == "0.0.0.0" || host == "::" || host == "[::]" { + host = "" + } + return "http://" + net.JoinHostPort(host, port) +} diff --git a/cmd/spinloop/gateway_test.go b/cmd/spinloop/gateway_test.go new file mode 100644 index 00000000..8bf82891 --- /dev/null +++ b/cmd/spinloop/gateway_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spinloop-ai/spinloop/internal/gateway" +) + +// The gateway starts with the fleet file it serves and answers, and its +// banner names the address a Spinloop puts in its FLEET. +func TestGatewayStartsAndAnswers(t *testing.T) { + isolateConfig(t) + t.Setenv("SPINLOOP_API_TOKEN", "") + node := newRoutableNode(t, "qwen3-27b", true, 300) + dir := t.TempDir() + fleetFileIn(t, dir, "nodes:\n"+node.entry("gpu-box")) + t.Chdir(dir) + + var srv *http.Server + var ln net.Listener + out := captureStdout(t, func() { + var err error + srv, ln, err = newGatewayServer("", "127.0.0.1:0", "", "") + if err != nil { + t.Fatal(err) + } + }) + defer ln.Close() + go srv.Serve(ln) + + port := ln.Addr().(*net.TCPAddr).Port + if !strings.Contains(out, "http://127.0.0.1:"+fmt.Sprint(port)) { + t.Errorf("the banner should name the address to put in a Spinloop's FLEET, got:\n%s", out) + } + + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/health", port)) + if err != nil { + t.Fatalf("health: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("health answered %d", resp.StatusCode) + } + + // The printed address is the one a Spinloop's FLEET names: asking it for + // the fleet's models gets the running node's. + resp, err = client.Get(fmt.Sprintf("http://127.0.0.1:%d/v1/models", port)) + if err != nil { + t.Fatalf("models: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("models answered %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), "qwen3-27b") { + t.Errorf("the running node's model should be listed, got: %s", body) + } +} + +// A missing fleet file fails naming the expected path, and nothing listens. +func TestGatewayFailsWithoutAFleetFile(t *testing.T) { + t.Chdir(t.TempDir()) + _, ln, err := newGatewayServer("", "127.0.0.1:0", "", "") + if err == nil { + t.Fatal("a gateway with no fleet file should fail") + } + if ln != nil { + t.Error("a failing gateway opened a listener") + } + if !strings.Contains(err.Error(), "fleet.yaml") { + t.Errorf("the failure should name the expected path, got: %v", err) + } +} + +// A fleet file naming a token variable set nowhere fails at startup, naming +// the node and the variable, rather than listening and failing per request. +func TestGatewayFailsOnAnUnsetTokenVariable(t *testing.T) { + dir := t.TempDir() + fleetFileIn(t, dir, "nodes:\n - name: gated\n host: 127.0.0.1\n port: 14242\n tokenEnv: GW_NODE_TOKEN_UNSET\n") + t.Chdir(dir) + + _, ln, err := newGatewayServer("", "127.0.0.1:0", "", "") + if err == nil { + t.Fatal("an unset token variable should fail the gateway at startup") + } + if ln != nil { + t.Error("a failing gateway opened a listener") + } + for _, want := range []string{"gated", "GW_NODE_TOKEN_UNSET"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the failure should name %q, got: %v", want, err) + } + } +} + +// Two token sources at once is a conflict, resolved the way the daemon's is. +func TestGatewayTokenSourcesConflict(t *testing.T) { + dir := t.TempDir() + fleetFileIn(t, dir, "nodes:\n - name: n\n host: 127.0.0.1\n port: 14242\n") + t.Chdir(dir) + + tokenFile := filepath.Join(t.TempDir(), "token") + mustWrite(t, tokenFile, "from-file\n") + _, _, err := newGatewayServer("", "127.0.0.1:0", "literal", tokenFile) + if err == nil { + t.Fatal("two token sources should be a conflict") + } + if !strings.Contains(err.Error(), "both given") { + t.Errorf("the conflict should name both sources, got: %v", err) + } +} + +// A tokenless non-loopback listen is refused at startup, naming every way a +// token can be supplied. +func TestGatewayRefusesTokenlessNonLoopback(t *testing.T) { + dir := t.TempDir() + fleetFileIn(t, dir, "nodes:\n - name: n\n host: 127.0.0.1\n port: 14242\n") + t.Chdir(dir) + t.Setenv("SPINLOOP_API_TOKEN", "") + + _, ln, err := newGatewayServer("", "0.0.0.0:0", "", "") + if err == nil { + t.Fatal("a tokenless non-loopback gateway should refuse to start") + } + if ln != nil { + t.Error("a refusing gateway opened a listener") + } + for _, want := range []string{"--api-token-file", "SPINLOOP_API_TOKEN", "--api-token"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal should name %q, got: %v", want, err) + } + } +} + +func TestGatewayListenAddr(t *testing.T) { + tests := []struct { + name string + listen string + explicit bool + loopback bool + want string + wantErr bool + }{ + {name: "neither", listen: gateway.DefaultListen, want: gateway.DefaultListen}, + {name: "typed address alone", listen: "10.0.0.5:9999", explicit: true, want: "10.0.0.5:9999"}, + {name: "loopback replaces the default", listen: gateway.DefaultListen, loopback: true, want: gateway.LoopbackListen}, + {name: "loopback and a typed address conflict", listen: "10.0.0.5:9999", explicit: true, loopback: true, wantErr: true}, + {name: "loopback and the repeated default conflict", listen: gateway.DefaultListen, explicit: true, loopback: true, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := gatewayListenAddr(tt.listen, tt.explicit, tt.loopback) + if (err != nil) != tt.wantErr { + t.Fatalf("gatewayListenAddr(%q, %v, %v) error = %v, wantErr %v", tt.listen, tt.explicit, tt.loopback, err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Fatalf("gatewayListenAddr = %q, want %q", got, tt.want) + } + }) + } +} + +// TestCmdGateway_LoopbackConflictsWithExplicitListen covers the rule through +// the real flag parsing, so the -l spelling and a --listen typed to the +// default's own value are both counted as explicit. The conflict is detected +// by fs.Changed, which is order-independent — the last case types the address +// first, pinning that a sequential rewrite can't make the rule depend on flag +// position. +func TestCmdGateway_LoopbackConflictsWithExplicitListen(t *testing.T) { + for _, args := range [][]string{ + {"--loopback", "--listen", "127.0.0.1:0"}, + {"--loopback", "--listen", gateway.DefaultListen}, + {"-l", "--listen", gateway.DefaultListen}, + {"--listen", "127.0.0.1:0", "--loopback"}, + } { + isolateConfig(t) + t.Setenv("SPINLOOP_API_TOKEN", "") + t.Chdir(t.TempDir()) + err := cmdGateway(args) + if err == nil || !strings.Contains(err.Error(), "--loopback") || !strings.Contains(err.Error(), "--listen") { + t.Fatalf("cmdGateway(%v) = %v, want a conflict naming both flags", args, err) + } + } +} + +// TestCmdGateway_LoopbackBindsLoopback checks the shorthand end to end: the +// gateway binds gateway.LoopbackListen and answers unauthenticated, because a +// loopback listen needs no token. The port is fixed, so the test declines +// rather than fights one — a developer in this repo often has a real gateway +// on it, and the rest of the suite never binds a fixed port for the same +// reason. +func TestCmdGateway_LoopbackBindsLoopback(t *testing.T) { + probe, err := net.DialTimeout("tcp", gateway.LoopbackListen, 500*time.Millisecond) + if err == nil { + probe.Close() + t.Skipf("%s is taken", gateway.LoopbackListen) + } + isolateConfig(t) + t.Setenv("SPINLOOP_API_TOKEN", "") + node := newRoutableNode(t, "qwen3-27b", true, 300) + dir := t.TempDir() + fleetFileIn(t, dir, "nodes:\n"+node.entry("gpu-box")) + t.Chdir(dir) + + // The banner goes to stdout; wait for it the way the daemon test waits + // for its stderr record. + out := filepath.Join(t.TempDir(), "stdout") + f, err := os.Create(out) + if err != nil { + t.Fatal(err) + } + old := os.Stdout + os.Stdout = f + t.Cleanup(func() { + os.Stdout = old + f.Close() + }) + + done := make(chan error, 1) + go func() { done <- cmdGateway([]string{"--loopback"}) }() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + data, _ := os.ReadFile(out) + if strings.Contains(string(data), "listening on "+gateway.LoopbackListen) { + break + } + time.Sleep(20 * time.Millisecond) + } + data, _ := os.ReadFile(out) + if !strings.Contains(string(data), "listening on "+gateway.LoopbackListen) { + t.Fatalf("gateway --loopback did not bind %s; stdout so far:\n%s", gateway.LoopbackListen, data) + } + + // No token was configured anywhere: an unauthenticated health answers. + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get("http://" + gateway.LoopbackListen + "/health") + if err != nil { + t.Fatalf("health: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("unauthenticated health answered %d, want 200", resp.StatusCode) + } + + interruptSelf(t) + select { + case err := <-done: + if err != nil { + t.Fatalf("gateway exited with %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("gateway did not exit on SIGINT") + } +} + +func TestFleetURL(t *testing.T) { + cases := map[string]string{ + "127.0.0.1:4000": "http://127.0.0.1:4000", + "gw.internal:4000": "http://gw.internal:4000", + ":4000": "http://:4000", + "0.0.0.0:4000": "http://:4000", + "[::]:4000": "http://:4000", + } + for got, want := range cases { + if url := fleetURL(got); url != want { + t.Errorf("fleetURL(%q) = %q, want %q", got, url, want) + } + } +} diff --git a/cmd/spinloop/main.go b/cmd/spinloop/main.go index d12660ca..dc6f3d1b 100644 --- a/cmd/spinloop/main.go +++ b/cmd/spinloop/main.go @@ -1192,6 +1192,21 @@ func applyBeforeLaunch(f spinloopPathFlag, providers string, h harness.Harness, if choice != nil && choice.APIKey != "" { resolve = fleetLaunchResolver(resolve, choice.APIKey) } + if choice != nil && choice.Gateway { + // A FLEET naming an endpoint authenticates with a token the client + // holds itself, resolved the way a key is resolved elsewhere: an ENV + // instruction, else the process environment, else the .env beside the + // Spinloop. Set nowhere, the launch cannot authenticate, and an + // endpoint that refuses every request is not one to point an agent at. + key := localKey(sel, localResolve) + if key == "" { + return spinloop.Selection{}, "", nil, nil, fmt.Errorf( + "no token to reach the FLEET endpoint %s: export %s, or set it in the .env beside %s", + choice.BaseURL, remoteAPIKeyEnv, path) + } + choice.APIKey = key + resolve = fleetLaunchResolver(resolve, key) + } if err := applySelection(sel, h, path, resolve); err != nil { return spinloop.Selection{}, "", nil, nil, err } diff --git a/cmd/spinloop/remote.go b/cmd/spinloop/remote.go index d712e2bf..a823b1f5 100644 --- a/cmd/spinloop/remote.go +++ b/cmd/spinloop/remote.go @@ -1242,27 +1242,34 @@ func deployConfigFor(sel spinloop.Selection, spinloopPath string) (remote.Deploy } // deployConfigForNode derives the same config for a machine that already -// exists — a fleet node being woken. Two things differ from a cloud -// deployment, both because the machine is the operator's rather than the +// exists — a fleet node being woken. Three things differ from a cloud +// deployment, all because the machine is the operator's rather than the // deployment's: sizing falls back to the engine's own default, so a Spinloop -// that `spinloop serve` runs happily needs no CONTEXT added merely to be routed; -// and the preset's bind survives, so an engine told to listen on 0.0.0.0 does. +// that `spinloop serve` runs happily needs no CONTEXT added merely to be +// routed; the preset's bind survives, so an engine told to listen on 0.0.0.0 +// does; and the Spinloop's own BASEURL is carried too, so a node with no +// preset does not wake onto the engine's default — llama.cpp's loopback, +// reachable from nobody but the node itself. func deployConfigForNode(sel spinloop.Selection, spinloopPath string) (remote.DeployConfig, error) { return deployConfig(sel, spinloopPath, deployTarget{ - runner: nodeRunnerFor, - owns: isNodeOwned, + runner: nodeRunnerFor, + owns: isNodeOwned, + carriesBaseURL: true, }) } // deployTarget is what the derivation cannot decide for itself: which runners // it accepts, whether a context size is required, which preset flags the -// destination assigns, and whether it fetches the weights itself (and so needs -// companions named). +// destination assigns, whether it fetches the weights itself (and so needs +// companions named), and whether the Spinloop's BASEURL is rendered into the +// engine's bind — a machine the operator owns listens wherever the Spinloop +// says; the cloud assigns its own port, so it keeps the field empty. type deployTarget struct { runner func(provider string) (string, error) requireContext bool seedsWeights bool owns func(key string) bool + carriesBaseURL bool } func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarget) (remote.DeployConfig, error) { @@ -1371,7 +1378,19 @@ func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarg dc.Companions = companionsFrom(global, params) } - dc.ServeArgs = preset.Flags(dropOwned(target.owns, global), dropOwned(target.owns, params)) + // The Spinloop's own bind is the final layer, where it travels at all: it + // wins over the preset's host and port exactly as it does for a local + // serve, so a woken engine listens wherever the Spinloop says rather than + // on the engine's default. + layers := [][]preset.Param{dropOwned(target.owns, global), dropOwned(target.owns, params)} + if target.carriesBaseURL { + bind, err := bindAddressParams(sel) + if err != nil { + return dc, err + } + layers = append(layers, bind) + } + dc.ServeArgs = preset.Flags(layers...) if dc.ServeArgs == nil { dc.ServeArgs = []string{} } diff --git a/cmd/spinloop/route.go b/cmd/spinloop/route.go index 280c3ca7..25c5daca 100644 --- a/cmd/spinloop/route.go +++ b/cmd/spinloop/route.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "net/url" "os" "path/filepath" "strings" @@ -62,12 +63,16 @@ func routeThroughFleet(sel spinloop.Selection, spinloopPath string, opts routeOp return nil, nil } // A FLEET naming a URL is the gateway shape: it has already done the - // choosing. Parsing accepts it so the eventual gateway needs no new - // keyword; nothing here can act on it yet. + // choosing, so there is no fleet file to read and no node to contact. The + // token is not resolved here: applyBeforeLaunch resolves it through the + // same chain the launch uses, so a missing value fails before anything is + // written. if isEndpoint(target) { - return nil, fmt.Errorf( - "FLEET %s names an endpoint, and gateway routing is not implemented yet: "+ - "name a fleet file to choose a node from", target) + return &fleet.Choice{ + Gateway: true, + BaseURL: endpointBaseURL(target), + Reason: "FLEET names an endpoint", + }, nil } cfg, err := fleet.Resolve(resolveFleetPath(target, opts.fleetPath != "", spinloopPath)) @@ -115,10 +120,22 @@ func routeThroughFleet(sel spinloop.Selection, spinloopPath string, opts routeOp if opts.noWake { return nil, fmt.Errorf("%w\nStart one with `spinloop fleet start `, or drop --no-wake to have spinloop do it", err) } + if !cfg.Wakes() { + // The fleet file says the machines are not to be started on demand. The + // refusal still names the node that would have been woken, the way a + // --no-wake refusal does: the setting decides whether to wake, not what + // would be woken. + if wake, ok := cfg.WouldWake(none.Results, fleet.ConstantConfig(dc, dcErr)); ok { + return nil, fmt.Errorf( + "%w\nwake is off in %s: start %s with `spinloop fleet start %s`", + err, cfg.Path, wake.Name, wake.Name) + } + return nil, fmt.Errorf("%w\nwake is off in %s", err, cfg.Path) + } if dcErr != nil { return nil, fmt.Errorf("%w\nand this Spinloop cannot be turned into something to start: %v", err, dcErr) } - choice, err = cfg.Wake(ctx, want, dc, none.Results, func(format string, args ...any) { + choice, err = cfg.Wake(ctx, want, fleet.ConstantConfig(dc, dcErr), none.Results, func(format string, args ...any) { fmt.Fprintf(os.Stderr, format, args...) }) if err != nil { @@ -131,9 +148,25 @@ func routeThroughFleet(sel spinloop.Selection, spinloopPath string, opts routeOp // announceChoice names the node a launch landed on before the agent starts, so // an unexpected route says so at the time rather than at the first request. func announceChoice(c *fleet.Choice) { + if c.Gateway { + fmt.Fprintf(os.Stderr, "Routing at the FLEET endpoint %s\n", c.BaseURL) + return + } fmt.Fprintf(os.Stderr, "Using %s at %s — %s\n", c.Node.Name, c.BaseURL, c.Reason) } +// endpointBaseURL is the address a launch gives an agent for a FLEET that names +// an endpoint: the value as given when it carries a path, and the OpenAI- +// compatible prefix added when it does not, so FLEET http://gw:4000 points the +// agent at http://gw:4000/v1. +func endpointBaseURL(target string) string { + u, err := url.Parse(target) + if err != nil || (u.Path != "" && u.Path != "/") { + return target + } + return strings.TrimRight(target, "/") + "/v1" +} + // resolveFleetPath resolves a fleet file's path. A relative FLEET is resolved // against the Spinloop that names it, the same rule PRESET and REMOTE follow — an // Spinloop and the fleet beside it travel together, and resolving against the diff --git a/cmd/spinloop/route_test.go b/cmd/spinloop/route_test.go index cd3134c4..057ffc28 100644 --- a/cmd/spinloop/route_test.go +++ b/cmd/spinloop/route_test.go @@ -314,20 +314,170 @@ func TestRoutingToARunningNodeToleratesAnUnusableParallel(t *testing.T) { }) } -// A FLEET naming a URL is the gateway shape: it parses, and says plainly that -// it is not implemented rather than being treated as a filename. -func TestFleetURLIsRefusedAsUnimplemented(t *testing.T) { +// A FLEET naming a URL is the gateway shape: the endpoint has already done the +// choosing, so no fleet file is read, no node is contacted, and the node- +// steering flags are inert. A value with no path gets the OpenAI-compatible +// prefix. +func TestFleetURLYieldsTheEndpoint(t *testing.T) { spinloopDir := routedSpinloop(t, "qwen3-27b", "http://gateway.internal:4000") sel, path, err := readSpinloop("test", spinloopDir) if err != nil { t.Fatal(err) } - _, err = routeThroughFleet(sel, path, routeOptions{}) - if err == nil { - t.Fatal("a gateway URL should fail for now") + captureStderr(t, func() { + c, err := routeThroughFleet(sel, path, routeOptions{node: "nobody", prefer: "sideways", noWake: true}) + if err != nil { + t.Fatalf("an endpoint FLEET should not consult any node: %v", err) + } + if !c.Gateway { + t.Fatalf("the choice should mark itself as an endpoint, got %+v", c) + } + if c.BaseURL != "http://gateway.internal:4000/v1" { + t.Errorf("an endpoint without a path gets the prefix, got %s", c.BaseURL) + } + }) +} + +func TestFleetURLWithAPathIsUsedAsGiven(t *testing.T) { + spinloopDir := routedSpinloop(t, "qwen3-27b", "http://gateway.internal:4000/proxy/v1") + sel, path, err := readSpinloop("test", spinloopDir) + if err != nil { + t.Fatal(err) + } + captureStderr(t, func() { + c, err := routeThroughFleet(sel, path, routeOptions{}) + if err != nil { + t.Fatal(err) + } + if !c.Gateway || c.BaseURL != "http://gateway.internal:4000/proxy/v1" { + t.Errorf("an endpoint carrying a path is used as given, got %+v", c) + } + }) +} + +// A pinned BASEURL wins over an endpoint FLEET, as it wins over a fleet file. +func TestPinnedBaseURLBeatsAnEndpointFleet(t *testing.T) { + spinloopDir := t.TempDir() + mustWrite(t, filepath.Join(spinloopDir, "Spinloop"), + "PROVIDER llamacpp\nMODEL qwen3-27b\nBASEURL http://pinned:9999/v1\nFLEET http://gateway.internal:4000\n") + sel, path, err := readSpinloop("test", spinloopDir) + if err != nil { + t.Fatal(err) + } + stderr := captureStderr(t, func() { + c, err := routeThroughFleet(sel, path, routeOptions{}) + if err != nil { + t.Fatal(err) + } + if c != nil { + t.Errorf("a pinned BASEURL is not routed, got %+v", c) + } + }) + if !strings.Contains(stderr, "Not routing") { + t.Errorf("spinloop should say it is not routing, got:\n%s", stderr) } - if !strings.Contains(err.Error(), "not implemented yet") { - t.Errorf("error should say it is not implemented, got: %v", err) +} + +// stubHarnessBinaryWithEnv is stubHarnessBinary plus a dump of the two +// variables a routed launch injects, for asserting what the agent actually +// got. +func stubHarnessBinaryWithEnv(t *testing.T, argsFile, envFile string) { + t.Helper() + dir := t.TempDir() + body := "#!/bin/sh\n" + + "printf '%s\\n' \"$@\" > " + argsFile + "\n" + + "printf 'BASE=%s\\nKEY=%s\\n' \"$OPENAI_BASE_URL\" \"$OPENAI_API_KEY\" > " + envFile + "\n" + if err := os.WriteFile(filepath.Join(dir, "opencode"), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// A FLEET naming an endpoint points the agent at it: the address gets the +// OpenAI-compatible prefix, and the token is resolved from the client's +// environment the way a key is resolved elsewhere. +func TestLaunchWithEndpointFleetPointsTheAgentAtTheGateway(t *testing.T) { + isolateConfig(t) + t.Setenv("OPENAI_API_KEY", "gw-token") + t.Setenv("OPENAI_BASE_URL", "") + argsFile := filepath.Join(t.TempDir(), "args") + envFile := filepath.Join(t.TempDir(), "env") + stubHarnessBinaryWithEnv(t, argsFile, envFile) + + spinloopDir := routedSpinloop(t, "qwen3-27b", "http://gateway.internal:4000") + captureStdout(t, func() { + if err := cmdHarness([]string{"--spinloop=" + spinloopDir, "--", "run"}); err != nil { + t.Fatalf("cmdHarness: %v", err) + } + }) + if _, err := os.ReadFile(argsFile); err != nil { + t.Fatalf("harness was not launched: %v", err) + } + data, err := os.ReadFile(envFile) + if err != nil { + t.Fatal(err) + } + out := string(data) + if !strings.Contains(out, "BASE=http://gateway.internal:4000/v1") { + t.Errorf("the agent's base URL should be the endpoint with the prefix, got:\n%s", out) + } + if !strings.Contains(out, "KEY=gw-token") { + t.Errorf("the agent should carry the gateway's token as its key, got:\n%s", out) + } +} + +// A FLEET naming an endpoint with no token anywhere fails before the agent +// launches and before the harness config is written, naming the variable. +func TestLaunchWithEndpointFleetFailsWithoutAToken(t *testing.T) { + home := isolateConfig(t) + t.Setenv("OPENAI_API_KEY", "") + argsFile := filepath.Join(t.TempDir(), "args") + stubHarnessBinary(t, "opencode", argsFile) + + spinloopDir := routedSpinloop(t, "qwen3-27b", "http://gateway.internal:4000") + captureStdout(t, func() { + err := cmdHarness([]string{"--spinloop=" + spinloopDir, "--", "run"}) + if err == nil { + t.Fatal("a launch that cannot authenticate the endpoint should fail") + } + if !strings.Contains(err.Error(), "OPENAI_API_KEY") { + t.Errorf("the failure should name the variable to set, got:\n%v", err) + } + }) + if _, err := os.ReadFile(argsFile); err == nil { + t.Error("the agent launched without a token to reach the endpoint") + } + if _, err := os.Stat(filepath.Join(home, ".config", "opencode", "opencode.json")); err == nil { + t.Error("the harness config was written for a launch that could not authenticate") + } +} + +// A fleet that declares wake: off refuses to start anything when nothing is +// serving, and names the node that would have been woken with the command that +// would start it. +func TestWakeOffRefusesNamingTheNode(t *testing.T) { + node := newRoutableNode(t, "", false, 0) + dir := t.TempDir() + fleetPath := fleetFileIn(t, dir, "wake: off\nnodes:\n"+node.entry("idle-box")) + spinloopDir := routedSpinloop(t, "qwen3-27b", fleetPath) + + sel, path, err := readSpinloop("test", spinloopDir) + if err != nil { + t.Fatal(err) + } + captureStderr(t, func() { + _, err := routeThroughFleet(sel, path, routeOptions{}) + if err == nil { + t.Fatal("wake: off with nothing serving should fail") + } + for _, want := range []string{"wake is off", "idle-box", "spinloop fleet start idle-box"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("message should mention %q, got:\n%s", want, err) + } + } + }) + if node.started { + t.Error("wake: off started an engine") } } @@ -456,6 +606,27 @@ func TestCmdFleetRouteExplainsTheChoice(t *testing.T) { } } +// A FLEET naming an endpoint has already chosen: the route says where a launch +// would point the agent, and queries nothing. +func TestCmdFleetRouteAgainstAnEndpoint(t *testing.T) { + spinloopDir := routedSpinloop(t, "qwen3-27b", "http://gw.internal:4000") + + out := captureStdout(t, func() { + if err := cmdFleetRoute([]string{filepath.Join(spinloopDir, "Spinloop")}); err != nil { + t.Fatal(err) + } + }) + for _, want := range []string{ + "gw.internal:4000 (an endpoint, not a fleet file)", + "would point the agent at http://gw.internal:4000/v1", + "nothing is started", + } { + if !strings.Contains(out, want) { + t.Errorf("output should mention %q, got:\n%s", want, out) + } + } +} + // The flag lets the two preferences be compared on a live fleet without // editing the file. func TestCmdFleetRoutePreferenceFlagBeatsTheFile(t *testing.T) { @@ -524,6 +695,30 @@ func TestCmdFleetRouteStartsNothing(t *testing.T) { } } +// A fleet that declares wake: off still reports, when nothing is running, the +// node whose source describes the model and the command that would start it — +// but says a launch would refuse. +func TestCmdFleetRouteWakeOffRefusal(t *testing.T) { + node := newRoutableNode(t, "", false, 0) + dir := t.TempDir() + fleetPath := fleetFileIn(t, dir, "wake: off\nnodes:\n"+node.entry("idle-box")) + spinloopDir := routedSpinloop(t, "qwen3-27b", fleetPath) + + out := captureStdout(t, func() { + if err := cmdFleetRoute([]string{filepath.Join(spinloopDir, "Spinloop")}); err != nil { + t.Fatal(err) + } + }) + for _, want := range []string{"wake is off", "idle-box", "spinloop fleet start idle-box", "Nothing has been started"} { + if !strings.Contains(out, want) { + t.Errorf("output should mention %q, got:\n%s", want, out) + } + } + if node.started { + t.Error("fleet route started an engine") + } +} + // A Spinloop naming no fleet, with no --fleet, has nothing to report. func TestCmdFleetRouteNeedsAFleet(t *testing.T) { spinloopDir := routedSpinloop(t, "qwen3-27b", "") diff --git a/cmd/spinloop/serve_daemon.go b/cmd/spinloop/serve_daemon.go index 5112900b..b970545a 100644 --- a/cmd/spinloop/serve_daemon.go +++ b/cmd/spinloop/serve_daemon.go @@ -245,7 +245,7 @@ func runServeForegroundAPI(sel spinloop.Selection, spinloopPath string, engine s if model == "" { model = sel.Alias } - d.SetServed(sel.Provider, model) + d.SetServed(sel.Provider, model, sel.Alias) d.SetScrape(scrapeTargetFor(engine, sel.BaseURL, argv)) d.SetEngineEndpoint(engineEndpointFor(engine, sel.BaseURL, argv)) diff --git a/cmd/spinloop/serve_daemon_test.go b/cmd/spinloop/serve_daemon_test.go index c597ccd8..92634609 100644 --- a/cmd/spinloop/serve_daemon_test.go +++ b/cmd/spinloop/serve_daemon_test.go @@ -990,6 +990,77 @@ ctx-size = 4096 } } +// The Spinloop's own BASEURL travels where the preset's bind does: a woken +// node with no preset must not land on the engine's default bind, which for +// llama.cpp is loopback — reachable from nobody but the node itself. +func TestNodeDeployConfigCarriesTheSpinloopBind(t *testing.T) { + spinloopPath := writeDeploySpinloop(t, + "PROVIDER llamacpp\nMODEL org/model:Q4_K_M\nCONTEXT 4096\nBASEURL http://0.0.0.0:8080/v1\n", "") + sel, _, err := readSpinloop("test", spinloopPath) + if err != nil { + t.Fatal(err) + } + + node, err := deployConfigForNode(sel, spinloopPath) + if err != nil { + t.Fatal(err) + } + args := strings.Join(node.ServeArgs, " ") + for _, want := range []string{"--host 0.0.0.0", "--port 8080"} { + if !strings.Contains(args, want) { + t.Errorf("a node's serve args should carry the Spinloop's bind %q, got: %s", want, args) + } + } + + // The cloud assigns its own bind, so the Spinloop's is not carried there. + cloud, err := deployConfigFor(sel, spinloopPath) + if err != nil { + t.Fatal(err) + } + cloudArgs := strings.Join(cloud.ServeArgs, " ") + for _, unwanted := range []string{"--host", "--port"} { + if strings.Contains(cloudArgs, unwanted) { + t.Errorf("the cloud sets its own bind, so %q should not be carried, got: %s", unwanted, cloudArgs) + } + } +} + +// Where the Spinloop states a bind and the preset states one too, the +// Spinloop's wins — the same precedence a local serve gives its own BASEURL +// over the preset — and the engine is told once, not twice. +func TestNodeDeployConfigBindBeatsThePreset(t *testing.T) { + spinloopPath := writeDeploySpinloop(t, + "PROVIDER llamacpp\nALIAS qwen\nPRESET ./preset.ini\nBASEURL http://0.0.0.0:8080/v1\n", + `[*] +host = 127.0.0.1 +port = 9090 + +[qwen] +hf = org/model:Q4_K_M +ctx-size = 4096 +`) + sel, _, err := readSpinloop("test", spinloopPath) + if err != nil { + t.Fatal(err) + } + + node, err := deployConfigForNode(sel, spinloopPath) + if err != nil { + t.Fatal(err) + } + args := strings.Join(node.ServeArgs, " ") + for _, want := range []string{"--host 0.0.0.0", "--port 8080"} { + if !strings.Contains(args, want) { + t.Errorf("the Spinloop's bind should win, got: %s", args) + } + } + for _, unwanted := range []string{"127.0.0.1", "9090"} { + if strings.Contains(args, unwanted) { + t.Errorf("the preset's bind should be overridden, got: %s", args) + } + } +} + // mtplxNodePreset is an MTPLX-vocabulary preset: long-form keys, the model under // `model`, the window under `context-window`, the cap under // `max-active-requests`, a served name under `model-id`, and a scheduling mode diff --git a/docs/README.md b/docs/README.md index 9d213c2f..82822615 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,8 +34,9 @@ Four words carry the whole tool: - [The HTTP control API](http-api.md) — driving a supervised engine over HTTP, and the [OpenAPI contract](openapi.yaml) for writing a client - [Running a fleet](commands/fleet.md) — one spinloop watching every machine you - run, with a [containerised fleet](../examples/fleet-docker/) you can bring up - on a laptop + run, with a [containerised fleet](../examples/fleet-docker/) and a + [containerised gateway](../examples/gateway-docker/) you can bring up on a + laptop - [Environment variables](env-vars.md) — every variable spinloop reads - [Runnable examples](../examples/) — ready-to-apply Spinloops with walkthroughs - [Deploying your own cloud GPU endpoint](../remote/) — the AWS project behind @@ -62,6 +63,7 @@ Four words carry the whole tool: | [`spinloop up`](commands/up.md) | Start the engine this directory holds: the fleet, or the `Spinloop`'s server | | [`spinloop daemon`](commands/serve.md#the-control-api---api-and-spinloop-daemon) | Supervise an engine over the [control API](http-api.md) | | [`spinloop fleet`](commands/fleet.md) | Observe and drive the engines on every machine you run | +| [`spinloop gateway`](commands/gateway.md) | Serve the fleet under one OpenAI-compatible endpoint | | [`spinloop remote`](commands/remote.md) | Run the model on a cloud GPU that stops when you do | | [`spinloop export`](commands/export.md) | Capture the current setup as a `Spinloop` | | [`spinloop harness`](commands/harness.md) | Launch the agent, optionally configuring it first | diff --git a/docs/commands/fleet.md b/docs/commands/fleet.md index 700e1636..25eb2aab 100644 --- a/docs/commands/fleet.md +++ b/docs/commands/fleet.md @@ -170,6 +170,29 @@ nodes: … override the file for one command, which is the cheap way to see what the other setting would do before committing to it. +### Waking + +`wake` decides whether routing may start an engine on a node that is not +running one: + +```yaml +wake: off # or: on +nodes: … +``` + +- **`on`** (the default, and the behaviour of a file that declares nothing) — + when nothing is serving, spinloop starts a node and waits for its engine to + answer before the agent launches or the request is answered. +- **`off`** — a request nothing is serving fails rather than starting + anything, naming the node that would have been woken and the `spinloop fleet + start ` command that would start it. Use it where the machines are not + to be started on demand — the models are loaded by hand, or someone else + drives the starts. + +An explicit `--no-wake` still refuses to start anything, whatever the file +says; an explicit `spinloop fleet start` does the opposite — it always starts, +because it was asked. + ### Tokens `tokenEnv` names an environment variable; the value is resolved from the diff --git a/docs/commands/gateway.md b/docs/commands/gateway.md new file mode 100644 index 00000000..d2a723e3 --- /dev/null +++ b/docs/commands/gateway.md @@ -0,0 +1,142 @@ +# spinloop gateway + +Serve a [fleet](fleet.md) under one OpenAI-compatible endpoint. The gateway is +the fleet client wearing a server: it holds a `fleet.yaml`, answers +`/v1/models` and completion requests by choosing a node with the fleet's own +selector, and wakes a node when nothing is serving what a request asks for — so +a machine running agents needs nothing but a URL and one token. + +```sh +spinloop gateway # serves ./fleet.yaml on :4000 +spinloop gateway --fleet ./fleet.yaml +spinloop gateway --listen 0.0.0.0:4000 +spinloop gateway --api-token-file /run/secrets/gateway +``` + +It runs in the foreground, the way [`spinloop serve`](serve.md) does: it holds +the fleet file it serves, and a signal shuts it down cleanly. On startup it +resolves the fleet file and its own token, and checks the token references the +file names — a `tokenEnv` or `engineTokenEnv` variable set nowhere fails here, +naming the node, rather than surfacing later as a per-request authentication +failure. It prints the address a Spinloop names in its `FLEET`: + +``` +Gateway for fleet.yaml is listening on [::]:4000 +Name http://:4000 in a Spinloop's FLEET +``` + +The host it can know is the one it was told to bind; for a wildcard bind the +machine's own name is only the operator's to know, so the printed address says +`` and you fill in whatever this machine is called from the other +side. + +## Pointing an agent at it + +A Spinloop names the gateway's address in its `FLEET` — a URL, not a file: + +```dockerfile +PROVIDER llamacpp +MODEL qwen3-27b +FLEET http://gateway.internal:4000 +``` + +The launch reads no fleet file and contacts no node — the endpoint has already +done the choosing — and the agent it launches authenticates with the +gateway's token, as `OPENAI_API_KEY`, resolved the way a key is resolved +elsewhere: an `ENV` instruction, then the process environment, then the `.env` +beside the Spinloop. An agent pointed at the gateway holds exactly that one +credential; the node tokens and engine keys live with the gateway, which +presents them to the nodes and the engines. See +[The `Spinloop` file](../spinloop-file.md#running-the-model-on-another-machine-you-own). + +## What it answers + +| Path | Meaning | +| ---- | ------- | +| `GET /health` | That the gateway is up. It touches no node on purpose — it is how you tell the gateway down from the fleet down. | +| `GET /v1/models` | The OpenAI list of what the fleet is running: the served name when a running node reports one, else the model id, duplicates once. Nothing running is an empty list, not an error. | +| `POST /v1/chat/completions` | Routed to the node serving the request's `model`, the way a launch routes. | +| `POST /v1/completions` | The same, for the completions endpoint. | + +A request naming no `model` is refused saying so, and a path the gateway does +not serve is refused with a `404` naming the ones it does. + +### Routing a request + +A completion request is answered by the fleet's own selection: a node already +running the model wins, ranked by the fleet file's `prefer` with fleet-file +order breaking ties. A node whose engine is bound to loopback without an +[`engine` override](fleet.md#where-a-nodes-engine-answers) is never selected, +and when it is the only match the failure says so rather than holding the +request until the wake timeout. + +The request's body goes out unmodified and streamed replies are flushed as +they are produced, so a `stream: true` request streams through. The caller's +authorisation never travels past the gateway: the engine is reached with the +key its fleet entry names (`engineTokenEnv`, or the fleet-wide `apiKeyEnv` for +a `kind: remote` node), and an ungated engine is reached with none. The reply +the engine gives is the reply the caller gets — the gateway never retries +another node, and an upstream failure reaches the caller as an error naming +the node. + +### Waking a node + +When no running node serves the model and the fleet file's +[`wake`](fleet.md#waking) setting allows it, the gateway starts a node with +the config that node's own Spinloop source resolves to — only nodes whose +source describes the requested model are candidates, and a node whose stored +config already matches is tried first — and holds the request until the engine +answers, bounded by `--wake-timeout` (default 5m). A timeout fails the request +saying so and leaves the engine running, so a slow load is not thrown away. +Concurrent requests for the same model wake at most one engine: a request that +loses the start to the daemon's "already running" answer takes the node the +other one started. + +With `wake: off`, or when no node's source describes the model, a request +nothing is serving fails without starting anything, naming the node and the +`spinloop fleet start ` command that would start it. + +The gateway needs the same environment a machine running +`spinloop fleet start` would: the tokens the fleet file names, set in its +process environment or in a `.env` beside the fleet file. + +## The gateway's token + +Callers present the gateway's token as a bearer token on every request — a +wrong or missing one is a `401`. The token comes from one of three places, the +same rules the [daemon's](serve.md#the-control-api---api-and-spinloop-daemon) +token follows, and giving two at once is an error rather than a silent +precedence: + +| Source | Notes | +| ------ | ----- | +| `--api-token-file ` | The file's contents, trimmed. | +| `SPINLOOP_API_TOKEN` | The environment. | +| `--api-token ` | The token itself — readable by every local user through `ps`, like the daemon's. | + +A non-loopback listen with no token refuses to start, naming the three ways to +supply one; a loopback listen (`--loopback`, or `--listen 127.0.0.1:4000`) +needs none. The default, `:4000`, binds every interface — the shape a gateway +on a shared machine wants, and the reason the token is not optional there. + +## Flags + +| Flag | Meaning | +| ---- | ------- | +| `-f`, `--fleet` | The fleet file to serve (default `./fleet.yaml`) | +| `--listen` | The address to listen on (default `:4000`) | +| `-l`, `--loopback` | Bind to loopback on the default port (`127.0.0.1:4000`); needs no token | +| `--api-token-file` | Read the gateway's bearer token from this file | +| `--api-token` | The gateway's bearer token | +| `--wake-timeout` | How long to wait for a woken engine to answer (default 5m) | + +## See also + +- [`spinloop fleet`](fleet.md) — the file the gateway serves, and the nodes it + drives +- [`spinloop daemon`](serve.md#the-control-api---api-and-spinloop-daemon) — what + each node runs +- [`examples/gateway-docker/`](../../examples/gateway-docker/) — a gateway and + its fleet in containers, with the test suite that asserts all of this +- [The `Spinloop` file](../spinloop-file.md) — the `FLEET` that points an agent + here diff --git a/docs/env-vars.md b/docs/env-vars.md index 50d98041..9959b426 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -13,7 +13,7 @@ from the environment or a `.env` beside the Spinloop — never written into an | `SPINLOOP_ALIAS` | every command that takes a Spinloop path | A name registered with [`spinloop alias`](commands/alias.md), used when the command is given no path. Precedence: the path or alias argument > `SPINLOOP_ALIAS` > `./Spinloop`. It holds a registry name, never a path, and a same-named file in the working directory does not shadow it. It decides *which* Spinloop is the default, not *whether* one is applied — a bare `spinloop harness` still applies nothing, and `spinloop alias` ignores it. | | `SPINLOOP_PROVIDERS` | `list`, `add`, `apply`, … | Path to a `providers.yaml` that overrides the built-in catalogue. Precedence: `--providers` flag > `SPINLOOP_PROVIDERS` > embedded. | | `SPINLOOP_BASE_URL` | `add`, `apply` | Base-URL override for the provider being configured. Precedence: `--base-url`/`-u` > `SPINLOOP_BASE_URL` > the provider's own option var > the catalogue default. | -| `SPINLOOP_API_TOKEN` | `spinloop daemon`, `spinloop serve --api` | Bearer token for the daemon control API. One of three peer sources, alongside `--api-token-file` and `--api-token`; two at once is an error. From a service manager prefer the file form — see [serve](commands/serve.md). A non-loopback API listen without any of them refuses to start. | +| `SPINLOOP_API_TOKEN` | `spinloop daemon`, `spinloop serve --api`, `spinloop gateway` | Bearer token for the daemon control API — and the token a [gateway](commands/gateway.md)'s callers must present. One of three peer sources, alongside `--api-token-file` and `--api-token`; two at once is an error. From a service manager prefer the file form — see [serve](commands/serve.md). A non-loopback listen without any of them refuses to start. | | `SPINLOOP_LOG_LEVEL` | `spinloop daemon`, `spinloop serve` | How much spinloop records about the control API and the supervised engine: `debug`, `info` (default), `warn` or `error`. Precedence: `--log-level` flag > `SPINLOOP_LOG_LEVEL` > `info`. An unrecognised value refuses to start rather than falling back to the default. Under `spinloop serve` the `.env` beside the Spinloop can set it; the daemon reads no Spinloop, so there it comes from the environment its service manager gives it. Records go to stderr; see [what gets logged](commands/serve.md#what-gets-logged). | | *(per-node, named by `tokenEnv`)* | `spinloop fleet` | A fleet node's bearer token. `fleet.yaml` names the variable rather than holding the value; it resolves from the environment, then the `.env` beside the fleet file. See [fleet](commands/fleet.md). | | *(per-node, named by `engineTokenEnv`)* | `spinloop fleet`, `spinloop harness` | The key a fleet node's **engine** is gated with. Resolved the same way, and supplied by the client when it starts that engine — so the node holds no key of its own and the two ends cannot disagree. See [fleet](commands/fleet.md). | diff --git a/docs/openapi.yaml b/docs/openapi.yaml index fde05cee..c216528a 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -298,6 +298,12 @@ components: model: type: string description: What it is serving, when known. + servedName: + type: string + description: | + The name the engine answers to — the served name its deploy config + or Spinloop set — reported beside `model` when set. An aliased + engine answers to both, and a caller may know either. uptimeSeconds: type: integer description: How long the engine has been running. Zero unless running. @@ -344,9 +350,17 @@ components: purpose: the daemon knows its engine binds `127.0.0.1:8080`, which is useless to anyone else, and it cannot know the name a client reaches this host by — a LAN name, a tailscale name, a published container - port. The caller composes these against the host it already has. + port. The caller composes these against the host it already has. A node + that does know that name — a remote environment, whose control plane + publishes the instance's address — reports it in `host`. required: [port] properties: + host: + type: string + description: | + The name or address a client reaches the engine by, when the node + knows it. A daemon leaves it absent; a remote environment's status + fills it with the instance's published address. port: type: integer description: | diff --git a/docs/spinloop-file.md b/docs/spinloop-file.md index 182109c8..e015a85b 100644 --- a/docs/spinloop-file.md +++ b/docs/spinloop-file.md @@ -145,12 +145,28 @@ picking one. As with `REMOTE`, note the missing `BASEURL` — the address is whichever node gets chosen. Writing one pins the address and turns routing off, and spinloop says so rather than choosing a node and discarding it. -A `FLEET` may also name a URL rather than a file, for a single endpoint that has -already done the choosing. That is the shape the spinloop gateway will take; it is -not implemented yet, and naming one today fails saying so. +A `FLEET` may also name a URL rather than a file: a single endpoint that has +already done the choosing, the shape +[`spinloop gateway`](commands/gateway.md) serves: + +```dockerfile +PROVIDER llamacpp +MODEL qwen3-27b +FLEET http://gateway.internal:4000 +``` + +Naming one reads no fleet file and contacts no node. The launch is pointed at +the address as given — with the OpenAI-compatible `/v1` prefix added when it +carries no path, and a value that already carries one used as given — and the +agent it launches authenticates with the endpoint's token, resolved the way a +key is resolved elsewhere: an `ENV` instruction, then the process environment, +then the `.env` beside the Spinloop. A variable already set wins, as on the +remote path. Set nowhere, the launch fails before it writes anything, naming +`OPENAI_API_KEY`. See [`spinloop fleet route`](commands/fleet.md#which-node-would-i-get) to check -which node you would get before launching anything. +which node you would get before launching anything — a route against an +endpoint just names it, without querying a node or starting one. ## Syntax diff --git a/examples/gateway-docker/.env.example b/examples/gateway-docker/.env.example new file mode 100644 index 00000000..2f69794a --- /dev/null +++ b/examples/gateway-docker/.env.example @@ -0,0 +1,12 @@ +# Copy to .env beside fleet.yaml, then `docker compose up -d --build`. +# +# Every value is a bearer secret for one box: the NODE_*_TOKENs gate each +# node's daemon control API, the NODE_*_ENGINE_KEYs gate each engine, and +# GATEWAY_TOKEN gates the gateway (both gateway services share it). Any +# non-empty values work — these are example secrets for a local stack, not +# secrets to reuse anywhere real. +NODE_A_TOKEN=node-a-dev-token +NODE_B_TOKEN=node-b-dev-token +NODE_A_ENGINE_KEY=node-a-engine-dev-key +NODE_B_ENGINE_KEY=node-b-engine-dev-key +GATEWAY_TOKEN=gateway-dev-token diff --git a/examples/gateway-docker/Dockerfile b/examples/gateway-docker/Dockerfile new file mode 100644 index 00000000..2ccaf869 --- /dev/null +++ b/examples/gateway-docker/Dockerfile @@ -0,0 +1,60 @@ +# A fleet node, and the gateway that fronts it: one image, two entry points. +# +# The node is a real `spinloop daemon` supervising a fake engine — Imposter's +# native engine standing in for llama-server, so the stack needs no GPU and no +# model. The gateway is the same binary running `spinloop gateway` over the +# fleet file baked in beside it; compose points each service at its entry. + +# Build spinloop from the working tree, so the stack tests THIS commit rather +# than a published artifact. +FROM golang:1.25-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +# Only what the binary needs, so the image cannot depend on anything else in +# the tree and rebuilds stay cheap. +COPY cmd ./cmd +COPY internal ./internal +RUN CGO_ENABLED=0 go build -o /out/spinloop ./cmd/spinloop + +FROM alpine:3.20 +ARG TARGETARCH +# Pinned so a node image is reproducible; bump deliberately. +ARG IMPOSTER_VERSION=5.21.3 + +# procps gives the collector a vmstat whose output matches what it parses; +# BusyBox's differs, so CPU would otherwise be absent. +RUN apk add --no-cache ca-certificates curl bash procps + +# Imposter's native engine — a standalone binary, which is what lets the daemon +# supervise it as a child process the way it would a real engine. +RUN curl -fsSL -o /tmp/imposter.tar.gz \ + "https://github.com/imposter-project/imposter-go/releases/download/v${IMPOSTER_VERSION}/imposter-go_linux_${TARGETARCH}.tar.gz" \ + && curl -fsSL -o /tmp/checksums.txt \ + "https://github.com/imposter-project/imposter-go/releases/download/v${IMPOSTER_VERSION}/checksums.txt" \ + && (cd /tmp && grep " imposter-go_linux_${TARGETARCH}.tar.gz\$" checksums.txt | sed 's/imposter-go_linux_.*/imposter.tar.gz/' | sha256sum -c -) \ + && tar -xzf /tmp/imposter.tar.gz -C /usr/local/bin imposter-go \ + && chmod +x /usr/local/bin/imposter-go \ + && rm -f /tmp/imposter.tar.gz /tmp/checksums.txt + +COPY --from=build /out/spinloop /usr/local/bin/spinloop +COPY examples/gateway-docker/shim/llama-server /usr/local/bin/llama-server +COPY examples/gateway-docker/engine /opt/engine +# The gateway's own files: the two fleet files it can serve, and the nodes' +# Spinloop source beside them, so a wake resolves what a node runs from inside +# the container. The host-side fleet.yaml points at the same file where it +# lives in the tree, node/Spinloop. +COPY examples/gateway-docker/gateway /opt/gw +COPY examples/gateway-docker/node/Spinloop /opt/gw/Spinloop +RUN chmod +x /usr/local/bin/llama-server + +# A container has no useful $HOME, so pin spinloop's config directory — the +# same reason the cloud instance's daemon unit pins it. +ENV SPINLOOP_CONFIG_DIR=/var/lib/spinloop +WORKDIR /opt/node + +EXPOSE 4242 +# The default entry is the node; the gateway services override it in compose. +# Bind the control API on all interfaces so the gateway container can reach +# it; a token is required for that, supplied per-node by compose. +CMD ["spinloop", "daemon", "--api-addr", "0.0.0.0:4242"] diff --git a/examples/gateway-docker/README.md b/examples/gateway-docker/README.md new file mode 100644 index 00000000..5c279f87 --- /dev/null +++ b/examples/gateway-docker/README.md @@ -0,0 +1,153 @@ +# A gateway you can actually run + +Two `spinloop daemon` nodes and a `spinloop gateway` in front of them, on your +laptop, in containers — so you can see what +[`spinloop gateway`](../../docs/commands/gateway.md) does before pointing a real +fleet at it. No GPUs, no cloud, no model downloads. + +```sh +cp .env.example .env +docker compose up -d --build + +# from this directory, with the tokens exported +set -a && . ./.env && set +a + +# the gateway's own surface +curl -H "Authorization: Bearer $GATEWAY_TOKEN" http://127.0.0.1:4000/v1/models +curl -X POST -H "Authorization: Bearer $GATEWAY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"model":"fake-model","messages":[{"role":"user","content":"hi"}]}' \ + http://127.0.0.1:4000/v1/chat/completions + +# and the fleet underneath it, the way spinloop fleet drives any fleet +spinloop fleet status --fleet ./fleet.yaml +spinloop fleet start node-b --fleet ./fleet.yaml +``` + +The stack brings up **two gateways** over the same two nodes: `gateway` on port +4000, which wakes a node when nothing is serving, and `gateway-cold` on port +4001, which refuses to — the `wake: off` policy from its own fleet file, so the +two ways of running a fleet are visible side by side. + +## What is real and what is not + +**Real**: each node runs the actual `spinloop daemon` from this repository, +serving its control API over the network with bearer-token auth, and supervises +its engine as a real child process. The gateway is the same binary running +`spinloop gateway`: it chooses a node with the fleet's own selector, wakes one +when nothing is serving, holds the request until the engine answers, and swaps +the caller's authorisation for the engine key its fleet entry names. + +**Not real**: the engine. Instead of `llama-server` there is a +[`llama-server` shim](shim/llama-server) that starts +[Imposter](https://imposter.sh)'s native engine, which serves a canned +`/health`, a `/metrics` in llama.cpp's Prometheus dialect, and OpenAI-shaped +completion replies — streamed, when asked, in the server-sent-events shape. So +a request through the gateway genuinely travels to a woken node and back, and +the streamed reply you see is the one the fake engine produced. Nothing is +inferring anything. + +That trade is deliberate: what is being demonstrated (and tested) is the +gateway's routing, waking and key handling, not inference. + +**Also real**: the keys. Each engine is gated with the key its fleet entry +names — the node reports a key is required and never what it is, and the key +reaches the engine as a file path, so `docker compose exec node-a ps ax` shows +`--api-key-file`, not the key. The caller of the gateway presents only the +gateway's token; the node tokens and engine keys live with the gateway, which +is the one place that holds all of them. + +There are three Spinloops here: + +- [`client/Spinloop`](client/Spinloop) — what an *agent's* machine wears. Its + `FLEET` is the gateway's address, a URL rather than a file: the gateway has + done the choosing, and the agent is only pointed at it, with the gateway's + token as its key. +- [`node/Spinloop`](node/Spinloop) — what a *node* runs when started. Its + `BASEURL` binds the engine to every interface, which is why the gateway — a + different container — can reach it at all. +- the nodes hold no Spinloop of their own in the container; the image bakes a + copy of `node/Spinloop` beside the gateway's fleet files so the gateway's own + wakes resolve what a node runs. + +## Things worth trying + +```sh +# Cold request: nothing is serving, so the gateway wakes a node, holds the +# request until the engine answers, and streams the reply back. +curl -X POST -H "Authorization: Bearer $GATEWAY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"model":"fake-model","stream":true,"messages":[{"role":"user","content":"hi"}]}' \ + http://127.0.0.1:4000/v1/chat/completions + +# The same at the wake: off gateway: refused, naming the node and the command +# that would start it. +curl -i -X POST -H "Authorization: Bearer $GATEWAY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"model":"fake-model","messages":[{"role":"user","content":"hi"}]}' \ + http://127.0.0.1:4001/v1/chat/completions + +# wake: off still routes to what is already running — it decides whether to +# start, not whether to answer. +spinloop fleet start node-a --fleet ./fleet.yaml +curl -X POST -H "Authorization: Bearer $GATEWAY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"model":"fake-model","messages":[{"role":"user","content":"hi"}]}' \ + http://127.0.0.1:4001/v1/chat/completions + +# A wrong token is a 401, not a routing decision. +curl -i http://127.0.0.1:4000/v1/models + +# The engine, directly: gated, like any engine the fleet gates. +curl -i -X POST http://127.0.0.1:18080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"fake-model","messages":[{"role":"user","content":"hi"}]}' + +# Launch an agent against the gateway: the agent gets the gateway's address +# and the gateway's token, and nothing else. +OPENAI_API_KEY="$GATEWAY_TOKEN" spinloop harness ./client/Spinloop +``` + +## It is also the integration test + +`./run-tests.sh` drives this same stack and asserts the behaviours above: a +model is listed once running, a cold request wakes a node and streams a reply, +the engine key is injected and never reaches the caller, a wrong token is 401, +and `wake: off` refuses. CI runs it on every pull request, which is the point: +an example that is exercised cannot quietly stop working. + +```sh +./run-tests.sh # up, assert, tear down +./run-tests.sh --keep # leave the stack running to poke at +``` + +## How it fits together + +| File | What it is | +| --- | --- | +| `compose.yaml` | Two nodes, a gateway that wakes, a gateway that refuses to. Every service that listens on a non-loopback address needs a token, and the gateways need the fleet file's node tokens and engine keys in their environment. | +| `fleet.yaml` | The *operator's* view: the two nodes over their published ports, with `engine:` blocks because the engines are published on ports the daemons cannot know. | +| `gateway/fleet.yaml` | What the `gateway` service serves: the same two nodes, addressed by compose service name — where the gateway can reach them, with no `engine:` override needed. | +| `gateway/fleet-cold.yaml` | The same fleet with `wake: off`, served by `gateway-cold`. | +| `Dockerfile` | Builds spinloop from this working tree, adds the Imposter engine and the shim, and bakes the gateway's files in. | +| `shim/llama-server` | Stands in for the engine binary. Reads the key file the daemon passes and hands the mock its gate as an environment variable, so the value never rides on a command line. | +| `engine/` | What the fake engine serves: `/health` and `/metrics` for the daemon, and the gated, stream-answering OpenAI routes. | +| `node/Spinloop` | What a node runs when started: a model, and a `BASEURL` that binds the engine to every interface. | +| `client/Spinloop` | What an *agent's* machine wears: a model, and a `FLEET` that is the gateway's address. | + +Two details that are easy to get wrong, and matter: + +- **The node's `BASEURL` binds the engine wide.** Without it llama-server + binds `127.0.0.1`, the daemon reports the engine loopback-only, and the + gateway is right to refuse routing to it — an engine that answers only on + its own machine is not a candidate for a gateway on another. +- **The shim execs the engine binary, not `imposter up`.** The CLI wrapper + exits 0 when its child dies, which the daemon would correctly record as a + clean stop — so a crash test would pass while testing nothing. + +## See also + +- [`examples/fleet-docker/`](../fleet-docker/) — a plain fleet, no gateway +- [`docs/commands/gateway.md`](../../docs/commands/gateway.md) +- [`docs/spinloop-file.md`](../../docs/spinloop-file.md) — the endpoint form of `FLEET` +- [HTTP Control API](../../docs/http-api.md) diff --git a/examples/gateway-docker/client/Spinloop b/examples/gateway-docker/client/Spinloop new file mode 100644 index 00000000..c56f8a21 --- /dev/null +++ b/examples/gateway-docker/client/Spinloop @@ -0,0 +1,13 @@ +# What an agent's machine wears to use this stack: a model, and a FLEET that +# names the gateway's address rather than a fleet file. The gateway has done +# the choosing; the agent is only pointed at it. +# +# The agent holds exactly one credential — the gateway's token, as +# OPENAI_API_KEY (from the environment, or a .env beside this file). The node +# tokens and engine keys live with the gateway, which presents them to the +# nodes and the engines; none of them ever reach the agent. +PROVIDER llamacpp +MODEL org/fake-model +ALIAS fake-model +CONTEXT 4096 +FLEET http://127.0.0.1:4000 diff --git a/examples/gateway-docker/compose.yaml b/examples/gateway-docker/compose.yaml new file mode 100644 index 00000000..4f590506 --- /dev/null +++ b/examples/gateway-docker/compose.yaml @@ -0,0 +1,63 @@ +# Two fleet nodes and two gateways in front of them: one that wakes a node +# when nothing is serving, one that refuses to, so the two wake policies are +# visible side by side. +# +# cp .env.example .env +# docker compose up -d --build +# ./run-tests.sh # or curl the gateway yourself — see README.md +name: spinloop-gateway-example + +services: + node-a: + build: + context: ../.. + dockerfile: examples/gateway-docker/Dockerfile + image: spinloop-gateway-node:example + environment: + SPINLOOP_API_TOKEN: ${NODE_A_TOKEN:?set NODE_A_TOKEN (copy .env.example to .env)} + ports: + - "14242:4242" + # The engine, published for direct curls from the host. The gateways + # need no published port: inside the compose network they reach the + # engine where the node's Spinloop binds it, 8080. + - "18080:8080" + + node-b: + image: spinloop-gateway-node:example + depends_on: [node-a] + environment: + SPINLOOP_API_TOKEN: ${NODE_B_TOKEN:?set NODE_B_TOKEN (copy .env.example to .env)} + ports: + - "14243:4242" + - "18081:8080" + + gateway: + image: spinloop-gateway-node:example + depends_on: [node-a, node-b] + command: ["spinloop", "gateway", "--fleet", "/opt/gw/fleet.yaml", "--listen", "0.0.0.0:4000"] + environment: + # The gateway's own token, resolved the way a daemon's is — from the + # environment here. Callers present it; the node tokens and engine keys + # the fleet file names are resolved from the same environment, at + # startup and at every wake. + SPINLOOP_API_TOKEN: ${GATEWAY_TOKEN:?set GATEWAY_TOKEN (copy .env.example to .env)} + NODE_A_TOKEN: ${NODE_A_TOKEN:?} + NODE_B_TOKEN: ${NODE_B_TOKEN:?} + NODE_A_ENGINE_KEY: ${NODE_A_ENGINE_KEY:?} + NODE_B_ENGINE_KEY: ${NODE_B_ENGINE_KEY:?} + ports: + - "4000:4000" + + # The wake: off policy over the same two nodes, on its own port. + gateway-cold: + image: spinloop-gateway-node:example + depends_on: [node-a, node-b] + command: ["spinloop", "gateway", "--fleet", "/opt/gw/fleet-cold.yaml", "--listen", "0.0.0.0:4001"] + environment: + SPINLOOP_API_TOKEN: ${GATEWAY_TOKEN:?set GATEWAY_TOKEN (copy .env.example to .env)} + NODE_A_TOKEN: ${NODE_A_TOKEN:?} + NODE_B_TOKEN: ${NODE_B_TOKEN:?} + NODE_A_ENGINE_KEY: ${NODE_A_ENGINE_KEY:?} + NODE_B_ENGINE_KEY: ${NODE_B_ENGINE_KEY:?} + ports: + - "4001:4001" diff --git a/examples/gateway-docker/engine/engine-config.yaml b/examples/gateway-docker/engine/engine-config.yaml new file mode 100644 index 00000000..b20a5be6 --- /dev/null +++ b/examples/gateway-docker/engine/engine-config.yaml @@ -0,0 +1,79 @@ +# What the fake engine serves. +# +# The two ungated routes are for the daemon itself: /health is its readiness +# probe and /metrics the scrape its collector parses in llama.cpp's dialect. +# Both run on the node's own machine, so neither needs the credential that +# stands between the gateway and the engine. +# +# The three OpenAI routes are gated: the gateway presents the key its fleet +# entry names, and the shim hands it to this process as the ENGINE_API_KEY +# environment variable. gate.js applies the check and, for the completion +# routes, answers streaming requests in the server-sent-events shape. +plugin: rest +resources: + - path: /health + method: GET + response: + statusCode: 200 + headers: + Content-Type: application/json + content: '{"status":"ok"}' + + # The llamacpp: counters internal/metrics parses. Static values keep the + # assertions deterministic; what is being tested is the collection path, not + # the arithmetic of a real engine. + - path: /metrics + method: GET + response: + statusCode: 200 + headers: + Content-Type: text/plain + content: | + # HELP llamacpp:prompt_tokens_total Number of prompt tokens processed. + llamacpp:prompt_tokens_total 4096 + # HELP llamacpp:tokens_predicted_total Number of tokens predicted. + llamacpp:tokens_predicted_total 1024 + # HELP llamacpp:n_decode_total Number of decode runs. + llamacpp:n_decode_total 900 + # HELP llamacpp:requests_processing Number of processing requests. + llamacpp:requests_processing 2 + # HELP llamacpp:requests_deferred Number of deferred requests. + llamacpp:requests_deferred 1 + # HELP llamacpp:request_success_total Number of successful requests. + llamacpp:request_success_total 17 + + - path: /v1/models + method: GET + steps: + - type: script + lang: javascript + file: gate.js + response: + statusCode: 200 + headers: + Content-Type: application/json + content: '{"object":"list","data":[{"id":"fake-model","object":"model"}]}' + + - path: /v1/chat/completions + method: POST + steps: + - type: script + lang: javascript + file: gate.js + response: + statusCode: 200 + headers: + Content-Type: application/json + content: '{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"fake-model","choices":[{"index":0,"message":{"role":"assistant","content":"Hello from the fake engine"},"finish_reason":"stop"}],"usage":{"prompt_tokens":8,"completion_tokens":5,"total_tokens":13}}' + + - path: /v1/completions + method: POST + steps: + - type: script + lang: javascript + file: gate.js + response: + statusCode: 200 + headers: + Content-Type: application/json + content: '{"id":"cmpl-1","object":"text_completion","created":1,"model":"fake-model","choices":[{"index":0,"text":"Hello from the fake engine","finish_reason":"stop"}],"usage":{"prompt_tokens":8,"completion_tokens":5,"total_tokens":13}}' diff --git a/examples/gateway-docker/engine/gate.js b/examples/gateway-docker/engine/gate.js new file mode 100644 index 00000000..3fde923c --- /dev/null +++ b/examples/gateway-docker/engine/gate.js @@ -0,0 +1,53 @@ +// The fake engine's gate and stream branch, shared by every gated route. +// +// The key arrives as the ENGINE_API_KEY environment variable: the shim read it +// from the --api-key-file the daemon pointed the engine at, so the value never +// rides on an argument any local user could read. A request presenting a +// different bearer — or none — is refused. With no key supplied the engine is +// ungated, which is right for one reached only over loopback. +// +// A completion request that asks for a stream gets the server-sent-events +// shape; the reply is one canned body, because what is under test is the +// gateway passing a streamed reply through, not an engine tokenising. + +var key = env["ENGINE_API_KEY"] || ""; + +if (key !== "" && (context.request.headers["Authorization"] || "") !== "Bearer " + key) { + respond() + .withStatusCode(401) + .withHeader("Content-Type", "application/json") + .withContent('{"error":{"message":"incorrect API key provided","type":"invalid_request_error"}}') + .skipDefaultBehaviour(); +} else if (context.request.method === "POST" && + /"stream"\s*:\s*true/.test(context.request.body || "")) { + respond() + .withStatusCode(200) + .withHeader("Content-Type", "text/event-stream") + .withContent(streamReply(context.request.path)) + .skipDefaultBehaviour(); +} +// Otherwise the resource's own canned response applies. + +function streamReply(path) { + if (path === "/v1/completions") { + return "data: " + JSON.stringify({ + id: "cmpl-1", object: "text_completion", created: 1, model: "fake-model", + choices: [{ index: 0, text: "Hello from the fake engine", finish_reason: "stop" }] + }) + "\n\ndata: [DONE]\n\n"; + } + var first = { + id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "fake-model", + choices: [{ + index: 0, + delta: { role: "assistant", content: "Hello from the fake engine" }, + finish_reason: null + }] + }; + var last = { + id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "fake-model", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }] + }; + return "data: " + JSON.stringify(first) + + "\n\ndata: " + JSON.stringify(last) + + "\n\ndata: [DONE]\n\n"; +} diff --git a/examples/gateway-docker/fleet.yaml b/examples/gateway-docker/fleet.yaml new file mode 100644 index 00000000..76d5dd75 --- /dev/null +++ b/examples/gateway-docker/fleet.yaml @@ -0,0 +1,36 @@ +# The operator's view of this stack, from your machine: the two nodes over the +# ports compose publishes. +# +# The gateways are not nodes in it — they serve the fleet files baked into +# the image — and the client's Spinloop names the gateway's published address, +# not this file. This is how you watch and drive the fleet itself: status, +# start, stop, metrics, logs. +# +# The engine: blocks here are not optional: each engine binds 8080 inside its +# container and is published on 18080/18081 outside, which the daemons cannot +# know. (The gateways' own fleet files need no override — inside the compose +# network the engine is where it binds.) +# +# The nodes' Spinloop source is node/Spinloop, which is also why +# `spinloop fleet start` works from here: it resolves what to run on a node +# and pushes that. +prefer: idle + +nodes: + - name: node-a + host: 127.0.0.1 + port: 14242 + tokenEnv: NODE_A_TOKEN + engineTokenEnv: NODE_A_ENGINE_KEY + file: ./node/Spinloop + engine: + port: 18080 + + - name: node-b + host: 127.0.0.1 + port: 14243 + tokenEnv: NODE_B_TOKEN + engineTokenEnv: NODE_B_ENGINE_KEY + file: ./node/Spinloop + engine: + port: 18081 diff --git a/examples/gateway-docker/gateway/fleet-cold.yaml b/examples/gateway-docker/gateway/fleet-cold.yaml new file mode 100644 index 00000000..1245077c --- /dev/null +++ b/examples/gateway-docker/gateway/fleet-cold.yaml @@ -0,0 +1,22 @@ +# The same fleet as fleet.yaml, with waking refused: a request nothing is +# serving fails naming the node that would be started and the command that +# would start it, rather than starting it. It is the wake: off policy for the +# whole file — may work be started on these machines on demand, or only used +# where it is already running. +prefer: idle +wake: off + +nodes: + - name: node-a + host: node-a + port: 4242 + tokenEnv: NODE_A_TOKEN + engineTokenEnv: NODE_A_ENGINE_KEY + file: ./Spinloop + + - name: node-b + host: node-b + port: 4242 + tokenEnv: NODE_B_TOKEN + engineTokenEnv: NODE_B_ENGINE_KEY + file: ./Spinloop diff --git a/examples/gateway-docker/gateway/fleet.yaml b/examples/gateway-docker/gateway/fleet.yaml new file mode 100644 index 00000000..baf70f97 --- /dev/null +++ b/examples/gateway-docker/gateway/fleet.yaml @@ -0,0 +1,32 @@ +# What the gateway serves: this stack's two nodes, addressed the way the +# gateway's container reaches them — by compose service name, on the daemon's +# port. +# +# The gateway is the client that wakes a node, so it must resolve each node's +# Spinloop source from its own filesystem: the Dockerfile bakes node/Spinloop +# in beside this file as /opt/gw/Spinloop, which is what `file` names. +# +# No engine: blocks, for once: each engine binds 0.0.0.0:8080 — its Spinloop's +# BASEURL says so, the daemon reports it — and the gateway dials :8080 +# over the compose network. The client-side fleet.yaml needs the override, +# because the engine is published to the host on another port. +# +# Tokens are named by variable, as everywhere: the gateway container's +# environment supplies the values (compose.yaml), so nothing secret is in this +# file. +prefer: idle + +nodes: + - name: node-a + host: node-a + port: 4242 + tokenEnv: NODE_A_TOKEN + engineTokenEnv: NODE_A_ENGINE_KEY + file: ./Spinloop + + - name: node-b + host: node-b + port: 4242 + tokenEnv: NODE_B_TOKEN + engineTokenEnv: NODE_B_ENGINE_KEY + file: ./Spinloop diff --git a/examples/gateway-docker/node/Spinloop b/examples/gateway-docker/node/Spinloop new file mode 100644 index 00000000..ac359350 --- /dev/null +++ b/examples/gateway-docker/node/Spinloop @@ -0,0 +1,14 @@ +# What each node runs when a start request gives it work — the source both +# fleet files point at. The image bakes a copy beside the gateway's fleet +# files, so the gateway's own wakes resolve it from inside the container. +# +# The BASEURL is the point: it binds the engine to every interface. Without +# it llama-server binds 127.0.0.1, the daemon reports the engine loopback-only, +# and the gateway — a different container — is right to refuse routing to it. +# The client's Spinloop has no BASEURL at all: it names no engine, because it +# never talks to one. +PROVIDER llamacpp +MODEL org/fake-model +ALIAS fake-model +CONTEXT 4096 +BASEURL http://0.0.0.0:8080/v1 diff --git a/examples/gateway-docker/run-tests.sh b/examples/gateway-docker/run-tests.sh new file mode 100755 index 00000000..5e319970 --- /dev/null +++ b/examples/gateway-docker/run-tests.sh @@ -0,0 +1,616 @@ +#!/usr/bin/env bash +# +# Drives the dockerised gateway stack and asserts the behaviours the gateway +# promises: a model is listed once running, a cold request wakes a node and +# streams a reply, the engine key is injected and never reaches the caller, a +# wrong token is 401, and wake: off refuses. This is both the CI integration +# test and something a maintainer can run locally — there is no CI-only path +# that can drift from what you run by hand. +# +# Usage: ./run-tests.sh [--keep] +# --keep leave the stack running afterwards, to poke at it yourself + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly HERE +REPO_ROOT="$(cd "${HERE}/../.." && pwd)" +readonly REPO_ROOT +# Where the built binary lands; the stack is driven by the spinloop built from +# this working tree, so the test covers this commit. +readonly SPINLOOP_BIN="${HERE}/.spinloop-test-bin" +readonly READY_TIMEOUT_SECS=90 +readonly GATEWAY_URL=http://127.0.0.1:4000 +readonly GATEWAY_COLD_URL=http://127.0.0.1:4001 +readonly COMPLETION_BODY='{"model":"fake-model","messages":[{"role":"user","content":"hi"}]}' + +keep_stack=0 +failures=0 + +####################################### +# Report a passing assertion. +# Arguments: +# Description of what passed. +# Outputs: +# Writes the result to stdout. +####################################### +pass() { + echo " ok - $1" +} + +####################################### +# Report a failing assertion and record it, without aborting the run — one +# failure should not hide the rest. +# Globals: +# failures +# Arguments: +# Description, expected, actual. +# Outputs: +# Writes the failure to stderr. +####################################### +fail() { + echo " FAIL - $1" >&2 + echo " expected: $2" >&2 + echo " actual: $3" >&2 + failures=$((failures + 1)) +} + +####################################### +# Assert that a string contains a substring. +# Arguments: +# Description, haystack, needle. +####################################### +assert_contains() { + local description="$1" haystack="$2" needle="$3" + if [[ "${haystack}" == *"${needle}"* ]]; then + pass "${description}" + else + fail "${description}" "to contain '${needle}'" "${haystack}" + fi +} + +####################################### +# Assert that a string does not contain a substring. +# Arguments: +# Description, haystack, needle. +####################################### +assert_not_contains() { + local description="$1" haystack="$2" needle="$3" + if [[ "${haystack}" != *"${needle}"* ]]; then + pass "${description}" + else + fail "${description}" "not to contain '${needle}'" "${haystack}" + fi +} + +####################################### +# Assert an exact string equality. +# Arguments: +# Description, actual, expected. +####################################### +assert_equals() { + local description="$1" actual="$2" expected="$3" + if [[ "${actual}" == "${expected}" ]]; then + pass "${description}" + else + fail "${description}" "${expected}" "${actual}" + fi +} + +####################################### +# Run a docker compose command, showing its output only when it fails. These +# commands are noisy on success and the test's own output is the point, but a +# silent failure is worse than noise: a `compose up` that cannot pull leaves +# nothing behind but "Tearing down..." and an exit code. +# Globals: +# HERE +# Arguments: +# Arguments to pass to docker compose. +# Returns: +# The command's exit status. +####################################### +compose() { + local out rc=0 + out="$(docker compose -f "${HERE}/compose.yaml" "$@" 2>&1)" || rc=$? + if (( rc != 0 )); then + echo "Error: docker compose $* failed (exit ${rc}):" >&2 + echo "${out}" >&2 + fi + return "${rc}" +} + +####################################### +# The container state for one service, or "" when docker cannot say. +# Globals: +# HERE +# Arguments: +# Service name. +# Outputs: +# Writes the state to stdout. +####################################### +container_state() { + docker compose -f "${HERE}/compose.yaml" ps --format '{{.State}}' "$1" 2>/dev/null +} + +####################################### +# Dump what the containers are doing, for a wait that timed out. +# Globals: +# HERE +# Outputs: +# Writes container state and recent logs to stderr. +####################################### +diagnose_fleet() { + docker compose -f "${HERE}/compose.yaml" ps >&2 2>&1 || true + docker compose -f "${HERE}/compose.yaml" logs --tail 20 >&2 2>&1 || true +} + +####################################### +# Run `spinloop fleet` against the example's fleet.yaml. +# Globals: +# SPINLOOP_BIN, HERE +# Arguments: +# Arguments to pass to `spinloop fleet`. +# Outputs: +# The command's stdout; stderr is discarded so assertions read cleanly. +####################################### +fleet() { + "${SPINLOOP_BIN}" fleet "$@" --fleet "${HERE}/fleet.yaml" 2>/dev/null +} + +####################################### +# As fleet(), but merging stderr — for assertions about error messages. +# Globals: +# SPINLOOP_BIN, HERE +####################################### +fleet_with_stderr() { + "${SPINLOOP_BIN}" fleet "$@" --fleet "${HERE}/fleet.yaml" 2>&1 +} + +####################################### +# The state column for one node, or "" when the node is absent. +# Arguments: +# Node name. +####################################### +node_state() { + local name="$1" + fleet status | awk -v n="${name}" '$1 == n {print $2}' +} + +####################################### +# The gateway container's logs, for assertions about what it did. +# Globals: +# HERE +####################################### +gateway_logs() { + docker compose -f "${HERE}/compose.yaml" logs --no-color gateway 2>/dev/null || true +} + +####################################### +# The process list inside node-a. +# Globals: +# HERE +####################################### +node_a_processes() { + docker compose -f "${HERE}/compose.yaml" exec -T node-a ps ax 2>/dev/null || true +} + +####################################### +# GET a gateway path with the gateway's token. +# Globals: +# GATEWAY_URL, GATEWAY_TOKEN +# Arguments: +# Port (4000 or 4001), path. +####################################### +gateway_get() { + local port="$1" path="$2" + curl -fsS -H "Authorization: Bearer ${GATEWAY_TOKEN}" \ + "http://127.0.0.1:${port}${path}" 2>/dev/null || true +} + +####################################### +# POST a completion body at a gateway. Returns the body; a non-2xx makes the +# function fail, which is what the positive-path assertions want. +# Globals: +# GATEWAY_TOKEN +# Arguments: +# Port, path, body, extra curl arguments (e.g. a header dump file flag). +####################################### +gateway_post() { + local port="$1" path="$2" body="$3" + shift 3 + curl -fsS -X POST -H "Authorization: Bearer ${GATEWAY_TOKEN}" \ + -H "Content-Type: application/json" -d "${body}" \ + "http://127.0.0.1:${port}${path}" "$@" 2>/dev/null || true +} + +####################################### +# Wait until both node daemons answer, so assertions do not race the +# containers' startup. +# Globals: +# READY_TIMEOUT_SECS +# Returns: +# 0 once both nodes report a state, 1 on timeout. +####################################### +wait_for_fleet() { + local deadline=$((SECONDS + READY_TIMEOUT_SECS)) + while (( SECONDS < deadline )); do + # Read the table into a variable rather than piping it: under `pipefail` + # a `grep -q` that matches and exits first can leave the pipeline + # reporting the writer's SIGPIPE, which reads here as "nothing + # unreachable" — the opposite of what was found. + if [[ "$(fleet status)" != *unreachable* ]]; then + return 0 + fi + sleep 2 + done + echo "Error: the fleet did not become reachable in ${READY_TIMEOUT_SECS}s" >&2 + fleet status >&2 || true + diagnose_fleet + return 1 +} + +####################################### +# Wait until both gateways answer /health. +# Globals: +# READY_TIMEOUT_SECS, GATEWAY_TOKEN +####################################### +wait_for_gateways() { + local deadline=$((SECONDS + READY_TIMEOUT_SECS)) + while (( SECONDS < deadline )); do + if [[ -n "$(gateway_get 4000 /health)" && -n "$(gateway_get 4001 /health)" ]]; then + return 0 + fi + sleep 2 + done + echo "Error: the gateways did not come up in ${READY_TIMEOUT_SECS}s" >&2 + diagnose_fleet + return 1 +} + +####################################### +# Wait for one node to reach a state. +# Arguments: +# Node name, expected state, timeout in seconds. +# Returns: +# 0 when the state is reached, 1 on timeout. +####################################### +wait_for_state() { + local name="$1" want="$2" timeout="$3" + local deadline=$((SECONDS + timeout)) + while (( SECONDS < deadline )); do + if [[ "$(node_state "${name}")" == "${want}" ]]; then + return 0 + fi + sleep 1 + done + return 1 +} + +####################################### +# Tear the stack down unless --keep was given. Registered as an EXIT trap so a +# failure part-way through still cleans up. +# Globals: +# keep_stack, HERE +####################################### +cleanup() { + if (( keep_stack )); then + echo + echo "Stack left running (--keep). Try:" + echo " cd ${HERE} && set -a && . ./.env && set +a" + echo " curl -H 'Authorization: Bearer \$GATEWAY_TOKEN' http://127.0.0.1:4000/v1/models" + echo "Tear down with: docker compose -f ${HERE}/compose.yaml down -v" + return + fi + echo + echo "Tearing down..." + docker compose -f "${HERE}/compose.yaml" down -v >/dev/null 2>&1 || true + rm -f "${SPINLOOP_BIN}" +} + +####################################### +# Assert the gateway's own door: its token in, 401 out, and a 404 that names +# what it serves. +####################################### +test_gateway_auth() { + echo "The gateway's own door" + local ok + ok="$(gateway_get 4000 /health)" + assert_contains "health answers with the token" "${ok}" '"ok":true' + assert_equals "no token is 401" \ + "$(curl -s -o /dev/null -w '%{http_code}' "${GATEWAY_URL}/health")" "401" + assert_equals "a wrong token is 401" \ + "$(curl -s -o /dev/null -w '%{http_code}' -H 'Authorization: Bearer not-the-token' "${GATEWAY_URL}/health")" "401" + + local out + out="$(curl -s -w '\n%{http_code}' -H "Authorization: Bearer ${GATEWAY_TOKEN}" \ + "${GATEWAY_URL}/v1/nope" 2>/dev/null || true)" + assert_contains "an unknown path names the paths served" "${out}" "the gateway serves" + assert_contains "it names /v1/models" "${out}" "/v1/models" + assert_contains "and it is a 404" "${out}" "404" +} + +####################################### +# Assert nothing running is an empty list, not an error. +####################################### +test_cold_listing() { + echo "Nothing running is an empty list, not an error" + local models + models="$(gateway_get 4000 /v1/models)" + assert_contains "a cold fleet lists nothing" "${models}" '"data":[]' +} + +####################################### +# Assert the wake: off gateway refuses a cold request, names the node and the +# command that would start it, and starts nothing. +####################################### +test_wake_off_refuses_cold() { + echo "wake: off refuses a cold request" + local out + out="$(curl -s -w '\n%{http_code}' -X POST \ + -H "Authorization: Bearer ${GATEWAY_TOKEN}" \ + -H "Content-Type: application/json" -d "${COMPLETION_BODY}" \ + "${GATEWAY_COLD_URL}/v1/chat/completions" 2>/dev/null || true)" + assert_contains "nothing is serving, so the request fails" "${out}" "503" + assert_contains "the failure names the policy" "${out}" "wake is off" + assert_contains "it names the node and the start command" \ + "${out}" "spinloop fleet start node-a" + assert_equals "and node-a was not started" "$(node_state node-a)" "idle" + assert_equals "nor was node-b" "$(node_state node-b)" "idle" +} + +####################################### +# Assert the command the refusal names actually starts the node. +####################################### +test_suggested_start_works() { + echo "The command the refusal names starts the node" + fleet start node-a >/dev/null + if wait_for_state node-a running 30; then + pass "fleet start node-a brings it up" + else + fail "fleet start node-a brings it up" "running" "$(node_state node-a)" + fi + assert_contains "status shows what it serves" "$(fleet status)" "fake-model" +} + +####################################### +# Assert wake: off still routes to what is already running: it decides +# whether to start, not whether to answer. +####################################### +test_wake_off_routes_running() { + echo "wake: off still routes what is already running" + # The cold gateway's last reading is from before the start; let it go stale + # rather than race the two-second cache. + sleep 3 + local out + out="$(gateway_post 4001 /v1/chat/completions "${COMPLETION_BODY}")" + assert_contains "the running node answers through the cold gateway" \ + "${out}" "Hello from the fake engine" +} + +####################################### +# Assert the model list is what the fleet is running. +####################################### +test_models_listing() { + echo "The model list is what the fleet is running" + local models + models="$(gateway_get 4000 /v1/models)" + assert_contains "the gateway lists the served name" "${models}" '"id":"fake-model"' +} + +####################################### +# Assert a running node answers through the waking gateway, and the gateway +# logged the route with its node and its wake state. +####################################### +test_gateway_serves_running_node() { + echo "A running node answers through the gateway" + local out + out="$(gateway_post 4000 /v1/chat/completions "${COMPLETION_BODY}")" + assert_contains "the reply is the engine's" "${out}" "Hello from the fake engine" + + local logs + logs="$(gateway_logs)" + assert_contains "the gateway logged the route" "${logs}" "msg=routed" + assert_contains "it named the node" "${logs}" "node=node-a" + assert_contains "it knew the node was already running" "${logs}" "woken=false" +} + +####################################### +# Assert the engine key is injected and never reaches the caller: the engine +# is gated (a direct call without the key is refused, with it answered), the +# key arrived as a file path, and no reply, log or process list carries it. +####################################### +test_engine_key_gating() { + echo "The engine key is injected, and never reaches the caller" + local url="http://127.0.0.1:18080/v1/chat/completions" + assert_equals "a direct call with no key is refused" \ + "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + -d "${COMPLETION_BODY}" "${url}")" "401" + assert_equals "a direct call with a wrong key is refused" \ + "$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer not-the-key" -H 'Content-Type: application/json' \ + -d "${COMPLETION_BODY}" "${url}")" "401" + local reply + reply="$(curl -fsS -X POST -H "Authorization: Bearer ${NODE_A_ENGINE_KEY}" \ + -H 'Content-Type: application/json' -d "${COMPLETION_BODY}" "${url}" 2>/dev/null || true)" + assert_contains "the node's own key opens it" "${reply}" "Hello from the fake engine" + + local status + status="$(curl -fsS -H "Authorization: Bearer ${NODE_A_TOKEN}" \ + http://127.0.0.1:14242/v1/status 2>/dev/null || true)" + assert_contains "the node reports its engine needs a key" "${status}" '"requiresKey":true' + assert_not_contains "the node never discloses the key" "${status}" "${NODE_A_ENGINE_KEY}" + + local enginelog + enginelog="$(fleet logs node-a --limit 50 2>/dev/null || true)" + assert_contains "the engine was gated by file" "${enginelog}" "--api-key-file" + assert_not_contains "the key itself never reaches the command line" \ + "${enginelog}" "${NODE_A_ENGINE_KEY}" + assert_not_contains "the key is not in the node's process list" \ + "$(node_a_processes)" "${NODE_A_ENGINE_KEY}" + + local via_gateway + via_gateway="$(gateway_post 4000 /v1/chat/completions "${COMPLETION_BODY}")" + assert_not_contains "the key never appears in a reply through the gateway" \ + "${via_gateway}" "${NODE_A_ENGINE_KEY}" +} + +####################################### +# Assert a cold request to the waking gateway starts a node and the streamed +# reply passes through in the server-sent-events shape. +####################################### +test_cold_request_wakes_and_streams() { + echo "A cold request wakes a node and streams a reply" + fleet stop node-a >/dev/null + wait_for_state node-a stopped 30 || true + # Let the gateway's reading of the fleet go stale, so the request below + # sees the stop rather than a two-second-old reading of a running node. + sleep 3 + + local headers="${HERE}/.stream-headers" + local out + out="$(gateway_post 4000 /v1/chat/completions \ + '{"model":"fake-model","stream":true,"messages":[{"role":"user","content":"hi"}]}' \ + -D "${headers}" || true)" + assert_contains "a streamed reply is server-sent-events" \ + "$(cat "${headers}" 2>/dev/null)" "text/event-stream" + assert_contains "the chunks pass through" "${out}" "Hello from the fake engine" + assert_contains "the stream ends" "${out}" "data: [DONE]" + rm -f "${headers}" + + if wait_for_state node-a running 60; then + pass "the request left the node it woke running" + else + fail "the request left the node it woke running" "running" "$(node_state node-a)" + fi + assert_contains "the gateway logged the wake" \ + "$(gateway_logs)" "Waking node-a to serve fake-model" +} + +####################################### +# Assert a harness launch against the client's Spinloop points the agent at +# the gateway's address with the gateway's token as its key. +# Globals: +# HERE, SPINLOOP_BIN +####################################### +test_launch_points_agent_at_gateway() { + echo "A launch points the agent at the gateway" + local sandbox="${HERE}/.launch-sandbox" + rm -rf "${sandbox}" + mkdir -p "${sandbox}/bin" "${sandbox}/home" + cat > "${sandbox}/bin/opencode" <<'STUB' +#!/usr/bin/env bash +echo "HARNESS base_url=${OPENAI_BASE_URL:-} key=${OPENAI_API_KEY:-}" +STUB + chmod +x "${sandbox}/bin/opencode" + + local launch + launch="$(PATH="${sandbox}/bin:${PATH}" HOME="${sandbox}/home" \ + XDG_CONFIG_HOME="${sandbox}/home/.config" \ + OPENAI_BASE_URL="" \ + OPENAI_API_KEY="${GATEWAY_TOKEN}" \ + "${SPINLOOP_BIN}" harness -O="${HERE}/client/Spinloop" -H opencode 2>&1 || true)" + assert_contains "the agent is pointed at the gateway with its prefix" \ + "${launch}" "base_url=http://127.0.0.1:4000/v1" + assert_contains "the agent is given the gateway's token as its key" \ + "${launch}" "key=${GATEWAY_TOKEN}" + local config="${sandbox}/home/.config/opencode/opencode.json" + if [[ -f "${config}" ]]; then + pass "the harness config was written" + assert_contains "and it carries the gateway's address" \ + "$(cat "${config}")" "127.0.0.1:4000" + else + fail "the harness config was written" "${config}" "missing" + fi + rm -rf "${sandbox}" +} + +####################################### +# Assert a launch that cannot authenticate the gateway fails before the agent +# is started and before anything is written, naming the variable. +# Globals: +# HERE, SPINLOOP_BIN +####################################### +test_launch_fails_without_token() { + echo "A launch without the gateway's token fails, naming the variable" + local sandbox="${HERE}/.launch-sandbox" + rm -rf "${sandbox}" + mkdir -p "${sandbox}/bin" "${sandbox}/home" + cat > "${sandbox}/bin/opencode" <<'STUB' +#!/usr/bin/env bash +echo "HARNESS base_url=${OPENAI_BASE_URL:-} key=${OPENAI_API_KEY:-}" +STUB + chmod +x "${sandbox}/bin/opencode" + + local launch + launch="$(PATH="${sandbox}/bin:${PATH}" HOME="${sandbox}/home" \ + XDG_CONFIG_HOME="${sandbox}/home/.config" \ + OPENAI_BASE_URL="" \ + OPENAI_API_KEY="" \ + "${SPINLOOP_BIN}" harness -O="${HERE}/client/Spinloop" -H opencode 2>&1 || true)" + assert_contains "the failure names the variable to set" "${launch}" "OPENAI_API_KEY" + assert_not_contains "the agent was not started" "${launch}" "HARNESS" + if [[ ! -f "${sandbox}/home/.config/opencode/opencode.json" ]]; then + pass "no harness config was written" + else + fail "no harness config was written" "no opencode.json" "one was written" + fi + rm -rf "${sandbox}" +} + +main() { + if [[ "${1:-}" == "--keep" ]]; then + keep_stack=1 + fi + + cd "${HERE}" + if [[ ! -f .env ]]; then + echo "Using .env.example for tokens (no .env present)" + cp .env.example .env + fi + set -a + # shellcheck source=/dev/null + . ./.env + set +a + + trap cleanup EXIT + + echo "Building spinloop from the working tree..." + (cd "${REPO_ROOT}" && go build -o "${SPINLOOP_BIN}" ./cmd/spinloop) + + echo "Bringing the stack up..." + compose up -d --build + wait_for_fleet + wait_for_gateways + + echo + test_gateway_auth + echo + test_cold_listing + echo + test_wake_off_refuses_cold + echo + test_suggested_start_works + echo + test_wake_off_routes_running + echo + test_models_listing + echo + test_gateway_serves_running_node + echo + test_engine_key_gating + echo + test_cold_request_wakes_and_streams + echo + test_launch_points_agent_at_gateway + echo + test_launch_fails_without_token + + echo + if (( failures > 0 )); then + echo "${failures} assertion(s) failed" >&2 + return 1 + fi + echo "All assertions passed" +} + +main "$@" diff --git a/examples/gateway-docker/shim/llama-server b/examples/gateway-docker/shim/llama-server new file mode 100755 index 00000000..4c627d43 --- /dev/null +++ b/examples/gateway-docker/shim/llama-server @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Stands in for llama-server so `spinloop serve`/`spinloop daemon` can start an +# "engine" with no GPU and no model. The daemon execs this exactly as it would +# the real binary, so process supervision, log capture and crash detection are +# genuinely exercised. +# +# It execs the Imposter ENGINE BINARY directly, never `imposter up`. The CLI +# wrapper exits 0 when its child dies, which the daemon would correctly record +# as a clean stop — a crash assertion would then pass while testing nothing. +# Exec'ing the engine makes it the daemon's direct child, so an abnormal death +# is a real non-zero exit and reads as `crashed`. +set -euo pipefail + +# Echo the command line into the daemon's engine log before consuming it — the +# loop below shifts every argument away. The key arrives as +# --api-key-file and stays that way: the tests check this log for the +# flag and for the absence of the value, and the process list for neither. +echo "llama-server shim: argv: $*" + +port=8080 +keyfile="" +# llama-server takes many flags; only the port and the key file matter here. +# Everything else is ignored on purpose, so new engine flags can never break +# the shim. +while [ $# -gt 0 ]; do + case "$1" in + --port) port="${2:-8080}"; shift 2 ;; + --api-key-file) keyfile="${2:-}"; shift 2 ;; + *) shift ;; + esac +done + +# The mock reads its gate from the environment, so the key never appears on a +# command line inside the container either. An engine started with no key +# file is ungated, which is right for one reached only over loopback. +if [ -n "${keyfile}" ]; then + ENGINE_API_KEY="$(cat "${keyfile}")" + export ENGINE_API_KEY + echo "llama-server shim: engine key supplied by file (value not echoed)" +else + echo "llama-server shim: engine ungated (no key file supplied)" +fi + +echo "llama-server shim: starting the Imposter engine on port ${port}" +export IMPOSTER_PORT="${port}" +exec imposter-go /opt/engine diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 544ddaa3..613cd78b 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -67,11 +67,12 @@ type Daemon struct { ready readiness hist systemHistory - mu sync.Mutex - runner string - model string - scrape metrics.ScrapeTarget - endpoint *EngineEndpoint + mu sync.Mutex + runner string + model string + servedName string + scrape metrics.ScrapeTarget + endpoint *EngineEndpoint } // log reads the daemon's logger, defaulting to discarding. @@ -114,17 +115,19 @@ func (d *Daemon) engineEndpoint() *EngineEndpoint { return d.endpoint } -// SetServed records what the daemon is serving, for status and metrics. -func (d *Daemon) SetServed(runner, model string) { +// SetServed records what the daemon is serving, for status and metrics. The +// served name is the name the engine answers to — an alias when one is set — +// and is empty when the engine answers only to the model. +func (d *Daemon) SetServed(runner, model, servedName string) { d.mu.Lock() - d.runner, d.model = runner, model + d.runner, d.model, d.servedName = runner, model, servedName d.mu.Unlock() } -func (d *Daemon) served() (string, string) { +func (d *Daemon) served() (string, string, string) { d.mu.Lock() defer d.mu.Unlock() - return d.runner, d.model + return d.runner, d.model, d.servedName } // configPath is the stored deploy config's location in the state directory. @@ -168,7 +171,7 @@ func (d *Daemon) Push(dc remote.DeployConfig) error { if err := os.WriteFile(d.configPath(), append(data, '\n'), 0o600); err != nil { return err } - d.SetServed(dc.Runner, dc.ModelID) + d.SetServed(dc.Runner, dc.ModelID, dc.ServedModelName) return nil } @@ -248,9 +251,9 @@ func (d *Daemon) StartEngine() error { argv = append(argv, keyArgs...) } if dc != nil { - d.SetServed(dc.Runner, dc.ModelID) + d.SetServed(dc.Runner, dc.ModelID, dc.ServedModelName) } - runner, model := d.served() + runner, model, _ := d.served() d.log().Info("starting engine", slog.String("source", source), slog.String("runner", runner), @@ -275,9 +278,13 @@ func (d *Daemon) StartEngine() error { // StatusResponse is the control API's status reply. type StatusResponse struct { - State string `json:"state"` - Runner string `json:"runner,omitempty"` - Model string `json:"model,omitempty"` + State string `json:"state"` + Runner string `json:"runner,omitempty"` + Model string `json:"model,omitempty"` + // ServedName is the name the running engine answers to — the served name + // its deploy config or Spinloop set — reported beside the model id when + // set: an aliased engine answers to both, and a caller may know either. + ServedName string `json:"servedName,omitempty"` UptimeSeconds int `json:"uptimeSeconds,omitempty"` LogPath string `json:"logPath,omitempty"` // LastActiveAt is when the engine last did any work, RFC 3339. Empty @@ -310,8 +317,15 @@ type StatusResponse struct { // binds 127.0.0.1:8080, which is useless to anyone else, and it cannot know // the name a client reaches this host by — a LAN name, a tailscale name, a // published container port. The caller composes these against the host it -// already has. +// already has. A node that does know that name — a remote environment, whose +// control plane publishes the instance's address — reports it in Host, and the +// caller uses it in place of the host it would otherwise supply. type EngineEndpoint struct { + // Host is the name or address a client reaches the engine by, when the + // node knows it. A daemon leaves it empty — it cannot know a client-facing + // name — but a remote environment's status fills it with the instance's + // published address, which is all a caller needs. + Host string `json:"host,omitempty"` // Port is the port the engine listens on — the engine's, never the // control API's. Port int `json:"port"` @@ -333,11 +347,12 @@ type EngineEndpoint struct { // engine's log lives, and how long the engine has been idle. func (d *Daemon) Status() StatusResponse { state, _, uptime := d.Sup.Status() - runner, model := d.served() + runner, model, servedName := d.served() resp := StatusResponse{ State: string(state), Runner: runner, Model: model, + ServedName: servedName, UptimeSeconds: uptime, LogPath: d.Sup.LogPath, Version: d.Version, @@ -400,7 +415,7 @@ func (d *Daemon) activity() (lastActiveAt string, idleSeconds int) { // Errors; an absent source is simply omitted, per the engine-metrics spec. func (d *Daemon) Metrics(ctx context.Context) metrics.Stats { state, _, uptime := d.Sup.Status() - runner, model := d.served() + runner, model, _ := d.served() stats := metrics.Stats{ State: string(state), Runner: runner, diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index a5105351..826c6183 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -339,13 +339,19 @@ while true; do sleep 0.05; done`) } // A start carrying its config pushes and starts in one call. - dc := `{"runner":"llamacpp","modelId":"org/model","serveArgs":[]}` + dc := `{"runner":"llamacpp","modelId":"org/model","serveArgs":[],"servedModelName":"org/model-alias"}` if resp, body := do("POST", "/v1/start", "sekrit", dc); resp.StatusCode != 200 || body["state"] != "running" { t.Fatalf("start with body = %d %v", resp.StatusCode, body) } if stored, _ := d.StoredConfig(); stored == nil || stored.ModelID != "org/model" { t.Fatalf("start body not persisted: %+v", stored) } + // The served name crosses the wire beside the model id: an aliased engine + // is addressed by either, and a caller may know only one of them. + if resp, body := do("GET", "/v1/status", "sekrit", ""); resp.StatusCode != 200 || + body["model"] != "org/model" || body["servedName"] != "org/model-alias" { + t.Fatalf("status after aliased start = %d %v", resp.StatusCode, body) + } // A start body while running is a 409 that stores nothing. if resp, body := do("POST", "/v1/start", "sekrit", @@ -407,8 +413,8 @@ while true; do sleep 0.05; done`) t.Fatal(err) } waitForState(t, crash.Sup, StateCrashed) - if got := crash.Status(); got.State != "crashed" { - t.Fatalf("status after crash = %+v", got) + if got := crash.Status(); got.State != "crashed" || got.ServedName != "" { + t.Fatalf("status after crash = %+v, want no served name without an alias", got) } } diff --git a/internal/daemon/readiness_test.go b/internal/daemon/readiness_test.go index 50cd45a1..7d1e7c0f 100644 --- a/internal/daemon/readiness_test.go +++ b/internal/daemon/readiness_test.go @@ -46,7 +46,7 @@ while true; do sleep 0.05; done`) t.Fatal(err) } waitForState(t, d.Sup, StateRunning) - d.SetServed(runner, "model") + d.SetServed(runner, "model", "") return d } diff --git a/internal/fleet/config.go b/internal/fleet/config.go index 1b702ea8..28cf480b 100644 --- a/internal/fleet/config.go +++ b/internal/fleet/config.go @@ -62,6 +62,36 @@ func ParsePrefer(s string) (Prefer, error) { return "", fmt.Errorf("unknown preference %q: use %q or %q", s, PreferIdle, PreferActive) } +// WakePolicy is whether routing may start an engine on a node that is not +// running one when no running node serves what is wanted. It sits in the fleet +// file beside prefer for the reason prefer does: it describes how this +// cluster is to be used — may work be started on its machines on demand, or +// only used where it is already running. +type WakePolicy string + +const ( + // WakeOn starts an engine on an idle node when nothing is serving. + WakeOn WakePolicy = "on" + // WakeOff never starts one: a request nothing is serving fails, naming + // the node that would have been woken and the command that would start it. + WakeOff WakePolicy = "off" +) + +// ParseWakePolicy validates a wake policy from a file. +func ParseWakePolicy(s string) (WakePolicy, error) { + switch WakePolicy(s) { + case WakeOn, WakeOff: + return WakePolicy(s), nil + } + return "", fmt.Errorf("unknown wake policy %q: use %q or %q", s, WakeOn, WakeOff) +} + +// Wakes reports whether routing may start an engine on a node that is not +// running one. A file that declares nothing wakes, as routing has always done. +func (c *Config) Wakes() bool { + return c.WakePolicy != WakeOff +} + // Config is a parsed fleet.yaml: the nodes, plus where the file was read from // (the directory whose .env supplies token values). type Config struct { @@ -71,6 +101,10 @@ type Config struct { // should be used — spread the work, or consolidate it. Empty means // PreferIdle. Prefer Prefer `yaml:"prefer"` + // WakePolicy is the fleet-wide wake policy: whether routing may start an + // engine on a node that is not running one. Empty means WakeOn, as + // routing has always done when the setting is absent. + WakePolicy WakePolicy `yaml:"wake"` // APIKeyEnv names the environment variable holding the key this fleet's // remote nodes require, shared by every one of them: a remote's engine is // always gated by its key, so a fleet of remotes can name the variable @@ -195,6 +229,11 @@ func (c *Config) validate() error { return err } } + if c.WakePolicy != "" { + if _, err := ParseWakePolicy(string(c.WakePolicy)); err != nil { + return err + } + } seen := map[string]bool{} for i := range c.Nodes { n := &c.Nodes[i] diff --git a/internal/fleet/config_test.go b/internal/fleet/config_test.go index d9f220fc..824ca5fa 100644 --- a/internal/fleet/config_test.go +++ b/internal/fleet/config_test.go @@ -570,3 +570,41 @@ func TestPreferRejectsUnknownValue(t *testing.T) { } } } + +func TestWakeSetting(t *testing.T) { + cases := []struct { + decl string + wake bool + }{ + {"wake: on\n", true}, + {"wake: off\n", false}, + {"", true}, // absent: routing wakes, as it always has + } + for _, c := range cases { + name := strings.TrimSpace(c.decl) + if name == "" { + name = "absent" + } + t.Run(name, func(t *testing.T) { + cfg, err := Load(writeFleet(t, c.decl+"nodes:\n - name: a\n host: a.local\n", "")) + if err != nil { + t.Fatal(err) + } + if got := cfg.Wakes(); got != c.wake { + t.Errorf("Wakes() = %v, want %v", got, c.wake) + } + }) + } +} + +func TestWakeRejectsUnknownValue(t *testing.T) { + _, err := Load(writeFleet(t, "wake: sometimes\nnodes:\n - name: a\n host: a.local\n", "")) + if err == nil { + t.Fatal("an unknown wake value should fail to parse") + } + for _, want := range []string{"on", "off"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should name %q, got %q", want, err) + } + } +} diff --git a/internal/fleet/remote_node.go b/internal/fleet/remote_node.go index 5425fbc2..c698388b 100644 --- a/internal/fleet/remote_node.go +++ b/internal/fleet/remote_node.go @@ -3,6 +3,8 @@ package fleet import ( "context" "fmt" + "net/url" + "strconv" "strings" "time" @@ -157,15 +159,36 @@ func (n *remoteNode) Logs(ctx context.Context, offset int64, limit int) (daemon. } // statusFromRemote maps the control plane's status reply onto the node's status. -// It carries only what a control-plane reply can honestly be mapped across: the -// state and its last-active record. The version is not in this reply — the stats -// reply carries it — so it is empty here, and runner/model are likewise absent. +// It carries what a status reply can honestly be mapped across: the state, what +// the engine is serving (runner, model, served name) and its last-active record. +// The version is not in this reply — the stats reply carries it — so it is empty +// here. The serving fields are empty when the daemon reports none: an engine that +// is not running one, or a daemon the control plane could not reach. +// +// A running environment's reply also names where its engine answers — the +// instance's published address, which a daemon on the instance cannot know for +// itself but the control plane can. It is carried as the engine's host, so +// routing resolves a remote node's address the way it resolves any node's. +// Absent (a stopped or undeployed environment reports none) means no engine +// address, exactly as the parts would be. func statusFromRemote(resp remote.Response) daemon.StatusResponse { - return daemon.StatusResponse{ + s := daemon.StatusResponse{ State: resp.State, + Runner: resp.Runner, + Model: resp.ModelID, + ServedName: resp.ServedName, LastActiveAt: resp.LastActiveAt, IdleSeconds: resp.IdleSeconds, } + if u, err := url.Parse(resp.BaseURL); resp.BaseURL != "" && err == nil && u.Host != "" { + port, _ := strconv.Atoi(u.Port()) + s.Engine = &daemon.EngineEndpoint{ + Host: u.Hostname(), + Port: port, + Path: u.Path, + } + } + return s } // statsFromRemote maps the stats Lambda's reply onto the shared stats shape. The diff --git a/internal/fleet/remote_node_test.go b/internal/fleet/remote_node_test.go index a617891f..5ded4f16 100644 --- a/internal/fleet/remote_node_test.go +++ b/internal/fleet/remote_node_test.go @@ -229,7 +229,8 @@ func TestRemoteNodeStartWithIsRefused(t *testing.T) { func TestRemoteNodeStatusOverTheControlPlane(t *testing.T) { stubAWSCreds(t) srv := remoteControlServer(t, - `{"state":"running","healthy":true,"lastActiveAt":"2026-01-02T00:00:00Z","idleSeconds":30}`, http.StatusOK) + `{"state":"running","healthy":true,"runner":"llamacpp","modelId":"org/m","servedName":"m",`+ + `"base_url":"http://1.2.3.4:8000/v1","lastActiveAt":"2026-01-02T00:00:00Z","idleSeconds":30}`, http.StatusOK) node, err := NewRemoteNode("env", remote.Config{StartURL: srv.URL, StopURL: srv.URL, Region: "us-east-1"}) if err != nil { t.Fatal(err) @@ -238,6 +239,16 @@ func TestRemoteNodeStatusOverTheControlPlane(t *testing.T) { if !r.OK() || r.Status.State != "running" || r.Status.IdleSeconds != 30 { t.Errorf("status result = %+v", r) } + // What the engine is serving rides the status, so a router can match a + // request to this node: the model id and the served name beside it. + if r.Status.Runner != "llamacpp" || r.Status.Model != "org/m" || r.Status.ServedName != "m" { + t.Errorf("serving facts not mapped: %+v", r.Status) + } + // And where the engine answers: the control plane's published address, so + // the router can reach the node rather than only name it. + if r.Status.Engine == nil || r.Status.Engine.Host != "1.2.3.4" || r.Status.Engine.Port != 8000 { + t.Errorf("engine address not mapped: %+v", r.Status.Engine) + } if r.Name != "env" { t.Errorf("name = %q", r.Name) } @@ -314,6 +325,46 @@ func TestKeeperIsRemoteOnly(t *testing.T) { } } +// statusFromRemote carries the serving facts when the daemon reports them, and +// leaves them empty when it does not — a running-but-unreachable daemon, or an +// engine that is not running a model, must not be invented into serving one. +func TestStatusFromRemoteServingFacts(t *testing.T) { + with := statusFromRemote(remote.Response{ + State: "running", + Runner: "llamacpp", + ModelID: "org/m", + ServedName: "m", + }) + if with.Runner != "llamacpp" || with.Model != "org/m" || with.ServedName != "m" { + t.Errorf("serving facts should map across, got %+v", with) + } + without := statusFromRemote(remote.Response{State: "running"}) + if without.Runner != "" || without.Model != "" || without.ServedName != "" { + t.Errorf("an absent serving fact must stay empty, got %+v", without) + } +} + +// statusFromRemote carries a running environment's engine address — the +// control plane's published base url — as the engine's host, so routing can +// reach it the way it reaches any node. A stopped or undeployed environment +// reports none, so its status carries no engine address. +func TestStatusFromRemoteCarriesTheEngineAddress(t *testing.T) { + got := statusFromRemote(remote.Response{ + State: "running", + BaseURL: "http://1.2.3.4:8000/v1", + }) + if got.Engine == nil { + t.Fatal("a running environment's engine address was not carried") + } + if got.Engine.Host != "1.2.3.4" || got.Engine.Port != 8000 || got.Engine.Path != "/v1" { + t.Errorf("engine endpoint = %+v, want host 1.2.3.4 port 8000 path /v1", got.Engine) + } + // No base url — a stopped or undeployed environment — means no address. + if got := statusFromRemote(remote.Response{State: "stopped"}); got.Engine != nil { + t.Errorf("a stopped environment should carry no engine address: %+v", got.Engine) + } +} + // A remote node drives start, stop and metrics over its control plane exactly // like a node would, mapping each reply onto the node's types. func TestRemoteNodeStartStopMetricsOverTheControlPlane(t *testing.T) { diff --git a/internal/fleet/select.go b/internal/fleet/select.go index bf0c5f2f..730ebf42 100644 --- a/internal/fleet/select.go +++ b/internal/fleet/select.go @@ -48,16 +48,33 @@ func (w Want) prefer() Prefer { return w.Prefer } -// matches reports whether a node serving `serving` is serving what is wanted. -// A launch that names no model wants any running engine. -func (w Want) matches(serving string) bool { +// matches reports whether a node reporting any of the given served names is +// serving what is wanted. A launch that names no model wants any running +// engine. +func (w Want) matches(serving ...string) bool { if w.Model == "" && w.Alias == "" && w.ModelID == "" { return true } - if serving == "" { - return false + for _, s := range serving { + if s != "" && (s == w.Model || s == w.Alias || s == w.ModelID) { + return true + } } - return serving == w.Model || serving == w.Alias || serving == w.ModelID + return false +} + +// servingNames is every name a node reports itself serving: the model id, and +// the served name it was started under when it reports one. An aliased engine +// answers to both, so either matching is a match. +func servingNames(s daemon.StatusResponse) []string { + var names []string + if s.Model != "" { + names = append(names, s.Model) + } + if s.ServedName != "" && s.ServedName != s.Model { + names = append(names, s.ServedName) + } + return names } // wanted names the model for a message, preferring the Spinloop's own MODEL. @@ -101,6 +118,10 @@ type Choice struct { Reason string // Woken records that this node was started to satisfy the launch. Woken bool + // Gateway records that FLEET named an endpoint rather than a fleet file: + // the endpoint has already done the choosing, Node is empty, and BaseURL + // is the endpoint's address rather than a node's engine. + Gateway bool } // candidate pairs a node's file entry with what it answered, keeping the @@ -133,7 +154,7 @@ func running(cands []candidate, w Want) []candidate { if !c.result.OK() || c.result.Status.State != string(daemon.StateRunning) { continue } - if w.matches(c.result.Status.Model) { + if w.matches(servingNames(c.result.Status)...) { out = append(out, c) } } @@ -184,7 +205,15 @@ func (c *Config) Select(ctx context.Context, w Want) (*Choice, error) { } } results := scope.FanOut(ctx, StatusCall) - return scope.choose(results, w) + return scope.Choose(results, w) +} + +// Choose applies the ranking to a fan-out's results the caller already holds, +// and resolves the winner's endpoint. It is Select without the query, for a +// caller that keeps its own reading of the fleet and reuses it across calls — +// the gateway does, because a burst of requests must not pay a fan-out each. +func (c *Config) Choose(results []NodeResult, w Want) (*Choice, error) { + return c.choose(results, w) } // ErrNoneServing reports that no node is serving what was wanted. It carries @@ -233,7 +262,7 @@ func (c *Config) choose(results []NodeResult, w Want) (*Choice, error) { if !only.result.OK() { return nil, fmt.Errorf("node %q: %s", only.entry.Name, describe(only.result)) } - if only.result.Status.State == string(daemon.StateRunning) && !w.matches(only.result.Status.Model) { + if only.result.Status.State == string(daemon.StateRunning) && !w.matches(servingNames(only.result.Status)...) { return nil, fmt.Errorf( "node %q is serving %s, not %s: it will not be restarted — pick another node, or stop it yourself", only.entry.Name, only.result.Status.Model, w.wanted()) @@ -302,6 +331,12 @@ func (c *Config) EngineBaseURL(n NodeConfig, status daemon.StatusResponse) (stri host, port, path := n.Host, 0, "" if ep := status.Engine; ep != nil { port, path = ep.Port, ep.Path + // A node that reports its engine's host — a remote environment, whose + // control plane knows the instance's published address — is reached + // there, in place of the host the fleet file supplies. + if ep.Host != "" { + host = ep.Host + } } if o := n.Engine; o != nil { if o.Host != "" { diff --git a/internal/fleet/select_test.go b/internal/fleet/select_test.go index 3190a733..2d01efad 100644 --- a/internal/fleet/select_test.go +++ b/internal/fleet/select_test.go @@ -299,6 +299,22 @@ func TestEngineBaseURL(t *testing.T) { status: reported, want: "https://engine.example:8080/v1", }, + { + name: "a reported engine host is used in place of the fleet file's", + node: NodeConfig{Name: "env", Kind: "remote"}, + status: daemon.StatusResponse{ + Engine: &daemon.EngineEndpoint{Host: "1.2.3.4", Port: 8000, Path: "/v1"}, + }, + want: "http://1.2.3.4:8000/v1", + }, + { + name: "an override still beats a reported engine host", + node: NodeConfig{Name: "env", Kind: "remote", Engine: &EngineOverride{Host: "proxy"}}, + status: daemon.StatusResponse{ + Engine: &daemon.EngineEndpoint{Host: "1.2.3.4", Port: 8000, Path: "/v1"}, + }, + want: "http://proxy:8000/v1", + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/fleet/wake.go b/internal/fleet/wake.go index c3c3cee3..2606f6ef 100644 --- a/internal/fleet/wake.go +++ b/internal/fleet/wake.go @@ -31,6 +31,44 @@ var wakePoll = 2 * time.Second // reads as a hang, so the caller is given something to print. type Waker func(format string, args ...any) +// ConfigFor resolves the deploy config a candidate node would be started +// with. The launch path resolves the same config for every candidate — one +// Spinloop describes what any node should start — while the gateway resolves +// each node's own Spinloop source, so different nodes may describe different +// engines. +type ConfigFor func(entry NodeConfig) (remote.DeployConfig, error) + +// ConstantConfig adapts one deploy config, already resolved, to a per-candidate +// resolver. The launch path uses it: its Spinloop describes what any node would +// start. +func ConstantConfig(dc remote.DeployConfig, err error) ConfigFor { + return func(NodeConfig) (remote.DeployConfig, error) { return dc, err } +} + +// configResolver resolves each candidate's deploy config at most once per wake, +// by node name. Resolving can cost more than a status read — the gateway +// parses each node's Spinloop source — and a wake asks for the same config +// twice per candidate: once to order them, once to start them. +type configResolver struct { + fn ConfigFor + dcs map[string]remote.DeployConfig + errs map[string]error +} + +func newConfigResolver(fn ConfigFor) *configResolver { + return &configResolver{fn: fn, dcs: map[string]remote.DeployConfig{}, errs: map[string]error{}} +} + +func (r *configResolver) config(entry NodeConfig) (remote.DeployConfig, error) { + if err, ok := r.errs[entry.Name]; ok { + return r.dcs[entry.Name], err + } + dc, err := r.fn(entry) + r.dcs[entry.Name] = dc + r.errs[entry.Name] = err + return dc, err +} + // Wake starts an engine for want on a node that is not running one, and waits // until its engine answers. Candidates are tried in fleet-file order, and a // node whose stored config already matches is preferred: it has the weights. @@ -38,17 +76,23 @@ type Waker func(format string, args ...any) // A node that refuses the config — a runner or model it cannot serve — is not // fatal while other candidates remain. When none succeeds, every refusal is // reported together. -func (c *Config) Wake(ctx context.Context, w Want, dc remote.DeployConfig, results []NodeResult, log Waker) (*Choice, error) { +func (c *Config) Wake(ctx context.Context, w Want, cfgFor ConfigFor, results []NodeResult, log Waker) (*Choice, error) { if log == nil { log = func(string, ...any) {} } - cands := wakeable(c.candidates(results), dc) + resolver := newConfigResolver(cfgFor) + cands := wakeable(c.candidates(results), resolver) if len(cands) == 0 { return nil, &ErrNoneServing{Results: results, Want: w, Path: c.Path} } var refused []string for _, cand := range cands { + dc, err := resolver.config(cand.entry) + if err != nil { + refused = append(refused, fmt.Sprintf("%s: %v", cand.entry.Name, err)) + continue + } node, err := c.NewNode(cand.entry) if err != nil { refused = append(refused, fmt.Sprintf("%s: %v", cand.entry.Name, err)) @@ -63,15 +107,22 @@ func (c *Config) Wake(ctx context.Context, w Want, dc remote.DeployConfig, resul return nil, err } log("Waking %s to serve %s...\n", cand.entry.Name, w.wanted()) - status, err := node.StartWith(ctx, &dc, engineKey) + _, err = node.StartWith(ctx, &dc, engineKey) if err != nil { // Another client may have woken this node first. That is // another route to the same place, not a failure — re-read // its state and take it if it is now serving what we want. + // The state alone is not the answer, though: the other start may + // still be loading, so the same readiness wait applies to a node + // we did not start ourselves. if isAlreadyRunning(err) { - if status, err = node.Status(ctx); err == nil && w.matches(status.Model) { - log("%s was already started by someone else; using it.\n", cand.entry.Name) - cand.result = NodeResult{Name: cand.entry.Name, Outcome: OutcomeOK, Status: status} + if status, err := node.Status(ctx); err == nil && w.matches(servingNames(status)...) { + log("%s was already started by someone else; waiting for its engine to answer...\n", cand.entry.Name) + ready, err := c.waitReady(ctx, node, cand.entry, w, log) + if err != nil { + return nil, err + } + cand.result = NodeResult{Name: cand.entry.Name, Outcome: OutcomeOK, Status: ready} return c.choiceFor(cand, w, true, engineKey) } } @@ -93,14 +144,15 @@ func (c *Config) Wake(ctx context.Context, w Want, dc remote.DeployConfig, resul // wakeable keeps the nodes that could be started, in the order to try them: a // node whose stored config already names the wanted model first, since it has // the weights and starts sooner. -func wakeable(cands []candidate, dc remote.DeployConfig) []candidate { +func wakeable(cands []candidate, resolver *configResolver) []candidate { var warm, cold []candidate for _, c := range cands { if !c.result.OK() || c.result.Status.State == string(daemon.StateRunning) { // A running engine is never displaced to make room. continue } - if c.result.Status.Model != "" && c.result.Status.Model == dc.ModelID { + dc, err := resolver.config(c.entry) + if err == nil && c.result.Status.Model != "" && c.result.Status.Model == dc.ModelID { warm = append(warm, c) continue } @@ -114,6 +166,11 @@ func wakeable(cands []candidate, dc remote.DeployConfig) []candidate { // weights, so a launch that trusted the state alone would hand the agent an // endpoint that refuses connections. // +// A daemon that reports its own readiness reading — the engine has answered +// its health check — is taken on that word; it checked from the same machine +// the engine runs on. A daemon that reports none (older builds, or a runner +// with no known health-check convention) falls back to the TCP probe. +// // On timeout the started engine is deliberately left running: it is probably // still loading, and stopping it throws away the only expensive part. func (c *Config) waitReady(ctx context.Context, node Node, entry NodeConfig, w Want, log Waker) (daemon.StatusResponse, error) { @@ -125,6 +182,9 @@ func (c *Config) waitReady(ctx context.Context, node Node, entry NodeConfig, w W if err == nil { last = status if status.State == string(daemon.StateRunning) { + if status.Ready == "ready" { + return status, nil + } baseURL, urlErr := c.EngineBaseURL(entry, status) if urlErr != nil { return status, urlErr @@ -191,8 +251,8 @@ func isAlreadyRunning(err error) bool { // is what lets a routing decision be explained before an agent depends on it — // and the reason the ordering lives in one place rather than being described // twice. -func (c *Config) WouldWake(results []NodeResult, dc remote.DeployConfig) (NodeConfig, bool) { - cands := wakeable(c.candidates(results), dc) +func (c *Config) WouldWake(results []NodeResult, cfgFor ConfigFor) (NodeConfig, bool) { + cands := wakeable(c.candidates(results), newConfigResolver(cfgFor)) if len(cands) == 0 { return NodeConfig{}, false } diff --git a/internal/fleet/wake_test.go b/internal/fleet/wake_test.go index 25ecc3df..e10f81f2 100644 --- a/internal/fleet/wake_test.go +++ b/internal/fleet/wake_test.go @@ -3,6 +3,7 @@ package fleet import ( "context" "encoding/json" + "fmt" "net" "net/http" "net/http/httptest" @@ -29,6 +30,11 @@ type fakeNode struct { startStatus int // engineDelay is how long after starting before the engine listens. engineDelay time.Duration + // ready, when set, is what /v1/status reports for `ready`. + ready bool + // noEngine keeps the engine's listener down even after an accepted start: + // readiness can only come from the daemon's own reading. + noEngine bool // started records whether a start was accepted. started bool // pushed is the deploy config the start carried. @@ -62,6 +68,9 @@ func newFakeNode(t *testing.T, state, model string) *fakeNode { resp := daemon.StatusResponse{State: f.state, Model: f.model} if f.state == string(daemon.StateRunning) { resp.Engine = &daemon.EngineEndpoint{Port: f.enginePort} + if f.ready { + resp.Ready = "ready" + } } json.NewEncoder(w).Encode(resp) }) @@ -87,11 +96,13 @@ func newFakeNode(t *testing.T, state, model string) *fakeNode { if dc.ModelID != "" { f.model = dc.ModelID } - delay := f.engineDelay - go func() { - time.Sleep(delay) - f.listenAsEngine() - }() + if !f.noEngine { + delay := f.engineDelay + go func() { + time.Sleep(delay) + f.listenAsEngine() + }() + } json.NewEncoder(w).Encode(daemon.StatusResponse{State: f.state, Model: f.model}) }) f.srv = httptest.NewServer(mux) @@ -153,7 +164,7 @@ func TestWakeStartsAnIdleNode(t *testing.T) { cfg := fleetOf(t, []string{"box"}, node) dc := remote.DeployConfig{Runner: "llamacpp", ModelID: "qwen3-27b"} - choice, err := cfg.Wake(context.Background(), Want{Model: "qwen3-27b"}, dc, statusOf(t, cfg), nil) + choice, err := cfg.Wake(context.Background(), Want{Model: "qwen3-27b"}, ConstantConfig(dc, nil), statusOf(t, cfg), nil) if err != nil { t.Fatal(err) } @@ -179,7 +190,7 @@ func TestWakeSkipsANodeThatRefusesTheConfig(t *testing.T) { cfg := fleetOf(t, []string{"wrong-box", "right-box"}, refuses, accepts) choice, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), nil) if err != nil { t.Fatal(err) } @@ -197,7 +208,7 @@ func TestWakeReportsEveryRefusal(t *testing.T) { cfg := fleetOf(t, []string{"a", "b"}, a, b) _, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "vllm", ModelID: "m"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "vllm", ModelID: "m"}, nil), statusOf(t, cfg), nil) if err == nil { t.Fatal("expected a failure when every node refuses") } @@ -220,7 +231,7 @@ func TestWakeWaitsForTheEngineToAnswer(t *testing.T) { start := time.Now() choice, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, statusOf(t, cfg), log) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), log) if err != nil { t.Fatal(err) } @@ -245,7 +256,7 @@ func TestWakeTimesOutWithoutStopping(t *testing.T) { cfg := fleetOf(t, []string{"stuck"}, node) _, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), nil) if err == nil { t.Fatal("expected a timeout") } @@ -279,7 +290,7 @@ func TestWakeLosingTheRaceUsesTheNode(t *testing.T) { Status: daemon.StatusResponse{State: string(daemon.StateIdle)}, }} choice, err := cfg.Wake(context.Background(), Want{Model: "qwen3-27b"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "qwen3-27b"}, stale, nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "qwen3-27b"}, nil), stale, nil) if err != nil { t.Fatalf("losing the race should not fail the launch: %v", err) } @@ -295,7 +306,7 @@ func TestWakeNeverDisplacesARunningEngine(t *testing.T) { cfg := fleetOf(t, []string{"busy"}, busy) _, err := cfg.Wake(context.Background(), Want{Model: "mine"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "mine"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "mine"}, nil), statusOf(t, cfg), nil) if err == nil { t.Fatal("expected a failure rather than a restart") } @@ -318,7 +329,7 @@ func TestWakePrefersANodeThatAlreadyHasTheModel(t *testing.T) { cfg := fleetOf(t, []string{"cold", "warm"}, cold, warm) choice, err := cfg.Wake(context.Background(), Want{Model: "qwen3-27b"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "qwen3-27b"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "qwen3-27b"}, nil), statusOf(t, cfg), nil) if err != nil { t.Fatal(err) } @@ -343,7 +354,7 @@ func TestWakeGatesTheEngineWithTheClientsKey(t *testing.T) { cfg.Nodes[0].EngineTokenEnv = "BOX_ENGINE_KEY" choice, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), nil) if err != nil { t.Fatal(err) } @@ -365,7 +376,7 @@ func TestWakeWithoutAKeyIsUngated(t *testing.T) { cfg := fleetOf(t, []string{"box"}, node) choice, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), nil) if err != nil { t.Fatal(err) } @@ -393,6 +404,94 @@ func TestRemoteRefusesToBeWoken(t *testing.T) { } } +// engineUpAfter makes the "engine" of a node started by someone else come +// listening after d: the node already reports running, so the engine's delay +// is not the start's. +func (f *fakeNode) engineUpAfter(d time.Duration) { + go func() { + time.Sleep(d) + f.listenAsEngine() + }() +} + +// The per-candidate resolver lets different nodes be started with different +// configs — the gateway shape, where each node's own Spinloop source decides +// what it would run. +func TestWakeTakesPerCandidateConfigs(t *testing.T) { + shortWake(t) + a := newFakeNode(t, string(daemon.StateIdle), "") + b := newFakeNode(t, string(daemon.StateIdle), "") + cfg := fleetOf(t, []string{"a", "b"}, a, b) + + cfgFor := func(entry NodeConfig) (remote.DeployConfig, error) { + if entry.Name == "a" { + return remote.DeployConfig{}, fmt.Errorf("node %q names no Spinloop source", entry.Name) + } + return remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil + } + choice, err := cfg.Wake(context.Background(), Want{Model: "m"}, cfgFor, statusOf(t, cfg), nil) + if err != nil { + t.Fatal(err) + } + if choice.Node.Name != "b" { + t.Errorf("chose %q, want the node whose source resolved", choice.Node.Name) + } + b.mu.Lock() + defer b.mu.Unlock() + if b.pushed == nil || b.pushed.ModelID != "m" { + t.Errorf("the node's own config did not reach it: %+v", b.pushed) + } +} + +// A node woken by someone else is not taken on state alone: its engine may +// still be loading, so the same readiness wait applies to a node we did not +// start ourselves. +func TestWakeWaitsForARacedNodeToAnswer(t *testing.T) { + shortWake(t) + node := newFakeNode(t, string(daemon.StateRunning), "qwen3-27b") + node.startErr = "an engine is already running" + node.startStatus = http.StatusConflict + node.engineUpAfter(150 * time.Millisecond) + cfg := fleetOf(t, []string{"contested"}, node) + + stale := []NodeResult{{ + Name: "contested", + Outcome: OutcomeOK, + Status: daemon.StatusResponse{State: string(daemon.StateIdle)}, + }} + start := time.Now() + choice, err := cfg.Wake(context.Background(), Want{Model: "qwen3-27b"}, + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "qwen3-27b"}, nil), stale, nil) + if err != nil { + t.Fatalf("losing the race should not fail the launch: %v", err) + } + if time.Since(start) < 150*time.Millisecond { + t.Error("returned before the raced node's engine was listening") + } + if choice.Node.Name != "contested" { + t.Errorf("chose %q", choice.Node.Name) + } +} + +// A daemon that reports its own readiness reading is trusted without a probe: +// it checked from the same machine the engine runs on. +func TestWakeTrustsTheDaemonReadinessReading(t *testing.T) { + shortWake(t) + node := newFakeNode(t, string(daemon.StateIdle), "") + node.ready = true + node.noEngine = true // the probe could never succeed; only the reading could + cfg := fleetOf(t, []string{"box"}, node) + + choice, err := cfg.Wake(context.Background(), Want{Model: "m"}, + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), nil) + if err != nil { + t.Fatalf("the daemon's own readiness reading should have been enough: %v", err) + } + if choice.Node.Name != "box" { + t.Errorf("chose %q", choice.Node.Name) + } +} + // A variable that resolves to nothing fails before any engine is started. func TestWakeFailsOnAnUnresolvableKey(t *testing.T) { shortWake(t) @@ -401,7 +500,7 @@ func TestWakeFailsOnAnUnresolvableKey(t *testing.T) { cfg.Nodes[0].EngineTokenEnv = "NOWHERE_ENGINE_KEY" _, err := cfg.Wake(context.Background(), Want{Model: "m"}, - remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, statusOf(t, cfg), nil) + ConstantConfig(remote.DeployConfig{Runner: "llamacpp", ModelID: "m"}, nil), statusOf(t, cfg), nil) if err == nil { t.Fatal("expected a failure") } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go new file mode 100644 index 00000000..59d65bc3 --- /dev/null +++ b/internal/gateway/gateway.go @@ -0,0 +1,419 @@ +// The fleet gateway: one OpenAI-compatible endpoint in front of a fleet. It +// answers agent requests with the fleet's own selector, holds each node's +// engine key, and wakes a node when nothing is serving what a request asks +// for — so a machine running an agent needs nothing but a URL and one token. +// +// It is a foreground process, the way `spinloop serve` is: it holds the fleet +// file it serves, and a machine that hosts agents points its Spinloop's FLEET +// at the address it prints. + +package gateway + +import ( + "bytes" + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync" + "time" + + "github.com/spinloop-ai/spinloop/internal/daemon" + "github.com/spinloop-ai/spinloop/internal/fleet" + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// DefaultListen is where the gateway answers when --listen is not given: a +// fixed port on every interface, so a Spinloop's FLEET can name one address +// without knowing the machine it lands on. +const DefaultListen = ":4000" + +// LoopbackListen is where `--loopback` binds the gateway: the default port on +// loopback, the safe bind a local-only gateway wants — one that Listen's token +// check accepts without a token. +const LoopbackListen = "127.0.0.1:4000" + +// cacheTTL is how long a fan-out reading stays fresh for routing. A burst of +// requests pays one fan-out, not one each; a request older than this sees the +// fleet as it is now. A variable so tests do not wait. +var cacheTTL = 2 * time.Second + +// pathsServed is the surface the gateway answers, for the 404 that names it. +var pathsServed = []string{"/v1/models", "/v1/chat/completions", "/v1/completions", "/health"} + +// Options shapes a handler beyond the fleet it serves. +type Options struct { + // ConfigFor resolves the deploy config a candidate node would be started + // with — each node's own Spinloop source. nil disables waking: nothing can + // be started, and a request nothing is serving fails saying so. + ConfigFor fleet.ConfigFor + // Log receives the gateway's log lines; nil discards them. + Log *slog.Logger + // Now is the clock the reading cache ages against; nil uses time.Now. + Now func() time.Time +} + +// Handler is the gateway: the fleet it serves, the token its callers present, +// and the state a burst of requests shares — the last reading of the fleet, +// and the reading's age. +type Handler struct { + cfg *fleet.Config + token string + cfgFor fleet.ConfigFor + log *slog.Logger + now func() time.Time + + mu sync.Mutex + results []fleet.NodeResult + at time.Time +} + +// New builds a gateway handler over a resolved fleet file. The token is the +// one callers must present — empty on loopback, where none is needed, as the +// daemon's control API allows. +func New(cfg *fleet.Config, token string, opts Options) *Handler { + log := opts.Log + if log == nil { + log = slog.New(slog.DiscardHandler) + } + now := opts.Now + if now == nil { + now = time.Now + } + return &Handler{cfg: cfg, token: token, cfgFor: opts.ConfigFor, log: log, now: now} +} + +// Listen opens the gateway's listener, applying the daemon's exposure rule: +// a non-loopback address is refused without a token, because it would put an +// engine's full output on the network for anyone to read. +func Listen(addr, token string) (net.Listener, error) { + if token == "" && !loopbackAddr(addr) { + return nil, fmt.Errorf( + "refusing to listen on non-loopback %q without a token: "+ + "pass --api-token-file , set %s, or pass --api-token — "+ + "or bind loopback, e.g. --listen 127.0.0.1:4000, which needs none", + addr, daemon.TokenEnvVar) + } + return net.Listen("tcp", addr) +} + +// loopbackAddr reports whether a listen address binds only loopback. An empty +// or wildcard host binds every interface, so it is not loopback. +func loopbackAddr(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + if host == "" { + return false + } + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// ServeHTTP is the gateway's whole surface: the three paths it serves, a +// health check that touches no node, and a 404 that names the rest. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var handler http.HandlerFunc + switch { + case r.Method == http.MethodGet && r.URL.Path == "/health": + handler = h.handleHealth + case r.Method == http.MethodGet && r.URL.Path == "/v1/models": + handler = h.handleModels + case r.Method == http.MethodPost && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/completions"): + handler = h.handleCompletion + default: + handler = func(w http.ResponseWriter, r *http.Request) { + writeError(w, http.StatusNotFound, fmt.Errorf( + "the gateway serves %s, not %s %s", strings.Join(pathsServed, ", "), r.Method, r.URL.Path)) + } + } + h.authenticate(handler)(w, r) +} + +// authenticate gates every request behind the bearer token, on the daemon's +// terms: an empty token means no auth, which Listen permits on loopback only. +func (h *Handler) authenticate(next http.HandlerFunc) http.HandlerFunc { + if h.token == "" { + return next + } + want := []byte("Bearer " + h.token) + return func(w http.ResponseWriter, r *http.Request) { + got := []byte(r.Header.Get("Authorization")) + if subtle.ConstantTimeCompare(got, want) != 1 { + writeError(w, http.StatusUnauthorized, fmt.Errorf("missing or invalid bearer token")) + return + } + next(w, r) + } +} + +// handleHealth answers that the gateway is up. It contacts no node on +// purpose: it is how an operator tells the gateway down from the fleet down. +func (h *Handler) handleHealth(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) +} + +// handleModels lists what the fleet is running, in the OpenAI list shape: the +// served name when a running node reports one, else the model id, duplicates +// once. Nothing running is an empty list, not an error. +func (h *Handler) handleModels(w http.ResponseWriter, r *http.Request) { + results := h.reading(r.Context()) + seen := map[string]bool{} + data := []map[string]any{} + for _, res := range results { + if !res.OK() || res.Status.State != string(daemon.StateRunning) { + continue + } + name := res.Status.ServedName + if name == "" { + name = res.Status.Model + } + if name == "" || seen[name] { + continue + } + seen[name] = true + data = append(data, map[string]any{"id": name, "object": "model"}) + } + writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data}) +} + +// reading returns the fleet's last fan-out, taking one when the last is stale. +// It is the whole freshness story: a burst of requests shares one reading, +// and a request older than cacheTTL sees the fleet as it is now. The mutex is +// held for the fan-out, so requests that arrive mid-fan-out wait for it and +// take its result rather than fanning out again. +func (h *Handler) reading(ctx context.Context) []fleet.NodeResult { + h.mu.Lock() + defer h.mu.Unlock() + if h.results != nil && h.now().Sub(h.at) < cacheTTL { + return h.results + } + h.results = h.cfg.FanOut(ctx, fleet.StatusCall) + h.at = h.now() + return h.results +} + +// handleCompletion routes a completion request to the node serving its model, +// waking one when nothing is and the fleet file allows it. +func (h *Handler) handleCompletion(w http.ResponseWriter, r *http.Request) { + model, body, err := requestModel(r) + if err != nil { + writeError(w, http.StatusBadRequest, err) + return + } + if model == "" { + writeError(w, http.StatusBadRequest, fmt.Errorf( + "the request names no model: completion requests need a `model` field")) + return + } + ctx := r.Context() + prefer, err := h.cfg.Preference("") + if err != nil { + writeError(w, http.StatusBadRequest, err) + return + } + want := fleet.Want{Model: model, Prefer: prefer} + + results := h.reading(ctx) + choice, err := h.cfg.Choose(h.reachable(results), want) + if err != nil { + var none *fleet.ErrNoneServing + if !errors.As(err, &none) { + writeError(w, http.StatusBadGateway, err) + return + } + choice, err = h.wakeFor(ctx, want, none.Results) + if err != nil { + writeError(w, http.StatusServiceUnavailable, err) + return + } + } + + h.log.Info("routed", + slog.String("model", model), + slog.String("node", choice.Node.Name), + slog.Bool("woken", choice.Woken)) + h.proxy(w, r, body, choice) +} + +// requestModel pulls the model field out of a completion request and returns +// it with the full body, which the proxy must forward unmodified. A body that +// is not a JSON object fails saying so, rather than being routed at a guess. +func requestModel(r *http.Request) (string, []byte, error) { + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return "", nil, fmt.Errorf("reading the request: %w", err) + } + var req struct { + Model string `json:"model"` + } + if len(bytes.TrimSpace(body)) > 0 { + if err := json.Unmarshal(body, &req); err != nil { + return "", nil, fmt.Errorf("the request is not a JSON body: %v", err) + } + } + return req.Model, body, nil +} + +// reachable marks the running engines the gateway cannot reach as not +// candidates: a node whose engine is bound to loopback, without an override +// taking responsibility for reachability, answers only on its own machine, so +// selecting it would hold a request for the wake timeout at best. The mark +// carries the daemon's own explanation, which names the bind and the fix, so +// a fleet where that is the only match fails saying so. +func (h *Handler) reachable(results []fleet.NodeResult) []fleet.NodeResult { + out := make([]fleet.NodeResult, len(results)) + for i, res := range results { + out[i] = res + if !res.OK() || res.Status.State != string(daemon.StateRunning) { + continue + } + entry, ok := h.cfg.Node(res.Name) + if !ok { + continue + } + if _, err := h.cfg.EngineBaseURL(entry, res.Status); err != nil { + out[i] = fleet.NodeResult{ + Name: res.Name, + Outcome: fleet.OutcomeUnreachable, + Err: err, + Status: res.Status, + At: res.At, + } + } + } + return out +} + +// wakeFor starts a node for a request nothing is serving, when the fleet file +// allows it, and holds the request until the engine answers. A concurrent +// request waking the same node loses its start to the daemon's 409 and takes +// the node the other one started — the same engine, the same wait. +func (h *Handler) wakeFor(ctx context.Context, want fleet.Want, results []fleet.NodeResult) (*fleet.Choice, error) { + none := &fleet.ErrNoneServing{Results: results, Want: want, Path: h.cfg.Path} + if h.cfgFor == nil { + return nil, fmt.Errorf("%s\nthis gateway can wake no node: it has no way to resolve a node's Spinloop source", none) + } + if !h.cfg.Wakes() { + return nil, h.refuseWake(want, none) + } + log := func(format string, args ...any) { + h.log.Info(strings.TrimSuffix(fmt.Sprintf(format, args...), "\n")) + } + return h.cfg.Wake(ctx, want, h.matchingConfigFor(want.Model), results, log) +} + +// refuseWake is the wake-off answer: nothing is started, and the failure names +// the node whose source describes the model and the command that would start +// it — or, when no source describes it, that there is nothing to start. +func (h *Handler) refuseWake(want fleet.Want, none error) error { + cfgFor := h.matchingConfigFor(want.Model) + for _, entry := range h.cfg.Nodes { + if _, err := cfgFor(entry); err == nil { + return fmt.Errorf("%s\nwake is off in %s: %q's source describes %s; start it with `spinloop fleet start %s`", + none, h.cfg.Path, entry.Name, want.Model, entry.Name) + } + } + return fmt.Errorf("%s\nwake is off in %s, and no node's source describes %s", none, h.cfg.Path, want.Model) +} + +// matchingConfigFor wraps the per-node source resolver with the one condition +// a wake has to meet: the source's config is the model the request asks for. +// A node whose source describes a different model is not a candidate — it +// would be started with the wrong engine — and its refusal says so. +func (h *Handler) matchingConfigFor(model string) fleet.ConfigFor { + base := h.cfgFor + return func(entry fleet.NodeConfig) (remote.DeployConfig, error) { + dc, err := base(entry) + if err != nil { + return dc, err + } + if dc.ModelID != model && dc.ServedModelName != model { + described := dc.ServedModelName + if described == "" { + described = dc.ModelID + } + return dc, fmt.Errorf("its source describes %s, not %s", described, model) + } + return dc, nil + } +} + +// proxy forwards the request to the chosen engine and the engine's reply back +// to the caller, unmodified: the body goes out as it came in, a streamed +// reply passes through as it is produced, and the reply the engine gives is +// the reply the caller gets — the gateway never retries another node. +func (h *Handler) proxy(w http.ResponseWriter, r *http.Request, body []byte, choice *fleet.Choice) { + target, err := url.Parse(choice.BaseURL) + if err != nil { + writeError(w, http.StatusBadGateway, fmt.Errorf("node %q's engine address is not a URL: %v", choice.Node.Name, err)) + return + } + p := &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.SetURL(target) + // The caller asks the gateway for /v1/; the engine's base + // URL already carries its own prefix, so /v1 comes off the + // request and the rest goes on the base. + pr.Out.URL.Path = joinPath(target.Path, strings.TrimPrefix(pr.In.URL.Path, "/v1")) + pr.Out.URL.RawQuery = pr.In.URL.RawQuery + }, + // -1 flushes after every write: a streamed reply reaches the caller + // as the engine produces it, not when a buffer fills. + FlushInterval: -1, + // The default transport, with its connection pooling and no overall + // request timeout: a completion may take as long as the model takes. + ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + h.log.Error("route failed", + slog.String("node", choice.Node.Name), + slog.String("error", err.Error())) + writeError(w, http.StatusBadGateway, fmt.Errorf( + "the engine on %s failed to answer: %v", choice.Node.Name, err)) + }, + } + + out := r.Clone(r.Context()) + out.Body = io.NopCloser(bytes.NewReader(body)) + out.ContentLength = int64(len(body)) + if choice.APIKey != "" { + // The caller's authoriser never travels past the gateway: the engine + // is reached with the key its fleet entry names, resolved the way + // every other fleet client resolves it. + out.Header.Set("Authorization", "Bearer "+choice.APIKey) + } else { + out.Header.Del("Authorization") + } + p.ServeHTTP(w, out) +} + +// joinPath joins a base path and a request path with at most one slash +// between them, either piece absent. +func joinPath(base, rest string) string { + base = strings.TrimRight(base, "/") + rest = "/" + strings.TrimLeft(rest, "/") + return base + rest +} + +// writeJSON sends a JSON reply. +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) +} + +// writeError sends a JSON error in the shape the OpenAI surface uses. +func writeError(w http.ResponseWriter, status int, err error) { + writeJSON(w, status, map[string]any{"error": map[string]any{"message": err.Error(), "type": "gateway_error"}}) +} diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go new file mode 100644 index 00000000..7288c2e4 --- /dev/null +++ b/internal/gateway/gateway_test.go @@ -0,0 +1,821 @@ +package gateway + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/spinloop-ai/spinloop/internal/daemon" + "github.com/spinloop-ai/spinloop/internal/fleet" + "github.com/spinloop-ai/spinloop/internal/remote" +) + +// fakeNode is one machine: its daemon's control API and, once running, its +// engine's real HTTP endpoint on the port the daemon reports — so the +// gateway's proxy path is exercised end to end against a listener, not a +// stub. The engine's port is reserved at construction and occupied only when +// the engine is meant to be up, which is what readiness means here. +type fakeNode struct { + mu sync.Mutex + // state and what the daemon reports serving. + state string + model string + servedName string + // ready, when set, is what the daemon reports for `ready`. + ready bool + // engineAuth, when set, is what the engine requires as its key. + engineAuth string + // startErr and startStatus are a refused start's reply. + startErr string + startStatus int + // engineDelay is how long after a start the engine listens. + engineDelay time.Duration + // noEngine keeps the engine down even after an accepted start: + // readiness can only come from the daemon's own reading. + noEngine bool + // loopbackOnly is what the daemon reports for its engine's bind. + loopbackOnly bool + // started counts accepted starts; pushed is the last config and key. + started int + pushed *remote.DeployConfig + pushedKey string + // engineGotAuth is the last authorisation the engine itself saw. + engineGotAuth string + // statusHits counts status calls, so a burst's fan-out is countable. + statusHits int + + daemonSrv *httptest.Server + engine *http.Server + engineLn net.Listener + enginePort int +} + +func newFakeNode(t *testing.T, state, model string) *fakeNode { + t.Helper() + f := &fakeNode{state: state, model: model} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f.enginePort = ln.Addr().(*net.TCPAddr).Port + ln.Close() // free it until the engine is meant to be up + f.engine = &http.Server{Handler: f.engineHandler()} + // A node already running has its engine up: its port answers. + if state == string(daemon.StateRunning) { + f.upAsEngine() + } + f.daemonSrv = httptest.NewServer(f.daemonMux()) + t.Cleanup(func() { + f.daemonSrv.Close() + f.mu.Lock() + defer f.mu.Unlock() + if f.engineLn != nil { + f.engineLn.Close() + } + }) + return f +} + +// engineHandler is the engine itself: it checks its key, streams when asked, +// and fails when told to. +func (f *fakeNode) engineHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.engineGotAuth = r.Header.Get("Authorization") + auth := f.engineAuth + f.mu.Unlock() + if auth != "" && f.engineGotAuth != "Bearer "+auth { + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, `{"error":"bad key"}`) + return + } + body, _ := io.ReadAll(r.Body) + switch { + case strings.Contains(string(body), `"stream":true`): + w.Header().Set("Content-Type", "text/event-stream") + fl := w.(http.Flusher) + for _, chunk := range []string{"data: one\n\n", "data: two\n\n", "data: [DONE]\n\n"} { + io.WriteString(w, chunk) + fl.Flush() + } + case strings.Contains(string(body), `"fail":true`): + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error":"the engine is on fire"}`) + default: + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"cmpl-1","object":"chat.completion","choices":[{"message":{"role":"assistant","content":"hello"}}]}`) + } + }) +} + +func (f *fakeNode) daemonMux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/v1/status", func(w http.ResponseWriter, _ *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.statusHits++ + resp := daemon.StatusResponse{State: f.state, Model: f.model, ServedName: f.servedName} + if f.state == string(daemon.StateRunning) { + resp.Engine = &daemon.EngineEndpoint{ + Port: f.enginePort, + LoopbackOnly: f.loopbackOnly, + RequiresKey: f.engineAuth != "", + } + if f.ready { + resp.Ready = "ready" + } + } + json.NewEncoder(w).Encode(resp) + }) + mux.HandleFunc("/v1/start", func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + if f.started > 0 && f.state == string(daemon.StateRunning) && f.startErr == "" { + // The daemon's own conflict: another start while one runs. + w.WriteHeader(http.StatusConflict) + json.NewEncoder(w).Encode(daemon.Error{Error: "an engine is already running"}) + f.mu.Unlock() + return + } + if f.startErr != "" { + status := f.startStatus + if status == 0 { + status = http.StatusBadRequest + } + w.WriteHeader(status) + json.NewEncoder(w).Encode(daemon.Error{Error: f.startErr}) + f.mu.Unlock() + return + } + var req daemon.StartRequest + json.NewDecoder(r.Body).Decode(&req) + dc := req.DeployConfig + f.pushed = &dc + f.pushedKey = req.EngineAPIKey + f.started++ + f.state = string(daemon.StateRunning) + if dc.ModelID != "" { + f.model = dc.ModelID + } + f.servedName = dc.ServedModelName + delay := f.engineDelay + noEngine := f.noEngine + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(daemon.StatusResponse{State: f.state, Model: f.model}) + f.mu.Unlock() + if !noEngine { + go func() { + time.Sleep(delay) + f.upAsEngine() + }() + } + }) + return mux +} + +// upAsEngine occupies the reserved port, which readiness probes and the proxy +// both dial. +func (f *fakeNode) upAsEngine() { + ln, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(f.enginePort))) + if err != nil { + return + } + f.mu.Lock() + f.engineLn = ln + f.mu.Unlock() + go f.engine.Serve(ln) +} + +// nodeConfig is the fleet-file entry pointing at this fake. +func (f *fakeNode) nodeConfig(name string) fleet.NodeConfig { + host, port, _ := net.SplitHostPort(strings.TrimPrefix(f.daemonSrv.URL, "http://")) + p, _ := strconv.Atoi(port) + return fleet.NodeConfig{Name: name, Host: host, Port: p, Kind: fleet.KindDaemon} +} + +// fleetOf builds a Config over the fakes, in the order given. +func fleetOf(t *testing.T, names []string, nodes ...*fakeNode) *fleet.Config { + t.Helper() + cfg := &fleet.Config{Path: "fleet.yaml", Dir: t.TempDir()} + for i, n := range nodes { + cfg.Nodes = append(cfg.Nodes, n.nodeConfig(names[i])) + } + return cfg +} + +// cfgForOf is a per-node source resolver from a table: what each node's own +// Spinloop source would resolve to, or its refusal. +func cfgForOf(t *testing.T, table map[string]remote.DeployConfig, refused map[string]string) fleet.ConfigFor { + return func(entry fleet.NodeConfig) (remote.DeployConfig, error) { + if msg, ok := refused[entry.Name]; ok { + return remote.DeployConfig{}, fmt.Errorf("%s", msg) + } + dc, ok := table[entry.Name] + if !ok { + return remote.DeployConfig{}, fmt.Errorf("node %q names no Spinloop source: no `file` field, no alias, no subdirectory", entry.Name) + } + return dc, nil + } +} + +// post sends one completion request to a handler and returns the reply. +func post(t *testing.T, h http.Handler, token string, body string) (*http.Response, string) { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "http://gw/v1/chat/completions", strings.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + data, _ := io.ReadAll(rec.Result().Body) + return rec.Result(), string(data) +} + +// --- the surface ---------------------------------------------------------- + +func TestListenRefusesTokenlessNonLoopback(t *testing.T) { + _, err := Listen("0.0.0.0:0", "") + if err == nil { + t.Fatal("a tokenless non-loopback listen should be refused") + } + for _, want := range []string{"--api-token-file", "SPINLOOP_API_TOKEN", "--api-token"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should name %q, got: %v", want, err) + } + } + ln, err := Listen("127.0.0.1:0", "") + if err != nil { + t.Fatalf("a tokenless loopback listen is allowed: %v", err) + } + ln.Close() +} + +func TestCallerAuthentication(t *testing.T) { + cfg := fleetOf(t, []string{"box"}, newFakeNode(t, string(daemon.StateIdle), "")) + h := New(cfg, "secret", Options{}) + + // Missing and wrong tokens are 401, and no node is contacted. + for _, tok := range []string{"", "wrong"} { + req := httptest.NewRequest(http.MethodGet, "http://gw/health", nil) + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("token %q: HTTP %d, want 401", tok, rec.Code) + } + } + + // The right one is through. + req := httptest.NewRequest(http.MethodGet, "http://gw/health", nil) + req.Header.Set("Authorization", "Bearer secret") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("right token: HTTP %d, want 200", rec.Code) + } +} + +func TestTokenlessLoopbackServesWithoutAuth(t *testing.T) { + cfg := fleetOf(t, []string{"box"}, newFakeNode(t, string(daemon.StateIdle), "")) + h := New(cfg, "", Options{}) + req := httptest.NewRequest(http.MethodGet, "http://gw/health", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("HTTP %d, want 200 with no token", rec.Code) + } +} + +func TestHealthTouchesNoNode(t *testing.T) { + // A node on a dead port: unreachable, but health must not care. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + ln.Close() + cfg := &fleet.Config{Path: "fleet.yaml", Dir: t.TempDir(), Nodes: []fleet.NodeConfig{ + {Name: "down", Host: "127.0.0.1", Port: port, Kind: fleet.KindDaemon}, + }} + h := New(cfg, "", Options{}) + req := httptest.NewRequest(http.MethodGet, "http://gw/health", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("health with every node down: HTTP %d, want 200", rec.Code) + } +} + +func TestUnknownPathNamesTheSurface(t *testing.T) { + cfg := fleetOf(t, []string{"box"}, newFakeNode(t, string(daemon.StateIdle), "")) + h := New(cfg, "", Options{}) + req := httptest.NewRequest(http.MethodGet, "http://gw/v1/embeddings", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusNotFound { + t.Fatalf("HTTP %d, want 404", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{"/v1/models", "/v1/chat/completions", "/v1/completions", "/health"} { + if !strings.Contains(body, want) { + t.Errorf("404 should name %s, got: %s", want, body) + } + } +} + +// --- models ---------------------------------------------------------------- + +func TestModelsListsWhatIsRunning(t *testing.T) { + aliased := newFakeNode(t, string(daemon.StateRunning), "org/model") + aliased.servedName = "the-alias" + plain := newFakeNode(t, string(daemon.StateRunning), "org/other") + stopped := newFakeNode(t, string(daemon.StateStopped), "org/stale") + cfg := fleetOf(t, []string{"aliased", "plain", "stopped"}, aliased, plain, stopped) + h := New(cfg, "", Options{}) + + req := httptest.NewRequest(http.MethodGet, "http://gw/v1/models", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("HTTP %d, want 200", rec.Code) + } + var list struct { + Object string `json:"object"` + Data []map[string]any `json:"data"` + } + json.NewDecoder(rec.Body).Decode(&list) + if list.Object != "list" { + t.Errorf("object = %q, want list", list.Object) + } + var ids []string + for _, m := range list.Data { + ids = append(ids, m["id"].(string)) + } + for _, want := range []string{"the-alias", "org/other"} { + found := false + for _, id := range ids { + if id == want { + found = true + } + } + if !found { + t.Errorf("models should list %q, got %v", want, ids) + } + } + for _, id := range ids { + if id == "org/stale" { + t.Error("a stopped node's model is not running, and is not listed") + } + } +} + +func TestModelsEmptyWhenNothingRuns(t *testing.T) { + cfg := fleetOf(t, []string{"box"}, newFakeNode(t, string(daemon.StateIdle), "")) + h := New(cfg, "", Options{}) + req := httptest.NewRequest(http.MethodGet, "http://gw/v1/models", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("HTTP %d, want 200: nothing running is an empty list, not an error", rec.Code) + } + var list struct { + Data []map[string]any `json:"data"` + } + json.NewDecoder(rec.Body).Decode(&list) + if len(list.Data) != 0 { + t.Errorf("data = %v, want empty", list.Data) + } +} + +// --- routing ---------------------------------------------------------------- + +func TestRequestGoesToTheNodeServingItsModel(t *testing.T) { + t.Setenv("RIGHT_ENGINE_KEY", "engine-key") + right := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + wrong := newFakeNode(t, string(daemon.StateRunning), "org/other") + cfg := fleetOf(t, []string{"right", "wrong"}, right, wrong) + cfg.Nodes[0].EngineTokenEnv = "RIGHT_ENGINE_KEY" + right.engineAuth = "engine-key" + h := New(cfg, "caller-token", Options{}) + + resp, body := post(t, h, "caller-token", `{"model":"org/wanted","messages":[]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("HTTP %d, body %s", resp.StatusCode, body) + } + if !strings.Contains(body, "hello") { + t.Errorf("the engine's reply did not reach the caller: %s", body) + } + right.mu.Lock() + defer right.mu.Unlock() + // The engine got its own key, not the caller's token. + if right.engineGotAuth != "Bearer engine-key" { + t.Errorf("engine saw %q, want the fleet's key", right.engineGotAuth) + } + if strings.Contains(right.engineGotAuth, "caller-token") { + t.Error("the caller's token travelled past the gateway") + } +} + +func TestRequestNamesNoModel(t *testing.T) { + node := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + cfg := fleetOf(t, []string{"box"}, node) + h := New(cfg, "", Options{}) + resp, body := post(t, h, "", `{"messages":[]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("HTTP %d, want 400", resp.StatusCode) + } + if !strings.Contains(body, "no model") { + t.Errorf("the refusal should say the request names no model: %s", body) + } + node.mu.Lock() + defer node.mu.Unlock() + if node.statusHits != 0 { + t.Error("a refused request contacted a node") + } +} + +func TestStreamedReplyPassesThrough(t *testing.T) { + node := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + cfg := fleetOf(t, []string{"box"}, node) + h := New(cfg, "", Options{}) + resp, body := post(t, h, "", `{"model":"org/wanted","stream":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("HTTP %d, body %s", resp.StatusCode, body) + } + if resp.Header.Get("Content-Type") != "text/event-stream" { + t.Errorf("content type = %q, want the engine's stream type", resp.Header.Get("Content-Type")) + } + for _, want := range []string{"data: one", "data: two", "data: [DONE]"} { + if !strings.Contains(body, want) { + t.Errorf("stream lost %q: %s", want, body) + } + } +} + +func TestEngineRefusalIsTheCallersError(t *testing.T) { + t.Setenv("KEY", "k") + failing := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + other := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + cfg := fleetOf(t, []string{"failing", "other"}, failing, other) + h := New(cfg, "", Options{}) + + resp, body := post(t, h, "", `{"model":"org/wanted","fail":true}`) + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("the engine's 500 is the caller's: HTTP %d, body %s", resp.StatusCode, body) + } + if !strings.Contains(body, "on fire") { + t.Errorf("the engine's own refusal should reach the caller: %s", body) + } + other.mu.Lock() + defer other.mu.Unlock() + // The other node was never tried: its engine saw no request at all. + if other.engineGotAuth != "" { + t.Error("a failed request was retried at another node") + } +} + +func TestEngineDownFailsNamingTheNode(t *testing.T) { + node := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + cfg := fleetOf(t, []string{"box"}, node) + h := New(cfg, "", Options{}) + node.mu.Lock() + node.engineLn.Close() // the engine dies while the daemon still reports it + node.mu.Unlock() + + resp, body := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("HTTP %d, want 502: %s", resp.StatusCode, body) + } + if !strings.Contains(body, "box") { + t.Errorf("the failure should name the node: %s", body) + } +} + +// A node reached by a non-loopback name with a loopback-bound engine, without +// an override, is not a candidate; the mark carries the daemon's explanation, +// which names the bind and the fix. +func TestReachableMarksLoopbackBoundEngines(t *testing.T) { + cfg := &fleet.Config{Path: "fleet.yaml", Dir: t.TempDir(), Nodes: []fleet.NodeConfig{ + {Name: "bound", Host: "remote-box", Kind: fleet.KindDaemon, Port: 14242}, + {Name: "overridden", Host: "remote-box", Kind: fleet.KindDaemon, Port: 14243, + Engine: &fleet.EngineOverride{Host: "proxy.local", Port: 9000, Path: "/v1"}}, + }} + h := New(cfg, "", Options{}) + results := []fleet.NodeResult{ + {Name: "bound", Outcome: fleet.OutcomeOK, Status: daemon.StatusResponse{ + State: string(daemon.StateRunning), Model: "m", + Engine: &daemon.EngineEndpoint{Port: 8080, LoopbackOnly: true}, + }}, + {Name: "overridden", Outcome: fleet.OutcomeOK, Status: daemon.StatusResponse{ + State: string(daemon.StateRunning), Model: "m", + Engine: &daemon.EngineEndpoint{Port: 8080, LoopbackOnly: true}, + }}, + } + marked := h.reachable(results) + if marked[0].OK() { + t.Fatal("a loopback-bound engine without an override is not a candidate") + } + if !strings.Contains(marked[0].Err.Error(), "loopback") { + t.Errorf("the mark should carry the explanation naming the bind: %v", marked[0].Err) + } + if !marked[1].OK() { + t.Errorf("an override takes responsibility for reachability: %v", marked[1].Err) + } +} + +// With nothing else matching, the failure the gateway gives names the bind +// and the fix rather than a bare "no node serving". +func TestLoopbackBoundEngineFailsNamingTheFix(t *testing.T) { + cfg := &fleet.Config{Path: "fleet.yaml", Dir: t.TempDir(), Nodes: []fleet.NodeConfig{ + {Name: "bound", Host: "remote-box", Kind: fleet.KindDaemon, Port: 14242}, + }} + h := New(cfg, "", Options{}) + results := []fleet.NodeResult{ + {Name: "bound", Outcome: fleet.OutcomeOK, Status: daemon.StatusResponse{ + State: string(daemon.StateRunning), Model: "org/wanted", + Engine: &daemon.EngineEndpoint{Port: 8080, LoopbackOnly: true}, + }}, + } + _, err := cfg.Choose(h.reachable(results), fleet.Want{Model: "org/wanted"}) + if err == nil { + t.Fatal("nothing reachable serves the model, so the request must fail") + } + for _, want := range []string{"bound", "loopback"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("failure should name %q: %v", want, err) + } + } +} + +// A remote environment's engine address arrives as its status's reported host — +// the control plane's published endpoint — so it is a candidate, not marked +// unreachable the way a loopback-bound engine is. Without this the gateway +// could list a remote node's model and then refuse to route a request to it. +func TestReachableRoutesToARemoteNode(t *testing.T) { + t.Setenv("TEST_ENGINE_KEY", "secret") + cfg := &fleet.Config{Path: "fleet.yaml", Dir: t.TempDir(), APIKeyEnv: "TEST_ENGINE_KEY", Nodes: []fleet.NodeConfig{ + {Name: "env", Kind: fleet.KindRemote}, + }} + h := New(cfg, "", Options{}) + results := []fleet.NodeResult{ + {Name: "env", Outcome: fleet.OutcomeOK, Status: daemon.StatusResponse{ + State: string(daemon.StateRunning), + Model: "org/wanted", + ServedName: "wanted", + Engine: &daemon.EngineEndpoint{Host: "1.2.3.4", Port: 8000, Path: "/v1"}, + }}, + } + choice, err := cfg.Choose(h.reachable(results), fleet.Want{Model: "wanted"}) + if err != nil { + t.Fatalf("a request for the remote node's model should route to it: %v", err) + } + if choice.Node.Name != "env" { + t.Errorf("choice = %q, want env", choice.Node.Name) + } + if choice.BaseURL != "http://1.2.3.4:8000/v1" { + t.Errorf("base URL = %q, want the control plane's published address", choice.BaseURL) + } + if choice.APIKey != "secret" { + t.Errorf("API key = %q, want the resolved engine key", choice.APIKey) + } +} + +func TestBurstFansOutOnce(t *testing.T) { + node := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + cfg := fleetOf(t, []string{"box"}, node) + now := time.Now() + h := New(cfg, "", Options{Now: func() time.Time { return now }}) + + for i := range 3 { + resp, _ := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("request %d: HTTP %d", i, resp.StatusCode) + } + } + node.mu.Lock() + hits := node.statusHits + node.mu.Unlock() + if hits != 1 { + t.Fatalf("a burst of three fanned out %d times, want once", hits) + } + + // A request after the reading ages fans out again. + now = now.Add(3 * time.Second) + resp, _ := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("stale-reading request: HTTP %d", resp.StatusCode) + } + node.mu.Lock() + hits = node.statusHits + node.mu.Unlock() + if hits != 2 { + t.Fatalf("the aged reading should have fanned out once more, got %d total", hits) + } +} + +// --- waking ----------------------------------------------------------------- + +func TestColdRequestWakesANode(t *testing.T) { + t.Setenv("BOX_ENGINE_KEY", "engine-key") + node := newFakeNode(t, string(daemon.StateIdle), "") + node.engineDelay = 100 * time.Millisecond + cfg := fleetOf(t, []string{"box"}, node) + cfg.Nodes[0].EngineTokenEnv = "BOX_ENGINE_KEY" + h := New(cfg, "", Options{ + ConfigFor: cfgForOf(t, + map[string]remote.DeployConfig{"box": {Runner: "llamacpp", ModelID: "org/wanted", ServedModelName: "org/wanted"}}, + nil), + }) + + start := time.Now() + resp, body := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("HTTP %d, body %s", resp.StatusCode, body) + } + if time.Since(start) < 100*time.Millisecond { + t.Error("answered before the engine was up") + } + node.mu.Lock() + defer node.mu.Unlock() + if node.started != 1 { + t.Fatalf("node started %d times, want once", node.started) + } + // Started with its own source's config, gated with the fleet's key. + if node.pushed == nil || node.pushed.ModelID != "org/wanted" { + t.Errorf("the node's own config did not reach it: %+v", node.pushed) + } + if node.pushedKey != "engine-key" { + t.Errorf("the engine was gated with %q, want the fleet's key", node.pushedKey) + } +} + +func TestConcurrentColdRequestsShareOneWake(t *testing.T) { + node := newFakeNode(t, string(daemon.StateIdle), "") + node.engineDelay = 150 * time.Millisecond + cfg := fleetOf(t, []string{"box"}, node) + h := New(cfg, "", Options{ + ConfigFor: cfgForOf(t, + map[string]remote.DeployConfig{"box": {Runner: "llamacpp", ModelID: "org/wanted"}}, + nil), + }) + + var wg sync.WaitGroup + codes := make([]int, 2) + for i := range codes { + wg.Add(1) + go func(i int) { + defer wg.Done() + resp, _ := post(t, h, "", `{"model":"org/wanted"}`) + codes[i] = resp.StatusCode + }(i) + } + wg.Wait() + for i, code := range codes { + if code != http.StatusOK { + t.Errorf("request %d: HTTP %d, want 200 from the shared wake", i, code) + } + } + node.mu.Lock() + defer node.mu.Unlock() + if node.started != 1 { + t.Fatalf("the node was started %d times, want once", node.started) + } +} + +func TestWakeTimeoutFailsTheRequestAndLeavesTheEngine(t *testing.T) { + old := fleet.WakeTimeout + fleet.WakeTimeout = 300 * time.Millisecond + t.Cleanup(func() { fleet.WakeTimeout = old }) + + node := newFakeNode(t, string(daemon.StateIdle), "") + node.noEngine = true // the engine never answers + cfg := fleetOf(t, []string{"box"}, node) + h := New(cfg, "", Options{ + ConfigFor: cfgForOf(t, + map[string]remote.DeployConfig{"box": {Runner: "llamacpp", ModelID: "org/wanted"}}, + nil), + }) + + resp, body := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("HTTP %d, want 503: %s", resp.StatusCode, body) + } + if !strings.Contains(body, "box") { + t.Errorf("the timeout should name the node: %s", body) + } + node.mu.Lock() + defer node.mu.Unlock() + if node.started != 1 || node.state != string(daemon.StateRunning) { + t.Error("the started engine is left running on timeout, not stopped") + } +} + +func TestWakeOffRefusesNamingTheNodeAndCommand(t *testing.T) { + node := newFakeNode(t, string(daemon.StateIdle), "") + cfg := fleetOf(t, []string{"box"}, node) + cfg.WakePolicy = fleet.WakeOff + h := New(cfg, "", Options{ + ConfigFor: cfgForOf(t, + map[string]remote.DeployConfig{"box": {Runner: "llamacpp", ModelID: "org/wanted"}}, + nil), + }) + + resp, body := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("HTTP %d, want 503", resp.StatusCode) + } + for _, want := range []string{"box", "spinloop fleet start box"} { + if !strings.Contains(body, want) { + t.Errorf("refusal should name %q: %s", want, body) + } + } + node.mu.Lock() + defer node.mu.Unlock() + if node.started != 0 { + t.Error("wake off starts nothing") + } +} + +func TestWakeOffWithNoMatchingSource(t *testing.T) { + node := newFakeNode(t, string(daemon.StateIdle), "") + cfg := fleetOf(t, []string{"box"}, node) + cfg.WakePolicy = fleet.WakeOff + h := New(cfg, "", Options{ + ConfigFor: cfgForOf(t, + map[string]remote.DeployConfig{"box": {Runner: "llamacpp", ModelID: "org/something-else"}}, + nil), + }) + resp, body := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("HTTP %d, want 503", resp.StatusCode) + } + if !strings.Contains(body, "no node's source describes") { + t.Errorf("the refusal should say nothing describes the model: %s", body) + } +} + +func TestNothingCanServeNamesEveryRefusal(t *testing.T) { + a := newFakeNode(t, string(daemon.StateIdle), "") + b := newFakeNode(t, string(daemon.StateIdle), "") + cfg := fleetOf(t, []string{"a", "b"}, a, b) + h := New(cfg, "", Options{ + ConfigFor: cfgForOf(t, + map[string]remote.DeployConfig{"a": {Runner: "llamacpp", ModelID: "org/other"}}, + map[string]string{"b": "node \"b\" names no Spinloop source: no `file` field, no alias, no subdirectory"}, + ), + }) + + resp, body := post(t, h, "", `{"model":"org/wanted"}`) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("HTTP %d, want 503", resp.StatusCode) + } + for _, want := range []string{"a", "b", "org/other", "Spinloop source"} { + if !strings.Contains(body, want) { + t.Errorf("the failure should name %q: %s", want, body) + } + } + a.mu.Lock() + b.mu.Lock() + defer func() { a.mu.Unlock(); b.mu.Unlock() }() + if a.started != 0 || b.started != 0 { + t.Error("nothing is started when nothing can serve") + } +} + +// --- logging ------------------------------------------------------------------ + +func TestRoutedRequestLeavesOneLogLine(t *testing.T) { + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, nil)) + node := newFakeNode(t, string(daemon.StateRunning), "org/wanted") + cfg := fleetOf(t, []string{"box"}, node) + h := New(cfg, "", Options{Log: log}) + + resp, _ := post(t, h, "", `{"model":"org/wanted","messages":[{"content":"a secret prompt"}]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("HTTP %d", resp.StatusCode) + } + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != 1 { + t.Fatalf("one routed request leaves one log line, got %d: %q", len(lines), buf.String()) + } + for _, want := range []string{"org/wanted", "box"} { + if !strings.Contains(lines[0], want) { + t.Errorf("the log line should name %q: %s", want, lines[0]) + } + } + if strings.Contains(buf.String(), "secret prompt") { + t.Error("the log carries the request body") + } +} diff --git a/internal/remote/remote.go b/internal/remote/remote.go index 189150a4..c1f8db6c 100644 --- a/internal/remote/remote.go +++ b/internal/remote/remote.go @@ -203,21 +203,28 @@ type Response struct { Environment string `json:"environment"` Message string `json:"message"` RetryAfterSeconds int `json:"retry_after_seconds"` - // Status-specific fields: the on-instance daemon's activity record, - // relayed by the status branch of the start Lambda. camelCase to match the - // daemon's own names, since these are copied through untouched — this - // struct is already mixed (see modelId, contextSize below). Absent when - // the instance is not running, when its daemon could not be reached, or - // when no engine has yet done any work. + // The on-instance daemon's activity record, relayed by the status branch of + // the start Lambda: when the engine last did work, and how long ago. + // camelCase to match the daemon's own names — this struct is already mixed + // (see modelId, contextSize below). Absent when the instance is not + // running, when its daemon could not be reached, or when no engine has yet + // done any work. LastActiveAt string `json:"lastActiveAt"` IdleSeconds int `json:"idleSeconds"` - // Deploy-specific fields. + // Deploy-specific fields. Runner, ModelID and ServedName are also relayed + // by the status reply, which reads them from the environment's deploy + // config — the same source the stats reply reads. Deployed bool `json:"deployed"` Seeding bool `json:"seeding"` // SeedID identifies the seed a deploy started, so it can be followed with // `spinloop remote seed status`. The instance id it replaces was an // implementation detail that changes if the seed is relaunched. - SeedID string `json:"seedId"` + SeedID string `json:"seedId"` + // ServedName is the name the engine answers to beside the model id — the + // served name the deploy gave it — relayed by the status reply from the + // environment's deploy config, so a caller may know the engine by either + // name. + ServedName string `json:"servedName"` Runner string `json:"runner"` ModelID string `json:"modelId"` ContextSize int `json:"contextSize"` diff --git a/openspec/changes/add-fleet-gateway/.openspec.yaml b/openspec/changes/add-fleet-gateway/.openspec.yaml new file mode 100644 index 00000000..1a62d62b --- /dev/null +++ b/openspec/changes/add-fleet-gateway/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-06 diff --git a/openspec/changes/add-fleet-gateway/design.md b/openspec/changes/add-fleet-gateway/design.md new file mode 100644 index 00000000..262a2ba5 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/design.md @@ -0,0 +1,241 @@ +## Context + +See proposal.md — Why. What matters for the approach is what already exists: + +- `internal/fleet` has the whole routing half: `Select` (fan-out plus the pure + `rank`), `Wake` (start-with-config, the warm-first candidate ordering, the + already-running race, `waitReady`), `EngineBaseURL` (composing the engine + address from the fleet file and the node's report), and `engineKeyFor` + (resolving a running engine's key). A gateway is a consumer of all of it. +- The wake path's one gap: when a start loses to another client's + already-running 409, `Wake` re-reads status and takes the node on the state + alone — a node can report `running` while still loading weights. Fine for a + client-side launch (the agent starts and the engine comes up moments later); + a gateway holding a caller's request must not proxy to an engine that is not + answering. +- The daemon's status says where the engine serves (`EngineEndpoint`) and, on + current daemons, whether it has answered its own health check (`Ready`), but + it reports the model id only, not the served name: a node started under an + alias answers requests for the alias and reports the id, so a router seeing + only status cannot match a request that carries the alias. +- The launch path consumes a `fleet.Choice`: `applyBeforeLaunch` writes + `choice.BaseURL` into the provider slot a `REMOTE` fills and wires + `choice.APIKey` through the same resolver the remote path uses. The endpoint + branch of `FLEET` is a new producer of the same `Choice`. +- The seam itself: `Selection.FleetIsEndpoint`, and the two + "gateway routing is not implemented yet" rejections in `routeThroughFleet` + and `fleet route`. + +## Goals / Non-Goals + +**Goals:** + +- One gateway that is the fleet client wearing a server: selection, waking, + endpoint resolution, and key handling all come from `internal/fleet`, so the + routing rules stay written once. +- A request held during a wake behaves like a slow first token, not an error: + the same readiness wait the launch path uses. +- The launch path's endpoint branch is a small new producer of `fleet.Choice`, + not a second routing mechanism. +- Every secret keeps its existing home: node tokens and engine keys in the + environment, named by the fleet file; the gateway's own token supplied the + way the daemon's is. + +**Non-Goals:** + +- TLS termination, budgets, spend tracking, per-user keys, retries across + nodes, request-body logging, queueing. The gateway stays a binary to run, + not a service to operate. +- New daemon endpoints, and new Spinloop keywords: `servedName` is one + additive status field, and `FLEET` already takes a URL. +- A gateway config file of its own: fleet.yaml plus flags is the configuration. + +## Decisions + +### A `gateway` command beside `serve`, a new `internal/gateway` package + +`spinloop gateway --fleet ./fleet.yaml --listen :4000` runs in the foreground +like `serve`: it holds a fleet file the way `serve` holds a Spinloop, and its +lifecycle is the process's. Top-level rather than `fleet gateway`, because it +does not observe the fleet the way `fleet status` does; it answers requests +through it, which is a different job. + +The HTTP surface lives in `internal/gateway` as an `http.Handler` built from a +`*fleet.Config`, the caller token, and a wake timeout. `cmd/spinloop/gateway.go` +resolves the fleet file, resolves the token, and serves. This keeps the +proxies, the model listing, the wake joining, and the status cache unit-testable +with a fake fleet, the same way the fleet client is tested. + +### The gateway wakes a node with the node's own config, never an invented one + +A completion request carries one name — the model — and a deploy config needs a +runner, a context, and serve args the request cannot name. The only config a +gateway can push is the one the node was told to run: its Spinloop source, +resolved exactly as `spinloop fleet start` resolves it (the `file` field, a +registered alias named after the node, a same-named directory beside the fleet +file). A node is therefore a wake candidate when it is not running and its +source's config names — as model or served name — the model the request asks +for. This is also the honest semantics: the gateway starts what the fleet file +says each node runs, and says so in the failures it reports. + +Consequently `fleet.Wake`'s single-config-per-all-candidates shape does not fit +the gateway, which needs one config per candidate. `Wake` is generalised to +take a per-candidate config function; the launch path passes a constant one +(derived from the Spinloop it wears), the gateway passes a resolver over the +node's own source. The ordering (warm-first), the refusal collection, the +already-running race, and the readiness wait all stay in `Wake`, written once. + +### Readiness is a gate before any request is proxied + +"Usable now" is three facts: the node reports `running`, the name it reports +serving matches the request, and its engine answers. The third comes from the +daemon's `Ready` field on current daemons and from the TCP probe the wake path +already uses (`engineAnswers`) where the field is absent, so older daemons are +not mistaken for stuck engines. + +The gateway's per-request shape is therefore: select (from the cached or fresh +fan-out) → wake if nothing is running and the policy allows → `WaitReady` on +the chosen node → proxy. A node that was already running but is still loading +goes through the same `WaitReady`, so "the request is held while the engine +loads" is one rule whether the gateway started the engine or found it loading. +`WaitReady` is lifted out of `Wake`'s private helper into an exported form that +both call. + +The same fix closes the client-side gap in the wake race: `Wake`'s +already-running branch currently takes a raced node on the state alone; it now +goes through the readiness wait, so no caller — launch or gateway — is handed +an engine that is still loading. The wait is bounded by the same wake timeout, +and a timeout still leaves the engine running. + +### Concurrent cold requests share one wake + +With the readiness fix, correctness already holds under concurrent cold +requests: the first starts the engine, the rest lose the 409, re-read, and +wait. But N requests would make N start attempts and N readiness polls against +one wake. The gateway keeps a single in-flight wake per (node, model) — a +mutex-guarded map — and joins it: the first request's wait is the wait the rest +take. At most one start per node, one poll loop, and every request in the +burst is answered from the same engine. + +### A short cache over the status fan-out + +Selecting fans out over every node, and the round waits on each producer with a +five-second per-node bound: one dead node would add up to five seconds to +*every* request an agent makes. The gateway therefore reuses a fan-out taken +within the last two seconds and re-runs the pure selection per request. The +ranking is on `idleSeconds`, which the fleet-harness-routing design already +calls a crude signal; a two-second-stale reading is unobservable, and the cache +also bounds how often wake decisions are made. The freshness is a package +variable so tests do not wait. A request that loses a race with a stopping node +fails with a connection error naming the node; the next request re-fans-out and +chooses elsewhere, which is the behaviour the no-retry rule gives. + +### The proxy: per-request, streaming, key-swapped + +Each routed request gets its own single-host reverse proxy at the resolved +engine address, with `FlushInterval` set so a streamed reply is flushed as the +engine sends it rather than in chunks. The proxy's client has no overall +timeout — a long generation is a long response — while the fan-out keeps the +fleet client's short one. The caller's `Authorization` is removed; the engine +key the gateway holds is set as the upstream's when the node reports its engine +gated, and nothing is set when it is not. The body and the reply pass through +unmodified, and a refused or failed upstream reply is the caller's reply: the +gateway does not retry at another node, because the engine's own error is the +honest one. + +### Auth and exposure mirror the daemon's rules + +One bearer token, the daemon's three sources (file, `SPINLOOP_API_TOKEN`, +command line), more than one a conflict, `401` without the right one, a +non-loopback listen without a token refused at startup, loopback without a +token allowed. The gateway is a longer-lived front door than the daemon's +control API and carries conversation content rather than control traffic, but +it sits on the fleet's own network, whose trust model is already +plain-HTTP-plus-bearer; TLS is an additive listener flag for later, not a +v1 gate. The rule is implemented in the gateway rather than imported from +`internal/daemon`, which keeps the daemon package a leaf the gateway depends +on rather than the other way round. + +### Status reports the served name beside the model id + +The daemon already stores the deploy config and reports its model id as the +served model; the served name is the same stored config's own field, so status +reports it when the config names one and omits it otherwise — one additive +`omitempty` field, no endpoint change. Older daemons and remotes omit it, and a +gateway matching a request then falls back to the model id alone, which is +exactly today's client-side behaviour. `docs/openapi.yaml` is updated to match; +`openapi_test.go` keeps the two in step. + +### The launch path's endpoint branch produces a `Choice` + +`routeThroughFleet` gains the branch its rejection used to occupy: a `FLEET` +with a scheme yields a `Choice` whose `BaseURL` is the named endpoint (with the +OpenAI-compatible prefix appended when the value carries no path) and a marker +that it names an endpoint rather than a node. The token is not resolved there: +`applyBeforeLaunch` already builds the resolver chain (environment, `.env` +beside the Spinloop, `ENV` instructions) and already special-cases a fleet +choice's key, so the endpoint's token is resolved through that same chain after +routing, a missing value fails before anything is written naming the variable, +and an already-set variable wins exactly as on the remote path. `fleet route` +gets the same branch as a report: the endpoint has already chosen, no node is +queried, nothing is started. + +### `wake:` sits in the fleet file beside `prefer:` + +Whether work may be started on the fleet's machines is a property of the fleet, +on the reasoning the `prefer` decision recorded: the same fleet shared by +several people and owned by one person wants different answers, and the fleet +file is the thing that differs. `wake: on|off`, default on (waking is today's +behaviour, so an existing fleet is unchanged), invalid values refused at parse +time. It decides *whether* to wake only: which node is chosen, the ranking, and +what a wake does are all untouched. The launch path's `--no-wake` still wins +over a file that allows waking — an explicit flag beats a file setting — and a +refused wake, by flag or by file, names the node that would have been woken and +the command that would start it. + +## Risks / Trade-offs + +- **A held request can outlive the caller's patience.** A cold wake is minutes + and the caller sees a slow response, with no progress to speak to. → It is + bounded by the wake timeout, the failure says the engine was left running, + and a retry is answered from the engine that is still loading. The + alternative — refuse and make the client retry — puts the burden on clients + that mostly will not retry. +- **A fleet whose nodes name no matching source cannot be woken through the + gateway**, even if a daemon could run the model from some other config. → + Deliberate: the gateway starts what the fleet file says a node runs, and the + failure names the node and the three ways a source could have been given. + The client-side wake, which carries its own Spinloop, is unaffected. +- **The two-second cache can pick a node that has just stopped.** → The proxy + fails with a connection error naming the node; the next request re-fans-out + and chooses elsewhere. No retry, by design. +- **Remotes in the fleet make cold wakes slow** — a scale-from-zero can exceed + the five-minute default. → Remotes wake through the same uniform `Node` + interface, and the timeout is a flag: a fleet with remotes sets a longer one + or declares `wake: off`. A wake that times out on a remote leaves the + instance starting, as the client-side path already does. +- **`servedName` is absent on older daemons and on remotes**, so aliases do not + match against them. → Matching falls back to the model id, which is today's + behaviour; the `no node is serving` failure lists each node's reported model, + which names the mismatch to anyone who knows the fleet. +- **The gateway holds every node's engine key in its environment.** A machine + compromise exposes all of them at once. → That is the trade the gateway is + bought for — the keys stop being distributed to every agent machine — and the + gateway is the one process the operator runs where the secrets already live. + +## Migration Plan + +Additive throughout. A fleet file with no `wake:` and a Spinloop with no +endpoint `FLEET` behave exactly as they do now; an older daemon that omits +`servedName` is matched on the model id; the wake race's readiness wait only +ever makes a launch wait longer, never shorter, and only in the race that +previously handed out a loading engine. Rollback is removing the `FLEET` URL +and stopping the gateway process; nothing else changes behaviour unless +something asks it to. + +## Open Questions + +None that block: TLS, budgets, per-user keys, and request logging beyond a line +per route are excluded by the proposal rather than deferred, and the wake +timeout default (five minutes, shared with the launch path) is a flag, not a +question. diff --git a/openspec/changes/add-fleet-gateway/proposal.md b/openspec/changes/add-fleet-gateway/proposal.md new file mode 100644 index 00000000..0372fc15 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/proposal.md @@ -0,0 +1,112 @@ +## Why + +A machine running an agent against a fleet today needs the fleet file, every +node's bearer token, and every node's engine key. A single +OpenAI-compatible endpoint in front of the fleet removes all of it: the agent +needs only a URL and one token, and the secrets stop at the gateway instead of +being distributed to every machine. The routing half was deliberately not built +when the fleet gained client-side selection (`fleet-harness-routing`, archived +2026-08-12) because it had nothing to stand on; the selector, the engine +endpoint reporting, and the wake path it assembles are all in place now, so the +gateway is mostly assembly. + +## What Changes + +- New `spinloop gateway` command: a foreground process that holds a fleet file + and serves one OpenAI-compatible endpoint for it — `/v1/models` (the union of + what the fleet's nodes are running) and reverse proxies for + `/v1/chat/completions` and `/v1/completions`, picking a node with the existing + fleet selector and streaming the reply through untouched. +- The gateway authenticates callers with one bearer token, the way the daemon + does (file, environment, or command line; a non-loopback listen without one + refuses to start), and holds each node's engine key itself, supplying it to a + node when it wakes one — the client-side rule that the starter supplies the + key now applies to the gateway as the starter. +- The gateway wakes a node when a request names a model nothing is serving, + holding the request while the engine loads, bounded by a wake timeout. A + fleet file's new top-level `wake:` setting (`on`/`off`, default `on`) + controls this; a model no node can serve fails fast, naming each node's + refusal. +- A `FLEET` value carrying a scheme — accepted at parse since + `fleet-harness-routing` and rejected as not implemented — now routes: a + launch with `FLEET http://gw:4000` points the agent at that gateway, with the + gateway's token taken from the client's environment. `spinloop fleet route` + answers such a Spinloop by saying the gateway has already chosen. +- Daemon status reports the name an engine serves its model under (the deploy + config's served name) beside the model id, so a request that names an alias + can be matched against what a node reports. +- A remote environment's status now carries what it is serving — the model id, + and the served name beside it — read from the environment's stored deploy + config, the same source the stats reply uses, so a gateway (or `fleet + status`) can match a request to a running remote node the way it matches a + local one and the fleet and remote views name it the same. Without this a + running remote node reported its state but no model, and was invisible to + model-based routing. +- A remote environment's status also carries where its engine answers — the + instance's published address the control plane reports, which a daemon on the + instance cannot know. Routing resolves it as the engine's host, so the gateway + reaches a running remote node instead of listing its model and then refusing + to route a request to it. +- Waking: a start refused because another client woke the node first is + re-read, and the winner of that race is used only once its engine answers — + not on state alone — so no caller, client or gateway, is handed an engine + that is still loading weights. +- New standalone example `examples/gateway-docker/`: a fleet plus a gateway in + containers, with a client that holds nothing but a URL and one token. + +## Capabilities + +### New Capabilities + +- `fleet-gateway`: the `spinloop gateway` command — its OpenAI-compatible + surface, caller authentication, node selection and proxying, wake behaviour + and its timeout, and what it deliberately is not. + +### Modified Capabilities + +- `spinloop-files`: the "FLEET names a file or an endpoint" requirement no + longer fails a URL as unimplemented; a `FLEET` naming an endpoint routes the + launch at that endpoint. +- `fleet-routing`: a launch whose `FLEET` names an endpoint is routed to that + gateway (base URL and key resolution); `fleet route` answers such a Spinloop; + waking's race rule now requires the engine to answer before a raced node is + used. +- `fleet-config`: a fleet file MAY declare a top-level `wake` setting + (`on`/`off`) deciding whether routing starts an engine on a node that is not + running one. +- `daemon-api`: status reports the served name of the model an engine runs, + beside the model id it already reports. +- `remote-node`: a running environment's status carries what it is serving + (model id and served name), read from the same stored deploy config the + stats reply reads, so the fleet and remote views name it the same. + +## Impact + +- `cmd/spinloop`: new `gateway.go` command; the launch path's route step and + `fleet route` gain the endpoint branch; completion and help cover the new + command. +- `internal/gateway` (new): the HTTP surface — auth, model listing, the + reverse proxy, the in-flight wake joining, and a short cache over the status + fan-out so a burst of requests does not fan out per request. +- `internal/fleet`: the wake race winner waits for the engine to answer; the + "usable now" test (running, matching, ready) is shared with the gateway; the + fleet file parses `wake:`. +- `internal/daemon`: status gains the served-name field, set from the stored + deploy config, and `EngineEndpoint` gains the host a node reports when it + knows its client-facing address (a remote environment does; a daemon never + will); `docs/openapi.yaml` updated to match (checked by `openapi_test.go`). +- `internal/fleet`: the remote node's status mapping carries the serving facts + the control plane now relays and the engine's published address as the + endpoint's host, and routing resolves a reported engine host in place of the + fleet file's; `internal/remote`: `Response` gains the served name. +- `remote/` (control plane): the start Lambda's status branch reads the + environment's stored deploy config for the serving facts (runner, model id, + served name) and the daemon for its activity. This changes what a deployed + start Lambda answers, so a control-plane redeploy is needed for live + environments to report a model; an environment deployed before the served-name + feature reports its model id but no served name until it is redeployed. +- `docs/`: command reference for `spinloop gateway` and the `FLEET` endpoint + form; `examples/gateway-docker/` is the runnable end-to-end demonstration. +- No daemon endpoint changes, no new dependencies, no Spinloop keyword, no + change to the launch path's ordering (route before apply). A fleet with no + `wake:` and a Spinloop with no `FLEET` URL behave exactly as they do now. diff --git a/openspec/changes/add-fleet-gateway/specs/daemon-api/spec.md b/openspec/changes/add-fleet-gateway/specs/daemon-api/spec.md new file mode 100644 index 00000000..b9f66e2a --- /dev/null +++ b/openspec/changes/add-fleet-gateway/specs/daemon-api/spec.md @@ -0,0 +1,24 @@ +## ADDED Requirements + +### Requirement: Status reports the name the model is served under + +Status SHALL report, beside the model id it already reports, the name the +engine serves the model under — the deploy config's served name — when the +stored config names one. A router that sees only status needs the name a +request will carry: a client that started the engine under an alias addresses +it by that alias, and matching a request only against the model id would not +recognise the node. The served name SHALL be omitted when the stored config +names none, and the model id SHALL be reported on the same terms it is today +whatever the served name is. + +#### Scenario: An aliased engine reports both names + +- **WHEN** an engine was started from a config that names both a model and a + served name, and a status request is made +- **THEN** the response reports the model id and the served name + +#### Scenario: No served name, no field + +- **WHEN** an engine was started from a config that names no served name, and a + status request is made +- **THEN** the response reports the model id and carries no served name diff --git a/openspec/changes/add-fleet-gateway/specs/fleet-config/spec.md b/openspec/changes/add-fleet-gateway/specs/fleet-config/spec.md new file mode 100644 index 00000000..b42e6cf7 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/specs/fleet-config/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Fleet-wide wake policy + +A fleet file MAY declare a top-level `wake` value of `on` or `off`, deciding +whether routing starts an engine on a node that is not running one when no +running node serves what is wanted. It belongs to the file rather than to each +node, for the reason `prefer` does: it describes how this cluster is to be +used — may work be started on its machines on demand, or only used where it is +already running — which is a property of the fleet, not of any one machine in +it. + +A file declaring nothing SHALL wake, as routing does when the setting is +absent: waking is the difference between a fleet that answers a request and +one that must be prepared by hand, and the file's author is the one who owns +the machines it names. A file declaring anything other than `on` or `off` SHALL +fail to parse, naming both accepted values, in keeping with the file's other +validation. + +The setting SHALL decide whether to wake only. It SHALL NOT change which node +is chosen, how matching nodes are ranked, or what a wake does: a fleet that +declares `wake: off` still reports, when nothing is running, the node whose +source describes the wanted model and the command that would start it. + +#### Scenario: A fleet that declares nothing wakes + +- **WHEN** a fleet file declares no `wake` setting and routing finds no node + serving what is wanted +- **THEN** routing starts an engine on a suitable node, as it does today + +#### Scenario: A fleet that refuses to wake + +- **WHEN** a fleet file declares `wake: off` and routing finds no node serving + what is wanted +- **THEN** nothing is started, and the failure names the node that would be + woken and the command that would start it + +#### Scenario: An unknown value is rejected at parse time + +- **WHEN** a fleet file declares `wake: sometimes` +- **THEN** parsing fails naming `on` and `off` diff --git a/openspec/changes/add-fleet-gateway/specs/fleet-gateway/spec.md b/openspec/changes/add-fleet-gateway/specs/fleet-gateway/spec.md new file mode 100644 index 00000000..4d6d942e --- /dev/null +++ b/openspec/changes/add-fleet-gateway/specs/fleet-gateway/spec.md @@ -0,0 +1,309 @@ +## Purpose + +One OpenAI-compatible endpoint in front of a fleet: a foreground process that +answers agent requests by choosing a node with the fleet's own selector, holds +each node's engine key, and wakes a node when nothing is serving what a request +asks for — so a machine running an agent needs nothing but a URL and one token. + +## ADDED Requirements + +### Requirement: The gateway command + +`spinloop gateway` SHALL run in the foreground, the way `spinloop serve` does, +holding the fleet file it serves: `--fleet ` (with the `-f ` short +form) when given, otherwise `./fleet.yaml` in the working directory, and a +missing file SHALL fail naming the expected path, as the fleet commands do. +The gateway SHALL listen on an address given by `--listen`, defaulting to +port 4000 on all interfaces, and SHALL print the address a Spinloop should name +in its `FLEET` instruction when it starts. + +The gateway's own startup failures SHALL be the fleet file's: a fleet file that +does not parse, or that names a token variable set nowhere, SHALL fail the +gateway at startup naming the problem, rather than listening and failing per +request. + +#### Scenario: The gateway starts and answers + +- **WHEN** the user runs `spinloop gateway --listen :4000` in a directory + holding a `fleet.yaml` +- **THEN** it prints the address to name in a Spinloop's `FLEET` and serves + requests until it is stopped + +#### Scenario: An explicit fleet file is served + +- **WHEN** the gateway is given a `--fleet` path +- **THEN** that file is the one it serves + +#### Scenario: A missing fleet file names itself + +- **WHEN** the user runs `spinloop gateway` in a directory holding no + `fleet.yaml` and passes no `--fleet` +- **THEN** it fails naming `./fleet.yaml`, and nothing listens + +#### Scenario: A broken fleet file fails at startup + +- **WHEN** the gateway's fleet file names a token variable that is set nowhere +- **THEN** the gateway fails at startup naming the node and the variable, + rather than listening + +### Requirement: Caller authentication + +The gateway SHALL authenticate callers with one bearer token, on the same +terms the daemon's control API authenticates: the token MAY be supplied by a +file (`--api-token-file`), the environment (`SPINLOOP_API_TOKEN`), or the +command line (`--api-token`); giving more than one SHALL fail naming the +conflict. Requests without the correct token SHALL be rejected with `401`. +When no token is configured, the gateway SHALL refuse to listen on a +non-loopback address and SHALL say why, and listening on loopback without a +token SHALL be allowed. + +The caller's token authorises use of the gateway only. It SHALL NOT be +forwarded to a node's engine or control API: the gateway reaches a node with +the node's own credentials, resolved from the fleet file the way every other +fleet client does. + +#### Scenario: A wrong token is rejected + +- **WHEN** a request carries a missing or incorrect bearer token +- **THEN** the response is `401` and no node is contacted + +#### Scenario: A tokenless non-loopback listen refuses to start + +- **WHEN** the gateway would listen on a non-loopback address and no token is + configured +- **THEN** startup fails saying a token is required for non-loopback exposure, + naming every way one can be supplied + +#### Scenario: A tokenless loopback is permitted + +- **WHEN** the gateway listens on a loopback address with no token configured +- **THEN** it serves requests without authentication + +#### Scenario: The caller's token stops at the gateway + +- **WHEN** an authenticated request is routed to a node +- **THEN** the node is reached with the node's own credentials from the fleet + file, and the caller's token is not sent to it + +### Requirement: Listing the fleet's models + +The gateway SHALL serve `GET /v1/models` returning, in the OpenAI list shape, +the union of the models the fleet's nodes are running: for each node whose +state is `running`, the name it reports serving — the served name when it +reports one, otherwise the model id — with duplicates listed once. A node that +is not running contributes nothing, however many models it could serve after a +wake: the list is what answers now, and requesting something else is the wake's +concern. + +#### Scenario: Running models are listed + +- **WHEN** two nodes are running, one serving a model under an alias and one + under its id, and a models request is made +- **THEN** the response lists the alias and the id, each once + +#### Scenario: A stopped node's model is not listed + +- **WHEN** a node is stopped after serving a model, and a models request is + made +- **THEN** the response lists only what the running nodes serve + +#### Scenario: Nothing running lists nothing + +- **WHEN** no node is running and a models request is made +- **THEN** the response is an empty list, not an error + +### Requirement: Routing a request to a node + +The gateway SHALL serve `POST /v1/chat/completions` and `POST /v1/completions` +by choosing a node and reverse-proxying the request to that node's engine. The +choice SHALL be the fleet's own selection: every node's state is considered, a +node matches when what it reports serving — its model id or its served name — +equals the model the request names, and matching nodes are ranked by the fleet +file's activity preference, ties broken by fleet-file order. A request naming +no model SHALL be refused saying so, not routed at a guess. + +The request's body SHALL reach the chosen engine unmodified, and the engine's +reply SHALL reach the caller unmodified, including a streamed reply, which the +gateway SHALL pass through without holding it back. The gateway SHALL replace +the caller's authorisation with the engine key it holds for that node — the +value the node's fleet entry names, resolved the way every other fleet client +resolves it — and SHALL send no authorisation at all to an engine that needs +none. The gateway SHALL NOT retry a failed request at another node: the reply +the chosen engine gives is the reply the caller gets. + +The engine's address SHALL be resolved the way routing resolves it: the node's +declared engine override as given, otherwise the node's host with the port and +path the engine reports. Where a node reports its engine's host — a remote +environment, whose control plane publishes the instance's address and which the +fleet file names by environment alone — that reported host SHALL be used in +place of the node's host, so the request reaches the instance rather than an +address the fleet file never held. A node that reports its engine bound to +loopback, without an override taking responsibility for reachability, SHALL NOT +be selected: it answers only on its own machine, and where no other node serves +the wanted model the request SHALL fail saying so and naming the fix. + +The gateway SHALL reuse node state it has recently read — a reading taken +within the last couple of seconds — rather than query every node on every +request, so a burst of requests does not pay a fan-out each. The freshness of +the reading SHALL NOT matter to the choice: the same fleet in the same state +chooses the same node. + +Each routed request SHALL be logged as one line — the model, the node chosen, +and the outcome — and the gateway SHALL log no request body. + +#### Scenario: A request goes to the node serving its model + +- **WHEN** one node is running the model a request names and another is + running a different model +- **THEN** the request is proxied to the first node and the engine's reply + reaches the caller unmodified + +#### Scenario: The engine key is swapped, not the caller's token + +- **WHEN** a request is routed to a node whose engine is gated, and the node's + fleet entry names the key +- **THEN** the engine receives the key's value as its authorisation, and the + caller's token is not sent to the engine + +#### Scenario: A streamed reply passes through + +- **WHEN** a streamed completion is requested and the chosen engine streams its + reply +- **THEN** the caller receives the stream as the engine sent it + +#### Scenario: An engine error is the caller's error + +- **WHEN** the chosen engine refuses the request +- **THEN** the engine's refusal reaches the caller and no other node is tried + +#### Scenario: A request naming no model is refused + +- **WHEN** a completion request carries no model +- **THEN** it is refused saying the request names no model, and no node is + contacted + +#### Scenario: A loopback-bound engine is not selected + +- **WHEN** the only node serving the wanted model reports its engine bound to + loopback and names no engine override, and the node is not reached over + loopback +- **THEN** the request fails saying the engine answers only on that machine, + naming the bind and the override as the fixes + +#### Scenario: A burst of requests does not fan out per request + +- **WHEN** several requests arrive within a couple of seconds of each other +- **THEN** the fleet is queried once for the burst, and the requests are + answered from that reading + +#### Scenario: A routed request leaves one log line + +- **WHEN** a request is routed and answered +- **THEN** the gateway's log gains one line naming the model, the node, and the + outcome, and carries no part of the request body + +### Requirement: Waking a node for a request + +When no running node serves the model a request names, and the fleet file's +wake policy allows it, the gateway SHALL start an engine on a node that is not +running one, and SHALL hold the request until the engine answers. A node is a +wake candidate when it is not running and the Spinloop source it names — its +`file` field, a registered alias named after it, or a same-named directory +beside the fleet file, resolved the way `spinloop fleet start` resolves it — +describes a config whose model or served name is the one the request asks for: +a node is started with what it was told to run, never with a config invented +for the request. Candidates whose stored config already names the model SHALL +be tried first, since they have the weights, and the rest in fleet-file order. +A node that refuses the start — a runner or model it cannot serve — SHALL NOT +fail the request while other candidates remain. + +The started engine SHALL be gated with the key the node's fleet entry names, +supplied by the gateway: the gateway is the client that starts the engine, so +the key the client sets is the key the engine takes. The wait SHALL be bounded +by a wake timeout, defaulting to five minutes and overridable by +`--wake-timeout`; exceeding it SHALL fail the request saying the engine did +not answer in time, and the started engine SHALL be left running rather than +stopped, so a slow load is not thrown away. + +When several requests ask for a model nothing is serving at once, the gateway +SHALL start at most one engine per node and answer every request from it: the +first request's wait is the wait the rest join. A node another request woke +first SHALL be used the same way, and only once its engine answers. + +With the wake policy off, a request for a model nothing is serving SHALL fail +without starting anything, naming the nodes and what they could serve, and the +command that would start one. A model no node is running and no node's source +describes SHALL fail the same way, whatever the policy: nothing to wake with, +and the failure SHALL say so rather than trying to start a node with nothing. + +#### Scenario: A cold request wakes a node and is served + +- **WHEN** no node is running the model a request names, one node's Spinloop + source describes it, and the wake policy allows it +- **THEN** that node is started with its own config, gated with the key its + fleet entry names, and the request is answered once the engine answers + +#### Scenario: The request is held while the engine loads + +- **WHEN** the woken node reports running while its engine is still loading +- **THEN** the request waits, and is answered when the engine answers, rather + than failing against an endpoint that refuses connections + +#### Scenario: A wake that does not finish in time fails the request + +- **WHEN** a woken node's engine does not answer within the wake timeout +- **THEN** the request fails saying so, naming the node, and the engine is left + running + +#### Scenario: Concurrent cold requests share one wake + +- **WHEN** two requests arrive at once for a model nothing is serving, and one + node's source describes it +- **THEN** that node is started once, and both requests are answered from the + same engine + +#### Scenario: A woken engine takes the gateway's key + +- **WHEN** the gateway starts an engine on a node whose fleet entry names an + engine key +- **THEN** the engine is gated with that value, the gateway's requests to it + carry it, and no reply to any caller contains it + +#### Scenario: Wake refused by the fleet file + +- **WHEN** the fleet file declares the wake policy off and no node is serving + the model a request names +- **THEN** nothing is started, and the request fails naming the node whose + source describes the model and the command that would start it + +#### Scenario: Nothing can serve the model + +- **WHEN** no node is running the model a request names and no node's Spinloop + source describes it +- **THEN** the request fails, naming each node and why it cannot serve the + model, and nothing is started + +#### Scenario: A sourceless node is not woken + +- **WHEN** the only node that could take a request names no Spinloop source + that resolves +- **THEN** it is not started, and the failure names it and the ways a source + could have been given + +### Requirement: Paths the gateway does not serve + +A path other than `/v1/models`, `/v1/chat/completions`, `/v1/completions`, and +the gateway's own health path SHALL be answered with `404` naming the paths the +gateway serves. The health path SHALL answer that the gateway is up without +contacting any node, so an operator can tell the gateway down from the fleet +down. + +#### Scenario: An unknown path is named as such + +- **WHEN** a request is made to a path the gateway does not serve +- **THEN** the response is `404` and names the paths it does serve + +#### Scenario: The health path does not touch the fleet + +- **WHEN** the health path is requested and every node is unreachable +- **THEN** it still answers that the gateway is up diff --git a/openspec/changes/add-fleet-gateway/specs/fleet-routing/spec.md b/openspec/changes/add-fleet-gateway/specs/fleet-routing/spec.md new file mode 100644 index 00000000..54440aa3 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/specs/fleet-routing/spec.md @@ -0,0 +1,175 @@ +## ADDED Requirements + +### Requirement: A FLEET naming an endpoint routes the launch at it + +A launch whose `FLEET` — instruction or `--fleet`/`-f` flag — names an +endpoint, a value carrying a scheme, SHALL NOT read a fleet file and SHALL NOT +contact any node: the endpoint has already done the choosing. The endpoint's +address SHALL be written as the applied provider's base URL, in the same place +a `REMOTE` endpoint's address is written, and SHALL be placed in the launched +agent's environment as `OPENAI_BASE_URL`. An endpoint value with no path SHALL +be given the OpenAI-compatible prefix, so `FLEET http://gw:4000` points the +agent at `http://gw:4000/v1`, and a value that already carries a path SHALL be +used as given. + +The key the agent is given SHALL be the endpoint's token, resolved from the +client's environment the way a key is resolved elsewhere — the process +environment first, then a `.env` beside the Spinloop — with a variable already +set in spinloop's environment winning. When the variable is set nowhere, the +launch SHALL fail before the agent launches, naming the variable to set: a +gateway that cannot be authenticated is not one to point an agent at. + +Flags that steer node selection — `--node`, `--prefer`, `--no-wake` — SHALL be +inert when the `FLEET` names an endpoint: there is no node to steer. A pinned +`BASEURL` SHALL still win over an endpoint `FLEET`, as it wins over a fleet +file. + +#### Scenario: A launch is pointed at the gateway + +- **WHEN** the user runs `spinloop harness` with a Spinloop containing + `FLEET http://gw.internal:4000`, and the gateway's token is set in the + environment +- **THEN** the launched agent's environment carries `OPENAI_BASE_URL` of + `http://gw.internal:4000/v1` and the token as its API key, and the applied + provider's base URL is the same address + +#### Scenario: A gateway URL with a path is used as given + +- **WHEN** a Spinloop contains `FLEET http://gw.internal:4000/proxy/v1` +- **THEN** the launched agent's base URL is that address, unchanged + +#### Scenario: No fleet file is read, no node contacted + +- **WHEN** a launch with an endpoint `FLEET` runs in a directory holding no + fleet file +- **THEN** the launch proceeds: the endpoint is not a file to resolve + +#### Scenario: A missing gateway token fails early + +- **WHEN** a launch with an endpoint `FLEET` runs and no variable holding the + gateway's token is set anywhere +- **THEN** the launch fails before the agent launches, naming the variable to + set, and the harness config is not written + +#### Scenario: An exported token wins + +- **WHEN** the variable the gateway's key is resolved under is already set in + the user's environment and a launch with an endpoint `FLEET` runs +- **THEN** the existing value reaches the agent unchanged + +### Requirement: fleet route answers a FLEET naming an endpoint + +`spinloop fleet route` on a Spinloop whose `FLEET` names an endpoint SHALL +report that the endpoint has already chosen: it SHALL name the address the +launch will be given, SHALL NOT query any node, and SHALL NOT wake one. + +#### Scenario: A route against an endpoint names it + +- **WHEN** the user runs `spinloop fleet route` on a Spinloop containing + `FLEET http://gw.internal:4000` +- **THEN** the output names that address as the one the launch will use, no + node is queried, and nothing is started + +## MODIFIED Requirements + +### Requirement: Waking a node + +When no running node is serving what is wanted, routing SHALL wake one: it SHALL +choose a node that is not running, push what the Spinloop asks for as that node's +deploy config, start it through the daemon's start endpoint, and wait before +launching the agent — not merely until the node reports `running`, which says +only that a process exists, but until its engine endpoint answers. A node whose +stored config already matches the wanted model SHALL be preferred, since it has +the weights. + +The pushed config is the node-side counterpart of what `spinloop serve` would run +for that Spinloop, translated per engine. A node may be woken for an engine that +binds its model at launch — `llamacpp`, `vllm`, and `mtplx` — and a `MODEL` that +names a file on the node's own disk is a valid wake for it: the node has the +file, and only a destination that fetches its weights itself refuses a local +path. + +A node that refuses the config — a runner or model it cannot serve — SHALL NOT +fail the launch while other candidates remain: the next candidate SHALL be +tried, and the refusals SHALL be reported when none succeeds. + +Two clients may wake the same node at once. A start refused because an engine is +already running SHALL NOT fail the launch: the node's state SHALL be re-read, +and a node now serving what was wanted SHALL be used — and the launch SHALL +wait for that node's engine to answer before launching the agent, exactly as it +waits for a node it woken itself: the node that won the race may still be +loading weights, and the wait is bounded by the same timeout. Losing that race +is another route to the same place, not an error. + +The wait SHALL be bounded by a timeout and SHALL report what it is waiting for, +because a cold node loads weights before it answers. Exceeding the timeout SHALL +fail naming the node and the endpoint that did not come up; the started engine +SHALL be left running rather than stopped, so a slow load is not thrown away. + +`--no-wake` SHALL turn waking off: with no running node serving what is wanted +the command SHALL then fail, listing the nodes and their states and naming the +command that would start one. + +#### Scenario: An idle node is woken and used + +- **WHEN** a fleet-routed launch finds no node serving the wanted model and one + node is idle and able to serve it +- **THEN** that node is given the Spinloop's model as its deploy config, started, + and the agent launches against it once its engine answers + +#### Scenario: A node is woken for a Mac-only engine + +- **WHEN** a fleet-routed launch finds no node serving the wanted model, and an + idle node's daemon can run MTPLX +- **THEN** that node is woken with a config that runs the wanted model under + `mtplx serve`, and the agent launches against it once its engine answers + +#### Scenario: A local model path wakes the node that has it + +- **WHEN** the Spinloop's `MODEL` names a file on the woken node's disk +- **THEN** the wake carries that path as the model to load, rather than + refusing it as a local file + +#### Scenario: A Spinloop that pins a bind wakes a node bound to it + +- **WHEN** the Spinloop names a `BASEURL` and a node is woken for it +- **THEN** the engine the node starts binds to the address the `BASEURL` names, + exactly as `spinloop serve` would bind it, and the node reports that engine + as reachable rather than on the engine's own default + +#### Scenario: A started engine that is not yet loaded is waited for + +- **WHEN** a woken node reports `running` while its engine is still loading + weights and not yet answering +- **THEN** the launch waits for the engine to answer rather than launching the + agent against an endpoint that refuses connections + +#### Scenario: A node that cannot serve the model is passed over + +- **WHEN** the first idle candidate rejects the pushed config as unservable and + a second idle node accepts it +- **THEN** the second node is started and used + +#### Scenario: No node can serve it + +- **WHEN** every idle node rejects the config +- **THEN** the command fails, naming each node and the reason it refused + +#### Scenario: Losing the race to another client + +- **WHEN** a start is refused because another client woke the same node first, + and that node is now serving the wanted model +- **THEN** the launch uses that node rather than failing, waiting for its + engine to answer first if it is still loading + +#### Scenario: A node that never comes up + +- **WHEN** a woken node does not report running within the timeout +- **THEN** the command fails naming the node, and the engine it started is left + running rather than stopped + +#### Scenario: Waking is refused + +- **WHEN** `--no-wake` is passed and no node is serving the wanted model +- **THEN** the command fails, listing the nodes with their states and naming the + `spinloop fleet start` command that would start one, and nothing is started diff --git a/openspec/changes/add-fleet-gateway/specs/remote-node/spec.md b/openspec/changes/add-fleet-gateway/specs/remote-node/spec.md new file mode 100644 index 00000000..f42a6445 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/specs/remote-node/spec.md @@ -0,0 +1,59 @@ +## MODIFIED Requirements + +### Requirement: A remote environment is a fleet node + +A registered remote environment SHALL be representable as one member of the fleet's node +set, answering the same operations a local node answers: its status, its metrics, and +being started, stopped, and read for logs. The control plane's replies SHALL be mapped +onto the same status and metrics shapes a local node yields, so downstream fan-out and +rendering treat the two identically. A running environment's status SHALL in particular +carry what its engine is serving — the model it runs, and the served name the deploy gave +it beside the model id when there is one — so a client choosing a node by model matches +it the way it matches a local node, and the fleet view and the remote view name it the +same. It SHALL also carry where its engine answers — the instance's published address, +which the control plane knows and a daemon on the instance cannot — so a client can +reach the engine, not only name it; a stopped or undeployed environment reports none. + +A remote environment that cannot be reached, or whose control call is rejected — +including a rejected AWS credential — SHALL be reported as a typed outcome against that +environment, the same way an unreachable or unauthorized node is, rather than failing the +command or being silently dropped. + +Because a remote endpoint is provisioned by deployment rather than woken like a node, a +node-level start asked to run on a supplied deploy configuration SHALL be refused with a +message naming the deployment path, rather than attempted. + +#### Scenario: A remote environment answers status like a node + +- **WHEN** a remote environment is asked for its status as a member of a node set +- **THEN** it returns a status carrying the endpoint's state, what its engine is serving + (the model, and the served name beside it when the deploy gave one), where its engine + answers (the instance's published address), and, when the engine has done work, its + last-active time, in the same shape a local node's status carries + +#### Scenario: A freshly loaded engine shows its model before it has done work + +- **WHEN** a remote environment's engine is serving a model but has not yet answered a + request, so it reports no last-active time +- **THEN** its status still carries the model it is serving, so a router can match a + request to it before the first request has landed + +#### Scenario: A remote environment answers metrics like a node + +- **WHEN** a running remote environment is asked for its metrics as a member of a node set +- **THEN** it returns the token and system figures in the same stats shape a local node + returns + +#### Scenario: A rejected control call is a typed outcome + +- **WHEN** a remote environment's status or metrics call is rejected, for example because + the caller's credentials are not valid +- **THEN** the environment is reported with a failure outcome and the reason, and it does + not abort or blank the rest of the node set + +#### Scenario: Waking a remote environment is refused + +- **WHEN** a node-level start is requested for a remote environment, carrying a deploy + configuration +- **THEN** it is refused with a message naming the deployment path, and the environment + is not started diff --git a/openspec/changes/add-fleet-gateway/specs/spinloop-files/spec.md b/openspec/changes/add-fleet-gateway/specs/spinloop-files/spec.md new file mode 100644 index 00000000..127a5ed3 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/specs/spinloop-files/spec.md @@ -0,0 +1,25 @@ +## MODIFIED Requirements + +### Requirement: FLEET names a file or an endpoint + +A `FLEET` value SHALL be either a path to a fleet file, which routing reads to +choose a node, or a URL, which names an endpoint that has already done the +choosing. A value carrying a scheme SHALL be read as the latter; anything else +as a path. Both SHALL parse, so the two ways of routing are one instruction +rather than two. + +A launch against a `FLEET` naming an endpoint SHALL route the agent at that +endpoint rather than fail as unimplemented: the endpoint is the agent's +OpenAI-compatible address, and the launch resolves the key to reach it with +from the client's own environment, as the remote path resolves a key. + +#### Scenario: A path names a fleet file + +- **WHEN** a Spinloop contains `FLEET ./fleet.yaml` +- **THEN** it parses as a fleet file to route through + +#### Scenario: A URL names an endpoint + +- **WHEN** a Spinloop contains `FLEET http://gateway.internal:4000` +- **THEN** it parses as an endpoint, and a launch against it points the agent + at that endpoint diff --git a/openspec/changes/add-fleet-gateway/tasks.md b/openspec/changes/add-fleet-gateway/tasks.md new file mode 100644 index 00000000..26a587c6 --- /dev/null +++ b/openspec/changes/add-fleet-gateway/tasks.md @@ -0,0 +1,41 @@ +# Tasks + +## 1. Fleet foundations + +- [x] 1.1 Parse a top-level `wake` setting in fleet.yaml (`on`/`off`, default on when absent, any other value refused naming both) with a `fleet.Config` accessor for the policy in force, and verify with `internal/fleet` config tests: a file with `wake: off` reports off, a bare file reports on, and `wake: sometimes` fails naming `on` and `off` +- [x] 1.2 Generalise `fleet.Wake` from one deploy config for all candidates to a per-candidate config function, keeping the existing warm-first ordering, refusal collection, race handling, and readiness wait, and verify the existing `wake_test.go` suite passes with the launch path's constant config plus a new test where two candidates take two different configs +- [x] 1.3 Gate every use of a woken or raced node on its engine answering: lift the readiness wait out of `Wake`'s private helper into an exported form both call, make `Wake`'s already-running branch wait for the engine to answer rather than taking the node on state alone, and add a "usable now" test of running + name match (model id or served name) + ready-or-answering, verified by a wake test where a node that won the race is still loading is waited for, not returned +- [x] 1.4 Make daemon status report the served name beside the model id: the stored deploy config's served name is recorded with the served model and reported as an omitted-unless-set `servedName` field, and `docs/openapi.yaml` is updated to match, verified by a daemon test (an aliased start reports both names, an unaliased one reports no field) and `openapi_test.go` passing + +## 2. The gateway + +- [x] 2.1 Build the `internal/gateway` handler: constructed from a `*fleet.Config`, a caller token, a wake timeout, and a per-node config callback, it authenticates with the daemon's token rules (file, `SPINLOOP_API_TOKEN`, command line; more than one a conflict; `401` without the right one; a non-loopback listen without a token refuses to start, loopback allowed), answers `GET /health` without touching any node, and refuses unknown paths with `404` naming the paths served, verified by handler tests against each rule +- [x] 2.2 Serve `GET /v1/models` as the OpenAI list of what the fleet's nodes are running — the served name when a running node reports one, otherwise the model id, duplicates once, nothing for a stopped node, an empty list not an error when nothing runs — verified by handler tests over a fake fleet in each state +- [x] 2.3 Reuse a status fan-out taken within the last two seconds rather than fanning out per request, the freshness a package variable tests can shorten, verified by a test counting fan-outs: a burst inside the window fans out once, a request after it fans out again +- [x] 2.4 Route `POST /v1/chat/completions` and `POST /v1/completions`: read the model from the body (a request naming none is refused saying so), select with the fleet's own ranking (name match on model id or served name, the fleet file's preference, fleet-file tie-break, a loopback-bound engine without an override never selected and named in the failure when it is the only candidate), wait for the chosen node to be usable, then reverse-proxy per request with streaming flushed as sent, the caller's authorisation swapped for the engine key the fleet entry names (none sent to an ungated engine), no read timeout on the upstream, no retry at another node, an upstream failure reaching the caller as an error naming the node, and one log line per routed request naming model, node, and outcome with no request body, verified by handler tests over fake engines: right node by model, key swapped not forwarded, a streamed reply passes through, an engine error is the caller's error, and the log line appears +- [x] 2.5 Wake for a request: when no running node matches and the fleet's wake policy allows it, start a node with the config its own Spinloop source resolves to (only nodes whose source names the requested model or served name are candidates, a node whose stored config already matches tried first), gate it with the key its fleet entry names, hold the request until the engine answers bounded by the wake timeout (a timeout failing the request saying so and leaving the engine running), share one in-flight wake per node between concurrent requests so a burst starts at most one engine, and with the policy off — or no node's source describing the model — fail without starting anything, naming the nodes and the ways a source could have been given, verified by handler tests: a cold request is woken and answered, concurrent cold requests produce one start, a timeout leaves the engine running, `wake: off` refuses naming the node, and a sourceless node is named in the failure + +## 3. The CLI + +- [x] 3.1 Add the `spinloop gateway` command beside `serve`: `--fleet`/`-f`, `--listen` defaulting to port 4000 on all interfaces, the daemon's three token sources, `--wake-timeout`; it resolves the fleet file and token at startup (failures naming the fix, nothing listening on failure), prints the address to name in a Spinloop's `FLEET`, and serves until stopped, verified by a command test (a gateway answers `/v1/models` on the printed address, a non-loopback listen without a token refuses naming the ways to supply one, a missing fleet file names `./fleet.yaml`) and by the completion dispatch-coverage scan passing with the new command +- [x] 3.2 Implement the launch path's endpoint branch: a `FLEET` — instruction or flag — carrying a scheme yields a choice naming the endpoint (the OpenAI-compatible prefix appended when the value has no path, a value with a path used as given), no fleet file read and no node contacted, the endpoint's token resolved through the launch's existing key chain (an ENV instruction, then the environment, then the `.env` beside the Spinloop, an already-set variable winning) with a missing value failing before anything is written and naming `OPENAI_API_KEY`, node-steering flags inert, and a pinned `BASEURL` still winning, verified by launch tests: a Spinloop with `FLEET http://gw:4000` gives the agent `OPENAI_BASE_URL` of `http://gw:4000/v1` and the token as its key, a Spinloop with a path in the URL is used as given, a missing token fails naming the variable and writes nothing, and a pinned `BASEURL` is not routed +- [x] 3.3 Make `spinloop fleet route` answer a `FLEET` that names an endpoint: it reports the address the launch will be given, queries no node, and starts nothing, replacing its "not implemented yet" refusal, verified by a command test on a Spinloop with an endpoint `FLEET` in a directory holding no fleet file +- [x] 3.4 Gate the launch path's waking on the fleet's `wake` setting: `wake: off` refuses to start a node the way `--no-wake` does — the failure names the node that would be woken and the `spinloop fleet start` command that would start it — and an explicit `--no-wake` still wins over a file that allows waking, verified by launch tests for each of the three combinations + +## 4. Example and documentation + +- [x] 4.1 Add a standalone `examples/gateway-docker/` — its own Dockerfile, shim, engine files, `compose.yaml` (two fleet nodes plus a gateway service, no reference to `examples/fleet-docker/`), a `fleet.yaml` whose nodes name their tokens and engine keys by variable, a `.env.example`, a client Spinloop whose `FLEET` is the gateway's address, a `run-tests.sh` asserting the gateway's surface (a model is listed once running, a cold request wakes a node and streams a reply, the engine key is injected and never reaches the caller, a wrong token is `401`, `wake: off` refuses), and a `README.md` with the run steps, verified by `./run-tests.sh` passing from a clean checkout +- [x] 4.2 Document the change where the rest of it is documented: `docs/commands/gateway.md` for the new command (flags, the token's three sources and the non-loopback rule, the address it prints), the endpoint form of `FLEET` and the token it resolves in `docs/spinloop-file.md`, the `wake` setting in the fleet file documentation, and `docs/README.md` pointing at the new example, verified by the docs links resolving and the new pages reading against the implemented flags + +## 5. Remote nodes report what they serve + +A running remote node reported its state but no model — the control plane's status branch read the on-instance daemon's status but relayed only its activity, dropping the serving facts. A gateway (or `fleet status`) matching a request by model could not see a running remote node. + +- [x] 5.1 Make the start Lambda's status branch read the environment's stored deploy config for the serving facts: `readDeployFacts` returns the runner, model id and served name from the same config the stats reply reads (the daemon's own model is the on-disk weights path, which a router cannot match, so it is never reported), read concurrently with the daemon's activity and never turning a working status into a failing one, verified by the start-status tests (facts reported with and without activity, a model present before the engine has done work, an absent served name stays absent, a failed config read drops only the facts) +- [x] 5.2 Map the relayed serving facts onto the node's status: `remote.Response` gains the served name, and `statusFromRemote` carries runner, model and served name, leaving them empty when the daemon reports none, verified by a remote-node status test (facts mapped across, and an absent fact stays empty) +- [x] 5.3 Note in the proposal and the `remote-node` spec that a running environment's status carries what its engine is serving, and that a control-plane redeploy is needed for live environments to report it +- [x] 5.4 Carry the engine's address on a remote node's status and resolve it in routing: `EngineEndpoint` gains the host a node reports, `statusFromRemote` maps the control plane's published base url to the endpoint's host, port and path (none when the environment is not running), `EngineBaseURL` uses a reported engine host in place of the fleet file's, and `docs/openapi.yaml` is updated to match, verified by the remote-node status tests (the address maps across, absent when stopped), the `EngineBaseURL` tests (a reported host is used, an override still wins), a gateway test (a remote node is a candidate and a request is routed to its published address with the resolved engine key), and `openapi_test.go` passing + +## 6. Final verification + +- [x] 6.1 Run the full gate: `go test ./... -cover` at or above the 80% total, `go vet ./...`, `gofmt -l .` clean, the control plane's `pnpm build` and `pnpm test`, and `openspec validate add-fleet-gateway` passing, fixing whatever each reports diff --git a/remote/lambda/shared/daemon.ts b/remote/lambda/shared/daemon.ts index 3f3a72de..03844e3b 100644 --- a/remote/lambda/shared/daemon.ts +++ b/remote/lambda/shared/daemon.ts @@ -80,6 +80,11 @@ export interface DaemonStatus { state: string; runner?: string; model?: string; + /** + * The name the engine answers to beside the model id — the served name the + * deploy named it. Absent when the deploy gave the engine no other name. + */ + servedName?: string; uptimeSeconds?: number; logPath?: string; lastActiveAt?: string; diff --git a/remote/lambda/start/index.ts b/remote/lambda/start/index.ts index 21cb4ec6..2e9716a2 100644 --- a/remote/lambda/start/index.ts +++ b/remote/lambda/start/index.ts @@ -158,17 +158,20 @@ async function status(env: string): Promise { }); } // Concurrently, not in sequence: status is what you type repeatedly while - // waiting for a box, and this branch should cost the slower of the two SSM - // calls rather than their sum. - const [healthy, activity] = await Promise.all([ + // waiting for a box, and this branch should cost the slowest of these calls + // rather than their sum. The model facts come from the deploy config — the + // same source the stats reply reads — and the activity from the daemon. + const [healthy, activity, deploy] = await Promise.all([ checkHealth(instance.instanceId), readDaemonActivity(instance.instanceId), + readDeployFacts(env), ]); const result: Record = { state: 'running', environment: env, healthy, base_url: baseUrl, + ...deploy, ...activity, }; if (instance.retainUntil) { @@ -178,12 +181,12 @@ async function status(env: string): Promise { } /** - * Ask the instance's daemon when its engine last did work. Every failure — - * SSM error, unreachable daemon, unparseable reply, an engine that has done - * nothing yet — yields an empty object, so the caller spreads nothing and the - * report is exactly what it would have been. This must never be able to turn - * a working status into a failing one, which is why it is kept out of the - * `healthy` expression. + * Ask the instance's daemon when its engine last did work. Every failure — an + * SSM error, an unreachable daemon, an unparseable reply, an engine that has + * not done any work yet — yields an empty object, so the caller spreads + * nothing and the report is exactly what it would have been without this. It + * is kept out of the `healthy` expression so it can never turn a working + * status into a failing one. */ async function readDaemonActivity( instanceId: string, @@ -206,6 +209,40 @@ async function readDaemonActivity( } } +/** + * Read the environment's stored deploy config and keep the facts that name + * what it is serving: the runner, the model id, and the served name where the + * deploy gave one. This is the same source the stats reply — and so the remote + * status view — reads, so the fleet and remote views name a model identically. + * The daemon's own model is the on-disk weights path, which a router cannot + * match a request against, so the path is never reported here. A failed read + * yields an empty object, so an environment whose config cannot be read still + * reports its state and activity without a made-up model. + */ +async function readDeployFacts(env: string): Promise<{ + runner?: string; + modelId?: string; + servedName?: string; +}> { + try { + const cfg = await readDeployConfig(deployConfigParam(env)); + const facts: { runner?: string; modelId?: string; servedName?: string } = {}; + if (cfg.runner) { + facts.runner = cfg.runner; + } + if (cfg.modelId) { + facts.modelId = cfg.modelId; + } + if (cfg.servedModelName) { + facts.servedName = cfg.servedModelName; + } + return facts; + } catch (err) { + console.log(JSON.stringify({ phase: 'deploy-facts', error: errorName(err) })); + return {}; + } +} + /** POST — launch the environment's instance if needed and block until serving. */ async function wake( env: string, diff --git a/remote/test/start-status.test.ts b/remote/test/start-status.test.ts index 3f8f527b..3706a155 100644 --- a/remote/test/start-status.test.ts +++ b/remote/test/start-status.test.ts @@ -28,6 +28,7 @@ const LAMBDA_ENV = { const findManagedInstance = vi.fn(); const isSsmAgentOnline = vi.fn(); const runShellCommand = vi.fn(); +const readDeployConfig = vi.fn(); const findEnvEip = vi.fn(); vi.mock('../lambda/shared/aws', async (importOriginal) => ({ @@ -35,6 +36,7 @@ vi.mock('../lambda/shared/aws', async (importOriginal) => ({ findManagedInstance: (...args: unknown[]) => findManagedInstance(...args), isSsmAgentOnline: (...args: unknown[]) => isSsmAgentOnline(...args), runShellCommand: (...args: unknown[]) => runShellCommand(...args), + readDeployConfig: (...args: unknown[]) => readDeployConfig(...args), })); vi.mock('../lambda/shared/environments', async (importOriginal) => ({ @@ -72,11 +74,23 @@ const daemonReply = (fields: Record) => ({ stdout: JSON.stringify({ state: 'running', ...fields }), }); +/** The stored deploy config the status branch reads its model facts from. */ +const deployConfig = (fields: Record = {}) => + readDeployConfig.mockResolvedValue({ + runner: 'llamacpp', + modelId: 'org/Qwen3.8-27B', + servedModelName: 'qwen3.8-27b', + ...fields, + }); + beforeEach(() => { vi.clearAllMocks(); findEnvEip.mockResolvedValue({ publicIp: '198.51.100.7' }); findManagedInstance.mockResolvedValue({ instanceId: 'i-abc', state: 'running' }); isSsmAgentOnline.mockResolvedValue(true); + // A deploy config is present for a running environment by default; tests + // that care about the model facts override it, the rest ignore it. + deployConfig(); }); /** Route each SSM invocation by the command it was given. */ @@ -98,6 +112,65 @@ describe('remote status activity reporting', () => { expect(body.idleSeconds).toBe(42); }); + it('reports what the environment is serving alongside its activity', async () => { + // The model facts come from the stored deploy config (the default one + // here); the daemon supplies only the activity. + runShellCommand.mockImplementation( + ssmRouter(daemonReply({ lastActiveAt: '2026-08-09T12:00:00Z', idleSeconds: 42 })), + ); + + const body = bodyOf(await handler(statusEvent, {} as Context)); + expect(body.state).toBe('running'); + expect(body.runner).toBe('llamacpp'); + expect(body.modelId).toBe('org/Qwen3.8-27B'); + expect(body.servedName).toBe('qwen3.8-27b'); + expect(body.lastActiveAt).toBe('2026-08-09T12:00:00Z'); + expect(body.idleSeconds).toBe(42); + }); + + it('reports the model before the engine has done any work', async () => { + // The model facts come from the deploy config, so they are present + // whenever the instance is running — even before the engine has answered a + // request. That is the case a router needs, so the model is not gated on + // activity. + runShellCommand.mockImplementation(ssmRouter(daemonReply({}))); + + const body = bodyOf(await handler(statusEvent, {} as Context)); + expect(body.state).toBe('running'); + expect(body.modelId).toBe('org/Qwen3.8-27B'); + expect(body.servedName).toBe('qwen3.8-27b'); + expect(body).not.toHaveProperty('lastActiveAt'); + expect(body).not.toHaveProperty('idleSeconds'); + }); + + it('omits the served name when the deploy named none', async () => { + // A deploy before the served-name feature stored no servedModelName, so + // the report carries the model id but no served name. + deployConfig({ servedModelName: '' }); + runShellCommand.mockImplementation( + ssmRouter(daemonReply({ lastActiveAt: '2026-08-09T12:00:00Z', idleSeconds: 5 })), + ); + + const body = bodyOf(await handler(statusEvent, {} as Context)); + expect(body.modelId).toBe('org/Qwen3.8-27B'); + expect(body).not.toHaveProperty('servedName'); + }); + + it('omits the model facts when the deploy config cannot be read', async () => { + readDeployConfig.mockRejectedValue(new Error('InvalidParameter')); + runShellCommand.mockImplementation( + ssmRouter(daemonReply({ lastActiveAt: '2026-08-09T12:00:00Z', idleSeconds: 5 })), + ); + + const result = await handler(statusEvent, {} as Context); + expect(structured(result).statusCode).toBe(200); + const body = bodyOf(result); + expect(body.lastActiveAt).toBe('2026-08-09T12:00:00Z'); + expect(body).not.toHaveProperty('runner'); + expect(body).not.toHaveProperty('modelId'); + expect(body).not.toHaveProperty('servedName'); + }); + it('treats an omitted idleSeconds as zero, not as absent', async () => { // The daemon omits idleSeconds rather than sending 0 while the engine is // working right now. The timestamp is the gate, so this must survive.