Add State Persistence: Snapshot on Stop, Restore on Start (#335 P0-b)

cmd/cloudemu/lifecycle.go

@@ -22,6 +22,7 @@ const (
 	stateFileName     = "state.json"
 	logFileName       = "cloudemu.log"
 	endpointsFileName = "endpoints.json"
+	persistFileName   = "snapshot.json"
 
 	startupTimeout = 15 * time.Second
 	stopTimeout    = 12 * time.Second
@@ -68,6 +69,19 @@ func runDir(home string) (string, error) {
 func statePath(dir string) string     { return filepath.Join(dir, stateFileName) }
 func logPath(dir string) string       { return filepath.Join(dir, logFileName) }
 func endpointsPath(dir string) string { return filepath.Join(dir, endpointsFileName) }
+func persistPath(dir string) string   { return filepath.Join(dir, persistFileName) }
+
+// hasFlag reports whether args contains --name / -name (bare or =value form).
+func hasFlag(args []string, name string) bool {
+	for _, a := range args {
+		if a == "--"+name || a == "-"+name ||
+			strings.HasPrefix(a, "--"+name+"=") || strings.HasPrefix(a, "-"+name+"=") {
+			return true
+		}
+	}
+
+	return false
+}
 
 func writeState(dir string, s daemonState) error {
 	if err := os.MkdirAll(dir, dirPerm); err != nil {
@@ -364,12 +378,26 @@ func runStart(args []string) error {
 	// so last-wins flag parsing can't redirect the file or duplicate the flag.
 	rest = stripFlag(rest, "endpoints-file", true)
 	rest = stripFlag(rest, "quiet", false)
+	// start also manages the persistence snapshot path under the run dir; drop a
+	// user --state-file so it can't point elsewhere.
+	rest = stripFlag(rest, "state-file", true)
 
 	dir, err := runDir(home)
 	if err != nil {
 		return err
 	}
 
+	// Opt-in persistence: if the user asked to persist, point serve at a snapshot
+	// file in the run dir (and imply --persist when only --persist-metadata-only
+	// is given).
+	if hasFlag(rest, "persist") || hasFlag(rest, "persist-metadata-only") {
+		if !hasFlag(rest, "persist") {
+			rest = append(rest, "--persist")
+		}
+
+		rest = append(rest, "--state-file", persistPath(dir))
+	}
+
 	if s, rErr := readState(dir); rErr == nil && processAlive(s.PID) && daemonReachable(s.Endpoints) {
 		fmt.Printf("cloudemu already running (pid %d)\n", s.PID)
 		printEndpoints(s.Endpoints)
@@ -574,7 +602,7 @@ func runDelete(args []string) error {
 		return err
 	}
 
-	for _, p := range []string{statePath(dir), logPath(dir), endpointsPath(dir)} {
+	for _, p := range []string{statePath(dir), logPath(dir), endpointsPath(dir), persistPath(dir)} {
 		if rmErr := os.Remove(p); rmErr != nil && !os.IsNotExist(rmErr) {
 			return rmErr
 		}

cmd/cloudemu/persist_serve_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stackshy/cloudemu/v2/seed"
+)
+
+// TestRestoreStateIgnoresUnreadableFile covers M1's fail-open half: a corrupt or
+// missing snapshot must not wedge startup — restoreState warns and starts empty
+// rather than returning an error that aborts runServe.
+func TestRestoreStateIgnoresUnreadableFile(t *testing.T) {
+	dir := t.TempDir()
+
+	corrupt := filepath.Join(dir, "snapshot.json")
+	if err := os.WriteFile(corrupt, []byte("{ not valid json"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := restoreState(context.Background(), corrupt, map[string]seed.Target{}); err != nil {
+		t.Fatalf("restoreState(corrupt) = %v, want nil (start empty)", err)
+	}
+
+	missing := filepath.Join(dir, "does-not-exist.json")
+	if err := restoreState(context.Background(), missing, map[string]seed.Target{}); err != nil {
+		t.Fatalf("restoreState(missing) = %v, want nil", err)
+	}
+}

cmd/cloudemu/serve.go

@@ -6,7 +6,6 @@
 	"errors"
 	"flag"
 	"fmt"
-	eksprov "github.com/stackshy/cloudemu/v2/providers/aws/eks"
 	"net"
 	"net/http"
 	"os"
@@ -18,6 +17,8 @@
 
 	"github.com/stackshy/cloudemu/v2"
 	"github.com/stackshy/cloudemu/v2/config"
+	"github.com/stackshy/cloudemu/v2/persist"
+	eksprov "github.com/stackshy/cloudemu/v2/providers/aws/eks"
 	"github.com/stackshy/cloudemu/v2/seed"
 	"github.com/stackshy/cloudemu/v2/server/admin"
 	awsserver "github.com/stackshy/cloudemu/v2/server/aws"
@@ -26,26 +27,32 @@
 	"github.com/stackshy/cloudemu/v2/services/kubernetes"
 )
 
+// errStateFileRequired is returned when --persist is set without --state-file.
+var errStateFileRequired = errors.New("--persist requires --state-file")
+
 // serveConfig holds the resolved serve flags.
 type serveConfig struct {
-	providers  string
-	host       string
-	awsPort    string
-	azurePort  string
-	gcpPort    string
-	k8sPort    string
-	accountID  string
-	region     string
-	projectID  string
-	latency    time.Duration
-	tlsCert    string
-	tlsKey     string
-	tlsHosts   stringList
-	endpoints  string
-	admin      bool
-	logReqs    bool
-	quiet      bool
-	shutdownTO time.Duration
+	providers       string
+	host            string
+	awsPort         string
+	azurePort       string
+	gcpPort         string
+	k8sPort         string
+	accountID       string
+	region          string
+	projectID       string
+	latency         time.Duration
+	tlsCert         string
+	tlsKey          string
+	tlsHosts        stringList
+	endpoints       string
+	admin           bool
+	logReqs         bool
+	quiet           bool
+	shutdownTO      time.Duration
+	persist         bool
+	stateFile       string
+	persistMetaOnly bool
 }
 
 // stringList is a repeatable string flag (e.g. --tls-host a --tls-host b).
@@ -78,6 +85,9 @@
 	fs.BoolVar(&c.logReqs, "log-requests", false, "log every HTTP request (method, path, status, duration)")
 	fs.BoolVar(&c.quiet, "quiet", false, "suppress the startup banner")
 	fs.DurationVar(&c.shutdownTO, "shutdown-timeout", 10*time.Second, "grace period for in-flight requests on shutdown")
+	fs.BoolVar(&c.persist, "persist", false, "save state to --state-file on shutdown and restore it on startup (includes object bodies)")
+	fs.StringVar(&c.stateFile, "state-file", "", "path to the JSON state snapshot (required with --persist)")
+	fs.BoolVar(&c.persistMetaOnly, "persist-metadata-only", false, "persist resource structure but omit object bodies (smaller snapshot)")
 	fs.Usage = func() {
 		fmt.Fprintf(fs.Output(), "Usage: cloudemu serve [flags]\n\nStart the standalone emulator. Flags:\n")
 		fs.PrintDefaults()
@@ -86,9 +96,13 @@
 		return err
 	}
 	if (c.tlsCert == "") != (c.tlsKey == "") {
 		return errors.New("--tls-cert and --tls-key must be given together")
 	}
 
+	if c.persist && c.stateFile == "" {
+		return errStateFileRequired
+	}
+
 	sel, err := parseProviders(c.providers)
 	if err != nil {
 		return err
@@ -196,6 +210,14 @@
 	}
 	rebuild() // populate the backends before serving
 
+	// Restore persisted state into the freshly-built providers before serving,
+	// so the first request already sees the resources from the last run.
+	if c.persist {
+		if err := restoreState(context.Background(), c.stateFile, targets); err != nil {
+			return fmt.Errorf("restore persisted state: %w", err)
+		}
+	}
+
 	// seedFor applies a fixture body to a provider's current drivers. It shares
 	// rebuildMu with reset so a seed and a reset can't run against each other's
 	// half-built state.
@@ -299,7 +321,7 @@
 
 	errCh := make(chan error, len(servers))
 	for i, s := range servers {
 		s := s
 		ln := listeners[i]
 		go func() {
 			var err error
@@ -343,9 +365,73 @@
 			shutErr = err
 		}
 	}
+
+	// Snapshot after Shutdown so no in-flight request can mutate state mid-read.
+	if c.persist {
+		if err := snapshotState(context.Background(), c.stateFile, !c.persistMetaOnly, targets); err != nil {
+			fmt.Fprintf(os.Stderr, "warning: failed to save state to %s: %v\n", c.stateFile, err)
+		} else if !c.quiet {
+			fmt.Fprintf(os.Stdout, "state saved to %s\n", c.stateFile)
+		}
+	}
+
 	return shutErr
 }
 
+// restoreState loads the snapshot at path (if any) into the freshly-built
+// providers. A missing file is not an error — the server just starts empty,
+// exactly as it does without --persist. Providers present in the snapshot but
+// not running now are skipped.
+func restoreState(ctx context.Context, path string, targets map[string]seed.Target) error {
+	snap, err := persist.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil // first run — nothing to restore
+		}
+
+		// A corrupt / truncated / unknown-schema snapshot must not wedge startup
+		// on the very stop→start path this feature serves: warn and start empty
+		// rather than aborting.
+		fmt.Fprintf(os.Stderr, "warning: ignoring unreadable state file %s: %v\n", path, err)
+
+		return nil
+	}
+
+	for name := range snap.Providers {
+		t, ok := targets[name]
+		if !ok {
+			continue
+		}
+
+		ps := snap.Providers[name]
+		if err := persist.Restore(ctx, t, &ps); err != nil {
+			return fmt.Errorf("restore %s: %w", name, err)
+		}
+	}
+
+	return nil
+}
+
+// snapshotState exports every running provider's state and writes the snapshot
+// file. Called after Shutdown, so the providers are quiescent.
+func snapshotState(ctx context.Context, path string, includeAssets bool, targets map[string]seed.Target) error {
+	snap := persist.Snapshot{
+		SchemaVersion: persist.SchemaVersion,
+		Providers:     make(map[string]persist.ProviderState, len(targets)),
+	}
+
+	for name, t := range targets {
+		ps, err := persist.Export(ctx, t, persist.Options{IncludeAssets: includeAssets})
+		if err != nil {
+			return fmt.Errorf("export %s: %w", name, err)
+		}
+
+		snap.Providers[name] = ps
+	}
+
+	return snap.WriteFile(path)
+}
+
 type namedServer struct {
 	name string
 	srv  *http.Server
@@ -373,7 +459,7 @@
 			continue
 		}
 		if p != "aws" && p != "azure" && p != "gcp" {
 			return nil, fmt.Errorf("unknown provider %q (want aws, azure, or gcp)", p)
 		}
 		if !seen[p] {
 			seen[p] = true
@@ -381,7 +467,7 @@
 		}
 	}
 	if len(out) == 0 {
 		return nil, errors.New("no providers selected")
 	}
 	return out, nil
 }

docs/standalone-server.md

@@ -83,8 +83,45 @@ already-running instance). `--endpoints-file` and `--quiet` are managed by
 
 Run state (pid, log, resolved endpoints) lives under `~/.cloudemu/` by default;
 point it elsewhere with `--home <dir>` (pass the same `--home` to the other
-lifecycle commands). State is **not** yet persisted across `stop`/`start` — the
-server still starts empty each time (see "Not yet included").
+lifecycle commands).
+
+### Persistence across restarts
+
+By default the emulator starts empty every time. Pass `--persist` to `start` and
+your resources survive `stop`→`start`:
+
+```sh
+cloudemu start --persist          # save on stop, restore on start
+# create buckets/tables/objects…
+cloudemu stop                     # writes ~/.cloudemu/<home>/snapshot.json
+cloudemu start --persist          # your resources are back
+cloudemu delete                   # also removes the snapshot + assets
+```
+
+`start` manages the snapshot path for you (in the run dir). Persistence is
+**opt-in**; when on, it saves your resources *including* object bodies, so an S3
+object comes back with its contents intact. If you only care about the resource
+structure and want a smaller snapshot, add `--persist-metadata-only` to skip
+object bytes:
+
+```sh
+cloudemu start --persist                       # full: structure + object bodies
+cloudemu start --persist --persist-metadata-only   # smaller: structure only
+```
+
+Coverage is currently the data-bearing services that share a cross-provider
+driver interface — object storage (S3/Blob/GCS), NoSQL tables
+(DynamoDB/Firestore/Cosmos), secrets (Secrets Manager/Key Vault/Secret Manager),
+and compute instances (EC2/VMs/GCE); other services still start empty. The
+snapshot is a single human-readable JSON file spanning all three providers, so
+you can inspect or `git diff` it.
+
+Fidelity notes: object bodies, secret values, and table items are all saved by
+default, along with any secondary indexes present in a table's configuration.
+Pass `--persist-metadata-only` to drop object *bodies* for a smaller snapshot —
+restored objects then come back as zero-byte keys until you re-upload them.
+Compute instances are recreated via `RunInstances`, so image/type/tags are
+preserved but the emulator assigns fresh instance IDs and IPs on restore.
 
 ## Ports
 
@@ -169,6 +206,9 @@ cloudemu serve --tls-host myhost.local --tls-host 192.168.1.10
 | `--tls-cert` / `--tls-key` | — | supply your own Azure cert (else self-signed) |
 | `--tls-host` | — | extra SAN for the generated cert (repeatable) |
 | `--endpoints-file` | — | write resolved endpoints as JSON |
+| `--persist` | `false` | save state on shutdown and restore it on startup, including object bodies (requires `--state-file`) |
+| `--state-file` | — | path to the JSON state snapshot (`start` manages this for you) |
+| `--persist-metadata-only` | `false` | persist resource structure but omit object bodies (smaller snapshot) |
 | `--log-requests` | `false` | log every request |
 | `--quiet` | `false` | suppress the startup banner |
 | `--shutdown-timeout` | `10s` | grace period for in-flight requests on Ctrl-C |
@@ -239,7 +279,7 @@ seed.Apply(ctx, f, seed.Target{Storage: aws.S3, Database: aws.DynamoDB})
 
 ## Not yet included
 
-**Persistence** across restarts and **snapshot/restore** aren't part of this
-mode yet — snapshot/restore needs the state model tracked in #107. Docker
-packaging (#247) and a Testcontainers module (#248) build directly on this
-binary.
+**Persistence** covers object storage and NoSQL tables today (see "Persistence
+across restarts" above); the remaining services and full **snapshot/restore**
+fidelity are tracked in #107. Docker packaging (#247) and a Testcontainers
+module (#248) build directly on this binary.