From 30c7341e496b2cc80056cb73c9e8cd49dedd17c4 Mon Sep 17 00:00:00 2001 From: Gajendra Malviya Date: Fri, 7 Aug 2026 17:25:32 +0530 Subject: [PATCH 1/2] Add cloud snapshots: named save/load/list/delete of emulator state (#335 P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persistence (#337) auto-saves one state on stop; this adds multiple named, restorable snapshots on a live server — a local, free equivalent of LocalStack Cloud Pods, built on the same persist engine. - persist: add optional Meta header to Snapshot and whole-emulator ExportAll / RestoreAll helpers (serve's persist-on-stop path refactored to reuse them). - admin: GET /_cloudemu/snapshot exports the whole-emulator state as JSON, POST rebuilds to empty and restores from the posted state; both act on every provider like reset. Wired through NewControl (snapshot/restore may be nil to disable, e.g. --admin=false). - CLI: `cloudemu snapshot save|load|list|delete `. save/load talk to the running daemon's control plane; list/delete are file operations. Snapshots are single JSON files under ~/.cloudemu/snapshots/, atomic-written, with name sanitization to block path traversal and --force to overwrite. Covers the same services as persistence (object storage, NoSQL tables, secrets, compute). Tests: persist multi-provider ExportAll/RestoreAll; admin snapshot GET/POST/disabled; CLI name validation, flag parsing, and save/load/list/delete against a test server. --- cmd/cloudemu/lifecycle_other.go | 6 + cmd/cloudemu/main.go | 17 +- cmd/cloudemu/serve.go | 70 +++++--- cmd/cloudemu/snapshot.go | 302 ++++++++++++++++++++++++++++++++ cmd/cloudemu/snapshot_test.go | 104 +++++++++++ docs/standalone-server.md | 26 +++ persist/persist.go | 48 +++++ persist/persist_test.go | 61 +++++++ server/admin/admin.go | 72 +++++++- server/admin/admin_test.go | 40 ++++- 10 files changed, 705 insertions(+), 41 deletions(-) create mode 100644 cmd/cloudemu/snapshot.go create mode 100644 cmd/cloudemu/snapshot_test.go diff --git a/cmd/cloudemu/lifecycle_other.go b/cmd/cloudemu/lifecycle_other.go index 8ec455b8..bad6d439 100644 --- a/cmd/cloudemu/lifecycle_other.go +++ b/cmd/cloudemu/lifecycle_other.go @@ -10,3 +10,9 @@ import "errors" func runLifecycle(_ string, _ []string) error { return errors.New("start/stop/status/logs/delete are only supported on Unix/macOS; run `cloudemu serve` instead") } + +// runSnapshot is unavailable off Unix for the same reason as the lifecycle +// commands: it operates on the background daemon's run directory. +func runSnapshot(_ []string) error { + return errors.New("snapshot is only supported on Unix/macOS") +} diff --git a/cmd/cloudemu/main.go b/cmd/cloudemu/main.go index 8b190a95..098f8c0f 100644 --- a/cmd/cloudemu/main.go +++ b/cmd/cloudemu/main.go @@ -20,6 +20,7 @@ Usage: cloudemu status Show whether the emulator is running and its endpoints cloudemu logs [-f] Print (or follow) the background emulator's log cloudemu delete Stop the emulator and remove its run directory + cloudemu snapshot ... Save/load/list/delete named state snapshots cloudemu serve [flags] Run the server in the foreground (see: cloudemu serve -h) cloudemu version Print the version cloudemu help Show this message @@ -31,11 +32,12 @@ default; override with --home . Run "cloudemu serve -h" for serve flags. // Lifecycle subcommand names. Defined here (not in the Unix-tagged // lifecycle.go) so the dispatch compiles on every platform. const ( - cmdStart = "start" - cmdStop = "stop" - cmdStatus = "status" - cmdLogs = "logs" - cmdDelete = "delete" + cmdStart = "start" + cmdStop = "stop" + cmdStatus = "status" + cmdLogs = "logs" + cmdDelete = "delete" + cmdSnapshot = "snapshot" ) // version is overridable at build time with @@ -59,6 +61,11 @@ func main() { fmt.Fprintln(os.Stderr, "cloudemu:", err) os.Exit(1) } + case cmdSnapshot: + if err := runSnapshot(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, "cloudemu:", err) + os.Exit(1) + } case "version", "-v", "--version": fmt.Println("cloudemu", version) case "help", "-h", "--help": diff --git a/cmd/cloudemu/serve.go b/cmd/cloudemu/serve.go index ba305de0..76f69bcc 100644 --- a/cmd/cloudemu/serve.go +++ b/cmd/cloudemu/serve.go @@ -3,6 +3,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -30,6 +31,10 @@ import ( // errStateFileRequired is returned when --persist is set without --state-file. var errStateFileRequired = errors.New("--persist requires --state-file") +// errUnsupportedSnapshot is returned when a posted snapshot has an unknown +// schema version. +var errUnsupportedSnapshot = errors.New("unsupported snapshot schema version") + // serveConfig holds the resolved serve flags. type serveConfig struct { providers string @@ -249,13 +254,47 @@ func runServe(args []string) error { c.host) } + // snapshotFn/restoreFn back /_cloudemu/snapshot; both act on the whole + // emulator like reset. snapshot captures current state as JSON; restore + // rebuilds to empty then loads the posted state. + snapshotFn := func() ([]byte, error) { + rebuildMu.Lock() + cur := targets + rebuildMu.Unlock() + + snap, err := persist.ExportAll(context.Background(), cur, persist.Options{IncludeAssets: true}) + if err != nil { + return nil, err + } + + return json.MarshalIndent(snap, "", " ") + } + restoreFn := func(body []byte) error { + var snap persist.Snapshot + if err := json.Unmarshal(body, &snap); err != nil { + return fmt.Errorf("parse snapshot: %w", err) + } + + if snap.SchemaVersion != persist.SchemaVersion { + return fmt.Errorf("%w: got %d, want %d", errUnsupportedSnapshot, snap.SchemaVersion, persist.SchemaVersion) + } + + rebuild() // wipe to empty before loading + + rebuildMu.Lock() + cur := targets + rebuildMu.Unlock() + + return persist.RestoreAll(context.Background(), &snap, cur) + } + // handlerFor fronts a backend with the /_cloudemu control plane. With the // admin API off the backend serves directly, so control paths fall through // to the wire handlers (whatever they return for an unrouted path). seedFn // may be nil (e.g. the Kubernetes port), which disables the seed endpoint. handlerFor := func(b *admin.Backend, seedFn func([]byte) (int, error)) http.Handler { if c.admin { - return admin.NewControl(b, rebuild, seedFn) + return admin.NewControl(b, rebuild, seedFn, snapshotFn, restoreFn) } return b } @@ -397,36 +436,15 @@ func restoreState(ctx context.Context, path string, targets map[string]seed.Targ return nil } - for name := range snap.Providers { - t, ok := targets[name] - if !ok { - continue - } - - ps := snap.Providers[name] - if err := persist.Restore(ctx, t, &ps); err != nil { - return fmt.Errorf("restore %s: %w", name, err) - } - } - - return nil + return persist.RestoreAll(ctx, &snap, targets) } // snapshotState exports every running provider's state and writes the snapshot // file. Called after Shutdown, so the providers are quiescent. func snapshotState(ctx context.Context, path string, includeAssets bool, targets map[string]seed.Target) error { - snap := persist.Snapshot{ - SchemaVersion: persist.SchemaVersion, - Providers: make(map[string]persist.ProviderState, len(targets)), - } - - for name, t := range targets { - ps, err := persist.Export(ctx, t, persist.Options{IncludeAssets: includeAssets}) - if err != nil { - return fmt.Errorf("export %s: %w", name, err) - } - - snap.Providers[name] = ps + snap, err := persist.ExportAll(ctx, targets, persist.Options{IncludeAssets: includeAssets}) + if err != nil { + return err } return snap.WriteFile(path) diff --git a/cmd/cloudemu/snapshot.go b/cmd/cloudemu/snapshot.go new file mode 100644 index 00000000..ff64eec3 --- /dev/null +++ b/cmd/cloudemu/snapshot.go @@ -0,0 +1,302 @@ +//go:build unix + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/stackshy/cloudemu/v2/persist" +) + +const ( + snapshotsDirName = "snapshots" + snapHTTPTimeout = 30 * time.Second +) + +var snapNameRE = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`) + +// Sentinel errors so callers (and err113) get static, wrappable failures. +var ( + errSnapUsage = errors.New("usage: cloudemu snapshot [name] [--home dir] [--force]") + errSnapName = errors.New("snapshot name must be 1-64 chars of [A-Za-z0-9._-], excluding the reserved dot and double-dot") + errSnapExists = errors.New("snapshot already exists (pass --force to overwrite)") + errSnapNotFound = errors.New("snapshot not found") + errSnapNoEndpoint = errors.New("no plain-HTTP endpoint available (start the aws or gcp provider)") + errSnapAdminOff = errors.New("the server's control plane is disabled (started with --admin=false)") + errSnapDaemonDown = errors.New("cloudemu is not running (snapshot save/load need a running server)") + errSnapServer = errors.New("snapshot request failed") +) + +func snapshotsDir(dir string) string { return filepath.Join(dir, snapshotsDirName) } +func snapshotFilePath(dir, n string) string { return filepath.Join(snapshotsDir(dir), n+".json") } + +func validSnapshotName(name string) bool { + return name != "." && name != ".." && snapNameRE.MatchString(name) +} + +// parseSnapshotFlags splits --home / --force out of args, returning the +// remaining positional args. +func parseSnapshotFlags(args []string) (home string, force bool, pos []string) { + home, rest := splitHomeFlag(args) + pos = make([]string, 0, len(rest)) + + for _, a := range rest { + if a == "--force" || a == "-force" { + force = true + + continue + } + + pos = append(pos, a) + } + + return home, force, pos +} + +// runSnapshot dispatches the snapshot save/load/list/delete subcommands. +func runSnapshot(args []string) error { + if len(args) == 0 { + return errSnapUsage + } + + home, force, pos := parseSnapshotFlags(args[1:]) + + dir, err := runDir(home) + if err != nil { + return err + } + + switch args[0] { + case "save": + return withName(pos, func(n string) error { return snapshotSave(dir, n, force) }) + case "load": + return withName(pos, func(n string) error { return snapshotLoad(dir, n) }) + case "delete": + return withName(pos, func(n string) error { return snapshotDelete(dir, n) }) + case "list": + return snapshotList(dir) + default: + return errSnapUsage + } +} + +// withName runs fn with the single positional name, or returns a usage error. +func withName(pos []string, fn func(string) error) error { + if len(pos) != 1 { + return errSnapUsage + } + + return fn(pos[0]) +} + +func snapshotSave(dir, name string, force bool) error { + if !validSnapshotName(name) { + return errSnapName + } + + path := snapshotFilePath(dir, name) + if !force { + if _, err := os.Stat(path); err == nil { + return errSnapExists + } + } + + base, err := adminBaseURL(dir) + if err != nil { + return err + } + + body, err := snapshotRequest(http.MethodGet, base, nil) + if err != nil { + return err + } + + var snap persist.Snapshot + if err := json.Unmarshal(body, &snap); err != nil { + return fmt.Errorf("parse snapshot from server: %w", err) + } + + providers := make([]string, 0, len(snap.Providers)) + for p := range snap.Providers { + providers = append(providers, p) + } + + sort.Strings(providers) + + snap.Meta = &persist.Meta{ + Name: name, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + CloudemuVersion: version, + Providers: providers, + } + + if err := snap.WriteFile(path); err != nil { + return err + } + + fmt.Printf("saved snapshot %q (providers: %s)\n", name, strings.Join(providers, ", ")) + + return nil +} + +func snapshotLoad(dir, name string) error { + if !validSnapshotName(name) { + return errSnapName + } + + body, err := os.ReadFile(snapshotFilePath(dir, name)) + if errors.Is(err, os.ErrNotExist) { + return errSnapNotFound + } + + if err != nil { + return err + } + + base, err := adminBaseURL(dir) + if err != nil { + return err + } + + if _, err := snapshotRequest(http.MethodPost, base, body); err != nil { + return err + } + + fmt.Printf("loaded snapshot %q\n", name) + + return nil +} + +func snapshotDelete(dir, name string) error { + if !validSnapshotName(name) { + return errSnapName + } + + err := os.Remove(snapshotFilePath(dir, name)) + if errors.Is(err, os.ErrNotExist) { + return errSnapNotFound + } + + if err != nil { + return err + } + + fmt.Printf("deleted snapshot %q\n", name) + + return nil +} + +func snapshotList(dir string) error { + entries, err := os.ReadDir(snapshotsDir(dir)) + if errors.Is(err, os.ErrNotExist) { + fmt.Println("no snapshots") + + return nil + } + + if err != nil { + return err + } + + printed := 0 + + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") { + continue + } + + if printed == 0 { + fmt.Printf("%-24s %-20s %10s\n", "NAME", "CREATED", "SIZE") + } + + info, iErr := e.Info() + if iErr != nil { + return iErr + } + + fmt.Printf("%-24s %-20s %10d\n", + strings.TrimSuffix(e.Name(), ".json"), + info.ModTime().UTC().Format(time.DateTime), + info.Size()) + + printed++ + } + + if printed == 0 { + fmt.Println("no snapshots") + } + + return nil +} + +// adminBaseURL reads the daemon's endpoints file and returns a plain-HTTP base +// URL for the control plane (avoids the self-signed HTTPS endpoints). +func adminBaseURL(dir string) (string, error) { + eps, err := readEndpoints(endpointsPath(dir)) + if errors.Is(err, os.ErrNotExist) { + return "", errSnapDaemonDown + } + + if err != nil { + return "", err + } + + for _, k := range []string{"aws", "gcp"} { + if ep := eps[k]; strings.HasPrefix(ep, "http://") { + return strings.TrimRight(ep, "/"), nil + } + } + + return "", errSnapNoEndpoint +} + +// snapshotRequest calls the daemon's /_cloudemu/snapshot endpoint. For GET body +// is nil and the response bytes are returned; for POST body is the snapshot. +func snapshotRequest(method, base string, body []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), snapHTTPTimeout) + defer cancel() + + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + + req, err := http.NewRequestWithContext(ctx, method, base+"/_cloudemu/snapshot", reader) + if err != nil { + return nil, err + } + + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: %w", errSnapDaemonDown, err) + } + defer resp.Body.Close() + + rb, _ := io.ReadAll(resp.Body) + + if resp.StatusCode == http.StatusNotImplemented { + return nil, errSnapAdminOff + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: %s: %s", errSnapServer, resp.Status, strings.TrimSpace(string(rb))) + } + + return rb, nil +} diff --git a/cmd/cloudemu/snapshot_test.go b/cmd/cloudemu/snapshot_test.go new file mode 100644 index 00000000..3eeece25 --- /dev/null +++ b/cmd/cloudemu/snapshot_test.go @@ -0,0 +1,104 @@ +//go:build unix + +package main + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestValidSnapshotName(t *testing.T) { + ok := []string{"baseline", "v1", "with-users", "a.b_c-1", strings.Repeat("x", 64)} + for _, n := range ok { + if !validSnapshotName(n) { + t.Errorf("validSnapshotName(%q) = false, want true", n) + } + } + + bad := []string{"", ".", "..", "a/b", "../etc", "has space", strings.Repeat("x", 65), "a*b"} + for _, n := range bad { + if validSnapshotName(n) { + t.Errorf("validSnapshotName(%q) = true, want false", n) + } + } +} + +func TestParseSnapshotFlags(t *testing.T) { + home, force, pos := parseSnapshotFlags([]string{"v1", "--home", "/tmp/h", "--force"}) + if home != "/tmp/h" || !force || len(pos) != 1 || pos[0] != "v1" { + t.Fatalf("parseSnapshotFlags = %q %v %v", home, force, pos) + } + + home, force, pos = parseSnapshotFlags([]string{"v2"}) + if home != "" || force || len(pos) != 1 || pos[0] != "v2" { + t.Fatalf("parseSnapshotFlags(bare) = %q %v %v", home, force, pos) + } +} + +func TestSnapshotSaveListLoadDelete(t *testing.T) { + var posted string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + _, _ = w.Write([]byte(`{"schemaVersion":1,"providers":{"aws":{}}}`)) + + return + } + + b, _ := io.ReadAll(r.Body) + posted = string(b) + _, _ = w.Write([]byte(`{"status":"restored"}`)) + })) + defer srv.Close() + + dir := t.TempDir() + if err := os.WriteFile(endpointsPath(dir), []byte(`{"aws":"`+srv.URL+`"}`), 0o600); err != nil { + t.Fatal(err) + } + + // save writes the snapshot file + if err := snapshotSave(dir, "s1", false); err != nil { + t.Fatalf("save: %v", err) + } + if _, err := os.Stat(snapshotFilePath(dir, "s1")); err != nil { + t.Fatalf("saved file missing: %v", err) + } + + // a second save without --force is refused + if err := snapshotSave(dir, "s1", false); !errors.Is(err, errSnapExists) { + t.Fatalf("save(exists) = %v, want errSnapExists", err) + } + + // load posts the saved file back to the server + if err := snapshotLoad(dir, "s1"); err != nil { + t.Fatalf("load: %v", err) + } + if !strings.Contains(posted, `"schemaVersion"`) || !strings.Contains(posted, `"name": "s1"`) { + t.Fatalf("posted snapshot missing schema/meta: %s", posted) + } + + // loading a missing snapshot is a clear error + if err := snapshotLoad(dir, "nope"); !errors.Is(err, errSnapNotFound) { + t.Fatalf("load(missing) = %v, want errSnapNotFound", err) + } + + // delete removes it; a second delete reports not found + if err := snapshotDelete(dir, "s1"); err != nil { + t.Fatalf("delete: %v", err) + } + if err := snapshotDelete(dir, "s1"); !errors.Is(err, errSnapNotFound) { + t.Fatalf("delete(again) = %v, want errSnapNotFound", err) + } +} + +func TestSnapshotSaveDaemonDown(t *testing.T) { + dir := t.TempDir() // no endpoints file → daemon considered down + if err := snapshotSave(dir, "x", false); !errors.Is(err, errSnapDaemonDown) { + t.Fatalf("save(no daemon) = %v, want errSnapDaemonDown", err) + } +} diff --git a/docs/standalone-server.md b/docs/standalone-server.md index 81f02ffe..cdb1fd7b 100644 --- a/docs/standalone-server.md +++ b/docs/standalone-server.md @@ -123,6 +123,32 @@ restored objects then come back as zero-byte keys until you re-upload them. Compute instances are recreated via `RunInstances`, so image/type/tags are preserved but the emulator assigns fresh instance IDs and IPs on restore. +### Named snapshots (`snapshot save` / `load` / `list` / `delete`) + +Persistence auto-saves a single state on stop. **Snapshots** let you capture, +name, and switch between *multiple* states on a running server — a local, free +equivalent of LocalStack's Cloud Pods. + +```sh +cloudemu start +# … create buckets / tables / secrets / instances … +cloudemu snapshot save baseline # capture current state as "baseline" +# … run a destructive test … +cloudemu snapshot load baseline # restore it instantly — no restart +cloudemu snapshot list # NAME CREATED SIZE +cloudemu snapshot delete baseline +``` + +Each snapshot is a single JSON file under `~/.cloudemu/snapshots/.json` +(override the dir with `--home`) — inspectable, `git`-diffable, and shareable: +copy the file to a teammate and they `snapshot load` the identical state. + +`save` and `load` talk to the running server's control plane, so they need the +`--admin` plane (on by default) and the `aws` or `gcp` provider running; `list` +and `delete` are file operations that work without a running server. Snapshots +cover the same services as persistence (object storage, NoSQL tables, secrets, +compute instances). Names must match `[A-Za-z0-9._-]` (1–64 chars). + ## Ports | Provider | Default | Protocol | Notes | diff --git a/persist/persist.go b/persist/persist.go index ecb8d1f4..3f05dad7 100644 --- a/persist/persist.go +++ b/persist/persist.go @@ -44,9 +44,20 @@ var ( // as one JSON document. type Snapshot struct { SchemaVersion int `json:"schemaVersion"` + Meta *Meta `json:"meta,omitempty"` Providers map[string]ProviderState `json:"providers,omitempty"` } +// Meta is optional descriptive header for a named snapshot (the auto +// persist-on-stop file leaves it nil). It lets tooling describe a snapshot +// without restoring it. +type Meta struct { + Name string `json:"name,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + CloudemuVersion string `json:"cloudemuVersion,omitempty"` + Providers []string `json:"providers,omitempty"` +} + // ProviderState is a single provider's persisted resources. type ProviderState struct { Buckets []Bucket `json:"buckets,omitempty"` @@ -105,6 +116,43 @@ type Options struct { IncludeAssets bool } +// ExportAll captures the state of every provider in targets into one Snapshot. +// It is the whole-emulator core shared by the persist-on-stop path and the +// snapshot admin endpoint. +func ExportAll(ctx context.Context, targets map[string]seed.Target, opts Options) (Snapshot, error) { + snap := Snapshot{SchemaVersion: SchemaVersion, Providers: make(map[string]ProviderState, len(targets))} + + for name, t := range targets { + ps, err := Export(ctx, t, opts) + if err != nil { + return Snapshot{}, fmt.Errorf("export %s: %w", name, err) + } + + snap.Providers[name] = ps + } + + return snap, nil +} + +// RestoreAll restores each provider present in snap into the matching target. +// Providers in the snapshot with no matching running target are skipped, and +// targets should be freshly rebuilt (empty) before calling. +func RestoreAll(ctx context.Context, snap *Snapshot, targets map[string]seed.Target) error { + for name := range snap.Providers { + t, ok := targets[name] + if !ok { + continue + } + + ps := snap.Providers[name] + if err := Restore(ctx, t, &ps); err != nil { + return fmt.Errorf("restore %s: %w", name, err) + } + } + + return nil +} + // Export reads a provider's current state through its drivers. A nil driver for // a kind simply contributes nothing (that kind isn't persisted). func Export(ctx context.Context, t seed.Target, opts Options) (ProviderState, error) { diff --git a/persist/persist_test.go b/persist/persist_test.go index 2cdf3017..ae974d98 100644 --- a/persist/persist_test.go +++ b/persist/persist_test.go @@ -11,6 +11,7 @@ import ( "github.com/stackshy/cloudemu/v2/persist" "github.com/stackshy/cloudemu/v2/seed" dbdriver "github.com/stackshy/cloudemu/v2/services/database/driver" + storagedriver "github.com/stackshy/cloudemu/v2/services/storage/driver" ) // TestExportRestoreRoundTrip is the core persistence guarantee: state exported @@ -101,6 +102,66 @@ func TestExportRestoreRoundTrip(t *testing.T) { } } +// TestExportAllRestoreAllMultiProvider covers the whole-emulator core used by +// both the persist-on-stop path and the snapshot admin endpoint: ExportAll +// captures every provider into one Snapshot, and RestoreAll puts each back into +// the matching target. +func TestExportAllRestoreAllMultiProvider(t *testing.T) { + ctx := context.Background() + + aws, gcp := cloudemu.NewAWS(), cloudemu.NewGCP() + src := map[string]seed.Target{ + "aws": {Storage: aws.S3, Database: aws.DynamoDB}, + "gcp": {Storage: gcp.GCS, Database: gcp.Firestore}, + } + if err := seed.Apply(ctx, seed.Fixtures{Buckets: []seed.Bucket{{Name: "a", Objects: []seed.Object{{Key: "k", Body: "av"}}}}}, src["aws"]); err != nil { + t.Fatalf("seed aws: %v", err) + } + if err := seed.Apply(ctx, seed.Fixtures{Buckets: []seed.Bucket{{Name: "g", Objects: []seed.Object{{Key: "k", Body: "gv"}}}}}, src["gcp"]); err != nil { + t.Fatalf("seed gcp: %v", err) + } + + snap, err := persist.ExportAll(ctx, src, persist.Options{IncludeAssets: true}) + if err != nil { + t.Fatalf("ExportAll: %v", err) + } + if snap.SchemaVersion != persist.SchemaVersion || len(snap.Providers) != 2 { + t.Fatalf("ExportAll snapshot = %+v", snap) + } + + raw, _ := json.Marshal(snap) + var got persist.Snapshot + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + aws2, gcp2 := cloudemu.NewAWS(), cloudemu.NewGCP() + dst := map[string]seed.Target{ + "aws": {Storage: aws2.S3, Database: aws2.DynamoDB}, + "gcp": {Storage: gcp2.GCS, Database: gcp2.Firestore}, + } + if err := persist.RestoreAll(ctx, &got, dst); err != nil { + t.Fatalf("RestoreAll: %v", err) + } + + ao, err := aws2.S3.GetObject(ctx, "a", "k") + if err != nil || string(ao.Data) != "av" { + t.Fatalf("aws restore = %v / %q", err, dataOf(ao)) + } + go_, err := gcp2.GCS.GetObject(ctx, "g", "k") + if err != nil || string(go_.Data) != "gv" { + t.Fatalf("gcp restore = %v / %q", err, dataOf(go_)) + } +} + +func dataOf(o *storagedriver.Object) []byte { + if o == nil { + return nil + } + + return o.Data +} + // TestExportMetadataOnlyOmitsBodies verifies the default (metadata-only) export // records object keys/metadata but drops the bytes, and that IncludeAssets keeps // them — the flag that keeps the snapshot file KB-sized by default. diff --git a/server/admin/admin.go b/server/admin/admin.go index 4f6211e8..1f5f6a79 100644 --- a/server/admin/admin.go +++ b/server/admin/admin.go @@ -28,6 +28,10 @@ const Prefix = "/_cloudemu/" // maxFixtureBytes caps a seed request body. const maxFixtureBytes = 32 << 20 // 32 MiB +// maxSnapshotBytes caps a restore request body. Snapshots can carry object +// bodies, so this is larger than a seed fixture. +const maxSnapshotBytes = 512 << 20 // 512 MiB + // Backend is a hot-swappable http.Handler. Requests read the current handler // under a read lock; Swap replaces it under a write lock. A zero Backend is not // usable — construct with NewBackend. @@ -63,15 +67,25 @@ func (b *Backend) ServeHTTP(w http.ResponseWriter, r *http.Request) { // seed applies a fixture body to this provider's drivers and returns how many // top-level resources it created; a nil seed disables the seed endpoint (501). type Control struct { - backend *Backend - reset func() - seed func(fixture []byte) (int, error) + backend *Backend + reset func() + seed func(fixture []byte) (int, error) + snapshot func() ([]byte, error) + restore func(snapshot []byte) error } // NewControl wraps backend with the control plane. reset must rebuild every -// backend (including this one) to a clean state. seed may be nil. -func NewControl(backend *Backend, reset func(), seed func(fixture []byte) (int, error)) *Control { - return &Control{backend: backend, reset: reset, seed: seed} +// backend (including this one) to a clean state. seed, snapshot, and restore +// may each be nil, which disables the corresponding endpoint. snapshot returns +// the whole-emulator state as JSON; restore replaces it from that JSON. +func NewControl( + backend *Backend, + reset func(), + seed func(fixture []byte) (int, error), + snapshot func() ([]byte, error), + restore func(snapshot []byte) error, +) *Control { + return &Control{backend: backend, reset: reset, seed: seed, snapshot: snapshot, restore: restore} } // ServeHTTP routes control-plane paths to the control handler and everything @@ -121,11 +135,57 @@ func (c *Control) serveControl(w http.ResponseWriter, r *http.Request) { return } writeJSON(w, http.StatusOK, map[string]any{"status": "seeded", "applied": applied}) + case "snapshot": + c.serveSnapshot(w, r) default: writeJSON(w, http.StatusNotFound, map[string]string{"error": "unknown control endpoint"}) } } +// serveSnapshot handles GET /_cloudemu/snapshot (export the whole-emulator state +// as JSON) and POST /_cloudemu/snapshot (replace it from the posted JSON). Both +// act on every provider, like reset, so a call to any provider port covers the +// whole emulator. +func (c *Control) serveSnapshot(w http.ResponseWriter, r *http.Request) { + if c.snapshot == nil || c.restore == nil { + writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "snapshots are not available on this server"}) + return + } + + switch r.Method { + case http.MethodGet: + data, err := c.snapshot() + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data) + case http.MethodPost: + body, err := io.ReadAll(io.LimitReader(r.Body, maxSnapshotBytes+1)) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "read snapshot: " + err.Error()}) + return + } + + if len(body) > maxSnapshotBytes { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "snapshot exceeds 512 MiB"}) + return + } + + if err := c.restore(body); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + + writeJSON(w, http.StatusOK, map[string]string{"status": "restored"}) + default: + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "snapshot requires GET or POST"}) + } +} + func writeJSON(w http.ResponseWriter, status int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) diff --git a/server/admin/admin_test.go b/server/admin/admin_test.go index 51ba9cae..bdf52e6d 100644 --- a/server/admin/admin_test.go +++ b/server/admin/admin_test.go @@ -61,7 +61,7 @@ func TestBackendConcurrentSwap(t *testing.T) { func TestControlReset(t *testing.T) { resets := 0 b := admin.NewBackend(handler("backend")) - c := admin.NewControl(b, func() { resets++; b.Swap(handler("rebuilt")) }, nil) + c := admin.NewControl(b, func() { resets++; b.Swap(handler("rebuilt")) }, nil, nil, nil) // Non-control paths pass through to the backend. if got := do(t, c, http.MethodGet, "/some/aws/request"); got != "backend" { @@ -84,7 +84,7 @@ func TestControlReset(t *testing.T) { func TestControlRoutes(t *testing.T) { b := admin.NewBackend(handler("backend")) - c := admin.NewControl(b, func() {}, nil) // nil seed → seed endpoint disabled + c := admin.NewControl(b, func() {}, nil, nil, nil) // nil seed → seed endpoint disabled cases := []struct { method, path string @@ -111,7 +111,7 @@ func TestControlSeed(t *testing.T) { c := admin.NewControl(b, func() {}, func(fixture []byte) (int, error) { gotFixture = string(fixture) return 4, nil - }) + }, nil, nil) rec := httptest.NewRecorder() c.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, admin.Prefix+"seed", strings.NewReader(`{"buckets":[]}`))) @@ -128,7 +128,7 @@ func TestControlSeed(t *testing.T) { // A seeder error surfaces as 400. cErr := admin.NewControl(b, func() {}, func([]byte) (int, error) { return 0, errFixture - }) + }, nil, nil) rec = httptest.NewRecorder() cErr.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, admin.Prefix+"seed", strings.NewReader(`{}`))) if rec.Code != http.StatusBadRequest { @@ -136,6 +136,38 @@ func TestControlSeed(t *testing.T) { } } +func TestControlSnapshot(t *testing.T) { + b := admin.NewBackend(handler("backend")) + + var restored string + c := admin.NewControl(b, func() {}, nil, + func() ([]byte, error) { return []byte(`{"schemaVersion":1}`), nil }, + func(body []byte) error { restored = string(body); return nil }, + ) + + // GET returns the snapshot bytes verbatim. + rec := httptest.NewRecorder() + c.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, admin.Prefix+"snapshot", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != `{"schemaVersion":1}` { + t.Fatalf("GET snapshot = %d %q", rec.Code, rec.Body.String()) + } + + // POST hands the body to restore. + rec = httptest.NewRecorder() + c.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, admin.Prefix+"snapshot", strings.NewReader(`{"schemaVersion":1}`))) + if rec.Code != http.StatusOK || restored != `{"schemaVersion":1}` { + t.Fatalf("POST snapshot = %d, restored %q", rec.Code, restored) + } + + // With nil snapshot/restore the endpoint is disabled (501). + off := admin.NewControl(b, func() {}, nil, nil, nil) + rec = httptest.NewRecorder() + off.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, admin.Prefix+"snapshot", nil)) + if rec.Code != http.StatusNotImplemented { + t.Fatalf("disabled snapshot = %d, want 501", rec.Code) + } +} + var errFixture = fmt.Errorf("bad fixture") func do(t *testing.T, h http.Handler, method, path string) string { From 0f5c5a9c0056b7bd939de472e96320160ee68214 Mon Sep 17 00:00:00 2001 From: Gajendra Malviya Date: Fri, 7 Aug 2026 17:51:32 +0530 Subject: [PATCH 2/2] Harden cloud snapshots: broaden admin warning, list Meta, document destructive load Address review on the cloud-snapshots PR: - Broaden the non-loopback --admin warning: GET /_cloudemu/snapshot dumps all emulated state (including secret values) to any caller, not just reset wiping. - `snapshot list` now reads each file's Meta header, showing the recorded createdAt and a providers column instead of the file mtime alone. - Document that `load` is destructive (wipe then restore) and note the future staging-and-swap hardening for a failed restore. --- cmd/cloudemu/serve.go | 9 +++++- cmd/cloudemu/snapshot.go | 58 +++++++++++++++++++++++++++++++++------ docs/standalone-server.md | 6 ++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/cmd/cloudemu/serve.go b/cmd/cloudemu/serve.go index 76f69bcc..222ecb7b 100644 --- a/cmd/cloudemu/serve.go +++ b/cmd/cloudemu/serve.go @@ -250,7 +250,10 @@ func runServe(args []string) error { // could POST /_cloudemu/reset. if c.admin && !isLoopbackHost(c.host) { fmt.Fprintf(os.Stderr, - "warning: --admin control plane (POST /_cloudemu/reset wipes all state) is reachable on non-loopback host %q; pass --admin=false to disable it\n", + "warning: --admin control plane is reachable on non-loopback host %q — "+ + "POST /_cloudemu/reset wipes all state, and GET /_cloudemu/snapshot dumps "+ + "all emulated state (including secret values) to any caller; "+ + "pass --admin=false to disable it\n", c.host) } @@ -279,6 +282,10 @@ func runServe(args []string) error { return fmt.Errorf("%w: got %d, want %d", errUnsupportedSnapshot, snap.SchemaVersion, persist.SchemaVersion) } + // Destructive load (reset semantics): wipe to empty, then repopulate. If + // RestoreAll fails partway the running state is already gone — acceptable + // for a local emulator, but a future hardening is to restore into a + // staging build and swap it in only on success. rebuild() // wipe to empty before loading rebuildMu.Lock() diff --git a/cmd/cloudemu/snapshot.go b/cmd/cloudemu/snapshot.go index ff64eec3..8af6e3b0 100644 --- a/cmd/cloudemu/snapshot.go +++ b/cmd/cloudemu/snapshot.go @@ -217,19 +217,17 @@ func snapshotList(dir string) error { continue } - if printed == 0 { - fmt.Printf("%-24s %-20s %10s\n", "NAME", "CREATED", "SIZE") - } - info, iErr := e.Info() if iErr != nil { return iErr } - fmt.Printf("%-24s %-20s %10d\n", - strings.TrimSuffix(e.Name(), ".json"), - info.ModTime().UTC().Format(time.DateTime), - info.Size()) + if printed == 0 { + fmt.Printf("%-24s %-20s %-18s %10s\n", "NAME", "CREATED", "PROVIDERS", "SIZE") + } + + name, created, providers := snapshotRow(snapshotsDir(dir), e, info) + fmt.Printf("%-24s %-20s %-18s %10d\n", name, created, providers, info.Size()) printed++ } @@ -241,6 +239,50 @@ func snapshotList(dir string) error { return nil } +// snapshotRow derives the display columns for one snapshot file, preferring the +// stored Meta header (accurate created-at + captured providers) and falling +// back to the filename / file mtime when Meta is absent or unreadable. +func snapshotRow(dir string, e os.DirEntry, info os.FileInfo) (name, created, providers string) { + name = strings.TrimSuffix(e.Name(), ".json") + created = info.ModTime().UTC().Format(time.DateTime) + providers = "-" + + meta := readSnapshotMeta(filepath.Join(dir, e.Name())) + if meta == nil { + return name, created, providers + } + + if meta.Name != "" { + name = meta.Name + } + + if meta.CreatedAt != "" { + created = meta.CreatedAt + } + + if len(meta.Providers) > 0 { + providers = strings.Join(meta.Providers, ",") + } + + return name, created, providers +} + +// readSnapshotMeta returns the Meta header of a snapshot file, or nil if the +// file can't be read or parsed. +func readSnapshotMeta(path string) *persist.Meta { + b, err := os.ReadFile(path) + if err != nil { + return nil + } + + var s persist.Snapshot + if err := json.Unmarshal(b, &s); err != nil { + return nil + } + + return s.Meta +} + // adminBaseURL reads the daemon's endpoints file and returns a plain-HTTP base // URL for the control plane (avoids the self-signed HTTPS endpoints). func adminBaseURL(dir string) (string, error) { diff --git a/docs/standalone-server.md b/docs/standalone-server.md index cdb1fd7b..efa4ee28 100644 --- a/docs/standalone-server.md +++ b/docs/standalone-server.md @@ -149,6 +149,12 @@ and `delete` are file operations that work without a running server. Snapshots cover the same services as persistence (object storage, NoSQL tables, secrets, compute instances). Names must match `[A-Za-z0-9._-]` (1–64 chars). +`load` is destructive: it wipes the running state (reset semantics) and then +repopulates from the snapshot, so anything created since the snapshot is +discarded. If a restore fails partway the running state is already cleared — +fine for a local emulator, but don't point `load` at a server whose current +state you haven't snapshotted. + ## Ports | Provider | Default | Protocol | Notes |