diff --git a/cmd/cloudemu/init_test.go b/cmd/cloudemu/init_test.go new file mode 100644 index 00000000..3a231478 --- /dev/null +++ b/cmd/cloudemu/init_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + cloudemu "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/seed" +) + +func TestApplyInitDirAppliesFixtures(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + + // Files apply in lexical order; content is provider-agnostic seed fixtures. + if err := os.WriteFile(filepath.Join(dir, "01-buckets.json"), []byte(`{"buckets":[{"name":"b"}]}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "02-tables.json"), []byte(`{"tables":[{"name":"t","partitionKey":"id"}]}`), 0o600); err != nil { + t.Fatal(err) + } + // A non-json file must be ignored. + if err := os.WriteFile(filepath.Join(dir, "README.txt"), []byte("ignore me"), 0o600); err != nil { + t.Fatal(err) + } + + aws := cloudemu.NewAWS() + targets := map[string]seed.Target{"aws": {Storage: aws.S3, Database: aws.DynamoDB}} + if err := applyInitDir(ctx, dir, targets); err != nil { + t.Fatalf("applyInitDir: %v", err) + } + + buckets, err := aws.S3.ListBuckets(ctx) + if err != nil || len(buckets) != 1 || buckets[0].Name != "b" { + t.Fatalf("bucket not created from init dir: %v %v", buckets, err) + } + + tables, err := aws.DynamoDB.ListTables(ctx) + if err != nil || len(tables) != 1 || tables[0] != "t" { + t.Fatalf("table not created from init dir: %v %v", tables, err) + } +} + +func TestApplyInitDirMissingIsNoOp(t *testing.T) { + aws := cloudemu.NewAWS() + targets := map[string]seed.Target{"aws": {Storage: aws.S3}} + if err := applyInitDir(context.Background(), filepath.Join(t.TempDir(), "absent"), targets); err != nil { + t.Fatalf("applyInitDir(missing) = %v, want nil", err) + } +} + +func TestApplyInitDirParseErrorFailsBoot(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "bad.json"), []byte("{ not valid"), 0o600); err != nil { + t.Fatal(err) + } + + aws := cloudemu.NewAWS() + targets := map[string]seed.Target{"aws": {Storage: aws.S3}} + if err := applyInitDir(context.Background(), dir, targets); err == nil { + t.Fatal("applyInitDir(bad json) = nil, want parse error") + } +} + +func TestApplyInitDirDuplicateWarnsNotFails(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + + // A fixture whose FIRST resource (bucket "dup") already exists, followed by a + // NEW resource (table "fresh"). The collision must not truncate the rest of + // the fixture — "fresh" must still be created. + fixture := `{"buckets":[{"name":"dup"}],"tables":[{"name":"fresh","partitionKey":"id"}]}` + if err := os.WriteFile(filepath.Join(dir, "b.json"), []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + + aws := cloudemu.NewAWS() + targets := map[string]seed.Target{"aws": {Storage: aws.S3, Database: aws.DynamoDB}} + // Pre-create the bucket so the init apply hits AlreadyExists on the first item. + if err := aws.S3.CreateBucket(ctx, "dup"); err != nil { + t.Fatal(err) + } + + // The collision must warn-and-continue, not fail boot. + if err := applyInitDir(ctx, dir, targets); err != nil { + t.Fatalf("applyInitDir(duplicate) = %v, want nil (warn+continue)", err) + } + + // The resource AFTER the collision must still have been created. + tables, err := aws.DynamoDB.ListTables(ctx) + if err != nil || len(tables) != 1 || tables[0] != "fresh" { + t.Fatalf("post-collision table not created: %v %v", tables, err) + } +} diff --git a/cmd/cloudemu/lifecycle.go b/cmd/cloudemu/lifecycle.go index 6cde4763..7043bba0 100644 --- a/cmd/cloudemu/lifecycle.go +++ b/cmd/cloudemu/lifecycle.go @@ -23,6 +23,7 @@ const ( logFileName = "cloudemu.log" endpointsFileName = "endpoints.json" persistFileName = "snapshot.json" + initDirName = "init.d" startupTimeout = 15 * time.Second stopTimeout = 12 * time.Second @@ -71,6 +72,13 @@ func logPath(dir string) string { return filepath.Join(dir, logFileName) } func endpointsPath(dir string) string { return filepath.Join(dir, endpointsFileName) } func persistPath(dir string) string { return filepath.Join(dir, persistFileName) } +// isDir reports whether path exists and is a directory. +func isDir(path string) bool { + fi, err := os.Stat(path) + + return err == nil && fi.IsDir() +} + // hasFlag reports whether args contains --name / -name (bare or =value form). func hasFlag(args []string, name string) bool { for _, a := range args { @@ -398,6 +406,14 @@ func runStart(args []string) error { rest = append(rest, "--state-file", persistPath(dir)) } + // Auto-load a drop-in init.d under the run dir (docker-entrypoint.d style) + // unless the user pointed --init-dir elsewhere. + if !hasFlag(rest, "init-dir") { + if d := filepath.Join(dir, initDirName); isDir(d) { + rest = append(rest, "--init-dir", d) + } + } + if s, rErr := readState(dir); rErr == nil && processAlive(s.PID) && daemonReachable(s.Endpoints) { fmt.Printf("cloudemu already running (pid %d)\n", s.PID) printEndpoints(s.Endpoints) diff --git a/cmd/cloudemu/serve.go b/cmd/cloudemu/serve.go index 222ecb7b..c398252c 100644 --- a/cmd/cloudemu/serve.go +++ b/cmd/cloudemu/serve.go @@ -11,6 +11,8 @@ import ( "net/http" "os" "os/signal" + "path/filepath" + "sort" "strings" "sync" "syscall" @@ -58,6 +60,7 @@ type serveConfig struct { persist bool stateFile string persistMetaOnly bool + initDir string } // stringList is a repeatable string flag (e.g. --tls-host a --tls-host b). @@ -93,6 +96,7 @@ func runServe(args []string) error { fs.BoolVar(&c.persist, "persist", false, "save state to --state-file on shutdown and restore it on startup (includes object bodies)") fs.StringVar(&c.stateFile, "state-file", "", "path to the JSON state snapshot (required with --persist)") fs.BoolVar(&c.persistMetaOnly, "persist-metadata-only", false, "persist resource structure but omit object bodies (smaller snapshot)") + fs.StringVar(&c.initDir, "init-dir", "", "apply every *.json seed fixture in this directory on startup") fs.Usage = func() { fmt.Fprintf(fs.Output(), "Usage: cloudemu serve [flags]\n\nStart the standalone emulator. Flags:\n") fs.PrintDefaults() @@ -223,6 +227,14 @@ func runServe(args []string) error { } } + // Apply init fixtures on top of the built (and possibly restored) providers, + // before serving, so the first request already sees the boot state. + if c.initDir != "" { + if err := applyInitDir(context.Background(), c.initDir, targets); err != nil { + return fmt.Errorf("apply init dir: %w", err) + } + } + // seedFor applies a fixture body to a provider's current drivers. It shares // rebuildMu with reset so a seed and a reset can't run against each other's // half-built state. @@ -446,6 +458,65 @@ func restoreState(ctx context.Context, path string, targets map[string]seed.Targ return persist.RestoreAll(ctx, &snap, targets) } +// applyInitDir applies every *.json fixture in dir (lexical order) to every +// running provider on boot, bringing the emulator up to a known state. A +// missing dir is a no-op. A parse error fails startup (clear misconfiguration); +// an apply error only warns and continues, so a fixture that collides with +// already-restored state can't wedge the boot. +func applyInitDir(ctx context.Context, dir string, targets map[string]seed.Target) error { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + + if err != nil { + return err + } + + names := make([]string, 0, len(entries)) + + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".json") { + names = append(names, e.Name()) + } + } + + sort.Strings(names) + + for _, name := range names { + if err := applyInitFile(ctx, filepath.Join(dir, name), name, targets); err != nil { + return err + } + } + + return nil +} + +// applyInitFile loads one fixture file and applies it to every provider. A load +// (parse) error is returned; per-provider apply errors are warned and skipped. +func applyInitFile(ctx context.Context, path, name string, targets map[string]seed.Target) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + + f, err := seed.Load(data) + if err != nil { + return fmt.Errorf("init fixture %s: %w", name, err) + } + + for prov, t := range targets { + // IgnoreExisting so a resource that already exists (from restored state or + // an earlier init file) is skipped rather than aborting the rest of the + // fixture; other errors still warn. + if err := seed.Apply(ctx, f, t, seed.IgnoreExisting()); err != nil { + fmt.Fprintf(os.Stderr, "warning: init %s on %s: %v\n", name, prov, err) + } + } + + return nil +} + // snapshotState exports every running provider's state and writes the snapshot // file. Called after Shutdown, so the providers are quiescent. func snapshotState(ctx context.Context, path string, includeAssets bool, targets map[string]seed.Target) error { diff --git a/docs/standalone-server.md b/docs/standalone-server.md index efa4ee28..c4e9f73a 100644 --- a/docs/standalone-server.md +++ b/docs/standalone-server.md @@ -155,6 +155,30 @@ discarded. If a restore fails partway the running state is already cleared — fine for a local emulator, but don't point `load` at a server whose current state you haven't snapshotted. +### Init hooks (auto-seed on boot) + +Drop `*.json` seed fixtures in an init directory and they're applied on every +startup, so the emulator comes up in a known state without manual seeding: + +```sh +mkdir -p ~/.cloudemu/init.d +echo '{"buckets":[{"name":"app-data"}],"tables":[{"name":"users","partitionKey":"id"}]}' \ + > ~/.cloudemu/init.d/01-baseline.json +cloudemu start # applies init.d automatically +``` + +`start` auto-loads `/init.d` when it exists (point `--home` elsewhere to +change the run dir). For the foreground server, pass the directory explicitly: +`cloudemu serve --init-dir ./fixtures`. + +Files are applied in lexical order (`01-…`, `02-…`) to **every** running provider +— the fixtures are provider-agnostic, so one file seeds S3, Blob, and GCS alike. +A malformed fixture fails startup; an apply error (e.g. a resource that already +exists from restored persistence) logs a warning and boot continues. Fixtures use +the same schema as [`/_cloudemu/seed`](#resetting-state-between-tests-_cloudemu) +(buckets, tables, secrets, instances). Running setup **scripts** on boot is a +planned follow-up. + ## Ports | Provider | Default | Protocol | Notes | diff --git a/seed/seed.go b/seed/seed.go index 6df7cad4..521ad689 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -14,6 +14,7 @@ import ( "fmt" "io/fs" + cerrors "github.com/stackshy/cloudemu/v2/errors" computedriver "github.com/stackshy/cloudemu/v2/services/compute/driver" dbdriver "github.com/stackshy/cloudemu/v2/services/database/driver" secretsdriver "github.com/stackshy/cloudemu/v2/services/secrets/driver" @@ -163,32 +164,63 @@ func (f Fixtures) Validate(t Target) error { return nil } +// Option configures Apply. +type Option func(*applyOptions) + +type applyOptions struct{ ignoreExisting bool } + +// IgnoreExisting makes Apply skip a resource that already exists (an +// AlreadyExists error) and continue with the rest of the fixture, instead of +// failing at the first collision. Use it for boot-time init, where a fixture may +// overlap resources already present from restored state or an earlier file — so +// a duplicate skips just that resource rather than truncating everything after +// it in the fixture. +func IgnoreExisting() Option { + return func(o *applyOptions) { o.ignoreExisting = true } +} + // Apply validates the whole fixture set, then writes it through t's drivers in // a fixed order (buckets, tables, secrets, instances). Validation runs first so // an invalid fixture is rejected before anything is created. Writes are not // transactional: on a mid-write failure (e.g. seeding a backend that isn't -// empty), earlier resources remain — reset and retry against a fresh backend. -func Apply(ctx context.Context, f Fixtures, t Target) error { +// empty), earlier resources remain — reset and retry against a fresh backend, +// or pass IgnoreExisting to tolerate resources that already exist. +// +//nolint:gocritic // hugeParam: Fixtures is passed by value to keep the stable public Apply signature. +func Apply(ctx context.Context, f Fixtures, t Target, opts ...Option) error { + var o applyOptions + for _, opt := range opts { + opt(&o) + } + if err := f.Validate(t); err != nil { return err } - if err := applyBuckets(ctx, f.Buckets, t.Storage); err != nil { + + if err := applyBuckets(ctx, f.Buckets, t.Storage, o.ignoreExisting); err != nil { return err } - if err := applyTables(ctx, f.Tables, t.Database); err != nil { + + if err := applyTables(ctx, f.Tables, t.Database, o.ignoreExisting); err != nil { return err } - if err := applySecrets(ctx, f.Secrets, t.Secrets); err != nil { + + if err := applySecrets(ctx, f.Secrets, t.Secrets, o.ignoreExisting); err != nil { return err } - return applyInstances(ctx, f.Instances, t.Compute) + + return applyInstances(ctx, f.Instances, t.Compute, o.ignoreExisting) } -func applyBuckets(ctx context.Context, buckets []Bucket, d storagedriver.Bucket) error { +func applyBuckets(ctx context.Context, buckets []Bucket, d storagedriver.Bucket, ignoreExisting bool) error { for _, b := range buckets { if err := d.CreateBucket(ctx, b.Name); err != nil { - return fmt.Errorf("seed bucket %q: %w", b.Name, err) + if !ignoreExisting || !cerrors.IsAlreadyExists(err) { + return fmt.Errorf("seed bucket %q: %w", b.Name, err) + } + // Bucket already exists — still (re)put its declared objects below. } + for _, o := range b.Objects { ct := o.ContentType if ct == "" { @@ -202,15 +234,19 @@ func applyBuckets(ctx context.Context, buckets []Bucket, d storagedriver.Bucket) return nil } -func applyTables(ctx context.Context, tables []Table, d dbdriver.Database) error { +func applyTables(ctx context.Context, tables []Table, d dbdriver.Database, ignoreExisting bool) error { for _, tb := range tables { if err := d.CreateTable(ctx, dbdriver.TableConfig{ Name: tb.Name, PartitionKey: tb.PartitionKey, SortKey: tb.SortKey, }); err != nil { - return fmt.Errorf("seed table %q: %w", tb.Name, err) + if !ignoreExisting || !cerrors.IsAlreadyExists(err) { + return fmt.Errorf("seed table %q: %w", tb.Name, err) + } + // Table already exists — still (re)put its declared items below. } + for i, item := range tb.Items { if err := d.PutItem(ctx, tb.Name, item); err != nil { return fmt.Errorf("seed table %q item %d: %w", tb.Name, i, err) @@ -220,19 +256,21 @@ func applyTables(ctx context.Context, tables []Table, d dbdriver.Database) error return nil } -func applySecrets(ctx context.Context, secrets []Secret, d secretsdriver.Secrets) error { +func applySecrets(ctx context.Context, secrets []Secret, d secretsdriver.Secrets, ignoreExisting bool) error { for _, s := range secrets { if _, err := d.CreateSecret(ctx, secretsdriver.SecretConfig{ Name: s.Name, Description: s.Description, }, []byte(s.Value)); err != nil { - return fmt.Errorf("seed secret %q: %w", s.Name, err) + if !ignoreExisting || !cerrors.IsAlreadyExists(err) { + return fmt.Errorf("seed secret %q: %w", s.Name, err) + } } } return nil } -func applyInstances(ctx context.Context, instances []Instance, d computedriver.Compute) error { +func applyInstances(ctx context.Context, instances []Instance, d computedriver.Compute, ignoreExisting bool) error { for _, in := range instances { count := in.Count if count < 1 { @@ -243,7 +281,9 @@ func applyInstances(ctx context.Context, instances []Instance, d computedriver.C cfg.Tags = map[string]string{"Name": in.Name} } if _, err := d.RunInstances(ctx, cfg, count); err != nil { - return fmt.Errorf("seed instance (%s): %w", in.ImageID, err) + if !ignoreExisting || !cerrors.IsAlreadyExists(err) { + return fmt.Errorf("seed instance (%s): %w", in.ImageID, err) + } } } return nil