Publish immutable config snapshots; atomic settings persistence, timing validation, payment/build-retry fixes

CLAUDE.md

@@ -539,9 +539,20 @@ Key config sections:
 
 ### Settings Service & State Persistence (`--state-db`)
 
-The **Settings Service** (`pkg/settings`) is the central authority for all *mutable*
-runtime config. It owns the single effective `config.Config` that every module reads
-and is the only writer. Setting values resolve from three layers:
+The **Settings Service** (`config.Service` in `pkg/config/settings.go`) is the
+central authority for all *mutable* runtime config and the dependency modules
+hold instead of a raw `*config.Config`. It publishes **immutable config
+snapshots**: `Current()` returns the latest generation, which never changes
+underneath its reader; every applied change builds a fresh `Config` (shallow
+copy — all fields are values) and swaps it in atomically. Consumers load
+exactly ONE snapshot per operation (HTTP request, scheduler tick, build,
+reconcile pass) and thread it down the call stack. The greppable rule:
+`*config.Service` may live in struct fields; `*config.Config` (and section
+pointers) appear only as function parameters and locals — storing a snapshot
+freezes that consumer on a stale generation. One-shot commands and tests wrap
+a fixed config via `config.NewStaticService(cfg)` (serves the same pointer
+every call, so single-threaded tests may still mutate it between operations).
+Setting values resolve from three layers:
 
 ```
 hardcoded defaults  <  CLI-supplied (flag/env/config)  <  UI override
@@ -555,12 +566,14 @@ CLI vs UI is resolved by **recency** (a monotonic seq), not fixed priority:
   form the CLI layer, so bumping a *hardcoded default* in a new release never
   clobbers a UI override.
 
-The mutable-setting registry lives in `pkg/settings/fields.go` (keys in `keys.go`);
-the per-module enable flags (`epbs_enabled`, `builder_api_enabled`,
-`lifecycle_enabled`) are ordinary settings too. Write handlers call
-`settingsSvc.SetMany`, which mutates the shared config in place, persists overrides,
-and fires `OnChange` callbacks (registered in `cmd/run.go`) that trigger module
-resets (`builder.UpdateConfig`, `epbs.UpdateConfig`) and `SetEnabled` syncs.
+The mutable-setting registry lives in `pkg/config/settings_fields.go` (keys in
+`settings_keys.go`); the per-module enable flags (`epbs_enabled`,
+`builder_api_enabled`, `lifecycle_enabled`) are ordinary settings too. Write
+handlers call `settingsSvc.SetMany`, which publishes the new snapshot
+generation (a `SetMany` batch is atomic: readers see all of it or none of it),
+persists overrides, and fires `OnChange` callbacks (registered in
+`cmd/run.go`) that trigger module resets (plan-service schedule accounting)
+and `SetEnabled` syncs from a fresh `Current()` snapshot.
 
 The optional **state-db** (`pkg/db`, mirrors spamoor: `glebarez/go-sqlite` + `sqlx`
 + goose migrations) persists across restarts when `--state-db <path>` is set:
@@ -593,7 +606,7 @@ numbered step comments there match this list 1:1):
 3. Initialize BLS signer
 4. Initialize RPC client and wallet (if lifecycle available)
 5. Fetch chain spec & genesis (wait for the beacon node), apply slot-time timing defaults
-6. Open the state-db (`--state-db`) and initialize the central Settings Service (applies persisted overrides into `cfg` in place before any module reads it)
+6. Open the state-db (`--state-db`) and initialize the central Settings Service (its first published snapshot already includes persisted overrides; the builder key registry is constructed right after it, since the fleet targets are mutable settings)
 7. Start chain service
 7b. Start the action plan service (the per-slot scheduling authority; persisted via the `kv_store` `slot_plans` namespace; a mandatory constructor dependency of every action module below)
 8. Initialize lifecycle manager (if prerequisites available)

cmd/deposit.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/ethpandaops/buildoor/pkg/chain"
+	"github.com/ethpandaops/buildoor/pkg/config"
 	"github.com/ethpandaops/buildoor/pkg/lifecycle"
 	"github.com/ethpandaops/buildoor/pkg/rpc/beacon"
 	"github.com/ethpandaops/buildoor/pkg/rpc/execution"
@@ -52,8 +53,12 @@ var depositCmd = &cobra.Command{
 		}
 		defer rpcClient.Close()
 
+		// One-shot command: wrap the resolved operator config in a static
+		// settings service (no UI overrides, no persistence).
+		cfgSvc := config.NewStaticService(cfg)
+
 		// Initialize the managed builder key set and select the key to deposit for
-		registry, err := newKeyRegistry(cfg, logger)
+		registry, err := newKeyRegistry(cfgSvc, logger)
 		if err != nil {
 			return err
 		}
@@ -91,7 +96,7 @@ var depositCmd = &cobra.Command{
 		}
 
 		// Initialize chain service
-		chainSvc := chain.NewService(cfg, clClient, chainSpec, genesis, logger)
+		chainSvc := chain.NewService(cfgSvc, clClient, chainSpec, genesis, logger)
 		if err := chainSvc.Start(ctx); err != nil {
 			return fmt.Errorf("failed to start chain service: %w", err)
 		}
@@ -116,7 +121,7 @@ var depositCmd = &cobra.Command{
 		timeout, _ := cmd.Flags().GetDuration("timeout")
 
 		// Initialize lifecycle manager
-		lifecycleMgr, err := lifecycle.NewManager(cfg, clClient, chainSvc, registry, w, logger)
+		lifecycleMgr, err := lifecycle.NewManager(cfgSvc, clClient, chainSvc, registry, w, logger)
 		if err != nil {
 			return fmt.Errorf("failed to initialize lifecycle manager: %w", err)
 		}

cmd/exit.go

@@ -7,6 +7,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/ethpandaops/buildoor/pkg/chain"
+	"github.com/ethpandaops/buildoor/pkg/config"
 	"github.com/ethpandaops/buildoor/pkg/lifecycle"
 	"github.com/ethpandaops/buildoor/pkg/rpc/beacon"
 	"github.com/ethpandaops/buildoor/pkg/rpc/execution"
@@ -52,7 +53,11 @@ var exitCmd = &cobra.Command{
 		defer rpcClient.Close()
 
 		// Initialize the managed builder key set and select the key to exit
-		registry, err := newKeyRegistry(cfg, logger)
+		// One-shot command: wrap the resolved operator config in a static
+		// settings service (no UI overrides, no persistence).
+		cfgSvc := config.NewStaticService(cfg)
+
+		registry, err := newKeyRegistry(cfgSvc, logger)
 		if err != nil {
 			return err
 		}
@@ -90,7 +95,7 @@ var exitCmd = &cobra.Command{
 		}
 
 		// Initialize chain service
-		chainSvc := chain.NewService(cfg, clClient, chainSpec, genesis, logger)
+		chainSvc := chain.NewService(cfgSvc, clClient, chainSpec, genesis, logger)
 		if err := chainSvc.Start(ctx); err != nil {
 			return fmt.Errorf("failed to start chain service: %w", err)
 		}

cmd/keys.go

@@ -13,13 +13,15 @@ import (
 // newKeyRegistry builds the managed builder key set from the configured entry
 // key (raw private key or mnemonic + account index). Internal key 0 is the entry
 // key itself, so a single-key deployment keeps its identity.
-func newKeyRegistry(cfg *config.Config, log logrus.FieldLogger) (*builder_keys.Registry, error) {
+func newKeyRegistry(cfgSvc *config.Service, log logrus.FieldLogger) (*builder_keys.Registry, error) {
+	cfg := cfgSvc.Current()
+
 	entryPrivkey, err := signer.ResolveEntryPrivkey(cfg.BuilderPrivkey, cfg.BuilderMnemonic, cfg.BuilderKeyIndex)
 	if err != nil {
 		return nil, fmt.Errorf("invalid builder key: %w", err)
 	}
 
-	registry, err := builder_keys.NewRegistry(cfg, entryPrivkey, log)
+	registry, err := builder_keys.NewRegistry(cfgSvc, entryPrivkey, log)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize builder key registry: %w", err)
 	}

cmd/run.go

@@ -83,16 +83,6 @@ and begins building blocks according to configuration.`,
 			return fmt.Errorf("failed to connect to EL engine API: %w", err)
 		}
 
-		// 3. Initialize the managed builder key set (derived from the raw hex key
-		// or the mnemonic; internal key 0 is that entry key itself)
-		keyRegistry, err := newKeyRegistry(cfg, logger)
-		if err != nil {
-			return err
-		}
-
-		pubkey := keyRegistry.Primary().Pubkey()
-		logger.WithField("pubkey", fmt.Sprintf("%x", pubkey[:8])).Info("Builder key loaded")
-
 		// 4. Initialize RPC client and wallet (if lifecycle enabled)
 		var rpcClient *execution.Client
 
@@ -178,6 +168,10 @@ and begins building blocks according to configuration.`,
 		slotTimeMs := chainSpec.SecondsPerSlot.Milliseconds()
 		cfg.ApplySlotDefaults(slotTimeMs)
 
+		if err := config.ValidateTimingBounds(cfg, chainSpec.SecondsPerSlot); err != nil {
+			return fmt.Errorf("invalid timing configuration: %w", err)
+		}
+
 		logger.WithFields(logrus.Fields{
 			"slot_time_ms":       slotTimeMs,
 			"build_start_time":   cfg.EPBS.BuildStartTime,
@@ -187,9 +181,12 @@ and begins building blocks according to configuration.`,
 		}).Info("Timing defaults applied")
 
 		// 6. Open the optional state-db and build the central settings service.
-		// The settings service applies persisted UI overrides (and detects CLI
-		// changes) into cfg in place BEFORE any service reads it, so every
-		// module starts from the effective configuration.
+		// The settings service publishes immutable config snapshots: persisted
+		// UI overrides (and detected CLI changes) are part of the first
+		// generation, so every module starts from the effective configuration.
+		// Modules receive the service itself and load one snapshot per
+		// operation; cfg below is rebound to the initial snapshot for the
+		// remaining startup reads.
 		stateDB := db.NewDatabase(&db.Config{File: cfg.StateDBPath}, logger)
 		if err := stateDB.Init(); err != nil {
 			return fmt.Errorf("failed to init state-db: %w", err)
@@ -205,13 +202,26 @@ and begins building blocks according to configuration.`,
 			supplied[f.Key] = v.IsSet(f.FlagKey)
 		}
 
-		settingsSvc, err := config.NewService(cfg, defaults, supplied, stateDB, logger)
+		settingsSvc, err := config.NewService(cfg, defaults, supplied, chainSpec.SecondsPerSlot, stateDB, logger)
 		if err != nil {
 			return fmt.Errorf("failed to init settings service: %w", err)
 		}
 
+		cfg = settingsSvc.Current()
+
+		// 6b. Initialize the managed builder key set (derived from the raw hex
+		// key or the mnemonic; internal key 0 is that entry key itself). Needs
+		// the settings service: the fleet targets are mutable settings.
+		keyRegistry, err := newKeyRegistry(settingsSvc, logger)
+		if err != nil {
+			return err
+		}
+
+		pubkey := keyRegistry.Primary().Pubkey()
+		logger.WithField("pubkey", fmt.Sprintf("%x", pubkey[:8])).Info("Builder key loaded")
+
 		// 7. Start chain service (epoch-level state management)
-		chainSvc := chain.NewService(cfg, clClient, chainSpec, genesis, logger)
+		chainSvc := chain.NewService(settingsSvc, clClient, chainSpec, genesis, logger)
 		if err := chainSvc.Start(ctx); err != nil {
 			return fmt.Errorf("failed to start chain service: %w", err)
 		}
@@ -228,7 +238,7 @@ and begins building blocks according to configuration.`,
 		// 7b. Initialize the per-slot action plan service. Decision points
 		// (build/bid/serve/reveal) freeze the slot's plan on first use; plans
 		// persist in the state-db's kv_store when --state-db is set.
-		planSvc := action_plan.NewPlanService(cfg, chainSvc, logger)
+		planSvc := action_plan.NewPlanService(settingsSvc, chainSvc, logger)
 		planSvc.SetPersistence(ctx, stateDB)
 
 		if err := planSvc.Start(ctx); err != nil {
@@ -242,7 +252,7 @@ and begins building blocks according to configuration.`,
 		var lifecycleMgr *lifecycle.Manager
 
 		if lifecycleAvailable {
-			lifecycleMgr, err = lifecycle.NewManager(cfg, clClient, chainSvc, keyRegistry, w, logger)
+			lifecycleMgr, err = lifecycle.NewManager(settingsSvc, clClient, chainSvc, keyRegistry, w, logger)
 			if err != nil {
 				return fmt.Errorf("failed to initialize lifecycle: %w", err)
 			}
@@ -278,7 +288,7 @@ and begins building blocks according to configuration.`,
 			defer validatorStore.Stop()
 		}
 
-		builderSvc, err := payload_builder.NewService(cfg, clClient, chainSvc, planSvc, engineClient, feeRecipient, logger)
+		builderSvc, err := payload_builder.NewService(settingsSvc, clClient, chainSvc, planSvc, engineClient, feeRecipient, logger)
 		if err != nil {
 			return fmt.Errorf("failed to initialize builder: %w", err)
 		}
@@ -311,7 +321,7 @@ and begins building blocks according to configuration.`,
 			// for payments settled since the last epoch snapshot.
 			keyRegistry.SetBalanceAdjuster(paymentTracker)
 
-			revealSvc = payload_bidder.NewRevealService(cfg, keyRegistry,
+			revealSvc = payload_bidder.NewRevealService(settingsSvc, keyRegistry,
 				clClient, chainSvc, builderSvc, paymentTracker, planSvc,
 				chainSvc.GetHeadVoteTracker(), logger)
 			if err := revealSvc.Start(ctx); err != nil {
@@ -353,7 +363,7 @@ and begins building blocks according to configuration.`,
 			gloasForkEpoch := chainSpec.GetForkEpoch(version.DataVersionGloas)
 			logger.WithField("gloas_fork_epoch", gloasForkEpoch).Info("Initializing p2p bidder service...")
 
-			epbsSvc, err = p2p_bidder.NewService(clClient, chainSvc, keyRegistry, propPrefSvc.GetStore(), planSvc, logger)
+			epbsSvc, err = p2p_bidder.NewService(settingsSvc, clClient, chainSvc, keyRegistry, propPrefSvc.GetStore(), planSvc, logger)
 			if err != nil {
 				return fmt.Errorf("failed to initialize p2p bidder: %w", err)
 			}
@@ -381,7 +391,7 @@ and begins building blocks according to configuration.`,
 				"genesis_validators_root": fmt.Sprintf("0x%x", genesisValidatorsRoot[:]),
 			}).Info("Using genesis parameters from beacon node")
 
-			builderAPISrv = builderapi.NewServer(&cfg.BuilderAPI, logger, chainSvc, planSvc, builderSvc.GetPayloadCache(), keyRegistry, validatorStore)
+			builderAPISrv = builderapi.NewServer(settingsSvc, logger, chainSvc, planSvc, builderSvc.GetPayloadCache(), keyRegistry, validatorStore)
 			builderAPISrv.SetCLClient(clClient)
 			builderAPISrv.SetEnabled(cfg.BuilderAPIEnabled)
 
@@ -405,7 +415,7 @@ and begins building blocks according to configuration.`,
 		// artifacts. Started before the producer services so its blocking
 		// subscriptions never miss an event; SetPersistence migrates any
 		// legacy won_blocks namespace into slot results.
-		resultTracker := slot_results.NewTracker(cfg, chainSvc, stateDB, planSvc,
+		resultTracker := slot_results.NewTracker(settingsSvc, chainSvc, stateDB, planSvc,
 			builderSvc, epbsSvc, revealSvc, inclusionTracker, keyRegistry, logger)
 		resultTracker.SetPersistence(ctx, stateDB)
 
@@ -425,28 +435,26 @@ and begins building blocks according to configuration.`,
 		valRanges.Start(ctx)
 
 		// 14. Register settings OnChange subscribers: route changes through the
-		// modules. The settings service has already mutated cfg in place; these
-		// callbacks trigger module-side resets (schedule counters, scheduler) and
-		// sync the enable flags.
+		// modules. The settings service has already published the new snapshot
+		// when these fire; they trigger module-side resets (schedule counters)
+		// and sync the enable flags from a fresh snapshot.
 		settingsSvc.OnChange(func() {
+			current := settingsSvc.Current()
+
 			// The plan service is the scheduling authority: schedule-mode
 			// changes reset its next_n accounting.
 			planSvc.UpdateConfig()
 
-			if err := builderSvc.UpdateConfig(cfg); err != nil {
-				logger.WithError(err).Warn("failed to apply builder config update")
-			}
-
 			if epbsSvc != nil {
-				epbsSvc.SetEnabled(cfg.EPBSEnabled)
+				epbsSvc.SetEnabled(current.EPBSEnabled)
 			}
 
 			if builderAPISrv != nil {
-				builderAPISrv.SetEnabled(cfg.BuilderAPIEnabled)
+				builderAPISrv.SetEnabled(current.BuilderAPIEnabled)
 			}
 
 			if lifecycleMgr != nil {
-				lifecycleMgr.SetEnabled(cfg.LifecycleEnabled)
+				lifecycleMgr.SetEnabled(current.LifecycleEnabled)
 				// A target key count change must act now, not at the next
 				// reconcile tick.
 				lifecycleMgr.Reconcile()

pkg/action_plan/service.go

@@ -47,7 +47,7 @@ type RuleChangeEvent struct {
 // that fill the slots it does not cover, their freeze state and their
 // persistence. It is the single writer; all reads return deep copies.
 type PlanService struct {
-	cfg      *config.Config
+	cfgSvc   *config.Service // settings source; one snapshot per decision
 	chainSvc chain.Service
 	store    *memstore.Store[phase0.Slot, *SlotPlan]
 	rules    *memstore.Store[string, *SlotRule]
@@ -75,9 +75,9 @@ type PlanService struct {
 // NewPlanService creates the plan service. The config pointer is the shared
 // live config; enable flags and default settings are read from it at freeze
 // time.
-func NewPlanService(cfg *config.Config, chainSvc chain.Service, log logrus.FieldLogger) *PlanService {
+func NewPlanService(cfgSvc *config.Service, chainSvc chain.Service, log logrus.FieldLogger) *PlanService {
 	return &PlanService{
-		cfg:      cfg,
+		cfgSvc:   cfgSvc,
 		chainSvc: chainSvc,
 		store:    memstore.New[phase0.Slot, *SlotPlan](),
 		rules:    memstore.New[string, *SlotRule](),
@@ -308,7 +308,7 @@ func (s *PlanService) Freeze(slot phase0.Slot) *FrozenPlan {
 	plan := s.PlanForSlot(slot)
 
 	fork := s.chainSvc.ActiveForkAtEpoch(s.chainSvc.GetEpochOfSlot(slot))
-	frozen := resolveFrozenPlan(slot, plan, s.cfg, fork, time.Now(), s.slotsBuilt)
+	frozen := resolveFrozenPlan(slot, plan, s.cfgSvc.Current(), fork, time.Now(), s.slotsBuilt)
 	s.frozen[slot] = frozen
 
 	return frozen
@@ -334,7 +334,7 @@ func (s *PlanService) UpdateConfig() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if s.cfg.Schedule.Mode == config.ScheduleModeNextN {
+	if s.cfgSvc.Current().Schedule.Mode == config.ScheduleModeNextN {
 		s.slotsBuilt = 0
 	}
 }
@@ -353,15 +353,17 @@ func (s *PlanService) GetSlotsRemaining() int {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if s.cfg.Schedule.Mode != config.ScheduleModeNextN {
+	cfg := s.cfgSvc.Current()
+
+	if cfg.Schedule.Mode != config.ScheduleModeNextN {
 		return -1
 	}
 
-	if s.slotsBuilt >= s.cfg.Schedule.NextN {
+	if s.slotsBuilt >= cfg.Schedule.NextN {
 		return 0
 	}
 
-	return int(s.cfg.Schedule.NextN - s.slotsBuilt)
+	return int(cfg.Schedule.NextN - s.slotsBuilt)
 }
 
 // IsFrozen reports whether the slot's plan has been frozen already.
@@ -588,7 +590,7 @@ func matchRule(rules []*SlotRule, slot phase0.Slot, slotsPerEpoch uint64) *SlotR
 // pruneForEpoch drops past plans outside the retention window and stale
 // freeze markers. Future plans never match the cutoff and are never pruned.
 func (s *PlanService) pruneForEpoch(epoch phase0.Epoch) {
-	retention := s.cfg.SlotResultRetentionEpochs // live read; mutable setting
+	retention := s.cfgSvc.Current().SlotResultRetentionEpochs // mutable setting; fresh snapshot per prune
 	if retention == 0 || uint64(epoch) <= retention {
 		return
 	}