From d8717177796cb24913d9dcf21709519f085a46c5 Mon Sep 17 00:00:00 2001 From: kewei Date: Thu, 23 Jul 2026 09:58:17 +0800 Subject: [PATCH 01/17] cl/forkchoice: fix skipped checkpoint state root --- .../forkchoice/fork_graph/fork_graph_disk.go | 25 +++--------- .../fork_graph/fork_graph_disk_fs.go | 10 ++--- .../forkchoice/fork_graph/fork_graph_test.go | 38 +++++++++++++++++++ 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index 065b0e5f944..d363cd3bccd 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -150,30 +150,17 @@ func NewForkGraphDisk(anchorState *state.CachingBeaconState, syncedData synced_d } anchorHeader := anchorState.LatestBlockHeader() if anchorState.Version() >= clparams.GloasVersion && anchorState.Slot() > 0 { - // GLOAS checkpoint/anchor sync fix: the first transitionSlot for this - // anchor needs to record the correct state root (computed with - // LatestBlockHeader.Root == zero) into stateRoots. Two cases arise: - // - // Fresh checkpoint sync: Root is zero per spec (process_block_header - // zeroes it). We compute HashSSZ with Root=0 (the correct value), - // fill in Root, and cache it as PreviousStateRoot. - // - // Restart from disk: a previous run already filled in Root and - // serialized the state. Root is now that same correct hash (the one - // originally computed with Root=0). HashSSZ would return a different - // (wrong) value because Root is non-zero, so we must NOT recompute; - // instead we use the stored Root directly as PreviousStateRoot. - if anchorHeader.Root == [32]byte{} { - stateHash, err := anchorState.HashSSZ() + stateHash := anchorState.PeekPreviousStateRoot() + if stateHash == (common.Hash{}) || anchorHeader.Root == (common.Hash{}) || anchorHeader.Slot < anchorState.Slot() { + stateHash, err = anchorState.HashSSZ() if err != nil { panic(err) } + } + if anchorHeader.Root == (common.Hash{}) { anchorHeader.Root = stateHash - anchorState.SetLatestBlockHeader(&anchorHeader) - anchorState.SetPreviousStateRoot(stateHash) - } else { - anchorState.SetPreviousStateRoot(anchorHeader.Root) } + anchorState.SetPreviousStateRoot(stateHash) } else { if anchorHeader.Root, err = anchorState.HashSSZ(); err != nil { panic(err) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go index f8b728db142..6eb756f07c1 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go @@ -156,13 +156,11 @@ func (f *forkGraphDisk) DumpBeaconStateOnDisk(blockRoot common.Hash, bs *state.C log.Error("failed to write ssz buffer", "err", err) return err } - // Write the authoritative state root so it can be restored on load. - // Use the stored block header's Root (set from block.StateRoot in AddChainSegment) - // rather than the state's PreviousStateRoot cache field, which can be stale if - // a concurrent block arrival modified f.currentState between GetStateAtBlockRoot - // and the copy in OnHeadStateWithBlockRoot. + // A skipped-slot state root differs from the latest block header's state root. var stateRootToWrite common.Hash - if hdr, ok := f.GetHeader(blockRoot); ok { + if bs.Version() >= clparams.GloasVersion && bs.LatestBlockHeader().Slot < bs.Slot() { + stateRootToWrite = bs.PeekPreviousStateRoot() + } else if hdr, ok := f.GetHeader(blockRoot); ok { stateRootToWrite = hdr.Root } else { // Fallback for anchor state or cases where header isn't stored yet diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go index 55cbbd11d83..2250e5bb462 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go @@ -68,6 +68,44 @@ func TestForkGraphInDisk(t *testing.T) { require.Equal(t, PreValidated, status) } +func TestNewForkGraphDiskCachesAnchorStateRoot(t *testing.T) { + for _, tc := range []struct { + name string + stateSlot uint64 + headerSlot uint64 + headerRoot common.Hash + cachedRoot common.Hash + }{ + {name: "skipped slot", stateSlot: 64, headerSlot: 63, headerRoot: common.Hash{1}}, + {name: "block slot", stateSlot: 64, headerSlot: 64}, + {name: "legacy block slot", stateSlot: 64, headerSlot: 64, headerRoot: common.Hash{1}, cachedRoot: common.Hash{1}}, + } { + t.Run(tc.name, func(t *testing.T) { + anchorState := state.New(&clparams.MainnetBeaconConfig) + anchorState.SetVersion(clparams.GloasVersion) + anchorState.SetSlot(tc.stateSlot) + header := &cltypes.BeaconBlockHeader{Slot: tc.headerSlot, Root: tc.headerRoot} + anchorState.SetLatestBlockHeader(header) + expectedStateRoot, err := anchorState.HashSSZ() + require.NoError(t, err) + if tc.cachedRoot != (common.Hash{}) { + expectedStateRoot = tc.cachedRoot + anchorState.SetPreviousStateRoot(tc.cachedRoot) + } + anchorRoot, err := anchorState.BlockRoot() + require.NoError(t, err) + + graph := NewForkGraphDisk(anchorState, nil, afero.NewMemMapFs(), beacon_router_configuration.RouterConfiguration{}).(*forkGraphDisk) + + require.Equal(t, common.Hash(expectedStateRoot), anchorState.PeekPreviousStateRoot()) + require.Equal(t, header.Root, anchorState.LatestBlockHeader().Root) + persistedState, err := graph.readBeaconStateFromDisk(anchorRoot) + require.NoError(t, err) + require.Equal(t, common.Hash(expectedStateRoot), persistedState.PeekPreviousStateRoot()) + }) + } +} + // A prune for an already-covered slot (e.g. from a concurrent lock-free drain) // must not move the lowest-available marker backward past deleted data. func TestPruneKeepsLowestAvailableBlockMonotonic(t *testing.T) { From dea442aae4acbfcd2210161dfcb7f3a55b89f8ea Mon Sep 17 00:00:00 2001 From: kewei Date: Thu, 23 Jul 2026 10:08:47 +0800 Subject: [PATCH 02/17] cl/stages: validate Gloas payloads with external EL --- cl/phase1/stages/chain_tip_sync.go | 18 ++++++++++-------- cl/phase1/stages/forward_sync.go | 8 ++++---- cl/phase1/stages/gloas_payload_test.go | 18 +++++++++--------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/cl/phase1/stages/chain_tip_sync.go b/cl/phase1/stages/chain_tip_sync.go index 02dffddf278..f3c6aeee955 100644 --- a/cl/phase1/stages/chain_tip_sync.go +++ b/cl/phase1/stages/chain_tip_sync.go @@ -60,8 +60,8 @@ func gloasEnvelopePayloadHash(envelope *cltypes.SignedExecutionPayloadEnvelope) return envelope.Message.Payload.BlockHash, true } -func canRetryGloasPayloads(cfg *Cfg) bool { - return cfg.executionClient != nil && cfg.executionClient.SupportInsertion() +func canValidateGloasPayloads(cfg *Cfg) bool { + return cfg.executionClient != nil } // waitForExecutionEngineToBeFinished checks if the execution engine is ready within a specified timeout. @@ -259,7 +259,7 @@ MainLoop: if block.Version() >= clparams.GloasVersion && len(envelopes) > 0 { parentRoot := block.Block.ParentRoot if env, ok := envelopes[common.Hash(parentRoot)]; ok { - if envErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, canRetryGloasPayloads(cfg)); envErr != nil { + if envErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, canValidateGloasPayloads(cfg)); envErr != nil { log.Debug("[chainTipSync] failed to apply parent envelope", "slot", block.Block.Slot, "err", envErr) } } @@ -296,7 +296,7 @@ func fetchAndApplyEnvelopes(ctx context.Context, cfg *Cfg, roots [][32]byte) { return } for _, env := range envelopes { - if err := cfg.forkChoice.OnExecutionPayload(ctx, env, true, canRetryGloasPayloads(cfg)); err != nil { + if err := cfg.forkChoice.OnExecutionPayload(ctx, env, true, canValidateGloasPayloads(cfg)); err != nil { log.Debug("[chainTipSync] failed to apply recovered GLOAS envelope", "beaconBlockRoot", env.Message.BeaconBlockRoot, "err", err) } } @@ -676,13 +676,15 @@ func chainTipSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) e // insertion — so it must run regardless of SupportInsertion(). recoverMissingEnvelopes(ctx, cfg) - if canRetryGloasPayloads(cfg) { + if canValidateGloasPayloads(cfg) { // [New in Gloas:EIP7732] Drain execution blocks whose CL transition succeeded // but whose EL newPayload previously returned SYNCING/ACCEPTED. drainPendingGloasPayloads(ctx, cfg) retryUnverifiedAnchorPayload(ctx, cfg) - if err := cfg.blockCollector.Flush(context.Background()); err != nil { - log.Warn("[chainTipSync] blockCollector.Flush failed (EL may still be catching up)", "err", err) + if cfg.executionClient.SupportInsertion() { + if err := cfg.blockCollector.Flush(context.Background()); err != nil { + log.Warn("[chainTipSync] blockCollector.Flush failed (EL may still be catching up)", "err", err) + } } } @@ -696,7 +698,7 @@ func chainTipSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) e if headRoot != (common.Hash{}) && !cfg.forkChoice.HasEnvelope(headRoot) { pollForEnvelope(ctx, cfg, headRoot, 2*time.Second) } - if canRetryGloasPayloads(cfg) { + if canValidateGloasPayloads(cfg) { verifyUnverifiedGloasPayloads(ctx, cfg) } // NOTE: recoverMissingEnvelopes runs unconditionally above (before diff --git a/cl/phase1/stages/forward_sync.go b/cl/phase1/stages/forward_sync.go index 9c59c469597..bdddf1f3ff3 100644 --- a/cl/phase1/stages/forward_sync.go +++ b/cl/phase1/stages/forward_sync.go @@ -210,7 +210,7 @@ func processDownloadedBlockBatches(ctx context.Context, logger log.Logger, cfg * if block.Version() >= clparams.GloasVersion { if env, ok := envelopes[blockRoot]; ok { // FULL block: update forkchoice with the envelope (updates eth2Roots, persists to disk). - if fceErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, false); fceErr != nil { + if fceErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, canValidateGloasPayloads(cfg)); fceErr != nil { logger.Warn("[Caplin] forward sync: failed to process GLOAS envelope", "slot", block.Block.Slot, "err", fceErr) } else if shouldInsert { if err = cfg.blockCollector.AddGloasBlock(block.Block, env); err != nil { @@ -515,7 +515,7 @@ func ensureAnchorEnvelopeOnce(ctx context.Context, cfg *Cfg) error { if err := cfg.forkChoice.StoreAnchorEnvelope(anchorRoot, env); err != nil { return fmt.Errorf("failed to store anchor envelope: %w", err) } - if err := validateAnchorPayloadIfLocalEL(ctx, cfg, anchorRoot, bid, env); err != nil { + if err := validateAnchorPayloadWithExecutionClient(ctx, cfg, anchorRoot, bid, env); err != nil { return err } @@ -524,8 +524,8 @@ func ensureAnchorEnvelopeOnce(ctx context.Context, cfg *Cfg) error { return nil } -func validateAnchorPayloadIfLocalEL(ctx context.Context, cfg *Cfg, anchorRoot common.Hash, bid *cltypes.ExecutionPayloadBid, env *cltypes.SignedExecutionPayloadEnvelope) error { - if !canRetryGloasPayloads(cfg) { +func validateAnchorPayloadWithExecutionClient(ctx context.Context, cfg *Cfg, anchorRoot common.Hash, bid *cltypes.ExecutionPayloadBid, env *cltypes.SignedExecutionPayloadEnvelope) error { + if !canValidateGloasPayloads(cfg) { return nil } status, err := validateAnchorPayloadWithEL(ctx, cfg, bid, env) diff --git a/cl/phase1/stages/gloas_payload_test.go b/cl/phase1/stages/gloas_payload_test.go index 46f1bd88b74..3aacb300e5c 100644 --- a/cl/phase1/stages/gloas_payload_test.go +++ b/cl/phase1/stages/gloas_payload_test.go @@ -181,31 +181,31 @@ func TestGloasPayloadHelpers(t *testing.T) { require.Equal(t, want, hash) } -func TestStandaloneExecutionClientDoesNotRunLocalGloasRetry(t *testing.T) { - require.False(t, canRetryGloasPayloads(&Cfg{})) - require.False(t, canRetryGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: false}})) - require.True(t, canRetryGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: true}})) +func TestGloasPayloadValidationRequiresExecutionClient(t *testing.T) { + require.False(t, canValidateGloasPayloads(&Cfg{})) + require.True(t, canValidateGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: false}})) + require.True(t, canValidateGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: true}})) } -func TestValidateAnchorPayloadIfLocalELFollowsSupportInsertion(t *testing.T) { +func TestValidateAnchorPayloadWithAnyExecutionClient(t *testing.T) { cfg, _, bid, env, anchorRoot := validAnchorEnvelopeFixture(t, 1) remoteEL := &testExecutionEngine{ supportInsertion: false, - payloadStatus: execution_client.PayloadStatusInvalidated, + payloadStatus: execution_client.PayloadStatusValidated, } - require.NoError(t, validateAnchorPayloadIfLocalEL(context.Background(), &Cfg{ + require.NoError(t, validateAnchorPayloadWithExecutionClient(context.Background(), &Cfg{ beaconCfg: cfg, executionClient: remoteEL, forkChoice: &forkchoice.ForkChoiceStore{}, }, anchorRoot, bid, env)) - require.Equal(t, 0, remoteEL.newPayloadCalls) + require.Equal(t, 1, remoteEL.newPayloadCalls) localEL := &testExecutionEngine{ supportInsertion: true, payloadStatus: execution_client.PayloadStatusValidated, } - require.NoError(t, validateAnchorPayloadIfLocalEL(context.Background(), &Cfg{ + require.NoError(t, validateAnchorPayloadWithExecutionClient(context.Background(), &Cfg{ beaconCfg: cfg, executionClient: localEL, forkChoice: &forkchoice.ForkChoiceStore{}, From 5a3b98709e14a68e7a4be67628e69bad1c2ae6d8 Mon Sep 17 00:00:00 2001 From: kewei Date: Thu, 23 Jul 2026 10:17:22 +0800 Subject: [PATCH 03/17] cl/execution_client: encode empty transactions as array --- .../execution_client_engine_test.go | 31 +++++++++++++++++++ execution/engineapi/engine_types/ssz.go | 1 + 2 files changed, 32 insertions(+) diff --git a/cl/phase1/execution_client/execution_client_engine_test.go b/cl/phase1/execution_client/execution_client_engine_test.go index 2af65f5e8b6..2de71c07375 100644 --- a/cl/phase1/execution_client/execution_client_engine_test.go +++ b/cl/phase1/execution_client/execution_client_engine_test.go @@ -114,6 +114,37 @@ func TestExecutionPayloadFromSSZBlock_BlockAccessListGloasOnly(t *testing.T) { } } +func TestExecutionPayloadFromSSZBlock_TransactionsAreJSONArray(t *testing.T) { + beaconCfg := clparams.MainnetBeaconConfig + tests := []struct { + name string + version clparams.StateVersion + json string + want []any + }{ + {name: "pre-Gloas empty", version: clparams.ElectraVersion, json: "[]", want: []any{}}, + {name: "pre-Gloas non-empty", version: clparams.ElectraVersion, json: `["0x0102"]`, want: []any{"0x0102"}}, + {name: "Gloas empty", version: clparams.GloasVersion, json: "[]", want: []any{}}, + {name: "Gloas non-empty", version: clparams.GloasVersion, json: `["0x0102"]`, want: []any{"0x0102"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload := cltypes.NewEth1Block(tt.version, &beaconCfg) + payload.Extra = solid.NewExtraData() + payload.Transactions = &solid.TransactionsSSZ{} + payload.Withdrawals = solid.NewStaticListSSZ[*cltypes.Withdrawal](int(beaconCfg.MaxWithdrawalsPerPayload), 44) + require.NoError(t, payload.Transactions.UnmarshalJSON([]byte(tt.json))) + + raw, err := json.Marshal(engine_types.ExecutionPayloadFromSSZBlock(payload, tt.version)) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, tt.want, decoded["transactions"]) + }) + } +} + func gloas(cfg *clparams.BeaconChainConfig, balData []byte) *cltypes.Eth1Block { block := cltypes.NewEth1Block(clparams.GloasVersion, cfg) block.Extra = solid.NewExtraData() diff --git a/execution/engineapi/engine_types/ssz.go b/execution/engineapi/engine_types/ssz.go index 3de43bfbf3d..34e92f3783d 100644 --- a/execution/engineapi/engine_types/ssz.go +++ b/execution/engineapi/engine_types/ssz.go @@ -162,6 +162,7 @@ func ExecutionPayloadFromSSZBlock(block *cltypes.Eth1Block, version clparams.Sta ExtraData: block.Extra.Bytes(), BaseFeePerGas: (*hexutil.Big)(baseFee.ToBig()), BlockHash: block.BlockHash, + Transactions: make([]hexutil.Bytes, 0, len(body.Transactions)), Withdrawals: body.Withdrawals, SSZVersion: version, } From 7b6f78afde9c5fa757778a42f496b89481da75a3 Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 27 Jul 2026 13:39:10 +0900 Subject: [PATCH 04/17] cl/services: bound payload attestation validation --- .../services/payload_attestation_service.go | 53 ++++- .../payload_attestation_service_test.go | 185 ++++++++++++++++++ 2 files changed, 234 insertions(+), 4 deletions(-) diff --git a/cl/phase1/network/services/payload_attestation_service.go b/cl/phase1/network/services/payload_attestation_service.go index 7bdaa1ed0d7..4066b4dbfe9 100644 --- a/cl/phase1/network/services/payload_attestation_service.go +++ b/cl/phase1/network/services/payload_attestation_service.go @@ -57,14 +57,19 @@ type pendingPayloadAttestationJob struct { creationTime time.Time } +type payloadAttestationValidation struct { + done chan struct{} +} + const ( // seenPayloadAttestationCacheSize: PTC has 512 validators per slot. // With clock disparity, we may see attestations for ~2 slots. // 512 * 4 = 2048 provides safety margin. - seenPayloadAttestationCacheSize = 2048 - pendingPayloadAttestationExpiry = 30 * time.Second - pendingPayloadAttestationCheckInterval = 100 * time.Millisecond - maxPendingAttestations = 2048 + seenPayloadAttestationCacheSize = 2048 + pendingPayloadAttestationExpiry = 30 * time.Second + pendingPayloadAttestationCheckInterval = 100 * time.Millisecond + maxPendingAttestations = 2048 + maxConcurrentPayloadAttestationValidations = 2 ) type payloadAttestationService struct { @@ -80,6 +85,8 @@ type payloadAttestationService struct { pendingAttestations sync.Map // pendingPayloadAttestationKey -> *pendingPayloadAttestationJob pendingCount atomic.Int32 pendingCond *sync.Cond + validationSlots chan struct{} + validationsInFlight sync.Map } // NewPayloadAttestationService creates a new payload attestation service. @@ -102,6 +109,7 @@ func NewPayloadAttestationService( emitters: emitters, seenAttestationsCache: seenCache, pendingCond: sync.NewCond(&sync.Mutex{}), + validationSlots: make(chan struct{}, maxConcurrentPayloadAttestationValidations), } go s.loop(ctx) return s @@ -167,6 +175,22 @@ func (s *payloadAttestationService) ProcessMessage(ctx context.Context, _ *uint6 return fmt.Errorf("%w: payload attestation slot %d does not match referenced block slot %d", ErrIgnore, slot, blockHeader.Slot) } + finishValidation, alreadySeen, err := s.beginValidation(ctx, seenKey) + if err != nil { + return err + } + if alreadySeen { + return fmt.Errorf("%w: already seen payload attestation from validator %d for slot %d", ErrIgnore, validatorIndex, slot) + } + defer finishValidation() + + select { + case s.validationSlots <- struct{}{}: + defer func() { <-s.validationSlots }() + case <-ctx.Done(): + return ctx.Err() + } + // Process through forkchoice which handles: // [IGNORE] block state not found // [REJECT] validator is not in PTC @@ -196,6 +220,27 @@ func (s *payloadAttestationService) ProcessMessage(ctx context.Context, _ *uint6 return nil } +func (s *payloadAttestationService) beginValidation(ctx context.Context, key seenPayloadAttestationKey) (func(), bool, error) { + for { + if s.seenAttestationsCache.Contains(key) { + return nil, true, nil + } + validation := &payloadAttestationValidation{done: make(chan struct{})} + existing, loaded := s.validationsInFlight.LoadOrStore(key, validation) + if !loaded { + return func() { + s.validationsInFlight.CompareAndDelete(key, validation) + close(validation.done) + }, false, nil + } + select { + case <-existing.(*payloadAttestationValidation).done: + case <-ctx.Done(): + return nil, false, ctx.Err() + } + } +} + // queuePendingAttestation adds an attestation to the pending queue for later processing. func (s *payloadAttestationService) queuePendingAttestation(blockRoot common.Hash, msg *cltypes.PayloadAttestationMessage) { if s.pendingCount.Add(1) > maxPendingAttestations { diff --git a/cl/phase1/network/services/payload_attestation_service_test.go b/cl/phase1/network/services/payload_attestation_service_test.go index 1b487d3bd5e..7f7eca2500e 100644 --- a/cl/phase1/network/services/payload_attestation_service_test.go +++ b/cl/phase1/network/services/payload_attestation_service_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "sync" + "sync/atomic" "testing" "time" @@ -30,11 +31,50 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/phase1/core/state/lru" + "github.com/erigontech/erigon/cl/phase1/forkchoice" "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" "github.com/erigontech/erigon/cl/utils/eth_clock" "github.com/erigontech/erigon/common" ) +type blockingPayloadAttestationForkchoice struct { + forkchoice.ForkChoiceStorage + active atomic.Int32 + max atomic.Int32 + started chan struct{} + release chan struct{} +} + +func (f *blockingPayloadAttestationForkchoice) OnPayloadAttestationMessage(*cltypes.PayloadAttestationMessage, bool) error { + active := f.active.Add(1) + defer f.active.Add(-1) + for { + maxActive := f.max.Load() + if active <= maxActive || f.max.CompareAndSwap(maxActive, active) { + break + } + } + f.started <- struct{}{} + <-f.release + return nil +} + +type retryPayloadAttestationForkchoice struct { + forkchoice.ForkChoiceStorage + calls atomic.Int32 + firstStarted chan struct{} + releaseFirst chan struct{} +} + +func (f *retryPayloadAttestationForkchoice) OnPayloadAttestationMessage(*cltypes.PayloadAttestationMessage, bool) error { + if f.calls.Add(1) == 1 { + close(f.firstStarted) + <-f.releaseFirst + return errors.New("invalid signature") + } + return nil +} + func setupPayloadAttestationService(t *testing.T, ctrl *gomock.Controller) (*payloadAttestationService, *mock_services.ForkChoiceStorageMock, *eth_clock.MockEthereumClock) { forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) ethClockMock := eth_clock.NewMockEthereumClock(ctrl) @@ -49,11 +89,156 @@ func setupPayloadAttestationService(t *testing.T, ctrl *gomock.Controller) (*pay seenAttestationsCache: seenCache, emitters: beaconevents.NewEventEmitter(), pendingCond: sync.NewCond(&sync.Mutex{}), // Needed for queuePendingAttestation + validationSlots: make(chan struct{}, maxConcurrentPayloadAttestationValidations), } return service, forkchoiceMock, ethClockMock } +func TestPayloadAttestationServiceBoundsConcurrentValidation(t *testing.T) { + const expectedMaxConcurrentValidations = 2 + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + + blockingForkchoice := &blockingPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan struct{}, 8), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(8) + + var wg sync.WaitGroup + for i := range 8 { + wg.Go(func() { + msg := newTestPayloadAttestationMessage(100, uint64(i), blockRoot) + require.NoError(t, service.ProcessMessage(context.Background(), nil, msg)) + }) + } + + for range expectedMaxConcurrentValidations + 1 { + select { + case <-blockingForkchoice.started: + case <-time.After(time.Second): + close(blockingForkchoice.release) + wg.Wait() + require.LessOrEqual(t, blockingForkchoice.max.Load(), int32(expectedMaxConcurrentValidations)) + return + } + } + close(blockingForkchoice.release) + wg.Wait() + require.LessOrEqual(t, blockingForkchoice.max.Load(), int32(expectedMaxConcurrentValidations)) +} + +func TestPayloadAttestationServiceSerializesDuplicateValidation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + + blockingForkchoice := &blockingPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan struct{}, 2), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + results := make(chan error, 2) + for range 2 { + go func() { + results <- service.ProcessMessage(context.Background(), nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + }() + } + + <-blockingForkchoice.started + select { + case <-blockingForkchoice.started: + case <-time.After(100 * time.Millisecond): + } + close(blockingForkchoice.release) + + firstErr := <-results + secondErr := <-results + require.Equal(t, 1, int(blockingForkchoice.max.Load())) + require.True(t, (firstErr == nil) != (secondErr == nil)) + if firstErr != nil { + require.ErrorIs(t, firstErr, ErrIgnore) + } else { + require.ErrorIs(t, secondErr, ErrIgnore) + } +} + +func TestPayloadAttestationServiceRetriesAfterInvalidDuplicate(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + + retryForkchoice := &retryPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + firstStarted: make(chan struct{}), + releaseFirst: make(chan struct{}), + } + service.forkchoiceStore = retryForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + firstResult := make(chan error, 1) + go func() { + firstResult <- service.ProcessMessage(context.Background(), nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + }() + <-retryForkchoice.firstStarted + + secondResult := make(chan error, 1) + go func() { + second := newTestPayloadAttestationMessage(100, 42, blockRoot) + second.Signature[0] = 1 + secondResult <- service.ProcessMessage(context.Background(), nil, second) + }() + + require.Never(t, func() bool { return retryForkchoice.calls.Load() > 1 }, 100*time.Millisecond, 10*time.Millisecond) + close(retryForkchoice.releaseFirst) + + require.ErrorContains(t, <-firstResult, "invalid signature") + require.NoError(t, <-secondResult) + require.Equal(t, int32(2), retryForkchoice.calls.Load()) +} + +func TestPayloadAttestationServiceCanceledWhileValidationIsFull(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + for range maxConcurrentPayloadAttestationValidations { + service.validationSlots <- struct{}{} + } + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := service.ProcessMessage(ctx, nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + + require.ErrorIs(t, err, context.Canceled) + inFlight := 0 + service.validationsInFlight.Range(func(_, _ any) bool { + inFlight++ + return true + }) + require.Zero(t, inFlight) +} + func newTestPayloadAttestationMessage(slot uint64, validatorIndex uint64, blockRoot common.Hash) *cltypes.PayloadAttestationMessage { return &cltypes.PayloadAttestationMessage{ ValidatorIndex: validatorIndex, From 39085370eee6fefad7887723c772b2672b73809f Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 27 Jul 2026 14:41:45 +0900 Subject: [PATCH 05/17] cl/forkchoice: share payload attestation validation context --- cl/phase1/forkchoice/forkchoice.go | 6 + .../on_payload_attestation_message.go | 58 +---- .../payload_attestation_validation.go | 176 +++++++++++++++ .../payload_attestation_validation_test.go | 205 ++++++++++++++++++ .../services/payload_attestation_service.go | 18 +- .../payload_attestation_service_test.go | 44 +--- 6 files changed, 413 insertions(+), 94 deletions(-) create mode 100644 cl/phase1/forkchoice/payload_attestation_validation.go create mode 100644 cl/phase1/forkchoice/payload_attestation_validation_test.go diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index 708fbae19ca..e988667bea2 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -176,6 +176,7 @@ type ForkChoiceStore struct { ptcVoteMu sync.Mutex // protects read-modify-write on payloadTimelinessVote and payloadDataAvailabilityVote payloadTimelinessVote sync.Map // map[common.Hash][clparams.PtcSize]int8 (0=unvoted, 1=true, -1=false) payloadDataAvailabilityVote sync.Map // map[common.Hash][clparams.PtcSize]int8 (0=unvoted, 1=true, -1=false) + payloadAttestationContexts *payloadAttestationValidationContexts // [New in Gloas:EIP7732] Block timeliness tracking. // Pre-GLOAS: stores [block_timely, false] (only index 0 is meaningful). // Post-GLOAS: stores [block_timely, payload_timely] — two independent booleans. @@ -335,6 +336,10 @@ func NewForkChoiceStore( if err != nil { return nil, err } + payloadAttestationContexts, err := newPayloadAttestationValidationContexts() + if err != nil { + return nil, err + } publicKeysRegistry.ResetAnchor(anchorState) participation.Add(state.Epoch(anchorState.BeaconState), anchorState.CurrentEpochParticipation().Copy()) @@ -392,6 +397,7 @@ func NewForkChoiceStore( executionPayloadStatus: executionPayloadStatus, payloadStatusByRoot: payloadStatusByRoot, executionPayloadGasLimit: executionPayloadGasLimit, + payloadAttestationContexts: payloadAttestationContexts, db: db, } f.justifiedCheckpoint.Store(anchorCheckpoint) diff --git a/cl/phase1/forkchoice/on_payload_attestation_message.go b/cl/phase1/forkchoice/on_payload_attestation_message.go index 3b36675881d..dfb11847d5f 100644 --- a/cl/phase1/forkchoice/on_payload_attestation_message.go +++ b/cl/phase1/forkchoice/on_payload_attestation_message.go @@ -22,8 +22,6 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" - "github.com/erigontech/erigon/cl/cltypes/solid" - "github.com/erigontech/erigon/cl/phase1/core/state" ) // OnPayloadAttestationMessage processes a payload attestation message and updates @@ -36,67 +34,35 @@ func (f *ForkChoiceStore) OnPayloadAttestationMessage( msg *cltypes.PayloadAttestationMessage, isFromBlock bool, ) error { - if msg.Data == nil { + if msg == nil || msg.Data == nil { return errors.New("nil payload attestation data") } data := msg.Data blockRoot := data.BeaconBlockRoot - blockState, err := f.GetStateAtBlockRoot(blockRoot, true) - if err != nil { - return err - } - if blockState == nil { - return fmt.Errorf("%w: block state not found for root %v", ErrIgnore, blockRoot) + if !isFromBlock { + // Wall-clock time is authoritative for gossip because store time can lag OnTick. + currentSlot := f.ethClock.GetCurrentSlot() + if data.Slot != currentSlot { + return fmt.Errorf("%w: attestation slot %d is not current slot %d", ErrIgnore, data.Slot, currentSlot) + } } - // Get the PTC for the attestation slot - ptc, err := blockState.GetPTC(data.Slot) + validationContext, err := f.payloadAttestationValidationContext(blockRoot, data.Slot) if err != nil { return err } - - // PTC votes can only change the vote for their assigned beacon block - if data.Slot != blockState.Slot() { - return fmt.Errorf("%w: attestation slot %d does not match block slot %d", ErrIgnore, data.Slot, blockState.Slot()) - } - - // [REJECT] Check that the attester is from the PTC - var ptcIndices []int - for i, idx := range ptc { - if idx == msg.ValidatorIndex { - ptcIndices = append(ptcIndices, i) - } - } - if len(ptcIndices) == 0 { - return fmt.Errorf("validator %d is not in PTC for slot %d", msg.ValidatorIndex, data.Slot) + ptcIndices, err := validationContext.ptcPositions(msg) + if err != nil { + return err } // Verify the signature and check that it's for the current slot if coming from wire if !isFromBlock { - // [IGNORE] Check that the attestation is for the current slot. - // Use ethClock.GetCurrentSlot() (wall-clock based) instead of f.Slot() - // (forkchoice-store time based) because f.Slot() depends on f.time which - // is only updated by OnTick and can be stale or uninitialized, causing - // uint64 underflow and an absurdly large slot number. - currentSlot := f.ethClock.GetCurrentSlot() - if data.Slot != currentSlot { - return fmt.Errorf("%w: attestation slot %d is not current slot %d", ErrIgnore, data.Slot, currentSlot) - } - // [REJECT] Verify the signature - indexedAttestation := &cltypes.IndexedPayloadAttestation{ - AttestingIndices: solid.NewRawUint64List(1, []uint64{msg.ValidatorIndex}), - Data: data, - Signature: msg.Signature, - } - valid, err := state.IsValidIndexedPayloadAttestation(blockState, indexedAttestation) - if err != nil { + if err := validationContext.validateSignature(msg); err != nil { return err } - if !valid { - return errors.New("invalid payload attestation signature") - } } // Atomically update PTC vote arrays under mutex to prevent concurrent diff --git a/cl/phase1/forkchoice/payload_attestation_validation.go b/cl/phase1/forkchoice/payload_attestation_validation.go new file mode 100644 index 00000000000..e5627b2ad16 --- /dev/null +++ b/cl/phase1/forkchoice/payload_attestation_validation.go @@ -0,0 +1,176 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package forkchoice + +import ( + "errors" + "fmt" + + "golang.org/x/sync/singleflight" + + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/fork" + "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/cl/phase1/core/state/lru" + "github.com/erigontech/erigon/cl/utils/bls" + "github.com/erigontech/erigon/common" +) + +const ( + payloadAttestationValidationContextCacheSize = 128 + maxConcurrentValidationContextBuilds = 1 +) + +type payloadAttestationValidationContext struct { + slot uint64 + domain common.Hash + positions map[uint64][]int + publicKeys map[uint64]common.Bytes48 +} + +type payloadAttestationValidationContexts struct { + cache *lru.Cache[common.Hash, *payloadAttestationValidationContext] + buildGroup singleflight.Group + buildSlots chan struct{} +} + +func newPayloadAttestationValidationContexts() (*payloadAttestationValidationContexts, error) { + cache, err := lru.New[common.Hash, *payloadAttestationValidationContext]( + "payload_attestation_validation_contexts", + payloadAttestationValidationContextCacheSize, + ) + if err != nil { + return nil, err + } + return &payloadAttestationValidationContexts{ + cache: cache, + buildSlots: make(chan struct{}, maxConcurrentValidationContextBuilds), + }, nil +} + +func (c *payloadAttestationValidationContexts) get( + blockRoot common.Hash, + build func() (*payloadAttestationValidationContext, error), +) (*payloadAttestationValidationContext, error) { + if validationContext, ok := c.cache.Get(blockRoot); ok { + return validationContext, nil + } + value, err, _ := c.buildGroup.Do(string(blockRoot[:]), func() (any, error) { + if validationContext, ok := c.cache.Get(blockRoot); ok { + return validationContext, nil + } + c.buildSlots <- struct{}{} + defer func() { <-c.buildSlots }() + validationContext, err := build() + if err != nil { + return nil, err + } + c.cache.Add(blockRoot, validationContext) + return validationContext, nil + }) + if err != nil { + return nil, err + } + return value.(*payloadAttestationValidationContext), nil +} + +func (f *ForkChoiceStore) payloadAttestationValidationContext( + blockRoot common.Hash, + slot uint64, +) (*payloadAttestationValidationContext, error) { + return f.payloadAttestationContexts.get(blockRoot, func() (*payloadAttestationValidationContext, error) { + blockState, err := f.GetStateAtBlockRoot(blockRoot, true) + if err != nil { + return nil, err + } + if blockState == nil { + return nil, fmt.Errorf("%w: block state not found for root %v", ErrIgnore, blockRoot) + } + if slot != blockState.Slot() { + return nil, fmt.Errorf("%w: attestation slot %d does not match block slot %d", ErrIgnore, slot, blockState.Slot()) + } + + ptc, err := blockState.GetPTC(slot) + if err != nil { + return nil, err + } + if len(ptc) != int(blockState.BeaconConfig().PtcSize) { + return nil, fmt.Errorf("invalid PTC length %d, expected %d", len(ptc), blockState.BeaconConfig().PtcSize) + } + domain, err := blockState.GetDomain(blockState.BeaconConfig().DomainPtcAttester, state.GetEpochAtSlot(blockState.BeaconConfig(), slot)) + if err != nil { + return nil, fmt.Errorf("unable to get the domain: %w", err) + } + if len(domain) != len(common.Hash{}) { + return nil, fmt.Errorf("invalid PTC attester domain length %d", len(domain)) + } + + validationContext := &payloadAttestationValidationContext{ + slot: slot, + positions: make(map[uint64][]int, len(ptc)), + publicKeys: make(map[uint64]common.Bytes48, len(ptc)), + } + copy(validationContext.domain[:], domain) + for position, validatorIndex := range ptc { + if validatorIndex >= uint64(blockState.ValidatorLength()) { + return nil, fmt.Errorf("PTC validator %d is out of range", validatorIndex) + } + validationContext.positions[validatorIndex] = append(validationContext.positions[validatorIndex], position) + if _, ok := validationContext.publicKeys[validatorIndex]; ok { + continue + } + validator, err := blockState.ValidatorForValidatorIndex(int(validatorIndex)) + if err != nil { + return nil, fmt.Errorf("failed to get PTC validator %d: %w", validatorIndex, err) + } + if len(validator.PublicKeyBytes()) != len(common.Bytes48{}) { + return nil, fmt.Errorf("invalid public key length for PTC validator %d", validatorIndex) + } + var publicKey common.Bytes48 + copy(publicKey[:], validator.PublicKeyBytes()) + validationContext.publicKeys[validatorIndex] = publicKey + } + return validationContext, nil + }) +} + +func (c *payloadAttestationValidationContext) ptcPositions(msg *cltypes.PayloadAttestationMessage) ([]int, error) { + if msg.Data.Slot != c.slot { + return nil, fmt.Errorf("%w: attestation slot %d does not match block slot %d", ErrIgnore, msg.Data.Slot, c.slot) + } + positions, ok := c.positions[msg.ValidatorIndex] + if !ok { + return nil, fmt.Errorf("validator %d is not in PTC for slot %d", msg.ValidatorIndex, msg.Data.Slot) + } + return positions, nil +} + +func (c *payloadAttestationValidationContext) validateSignature(msg *cltypes.PayloadAttestationMessage) error { + signingRoot, err := fork.ComputeSigningRoot(msg.Data, c.domain[:]) + if err != nil { + return fmt.Errorf("unable to get signing root: %w", err) + } + publicKey := c.publicKeys[msg.ValidatorIndex] + valid, err := bls.VerifyAggregate(msg.Signature[:], signingRoot[:], [][]byte{publicKey[:]}) + if err != nil { + return fmt.Errorf("error while validating signature: %w", err) + } + if !valid { + return errors.New("invalid payload attestation signature") + } + return nil +} diff --git a/cl/phase1/forkchoice/payload_attestation_validation_test.go b/cl/phase1/forkchoice/payload_attestation_validation_test.go new file mode 100644 index 00000000000..74070250554 --- /dev/null +++ b/cl/phase1/forkchoice/payload_attestation_validation_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package forkchoice + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/fork" + "github.com/erigontech/erigon/cl/utils/bls" + "github.com/erigontech/erigon/common" +) + +type payloadAttestationValidationContextResult struct { + validationContext *payloadAttestationValidationContext + err error +} + +func TestOnPayloadAttestationMessageRejectsNil(t *testing.T) { + f := &ForkChoiceStore{} + require.Error(t, f.OnPayloadAttestationMessage(nil, false)) + require.Error(t, f.OnPayloadAttestationMessage(&cltypes.PayloadAttestationMessage{}, false)) +} + +func TestPayloadAttestationValidationContextsCollapseConcurrentBuilds(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + root := common.HexToHash("0x1234") + expected := &payloadAttestationValidationContext{slot: 100} + started := make(chan struct{}) + release := make(chan struct{}) + var builds atomic.Int32 + + build := func() (*payloadAttestationValidationContext, error) { + if builds.Add(1) == 1 { + close(started) + } + <-release + return expected, nil + } + + results := make(chan payloadAttestationValidationContextResult, 16) + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + validationContext, getErr := contexts.get(root, build) + results <- payloadAttestationValidationContextResult{validationContext, getErr} + }) + } + <-started + require.Equal(t, int32(1), builds.Load()) + close(release) + wg.Wait() + close(results) + + for result := range results { + require.NoError(t, result.err) + require.Same(t, expected, result.validationContext) + } + _, err = contexts.get(root, build) + require.NoError(t, err) + require.Equal(t, int32(1), builds.Load()) +} + +func TestPayloadAttestationValidationContextsDoNotCacheBuildErrors(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + root := common.HexToHash("0x1234") + var builds atomic.Int32 + _, err = contexts.get(root, func() (*payloadAttestationValidationContext, error) { + builds.Add(1) + return nil, errors.New("state unavailable") + }) + require.ErrorContains(t, err, "state unavailable") + + expected := &payloadAttestationValidationContext{slot: 100} + actual, err := contexts.get(root, func() (*payloadAttestationValidationContext, error) { + builds.Add(1) + return expected, nil + }) + require.NoError(t, err) + require.Same(t, expected, actual) + require.Equal(t, int32(2), builds.Load()) +} + +func TestPayloadAttestationValidationContextsBoundDifferentRootBuilds(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + var active atomic.Int32 + var maxActive atomic.Int32 + started := make(chan struct{}, 3) + release := make(chan struct{}) + results := make(chan error, 3) + for i := range 3 { + go func() { + _, getErr := contexts.get(common.Hash{byte(i + 1)}, func() (*payloadAttestationValidationContext, error) { + current := active.Add(1) + defer active.Add(-1) + for { + maximum := maxActive.Load() + if current <= maximum || maxActive.CompareAndSwap(maximum, current) { + break + } + } + started <- struct{}{} + <-release + return &payloadAttestationValidationContext{}, nil + }) + results <- getErr + }() + } + + for range maxConcurrentValidationContextBuilds { + select { + case <-started: + case <-time.After(time.Second): + require.FailNow(t, "validation context build did not start") + } + } + select { + case <-started: + require.FailNow(t, "too many validation contexts built concurrently") + case <-time.After(100 * time.Millisecond): + } + close(release) + for range 3 { + require.NoError(t, <-results) + } + require.Equal(t, int32(maxConcurrentValidationContextBuilds), maxActive.Load()) +} + +func TestPayloadAttestationValidationContextPositions(t *testing.T) { + validationContext := &payloadAttestationValidationContext{ + slot: 100, + positions: map[uint64][]int{42: {1, 7}}, + } + msg := &cltypes.PayloadAttestationMessage{ + ValidatorIndex: 42, + Data: &cltypes.PayloadAttestationData{Slot: 100}, + } + + positions, err := validationContext.ptcPositions(msg) + require.NoError(t, err) + require.Equal(t, []int{1, 7}, positions) + + msg.Data.Slot = 99 + _, err = validationContext.ptcPositions(msg) + require.ErrorIs(t, err, ErrIgnore) +} + +func TestPayloadAttestationValidationContextSignature(t *testing.T) { + privateKey, err := bls.GenerateKey() + require.NoError(t, err) + + data := &cltypes.PayloadAttestationData{ + BeaconBlockRoot: common.HexToHash("0x1234"), + Slot: 100, + PayloadPresent: true, + BlobDataAvailable: true, + } + domain := common.HexToHash("0xabcd") + signingRoot, err := fork.ComputeSigningRoot(data, domain[:]) + require.NoError(t, err) + + var publicKey common.Bytes48 + copy(publicKey[:], bls.CompressPublicKey(privateKey.PublicKey())) + msg := &cltypes.PayloadAttestationMessage{ + ValidatorIndex: 42, + Data: data, + } + copy(msg.Signature[:], privateKey.Sign(signingRoot[:]).Bytes()) + validationContext := &payloadAttestationValidationContext{ + slot: 100, + domain: domain, + positions: map[uint64][]int{42: {1}}, + publicKeys: map[uint64]common.Bytes48{42: publicKey}, + } + + require.NoError(t, validationContext.validateSignature(msg)) + msg.Data.PayloadPresent = false + require.ErrorContains(t, validationContext.validateSignature(msg), "invalid payload attestation signature") +} diff --git a/cl/phase1/network/services/payload_attestation_service.go b/cl/phase1/network/services/payload_attestation_service.go index 4066b4dbfe9..962f6e9d9ee 100644 --- a/cl/phase1/network/services/payload_attestation_service.go +++ b/cl/phase1/network/services/payload_attestation_service.go @@ -65,11 +65,10 @@ const ( // seenPayloadAttestationCacheSize: PTC has 512 validators per slot. // With clock disparity, we may see attestations for ~2 slots. // 512 * 4 = 2048 provides safety margin. - seenPayloadAttestationCacheSize = 2048 - pendingPayloadAttestationExpiry = 30 * time.Second - pendingPayloadAttestationCheckInterval = 100 * time.Millisecond - maxPendingAttestations = 2048 - maxConcurrentPayloadAttestationValidations = 2 + seenPayloadAttestationCacheSize = 2048 + pendingPayloadAttestationExpiry = 30 * time.Second + pendingPayloadAttestationCheckInterval = 100 * time.Millisecond + maxPendingAttestations = 2048 ) type payloadAttestationService struct { @@ -85,7 +84,6 @@ type payloadAttestationService struct { pendingAttestations sync.Map // pendingPayloadAttestationKey -> *pendingPayloadAttestationJob pendingCount atomic.Int32 pendingCond *sync.Cond - validationSlots chan struct{} validationsInFlight sync.Map } @@ -109,7 +107,6 @@ func NewPayloadAttestationService( emitters: emitters, seenAttestationsCache: seenCache, pendingCond: sync.NewCond(&sync.Mutex{}), - validationSlots: make(chan struct{}, maxConcurrentPayloadAttestationValidations), } go s.loop(ctx) return s @@ -184,13 +181,6 @@ func (s *payloadAttestationService) ProcessMessage(ctx context.Context, _ *uint6 } defer finishValidation() - select { - case s.validationSlots <- struct{}{}: - defer func() { <-s.validationSlots }() - case <-ctx.Done(): - return ctx.Err() - } - // Process through forkchoice which handles: // [IGNORE] block state not found // [REJECT] validator is not in PTC diff --git a/cl/phase1/network/services/payload_attestation_service_test.go b/cl/phase1/network/services/payload_attestation_service_test.go index 7f7eca2500e..402f756f36e 100644 --- a/cl/phase1/network/services/payload_attestation_service_test.go +++ b/cl/phase1/network/services/payload_attestation_service_test.go @@ -89,15 +89,12 @@ func setupPayloadAttestationService(t *testing.T, ctrl *gomock.Controller) (*pay seenAttestationsCache: seenCache, emitters: beaconevents.NewEventEmitter(), pendingCond: sync.NewCond(&sync.Mutex{}), // Needed for queuePendingAttestation - validationSlots: make(chan struct{}, maxConcurrentPayloadAttestationValidations), } return service, forkchoiceMock, ethClockMock } -func TestPayloadAttestationServiceBoundsConcurrentValidation(t *testing.T) { - const expectedMaxConcurrentValidations = 2 - +func TestPayloadAttestationServiceAllowsConcurrentValidationForDifferentValidators(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -114,26 +111,30 @@ func TestPayloadAttestationServiceBoundsConcurrentValidation(t *testing.T) { ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(8) var wg sync.WaitGroup + results := make(chan error, 8) for i := range 8 { wg.Go(func() { msg := newTestPayloadAttestationMessage(100, uint64(i), blockRoot) - require.NoError(t, service.ProcessMessage(context.Background(), nil, msg)) + results <- service.ProcessMessage(context.Background(), nil, msg) }) } - for range expectedMaxConcurrentValidations + 1 { + for range 8 { select { case <-blockingForkchoice.started: case <-time.After(time.Second): close(blockingForkchoice.release) wg.Wait() - require.LessOrEqual(t, blockingForkchoice.max.Load(), int32(expectedMaxConcurrentValidations)) - return + require.FailNow(t, "validation was throttled") } } close(blockingForkchoice.release) wg.Wait() - require.LessOrEqual(t, blockingForkchoice.max.Load(), int32(expectedMaxConcurrentValidations)) + close(results) + for err := range results { + require.NoError(t, err) + } + require.Equal(t, int32(8), blockingForkchoice.max.Load()) } func TestPayloadAttestationServiceSerializesDuplicateValidation(t *testing.T) { @@ -214,31 +215,6 @@ func TestPayloadAttestationServiceRetriesAfterInvalidDuplicate(t *testing.T) { require.Equal(t, int32(2), retryForkchoice.calls.Load()) } -func TestPayloadAttestationServiceCanceledWhileValidationIsFull(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) - blockRoot := common.HexToHash("0x1234") - fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} - for range maxConcurrentPayloadAttestationValidations { - service.validationSlots <- struct{}{} - } - ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - err := service.ProcessMessage(ctx, nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) - - require.ErrorIs(t, err, context.Canceled) - inFlight := 0 - service.validationsInFlight.Range(func(_, _ any) bool { - inFlight++ - return true - }) - require.Zero(t, inFlight) -} - func newTestPayloadAttestationMessage(slot uint64, validatorIndex uint64, blockRoot common.Hash) *cltypes.PayloadAttestationMessage { return &cltypes.PayloadAttestationMessage{ ValidatorIndex: validatorIndex, From 7db3a04f04962df4fbf7ee2662096ea0186b7c8f Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 27 Jul 2026 15:55:46 +0900 Subject: [PATCH 06/17] cl/services: process aggregate votes on observer nodes --- cl/phase1/network/services/aggregate_and_proof_service.go | 6 +++++- .../network/services/aggregate_and_proof_service_test.go | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cl/phase1/network/services/aggregate_and_proof_service.go b/cl/phase1/network/services/aggregate_and_proof_service.go index 4431aebfbb8..6b19f7a3a6f 100644 --- a/cl/phase1/network/services/aggregate_and_proof_service.go +++ b/cl/phase1/network/services/aggregate_and_proof_service.go @@ -340,7 +340,7 @@ func (a *aggregateAndProofServiceImpl) ProcessMessage( localValidatorIsProposer = a.isLocalValidatorProposer(headState, currentEpoch, localValidators) } - if localValidatorIsProposer || aggregateAndProof.ImmediateProcess { + if shouldVerifyAggregate(localValidators, localValidatorIsProposer, aggregateAndProof.ImmediateProcess) { // Set beacon config on the aggregate's attestation so HashSSZ uses the correct // AggregationBits limit for the active preset (e.g. minimal: 8192, mainnet: 131072). aggregateAndProof.SignedAggregateAndProof.Message.Aggregate.SetBeaconConfig(a.beaconCfg) @@ -384,6 +384,10 @@ func (a *aggregateAndProofServiceImpl) ProcessMessage( return nil } +func shouldVerifyAggregate(localValidators []uint64, localValidatorIsProposer, immediateProcess bool) bool { + return len(localValidators) == 0 || localValidatorIsProposer || immediateProcess +} + func GetSignaturesOnAggregate( s *state.CachingBeaconState, aggregateAndProof *cltypes.SignedAggregateAndProof, diff --git a/cl/phase1/network/services/aggregate_and_proof_service_test.go b/cl/phase1/network/services/aggregate_and_proof_service_test.go index 4a8a29c80c9..409a259dd11 100644 --- a/cl/phase1/network/services/aggregate_and_proof_service_test.go +++ b/cl/phase1/network/services/aggregate_and_proof_service_test.go @@ -305,6 +305,13 @@ func TestAggregateAndProofSuccess(t *testing.T) { require.NoError(t, aggService.ProcessMessage(context.Background(), nil, agg)) } +func TestShouldVerifyAggregateForObserver(t *testing.T) { + require.True(t, shouldVerifyAggregate(nil, false, false)) + require.False(t, shouldVerifyAggregate([]uint64{1}, false, false)) + require.True(t, shouldVerifyAggregate([]uint64{1}, true, false)) + require.True(t, shouldVerifyAggregate([]uint64{1}, false, true)) +} + func TestSyncMapRangeDeadlock(t *testing.T) { var m sync.Map m.Store(1, 1) From 4110b8bd350141baccee8d6dace040370efe513e Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 13 Jul 2026 04:27:26 +0900 Subject: [PATCH 07/17] cl/das: start PeerDAS after gossip registration --- cl/das/mock_services/peer_das_mock.go | 36 +++++++++++++++++ cl/das/peer_das.go | 18 ++++++--- cl/das/peer_das_start_test.go | 45 ++++++++++++++++++++++ cl/spectest/consensus_tests/fork_choice.go | 2 +- cmd/caplin/caplin1/run.go | 3 +- 5 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 cl/das/peer_das_start_test.go diff --git a/cl/das/mock_services/peer_das_mock.go b/cl/das/mock_services/peer_das_mock.go index 090c5007d76..ab065a443d9 100644 --- a/cl/das/mock_services/peer_das_mock.go +++ b/cl/das/mock_services/peer_das_mock.go @@ -347,6 +347,42 @@ func (c *MockPeerDasSetForkChoiceCall) DoAndReturn(f func(das.BlockGetter)) *Moc return c } +// Start mocks base method. +func (m *MockPeerDas) Start(ctx context.Context) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start", ctx) +} + +// Start indicates an expected call of Start. +func (mr *MockPeerDasMockRecorder) Start(ctx any) *MockPeerDasStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockPeerDas)(nil).Start), ctx) + return &MockPeerDasStartCall{Call: call} +} + +// MockPeerDasStartCall wrap *gomock.Call +type MockPeerDasStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockPeerDasStartCall) Return() *MockPeerDasStartCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockPeerDasStartCall) Do(f func(context.Context)) *MockPeerDasStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockPeerDasStartCall) DoAndReturn(f func(context.Context)) *MockPeerDasStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // StateReader mocks base method. func (m *MockPeerDas) StateReader() peerdasstate.PeerDasStateReader { m.ctrl.T.Helper() diff --git a/cl/das/peer_das.go b/cl/das/peer_das.go index 6e625e43f2e..2575200c57d 100644 --- a/cl/das/peer_das.go +++ b/cl/das/peer_das.go @@ -48,6 +48,7 @@ type gloasBlockData struct { //go:generate mockgen -typed=true -destination=mock_services/peer_das_mock.go -package=mock_services . PeerDas type PeerDas interface { + Start(ctx context.Context) // [Modified in Gloas:EIP7732] Changed from []*SignedBlindedBeaconBlock to []ColumnSyncableSignedBlock // to support both pre-GLOAS (blinded) and GLOAS (non-blinded) blocks DownloadColumnsAndRecoverBlobs(ctx context.Context, blocks []cltypes.ColumnSyncableSignedBlock) error @@ -88,10 +89,10 @@ type peerdas struct { blockReader freezeblocks.BeaconSnapshotReader indiciesDB kv.RoDB gloasDataCache *lru.Cache[common.Hash, *gloasBlockData] // cache for GLOAS block data (~1KB per entry) + startOnce sync.Once } func NewPeerDas( - ctx context.Context, rpc *rpc.BeaconRpcP2P, beaconConfig *clparams.BeaconChainConfig, caplinConfig *clparams.CaplinConfig, @@ -128,14 +129,19 @@ func NewPeerDas( indiciesDB: indiciesDB, gloasDataCache: gloasDataCache, } - p.resubscribeGossip() - for range numOfBlobRecoveryWorkers { - go p.blobsRecoverWorker(ctx) - } - go p.syncColumnDataWorker(ctx) return p } +func (d *peerdas) Start(ctx context.Context) { + d.startOnce.Do(func() { + d.resubscribeGossip() + for range numOfBlobRecoveryWorkers { + go d.blobsRecoverWorker(ctx) + } + go d.syncColumnDataWorker(ctx) + }) +} + func (d *peerdas) StateReader() peerdasstate.PeerDasStateReader { return d.state } diff --git a/cl/das/peer_das_start_test.go b/cl/das/peer_das_start_test.go new file mode 100644 index 00000000000..3f13477ef45 --- /dev/null +++ b/cl/das/peer_das_start_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package das + +import ( + "context" + "testing" + + "go.uber.org/mock/gomock" + + "github.com/erigontech/erigon/cl/clparams" + peerdasstate "github.com/erigontech/erigon/cl/das/state" + gossipmock "github.com/erigontech/erigon/cl/phase1/network/gossip/mock_services" +) + +func TestPeerDasSubscribesOnlyAfterStart(t *testing.T) { + ctrl := gomock.NewController(t) + gossipManager := gossipmock.NewMockGossip(ctrl) + beaconConfig := clparams.MainnetBeaconConfig + beaconConfig.DataColumnSidecarSubnetCount = 2 + caplinConfig := clparams.CaplinConfig{ArchiveBlobs: true} + peerDasState := peerdasstate.NewPeerDasState(&beaconConfig, &clparams.NetworkConfig{}) + + peerDas := NewPeerDas(nil, &beaconConfig, &caplinConfig, nil, nil, nil, [32]byte{}, nil, peerDasState, gossipManager, nil, nil) + + gossipManager.EXPECT().SubscribeWithExpiry(gomock.Any(), gomock.Any()).Times(2) + ctx, cancel := context.WithCancel(context.Background()) + peerDas.Start(ctx) + peerDas.Start(ctx) + cancel() +} diff --git a/cl/spectest/consensus_tests/fork_choice.go b/cl/spectest/consensus_tests/fork_choice.go index 34371ac8fdb..2d3aaebe30e 100644 --- a/cl/spectest/consensus_tests/fork_choice.go +++ b/cl/spectest/consensus_tests/fork_choice.go @@ -312,7 +312,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err blobStorage := blob_storage.NewBlobStore(memdb.New(t, "/tmp", dbcfg.ChainDB), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock) columnStorage := blob_storage.NewDataColumnStore(afero.NewMemMapFs(), 1000, &clparams.MainnetBeaconConfig, ethClock, emitters) peerDasState := peerdasstate.NewPeerDasState(&clparams.MainnetBeaconConfig, &clparams.NetworkConfig{}) - peerDas := das.NewPeerDas(ctx, nil, &clparams.MainnetBeaconConfig, &clparams.CaplinConfig{}, columnStorage, blobStorage, nil, enode.ID{}, ethClock, peerDasState, nil, nil, nil) + peerDas := das.NewPeerDas(nil, &clparams.MainnetBeaconConfig, &clparams.CaplinConfig{}, columnStorage, blobStorage, nil, enode.ID{}, ethClock, peerDasState, nil, nil, nil) localValidators := validator_params.NewValidatorParams() forkStore, err := forkchoice.NewForkChoiceStore( diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index a744357bc8e..87f8d17fee9 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -460,7 +460,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi peerDasState.SetLocalNodeID(localNode) beaconRpc := rpc.NewBeaconRpcP2P(ctx, sentinel, beaconConfig, ethClock, state) gossipManager.SetPeerBanner(beaconRpc) - peerDas := das.NewPeerDas(ctx, beaconRpc, beaconConfig, &config, columnStorage, blobStorage, sentinel, localNode.ID(), ethClock, peerDasState, gossipManager, rcsn, indexDB) + peerDas := das.NewPeerDas(beaconRpc, beaconConfig, &config, columnStorage, blobStorage, sentinel, localNode.ID(), ethClock, peerDasState, gossipManager, rcsn, indexDB) forkChoice.InitPeerDas(peerDas) // hack init peerDas.SetForkChoice(forkChoice) // [New in Gloas:EIP7732] Set forkChoice for GLOAS kzg_commitments lookup committeeSub := committee_subscription.NewCommitteeSubscribeManagement(ctx, beaconConfig, networkConfig, ethClock, aggregationPool, syncedDataManager, gossipManager) @@ -501,6 +501,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi proposerPreferencesService, executionPayloadBidService, ) + peerDas.Start(ctx) { go batchSignatureVerifier.Start() From f4bd1014c911e853381b79879430cc99e09177c6 Mon Sep 17 00:00:00 2001 From: kewei Date: Wed, 29 Jul 2026 10:03:11 +0800 Subject: [PATCH 08/17] cl/forkchoice: reuse gloas contribution capacity --- cl/phase1/forkchoice/gloas_weight_tree.go | 6 +++++ .../forkchoice/gloas_weight_tree_test.go | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/cl/phase1/forkchoice/gloas_weight_tree.go b/cl/phase1/forkchoice/gloas_weight_tree.go index e0e92ac0afc..0aebf516fd0 100644 --- a/cl/phase1/forkchoice/gloas_weight_tree.go +++ b/cl/phase1/forkchoice/gloas_weight_tree.go @@ -228,6 +228,12 @@ func growGloasContributions(applied []gloasVoteContribution, size int) []gloasVo if len(applied) >= size { return applied } + if cap(applied) >= size { + previousLen := len(applied) + applied = applied[:size] + clear(applied[previousLen:]) + return applied + } nextCap := max(cap(applied)*2, size) next := make([]gloasVoteContribution, size, nextCap) copy(next, applied) diff --git a/cl/phase1/forkchoice/gloas_weight_tree_test.go b/cl/phase1/forkchoice/gloas_weight_tree_test.go index 4e229b36899..2d76fa24db5 100644 --- a/cl/phase1/forkchoice/gloas_weight_tree_test.go +++ b/cl/phase1/forkchoice/gloas_weight_tree_test.go @@ -191,6 +191,28 @@ func TestGrowGloasContributionsGrowsAmortized(t *testing.T) { require.Greater(t, cap(applied), len(applied)) } +func TestGrowGloasContributionsReusesSpareCapacity(t *testing.T) { + applied := make([]gloasVoteContribution, 1, 8) + applied[0].set = true + + grown := growGloasContributions(applied, 2) + + require.Len(t, grown, 2) + require.Equal(t, 8, cap(grown)) + require.True(t, &applied[0] == &grown[0]) + require.True(t, grown[0].set) +} + +func TestGrowGloasContributionsClearsNewlyExposedEntries(t *testing.T) { + applied := make([]gloasVoteContribution, 2, 8) + applied[1].set = true + applied = applied[:1] + + grown := growGloasContributions(applied, 2) + + require.False(t, grown[1].set) +} + func TestGloasMarksDirtyWeightTree(t *testing.T) { f := newGloasWeightTreeTestStore() f.gloasWeightTree.state = &checkpointState{} From fcf852779163439a7a43332a8dd43f07886d0a75 Mon Sep 17 00:00:00 2001 From: kewei Date: Wed, 29 Jul 2026 11:09:35 +0800 Subject: [PATCH 09/17] cl/services: retry Gloas envelopes after data availability --- .../services/execution_payload_service.go | 37 ++++++---- .../execution_payload_service_test.go | 72 +++++++++++++++++++ 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/cl/phase1/network/services/execution_payload_service.go b/cl/phase1/network/services/execution_payload_service.go index cdac0bf5c17..0b5c1f8e27a 100644 --- a/cl/phase1/network/services/execution_payload_service.go +++ b/cl/phase1/network/services/execution_payload_service.go @@ -41,7 +41,7 @@ type seenEnvelopeKey struct { builderIndex uint64 } -// pendingEnvelopeKey tracks envelopes waiting for their block to arrive. +// pendingEnvelopeKey tracks envelopes waiting for their block or data columns. // We use (blockRoot, envelopeHash) as key instead of just blockRoot because: // - Multiple envelopes (including forged ones) may arrive before the block // - Using only blockRoot would cause later arrivals to overwrite earlier ones @@ -52,7 +52,7 @@ type pendingEnvelopeKey struct { envelopeHash common.Hash } -// envelopeJob represents a pending envelope waiting for its block to arrive +// envelopeJob represents an envelope waiting for its dependencies. type envelopeJob struct { envelope *cltypes.SignedExecutionPayloadEnvelope creationTime time.Time @@ -73,7 +73,7 @@ type executionPayloadService struct { // Cache to track seen envelopes: (beaconBlockRoot, builderIndex) -> struct{} seenEnvelopesCache *lru.Cache[seenEnvelopeKey, struct{}] - // Pending envelopes waiting for block to arrive + // Pending envelopes waiting for their dependencies pendingEnvelopes sync.Map // pendingEnvelopeKey -> *envelopeJob pendingCount atomic.Int32 pendingCond *sync.Cond @@ -174,8 +174,12 @@ func (s *executionPayloadService) ProcessMessage(ctx context.Context, _ *uint64, // Process the execution payload through forkchoice // Note: bid matching and signature verification are done in OnExecutionPayload.validateEnvelopeAgainstBlock if err := s.forkchoiceStore.OnExecutionPayload(ctx, signedEnvelope, true, true); err != nil { - if errors.Is(err, forkchoice.ErrIgnore) || errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) { - return fmt.Errorf("%w: %v", ErrIgnore, err) + if errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) { + s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope) + return fmt.Errorf("%w: %w", ErrIgnore, err) + } + if errors.Is(err, forkchoice.ErrIgnore) { + return fmt.Errorf("%w: %w", ErrIgnore, err) } return fmt.Errorf("failed to process execution payload: %w", err) } @@ -270,7 +274,7 @@ func (s *executionPayloadService) loop(ctx context.Context) { } } -// processPendingEnvelopes checks and processes any pending envelopes whose blocks have arrived +// processPendingEnvelopes retries pending envelopes whose blocks have arrived. func (s *executionPayloadService) processPendingEnvelopes(ctx context.Context) { s.pendingEnvelopes.Range(func(key, value any) bool { pendingKey := key.(pendingEnvelopeKey) @@ -278,9 +282,10 @@ func (s *executionPayloadService) processPendingEnvelopes(ctx context.Context) { // Check expiry if time.Since(job.creationTime) > pendingEnvelopeExpiry { - s.pendingEnvelopes.Delete(pendingKey) - s.pendingCount.Add(-1) - log.Trace("Pending envelope expired", "blockRoot", pendingKey.blockRoot) + if _, loaded := s.pendingEnvelopes.LoadAndDelete(pendingKey); loaded { + s.pendingCount.Add(-1) + log.Trace("Pending envelope expired", "blockRoot", pendingKey.blockRoot) + } return true } @@ -290,12 +295,14 @@ func (s *executionPayloadService) processPendingEnvelopes(ctx context.Context) { return true // Block still not here, keep waiting } - // Block arrived, remove from pending and process - s.pendingEnvelopes.Delete(pendingKey) - s.pendingCount.Add(-1) - - // Re-run full validation via ProcessMessage - if err := s.ProcessMessage(ctx, nil, job.envelope); err != nil { + err := s.ProcessMessage(ctx, nil, job.envelope) + if errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) { + return true + } + if _, loaded := s.pendingEnvelopes.LoadAndDelete(pendingKey); loaded { + s.pendingCount.Add(-1) + } + if err != nil { log.Trace("Failed to process pending envelope", "blockRoot", pendingKey.blockRoot, "err", err) } return true diff --git a/cl/phase1/network/services/execution_payload_service_test.go b/cl/phase1/network/services/execution_payload_service_test.go index 9e32176d091..240101d4e10 100644 --- a/cl/phase1/network/services/execution_payload_service_test.go +++ b/cl/phase1/network/services/execution_payload_service_test.go @@ -30,6 +30,7 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/phase1/core/state/lru" + "github.com/erigontech/erigon/cl/phase1/forkchoice" "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" "github.com/erigontech/erigon/common" ) @@ -291,6 +292,77 @@ func TestExecutionPayloadServicePendingEnvelopeProcessing(t *testing.T) { require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) } +func TestExecutionPayloadServiceQueuesEnvelopeUntilDataAvailable(t *testing.T) { + service, fcu := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Slot: 100}, + } + fcu.OnExecutionPayloadErr = forkchoice.ErrEIP7594ColumnDataNotAvailable + + err := service.ProcessMessage(t.Context(), nil, envelope) + require.ErrorIs(t, err, forkchoice.ErrEIP7594ColumnDataNotAvailable) + require.Equal(t, int32(1), impl.pendingCount.Load()) + + fcu.OnExecutionPayloadErr = nil + impl.processPendingEnvelopes(t.Context()) + + require.Equal(t, int32(0), impl.pendingCount.Load()) + require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) +} + +func TestExecutionPayloadServiceRetainsPendingEnvelopeUntilDataAvailable(t *testing.T) { + service, fcu := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + key := pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash} + value, ok := impl.pendingEnvelopes.Load(key) + require.True(t, ok) + creationTime := value.(*envelopeJob).creationTime + + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Slot: 100}, + } + fcu.OnExecutionPayloadErr = forkchoice.ErrEIP7594ColumnDataNotAvailable + + impl.processPendingEnvelopes(t.Context()) + require.Equal(t, int32(1), impl.pendingCount.Load()) + value, ok = impl.pendingEnvelopes.Load(key) + require.True(t, ok) + require.Equal(t, creationTime, value.(*envelopeJob).creationTime) + + fcu.OnExecutionPayloadErr = nil + impl.processPendingEnvelopes(t.Context()) + + require.Equal(t, int32(0), impl.pendingCount.Load()) + require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) +} + +func TestExecutionPayloadServiceDropsPendingEnvelopeAfterValidationFailure(t *testing.T) { + service, fcu := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Slot: 100}, + } + fcu.OnExecutionPayloadErr = errors.New("invalid envelope") + + impl.processPendingEnvelopes(t.Context()) + + require.Equal(t, int32(0), impl.pendingCount.Load()) + require.False(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) +} + func TestExecutionPayloadServiceMultiplePendingForSameBlock(t *testing.T) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) From 25f577224672d0b43719e5677f1dcd96510ec1c4 Mon Sep 17 00:00:00 2001 From: kewei Date: Thu, 30 Jul 2026 12:10:36 +0700 Subject: [PATCH 10/17] cl/network: resolve first Gloas backfill lookahead --- .../network/backward_beacon_downloader.go | 192 ++++++++++++--- .../backward_beacon_downloader_test.go | 226 ++++++++++++++++-- cl/phase1/network/beacon_downloader.go | 3 + 3 files changed, 369 insertions(+), 52 deletions(-) diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index ef7cfad7315..d2520dc3c25 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -67,9 +67,10 @@ type BackwardBeaconDownloader struct { beaconCfg *clparams.BeaconChainConfig // [New in Gloas:EIP7732] highest block from the previous batch, used as lookahead // to determine FULL/EMPTY status of the highest block in the current batch. - prevBatchTopBlock *cltypes.SignedBeaconBlock - httpFallbackURL string // beacon API base URL for HTTP fallback when P2P fails - httpPreferred atomic.Bool // set after first HTTP success; skips P2P probing + prevBatchTopBlock *cltypes.SignedBeaconBlock + httpFallbackURL string // beacon API base URL for HTTP fallback when P2P fails + httpPreferred atomic.Bool // set after first HTTP success; skips P2P probing + consecutiveLookaheadFailures uint8 // Count consecutive batches where envelope fetch returned 0 for all FULL roots. // After enough failures, skip envelope requirements and process blocks as EMPTY. @@ -288,6 +289,12 @@ func (b *BackwardBeaconDownloader) sendBlockRequest( // processResponses processes downloaded blocks in reverse order. func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, responses []*cltypes.SignedBeaconBlock) error { + if err := b.prepareFirstBatchLookahead(ctx, responses); err != nil { + log.Warn("[BackwardBeaconDownloader] GLOAS lookahead unavailable", "err", err) + b.waitBeforeLookaheadRetry(ctx) + return nil + } + // [New in Gloas:EIP7732] Fetch envelopes for GLOAS FULL blocks before processing. log.Debug("[BackwardBeaconDownloader] processResponses start", "blocks", len(responses), "slotToDownload", b.slotToDownload.Load(), "expectedRoot", b.expectedRoot) envelopes, fullRootSet := b.fetchGloasEnvelopes(ctx, responses) @@ -302,6 +309,10 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons if b.finished.Load() { return nil } + if block == nil || block.Block == nil || block.Block.Body == nil { + log.Debug("[BackwardBeaconDownloader] ignoring incomplete block response") + continue + } blockRoot, err := block.Block.HashSSZ() if err != nil { @@ -345,7 +356,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons b.expectedRoot = block.Block.ParentRoot if block.Block.Slot == 0 { b.finished.Store(true) - b.prevBatchTopBlock = responses[0] + b.prevBatchTopBlock = firstCompleteBlock(responses) return nil } b.slotToDownload.Store(block.Block.Slot - 1) @@ -354,7 +365,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons // Update prevBatchTopBlock only when at least one block was processed, // so retries preserve the correct lookahead for FULL/EMPTY determination. if advanced && len(responses) > 0 { - b.prevBatchTopBlock = responses[0] + b.prevBatchTopBlock = firstCompleteBlock(responses) } if !matched { @@ -403,13 +414,143 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } +func firstCompleteBlock(responses []*cltypes.SignedBeaconBlock) *cltypes.SignedBeaconBlock { + for _, block := range responses { + if block != nil && block.Block != nil && block.Block.Body != nil { + return block + } + } + return nil +} + +func lastCompleteBlock(responses []*cltypes.SignedBeaconBlock) *cltypes.SignedBeaconBlock { + for _, block := range slices.Backward(responses) { + if block != nil && block.Block != nil && block.Block.Body != nil { + return block + } + } + return nil +} + +func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Context, responses []*cltypes.SignedBeaconBlock) error { + if b.prevBatchTopBlock != nil || len(responses) == 0 { + return nil + } + + anchor := lastCompleteBlock(responses) + if anchor == nil { + return errors.New("batch has no complete blocks") + } + if anchor.Version() < clparams.GloasVersion { + return nil + } + + anchorRoot, err := anchor.Block.HashSSZ() + if err != nil { + return fmt.Errorf("hash highest block: %w", err) + } + if anchorRoot != b.expectedRoot { + return nil + } + + lookahead, err := b.fetchGloasLookahead(ctx, anchor, anchorRoot) + if err != nil { + return err + } + b.prevBatchTopBlock = lookahead + b.consecutiveLookaheadFailures = 0 + return nil +} + +func (b *BackwardBeaconDownloader) fetchGloasLookahead( + ctx context.Context, + anchor *cltypes.SignedBeaconBlock, + anchorRoot common.Hash, +) (*cltypes.SignedBeaconBlock, error) { + if anchor.Block.Slot == math.MaxUint64 { + return nil, errors.New("cannot fetch lookahead after max slot") + } + + const lookaheadWindow = uint64(64) + start := anchor.Block.Slot + 1 + var httpErr error + if b.httpFallbackURL != "" { + candidates, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, lookaheadWindow, b.beaconCfg) + if err == nil { + if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { + return lookahead, nil + } + httpErr = errors.New("beacon API returned no direct child") + } else { + httpErr = err + } + } + + if b.rpc != nil { + candidates, _, err := b.rpc.SendBeaconBlocksByRangeReq(ctx, start, lookaheadWindow) + if err == nil { + if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { + return lookahead, nil + } + return nil, errors.New("peer returned no direct child") + } + if httpErr != nil { + return nil, errors.Join(httpErr, err) + } + return nil, err + } + if httpErr != nil { + return nil, httpErr + } + return nil, errors.New("no GLOAS lookahead source configured") +} + +func selectGloasLookahead( + anchor *cltypes.SignedBeaconBlock, + anchorRoot common.Hash, + candidates []*cltypes.SignedBeaconBlock, +) *cltypes.SignedBeaconBlock { + var selected *cltypes.SignedBeaconBlock + for _, candidate := range candidates { + if candidate == nil || candidate.Block == nil || candidate.Block.Body == nil { + continue + } + if candidate.Block.Slot <= anchor.Block.Slot || candidate.Block.ParentRoot != anchorRoot { + continue + } + bid := candidate.Block.Body.GetSignedExecutionPayloadBid() + if bid == nil || bid.Message == nil { + continue + } + if selected == nil || candidate.Block.Slot < selected.Block.Slot { + selected = candidate + } + } + return selected +} + +func (b *BackwardBeaconDownloader) waitBeforeLookaheadRetry(ctx context.Context) { + if b.consecutiveLookaheadFailures < 6 { + b.consecutiveLookaheadFailures++ + } + delay := time.Second << (b.consecutiveLookaheadFailures - 1) + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + // determineGloasFullRoots returns the block roots of GLOAS FULL blocks in the batch. // Uses the count+1 lookahead trick: block[i] is FULL if block[i+1].bid.ParentBlockHash == block[i].bid.BlockHash. // For the highest block in the batch, prevBatchTopBlock is used as the cross-batch lookahead. -// If prevBatchTopBlock is nil (first batch ever), the highest block is requested optimistically. func determineGloasFullRoots(responses []*cltypes.SignedBeaconBlock, prevBatchTopBlock *cltypes.SignedBeaconBlock) [][32]byte { var fullRoots [][32]byte for i, block := range responses { + if block == nil || block.Block == nil || block.Block.Body == nil { + continue + } if block.Version() < clparams.GloasVersion { continue } @@ -425,11 +566,9 @@ func determineGloasFullRoots(responses []*cltypes.SignedBeaconBlock, prevBatchTo lookahead = prevBatchTopBlock } if lookahead == nil { - // No lookahead for the highest block in the first batch: request optimistically. - root, err := block.Block.HashSSZ() - if err == nil { - fullRoots = append(fullRoots, root) - } + continue + } + if lookahead.Block == nil || lookahead.Block.Body == nil { continue } nextBid := lookahead.Block.Body.GetSignedExecutionPayloadBid() @@ -464,28 +603,27 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp return nil, fullRootSet } - // When HTTP has been working, skip the slow P2P envelope fetch entirely. + var envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope if b.httpPreferred.Load() && b.httpFallbackURL != "" { - envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fullRoots)) + envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fullRoots)) fetched := fetchEnvelopesFromBeaconAPI(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) if fetched > 0 { log.Debug("[BackwardBeaconDownloader] fetched envelopes from beacon API", "count", fetched) } - return envelopes, fullRootSet - } - - envelopes, err := RequestEnvelopesFrantically(ctx, b.rpc, fullRoots) - if err != nil { - log.Debug("[BackwardBeaconDownloader] failed to fetch GLOAS envelopes via P2P", "err", err) - } - // Fill in missing envelopes from the beacon API when an HTTP URL is configured. - if b.httpFallbackURL != "" && len(envelopes) < len(fullRoots) { - if envelopes == nil { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fullRoots)) + } else { + var err error + envelopes, err = RequestEnvelopesFrantically(ctx, b.rpc, fullRoots) + if err != nil { + log.Debug("[BackwardBeaconDownloader] failed to fetch GLOAS envelopes via P2P", "err", err) } - fetched := fetchEnvelopesFromBeaconAPI(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) - if fetched > 0 { - log.Debug("[BackwardBeaconDownloader] fetched envelopes from beacon API", "count", fetched) + if b.httpFallbackURL != "" && len(envelopes) < len(fullRoots) { + if envelopes == nil { + envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fullRoots)) + } + fetched := fetchEnvelopesFromBeaconAPI(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) + if fetched > 0 { + log.Debug("[BackwardBeaconDownloader] fetched envelopes from beacon API", "count", fetched) + } } } diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index 5d5a10e0b14..fd6b54cd657 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -20,6 +20,7 @@ import ( "context" "net/http" "net/http/httptest" + "sync/atomic" "testing" "time" @@ -65,6 +66,18 @@ func TestDetermineGloasFullRoots_EmptyBatch(t *testing.T) { assert.Empty(t, roots) } +func TestDetermineGloasFullRoots_IncompleteBlocks(t *testing.T) { + incomplete := &cltypes.SignedBeaconBlock{} + + assert.NotPanics(t, func() { + roots := determineGloasFullRoots( + []*cltypes.SignedBeaconBlock{nil, incomplete}, + nil, + ) + assert.Empty(t, roots) + }) +} + // TestDetermineGloasFullRoots_AllPreGloas verifies that pre-GLOAS blocks are ignored. func TestDetermineGloasFullRoots_AllPreGloas(t *testing.T) { responses := []*cltypes.SignedBeaconBlock{ @@ -76,18 +89,12 @@ func TestDetermineGloasFullRoots_AllPreGloas(t *testing.T) { assert.Empty(t, roots) } -// TestDetermineGloasFullRoots_SingleBlock_NilLookahead verifies that a single GLOAS block -// with no prevBatchTopBlock (first batch ever) is treated optimistically as FULL. -func TestDetermineGloasFullRoots_SingleBlock_NilLookahead(t *testing.T) { +func TestDetermineGloasFullRoots_SingleBlock_NilLookaheadDoesNotGuess(t *testing.T) { blk := makeGloasBlock(100, hash(0xAA), hash(0x00)) responses := []*cltypes.SignedBeaconBlock{blk} roots := determineGloasFullRoots(responses, nil) - require.Len(t, roots, 1) - - expected, err := blk.Block.HashSSZ() - require.NoError(t, err) - assert.Equal(t, expected, roots[0]) + assert.Empty(t, roots) } // TestDetermineGloasFullRoots_InBatch_Full verifies that a GLOAS block is identified as FULL @@ -99,15 +106,11 @@ func TestDetermineGloasFullRoots_InBatch_Full(t *testing.T) { responses := []*cltypes.SignedBeaconBlock{blk0, blk1} roots := determineGloasFullRoots(responses, nil) - // blk0 is FULL, blk1 is highest with nil prevBatchTopBlock → optimistic - require.Len(t, roots, 2) + require.Len(t, roots, 1) root0, err := blk0.Block.HashSSZ() require.NoError(t, err) - root1, err := blk1.Block.HashSSZ() - require.NoError(t, err) assert.Contains(t, roots, root0) - assert.Contains(t, roots, root1) } // TestDetermineGloasFullRoots_InBatch_Empty verifies that a GLOAS block is identified as EMPTY @@ -119,12 +122,7 @@ func TestDetermineGloasFullRoots_InBatch_Empty(t *testing.T) { responses := []*cltypes.SignedBeaconBlock{blk0, blk1} roots := determineGloasFullRoots(responses, nil) - // blk0 is EMPTY, blk1 is highest with nil prevBatchTopBlock → optimistic - require.Len(t, roots, 1) - - root1, err := blk1.Block.HashSSZ() - require.NoError(t, err) - assert.Equal(t, root1, roots[0]) + assert.Empty(t, roots) } // TestDetermineGloasFullRoots_CrossBatch_Full verifies the cross-batch lookahead: @@ -169,7 +167,7 @@ func TestDetermineGloasFullRoots_Mixed(t *testing.T) { responses := []*cltypes.SignedBeaconBlock{blk0, blk1, blk2, blk3} roots := determineGloasFullRoots(responses, nil) - require.Len(t, roots, 3) // blk0, blk2, blk3(optimistic) + require.Len(t, roots, 2) root0, _ := blk0.Block.HashSSZ() root1, _ := blk1.Block.HashSSZ() @@ -178,7 +176,7 @@ func TestDetermineGloasFullRoots_Mixed(t *testing.T) { assert.Contains(t, roots, root0) assert.NotContains(t, roots, root1) assert.Contains(t, roots, root2) - assert.Contains(t, roots, root3) + assert.NotContains(t, roots, root3) } // TestDetermineGloasFullRoots_MixedVersions verifies that pre-GLOAS blocks in a mixed @@ -191,13 +189,191 @@ func TestDetermineGloasFullRoots_MixedVersions(t *testing.T) { responses := []*cltypes.SignedBeaconBlock{deneb, gloasFull, lookahead} roots := determineGloasFullRoots(responses, nil) - // gloasFull FULL, lookahead optimistic (highest with nil prevBatchTopBlock) - require.Len(t, roots, 2) + require.Len(t, roots, 1) rootFull, _ := gloasFull.Block.HashSSZ() - rootLookahead, _ := lookahead.Block.HashSSZ() assert.Contains(t, roots, rootFull) - assert.Contains(t, roots, rootLookahead) +} + +func TestBackwardBeaconDownloaderFirstBatchUsesLookaheadAfterMissedSlot(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + + lookahead := makeGloasBlock(102, hash(0xBB), hash(0x10)) + lookahead.Block.ParentRoot = anchorRoot + encodedLookahead, err := lookahead.EncodeSSZ(nil) + require.NoError(t, err) + + var lookaheadRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v2/beacon/blocks/102" { + lookaheadRequests.Add(1) + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedLookahead) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + var processed atomic.Bool + downloader := &BackwardBeaconDownloader{ + expectedRoot: anchorRoot, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Store(true) + assert.Nil(t, envelope) + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{anchor})) + assert.True(t, processed.Load()) + assert.Equal(t, int32(1), lookaheadRequests.Load()) +} + +func TestBackwardBeaconDownloaderFirstBatchFullBlockWaitsForEnvelope(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + + lookahead := makeGloasBlock(102, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = anchorRoot + encodedLookahead, err := lookahead.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v2/beacon/blocks/102" { + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedLookahead) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + var processed atomic.Bool + downloader := &BackwardBeaconDownloader{ + expectedRoot: anchorRoot, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Store(true) + return true, nil + }, + } + downloader.httpPreferred.Store(true) + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{anchor})) + assert.False(t, processed.Load()) + assert.Equal(t, 1, downloader.consecutiveEnvelopeFailures) +} + +func TestBackwardBeaconDownloaderHTTPPreferredMissingEnvelopeTracksFailure(t *testing.T) { + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + lookahead := makeGloasBlock(102, hash(0xBB), hash(0xAA)) + downloader := &BackwardBeaconDownloader{ + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + downloader.httpPreferred.Store(true) + + envelopes, fullRoots := downloader.fetchGloasEnvelopes( + context.Background(), + []*cltypes.SignedBeaconBlock{block, lookahead}, + ) + + assert.Empty(t, envelopes) + require.Len(t, fullRoots, 1) + assert.Equal(t, 1, downloader.consecutiveEnvelopeFailures) +} + +func TestSelectGloasLookaheadRejectsUnlinkedAndIncompleteBlocks(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + + unlinked := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + unlinked.Block.ParentRoot = hash(0xFF) + linked := makeGloasBlock(102, hash(0xCC), hash(0xAA)) + linked.Block.ParentRoot = anchorRoot + laterLinked := makeGloasBlock(103, hash(0xDD), hash(0xCC)) + laterLinked.Block.ParentRoot = anchorRoot + + selected := selectGloasLookahead( + anchor, + anchorRoot, + []*cltypes.SignedBeaconBlock{nil, unlinked, laterLinked, linked}, + ) + assert.Same(t, linked, selected) +} + +func TestFetchEnvelopesFromBeaconAPIIncompleteBlock(t *testing.T) { + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + received := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) + + assert.NotPanics(t, func() { + fetched := fetchEnvelopesFromBeaconAPI( + context.Background(), + server.URL, + []*cltypes.SignedBeaconBlock{nil, block}, + [][32]byte{blockRoot}, + received, + &clparams.MainnetBeaconConfig, + ) + assert.Zero(t, fetched) + }) +} + +func TestBackwardBeaconDownloaderPreGloasFirstBatchNeedsNoLookahead(t *testing.T) { + block := makeDenebBlock(100) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + + var processed atomic.Bool + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Store(true) + assert.Nil(t, envelope) + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) + assert.True(t, processed.Load()) +} + +func TestBackwardBeaconDownloaderSkipsIncompleteResponse(t *testing.T) { + block := makeDenebBlock(100) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + return false, nil + }, + } + + assert.NotPanics(t, func() { + require.NoError(t, downloader.processResponses( + context.Background(), + []*cltypes.SignedBeaconBlock{nil, block}, + )) + }) } func TestBackwardBeaconDownloaderHTTPPreferredEmptyResponseFallsBack(t *testing.T) { diff --git a/cl/phase1/network/beacon_downloader.go b/cl/phase1/network/beacon_downloader.go index 259c9c75f85..009e4263f27 100644 --- a/cl/phase1/network/beacon_downloader.go +++ b/cl/phase1/network/beacon_downloader.go @@ -553,6 +553,9 @@ func fetchEnvelopesFromBeaconAPI( // Build root-to-slot mapping from blocks rootToSlot := make(map[common.Hash]uint64, len(blocks)) for _, blk := range blocks { + if blk == nil || blk.Block == nil || blk.Block.Body == nil { + continue + } root, err := blk.Block.HashSSZ() if err == nil { rootToSlot[root] = blk.Block.Slot From 2b661b53db747aede681f4bc99b894de099d0d0b Mon Sep 17 00:00:00 2001 From: kewei Date: Sat, 1 Aug 2026 04:56:44 +0700 Subject: [PATCH 11/17] cl: stabilize Gloas external EL sync --- cl/phase1/forkchoice/forkchoice.go | 12 +- cl/phase1/forkchoice/on_execution_payload.go | 90 +++- .../forkchoice/on_execution_payload_test.go | 319 ++++++++++++- .../network/backward_beacon_downloader.go | 451 +++++++++++++----- .../backward_beacon_downloader_test.go | 296 +++++++++++- cl/phase1/network/beacon_downloader.go | 73 +-- cl/phase1/stages/stage_history_download.go | 86 +++- .../stages/stage_history_download_test.go | 43 ++ 8 files changed, 1195 insertions(+), 175 deletions(-) diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index e988667bea2..6b9df95bc24 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -200,8 +200,10 @@ type ForkChoiceStore struct { // whose EL newPayload failed (e.g. because EL hasn't caught up after forward sync). // The stages layer drains these into blockCollector before each Flush() so EL // eventually receives the blocks. - pendingELPayloadsMu sync.Mutex - pendingELPayloads []PendingELPayload + pendingELPayloadsMu sync.Mutex + pendingELPayloads []PendingELPayload + payloadValidations map[common.Hash]*payloadValidationCall + payloadValidationSlots chan struct{} // db is used to persist execution payload indices (block number/hash) when an envelope // is accepted in OnExecutionPayload. May be nil (e.g. in tests), in which case the @@ -215,6 +217,12 @@ type PendingELPayload struct { Envelope *cltypes.SignedExecutionPayloadEnvelope } +type payloadValidationCall struct { + done chan struct{} + status execution_client.PayloadStatus + err error +} + type childrens struct { childrenHashes []common.Hash parentSlot uint64 // we keep this one for pruning diff --git a/cl/phase1/forkchoice/on_execution_payload.go b/cl/phase1/forkchoice/on_execution_payload.go index 780855a1cbd..62e93b20920 100644 --- a/cl/phase1/forkchoice/on_execution_payload.go +++ b/cl/phase1/forkchoice/on_execution_payload.go @@ -234,6 +234,9 @@ func (f *ForkChoiceStore) validatePayloadWithEL( if f.engine == nil { return nil } + if envelope == nil || envelope.Payload == nil || envelope.ExecutionRequests == nil { + return errors.New("validatePayloadWithEL: incomplete envelope") + } // Get committed bid from the block (not from state, since state transition hasn't happened yet) committedBid := block.Block.Body.GetSignedExecutionPayloadBid() @@ -266,11 +269,15 @@ func (f *ForkChoiceStore) validatePayloadWithEL( if executionRequestsList == nil { executionRequestsList = []hexutil.Bytes{} } + validationKey, err := envelope.HashSSZ() + if err != nil { + return fmt.Errorf("validatePayloadWithEL: failed to hash envelope: %w", err) + } // Call NewPayload to validate execution payload with EL timeStartExec := time.Now() parentBlockRoot := block.Block.ParentRoot - payloadStatus, err := f.engine.NewPayload(ctx, envelope.Payload, &parentBlockRoot, versionedHashes, executionRequestsList) + payloadStatus, err := f.newPayloadWithoutForkChoiceLock(ctx, common.Hash(validationKey), envelope.Payload, &parentBlockRoot, versionedHashes, executionRequestsList) monitor.ObserveNewPayloadTime(timeStartExec) log.Trace("[validatePayloadWithEL] NewPayload", "status", payloadStatus, "beaconBlockRoot", beaconBlockRoot) @@ -313,6 +320,65 @@ func (f *ForkChoiceStore) validatePayloadWithEL( return nil } +func (f *ForkChoiceStore) newPayloadWithoutForkChoiceLock( + ctx context.Context, + beaconBlockRoot common.Hash, + payload *cltypes.Eth1Block, + parentBlockRoot *common.Hash, + versionedHashes []common.Hash, + executionRequestsList []hexutil.Bytes, +) (execution_client.PayloadStatus, error) { + if f.payloadValidations == nil { + f.payloadValidations = make(map[common.Hash]*payloadValidationCall) + } + if f.payloadValidationSlots == nil { + f.payloadValidationSlots = make(chan struct{}, 2) + } + if call, ok := f.payloadValidations[beaconBlockRoot]; ok { + f.mu.Unlock() + select { + case <-call.done: + f.mu.Lock() + return call.status, call.err + case <-ctx.Done(): + f.mu.Lock() + return execution_client.PayloadStatusNone, ctx.Err() + } + } + + call := &payloadValidationCall{done: make(chan struct{})} + f.payloadValidations[beaconBlockRoot] = call + f.mu.Unlock() + select { + case f.payloadValidationSlots <- struct{}{}: + case <-ctx.Done(): + f.mu.Lock() + call.err = ctx.Err() + delete(f.payloadValidations, beaconBlockRoot) + close(call.done) + return execution_client.PayloadStatusNone, call.err + } + var status execution_client.PayloadStatus + var err error + var panicValue any + func() { + defer func() { + panicValue = recover() + }() + status, err = f.engine.NewPayload(ctx, payload, parentBlockRoot, versionedHashes, executionRequestsList) + }() + <-f.payloadValidationSlots + f.mu.Lock() + call.status = status + call.err = err + delete(f.payloadValidations, beaconBlockRoot) + close(call.done) + if panicValue != nil { + panic(panicValue) + } + return status, err +} + // applyEnvelope processes the envelope under f.mu: validates, verifies with CL and EL, // and persists the envelope to disk. No CL state transition is performed — the // execution effects are deferred to the next block's ProcessParentExecutionPayload. @@ -400,6 +466,17 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop return false, err } } + if f.forkGraph.HasEnvelope(beaconBlockRoot) { + return false, nil + } + blockState, err = f.forkGraph.GetState(beaconBlockRoot, false) + if err != nil { + return false, fmt.Errorf("OnExecutionPayload: failed to refresh block state: %w", err) + } + block, ok = f.forkGraph.GetBlock(beaconBlockRoot) + if blockState == nil || !ok || block == nil { + return false, fmt.Errorf("%w: block disappeared during payload validation for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) + } } // Ensure the correct state root is available for the beacon_block_root check @@ -607,6 +684,17 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, return false, err } } + if f.forkGraph.HasEnvelope(beaconBlockRoot) { + return false, nil + } + blockState, err = f.forkGraph.GetState(beaconBlockRoot, false) + if err != nil { + return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to refresh block state: %w", err) + } + block, ok = f.forkGraph.GetBlock(beaconBlockRoot) + if blockState == nil || !ok || block == nil { + return false, fmt.Errorf("%w: block disappeared during payload validation for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) + } blockState.SetPreviousStateRoot(block.Block.StateRoot) diff --git a/cl/phase1/forkchoice/on_execution_payload_test.go b/cl/phase1/forkchoice/on_execution_payload_test.go index 85dd3524a94..030151512cc 100644 --- a/cl/phase1/forkchoice/on_execution_payload_test.go +++ b/cl/phase1/forkchoice/on_execution_payload_test.go @@ -18,6 +18,8 @@ package forkchoice import ( "context" + "errors" + "sync/atomic" "testing" "time" @@ -30,6 +32,7 @@ import ( "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/hexutil" ) // TestValidateEnvelopeAgainstBlock_NoBid tests that validation fails when block has no bid @@ -323,9 +326,8 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { payloadStatusByRoot: payloadStatusByRoot, executionPayloadGasLimit: executionPayloadGasLimit, } - envelope := &cltypes.ExecutionPayloadEnvelope{ - Payload: &cltypes.Eth1Block{BlockHash: executionBlockHash}, - } + envelope := cltypes.NewExecutionPayloadEnvelope(cfg) + envelope.Payload.BlockHash = executionBlockHash body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{ Message: &cltypes.ExecutionPayloadBid{ @@ -362,3 +364,314 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { }) } } + +func TestValidatePayloadWithELReleasesForkChoiceMuDuringNewPayload(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + engineStarted := make(chan struct{}) + releaseEngine := make(chan struct{}) + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + close(engineStarted) + <-releaseEngine + return execution_client.PayloadStatusValidated, nil + }) + + verifiedExecutionPayload, err := lru.New[common.Hash, struct{}](16) + require.NoError(t, err) + executionPayloadStatus, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + payloadStatusByRoot, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + executionPayloadGasLimit, err := lru.New[common.Hash, uint64](16) + require.NoError(t, err) + + f := &ForkChoiceStore{ + beaconCfg: cfg, + engine: engine, + forkGraph: payloadVoteForkGraph{}, + verifiedExecutionPayload: verifiedExecutionPayload, + executionPayloadStatus: executionPayloadStatus, + payloadStatusByRoot: payloadStatusByRoot, + executionPayloadGasLimit: executionPayloadGasLimit, + } + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{ + Message: &cltypes.ExecutionPayloadBid{ + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + }, + } + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Body: body}} + envelope := cltypes.NewExecutionPayloadEnvelope(cfg) + envelope.Payload.BlockHash = common.HexToHash("0xabcd") + + validationDone := make(chan error, 1) + go func() { + f.mu.Lock() + defer f.mu.Unlock() + validationDone <- f.validatePayloadWithEL(context.Background(), envelope, block, common.HexToHash("0x1234")) + }() + <-engineStarted + + lockAcquired := make(chan struct{}) + go func() { + f.mu.Lock() + close(lockAcquired) + f.mu.Unlock() + }() + acquiredBeforeRelease := false + select { + case <-lockAcquired: + acquiredBeforeRelease = true + case <-time.After(100 * time.Millisecond): + } + close(releaseEngine) + require.NoError(t, <-validationDone) + require.True(t, acquiredBeforeRelease, "forkchoice mutex stayed locked during NewPayload") +} + +func TestValidatePayloadWithELDoesNotCoalesceDifferentPayloads(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + firstStarted := make(chan struct{}) + secondStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + var calls atomic.Int32 + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(2). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + if calls.Add(1) == 1 { + close(firstStarted) + <-releaseFirst + } else { + close(secondStarted) + } + return execution_client.PayloadStatusValidated, nil + }) + + verifiedExecutionPayload, err := lru.New[common.Hash, struct{}](16) + require.NoError(t, err) + executionPayloadStatus, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + payloadStatusByRoot, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + executionPayloadGasLimit, err := lru.New[common.Hash, uint64](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + beaconCfg: cfg, + engine: engine, + forkGraph: payloadVoteForkGraph{}, + verifiedExecutionPayload: verifiedExecutionPayload, + executionPayloadStatus: executionPayloadStatus, + payloadStatusByRoot: payloadStatusByRoot, + executionPayloadGasLimit: executionPayloadGasLimit, + } + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{ + Message: &cltypes.ExecutionPayloadBid{BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48)}, + } + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Body: body}} + blockRoot := common.HexToHash("0x1234") + first := cltypes.NewExecutionPayloadEnvelope(cfg) + first.BeaconBlockRoot = blockRoot + first.Payload.BlockHash = common.HexToHash("0xabcd") + second := cltypes.NewExecutionPayloadEnvelope(cfg) + second.BeaconBlockRoot = blockRoot + second.Payload.BlockHash = first.Payload.BlockHash + second.Payload.GasUsed = 1 + + results := make(chan error, 2) + validate := func(envelope *cltypes.ExecutionPayloadEnvelope) { + f.mu.Lock() + defer f.mu.Unlock() + results <- f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + } + go validate(first) + <-firstStarted + go validate(second) + select { + case <-secondStarted: + case <-time.After(time.Second): + close(releaseFirst) + t.Fatal("different payload was coalesced with in-flight validation") + } + close(releaseFirst) + require.NoError(t, <-results) + require.NoError(t, <-results) +} + +func TestNewPayloadCoalescesSameKey(t *testing.T) { + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + started := make(chan struct{}) + release := make(chan struct{}) + expectedErr := errors.New("payload rejected") + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(1). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + close(started) + <-release + return execution_client.PayloadStatusInvalidated, expectedErr + }) + f := &ForkChoiceStore{engine: engine} + key := hashWithFirstByte(1) + type result struct { + status execution_client.PayloadStatus + err error + } + results := make(chan result, 2) + validate := func(acquired chan<- struct{}) { + f.mu.Lock() + if acquired != nil { + close(acquired) + } + status, err := f.newPayloadWithoutForkChoiceLock(context.Background(), key, nil, nil, nil, nil) + f.mu.Unlock() + results <- result{status: status, err: err} + } + + go validate(nil) + <-started + followerAcquired := make(chan struct{}) + go validate(followerAcquired) + <-followerAcquired + f.mu.Lock() + close(release) + f.mu.Unlock() + for range 2 { + got := <-results + require.EqualValues(t, execution_client.PayloadStatusInvalidated, got.status) + require.ErrorIs(t, got.err, expectedErr) + } +} + +func TestNewPayloadCanceledWaiterDoesNotCancelLeader(t *testing.T) { + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + started := make(chan struct{}) + release := make(chan struct{}) + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(1). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + close(started) + <-release + return execution_client.PayloadStatusValidated, nil + }) + f := &ForkChoiceStore{engine: engine} + key := hashWithFirstByte(1) + leaderDone := make(chan error, 1) + go func() { + f.mu.Lock() + _, err := f.newPayloadWithoutForkChoiceLock(context.Background(), key, nil, nil, nil, nil) + f.mu.Unlock() + leaderDone <- err + }() + <-started + + waiterCtx, cancel := context.WithCancel(context.Background()) + cancel() + f.mu.Lock() + _, err := f.newPayloadWithoutForkChoiceLock(waiterCtx, key, nil, nil, nil, nil) + f.mu.Unlock() + require.ErrorIs(t, err, context.Canceled) + + close(release) + require.NoError(t, <-leaderDone) +} + +func TestNewPayloadConcurrencyIsBounded(t *testing.T) { + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + started := make(chan struct{}, 3) + release := make(chan struct{}) + var active atomic.Int32 + var maximum atomic.Int32 + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(3). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + current := active.Add(1) + for current > maximum.Load() && !maximum.CompareAndSwap(maximum.Load(), current) { + } + started <- struct{}{} + <-release + active.Add(-1) + return execution_client.PayloadStatusValidated, nil + }) + f := &ForkChoiceStore{engine: engine} + done := make(chan struct{}, 3) + for i := range 3 { + go func(key byte) { + f.mu.Lock() + _, _ = f.newPayloadWithoutForkChoiceLock(context.Background(), hashWithFirstByte(key), nil, nil, nil, nil) + f.mu.Unlock() + done <- struct{}{} + }(byte(i + 1)) + } + <-started + <-started + select { + case <-started: + t.Fatal("more than two NewPayload calls ran concurrently") + case <-time.After(100 * time.Millisecond): + } + close(release) + <-started + for range 3 { + <-done + } + require.Equal(t, int32(2), maximum.Load()) +} + +func TestNewPayloadPanicRestoresForkChoiceState(t *testing.T) { + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + var calls atomic.Int32 + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(3). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + if calls.Add(1) == 1 { + panic("engine panic") + } + return execution_client.PayloadStatusValidated, nil + }) + f := &ForkChoiceStore{engine: engine} + key := hashWithFirstByte(1) + func() { + defer func() { require.Equal(t, "engine panic", recover()) }() + f.mu.Lock() + defer f.mu.Unlock() + _, _ = f.newPayloadWithoutForkChoiceLock(context.Background(), key, nil, nil, nil, nil) + }() + + done := make(chan error, 2) + for _, retryKey := range []common.Hash{key, hashWithFirstByte(2)} { + go func() { + f.mu.Lock() + _, err := f.newPayloadWithoutForkChoiceLock(context.Background(), retryKey, nil, nil, nil, nil) + f.mu.Unlock() + done <- err + }() + } + for range 2 { + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("payload validation remained blocked after engine panic") + } + } +} + +func hashWithFirstByte(value byte) common.Hash { + var hash common.Hash + hash[0] = value + return hash +} diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index d2520dc3c25..d1621083fcc 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "maps" "math" "net/http" "slices" @@ -71,6 +72,9 @@ type BackwardBeaconDownloader struct { httpFallbackURL string // beacon API base URL for HTTP fallback when P2P fails httpPreferred atomic.Bool // set after first HTTP success; skips P2P probing consecutiveLookaheadFailures uint8 + lookaheadSearchOffset uint64 + lookaheadRescan bool + lookaheadAnchorRoot common.Hash // Count consecutive batches where envelope fetch returned 0 for all FULL roots. // After enough failures, skip envelope requirements and process blocks as EMPTY. @@ -84,10 +88,16 @@ type BackwardBeaconDownloader struct { mu sync.Mutex } +const ( + gloasLookaheadWindow = uint64(64) + maxSkippedFullBlocks = 65536 + maxBeaconAPIResponseBytes = 64 << 20 +) + // SkippedFullBlock records a GLOAS FULL block whose envelope was unavailable during backward download. type SkippedFullBlock struct { - Block *cltypes.SignedBeaconBlock - Root [32]byte + Slot uint64 + Root [32]byte } func NewBackwardBeaconDownloader(ctx context.Context, rpc *rpc.BeaconRpcP2P, sn *freezeblocks.CaplinSnapshots, engine execution_client.ExecutionEngine, db kv.RwDB, beaconCfg *clparams.BeaconChainConfig) *BackwardBeaconDownloader { @@ -297,7 +307,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons // [New in Gloas:EIP7732] Fetch envelopes for GLOAS FULL blocks before processing. log.Debug("[BackwardBeaconDownloader] processResponses start", "blocks", len(responses), "slotToDownload", b.slotToDownload.Load(), "expectedRoot", b.expectedRoot) - envelopes, fullRootSet := b.fetchGloasEnvelopes(ctx, responses) + envelopes, fullRootSet, knownRootSet := b.fetchGloasEnvelopes(ctx, responses) log.Debug("[BackwardBeaconDownloader] envelopes fetched", "count", len(envelopes), "fullRoots", len(fullRootSet)) // Track whether any block was successfully processed. Only update @@ -330,11 +340,23 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons if envelopes != nil { envelope = envelopes[common.Hash(blockRoot)] } + if block.Version() >= clparams.GloasVersion { + if _, known := knownRootSet[common.Hash(blockRoot)]; !known { + log.Warn("[BackwardBeaconDownloader] GLOAS block availability unknown, will retry", "slot", block.Block.Slot) + return nil + } + if envelope != nil { + if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), envelope); err != nil { + log.Warn("[BackwardBeaconDownloader] GLOAS envelope does not match block, will retry", "slot", block.Block.Slot, "err", err) + return nil + } + } + } // A FULL block whose envelope could not be fetched must not be treated as // EMPTY — unless we've exhausted retries (envelopesSkipped is set when // consecutive batches fail envelope fetch entirely). - if _, isFull := fullRootSet[common.Hash(blockRoot)]; isFull && envelope == nil && !b.envelopesSkipped { + if _, isFull := fullRootSet[common.Hash(blockRoot)]; isFull && envelope == nil && (!b.envelopesSkipped || !b.canTrackSkippedFullBlock(block)) { log.Warn("[BackwardBeaconDownloader] GLOAS FULL block envelope missing, will retry", "slot", block.Block.Slot, "consecutiveFailures", b.consecutiveEnvelopeFailures) return nil @@ -349,10 +371,11 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons // Record FULL blocks passing through without envelope for post-download recovery. if _, isFull := fullRootSet[common.Hash(blockRoot)]; isFull && envelope == nil { - b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Block: block, Root: blockRoot}) + b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) } advanced = true + b.prevBatchTopBlock = block b.expectedRoot = block.Block.ParentRoot if block.Block.Slot == 0 { b.finished.Store(true) @@ -364,10 +387,6 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons // Update prevBatchTopBlock only when at least one block was processed, // so retries preserve the correct lookahead for FULL/EMPTY determination. - if advanced && len(responses) > 0 { - b.prevBatchTopBlock = firstCompleteBlock(responses) - } - if !matched { log.Debug("[BackwardBeaconDownloader] no root match in batch", "expectedRoot", b.expectedRoot, "responses", len(responses), "advanced", advanced) } @@ -385,14 +404,38 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons log.Debug("[BackwardBeaconDownloader] block matched via root lookup", "slot", block.Block.Slot, "root", common.Hash(blockRoot)) var envelope *cltypes.SignedExecutionPayloadEnvelope - if block.Version() >= clparams.GloasVersion && !b.envelopesSkipped { - env, fetchErr := b.fetchSingleEnvelope(ctx, block) - if fetchErr != nil { - log.Warn("[BackwardBeaconDownloader] GLOAS envelope fetch failed for root-fetched block, treating as EMPTY", - "slot", block.Block.Slot, "err", fetchErr) + isFull := false + if block.Version() >= clparams.GloasVersion { + lookahead := b.prevBatchTopBlock + if _, known := gloasBlockAvailability(block, lookahead); !known { + lookahead, err = b.fetchGloasLookahead(ctx, block, common.Hash(blockRoot)) + if err != nil { + log.Warn("[BackwardBeaconDownloader] root-fetched GLOAS lookahead unavailable, will retry", "slot", block.Block.Slot, "err", err) + return nil + } + } + full, known := gloasBlockAvailability(block, lookahead) + if !known { + log.Warn("[BackwardBeaconDownloader] root-fetched GLOAS block availability unknown, will retry", "slot", block.Block.Slot) + return nil + } + isFull = full + if full { + env, fetchErr := b.fetchSingleEnvelope(ctx, block) + if fetchErr == nil && env != nil { + if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err == nil { + envelope = env + } else { + log.Warn("[BackwardBeaconDownloader] root-fetched envelope does not match block", "slot", block.Block.Slot, "err", err) + } + } + b.recordEnvelopeFetchResult(1, btoi(envelope != nil)) + if envelope == nil && (!b.envelopesSkipped || !b.canTrackSkippedFullBlock(block)) { + log.Warn("[BackwardBeaconDownloader] root-fetched FULL block envelope unavailable, will retry", + "slot", block.Block.Slot, "err", fetchErr, "consecutiveFailures", b.consecutiveEnvelopeFailures) + return nil + } } - // env == nil && fetchErr == nil means HTTP 404: genuinely EMPTY. - envelope = env } finished, err := b.onNewBlock(block, envelope) @@ -400,6 +443,10 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons if err != nil { log.Warn("Error processing root-fetched block", "err", err) } else { + if isFull && envelope == nil { + b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) + } + b.prevBatchTopBlock = block b.expectedRoot = block.Block.ParentRoot if block.Block.Slot == 0 { b.finished.Store(true) @@ -414,6 +461,13 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } +func btoi(value bool) int { + if value { + return 1 + } + return 0 +} + func firstCompleteBlock(responses []*cltypes.SignedBeaconBlock) *cltypes.SignedBeaconBlock { for _, block := range responses { if block != nil && block.Block != nil && block.Block.Body != nil { @@ -432,14 +486,27 @@ func lastCompleteBlock(responses []*cltypes.SignedBeaconBlock) *cltypes.SignedBe return nil } +func blockByRoot(responses []*cltypes.SignedBeaconBlock, expectedRoot common.Hash) *cltypes.SignedBeaconBlock { + for _, block := range responses { + if block == nil || block.Block == nil || block.Block.Body == nil { + continue + } + root, err := block.Block.HashSSZ() + if err == nil && root == expectedRoot { + return block + } + } + return nil +} + func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Context, responses []*cltypes.SignedBeaconBlock) error { if b.prevBatchTopBlock != nil || len(responses) == 0 { return nil } - anchor := lastCompleteBlock(responses) + anchor := blockByRoot(responses, b.expectedRoot) if anchor == nil { - return errors.New("batch has no complete blocks") + return nil } if anchor.Version() < clparams.GloasVersion { return nil @@ -449,7 +516,11 @@ func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Contex if err != nil { return fmt.Errorf("hash highest block: %w", err) } - if anchorRoot != b.expectedRoot { + if lookahead := selectGloasLookahead(anchor, anchorRoot, responses); lookahead != nil { + b.prevBatchTopBlock = lookahead + b.consecutiveLookaheadFailures = 0 + b.lookaheadSearchOffset = 0 + b.lookaheadRescan = false return nil } @@ -459,6 +530,8 @@ func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Contex } b.prevBatchTopBlock = lookahead b.consecutiveLookaheadFailures = 0 + b.lookaheadSearchOffset = 0 + b.lookaheadRescan = false return nil } @@ -467,42 +540,75 @@ func (b *BackwardBeaconDownloader) fetchGloasLookahead( anchor *cltypes.SignedBeaconBlock, anchorRoot common.Hash, ) (*cltypes.SignedBeaconBlock, error) { + if b.lookaheadAnchorRoot != anchorRoot { + b.lookaheadAnchorRoot = anchorRoot + b.lookaheadSearchOffset = 0 + b.lookaheadRescan = false + } if anchor.Block.Slot == math.MaxUint64 { return nil, errors.New("cannot fetch lookahead after max slot") } - const lookaheadWindow = uint64(64) - start := anchor.Block.Slot + 1 - var httpErr error + if b.lookaheadSearchOffset > math.MaxUint64-anchor.Block.Slot-1 { + return nil, errors.New("GLOAS lookahead search overflow") + } + offset := b.lookaheadSearchOffset + if b.lookaheadRescan { + offset = 0 + } + start := anchor.Block.Slot + 1 + offset + sources := make([]gloasLookaheadFetcher, 0, 2) if b.httpFallbackURL != "" { - candidates, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, lookaheadWindow, b.beaconCfg) - if err == nil { - if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { - return lookahead, nil - } - httpErr = errors.New("beacon API returned no direct child") - } else { - httpErr = err - } + sources = append(sources, func(ctx context.Context, start, count uint64) ([]*cltypes.SignedBeaconBlock, error) { + return fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, count, b.beaconCfg) + }) } - if b.rpc != nil { - candidates, _, err := b.rpc.SendBeaconBlocksByRangeReq(ctx, start, lookaheadWindow) - if err == nil { - if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { - return lookahead, nil - } - return nil, errors.New("peer returned no direct child") + sources = append(sources, func(ctx context.Context, start, count uint64) ([]*cltypes.SignedBeaconBlock, error) { + blocks, _, err := b.rpc.SendBeaconBlocksByRangeReq(ctx, start, count) + return blocks, err + }) + } + if len(sources) == 0 { + return nil, errors.New("no GLOAS lookahead source configured") + } + lookahead, err := fetchGloasLookaheadFromSources(ctx, anchor, anchorRoot, start, sources...) + if lookahead != nil { + return lookahead, nil + } + b.advanceLookaheadSearch() + return nil, err +} + +type gloasLookaheadFetcher func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) + +func fetchGloasLookaheadFromSources(ctx context.Context, anchor *cltypes.SignedBeaconBlock, anchorRoot common.Hash, start uint64, sources ...gloasLookaheadFetcher) (*cltypes.SignedBeaconBlock, error) { + errs := make([]error, 0, len(sources)) + for _, fetch := range sources { + candidates, err := fetch(ctx, start, gloasLookaheadWindow) + if err != nil { + errs = append(errs, err) + continue } - if httpErr != nil { - return nil, errors.Join(httpErr, err) + if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { + return lookahead, nil } - return nil, err + errs = append(errs, errors.New("lookahead source returned no direct child")) + } + return nil, errors.Join(errs...) +} + +func (b *BackwardBeaconDownloader) advanceLookaheadSearch() { + if b.lookaheadRescan { + b.lookaheadRescan = false + b.lookaheadSearchOffset += gloasLookaheadWindow + return } - if httpErr != nil { - return nil, httpErr + if b.lookaheadSearchOffset == 0 { + b.lookaheadSearchOffset = gloasLookaheadWindow + return } - return nil, errors.New("no GLOAS lookahead source configured") + b.lookaheadRescan = true } func selectGloasLookahead( @@ -542,56 +648,78 @@ func (b *BackwardBeaconDownloader) waitBeforeLookaheadRetry(ctx context.Context) } } -// determineGloasFullRoots returns the block roots of GLOAS FULL blocks in the batch. -// Uses the count+1 lookahead trick: block[i] is FULL if block[i+1].bid.ParentBlockHash == block[i].bid.BlockHash. -// For the highest block in the batch, prevBatchTopBlock is used as the cross-batch lookahead. func determineGloasFullRoots(responses []*cltypes.SignedBeaconBlock, prevBatchTopBlock *cltypes.SignedBeaconBlock) [][32]byte { + anchor := lastCompleteBlock(responses) + if anchor == nil { + return nil + } + expectedRoot, err := anchor.Block.HashSSZ() + if err != nil { + return nil + } + fullRoots, _ := determineGloasAvailability(responses, prevBatchTopBlock, expectedRoot) + return fullRoots +} + +func determineGloasAvailability( + responses []*cltypes.SignedBeaconBlock, + lookahead *cltypes.SignedBeaconBlock, + expectedRoot common.Hash, +) ([][32]byte, map[common.Hash]struct{}) { var fullRoots [][32]byte - for i, block := range responses { + knownRoots := make(map[common.Hash]struct{}) + for _, block := range slices.Backward(responses) { if block == nil || block.Block == nil || block.Block.Body == nil { continue } - if block.Version() < clparams.GloasVersion { - continue - } - bid := block.Block.Body.GetSignedExecutionPayloadBid() - if bid == nil || bid.Message == nil { - continue - } - // Determine the lookahead block (next higher slot in the chain). - var lookahead *cltypes.SignedBeaconBlock - if i+1 < len(responses) { - lookahead = responses[i+1] - } else { - lookahead = prevBatchTopBlock - } - if lookahead == nil { + root, err := block.Block.HashSSZ() + if err != nil || root != expectedRoot { continue } - if lookahead.Block == nil || lookahead.Block.Body == nil { + if block.Version() < clparams.GloasVersion { + lookahead = block + expectedRoot = block.Block.ParentRoot continue } - nextBid := lookahead.Block.Body.GetSignedExecutionPayloadBid() - if nextBid != nil && nextBid.Message != nil && nextBid.Message.ParentBlockHash == bid.Message.BlockHash { - root, err := block.Block.HashSSZ() - if err == nil { + full, known := gloasBlockAvailability(block, lookahead) + if known { + knownRoots[common.Hash(root)] = struct{}{} + if full { fullRoots = append(fullRoots, root) } } + lookahead = block + expectedRoot = block.Block.ParentRoot } - return fullRoots + return fullRoots, knownRoots +} + +func gloasBlockAvailability(block, lookahead *cltypes.SignedBeaconBlock) (bool, bool) { + if block == nil || block.Block == nil || block.Block.Body == nil || lookahead == nil || lookahead.Block == nil || lookahead.Block.Body == nil { + return false, false + } + root, err := block.Block.HashSSZ() + if err != nil || lookahead.Block.Slot <= block.Block.Slot || lookahead.Block.ParentRoot != root { + return false, false + } + bid := block.Block.Body.GetSignedExecutionPayloadBid() + nextBid := lookahead.Block.Body.GetSignedExecutionPayloadBid() + if bid == nil || bid.Message == nil || nextBid == nil || nextBid.Message == nil { + return false, false + } + return nextBid.Message.ParentBlockHash == bid.Message.BlockHash, true } // fetchGloasEnvelopes determines which GLOAS blocks in the batch are FULL and fetches their envelopes. // It returns the envelopes map and a set of block roots that were determined FULL by lookahead. // Callers must check: if a root is in fullRootSet but missing from envelopes, the fetch failed // and the block must NOT be treated as EMPTY. -func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, responses []*cltypes.SignedBeaconBlock) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, map[common.Hash]struct{}) { +func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, responses []*cltypes.SignedBeaconBlock) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, map[common.Hash]struct{}, map[common.Hash]struct{}) { if len(responses) == 0 { - return nil, nil + return nil, nil, nil } - fullRoots := determineGloasFullRoots(responses, b.prevBatchTopBlock) + fullRoots, knownRootSet := determineGloasAvailability(responses, b.prevBatchTopBlock, b.expectedRoot) // Build a set for O(1) lookup by callers. fullRootSet := make(map[common.Hash]struct{}, len(fullRoots)) @@ -599,36 +727,47 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp fullRootSet[common.Hash(r)] = struct{}{} } - if len(fullRoots) == 0 || b.envelopesSkipped { - return nil, fullRootSet + if len(fullRoots) == 0 { + return nil, fullRootSet, knownRootSet } var envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope if b.httpPreferred.Load() && b.httpFallbackURL != "" { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fullRoots)) - fetched := fetchEnvelopesFromBeaconAPI(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) - if fetched > 0 { - log.Debug("[BackwardBeaconDownloader] fetched envelopes from beacon API", "count", fetched) - } + envelopes = validateAndFetchMissingEnvelopes(ctx, b.httpFallbackURL, responses, fullRoots, nil, b.beaconCfg) } else { var err error envelopes, err = RequestEnvelopesFrantically(ctx, b.rpc, fullRoots) if err != nil { log.Debug("[BackwardBeaconDownloader] failed to fetch GLOAS envelopes via P2P", "err", err) } - if b.httpFallbackURL != "" && len(envelopes) < len(fullRoots) { - if envelopes == nil { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fullRoots)) - } - fetched := fetchEnvelopesFromBeaconAPI(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) - if fetched > 0 { - log.Debug("[BackwardBeaconDownloader] fetched envelopes from beacon API", "count", fetched) - } + envelopes = validateAndFetchMissingEnvelopes(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) + } + + b.recordEnvelopeFetchResult(len(fullRoots), len(envelopes)) + + return envelopes, fullRootSet, knownRootSet +} + +func validateFetchedEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks []*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + valid := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(envelopes)) + for _, block := range blocks { + if block == nil || block.Block == nil || block.Block.Body == nil { + continue + } + root, err := block.Block.HashSSZ() + if err != nil { + continue + } + envelope := envelopes[common.Hash(root)] + if ValidateFetchedEnvelope(beaconCfg, block, common.Hash(root), envelope) == nil { + valid[common.Hash(root)] = envelope } } + return valid +} - // Track consecutive batches where no envelopes could be fetched for FULL roots. - if len(envelopes) == 0 { +func (b *BackwardBeaconDownloader) recordEnvelopeFetchResult(requested, received int) { + if received < requested { b.consecutiveEnvelopeFailures++ const maxConsecutiveFailures = 3 if b.consecutiveEnvelopeFailures >= maxConsecutiveFailures && !b.envelopesSkipped { @@ -636,12 +775,10 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp log.Warn("[BackwardBeaconDownloader] too many consecutive envelope failures, treating FULL blocks as EMPTY", "consecutiveFailures", b.consecutiveEnvelopeFailures) } - } else { - b.consecutiveEnvelopeFailures = 0 - b.envelopesSkipped = false + return } - - return envelopes, fullRootSet + b.consecutiveEnvelopeFailures = 0 + b.envelopesSkipped = false } // SkippedFullBlocks returns FULL blocks that were processed without envelopes @@ -650,39 +787,100 @@ func (b *BackwardBeaconDownloader) SkippedFullBlocks() []SkippedFullBlock { return b.skippedFullBlocks } +func (b *BackwardBeaconDownloader) canTrackSkippedFullBlock(block *cltypes.SignedBeaconBlock) bool { + return block != nil && len(b.skippedFullBlocks) < maxSkippedFullBlocks +} + +func ValidateFetchedEnvelope(beaconCfg *clparams.BeaconChainConfig, block *cltypes.SignedBeaconBlock, blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) error { + if block == nil || block.Block == nil || block.Block.Body == nil || envelope == nil || envelope.Message == nil || envelope.Message.Payload == nil || envelope.Message.ExecutionRequests == nil { + return errors.New("incomplete block or envelope") + } + if envelope.Message.BeaconBlockRoot != blockRoot { + return fmt.Errorf("envelope beacon root %v != block root %v", envelope.Message.BeaconBlockRoot, blockRoot) + } + payload := envelope.Message.Payload + if payload.SlotNumber != block.Block.Slot { + return fmt.Errorf("envelope slot %d != block slot %d", envelope.Message.Payload.SlotNumber, block.Block.Slot) + } + bid := block.Block.Body.GetSignedExecutionPayloadBid() + if bid == nil || bid.Message == nil { + return errors.New("block missing execution payload bid") + } + committed := bid.Message + if envelope.Message.ParentBeaconBlockRoot != committed.ParentBlockRoot || envelope.Message.BuilderIndex != committed.BuilderIndex { + return errors.New("envelope metadata does not match committed bid") + } + if payload.BlockHash != committed.BlockHash || payload.ParentHash != committed.ParentBlockHash || payload.PrevRandao != committed.PrevRandao || payload.FeeRecipient != committed.FeeRecipient || payload.GasLimit != committed.GasLimit || payload.SlotNumber != committed.Slot { + return errors.New("envelope payload does not match committed bid") + } + requestsRoot, err := envelope.Message.ExecutionRequests.HashSSZ() + if err != nil { + return fmt.Errorf("hash execution requests: %w", err) + } + if requestsRoot != committed.ExecutionRequestsRoot { + return errors.New("envelope execution requests do not match committed bid") + } + requestsHash := cltypes.ComputeExecutionRequestHash(cltypes.GetExecutionRequestsList(beaconCfg, envelope.Message.ExecutionRequests)) + header, err := payload.RlpHeader(&envelope.Message.ParentBeaconBlockRoot, requestsHash) + if err != nil { + return fmt.Errorf("build execution payload header: %w", err) + } + if header.Hash() != payload.BlockHash { + return errors.New("execution payload block hash does not match payload contents") + } + return nil +} + // RecoverSkippedEnvelopes retries fetching envelopes for blocks that were // skipped during backward download. Returns a map of successfully fetched // envelopes keyed by beacon block root. -func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { - if len(b.skippedFullBlocks) == 0 { +func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, skipped []SkippedFullBlock) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + if len(skipped) == 0 { return nil } - roots := make([][32]byte, len(b.skippedFullBlocks)) - for i, s := range b.skippedFullBlocks { + roots := make([][32]byte, len(skipped)) + for i, s := range skipped { roots[i] = s.Root } - envelopes, err := RequestEnvelopesFrantically(ctx, b.rpc, roots) - if err != nil { - log.Debug("[BackwardBeaconDownloader] envelope recovery: P2P failed", "err", err) + envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) + if b.httpFallbackURL != "" { + b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped, envelopes) } - if envelopes == nil { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) + missingRoots := make([][32]byte, 0, len(roots)-len(envelopes)) + for _, root := range roots { + if _, ok := envelopes[common.Hash(root)]; !ok { + missingRoots = append(missingRoots, root) + } } - - // HTTP fallback for roots still missing after P2P. - if b.httpFallbackURL != "" && len(envelopes) < len(b.skippedFullBlocks) { - blocks := make([]*cltypes.SignedBeaconBlock, len(b.skippedFullBlocks)) - for i, s := range b.skippedFullBlocks { - blocks[i] = s.Block + if b.rpc != nil && len(missingRoots) > 0 && ctx.Err() == nil { + var err error + p2pEnvelopes, err := RequestEnvelopesFrantically(ctx, b.rpc, missingRoots) + if err != nil { + log.Debug("[BackwardBeaconDownloader] envelope recovery: P2P failed", "err", err) } - fetchEnvelopesFromBeaconAPI(ctx, b.httpFallbackURL, blocks, roots, envelopes, b.beaconCfg) + maps.Copy(envelopes, p2pEnvelopes) } return envelopes } +func (b *BackwardBeaconDownloader) fetchSkippedEnvelopesFromBeaconAPI(ctx context.Context, skipped []SkippedFullBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) { + for _, item := range skipped { + root := common.Hash(item.Root) + if _, ok := envelopes[root]; ok { + continue + } + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: item.Slot}} + envelope, err := b.fetchSingleEnvelope(ctx, block) + if err != nil || envelope == nil || envelope.Message == nil || envelope.Message.BeaconBlockRoot != root { + continue + } + envelopes[root] = envelope + } +} + // trySkipToExistingBlock attempts to skip ahead if the expected block already exists in the database. func (b *BackwardBeaconDownloader) trySkipToExistingBlock(ctx context.Context) error { tx, err := b.db.BeginRw(b.ctx) @@ -814,17 +1012,17 @@ func fetchBlockFromBeaconAPIByRoot(ctx context.Context, baseURL string, root com if err != nil { return nil, err } - body, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, err - } + defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return nil, nil } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("block fetch by root: status %d", resp.StatusCode) } + body, err := readBoundedBeaconAPIResponse(resp.Body, maxBeaconAPIResponseBytes) + if err != nil { + return nil, err + } version := httpConsensusVersion(resp.Header.Get("Eth-Consensus-Version")) block := cltypes.NewSignedBeaconBlock(beaconCfg, version) @@ -854,17 +1052,17 @@ func (b *BackwardBeaconDownloader) fetchSingleEnvelope(ctx context.Context, bloc if err != nil { return nil, err } - body, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, err - } + defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { - return nil, nil // genuinely EMPTY: beacon API confirms no envelope + return nil, nil } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("envelope fetch: HTTP %d", resp.StatusCode) } + body, err := readBoundedBeaconAPIResponse(resp.Body, maxBeaconAPIResponseBytes) + if err != nil { + return nil, err + } envelope := &cltypes.SignedExecutionPayloadEnvelope{ Message: cltypes.NewExecutionPayloadEnvelope(b.beaconCfg), @@ -874,3 +1072,14 @@ func (b *BackwardBeaconDownloader) fetchSingleEnvelope(ctx context.Context, bloc } return envelope, nil } + +func readBoundedBeaconAPIResponse(body io.Reader, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("beacon API response exceeds %d bytes", limit) + } + return data, nil +} diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index fd6b54cd657..c88d3ec6dda 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -17,6 +17,7 @@ package network import ( + "bytes" "context" "net/http" "net/http/httptest" @@ -29,6 +30,7 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/common" ) @@ -57,6 +59,42 @@ func hash(b byte) common.Hash { return h } +func linkGloasBlocks(t *testing.T, parent, child *cltypes.SignedBeaconBlock) { + root, err := parent.Block.HashSSZ() + require.NoError(t, err) + child.Block.ParentRoot = root +} + +func makeValidGloasEnvelope(t *testing.T, block *cltypes.SignedBeaconBlock) ([32]byte, *cltypes.SignedExecutionPayloadEnvelope) { + requests := cltypes.NewExecutionRequestsWithVersion(&clparams.MainnetBeaconConfig, clparams.GloasVersion) + requestsRoot, err := requests.HashSSZ() + require.NoError(t, err) + bid := block.Block.Body.GetSignedExecutionPayloadBid().Message + bid.ExecutionRequestsRoot = requestsRoot + bid.Slot = block.Block.Slot + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.Payload.Extra = solid.NewExtraData() + envelope.Message.Payload.Transactions = &solid.TransactionsSSZ{} + envelope.Message.Payload.Withdrawals = solid.NewStaticListSSZ[*cltypes.Withdrawal](int(clparams.MainnetBeaconConfig.MaxWithdrawalsPerPayload), 44) + envelope.Message.Payload.BlockAccessList = solid.NewByteListSSZ(clparams.MainnetBeaconConfig.MaxBytesPerTransaction) + envelope.Message.Payload.SlotNumber = bid.Slot + envelope.Message.Payload.ParentHash = bid.ParentBlockHash + envelope.Message.Payload.PrevRandao = bid.PrevRandao + envelope.Message.Payload.FeeRecipient = bid.FeeRecipient + envelope.Message.Payload.GasLimit = bid.GasLimit + envelope.Message.BuilderIndex = bid.BuilderIndex + envelope.Message.Payload.BlockHash = common.HexToHash("0x1da54a16ef5d8bd1d1559378bbdea3b084b58d1ff1e3db53c276a3ecd6c3ceb6") + requestsHash := cltypes.ComputeExecutionRequestHash(cltypes.GetExecutionRequestsList(&clparams.MainnetBeaconConfig, envelope.Message.ExecutionRequests)) + header, err := envelope.Message.Payload.RlpHeader(&envelope.Message.ParentBeaconBlockRoot, requestsHash) + require.NoError(t, err) + bid.BlockHash = header.Hash() + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + envelope.Message.BeaconBlockRoot = blockRoot + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, blockRoot, envelope)) + return blockRoot, envelope +} + // TestDetermineGloasFullRoots_EmptyBatch verifies that an empty batch returns no roots. func TestDetermineGloasFullRoots_EmptyBatch(t *testing.T) { roots := determineGloasFullRoots(nil, nil) @@ -103,6 +141,7 @@ func TestDetermineGloasFullRoots_InBatch_Full(t *testing.T) { // blk0 is FULL: blk1.ParentBlockHash == blk0.BlockHash blk0 := makeGloasBlock(100, hash(0xAA), hash(0x00)) blk1 := makeGloasBlock(101, hash(0xBB), hash(0xAA)) // ParentBlockHash = blk0.BlockHash + linkGloasBlocks(t, blk0, blk1) responses := []*cltypes.SignedBeaconBlock{blk0, blk1} roots := determineGloasFullRoots(responses, nil) @@ -119,6 +158,7 @@ func TestDetermineGloasFullRoots_InBatch_Empty(t *testing.T) { // blk0 is EMPTY: blk1.ParentBlockHash != blk0.BlockHash blk0 := makeGloasBlock(100, hash(0xAA), hash(0x00)) blk1 := makeGloasBlock(101, hash(0xBB), hash(0xCC)) // ParentBlockHash != blk0.BlockHash + linkGloasBlocks(t, blk0, blk1) responses := []*cltypes.SignedBeaconBlock{blk0, blk1} roots := determineGloasFullRoots(responses, nil) @@ -131,6 +171,7 @@ func TestDetermineGloasFullRoots_CrossBatch_Full(t *testing.T) { blk := makeGloasBlock(100, hash(0xAA), hash(0x00)) // prevBatchTopBlock is from the previous (higher-slot) batch; its ParentBlockHash = blk.BlockHash prevTop := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + linkGloasBlocks(t, blk, prevTop) responses := []*cltypes.SignedBeaconBlock{blk} roots := determineGloasFullRoots(responses, prevTop) @@ -147,6 +188,7 @@ func TestDetermineGloasFullRoots_CrossBatch_Empty(t *testing.T) { blk := makeGloasBlock(100, hash(0xAA), hash(0x00)) // prevBatchTopBlock's ParentBlockHash != blk.BlockHash → blk is EMPTY prevTop := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + linkGloasBlocks(t, blk, prevTop) responses := []*cltypes.SignedBeaconBlock{blk} roots := determineGloasFullRoots(responses, prevTop) @@ -164,6 +206,9 @@ func TestDetermineGloasFullRoots_Mixed(t *testing.T) { blk1 := makeGloasBlock(101, hash(0x20), hash(0x10)) // parent = blk0.hash → blk0 FULL blk2 := makeGloasBlock(102, hash(0x30), hash(0xFF)) // parent != blk1.hash → blk1 EMPTY blk3 := makeGloasBlock(103, hash(0x40), hash(0x30)) // parent = blk2.hash → blk2 FULL + linkGloasBlocks(t, blk0, blk1) + linkGloasBlocks(t, blk1, blk2) + linkGloasBlocks(t, blk2, blk3) responses := []*cltypes.SignedBeaconBlock{blk0, blk1, blk2, blk3} roots := determineGloasFullRoots(responses, nil) @@ -186,6 +231,7 @@ func TestDetermineGloasFullRoots_MixedVersions(t *testing.T) { gloasFull := makeGloasBlock(100, hash(0xAA), hash(0x00)) // lookahead confirms gloasFull is FULL lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + linkGloasBlocks(t, gloasFull, lookahead) responses := []*cltypes.SignedBeaconBlock{deneb, gloasFull, lookahead} roots := determineGloasFullRoots(responses, nil) @@ -283,7 +329,11 @@ func TestBackwardBeaconDownloaderHTTPPreferredMissingEnvelopeTracksFailure(t *te } downloader.httpPreferred.Store(true) - envelopes, fullRoots := downloader.fetchGloasEnvelopes( + linkGloasBlocks(t, block, lookahead) + lookaheadRoot, err := lookahead.Block.HashSSZ() + require.NoError(t, err) + downloader.expectedRoot = lookaheadRoot + envelopes, fullRoots, _ := downloader.fetchGloasEnvelopes( context.Background(), []*cltypes.SignedBeaconBlock{block, lookahead}, ) @@ -313,6 +363,250 @@ func TestSelectGloasLookaheadRejectsUnlinkedAndIncompleteBlocks(t *testing.T) { assert.Same(t, linked, selected) } +func TestGloasBlockAvailabilityRejectsUnlinkedLookahead(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = hash(0xFF) + + full, known := gloasBlockAvailability(block, lookahead) + assert.False(t, full) + assert.False(t, known) +} + +func TestDetermineFullGloasRootsRequiresCanonicalChild(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + child := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + child.Block.ParentRoot = hash(0xFF) + + assert.Empty(t, determineFullGloasRoots([]*cltypes.SignedBeaconBlock{block, child}, 1)) + linkGloasBlocks(t, block, child) + require.Len(t, determineFullGloasRoots([]*cltypes.SignedBeaconBlock{block, child}, 1), 1) + + child.Block.Slot = block.Block.Slot + assert.Empty(t, determineFullGloasRoots([]*cltypes.SignedBeaconBlock{block, child}, 1)) +} + +func TestForwardGloasHelpersIgnoreIncompleteBlocks(t *testing.T) { + incomplete := []*cltypes.SignedBeaconBlock{nil, {}} + assert.NotPanics(t, func() { + assert.False(t, anyGloasBlock(incomplete)) + assert.Empty(t, determineFullGloasRoots(incomplete, len(incomplete))) + }) +} + +func TestCompleteBeaconBlocksFiltersBeforeForwardProcessing(t *testing.T) { + valid := makeDenebBlock(100) + blocks := []*cltypes.SignedBeaconBlock{nil, {}, {Block: &cltypes.BeaconBlock{}}, valid} + + got := completeBeaconBlocks(blocks) + require.Len(t, got, 1) + assert.Same(t, valid, got[0]) +} + +func TestFetchGloasLookaheadAdvancesPastFirstWindow(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(165, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = anchorRoot + encoded, err := lookahead.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v2/beacon/blocks/165" { + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encoded) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig} + _, err = downloader.fetchGloasLookahead(context.Background(), anchor, anchorRoot) + require.Error(t, err) + require.Equal(t, uint64(64), downloader.lookaheadSearchOffset) + got, err := downloader.fetchGloasLookahead(context.Background(), anchor, anchorRoot) + require.NoError(t, err) + require.Equal(t, uint64(165), got.Block.Slot) +} + +func TestFetchGloasLookaheadRescansEarlierWindow(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = anchorRoot + encoded, err := lookahead.EncodeSSZ(nil) + require.NoError(t, err) + + var available atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if available.Load() && r.URL.Path == "/eth/v2/beacon/blocks/101" { + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encoded) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig} + _, err = downloader.fetchGloasLookahead(context.Background(), anchor, anchorRoot) + require.Error(t, err) + available.Store(true) + _, err = downloader.fetchGloasLookahead(context.Background(), anchor, anchorRoot) + require.Error(t, err) + got, err := downloader.fetchGloasLookahead(context.Background(), anchor, anchorRoot) + require.NoError(t, err) + require.Equal(t, uint64(101), got.Block.Slot) +} + +func TestFetchGloasLookaheadFromSourcesFallsBackAfterMissOrError(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + child := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + child.Block.ParentRoot = anchorRoot + + for _, first := range []gloasLookaheadFetcher{ + func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) { return nil, nil }, + func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) { + return nil, assert.AnError + }, + } { + secondCalled := false + got, fetchErr := fetchGloasLookaheadFromSources(context.Background(), anchor, anchorRoot, 101, first, + func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) { + secondCalled = true + return []*cltypes.SignedBeaconBlock{child}, nil + }) + require.NoError(t, fetchErr) + assert.True(t, secondCalled) + assert.Same(t, child, got) + } +} + +func TestBlockByRootFindsExpectedBlockInMiddle(t *testing.T) { + first := makeGloasBlock(100, hash(0x10), hash(0x00)) + expected := makeGloasBlock(101, hash(0x20), hash(0x10)) + last := makeGloasBlock(102, hash(0x30), hash(0x20)) + expectedRoot, err := expected.Block.HashSSZ() + require.NoError(t, err) + require.Same(t, expected, blockByRoot([]*cltypes.SignedBeaconBlock{first, expected, last}, expectedRoot)) +} + +func TestRootFallbackMissingEnvelopeEntersBoundedRecovery(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + child := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + linkGloasBlocks(t, block, child) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + encoded, err := block.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v2/beacon/blocks/"+common.Hash(blockRoot).Hex() { + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encoded) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + processed := 0 + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: child, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(_ *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed++ + assert.Nil(t, envelope) + return false, nil + }, + } + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), nil)) + } + assert.Equal(t, 1, processed) + require.Len(t, downloader.skippedFullBlocks, 1) + assert.Equal(t, common.Hash(blockRoot), common.Hash(downloader.skippedFullBlocks[0].Root)) +} + +func TestValidateFetchedEnvelopeRejectsDifferentBeaconRoot(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + mutations := map[string]func(*cltypes.ExecutionPayloadEnvelope){ + "beacon root": func(e *cltypes.ExecutionPayloadEnvelope) { e.BeaconBlockRoot[0]++ }, + "parent beacon root": func(e *cltypes.ExecutionPayloadEnvelope) { e.ParentBeaconBlockRoot[0]++ }, + "builder index": func(e *cltypes.ExecutionPayloadEnvelope) { e.BuilderIndex++ }, + "parent hash": func(e *cltypes.ExecutionPayloadEnvelope) { e.Payload.ParentHash[0]++ }, + "prev randao": func(e *cltypes.ExecutionPayloadEnvelope) { e.Payload.PrevRandao[0]++ }, + "fee recipient": func(e *cltypes.ExecutionPayloadEnvelope) { e.Payload.FeeRecipient[0]++ }, + "gas limit": func(e *cltypes.ExecutionPayloadEnvelope) { e.Payload.GasLimit++ }, + "slot": func(e *cltypes.ExecutionPayloadEnvelope) { e.Payload.SlotNumber++ }, + "payload contents": func(e *cltypes.ExecutionPayloadEnvelope) { e.Payload.GasUsed++ }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + changed := envelope.Clone().(*cltypes.SignedExecutionPayloadEnvelope) + mutate(changed.Message) + require.Error(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, blockRoot, changed)) + }) + } +} + +func TestMalformedP2PEnvelopeFallsBackToValidHTTPEnvelope(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, valid := makeValidGloasEnvelope(t, block) + encoded, err := valid.EncodeSSZ(nil) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/eth/v1/beacon/execution_payload_envelope/100" { + http.NotFound(w, r) + return + } + _, _ = w.Write(encoded) + })) + defer server.Close() + + malformed := valid.Clone().(*cltypes.SignedExecutionPayloadEnvelope) + malformed.Message.Payload.GasUsed++ + got := validateAndFetchMissingEnvelopes( + context.Background(), server.URL, []*cltypes.SignedBeaconBlock{block}, [][32]byte{blockRoot}, + map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{common.Hash(blockRoot): malformed}, &clparams.MainnetBeaconConfig, + ) + require.Len(t, got, 1) + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), got[common.Hash(blockRoot)])) +} + +func TestValidateFetchedEnvelopesDropsMalformedSameRoot(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + + got := validateFetchedEnvelopes(&clparams.MainnetBeaconConfig, []*cltypes.SignedBeaconBlock{block}, map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{blockRoot: envelope}) + assert.Empty(t, got) +} + +func TestSkippedFullBlockMemoryBudget(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + downloader := &BackwardBeaconDownloader{skippedFullBlocks: make([]SkippedFullBlock, maxSkippedFullBlocks-1)} + require.True(t, downloader.canTrackSkippedFullBlock(block)) + downloader.skippedFullBlocks = append(downloader.skippedFullBlocks, SkippedFullBlock{}) + require.False(t, downloader.canTrackSkippedFullBlock(block)) +} + +func TestReadBoundedBeaconAPIResponseRejectsOversize(t *testing.T) { + _, err := readBoundedBeaconAPIResponse(bytes.NewReader(make([]byte, 9)), 8) + require.Error(t, err) +} + func TestFetchEnvelopesFromBeaconAPIIncompleteBlock(t *testing.T) { server := httptest.NewServer(http.NotFoundHandler()) defer server.Close() diff --git a/cl/phase1/network/beacon_downloader.go b/cl/phase1/network/beacon_downloader.go index 009e4263f27..e04da00d13b 100644 --- a/cl/phase1/network/beacon_downloader.go +++ b/cl/phase1/network/beacon_downloader.go @@ -21,7 +21,6 @@ import ( "context" "errors" "fmt" - "io" "math" "net/http" "slices" @@ -258,8 +257,14 @@ func (f *ForwardBeaconDownloader) RequestMore(ctx context.Context) { Process: resp := atomicResp.Load().(peerAndBlocks) - processBlocks := resp.blocks + processBlocks := completeBeaconBlocks(resp.blocks) pid := resp.peerId + if len(processBlocks) == 0 { + if pid != "" && pid != "http-fallback" && f.rpc != nil { + f.rpc.BanPeer(pid) + } + return + } slices.SortFunc(processBlocks, func(a, b *cltypes.SignedBeaconBlock) int { return cmp.Compare(a.Block.Slot, b.Block.Slot) @@ -290,27 +295,14 @@ Process: // When blocks came from HTTP fallback, P2P is known-broken for this // batch — skip the 30s P2P envelope timeout and fetch directly via HTTP. if pid == "http-fallback" && f.httpFallbackURL != "" { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) - httpEnvs := fetchEnvelopesFromBeaconAPI(ctx, f.httpFallbackURL, processBlocks, fullRoots, envelopes, f.beaconCfg) - if httpEnvs > 0 { - log.Debug("[ForwardBeaconDownloader] fetched envelopes from beacon API", "count", httpEnvs) - } + envelopes = validateAndFetchMissingEnvelopes(ctx, f.httpFallbackURL, processBlocks, fullRoots, nil, f.beaconCfg) } else { var envErr error envelopes, envErr = RequestEnvelopesFrantically(ctx, f.rpc, fullRoots, processBlocks...) if envErr != nil { log.Debug("[ForwardBeaconDownloader] failed to get envelopes via P2P", "err", envErr) } - // HTTP fallback for envelopes when P2P returned incomplete results - if f.httpFallbackURL != "" && len(envelopes) < len(fullRoots) { - if envelopes == nil { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) - } - httpEnvs := fetchEnvelopesFromBeaconAPI(ctx, f.httpFallbackURL, processBlocks, fullRoots, envelopes, f.beaconCfg) - if httpEnvs > 0 { - log.Debug("[ForwardBeaconDownloader] fetched envelopes from beacon API", "count", httpEnvs) - } - } + envelopes = validateAndFetchMissingEnvelopes(ctx, f.httpFallbackURL, processBlocks, fullRoots, envelopes, f.beaconCfg) } log.Debug("[ForwardBeaconDownloader] envelope fetch result", "fullRoots", len(fullRoots), "received", len(envelopes), @@ -340,10 +332,28 @@ Process: } } +func validateAndFetchMissingEnvelopes(ctx context.Context, httpFallbackURL string, blocks []*cltypes.SignedBeaconBlock, fullRoots [][32]byte, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, beaconCfg *clparams.BeaconChainConfig) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + envelopes = validateFetchedEnvelopes(beaconCfg, blocks, envelopes) + if httpFallbackURL != "" && len(envelopes) < len(fullRoots) { + fetchEnvelopesFromBeaconAPI(ctx, httpFallbackURL, blocks, fullRoots, envelopes, beaconCfg) + } + return validateFetchedEnvelopes(beaconCfg, blocks, envelopes) +} + +func completeBeaconBlocks(blocks []*cltypes.SignedBeaconBlock) []*cltypes.SignedBeaconBlock { + complete := make([]*cltypes.SignedBeaconBlock, 0, len(blocks)) + for _, block := range blocks { + if block != nil && block.Block != nil && block.Block.Body != nil { + complete = append(complete, block) + } + } + return complete +} + // anyGloasBlock returns true if any block in the list is GLOAS version or later. func anyGloasBlock(blocks []*cltypes.SignedBeaconBlock) bool { for _, block := range blocks { - if block.Version() >= clparams.GloasVersion { + if block != nil && block.Block != nil && block.Block.Body != nil && block.Version() >= clparams.GloasVersion { return true } } @@ -362,6 +372,9 @@ func determineFullGloasRoots(blocks []*cltypes.SignedBeaconBlock, processCount i var roots [][32]byte for i := 0; i < processCount && i < len(blocks); i++ { block := blocks[i] + if block == nil || block.Block == nil || block.Block.Body == nil { + continue + } if block.Version() < clparams.GloasVersion { continue } @@ -373,7 +386,8 @@ func determineFullGloasRoots(blocks []*cltypes.SignedBeaconBlock, processCount i isFull := false if i+1 < len(blocks) { nextBlock := blocks[i+1] - if nextBlock.Version() >= clparams.GloasVersion { + blockRoot, err := block.Block.HashSSZ() + if err == nil && nextBlock != nil && nextBlock.Block != nil && nextBlock.Block.Body != nil && nextBlock.Block.Slot > block.Block.Slot && nextBlock.Block.ParentRoot == blockRoot && nextBlock.Version() >= clparams.GloasVersion { nextBid := nextBlock.Block.Body.GetSignedExecutionPayloadBid() if nextBid != nil && nextBid.Message != nil { isFull = nextBid.Message.ParentBlockHash == bid.Message.BlockHash @@ -479,19 +493,19 @@ func fetchBlocksFromBeaconAPI(ctx context.Context, baseURL string, startSlot, co results[idx].err = fmt.Errorf("HTTP block fetch slot %d: %w", slot, err) return } - body, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil { - results[idx].err = fmt.Errorf("HTTP block read slot %d: %w", slot, readErr) - return - } + defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { - return // Skipped slot — block stays nil + return } if resp.StatusCode != http.StatusOK { results[idx].err = fmt.Errorf("HTTP block fetch slot %d: status %d", slot, resp.StatusCode) return } + body, readErr := readBoundedBeaconAPIResponse(resp.Body, maxBeaconAPIResponseBytes) + if readErr != nil { + results[idx].err = fmt.Errorf("HTTP block read slot %d: %w", slot, readErr) + return + } version := httpConsensusVersion(resp.Header.Get("Eth-Consensus-Version")) block := cltypes.NewSignedBeaconBlock(beaconCfg, version) @@ -615,7 +629,7 @@ func fetchEnvelopesFromBeaconAPI( if err != nil { return } - body, err := io.ReadAll(resp.Body) + body, err := readBoundedBeaconAPIResponse(resp.Body, maxBeaconAPIResponseBytes) resp.Body.Close() if err != nil || resp.StatusCode != http.StatusOK { return @@ -628,6 +642,11 @@ func fetchEnvelopesFromBeaconAPI( log.Debug("[ForwardBeaconDownloader] HTTP envelope decode failed", "slot", slot, "err", err) return } + block := blockByRoot(blocks, common.Hash(root)) + if err := ValidateFetchedEnvelope(beaconCfg, block, common.Hash(root), envelope); err != nil { + log.Debug("[ForwardBeaconDownloader] HTTP envelope mismatch", "slot", slot, "err", err) + return + } results[idx] = envResult{hash: common.Hash(root), envelope: envelope} }) } diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index 8d3aaa904e7..d9f47be392f 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -64,8 +64,11 @@ type StageHistoryReconstructionCfg struct { const logIntervalTime = 30 * time.Second const ( - skippedEnvelopeRecoveryMaxAttempts = 3 - skippedEnvelopeRecoveryRetryInterval = 10 * time.Second + skippedEnvelopeRecoveryMaxAttempts = 3 + skippedEnvelopeRecoveryRetryInterval = 10 * time.Second + skippedEnvelopeRecoveryBatchSize = 2 + skippedEnvelopeRecoveryBatchTimeout = 5 * time.Second + skippedEnvelopeRecoveryAttemptTimeout = 2 * time.Minute ) func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, antiquary *antiquary.Antiquary, sn *freezeblocks.CaplinSnapshots, indiciesDB kv.RwDB, engine execution_client.ExecutionEngine, beaconCfg *clparams.BeaconChainConfig, caplinConfig clparams.CaplinConfig, waitForAllRoutines bool, startingRoot common.Hash, startinSlot uint64, tmpdir string, backfillingThrottling time.Duration, executionBlocksCollector block_collector.BlockCollector, blockReader freezeblocks.BeaconSnapshotReader, blobStorage blob_storage.BlobStorage, logger log.Logger, forkchoiceStore forkchoice.ForkChoiceStorage, blobDownloader *network.BlobHistoryDownloader) StageHistoryReconstructionCfg { @@ -430,7 +433,9 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co func recoverSkippedEnvelopesWithRetries(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock) bool { pending := skipped for attempt := 1; attempt <= skippedEnvelopeRecoveryMaxAttempts; attempt++ { - pending = recoverSkippedEnvelopes(ctx, cfg, pending) + attemptCtx, cancel := context.WithTimeout(ctx, skippedEnvelopeRecoveryAttemptTimeout) + pending = recoverSkippedEnvelopes(attemptCtx, cfg, pending) + cancel() if len(pending) == 0 { return true } @@ -460,40 +465,81 @@ func recoverSkippedEnvelopesWithRetries(ctx context.Context, cfg StageHistoryRec func recoverSkippedEnvelopes(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock) []network.SkippedFullBlock { log.Info("[BackwardBeaconDownloader] recovering skipped GLOAS envelopes", "count", len(skipped)) - envelopes := cfg.downloader.RecoverSkippedEnvelopes(ctx) + remaining := recoverSkippedEnvelopeBatches(ctx, skipped, skippedEnvelopeRecoveryBatchSize, skippedEnvelopeRecoveryBatchTimeout, + func(fetchCtx, persistCtx context.Context, batch []network.SkippedFullBlock) []network.SkippedFullBlock { + return recoverSkippedEnvelopeBatch(fetchCtx, persistCtx, cfg, batch) + }) + log.Info("[BackwardBeaconDownloader] envelope recovery complete", + "recovered", len(skipped)-len(remaining), "total", len(skipped)) + return remaining +} - recovered := 0 +func recoverSkippedEnvelopeBatches(ctx context.Context, skipped []network.SkippedFullBlock, batchSize int, batchTimeout time.Duration, recoverBatch func(context.Context, context.Context, []network.SkippedFullBlock) []network.SkippedFullBlock) []network.SkippedFullBlock { remaining := make([]network.SkippedFullBlock, 0, len(skipped)) - for _, s := range skipped { + for start := 0; start < len(skipped); start += batchSize { + end := min(start+batchSize, len(skipped)) + batch := skipped[start:end] + if ctx.Err() != nil { + return rotateTimedOutEnvelopeRecovery(skipped, start, remaining) + } + batchCtx, cancel := context.WithTimeout(ctx, batchTimeout) + remaining = append(remaining, recoverBatch(batchCtx, ctx, batch)...) + cancel() + } + return remaining +} + +func rotateTimedOutEnvelopeRecovery(skipped []network.SkippedFullBlock, unattemptedStart int, failed []network.SkippedFullBlock) []network.SkippedFullBlock { + pending := make([]network.SkippedFullBlock, 0, len(skipped)-unattemptedStart+len(failed)) + pending = append(pending, skipped[unattemptedStart:]...) + pending = append(pending, failed...) + return pending +} + +func recoverSkippedEnvelopeBatch(fetchCtx, persistCtx context.Context, cfg StageHistoryReconstructionCfg, batch []network.SkippedFullBlock) []network.SkippedFullBlock { + envelopes := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, batch) + if cfg.indiciesDB == nil { + return append([]network.SkippedFullBlock(nil), batch...) + } + tx, err := cfg.indiciesDB.BeginRo(persistCtx) + if err != nil { + return append([]network.SkippedFullBlock(nil), batch...) + } + defer tx.Rollback() + + remaining := make([]network.SkippedFullBlock, 0, len(batch)) + for _, s := range batch { env := envelopes[common.Hash(s.Root)] - if env == nil { - log.Warn("[BackwardBeaconDownloader] envelope still missing after recovery", - "slot", s.Block.Block.Slot, "root", common.Hash(s.Root)) + if env == nil || cfg.blockReader == nil { remaining = append(remaining, s) continue } - if env.Message == nil || env.Message.Payload == nil { - log.Warn("[BackwardBeaconDownloader] recovered envelope is malformed", - "slot", s.Block.Block.Slot, "root", common.Hash(s.Root)) + block, err := cfg.blockReader.ReadBlockByRoot(persistCtx, tx, common.Hash(s.Root)) + if err != nil || block == nil || block.Block == nil || block.Block.Body == nil { + log.Warn("[BackwardBeaconDownloader] skipped block unavailable during recovery", "slot", s.Slot, "root", common.Hash(s.Root), "err", err) remaining = append(remaining, s) continue } - - if !recoverSkippedEnvelope(ctx, cfg, s, env) { + root, err := block.Block.HashSSZ() + if err != nil || root != s.Root { remaining = append(remaining, s) continue } - recovered++ + if err := network.ValidateFetchedEnvelope(cfg.beaconCfg, block, common.Hash(s.Root), env); err != nil { + log.Warn("[BackwardBeaconDownloader] recovered envelope does not match block", "slot", s.Slot, "root", common.Hash(s.Root), "err", err) + remaining = append(remaining, s) + continue + } + if !recoverSkippedEnvelope(persistCtx, cfg, s, block, env) { + remaining = append(remaining, s) + } } - - log.Info("[BackwardBeaconDownloader] envelope recovery complete", - "recovered", recovered, "total", len(skipped)) return remaining } -func recoverSkippedEnvelope(ctx context.Context, cfg StageHistoryReconstructionCfg, s network.SkippedFullBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { +func recoverSkippedEnvelope(ctx context.Context, cfg StageHistoryReconstructionCfg, s network.SkippedFullBlock, block *cltypes.SignedBeaconBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { if cfg.executionBlocksCollector != nil { - if err := cfg.executionBlocksCollector.AddGloasBlock(s.Block.Block, env); err != nil { + if err := cfg.executionBlocksCollector.AddGloasBlock(block.Block, env); err != nil { log.Warn("[BackwardBeaconDownloader] envelope recovery: add block failed", "err", err) return false } diff --git a/cl/phase1/stages/stage_history_download_test.go b/cl/phase1/stages/stage_history_download_test.go index a29a26e6b66..53e1cfd036a 100644 --- a/cl/phase1/stages/stage_history_download_test.go +++ b/cl/phase1/stages/stage_history_download_test.go @@ -17,10 +17,53 @@ package stages import ( + "context" "math" "testing" + "time" + + "github.com/erigontech/erigon/cl/phase1/network" ) +func TestRecoverSkippedEnvelopeBatchesDoesNotStarveLaterBatches(t *testing.T) { + skipped := []network.SkippedFullBlock{{Slot: 1}, {Slot: 2}, {Slot: 3}, {Slot: 4}, {Slot: 5}, {Slot: 6}, {Slot: 7}, {Slot: 8}} + attempted := make([]uint64, 0, len(skipped)) + recoverBatch := func(ctx, _ context.Context, batch []network.SkippedFullBlock) []network.SkippedFullBlock { + attempted = append(attempted, batch[0].Slot) + if batch[0].Slot < 7 { + <-ctx.Done() + return batch + } + return nil + } + + pending := recoverSkippedEnvelopeBatches(context.Background(), skipped, 2, time.Millisecond, recoverBatch) + if len(attempted) != 4 || attempted[3] != 7 { + t.Fatalf("attempted batch starts = %v, want [1 3 5 7]", attempted) + } + for _, item := range pending { + if item.Slot >= 7 { + t.Fatalf("later recoverable item %d remained pending", item.Slot) + } + } +} + +func TestRecoverSkippedEnvelopeBatchesKeepsPartialSuccess(t *testing.T) { + skipped := []network.SkippedFullBlock{{Slot: 1}, {Slot: 2}} + recoverBatch := func(fetchCtx, persistCtx context.Context, batch []network.SkippedFullBlock) []network.SkippedFullBlock { + <-fetchCtx.Done() + if persistCtx.Err() != nil { + t.Fatalf("persist context expired with fetch context: %v", persistCtx.Err()) + } + return batch[1:] + } + + pending := recoverSkippedEnvelopeBatches(context.Background(), skipped, 2, time.Millisecond, recoverBatch) + if len(pending) != 1 || pending[0].Slot != 2 { + t.Fatalf("pending = %v, want only slot 2", pending) + } +} + // clampProgress must never report a total below processed nor underflow, even // when the floor and current counters drift past the frozen highestBlockSeen. // The last case mirrors the field report where the live EL head advanced past From 2916b5372905c6f883d50a49843cd5a8e064b69e Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 3 Aug 2026 16:36:34 +0700 Subject: [PATCH 12/17] cl: harden Gloas payload validation and recovery --- .../payload_validation_coordinator.go | 107 ++++++++++++++++++ .../payload_validation_coordinator_test.go | 90 +++++++++++++++ cl/phase1/forkchoice/forkchoice.go | 18 ++- cl/phase1/forkchoice/on_execution_payload.go | 52 +-------- .../network/backward_beacon_downloader.go | 17 ++- .../backward_beacon_downloader_test.go | 23 ++++ cl/phase1/stages/chain_tip_sync.go | 18 ++- cl/phase1/stages/clstages.go | 6 + cl/phase1/stages/forward_sync.go | 6 +- cl/phase1/stages/gloas_payload_test.go | 63 ++++++++++- cl/phase1/stages/stage_history_download.go | 26 ++++- 11 files changed, 358 insertions(+), 68 deletions(-) create mode 100644 cl/phase1/execution_client/payload_validation_coordinator.go create mode 100644 cl/phase1/execution_client/payload_validation_coordinator_test.go diff --git a/cl/phase1/execution_client/payload_validation_coordinator.go b/cl/phase1/execution_client/payload_validation_coordinator.go new file mode 100644 index 00000000000..2e67d1de310 --- /dev/null +++ b/cl/phase1/execution_client/payload_validation_coordinator.go @@ -0,0 +1,107 @@ +package execution_client + +import ( + "context" + "fmt" + "sync" + + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/hexutil" +) + +type payloadValidationCall struct { + done chan struct{} + status PayloadStatus + err error +} + +// PayloadValidationCoordinator bounds and coalesces NewPayload calls to one execution client. +type PayloadValidationCoordinator struct { + engine ExecutionEngine + slots chan struct{} + mu sync.Mutex + calls map[common.Hash]*payloadValidationCall +} + +// NewPayloadValidationCoordinator creates a coordinator allowing two concurrent engine calls. +func NewPayloadValidationCoordinator(engine ExecutionEngine) *PayloadValidationCoordinator { + return &PayloadValidationCoordinator{ + engine: engine, + slots: make(chan struct{}, 2), + calls: make(map[common.Hash]*payloadValidationCall), + } +} + +// NewPayload validates a payload through the shared concurrency and singleflight gate. +func (c *PayloadValidationCoordinator) NewPayload( + ctx context.Context, + key common.Hash, + payload *cltypes.Eth1Block, + parentBlockRoot *common.Hash, + versionedHashes []common.Hash, + executionRequestsList []hexutil.Bytes, +) (PayloadStatus, error) { + c.mu.Lock() + if call, ok := c.calls[key]; ok { + c.mu.Unlock() + return waitForPayloadValidation(ctx, call) + } + c.mu.Unlock() + + select { + case c.slots <- struct{}{}: + case <-ctx.Done(): + return PayloadStatusNone, ctx.Err() + } + c.mu.Lock() + if call, ok := c.calls[key]; ok { + c.mu.Unlock() + <-c.slots + return waitForPayloadValidation(ctx, call) + } + call := &payloadValidationCall{done: make(chan struct{})} + c.calls[key] = call + c.mu.Unlock() + + var ( + status PayloadStatus + err error + panicValue any + ) + func() { + defer func() { + panicValue = recover() + }() + status, err = c.engine.NewPayload(ctx, payload, parentBlockRoot, versionedHashes, executionRequestsList) + }() + <-c.slots + if panicValue != nil { + err = fmt.Errorf("execution client NewPayload panicked: %v", panicValue) + } + c.complete(key, call, status, err) + if panicValue != nil { + panic(panicValue) + } + return status, err +} + +func waitForPayloadValidation(ctx context.Context, call *payloadValidationCall) (PayloadStatus, error) { + select { + case <-call.done: + return call.status, call.err + case <-ctx.Done(): + return PayloadStatusNone, ctx.Err() + } +} + +func (c *PayloadValidationCoordinator) complete(key common.Hash, call *payloadValidationCall, status PayloadStatus, err error) { + c.mu.Lock() + call.status = status + call.err = err + if c.calls[key] == call { + delete(c.calls, key) + } + close(call.done) + c.mu.Unlock() +} diff --git a/cl/phase1/execution_client/payload_validation_coordinator_test.go b/cl/phase1/execution_client/payload_validation_coordinator_test.go new file mode 100644 index 00000000000..9ab3c3eb049 --- /dev/null +++ b/cl/phase1/execution_client/payload_validation_coordinator_test.go @@ -0,0 +1,90 @@ +package execution_client + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/hexutil" +) + +func TestPayloadValidationCoordinatorBoundsDistinctCalls(t *testing.T) { + ctrl := gomock.NewController(t) + engine := NewMockExecutionEngine(ctrl) + started := make(chan struct{}, 3) + release := make(chan struct{}) + var active atomic.Int32 + var maximum atomic.Int32 + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(3). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (PayloadStatus, error) { + current := active.Add(1) + for current > maximum.Load() && !maximum.CompareAndSwap(maximum.Load(), current) { + } + started <- struct{}{} + <-release + active.Add(-1) + return PayloadStatusValidated, nil + }) + + coordinator := NewPayloadValidationCoordinator(engine) + done := make(chan struct{}, 3) + for i := range 3 { + go func(key byte) { + _, _ = coordinator.NewPayload(context.Background(), common.Hash{key}, nil, nil, nil, nil) + done <- struct{}{} + }(byte(i + 1)) + } + <-started + <-started + select { + case <-started: + t.Fatal("more than two NewPayload calls ran concurrently") + case <-time.After(100 * time.Millisecond): + } + close(release) + <-started + for range 3 { + <-done + } + require.Equal(t, int32(2), maximum.Load()) +} + +func TestPayloadValidationCoordinatorReportsLeaderPanicToWaiter(t *testing.T) { + ctrl := gomock.NewController(t) + engine := NewMockExecutionEngine(ctrl) + started := make(chan struct{}) + release := make(chan struct{}) + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (PayloadStatus, error) { + close(started) + <-release + panic("engine panic") + }) + + coordinator := NewPayloadValidationCoordinator(engine) + key := common.Hash{1} + leaderPanic := make(chan any, 1) + go func() { + defer func() { leaderPanic <- recover() }() + _, _ = coordinator.NewPayload(context.Background(), key, nil, nil, nil, nil) + }() + <-started + waiterDone := make(chan error, 1) + go func() { + _, err := coordinator.NewPayload(context.Background(), key, nil, nil, nil, nil) + waiterDone <- err + }() + time.Sleep(10 * time.Millisecond) + close(release) + require.Equal(t, "engine panic", <-leaderPanic) + require.Error(t, <-waiterDone) +} diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index 6b9df95bc24..01c6e86bb08 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -200,10 +200,9 @@ type ForkChoiceStore struct { // whose EL newPayload failed (e.g. because EL hasn't caught up after forward sync). // The stages layer drains these into blockCollector before each Flush() so EL // eventually receives the blocks. - pendingELPayloadsMu sync.Mutex - pendingELPayloads []PendingELPayload - payloadValidations map[common.Hash]*payloadValidationCall - payloadValidationSlots chan struct{} + pendingELPayloadsMu sync.Mutex + pendingELPayloads []PendingELPayload + payloadValidator *execution_client.PayloadValidationCoordinator // db is used to persist execution payload indices (block number/hash) when an envelope // is accepted in OnExecutionPayload. May be nil (e.g. in tests), in which case the @@ -217,12 +216,6 @@ type PendingELPayload struct { Envelope *cltypes.SignedExecutionPayloadEnvelope } -type payloadValidationCall struct { - done chan struct{} - status execution_client.PayloadStatus - err error -} - type childrens struct { childrenHashes []common.Hash parentSlot uint64 // we keep this one for pruning @@ -406,6 +399,7 @@ func NewForkChoiceStore( payloadStatusByRoot: payloadStatusByRoot, executionPayloadGasLimit: executionPayloadGasLimit, payloadAttestationContexts: payloadAttestationContexts, + payloadValidator: execution_client.NewPayloadValidationCoordinator(engine), db: db, } f.justifiedCheckpoint.Store(anchorCheckpoint) @@ -438,6 +432,10 @@ func NewForkChoiceStore( return f, nil } +func (f *ForkChoiceStore) PayloadValidationCoordinator() *execution_client.PayloadValidationCoordinator { + return f.payloadValidator +} + func (f *ForkChoiceStore) InitPeerDas(peerDas das.PeerDas) { // this is a hack to inject the peer das f.peerDas = peerDas diff --git a/cl/phase1/forkchoice/on_execution_payload.go b/cl/phase1/forkchoice/on_execution_payload.go index 62e93b20920..9c461214ba5 100644 --- a/cl/phase1/forkchoice/on_execution_payload.go +++ b/cl/phase1/forkchoice/on_execution_payload.go @@ -328,55 +328,13 @@ func (f *ForkChoiceStore) newPayloadWithoutForkChoiceLock( versionedHashes []common.Hash, executionRequestsList []hexutil.Bytes, ) (execution_client.PayloadStatus, error) { - if f.payloadValidations == nil { - f.payloadValidations = make(map[common.Hash]*payloadValidationCall) + if f.payloadValidator == nil { + f.payloadValidator = execution_client.NewPayloadValidationCoordinator(f.engine) } - if f.payloadValidationSlots == nil { - f.payloadValidationSlots = make(chan struct{}, 2) - } - if call, ok := f.payloadValidations[beaconBlockRoot]; ok { - f.mu.Unlock() - select { - case <-call.done: - f.mu.Lock() - return call.status, call.err - case <-ctx.Done(): - f.mu.Lock() - return execution_client.PayloadStatusNone, ctx.Err() - } - } - - call := &payloadValidationCall{done: make(chan struct{})} - f.payloadValidations[beaconBlockRoot] = call + payloadValidator := f.payloadValidator f.mu.Unlock() - select { - case f.payloadValidationSlots <- struct{}{}: - case <-ctx.Done(): - f.mu.Lock() - call.err = ctx.Err() - delete(f.payloadValidations, beaconBlockRoot) - close(call.done) - return execution_client.PayloadStatusNone, call.err - } - var status execution_client.PayloadStatus - var err error - var panicValue any - func() { - defer func() { - panicValue = recover() - }() - status, err = f.engine.NewPayload(ctx, payload, parentBlockRoot, versionedHashes, executionRequestsList) - }() - <-f.payloadValidationSlots - f.mu.Lock() - call.status = status - call.err = err - delete(f.payloadValidations, beaconBlockRoot) - close(call.done) - if panicValue != nil { - panic(panicValue) - } - return status, err + defer f.mu.Lock() + return payloadValidator.NewPayload(ctx, beaconBlockRoot, payload, parentBlockRoot, versionedHashes, executionRequestsList) } // applyEnvelope processes the envelope under f.mu: validates, verifies with CL and EL, diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index d1621083fcc..40d27079d37 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "io" - "maps" "math" "net/http" "slices" @@ -834,7 +833,7 @@ func ValidateFetchedEnvelope(beaconCfg *clparams.BeaconChainConfig, block *cltyp // RecoverSkippedEnvelopes retries fetching envelopes for blocks that were // skipped during backward download. Returns a map of successfully fetched // envelopes keyed by beacon block root. -func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, skipped []SkippedFullBlock) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { +func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, skipped []SkippedFullBlock, blocks map[common.Hash]*cltypes.SignedBeaconBlock) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { if len(skipped) == 0 { return nil } @@ -846,7 +845,13 @@ func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) if b.httpFallbackURL != "" { - b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped, envelopes) + httpEnvelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) + b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped, httpEnvelopes) + for root, envelope := range httpEnvelopes { + if ValidateFetchedEnvelope(b.beaconCfg, blocks[root], root, envelope) == nil { + envelopes[root] = envelope + } + } } missingRoots := make([][32]byte, 0, len(roots)-len(envelopes)) for _, root := range roots { @@ -860,7 +865,11 @@ func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, if err != nil { log.Debug("[BackwardBeaconDownloader] envelope recovery: P2P failed", "err", err) } - maps.Copy(envelopes, p2pEnvelopes) + for root, envelope := range p2pEnvelopes { + if ValidateFetchedEnvelope(b.beaconCfg, blocks[root], root, envelope) == nil { + envelopes[root] = envelope + } + } } return envelopes diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index c88d3ec6dda..8c358ec946c 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -583,6 +583,29 @@ func TestMalformedP2PEnvelopeFallsBackToValidHTTPEnvelope(t *testing.T) { require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), got[common.Hash(blockRoot)])) } +func TestMalformedHTTPRecoveryEnvelopeRemainsMissing(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, malformed := makeValidGloasEnvelope(t, block) + malformed.Message.Payload.GasUsed++ + encoded, err := malformed.EncodeSSZ(nil) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(encoded) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{ + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + got := downloader.RecoverSkippedEnvelopes( + context.Background(), + []SkippedFullBlock{{Slot: block.Block.Slot, Root: blockRoot}}, + map[common.Hash]*cltypes.SignedBeaconBlock{common.Hash(blockRoot): block}, + ) + require.Empty(t, got) +} + func TestValidateFetchedEnvelopesDropsMalformedSameRoot(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, err := block.Block.HashSSZ() diff --git a/cl/phase1/stages/chain_tip_sync.go b/cl/phase1/stages/chain_tip_sync.go index f3c6aeee955..38aea2134ca 100644 --- a/cl/phase1/stages/chain_tip_sync.go +++ b/cl/phase1/stages/chain_tip_sync.go @@ -511,7 +511,23 @@ func retryGloasPayloadWithEL(ctx context.Context, cfg *Cfg, block *cltypes.Signe return execution_client.PayloadStatusNone, err } parentRoot := block.Block.ParentRoot - return cfg.executionClient.NewPayload(ctx, envelope.Message.Payload, &parentRoot, versionedHashes, executionRequestsList) + validationKey, err := envelope.Message.HashSSZ() + if err != nil { + return execution_client.PayloadStatusNone, err + } + return cfg.newPayloadCoordinator().NewPayload(ctx, common.Hash(validationKey), envelope.Message.Payload, &parentRoot, versionedHashes, executionRequestsList) +} + +func (c *Cfg) newPayloadCoordinator() *execution_client.PayloadValidationCoordinator { + if c.payloadValidator != nil { + return c.payloadValidator + } + if c.forkChoice != nil && c.forkChoice.PayloadValidationCoordinator() != nil { + c.payloadValidator = c.forkChoice.PayloadValidationCoordinator() + } else { + c.payloadValidator = execution_client.NewPayloadValidationCoordinator(c.executionClient) + } + return c.payloadValidator } func isGloasPayloadKnownInvalid(cfg *Cfg, envelope *cltypes.SignedExecutionPayloadEnvelope) bool { diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index 84746b66d00..e66f53a638f 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -52,6 +52,7 @@ type Cfg struct { ethClock eth_clock.EthereumClock beaconCfg *clparams.BeaconChainConfig executionClient execution_client.ExecutionEngine + payloadValidator *execution_client.PayloadValidationCoordinator state *state.CachingBeaconState forkChoice *forkchoice.ForkChoiceStore indiciesDB kv.RwDB @@ -99,6 +100,10 @@ func ClStagesCfg( attestationDataProducer attestation_producer.AttestationDataProducer, peerDas das.PeerDas, ) *Cfg { + payloadValidator := execution_client.NewPayloadValidationCoordinator(executionClient) + if forkChoice != nil && forkChoice.PayloadValidationCoordinator() != nil { + payloadValidator = forkChoice.PayloadValidationCoordinator() + } blobDownloader := network2.NewBlobHistoryDownloader( ctx, beaconCfg, @@ -122,6 +127,7 @@ func ClStagesCfg( beaconCfg: beaconCfg, state: state, executionClient: executionClient, + payloadValidator: payloadValidator, forkChoice: forkChoice, dirs: dirs, indiciesDB: indiciesDB, diff --git a/cl/phase1/stages/forward_sync.go b/cl/phase1/stages/forward_sync.go index bdddf1f3ff3..9542444a003 100644 --- a/cl/phase1/stages/forward_sync.go +++ b/cl/phase1/stages/forward_sync.go @@ -649,7 +649,11 @@ func validateAnchorPayloadWithEL(ctx context.Context, cfg *Cfg, bid *cltypes.Exe if err != nil { return execution_client.PayloadStatusNone, err } - return cfg.executionClient.NewPayload(ctx, env.Message.Payload, &bid.ParentBlockRoot, versionedHashes, executionRequestsList) + validationKey, err := env.Message.HashSSZ() + if err != nil { + return execution_client.PayloadStatusNone, err + } + return cfg.newPayloadCoordinator().NewPayload(ctx, common.Hash(validationKey), env.Message.Payload, &bid.ParentBlockRoot, versionedHashes, executionRequestsList) } func buildAnchorNewPayloadArgs(beaconCfg *clparams.BeaconChainConfig, bid *cltypes.ExecutionPayloadBid, env *cltypes.SignedExecutionPayloadEnvelope) ([]common.Hash, []hexutil.Bytes, error) { diff --git a/cl/phase1/stages/gloas_payload_test.go b/cl/phase1/stages/gloas_payload_test.go index 3aacb300e5c..609c28ba59f 100644 --- a/cl/phase1/stages/gloas_payload_test.go +++ b/cl/phase1/stages/gloas_payload_test.go @@ -3,7 +3,10 @@ package stages import ( "context" "math/big" + "sync" + "sync/atomic" "testing" + "time" "github.com/holiman/uint256" "github.com/stretchr/testify/require" @@ -213,6 +216,52 @@ func TestValidateAnchorPayloadWithAnyExecutionClient(t *testing.T) { require.Equal(t, 1, localEL.newPayloadCalls) } +func TestStageNewPayloadUsesSharedCoordinator(t *testing.T) { + beaconCfg, _, bid, env, _ := validAnchorEnvelopeFixture(t, 1) + started := make(chan struct{}, 3) + release := make(chan struct{}) + var active atomic.Int32 + var maximum atomic.Int32 + engine := &testExecutionEngine{ + payloadStatus: execution_client.PayloadStatusValidated, + newPayloadFn: func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + current := active.Add(1) + for current > maximum.Load() && !maximum.CompareAndSwap(maximum.Load(), current) { + } + started <- struct{}{} + <-release + active.Add(-1) + return execution_client.PayloadStatusValidated, nil + }, + } + coordinator := execution_client.NewPayloadValidationCoordinator(engine) + cfg := &Cfg{beaconCfg: beaconCfg, executionClient: engine, payloadValidator: coordinator} + done := make(chan struct{}, 3) + for _, key := range []common.Hash{{1}, {2}} { + go func() { + _, _ = coordinator.NewPayload(context.Background(), key, nil, nil, nil, nil) + done <- struct{}{} + }() + } + <-started + <-started + go func() { + _, _ = validateAnchorPayloadWithEL(context.Background(), cfg, bid, env) + done <- struct{}{} + }() + select { + case <-started: + t.Fatal("stage bypassed the shared NewPayload bound") + case <-time.After(100 * time.Millisecond): + } + close(release) + <-started + for range 3 { + <-done + } + require.Equal(t, int32(2), maximum.Load()) +} + func TestDrainPendingGloasPayloadsRequeuesNotValidatedPayload(t *testing.T) { cfg := clparams.MainnetBeaconConfig clparams.ApplyMinimalPreset(&cfg) @@ -243,8 +292,9 @@ func TestDrainPendingGloasPayloadsRequeuesNotValidatedPayload(t *testing.T) { }, Envelope: &cltypes.SignedExecutionPayloadEnvelope{ Message: &cltypes.ExecutionPayloadEnvelope{ - BeaconBlockRoot: blockRoot, - Payload: payload, + BeaconBlockRoot: blockRoot, + Payload: payload, + ExecutionRequests: cltypes.NewExecutionRequestsWithVersion(&cfg, clparams.GloasVersion), }, }, } @@ -392,10 +442,17 @@ type testExecutionEngine struct { supportInsertion bool payloadStatus execution_client.PayloadStatus newPayloadCalls int + newPayloadMu sync.Mutex + newPayloadFn func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) } -func (t *testExecutionEngine) NewPayload(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { +func (t *testExecutionEngine) NewPayload(ctx context.Context, payload *cltypes.Eth1Block, parentRoot *common.Hash, versionedHashes []common.Hash, requests []hexutil.Bytes) (execution_client.PayloadStatus, error) { + t.newPayloadMu.Lock() t.newPayloadCalls++ + t.newPayloadMu.Unlock() + if t.newPayloadFn != nil { + return t.newPayloadFn(ctx, payload, parentRoot, versionedHashes, requests) + } return t.payloadStatus, nil } diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index d9f47be392f..b5a504311d6 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -497,10 +497,11 @@ func rotateTimedOutEnvelopeRecovery(skipped []network.SkippedFullBlock, unattemp } func recoverSkippedEnvelopeBatch(fetchCtx, persistCtx context.Context, cfg StageHistoryReconstructionCfg, batch []network.SkippedFullBlock) []network.SkippedFullBlock { - envelopes := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, batch) - if cfg.indiciesDB == nil { + if cfg.indiciesDB == nil || cfg.blockReader == nil { return append([]network.SkippedFullBlock(nil), batch...) } + blocks := readSkippedEnvelopeBlocks(persistCtx, cfg, batch) + envelopes := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, batch, blocks) tx, err := cfg.indiciesDB.BeginRo(persistCtx) if err != nil { return append([]network.SkippedFullBlock(nil), batch...) @@ -537,6 +538,27 @@ func recoverSkippedEnvelopeBatch(fetchCtx, persistCtx context.Context, cfg Stage return remaining } +func readSkippedEnvelopeBlocks(ctx context.Context, cfg StageHistoryReconstructionCfg, batch []network.SkippedFullBlock) map[common.Hash]*cltypes.SignedBeaconBlock { + blocks := make(map[common.Hash]*cltypes.SignedBeaconBlock, len(batch)) + tx, err := cfg.indiciesDB.BeginRo(ctx) + if err != nil { + return blocks + } + defer tx.Rollback() + for _, item := range batch { + root := common.Hash(item.Root) + block, err := cfg.blockReader.ReadBlockByRoot(ctx, tx, root) + if err != nil || block == nil || block.Block == nil || block.Block.Body == nil { + continue + } + decodedRoot, err := block.Block.HashSSZ() + if err == nil && decodedRoot == item.Root { + blocks[root] = block + } + } + return blocks +} + func recoverSkippedEnvelope(ctx context.Context, cfg StageHistoryReconstructionCfg, s network.SkippedFullBlock, block *cltypes.SignedBeaconBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { if cfg.executionBlocksCollector != nil { if err := cfg.executionBlocksCollector.AddGloasBlock(block.Block, env); err != nil { From 061809ac21cc05d8bbfc6984a0eebada64d7c55a Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 3 Aug 2026 17:33:15 +0700 Subject: [PATCH 13/17] cl/network: do not ban peers for empty ranges --- cl/phase1/network/beacon_downloader.go | 6 +++++- cl/phase1/network/beacon_downloader_test.go | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 cl/phase1/network/beacon_downloader_test.go diff --git a/cl/phase1/network/beacon_downloader.go b/cl/phase1/network/beacon_downloader.go index e04da00d13b..4ee8460e231 100644 --- a/cl/phase1/network/beacon_downloader.go +++ b/cl/phase1/network/beacon_downloader.go @@ -260,7 +260,7 @@ Process: processBlocks := completeBeaconBlocks(resp.blocks) pid := resp.peerId if len(processBlocks) == 0 { - if pid != "" && pid != "http-fallback" && f.rpc != nil { + if shouldBanIncompleteBlockResponse(pid, len(resp.blocks), len(processBlocks)) && f.rpc != nil { f.rpc.BanPeer(pid) } return @@ -340,6 +340,10 @@ func validateAndFetchMissingEnvelopes(ctx context.Context, httpFallbackURL strin return validateFetchedEnvelopes(beaconCfg, blocks, envelopes) } +func shouldBanIncompleteBlockResponse(peerID string, received, complete int) bool { + return peerID != "" && peerID != "http-fallback" && received > 0 && complete == 0 +} + func completeBeaconBlocks(blocks []*cltypes.SignedBeaconBlock) []*cltypes.SignedBeaconBlock { complete := make([]*cltypes.SignedBeaconBlock, 0, len(blocks)) for _, block := range blocks { diff --git a/cl/phase1/network/beacon_downloader_test.go b/cl/phase1/network/beacon_downloader_test.go new file mode 100644 index 00000000000..0f4f14b9b64 --- /dev/null +++ b/cl/phase1/network/beacon_downloader_test.go @@ -0,0 +1,15 @@ +package network + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestShouldBanIncompleteBlockResponse(t *testing.T) { + require.False(t, shouldBanIncompleteBlockResponse("peer", 0, 0)) + require.True(t, shouldBanIncompleteBlockResponse("peer", 1, 0)) + require.False(t, shouldBanIncompleteBlockResponse("peer", 2, 1)) + require.False(t, shouldBanIncompleteBlockResponse("", 1, 0)) + require.False(t, shouldBanIncompleteBlockResponse("http-fallback", 1, 0)) +} From 0f3e53ea378037126799836c9010d31255c545d1 Mon Sep 17 00:00:00 2001 From: kewei Date: Wed, 5 Aug 2026 13:53:38 +0700 Subject: [PATCH 14/17] cl: address gloas sync review findings --- .../forkchoice/fork_graph/fork_graph_disk.go | 12 +- .../forkchoice/fork_graph/fork_graph_test.go | 14 +- cl/phase1/forkchoice/on_block.go | 20 ++ .../on_block_fork_consistency_test.go | 35 ++++ .../network/backward_beacon_downloader.go | 192 ++++++++++++++---- .../backward_beacon_downloader_test.go | 149 ++++++++++++++ 6 files changed, 378 insertions(+), 44 deletions(-) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index d363cd3bccd..0b1de74168a 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -151,10 +151,14 @@ func NewForkGraphDisk(anchorState *state.CachingBeaconState, syncedData synced_d anchorHeader := anchorState.LatestBlockHeader() if anchorState.Version() >= clparams.GloasVersion && anchorState.Slot() > 0 { stateHash := anchorState.PeekPreviousStateRoot() - if stateHash == (common.Hash{}) || anchorHeader.Root == (common.Hash{}) || anchorHeader.Slot < anchorState.Slot() { - stateHash, err = anchorState.HashSSZ() - if err != nil { - panic(err) + if stateHash == (common.Hash{}) { + if anchorHeader.Root != (common.Hash{}) && anchorHeader.Slot == anchorState.Slot() { + stateHash = anchorHeader.Root + } else { + stateHash, err = anchorState.HashSSZ() + if err != nil { + panic(err) + } } } if anchorHeader.Root == (common.Hash{}) { diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go index 2250e5bb462..313b9a99433 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go @@ -70,15 +70,17 @@ func TestForkGraphInDisk(t *testing.T) { func TestNewForkGraphDiskCachesAnchorStateRoot(t *testing.T) { for _, tc := range []struct { - name string - stateSlot uint64 - headerSlot uint64 - headerRoot common.Hash - cachedRoot common.Hash + name string + stateSlot uint64 + headerSlot uint64 + headerRoot common.Hash + cachedRoot common.Hash + expectedRoot common.Hash }{ {name: "skipped slot", stateSlot: 64, headerSlot: 63, headerRoot: common.Hash{1}}, {name: "block slot", stateSlot: 64, headerSlot: 64}, {name: "legacy block slot", stateSlot: 64, headerSlot: 64, headerRoot: common.Hash{1}, cachedRoot: common.Hash{1}}, + {name: "legacy block slot without cached root", stateSlot: 64, headerSlot: 64, headerRoot: common.Hash{2}, expectedRoot: common.Hash{2}}, } { t.Run(tc.name, func(t *testing.T) { anchorState := state.New(&clparams.MainnetBeaconConfig) @@ -91,6 +93,8 @@ func TestNewForkGraphDiskCachesAnchorStateRoot(t *testing.T) { if tc.cachedRoot != (common.Hash{}) { expectedStateRoot = tc.cachedRoot anchorState.SetPreviousStateRoot(tc.cachedRoot) + } else if tc.expectedRoot != (common.Hash{}) { + expectedStateRoot = tc.expectedRoot } anchorRoot, err := anchorState.BlockRoot() require.NoError(t, err) diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go index a31e3b4cff1..0df6918d0f2 100644 --- a/cl/phase1/forkchoice/on_block.go +++ b/cl/phase1/forkchoice/on_block.go @@ -334,6 +334,7 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // [New in Gloas:EIP7732] GLOAS-specific on_block logic (post state transition) var appliedEnvelope *cltypes.ExecutionPayloadEnvelope + stateMayBeStale := false if blockVersion >= clparams.GloasVersion { // Initialize payload timeliness and data availability votes for this block f.payloadTimelinessVote.Store(common.Hash(blockRoot), [clparams.PtcSize]int8{}) @@ -360,6 +361,7 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac if pending, ok := f.pendingLocalSelfBuildEnvelopes.Get(common.Hash(blockRoot)); ok { f.pendingLocalSelfBuildEnvelopes.Remove(common.Hash(blockRoot)) log.Trace("OnBlock: processing pending local self-build envelope", "blockRoot", common.Hash(blockRoot)) + stateMayBeStale = true applied, applyErr := f.applyLocalSelfBuildEnvelopeLocked(ctx, pending) if applyErr != nil { log.Warn("OnBlock: failed to process pending local self-build envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) @@ -372,6 +374,7 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // Always validate payload with EL for pending envelopes, regardless of the caller's newPayload flag. // During forward sync newPayload is false, but the envelope still needs to reach the EL; // otherwise the EL never learns about this block and the chain stalls. + stateMayBeStale = true applied, applyErr := f.applyEnvelopeLocked(ctx, pending, checkDataAvaiability, true) if applyErr != nil { log.Warn("OnBlock: failed to process pending envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) @@ -380,6 +383,12 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac } } } + if stateMayBeStale { + lastProcessedState, err = f.refreshBlockStateAfterPayloadValidation(common.Hash(blockRoot)) + if err != nil { + return err + } + } if lastProcessedState.Slot()%f.beaconCfg.SlotsPerEpoch == 0 { // Update randao mixes r := solid.NewHashVector(int(f.beaconCfg.EpochsPerHistoricalVector)) @@ -492,6 +501,17 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac return nil } +func (f *ForkChoiceStore) refreshBlockStateAfterPayloadValidation(blockRoot common.Hash) (*state.CachingBeaconState, error) { + blockState, err := f.forkGraph.GetState(blockRoot, false) + if err != nil { + return nil, fmt.Errorf("OnBlock: failed to refresh block state after payload validation: %w", err) + } + if blockState == nil { + return nil, fmt.Errorf("OnBlock: block state disappeared after payload validation for block %v", blockRoot) + } + return blockState, nil +} + func (f *ForkChoiceStore) addChainSegmentAndQueueLightClientEvents(block *cltypes.SignedBeaconBlock, fullValidation bool) (*state.CachingBeaconState, fork_graph.ChainSegmentInsertionResult, error) { lcUpdateBefore := f.forkGraph.NewestLightClientUpdate() lastProcessedState, status, err := f.forkGraph.AddChainSegment(block, fullValidation) diff --git a/cl/phase1/forkchoice/on_block_fork_consistency_test.go b/cl/phase1/forkchoice/on_block_fork_consistency_test.go index 7a22f7e23e4..ec2f76bc8ea 100644 --- a/cl/phase1/forkchoice/on_block_fork_consistency_test.go +++ b/cl/phase1/forkchoice/on_block_fork_consistency_test.go @@ -18,15 +18,29 @@ package forkchoice import ( "context" + "errors" "testing" "github.com/stretchr/testify/require" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" "github.com/erigontech/erigon/cl/utils" + "github.com/erigontech/erigon/common" ) +type refreshStateForkGraph struct { + fork_graph.ForkGraph + latest *state.CachingBeaconState + err error +} + +func (f *refreshStateForkGraph) GetState(common.Hash, bool) (*state.CachingBeaconState, error) { + return f.latest, f.err +} + // A response's decoded schema comes from the peer-chosen fork digest, so it is // independent of the slot the block claims. Gloas removed ExecutionPayload and // BlobKzgCommitments from BeaconBody, so a Gloas-decoded block whose slot maps @@ -55,3 +69,24 @@ func TestOnBlockRejectsForkSchemaSlotMismatch(t *testing.T) { err := store.OnBlock(context.Background(), mismatched, false, true, true) require.ErrorIs(t, err, ErrForkSchemaSlotMismatch) } + +func TestRefreshBlockStateAfterPayloadValidationUsesLatestState(t *testing.T) { + latest := state.New(&clparams.MainnetBeaconConfig) + latest.SetSlot(12) + store := &ForkChoiceStore{forkGraph: &refreshStateForkGraph{latest: latest}} + + got, err := store.refreshBlockStateAfterPayloadValidation(common.Hash{1}) + require.NoError(t, err) + require.Same(t, latest, got) +} + +func TestRefreshBlockStateAfterPayloadValidationRejectsMissingState(t *testing.T) { + store := &ForkChoiceStore{forkGraph: &refreshStateForkGraph{}} + _, err := store.refreshBlockStateAfterPayloadValidation(common.Hash{1}) + require.Error(t, err) + + expected := errors.New("read failed") + store.forkGraph = &refreshStateForkGraph{err: expected} + _, err = store.refreshBlockStateAfterPayloadValidation(common.Hash{1}) + require.ErrorIs(t, err, expected) +} diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index 40d27079d37..6384bee2196 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "maps" "math" "net/http" "slices" @@ -93,7 +94,7 @@ const ( maxBeaconAPIResponseBytes = 64 << 20 ) -// SkippedFullBlock records a GLOAS FULL block whose envelope was unavailable during backward download. +// SkippedFullBlock records a GLOAS block that may need an envelope after degraded backward download. type SkippedFullBlock struct { Slot uint64 Root [32]byte @@ -350,6 +351,10 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } } + if envelope == nil && b.envelopesSkipped && !b.canTrackSkippedFullBlock(block) { + log.Warn("[BackwardBeaconDownloader] skipped envelope recovery queue is full, will retry", "slot", block.Block.Slot) + return nil + } } // A FULL block whose envelope could not be fetched must not be treated as @@ -369,7 +374,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons } // Record FULL blocks passing through without envelope for post-download recovery. - if _, isFull := fullRootSet[common.Hash(blockRoot)]; isFull && envelope == nil { + if _, isFull := fullRootSet[common.Hash(blockRoot)]; envelope == nil && (isFull || b.envelopesSkipped) { b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) } @@ -419,7 +424,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } isFull = full - if full { + if full && !b.envelopesSkipped { env, fetchErr := b.fetchSingleEnvelope(ctx, block) if fetchErr == nil && env != nil { if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err == nil { @@ -434,6 +439,24 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons "slot", block.Block.Slot, "err", fetchErr, "consecutiveFailures", b.consecutiveEnvelopeFailures) return nil } + } else if !full && !b.envelopesSkipped { + env, fetchErr := b.fetchSingleEnvelope(ctx, block) + if fetchErr != nil { + log.Warn("[BackwardBeaconDownloader] root-fetched EMPTY confirmation failed, will retry", "slot", block.Block.Slot, "err", fetchErr) + return nil + } + if env != nil { + if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err != nil { + log.Warn("[BackwardBeaconDownloader] root-fetched envelope does not match block", "slot", block.Block.Slot, "err", err) + return nil + } + envelope = env + isFull = true + } + } + if envelope == nil && b.envelopesSkipped && !b.canTrackSkippedFullBlock(block) { + log.Warn("[BackwardBeaconDownloader] skipped envelope recovery queue is full, will retry", "slot", block.Block.Slot) + return nil } } @@ -442,7 +465,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons if err != nil { log.Warn("Error processing root-fetched block", "err", err) } else { - if isFull && envelope == nil { + if envelope == nil && (isFull || b.envelopesSkipped) { b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) } b.prevBatchTopBlock = block @@ -726,27 +749,104 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp fullRootSet[common.Hash(r)] = struct{}{} } - if len(fullRoots) == 0 { + if b.envelopesSkipped { return nil, fullRootSet, knownRootSet } var envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope - if b.httpPreferred.Load() && b.httpFallbackURL != "" { - envelopes = validateAndFetchMissingEnvelopes(ctx, b.httpFallbackURL, responses, fullRoots, nil, b.beaconCfg) - } else { - var err error - envelopes, err = RequestEnvelopesFrantically(ctx, b.rpc, fullRoots) - if err != nil { - log.Debug("[BackwardBeaconDownloader] failed to fetch GLOAS envelopes via P2P", "err", err) + if len(fullRoots) > 0 { + if b.httpPreferred.Load() && b.httpFallbackURL != "" { + envelopes = validateAndFetchMissingEnvelopes(ctx, b.httpFallbackURL, responses, fullRoots, nil, b.beaconCfg) + } else { + var err error + envelopes, err = RequestEnvelopesFrantically(ctx, b.rpc, fullRoots) + if err != nil { + log.Debug("[BackwardBeaconDownloader] failed to fetch GLOAS envelopes via P2P", "err", err) + } + envelopes = validateAndFetchMissingEnvelopes(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) } - envelopes = validateAndFetchMissingEnvelopes(ctx, b.httpFallbackURL, responses, fullRoots, envelopes, b.beaconCfg) + b.recordEnvelopeFetchResult(len(fullRoots), len(envelopes)) } - b.recordEnvelopeFetchResult(len(fullRoots), len(envelopes)) + inferredEmptyRoots := make(map[common.Hash]struct{}, len(knownRootSet)-len(fullRootSet)) + for root := range knownRootSet { + if _, full := fullRootSet[root]; !full { + inferredEmptyRoots[root] = struct{}{} + delete(knownRootSet, root) + } + } + if len(inferredEmptyRoots) > 0 && b.httpFallbackURL != "" { + probed, confirmedEmpty := b.probeGloasEmptyCandidates(ctx, responses, inferredEmptyRoots) + if envelopes == nil && len(probed) > 0 { + envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(probed)) + } + for root, envelope := range probed { + envelopes[root] = envelope + fullRootSet[root] = struct{}{} + knownRootSet[root] = struct{}{} + } + for root := range confirmedEmpty { + knownRootSet[root] = struct{}{} + } + } return envelopes, fullRootSet, knownRootSet } +func (b *BackwardBeaconDownloader) probeGloasEmptyCandidates(ctx context.Context, blocks []*cltypes.SignedBeaconBlock, roots map[common.Hash]struct{}) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, map[common.Hash]struct{}) { + type result struct { + root common.Hash + envelope *cltypes.SignedExecutionPayloadEnvelope + confirmed bool + } + results := make(chan result, len(roots)) + sem := make(chan struct{}, 8) + var wg sync.WaitGroup + for _, block := range blocks { + if block == nil || block.Block == nil || block.Block.Body == nil { + continue + } + root, err := block.Block.HashSSZ() + if err != nil { + continue + } + if _, ok := roots[common.Hash(root)]; !ok { + continue + } + wg.Go(func() { + sem <- struct{}{} + defer func() { <-sem }() + envelope, err := b.fetchSingleEnvelope(ctx, block) + if err != nil { + results <- result{root: common.Hash(root)} + return + } + if envelope == nil { + results <- result{root: common.Hash(root), confirmed: true} + return + } + if ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(root), envelope) != nil { + results <- result{root: common.Hash(root)} + return + } + results <- result{root: common.Hash(root), envelope: envelope} + }) + } + wg.Wait() + close(results) + + envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) + confirmedEmpty := make(map[common.Hash]struct{}) + for result := range results { + if result.envelope != nil { + envelopes[result.root] = result.envelope + } else if result.confirmed { + confirmedEmpty[result.root] = struct{}{} + } + } + return envelopes, confirmedEmpty +} + func validateFetchedEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks []*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { valid := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(envelopes)) for _, block := range blocks { @@ -843,35 +943,57 @@ func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, roots[i] = s.Root } - envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) + sources := make([]func(context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, 0, 2) if b.httpFallbackURL != "" { - httpEnvelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) - b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped, httpEnvelopes) - for root, envelope := range httpEnvelopes { - if ValidateFetchedEnvelope(b.beaconCfg, blocks[root], root, envelope) == nil { - envelopes[root] = envelope + sources = append(sources, func(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) + b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped, envelopes) + return validateRecoveryEnvelopes(b.beaconCfg, blocks, envelopes) + }) + } + if b.rpc != nil { + sources = append(sources, func(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + envelopes, err := RequestEnvelopesFrantically(ctx, b.rpc, roots) + if err != nil { + log.Debug("[BackwardBeaconDownloader] envelope recovery: P2P failed", "err", err) } - } + return validateRecoveryEnvelopes(b.beaconCfg, blocks, envelopes) + }) } - missingRoots := make([][32]byte, 0, len(roots)-len(envelopes)) - for _, root := range roots { - if _, ok := envelopes[common.Hash(root)]; !ok { - missingRoots = append(missingRoots, root) + + fetched := fetchEnvelopeRecoverySources(ctx, sources...) + envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fetched)) + for root, envelope := range fetched { + if ValidateFetchedEnvelope(b.beaconCfg, blocks[root], root, envelope) == nil { + envelopes[root] = envelope } } - if b.rpc != nil && len(missingRoots) > 0 && ctx.Err() == nil { - var err error - p2pEnvelopes, err := RequestEnvelopesFrantically(ctx, b.rpc, missingRoots) - if err != nil { - log.Debug("[BackwardBeaconDownloader] envelope recovery: P2P failed", "err", err) - } - for root, envelope := range p2pEnvelopes { - if ValidateFetchedEnvelope(b.beaconCfg, blocks[root], root, envelope) == nil { - envelopes[root] = envelope - } + return envelopes +} + +func validateRecoveryEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks map[common.Hash]*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + valid := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(envelopes)) + for root, envelope := range envelopes { + if ValidateFetchedEnvelope(beaconCfg, blocks[root], root, envelope) == nil { + valid[root] = envelope } } + return valid +} +func fetchEnvelopeRecoverySources(ctx context.Context, sources ...func(context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + results := make(chan map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(sources)) + var wg sync.WaitGroup + for _, source := range sources { + wg.Go(func() { results <- source(ctx) }) + } + wg.Wait() + close(results) + + envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) + for result := range results { + maps.Copy(envelopes, result) + } return envelopes } diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index 8c358ec946c..b4b8f6e0ca3 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -343,6 +343,97 @@ func TestBackwardBeaconDownloaderHTTPPreferredMissingEnvelopeTracksFailure(t *te assert.Equal(t, 1, downloader.consecutiveEnvelopeFailures) } +func TestFetchGloasEnvelopesProbesLookaheadInferredEmptyBlock(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + encoded, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/eth/v1/beacon/execution_payload_envelope/100" { + http.NotFound(w, r) + return + } + _, _ = w.Write(encoded) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{ + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + lookaheadRoot, err := lookahead.Block.HashSSZ() + require.NoError(t, err) + downloader.expectedRoot = lookaheadRoot + downloader.httpPreferred.Store(true) + + envelopes, fullRoots, knownRoots := downloader.fetchGloasEnvelopes( + context.Background(), + []*cltypes.SignedBeaconBlock{block, lookahead}, + ) + + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), envelopes[common.Hash(blockRoot)])) + assert.Contains(t, fullRoots, common.Hash(blockRoot)) + assert.Contains(t, knownRoots, common.Hash(blockRoot)) +} + +func TestFetchGloasEnvelopesSkipsNetworkAfterFailureThreshold(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + linkGloasBlocks(t, block, lookahead) + + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{ + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + envelopesSkipped: true, + } + lookaheadRoot, err := lookahead.Block.HashSSZ() + require.NoError(t, err) + downloader.expectedRoot = lookaheadRoot + downloader.httpPreferred.Store(true) + + envelopes, fullRoots, _ := downloader.fetchGloasEnvelopes( + context.Background(), + []*cltypes.SignedBeaconBlock{block, lookahead}, + ) + + assert.Empty(t, envelopes) + require.Len(t, fullRoots, 1) + assert.Zero(t, requests.Load()) +} + +func TestDegradedDownloadTracksLookaheadInferredEmptyBlock(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + beaconCfg: &clparams.MainnetBeaconConfig, + envelopesSkipped: true, + onNewBlock: func(_ *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + require.Nil(t, envelope) + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) + require.Len(t, downloader.skippedFullBlocks, 1) + assert.Equal(t, common.Hash(blockRoot), common.Hash(downloader.skippedFullBlocks[0].Root)) +} + func TestSelectGloasLookaheadRejectsUnlinkedAndIncompleteBlocks(t *testing.T) { anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) anchorRoot, err := anchor.Block.HashSSZ() @@ -536,6 +627,45 @@ func TestRootFallbackMissingEnvelopeEntersBoundedRecovery(t *testing.T) { assert.Equal(t, common.Hash(blockRoot), common.Hash(downloader.skippedFullBlocks[0].Root)) } +func TestRootFallbackProbesLookaheadInferredEmptyBlock(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + child := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + child.Block.ParentRoot = blockRoot + encodedBlock, err := block.EncodeSSZ(nil) + require.NoError(t, err) + encodedEnvelope, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/eth/v2/beacon/blocks/" + common.Hash(blockRoot).Hex(): + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedBlock) + case "/eth/v1/beacon/execution_payload_envelope/100": + _, _ = w.Write(encodedEnvelope) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + var processedEnvelope *cltypes.SignedExecutionPayloadEnvelope + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: child, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(_ *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processedEnvelope = envelope + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), nil)) + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), processedEnvelope)) +} + func TestValidateFetchedEnvelopeRejectsDifferentBeaconRoot(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, envelope := makeValidGloasEnvelope(t, block) @@ -606,6 +736,25 @@ func TestMalformedHTTPRecoveryEnvelopeRemainsMissing(t *testing.T) { require.Empty(t, got) } +func TestFetchEnvelopeRecoverySourcesStartsAllSourcesBeforeDeadline(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + wantRoot := hash(0x42) + wantEnvelope := &cltypes.SignedExecutionPayloadEnvelope{} + + got := fetchEnvelopeRecoverySources(ctx, + func(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + <-ctx.Done() + return nil + }, + func(context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + return map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{wantRoot: wantEnvelope} + }, + ) + + require.Same(t, wantEnvelope, got[wantRoot]) +} + func TestValidateFetchedEnvelopesDropsMalformedSameRoot(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, err := block.Block.HashSSZ() From 8f1b6529acfe595cb50c13efee8cb7374ca03070 Mon Sep 17 00:00:00 2001 From: kewei Date: Mon, 10 Aug 2026 03:22:08 +0800 Subject: [PATCH 15/17] cl/network: harden Gloas envelope fallback --- .../network/backward_beacon_downloader.go | 53 +++++--- .../backward_beacon_downloader_test.go | 116 +++++++++++++++++- cl/phase1/network/beacon_downloader.go | 34 +++-- 3 files changed, 160 insertions(+), 43 deletions(-) diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index 6384bee2196..491a64e47e2 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -425,7 +425,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons } isFull = full if full && !b.envelopesSkipped { - env, fetchErr := b.fetchSingleEnvelope(ctx, block) + env, fetchErr := b.fetchSingleEnvelope(ctx, common.Hash(blockRoot)) if fetchErr == nil && env != nil { if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err == nil { envelope = env @@ -440,7 +440,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } } else if !full && !b.envelopesSkipped { - env, fetchErr := b.fetchSingleEnvelope(ctx, block) + env, fetchErr := b.fetchSingleEnvelope(ctx, common.Hash(blockRoot)) if fetchErr != nil { log.Warn("[BackwardBeaconDownloader] root-fetched EMPTY confirmation failed, will retry", "slot", block.Block.Slot, "err", fetchErr) return nil @@ -771,6 +771,9 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp inferredEmptyRoots := make(map[common.Hash]struct{}, len(knownRootSet)-len(fullRootSet)) for root := range knownRootSet { if _, full := fullRootSet[root]; !full { + if b.httpFallbackURL == "" { + continue + } inferredEmptyRoots[root] = struct{}{} delete(knownRootSet, root) } @@ -799,9 +802,7 @@ func (b *BackwardBeaconDownloader) probeGloasEmptyCandidates(ctx context.Context envelope *cltypes.SignedExecutionPayloadEnvelope confirmed bool } - results := make(chan result, len(roots)) - sem := make(chan struct{}, 8) - var wg sync.WaitGroup + uniqueBlocks := make(map[common.Hash]*cltypes.SignedBeaconBlock, len(roots)) for _, block := range blocks { if block == nil || block.Block == nil || block.Block.Body == nil { continue @@ -810,26 +811,40 @@ func (b *BackwardBeaconDownloader) probeGloasEmptyCandidates(ctx context.Context if err != nil { continue } - if _, ok := roots[common.Hash(root)]; !ok { + hash := common.Hash(root) + if _, ok := roots[hash]; !ok { continue } + if _, exists := uniqueBlocks[hash]; !exists { + uniqueBlocks[hash] = block + } + } + + results := make(chan result, len(uniqueBlocks)) + sem := make(chan struct{}, 8) + var wg sync.WaitGroup + for root, block := range uniqueBlocks { wg.Go(func() { - sem <- struct{}{} + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } defer func() { <-sem }() - envelope, err := b.fetchSingleEnvelope(ctx, block) + envelope, err := b.fetchSingleEnvelope(ctx, root) if err != nil { - results <- result{root: common.Hash(root)} + results <- result{root: root} return } if envelope == nil { - results <- result{root: common.Hash(root), confirmed: true} + results <- result{root: root, confirmed: true} return } - if ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(root), envelope) != nil { - results <- result{root: common.Hash(root)} + if ValidateFetchedEnvelope(b.beaconCfg, block, root, envelope) != nil { + results <- result{root: root} return } - results <- result{root: common.Hash(root), envelope: envelope} + results <- result{root: root, envelope: envelope} }) } wg.Wait() @@ -880,8 +895,7 @@ func (b *BackwardBeaconDownloader) recordEnvelopeFetchResult(requested, received b.envelopesSkipped = false } -// SkippedFullBlocks returns FULL blocks that were processed without envelopes -// due to consecutive fetch failures during backward download. +// SkippedFullBlocks returns blocks that may still need envelopes after backward download. func (b *BackwardBeaconDownloader) SkippedFullBlocks() []SkippedFullBlock { return b.skippedFullBlocks } @@ -1003,8 +1017,7 @@ func (b *BackwardBeaconDownloader) fetchSkippedEnvelopesFromBeaconAPI(ctx contex if _, ok := envelopes[root]; ok { continue } - block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: item.Slot}} - envelope, err := b.fetchSingleEnvelope(ctx, block) + envelope, err := b.fetchSingleEnvelope(ctx, root) if err != nil || envelope == nil || envelope.Message == nil || envelope.Message.BeaconBlockRoot != root { continue } @@ -1164,15 +1177,15 @@ func fetchBlockFromBeaconAPIByRoot(ctx context.Context, baseURL string, root com } // fetchSingleEnvelope fetches the execution payload envelope for a single GLOAS block. -// Returns (envelope, nil) on success, (nil, nil) when the beacon API confirms the slot +// Returns (envelope, nil) on success, (nil, nil) when the beacon API confirms the root // has no envelope (HTTP 404 = genuinely EMPTY), or (nil, err) on fetch failure. -func (b *BackwardBeaconDownloader) fetchSingleEnvelope(ctx context.Context, block *cltypes.SignedBeaconBlock) (*cltypes.SignedExecutionPayloadEnvelope, error) { +func (b *BackwardBeaconDownloader) fetchSingleEnvelope(ctx context.Context, blockRoot common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { if b.httpFallbackURL == "" { return nil, fmt.Errorf("no HTTP fallback URL configured") } client := &http.Client{Timeout: 10 * time.Second} - reqURL := fmt.Sprintf("%s/eth/v1/beacon/execution_payload_envelope/%d", b.httpFallbackURL, block.Block.Slot) + reqURL := fmt.Sprintf("%s/eth/v1/beacon/execution_payload_envelope/%s", b.httpFallbackURL, blockRoot.Hex()) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { return nil, err diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index b4b8f6e0ca3..0b4807879c6 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -352,7 +352,7 @@ func TestFetchGloasEnvelopesProbesLookaheadInferredEmptyBlock(t *testing.T) { require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/eth/v1/beacon/execution_payload_envelope/100" { + if r.URL.Path != "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex() { http.NotFound(w, r) return } @@ -379,6 +379,89 @@ func TestFetchGloasEnvelopesProbesLookaheadInferredEmptyBlock(t *testing.T) { assert.Contains(t, knownRoots, common.Hash(blockRoot)) } +func TestP2POnlyDownloadTracksLookaheadInferredEmptyBlock(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(_ *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + require.Nil(t, envelope) + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) + require.True(t, processed) + assert.Empty(t, downloader.skippedFullBlocks) +} + +func TestProbeGloasEmptyCandidatesDeduplicatesRoots(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.NotFound(w, r) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{ + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + done := make(chan struct{}) + go func() { + downloader.probeGloasEmptyCandidates( + context.Background(), + []*cltypes.SignedBeaconBlock{block, block}, + map[common.Hash]struct{}{common.Hash(blockRoot): {}}, + ) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("duplicate root deadlocked envelope probing") + } + assert.Equal(t, int32(1), requests.Load()) +} + +func TestFetchEnvelopesFromBeaconAPIUsesBlockRoot(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + encoded, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + var requestedPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestedPath = r.URL.Path + _, _ = w.Write(encoded) + })) + defer server.Close() + + received := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) + require.Equal(t, 1, fetchEnvelopesFromBeaconAPI( + context.Background(), + server.URL, + []*cltypes.SignedBeaconBlock{block}, + [][32]byte{blockRoot}, + received, + &clparams.MainnetBeaconConfig, + )) + assert.Equal(t, "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex(), requestedPath) +} + func TestFetchGloasEnvelopesSkipsNetworkAfterFailureThreshold(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) @@ -642,7 +725,7 @@ func TestRootFallbackProbesLookaheadInferredEmptyBlock(t *testing.T) { case "/eth/v2/beacon/blocks/" + common.Hash(blockRoot).Hex(): w.Header().Set("Eth-Consensus-Version", "gloas") _, _ = w.Write(encodedBlock) - case "/eth/v1/beacon/execution_payload_envelope/100": + case "/eth/v1/beacon/execution_payload_envelope/" + common.Hash(blockRoot).Hex(): _, _ = w.Write(encodedEnvelope) default: http.NotFound(w, r) @@ -695,7 +778,7 @@ func TestMalformedP2PEnvelopeFallsBackToValidHTTPEnvelope(t *testing.T) { encoded, err := valid.EncodeSSZ(nil) require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/eth/v1/beacon/execution_payload_envelope/100" { + if r.URL.Path != "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex() { http.NotFound(w, r) return } @@ -736,6 +819,33 @@ func TestMalformedHTTPRecoveryEnvelopeRemainsMissing(t *testing.T) { require.Empty(t, got) } +func TestHTTPRecoveryUsesBlockRoot(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + encoded, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + var requestedPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestedPath = r.URL.Path + _, _ = w.Write(encoded) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{ + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + got := downloader.RecoverSkippedEnvelopes( + context.Background(), + []SkippedFullBlock{{Slot: block.Block.Slot, Root: blockRoot}}, + map[common.Hash]*cltypes.SignedBeaconBlock{common.Hash(blockRoot): block}, + ) + + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), got[common.Hash(blockRoot)])) + assert.Equal(t, "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex(), requestedPath) +} + func TestFetchEnvelopeRecoverySourcesStartsAllSourcesBeforeDeadline(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() diff --git a/cl/phase1/network/beacon_downloader.go b/cl/phase1/network/beacon_downloader.go index 4ee8460e231..5483f56c8e0 100644 --- a/cl/phase1/network/beacon_downloader.go +++ b/cl/phase1/network/beacon_downloader.go @@ -568,15 +568,14 @@ func fetchEnvelopesFromBeaconAPI( received map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, beaconCfg *clparams.BeaconChainConfig, ) int { - // Build root-to-slot mapping from blocks - rootToSlot := make(map[common.Hash]uint64, len(blocks)) + availableRoots := make(map[common.Hash]struct{}, len(blocks)) for _, blk := range blocks { if blk == nil || blk.Block == nil || blk.Block.Body == nil { continue } root, err := blk.Block.HashSSZ() if err == nil { - rootToSlot[root] = blk.Block.Slot + availableRoots[root] = struct{}{} } } @@ -585,24 +584,17 @@ func fetchEnvelopesFromBeaconAPI( envelope *cltypes.SignedExecutionPayloadEnvelope } - // Filter roots that need fetching - var toFetch []struct { - root [32]byte - slot uint64 - } + toFetch := make([][32]byte, 0, len(fullRoots)) for _, root := range fullRoots { h := common.Hash(root) if _, ok := received[h]; ok { continue } - slot, ok := rootToSlot[h] + _, ok := availableRoots[h] if !ok { continue } - toFetch = append(toFetch, struct { - root [32]byte - slot uint64 - }{root, slot}) + toFetch = append(toFetch, root) } if len(toFetch) == 0 { @@ -614,15 +606,17 @@ func fetchEnvelopesFromBeaconAPI( sem := make(chan struct{}, 8) var wg sync.WaitGroup - for i, item := range toFetch { + for i, root := range toFetch { idx := i - slot := item.slot - root := item.root wg.Go(func() { - sem <- struct{}{} + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } defer func() { <-sem }() - reqURL := fmt.Sprintf("%s/eth/v1/beacon/execution_payload_envelope/%d", baseURL, slot) + reqURL := fmt.Sprintf("%s/eth/v1/beacon/execution_payload_envelope/%s", baseURL, common.Hash(root).Hex()) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { return @@ -643,12 +637,12 @@ func fetchEnvelopesFromBeaconAPI( Message: cltypes.NewExecutionPayloadEnvelope(beaconCfg), } if err := envelope.DecodeSSZ(body, int(clparams.GloasVersion)); err != nil { - log.Debug("[ForwardBeaconDownloader] HTTP envelope decode failed", "slot", slot, "err", err) + log.Debug("[ForwardBeaconDownloader] HTTP envelope decode failed", "root", common.Hash(root), "err", err) return } block := blockByRoot(blocks, common.Hash(root)) if err := ValidateFetchedEnvelope(beaconCfg, block, common.Hash(root), envelope); err != nil { - log.Debug("[ForwardBeaconDownloader] HTTP envelope mismatch", "slot", slot, "err", err) + log.Debug("[ForwardBeaconDownloader] HTTP envelope mismatch", "root", common.Hash(root), "err", err) return } results[idx] = envResult{hash: common.Hash(root), envelope: envelope} From dc5ce83c409c0bb9f54c6be43c979b8d3a2b3e1f Mon Sep 17 00:00:00 2001 From: kewei Date: Wed, 12 Aug 2026 00:06:58 +0800 Subject: [PATCH 16/17] cl: harden Gloas payload recovery lifecycle --- .../payload_validation_coordinator.go | 91 +-- .../payload_validation_coordinator_test.go | 74 +++ .../forkchoice/fork_graph/fork_graph_disk.go | 13 + .../fork_graph/fork_graph_disk_fs.go | 126 +++- .../forkchoice/fork_graph/fork_graph_test.go | 138 ++++ cl/phase1/forkchoice/fork_graph/interface.go | 3 + cl/phase1/forkchoice/forkchoice.go | 29 + cl/phase1/forkchoice/forkchoice_test.go | 12 + cl/phase1/forkchoice/interface.go | 3 +- .../mock_services/forkchoice_mock.go | 6 +- cl/phase1/forkchoice/on_block.go | 79 ++- cl/phase1/forkchoice/on_execution_payload.go | 473 ++++++++++++-- .../forkchoice/on_execution_payload_test.go | 559 ++++++++++++++++- .../on_payload_attestation_message.go | 4 +- .../payload_attestation_validation.go | 32 +- .../payload_attestation_validation_test.go | 30 +- cl/phase1/forkchoice/payload_vote_test.go | 83 ++- .../forkchoice/pending_el_payload_test.go | 17 + .../network/backward_beacon_downloader.go | 306 +++++++-- .../backward_beacon_downloader_test.go | 589 +++++++++++++++++- .../services/execution_payload_service.go | 149 ++++- .../execution_payload_service_test.go | 315 +++++++++- .../services/payload_attestation_service.go | 7 +- .../payload_attestation_service_test.go | 4 +- cl/phase1/stages/chain_tip_sync.go | 13 +- cl/phase1/stages/stage_history_download.go | 115 +++- .../stages/stage_history_download_test.go | 164 +++++ 27 files changed, 3079 insertions(+), 355 deletions(-) diff --git a/cl/phase1/execution_client/payload_validation_coordinator.go b/cl/phase1/execution_client/payload_validation_coordinator.go index 2e67d1de310..d120fd57f75 100644 --- a/cl/phase1/execution_client/payload_validation_coordinator.go +++ b/cl/phase1/execution_client/payload_validation_coordinator.go @@ -2,6 +2,7 @@ package execution_client import ( "context" + "errors" "fmt" "sync" @@ -14,6 +15,7 @@ type payloadValidationCall struct { done chan struct{} status PayloadStatus err error + retry bool } // PayloadValidationCoordinator bounds and coalesces NewPayload calls to one execution client. @@ -42,56 +44,66 @@ func (c *PayloadValidationCoordinator) NewPayload( versionedHashes []common.Hash, executionRequestsList []hexutil.Bytes, ) (PayloadStatus, error) { - c.mu.Lock() - if call, ok := c.calls[key]; ok { + for { + c.mu.Lock() + if call, ok := c.calls[key]; ok { + c.mu.Unlock() + status, err, retry := waitForPayloadValidation(ctx, call) + if retry && ctx.Err() == nil { + continue + } + return status, err + } c.mu.Unlock() - return waitForPayloadValidation(ctx, call) - } - c.mu.Unlock() - select { - case c.slots <- struct{}{}: - case <-ctx.Done(): - return PayloadStatusNone, ctx.Err() - } - c.mu.Lock() - if call, ok := c.calls[key]; ok { + select { + case c.slots <- struct{}{}: + case <-ctx.Done(): + return PayloadStatusNone, ctx.Err() + } + c.mu.Lock() + if call, ok := c.calls[key]; ok { + c.mu.Unlock() + <-c.slots + status, err, retry := waitForPayloadValidation(ctx, call) + if retry && ctx.Err() == nil { + continue + } + return status, err + } + call := &payloadValidationCall{done: make(chan struct{})} + c.calls[key] = call c.mu.Unlock() - <-c.slots - return waitForPayloadValidation(ctx, call) - } - call := &payloadValidationCall{done: make(chan struct{})} - c.calls[key] = call - c.mu.Unlock() - var ( - status PayloadStatus - err error - panicValue any - ) - func() { - defer func() { - panicValue = recover() + var ( + status PayloadStatus + err error + panicValue any + ) + func() { + defer func() { + panicValue = recover() + }() + status, err = c.engine.NewPayload(ctx, payload, parentBlockRoot, versionedHashes, executionRequestsList) }() - status, err = c.engine.NewPayload(ctx, payload, parentBlockRoot, versionedHashes, executionRequestsList) - }() - <-c.slots - if panicValue != nil { - err = fmt.Errorf("execution client NewPayload panicked: %v", panicValue) - } - c.complete(key, call, status, err) - if panicValue != nil { - panic(panicValue) + <-c.slots + if panicValue != nil { + err = fmt.Errorf("execution client NewPayload panicked: %v", panicValue) + } + c.complete(key, call, status, err) + if panicValue != nil { + panic(panicValue) + } + return status, err } - return status, err } -func waitForPayloadValidation(ctx context.Context, call *payloadValidationCall) (PayloadStatus, error) { +func waitForPayloadValidation(ctx context.Context, call *payloadValidationCall) (PayloadStatus, error, bool) { select { case <-call.done: - return call.status, call.err + return call.status, call.err, call.retry case <-ctx.Done(): - return PayloadStatusNone, ctx.Err() + return PayloadStatusNone, ctx.Err(), false } } @@ -99,6 +111,7 @@ func (c *PayloadValidationCoordinator) complete(key common.Hash, call *payloadVa c.mu.Lock() call.status = status call.err = err + call.retry = errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) if c.calls[key] == call { delete(c.calls, key) } diff --git a/cl/phase1/execution_client/payload_validation_coordinator_test.go b/cl/phase1/execution_client/payload_validation_coordinator_test.go index 9ab3c3eb049..a00a20cb998 100644 --- a/cl/phase1/execution_client/payload_validation_coordinator_test.go +++ b/cl/phase1/execution_client/payload_validation_coordinator_test.go @@ -2,6 +2,7 @@ package execution_client import ( "context" + "errors" "sync/atomic" "testing" "time" @@ -14,6 +15,79 @@ import ( "github.com/erigontech/erigon/common/hexutil" ) +func TestPayloadValidationCoordinatorLeaderCancellationDoesNotPoisonWaiter(t *testing.T) { + ctrl := gomock.NewController(t) + engine := NewMockExecutionEngine(ctrl) + started := make(chan struct{}) + var calls atomic.Int32 + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(2). + DoAndReturn(func(ctx context.Context, _ *cltypes.Eth1Block, _ *common.Hash, _ []common.Hash, _ []hexutil.Bytes) (PayloadStatus, error) { + if calls.Add(1) == 1 { + close(started) + <-ctx.Done() + return PayloadStatusNone, ctx.Err() + } + return PayloadStatusValidated, nil + }) + + coordinator := NewPayloadValidationCoordinator(engine) + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderDone := make(chan error, 1) + go func() { + _, err := coordinator.NewPayload(leaderCtx, common.Hash{1}, nil, nil, nil, nil) + leaderDone <- err + }() + <-started + + waiterDone := make(chan payloadValidationContextResult, 1) + go func() { + status, err := coordinator.NewPayload(context.Background(), common.Hash{1}, nil, nil, nil, nil) + waiterDone <- payloadValidationContextResult{status: status, err: err} + }() + cancelLeader() + + require.ErrorIs(t, <-leaderDone, context.Canceled) + waiter := <-waiterDone + require.NoError(t, waiter.err) + require.EqualValues(t, PayloadStatusValidated, waiter.status) + require.Equal(t, int32(2), calls.Load()) +} + +type payloadValidationContextResult struct { + status PayloadStatus + err error +} + +func TestPayloadValidationCoordinatorWaiterCancellationIsLocal(t *testing.T) { + ctrl := gomock.NewController(t) + engine := NewMockExecutionEngine(ctrl) + started := make(chan struct{}) + release := make(chan struct{}) + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (PayloadStatus, error) { + close(started) + <-release + return PayloadStatusValidated, nil + }) + + coordinator := NewPayloadValidationCoordinator(engine) + leaderDone := make(chan error, 1) + go func() { + _, err := coordinator.NewPayload(context.Background(), common.Hash{1}, nil, nil, nil, nil) + leaderDone <- err + }() + <-started + waiterCtx, cancelWaiter := context.WithCancel(context.Background()) + cancelWaiter() + _, err := coordinator.NewPayload(waiterCtx, common.Hash{1}, nil, nil, nil, nil) + require.True(t, errors.Is(err, context.Canceled)) + close(release) + require.NoError(t, <-leaderDone) +} + func TestPayloadValidationCoordinatorBoundsDistinctCalls(t *testing.T) { ctrl := gomock.NewController(t) engine := NewMockExecutionEngine(ctrl) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index 0b1de74168a..41db8714242 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -597,6 +597,12 @@ func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) { } } for _, root := range oldRoots { + f.stateDumpLock.Lock() + block, ok := f.blocks.Load(root) + if !ok || block.(*cltypes.SignedBeaconBlock).Block.Slot >= pruneSlot { + f.stateDumpLock.Unlock() + continue + } f.badBlocks.Delete(root) f.blocks.Delete(root) f.lightclientBootstraps.Delete(root) @@ -608,6 +614,13 @@ func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) { // [New in Gloas:EIP7732] Also remove envelope files f.envelopeExists.Delete(root) f.fs.Remove(getEnvelopeFilename(root)) + f.fs.Remove(getEnvelopeIndexMarkerFilename(root)) + if temporaryFiles, err := afero.Glob(f.fs, getEnvelopeFilename(root)+".tmp-*"); err == nil { + for _, temporaryFile := range temporaryFiles { + f.fs.Remove(temporaryFile) + } + } + f.stateDumpLock.Unlock() } log.Debug("Pruned old blocks", "pruneSlot", pruneSlot) return diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go index 6eb756f07c1..b508a50506d 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go @@ -18,9 +18,11 @@ package fork_graph import ( "encoding/binary" + "encoding/hex" "fmt" "io" "os" + "strings" "github.com/golang/snappy" "github.com/spf13/afero" @@ -48,6 +50,10 @@ func getEnvelopeFilename(blockRoot common.Hash) string { return fmt.Sprintf("%x.envelope.snappy_ssz", blockRoot) } +func getEnvelopeIndexMarkerFilename(blockRoot common.Hash) string { + return fmt.Sprintf("%x.envelope.indices-pending", blockRoot) +} + func (f *forkGraphDisk) readBeaconStateFromDisk(blockRoot common.Hash) (bs *state.CachingBeaconState, err error) { var file afero.File f.stateDumpLock.Lock() @@ -194,6 +200,10 @@ func (f *forkGraphDisk) HasEnvelope(blockRoot common.Hash) bool { // Slow path: fall back to disk and populate cache on hit exists, err := afero.Exists(f.fs, getEnvelopeFilename(blockRoot)) if err == nil && exists { + envelope, readErr := f.ReadEnvelopeFromDisk(blockRoot) + if readErr != nil || envelope == nil || envelope.Message == nil || envelope.Message.BeaconBlockRoot != blockRoot { + return false + } f.envelopeExists.Store(blockRoot, struct{}{}) return true } @@ -258,15 +268,20 @@ func (f *forkGraphDisk) ReadEnvelopeFromDisk(blockRoot common.Hash) (envelope *c // DumpEnvelopeOnDisk dumps an execution payload envelope to disk. // [New in Gloas:EIP7732] func (f *forkGraphDisk) DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) (err error) { + publish, err := f.PrepareEnvelopeOnDisk(blockRoot, envelope, false) + if err != nil { + return err + } + return publish() +} + +func (f *forkGraphDisk) PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, requireBlock bool) (publish func() error, err error) { f.stateDumpLock.Lock() defer f.stateDumpLock.Unlock() - - // Populate in-memory cache on successful write - defer func() { - if err == nil { - f.envelopeExists.Store(blockRoot, struct{}{}) - } - }() + _, blockWasPresent := f.blocks.Load(blockRoot) + if requireBlock && !blockWasPresent { + return nil, fmt.Errorf("cannot prepare envelope for missing block %x", blockRoot) + } // Encode the envelope f.sszBuffer, err = envelope.EncodeSSZ(f.sszBuffer[:0]) @@ -274,11 +289,19 @@ func (f *forkGraphDisk) DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *clty return } - dumpedFile, err := f.fs.OpenFile(getEnvelopeFilename(blockRoot), os.O_TRUNC|os.O_CREATE|os.O_RDWR, 0o755) + filename := getEnvelopeFilename(blockRoot) + dumpedFile, err := afero.TempFile(f.fs, "", filename+".tmp-") if err != nil { - return err + return nil, err } - defer dumpedFile.Close() + temporaryFilename := dumpedFile.Name() + keepTemporary := false + defer func() { + _ = dumpedFile.Close() + if !keepTemporary { + _ = f.fs.Remove(temporaryFilename) + } + }() if f.sszSnappyWriter == nil { f.sszSnappyWriter = snappy.NewBufferedWriter(dumpedFile) @@ -291,22 +314,97 @@ func (f *forkGraphDisk) DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *clty binary.BigEndian.PutUint64(length, uint64(len(f.sszBuffer))) if _, err := f.sszSnappyWriter.Write(length); err != nil { log.Error("failed to write length", "err", err) - return err + return nil, err } // Write the envelope if _, err := f.sszSnappyWriter.Write(f.sszBuffer); err != nil { log.Error("failed to write ssz buffer", "err", err) - return err + return nil, err } if err = f.sszSnappyWriter.Flush(); err != nil { log.Error("failed to flush snappy writer", "err", err) - return err + return nil, err } if err = dumpedFile.Sync(); err != nil { log.Error("failed to sync dumped file", "err", err) return } + if err = dumpedFile.Close(); err != nil { + return nil, err + } + markerFilename := getEnvelopeIndexMarkerFilename(blockRoot) + marker, err := f.fs.OpenFile(markerFilename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + if err = marker.Sync(); err != nil { + _ = marker.Close() + _ = f.fs.Remove(markerFilename) + return nil, err + } + if err = marker.Close(); err != nil { + _ = f.fs.Remove(markerFilename) + return nil, err + } + keepTemporary = true + + return func() error { + f.stateDumpLock.Lock() + defer f.stateDumpLock.Unlock() + if _, blockPresent := f.blocks.Load(blockRoot); requireBlock && !blockPresent { + _ = f.fs.Remove(temporaryFilename) + _ = f.fs.Remove(markerFilename) + return fmt.Errorf("cannot publish envelope for pruned block %x", blockRoot) + } + if err := f.fs.Rename(temporaryFilename, filename); err != nil { + _ = f.fs.Remove(temporaryFilename) + _ = f.fs.Remove(markerFilename) + return err + } + f.envelopeExists.Store(blockRoot, struct{}{}) + return nil + }, nil +} - return +func (f *forkGraphDisk) PendingEnvelopeIndexRoots() ([]common.Hash, error) { + f.stateDumpLock.Lock() + defer f.stateDumpLock.Unlock() + entries, err := afero.ReadDir(f.fs, ".") + if err != nil { + return nil, err + } + const suffix = ".envelope.indices-pending" + roots := make([]common.Hash, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, suffix) { + continue + } + rootBytes, err := hex.DecodeString(strings.TrimSuffix(name, suffix)) + if err != nil || len(rootBytes) != len(common.Hash{}) { + continue + } + roots = append(roots, common.BytesToHash(rootBytes)) + } + return roots, nil +} + +func (f *forkGraphDisk) MarkEnvelopeIndicesCommitted(blockRoot common.Hash) error { + f.stateDumpLock.Lock() + defer f.stateDumpLock.Unlock() + matches, err := afero.Glob(f.fs, getEnvelopeFilename(blockRoot)+".tmp-*") + if err != nil { + return err + } + for _, match := range matches { + if err := f.fs.Remove(match); err != nil && !os.IsNotExist(err) { + return err + } + } + err = f.fs.Remove(getEnvelopeIndexMarkerFilename(blockRoot)) + if os.IsNotExist(err) { + return nil + } + return err } diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go index 313b9a99433..65a7b00f59d 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go @@ -18,7 +18,11 @@ package fork_graph import ( _ "embed" + "errors" + "os" + "sync" "testing" + "time" "github.com/erigontech/erigon/cl/beacon/beacon_router_configuration" "github.com/erigontech/erigon/cl/phase1/core/state" @@ -31,6 +35,30 @@ import ( "github.com/stretchr/testify/require" ) +type renameFailFS struct { + afero.Fs +} + +func (renameFailFS) Rename(string, string) error { + return errors.New("injected rename failure") +} + +type blockingStatFS struct { + afero.Fs + target string + started chan struct{} + release chan struct{} + once sync.Once +} + +func (f *blockingStatFS) Stat(name string) (os.FileInfo, error) { + if name == f.target { + f.once.Do(func() { close(f.started) }) + <-f.release + } + return f.Fs.Stat(name) +} + //go:embed test_data/block_0xe2a37a22d208ebe969c50e9d44bb3f1f63c5404787b9c214a5f2f28fb9835feb.ssz_snappy var block1 []byte @@ -110,6 +138,45 @@ func TestNewForkGraphDiskCachesAnchorStateRoot(t *testing.T) { } } +func TestHasEnvelopeRejectsTruncatedFile(t *testing.T) { + fs := afero.NewMemMapFs() + root := common.Hash{1} + require.NoError(t, afero.WriteFile(fs, getEnvelopeFilename(root), []byte{1, 2, 3}, 0o644)) + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + + require.False(t, graph.HasEnvelope(root)) + _, cached := graph.envelopeExists.Load(root) + require.False(t, cached) +} + +func TestDumpEnvelopeOnDiskKeepsPreviousFileWhenRenameFails(t *testing.T) { + fs := afero.NewMemMapFs() + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + original := testExecutionPayloadEnvelope(root, common.Hash{2}) + require.NoError(t, graph.DumpEnvelopeOnDisk(root, original)) + + graph.fs = renameFailFS{Fs: fs} + replacement := testExecutionPayloadEnvelope(root, common.Hash{3}) + require.ErrorContains(t, graph.DumpEnvelopeOnDisk(root, replacement), "injected rename failure") + + stored, err := graph.ReadEnvelopeFromDisk(root) + require.NoError(t, err) + require.Equal(t, original.Message.Payload.BlockHash, stored.Message.Payload.BlockHash) + exists, err := afero.Exists(fs, getEnvelopeFilename(root)+".tmp") + require.NoError(t, err) + require.False(t, exists) +} + +func testExecutionPayloadEnvelope(root, executionHash common.Hash) *cltypes.SignedExecutionPayloadEnvelope { + envelope := &cltypes.SignedExecutionPayloadEnvelope{ + Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig), + } + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload.BlockHash = executionHash + return envelope +} + // A prune for an already-covered slot (e.g. from a concurrent lock-free drain) // must not move the lowest-available marker backward past deleted data. func TestPruneKeepsLowestAvailableBlockMonotonic(t *testing.T) { @@ -129,3 +196,74 @@ func TestPruneKeepsLowestAvailableBlockMonotonic(t *testing.T) { require.NoError(t, f.Prune(120)) require.Equal(t, uint64(151), f.LowestAvailableSlot()) } + +func TestPruneScanDoesNotBlockUnrelatedEnvelopePersistence(t *testing.T) { + baseFS := afero.NewMemMapFs() + oldRoot := common.Hash{1} + newRoot := common.Hash{2} + oldBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + oldBlock.Block.Slot = 100 + newBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + newBlock.Block.Slot = 200 + require.NoError(t, afero.WriteFile(baseFS, getBeaconStateFilename(oldRoot), []byte{1}, 0o644)) + require.NoError(t, afero.WriteFile(baseFS, getBeaconStateFilename(newRoot), []byte{1}, 0o644)) + + fs := &blockingStatFS{ + Fs: baseFS, + target: getBeaconStateFilename(oldRoot), + started: make(chan struct{}), + release: make(chan struct{}), + } + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + graph.blocks.Store(oldRoot, oldBlock) + graph.blocks.Store(newRoot, newBlock) + + pruneDone := make(chan error, 1) + go func() { pruneDone <- graph.Prune(150) }() + <-fs.started + + envelopeRoot := common.Hash{3} + dumpDone := make(chan error, 1) + go func() { + dumpDone <- graph.DumpEnvelopeOnDisk(envelopeRoot, testExecutionPayloadEnvelope(envelopeRoot, common.Hash{4})) + }() + + var dumpErr error + blocked := false + select { + case dumpErr = <-dumpDone: + case <-time.After(100 * time.Millisecond): + blocked = true + } + close(fs.release) + require.NoError(t, <-pruneDone) + if blocked { + require.NoError(t, <-dumpDone) + t.Fatal("prune held the global persistence lock while scanning the filesystem") + } + require.NoError(t, dumpErr) +} + +func TestPreparedEnvelopeCannotPublishAfterItsBlockIsPruned(t *testing.T) { + fs := afero.NewMemMapFs() + oldRoot := common.Hash{1} + newRoot := common.Hash{2} + oldBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + oldBlock.Block.Slot = 100 + newBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + newBlock.Block.Slot = 200 + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + graph.blocks.Store(oldRoot, oldBlock) + graph.blocks.Store(newRoot, newBlock) + require.NoError(t, afero.WriteFile(fs, getBeaconStateFilename(oldRoot), []byte{1}, 0o644)) + require.NoError(t, afero.WriteFile(fs, getBeaconStateFilename(newRoot), []byte{1}, 0o644)) + + publish, err := graph.PrepareEnvelopeOnDisk(oldRoot, testExecutionPayloadEnvelope(oldRoot, common.Hash{3}), true) + require.NoError(t, err) + require.NoError(t, graph.Prune(150)) + require.ErrorContains(t, publish(), "pruned block") + require.False(t, graph.HasEnvelope(oldRoot)) + pendingRoots, err := graph.PendingEnvelopeIndexRoots() + require.NoError(t, err) + require.NotContains(t, pendingRoots, oldRoot) +} diff --git a/cl/phase1/forkchoice/fork_graph/interface.go b/cl/phase1/forkchoice/fork_graph/interface.go index 17c6ce89834..5819b6c639d 100644 --- a/cl/phase1/forkchoice/fork_graph/interface.go +++ b/cl/phase1/forkchoice/fork_graph/interface.go @@ -63,6 +63,9 @@ type ForkGraph interface { // and for the store.payloads membership check (HasEnvelope), but no separate // execution_payload_state is maintained. DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) error + PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, requireBlock bool) (publish func() error, err error) ReadEnvelopeFromDisk(blockRoot common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) HasEnvelope(blockRoot common.Hash) bool + PendingEnvelopeIndexRoots() ([]common.Hash, error) + MarkEnvelopeIndicesCommitted(blockRoot common.Hash) error } diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index 01c6e86bb08..734c7704d7a 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -18,6 +18,8 @@ package forkchoice import ( "cmp" + "context" + "fmt" "slices" "sync" "sync/atomic" @@ -195,6 +197,8 @@ type ForkChoiceStore struct { // Separate from pendingEnvelopes so that OnBlock replay can distinguish local origin // (skip BLS) from gossip origin (full verification) without inspecting envelope contents. pendingLocalSelfBuildEnvelopes *lru.Cache[common.Hash, *cltypes.SignedExecutionPayloadEnvelope] + pendingEnvelopeRetryMu sync.Mutex + pendingEnvelopeRetryLocal bool // [New in Gloas:EIP7732] Execution blocks whose CL state transition succeeded but // whose EL newPayload failed (e.g. because EL hasn't caught up after forward sync). @@ -203,6 +207,8 @@ type ForkChoiceStore struct { pendingELPayloadsMu sync.Mutex pendingELPayloads []PendingELPayload payloadValidator *execution_client.PayloadValidationCoordinator + envelopeOwnersMu sync.Mutex + envelopeOwners map[common.Hash]*envelopeOwner // db is used to persist execution payload indices (block number/hash) when an envelope // is accepted in OnExecutionPayload. May be nil (e.g. in tests), in which case the @@ -210,6 +216,11 @@ type ForkChoiceStore struct { db kv.RwDB } +type envelopeOwner struct { + mu sync.Mutex + refs int +} + // PendingELPayload holds a block+envelope pair that needs to be fed to the EL. type PendingELPayload struct { Block *cltypes.SignedBeaconBlock @@ -428,6 +439,9 @@ func NewForkChoiceStore( f.payloadDataAvailabilityVote.Store(common.Hash(anchorRoot), anchorDataAvailabilityVotes) f.gloasWeightTree = newGloasWeightTree(f) + if err := f.reconcilePendingEnvelopeIndices(context.Background()); err != nil { + return nil, fmt.Errorf("reconcile execution payload envelope indices: %w", err) + } return f, nil } @@ -1103,11 +1117,26 @@ func (f *ForkChoiceStore) RequeuePendingELPayload(p PendingELPayload) { // DrainPendingELPayloads returns and clears all queued EL payloads. // The stages layer calls this before Flush() to retry them with engine.NewPayload. func (f *ForkChoiceStore) DrainPendingELPayloads() []PendingELPayload { + return f.DrainPendingELPayloadsLimit(maxPendingELPayloads) +} + +func (f *ForkChoiceStore) DrainPendingELPayloadsLimit(limit int) []PendingELPayload { + if limit <= 0 { + return nil + } f.pendingELPayloadsMu.Lock() defer f.pendingELPayloadsMu.Unlock() if len(f.pendingELPayloads) == 0 { return nil } + if len(f.pendingELPayloads) > limit { + result := make([]PendingELPayload, limit) + copy(result, f.pendingELPayloads[:limit]) + copy(f.pendingELPayloads, f.pendingELPayloads[limit:]) + clear(f.pendingELPayloads[len(f.pendingELPayloads)-limit:]) + f.pendingELPayloads = f.pendingELPayloads[:len(f.pendingELPayloads)-limit] + return result + } if cap(f.pendingELPayloads) > pendingELPayloadsShrinkCap { result := f.pendingELPayloads f.pendingELPayloads = nil diff --git a/cl/phase1/forkchoice/forkchoice_test.go b/cl/phase1/forkchoice/forkchoice_test.go index 8ae73173608..587e0e36510 100644 --- a/cl/phase1/forkchoice/forkchoice_test.go +++ b/cl/phase1/forkchoice/forkchoice_test.go @@ -514,6 +514,18 @@ func (g *getFinalizedExecutionHashForkGraph) DumpEnvelopeOnDisk(common.Hash, *cl panic("not used") } +func (g *getFinalizedExecutionHashForkGraph) PrepareEnvelopeOnDisk(common.Hash, *cltypes.SignedExecutionPayloadEnvelope, bool) (func() error, error) { + panic("not used") +} + +func (g *getFinalizedExecutionHashForkGraph) PendingEnvelopeIndexRoots() ([]common.Hash, error) { + panic("not used") +} + +func (g *getFinalizedExecutionHashForkGraph) MarkEnvelopeIndicesCommitted(common.Hash) error { + panic("not used") +} + func (g *getFinalizedExecutionHashForkGraph) ReadEnvelopeFromDisk(common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { panic("not used") } diff --git a/cl/phase1/forkchoice/interface.go b/cl/phase1/forkchoice/interface.go index e94aa336463..fb46297fa20 100644 --- a/cl/phase1/forkchoice/interface.go +++ b/cl/phase1/forkchoice/interface.go @@ -141,9 +141,10 @@ type ForkChoiceStorageWriter interface { // self-build envelope, skipping BLS signature verification. EL validation still runs. // MUST only be called from the local block production path. ApplyLocalSelfBuildEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) error + RetryPendingExecutionPayloadEnvelopes(ctx context.Context, limit int) int // [New in Gloas:EIP7732] OnPayloadAttestationMessage processes a PTC attestation message from gossip. // Returns error if validation fails (REJECT), nil if accepted or ignored. - OnPayloadAttestationMessage(msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error + OnPayloadAttestationMessage(ctx context.Context, msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error // [New in Gloas:EIP7732] StoreAnchorEnvelope persists an envelope to disk and updates // eth2Roots without running state transition. Used during checkpoint sync where the // finalized state already incorporates the envelope's effects but subsequent blocks diff --git a/cl/phase1/forkchoice/mock_services/forkchoice_mock.go b/cl/phase1/forkchoice/mock_services/forkchoice_mock.go index 5c283b4a6a8..337b335a050 100644 --- a/cl/phase1/forkchoice/mock_services/forkchoice_mock.go +++ b/cl/phase1/forkchoice/mock_services/forkchoice_mock.go @@ -359,11 +359,15 @@ func (f *ForkChoiceStorageMock) ApplyLocalSelfBuildEnvelope(ctx context.Context, return nil } +func (f *ForkChoiceStorageMock) RetryPendingExecutionPayloadEnvelopes(context.Context, int) int { + return 0 +} + func (f *ForkChoiceStorageMock) StoreAnchorEnvelope(blockRoot common.Hash, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) error { return nil } -func (f *ForkChoiceStorageMock) OnPayloadAttestationMessage(msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error { +func (f *ForkChoiceStorageMock) OnPayloadAttestationMessage(_ context.Context, msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error { return nil } diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go index 0df6918d0f2..4044a850ff4 100644 --- a/cl/phase1/forkchoice/on_block.go +++ b/cl/phase1/forkchoice/on_block.go @@ -29,7 +29,6 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/monitor" - "github.com/erigontech/erigon/cl/persistence/beacon_indicies" "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" @@ -39,7 +38,6 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/execution/protocol/misc" "github.com/erigontech/erigon/execution/types" ) @@ -47,13 +45,14 @@ import ( const foreseenProposers = 16 var ( - ErrEIP4844DataNotAvailable = errors.New("EIP-4844 blob data is not available") - ErrEIP7594ColumnDataNotAvailable = errors.New("EIP-7594 column data is not available") - ErrNewPayloadNoStatus = errors.New("newPayload returned no status") - ErrMissingSegment = errors.New("missing segment: parent state not available") - ErrParentEnvelopePending = errors.New("parent execution payload envelope not yet available") - ErrNotFinalizedDescendant = errors.New("block is not a descendant of the finalized checkpoint") - ErrForkSchemaSlotMismatch = errors.New("block schema fork disagrees with the fork implied by its slot") + ErrEIP4844DataNotAvailable = errors.New("EIP-4844 blob data is not available") + ErrEIP7594ColumnDataNotAvailable = errors.New("EIP-7594 column data is not available") + ErrNewPayloadNoStatus = errors.New("newPayload returned no status") + ErrMissingSegment = errors.New("missing segment: parent state not available") + ErrParentEnvelopePending = errors.New("parent execution payload envelope not yet available") + ErrNotFinalizedDescendant = errors.New("block is not a descendant of the finalized checkpoint") + ErrForkSchemaSlotMismatch = errors.New("block schema fork disagrees with the fork implied by its slot") + ErrELPayloadValidationUnavailable = errors.New("execution payload validation unavailable") ) func verifyKzgCommitmentsAgainstTransactions(cfg *clparams.BeaconChainConfig, block *cltypes.BeaconBlock) error { @@ -334,7 +333,8 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // [New in Gloas:EIP7732] GLOAS-specific on_block logic (post state transition) var appliedEnvelope *cltypes.ExecutionPayloadEnvelope - stateMayBeStale := false + var pendingEnvelope *cltypes.SignedExecutionPayloadEnvelope + var pendingLocalSelfBuild bool if blockVersion >= clparams.GloasVersion { // Initialize payload timeliness and data availability votes for this block f.payloadTimelinessVote.Store(common.Hash(blockRoot), [clparams.PtcSize]int8{}) @@ -359,34 +359,10 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // queues so that origin is determined by which queue wrote the entry, not by // inspecting envelope contents (which an attacker could forge). if pending, ok := f.pendingLocalSelfBuildEnvelopes.Get(common.Hash(blockRoot)); ok { - f.pendingLocalSelfBuildEnvelopes.Remove(common.Hash(blockRoot)) - log.Trace("OnBlock: processing pending local self-build envelope", "blockRoot", common.Hash(blockRoot)) - stateMayBeStale = true - applied, applyErr := f.applyLocalSelfBuildEnvelopeLocked(ctx, pending) - if applyErr != nil { - log.Warn("OnBlock: failed to process pending local self-build envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) - } else if applied { - appliedEnvelope = pending.Message - } + pendingEnvelope = pending + pendingLocalSelfBuild = true } else if pending, ok := f.pendingEnvelopes.Get(common.Hash(blockRoot)); ok { - f.pendingEnvelopes.Remove(common.Hash(blockRoot)) - log.Trace("OnBlock: processing pending envelope", "blockRoot", common.Hash(blockRoot)) - // Always validate payload with EL for pending envelopes, regardless of the caller's newPayload flag. - // During forward sync newPayload is false, but the envelope still needs to reach the EL; - // otherwise the EL never learns about this block and the chain stalls. - stateMayBeStale = true - applied, applyErr := f.applyEnvelopeLocked(ctx, pending, checkDataAvaiability, true) - if applyErr != nil { - log.Warn("OnBlock: failed to process pending envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) - } else if applied { - appliedEnvelope = pending.Message - } - } - } - if stateMayBeStale { - lastProcessedState, err = f.refreshBlockStateAfterPayloadValidation(common.Hash(blockRoot)) - if err != nil { - return err + pendingEnvelope = pending } } if lastProcessedState.Slot()%f.beaconCfg.SlotsPerEpoch == 0 { @@ -486,13 +462,30 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac unlocked = true f.mu.Unlock() f.drainQueuedWork() + if pendingEnvelope != nil { + var applied bool + var applyErr error + if pendingLocalSelfBuild { + applied, applyErr = f.applyLocalSelfBuildEnvelope(ctx, pendingEnvelope) + } else { + applied, applyErr = f.applyEnvelope(ctx, pendingEnvelope, checkDataAvaiability, true) + } + if applyErr != nil { + log.Warn("OnBlock: failed to process pending envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) + } else { + f.removePendingExecutionPayloadEnvelope(pendingExecutionPayloadEnvelope{ + root: common.Hash(blockRoot), + envelope: pendingEnvelope, + local: pendingLocalSelfBuild, + }) + if applied { + appliedEnvelope = pendingEnvelope.Message + } + } + } - // Write execution payload envelope indices outside f.mu to avoid deadlock - // with postForkchoiceOperations (which holds MDBX tx then needs f.mu.RLock). - if appliedEnvelope != nil && f.db != nil { - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, common.Hash(blockRoot), appliedEnvelope) - }); err != nil { + if appliedEnvelope != nil { + if err := f.writeEnvelopeIndices(ctx, common.Hash(blockRoot), appliedEnvelope, pendingEnvelope, pendingLocalSelfBuild); err != nil { log.Warn("OnBlock: failed to write execution payload indices for pending envelope", "blockRoot", common.Hash(blockRoot), "err", err) } diff --git a/cl/phase1/forkchoice/on_execution_payload.go b/cl/phase1/forkchoice/on_execution_payload.go index 9c461214ba5..49077375909 100644 --- a/cl/phase1/forkchoice/on_execution_payload.go +++ b/cl/phase1/forkchoice/on_execution_payload.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "os" "time" "github.com/erigontech/erigon/cl/abstract" @@ -44,7 +45,16 @@ import ( // the payload because it hasn't caught up yet (e.g. parent block not available). // applyEnvelope treats this as non-fatal: it proceeds with persisting the envelope // and queues the execution block for later EL insertion. -var errELBehind = errors.New("EL behind: payload not processable yet") +var ( + errELBehind = errors.New("EL behind: payload not processable yet") + errExecutionPayloadInvalid = errors.New("execution payload envelope is invalid") +) + +type pendingExecutionPayloadEnvelope struct { + root common.Hash + envelope *cltypes.SignedExecutionPayloadEnvelope + local bool +} // validateEnvelopeAgainstBlock validates the envelope against the block and state. // This includes: @@ -230,18 +240,18 @@ func (f *ForkChoiceStore) validatePayloadWithEL( envelope *cltypes.ExecutionPayloadEnvelope, block *cltypes.SignedBeaconBlock, beaconBlockRoot common.Hash, -) error { +) (execution_client.PayloadStatus, error) { if f.engine == nil { - return nil + return execution_client.PayloadStatusNone, nil } if envelope == nil || envelope.Payload == nil || envelope.ExecutionRequests == nil { - return errors.New("validatePayloadWithEL: incomplete envelope") + return execution_client.PayloadStatusNone, errors.New("validatePayloadWithEL: incomplete envelope") } // Get committed bid from the block (not from state, since state transition hasn't happened yet) committedBid := block.Block.Body.GetSignedExecutionPayloadBid() if committedBid == nil || committedBid.Message == nil { - return errors.New("validatePayloadWithEL: block missing execution payload bid") + return execution_client.PayloadStatusNone, errors.New("validatePayloadWithEL: block missing execution payload bid") } // Calculate versioned hashes from committed bid's blob_kzg_commitments @@ -257,7 +267,7 @@ func (f *ForkChoiceStore) validatePayloadWithEL( versionedHashes = append(versionedHashes, versionedHash) return nil }); err != nil { - return fmt.Errorf("validatePayloadWithEL: failed to compute versioned hashes: %w", err) + return execution_client.PayloadStatusNone, fmt.Errorf("validatePayloadWithEL: failed to compute versioned hashes: %w", err) } } @@ -271,7 +281,7 @@ func (f *ForkChoiceStore) validatePayloadWithEL( } validationKey, err := envelope.HashSSZ() if err != nil { - return fmt.Errorf("validatePayloadWithEL: failed to hash envelope: %w", err) + return execution_client.PayloadStatusNone, fmt.Errorf("validatePayloadWithEL: failed to hash envelope: %w", err) } // Call NewPayload to validate execution payload with EL @@ -285,6 +295,17 @@ func (f *ForkChoiceStore) validatePayloadWithEL( executionBlockHash := envelope.Payload.BlockHash f.executionPayloadStatus.Add(executionBlockHash, payloadStatus) f.executionPayloadGasLimit.Add(executionBlockHash, envelope.Payload.GasLimit) + if payloadStatus == execution_client.PayloadStatusInvalidated { + log.Warn("validatePayloadWithEL: payload is invalid", "beaconBlockRoot", beaconBlockRoot, "err", err) + f.markPayloadInvalidLocked(beaconBlockRoot, executionBlockHash) + if err != nil { + return payloadStatus, fmt.Errorf("execution payload is invalid: %w", err) + } + return payloadStatus, errors.New("execution payload is invalid") + } + if err != nil { + return payloadStatus, fmt.Errorf("%w: %w", ErrELPayloadValidationUnavailable, err) + } switch payloadStatus { case execution_client.PayloadStatusNone: @@ -295,29 +316,20 @@ func (f *ForkChoiceStore) validatePayloadWithEL( log.Warn("validatePayloadWithEL: EL could not process payload (EL behind)", "beaconBlockRoot", beaconBlockRoot, "blockHash", executionBlockHash, "err", err) if optErr := f.optimisticStore.AddOptimisticCandidate(beaconBlockRoot, block.Block); optErr != nil { - return fmt.Errorf("failed to add block to optimistic store: %v", optErr) + return payloadStatus, fmt.Errorf("failed to add block to optimistic store: %v", optErr) } - return errELBehind + return payloadStatus, errELBehind case execution_client.PayloadStatusNotValidated: log.Trace("validatePayloadWithEL: payload is not validated yet", "beaconBlockRoot", beaconBlockRoot) // optimistic block candidate if err := f.optimisticStore.AddOptimisticCandidate(beaconBlockRoot, block.Block); err != nil { - return fmt.Errorf("failed to add block to optimistic store: %v", err) + return payloadStatus, fmt.Errorf("failed to add block to optimistic store: %v", err) } - case execution_client.PayloadStatusInvalidated: - log.Warn("validatePayloadWithEL: payload is invalid", "beaconBlockRoot", beaconBlockRoot, "err", err) - f.markPayloadInvalidLocked(beaconBlockRoot, executionBlockHash) - return errors.New("execution payload is invalid") case execution_client.PayloadStatusValidated: log.Trace("validatePayloadWithEL: payload is validated", "beaconBlockRoot", beaconBlockRoot) - f.markPayloadVerifiedLocked(beaconBlockRoot, executionBlockHash) } - if err != nil { - return fmt.Errorf("validatePayloadWithEL: newPayload failed: %v", err) - } - - return nil + return payloadStatus, nil } func (f *ForkChoiceStore) newPayloadWithoutForkChoiceLock( @@ -337,6 +349,206 @@ func (f *ForkChoiceStore) newPayloadWithoutForkChoiceLock( return payloadValidator.NewPayload(ctx, beaconBlockRoot, payload, parentBlockRoot, versionedHashes, executionRequestsList) } +func (f *ForkChoiceStore) lockEnvelopeOwner(blockRoot common.Hash) func() { + f.envelopeOwnersMu.Lock() + if f.envelopeOwners == nil { + f.envelopeOwners = make(map[common.Hash]*envelopeOwner) + } + owner := f.envelopeOwners[blockRoot] + if owner == nil { + owner = &envelopeOwner{} + f.envelopeOwners[blockRoot] = owner + } + owner.refs++ + f.envelopeOwnersMu.Unlock() + + owner.mu.Lock() + return func() { + owner.mu.Unlock() + f.envelopeOwnersMu.Lock() + owner.refs-- + if owner.refs == 0 { + delete(f.envelopeOwners, blockRoot) + } + f.envelopeOwnersMu.Unlock() + } +} + +// RetryPendingExecutionPayloadEnvelopes retries a bounded, fair batch of envelopes retained by fork choice. +func (f *ForkChoiceStore) RetryPendingExecutionPayloadEnvelopes(ctx context.Context, limit int) int { + if limit <= 0 || ctx.Err() != nil { + return 0 + } + + f.pendingEnvelopeRetryMu.Lock() + defer f.pendingEnvelopeRetryMu.Unlock() + + candidates := f.pendingExecutionPayloadEnvelopeCandidates(limit) + attempted := 0 + for _, candidate := range candidates { + if ctx.Err() != nil { + break + } + attempted++ + f.pendingEnvelopeRetryLocal = !candidate.local + var err error + if candidate.local { + err = f.ApplyLocalSelfBuildEnvelope(ctx, candidate.envelope) + } else { + err = f.OnExecutionPayload(ctx, candidate.envelope, true, true) + } + if err != nil && !errors.Is(err, errExecutionPayloadInvalid) { + f.rotatePendingExecutionPayloadEnvelope(candidate) + log.Debug("pending execution payload envelope retry deferred", "blockRoot", candidate.root, "local", candidate.local, "err", err) + continue + } + f.removePendingExecutionPayloadEnvelope(candidate) + } + return attempted +} + +func (f *ForkChoiceStore) pendingExecutionPayloadEnvelopeCandidates(limit int) []pendingExecutionPayloadEnvelope { + var gossipRoots, localRoots []common.Hash + if f.pendingEnvelopes != nil { + gossipRoots = f.pendingEnvelopes.Keys() + } + if f.pendingLocalSelfBuildEnvelopes != nil { + localRoots = f.pendingLocalSelfBuildEnvelopes.Keys() + } + candidates := make([]pendingExecutionPayloadEnvelope, 0, min(limit, len(gossipRoots)+len(localRoots))) + gossipIndex, localIndex := 0, 0 + preferLocal := f.pendingEnvelopeRetryLocal + for len(candidates) < limit && (gossipIndex < len(gossipRoots) || localIndex < len(localRoots)) { + local := preferLocal + if local && localIndex >= len(localRoots) { + local = false + } else if !local && gossipIndex >= len(gossipRoots) { + local = true + } + + var root common.Hash + var envelope *cltypes.SignedExecutionPayloadEnvelope + var ok bool + if local { + root = localRoots[localIndex] + localIndex++ + envelope, ok = f.pendingLocalSelfBuildEnvelopes.Peek(root) + } else { + root = gossipRoots[gossipIndex] + gossipIndex++ + envelope, ok = f.pendingEnvelopes.Peek(root) + } + if !ok { + continue + } + candidates = append(candidates, pendingExecutionPayloadEnvelope{root: root, envelope: envelope, local: local}) + preferLocal = !local + } + return candidates +} + +func (f *ForkChoiceStore) removePendingExecutionPayloadEnvelope(candidate pendingExecutionPayloadEnvelope) { + unlockOwner := f.lockEnvelopeOwner(candidate.root) + defer unlockOwner() + + cache := f.pendingEnvelopes + if candidate.local { + cache = f.pendingLocalSelfBuildEnvelopes + } + if cache == nil { + return + } + current, ok := cache.Peek(candidate.root) + if ok && current == candidate.envelope { + cache.Remove(candidate.root) + } +} + +func (f *ForkChoiceStore) rotatePendingExecutionPayloadEnvelope(candidate pendingExecutionPayloadEnvelope) { + unlockOwner := f.lockEnvelopeOwner(candidate.root) + defer unlockOwner() + + cache := f.pendingEnvelopes + if candidate.local { + cache = f.pendingLocalSelfBuildEnvelopes + } + if cache == nil { + return + } + current, ok := cache.Peek(candidate.root) + if ok && current == candidate.envelope { + cache.Get(candidate.root) + } +} + +func (f *ForkChoiceStore) retainPendingExecutionPayloadEnvelope(signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, local bool) { + if signedEnvelope == nil || signedEnvelope.Message == nil { + return + } + cache := f.pendingEnvelopes + if local { + cache = f.pendingLocalSelfBuildEnvelopes + } + if cache != nil { + cache.Add(signedEnvelope.Message.BeaconBlockRoot, signedEnvelope) + } +} + +func (f *ForkChoiceStore) writeEnvelopeIndices(ctx context.Context, blockRoot common.Hash, envelope *cltypes.ExecutionPayloadEnvelope, retryEnvelope *cltypes.SignedExecutionPayloadEnvelope, local bool) error { + if f.db == nil { + return nil + } + if err := f.persistEnvelopeIndices(ctx, blockRoot, envelope); err != nil { + f.retainPendingExecutionPayloadEnvelope(retryEnvelope, local) + return err + } + return nil +} + +func (f *ForkChoiceStore) persistEnvelopeIndices(ctx context.Context, blockRoot common.Hash, envelope *cltypes.ExecutionPayloadEnvelope) error { + if err := f.db.Update(ctx, func(tx kv.RwTx) error { + return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, blockRoot, envelope) + }); err != nil { + return err + } + return f.forkGraph.MarkEnvelopeIndicesCommitted(blockRoot) +} + +func (f *ForkChoiceStore) reconcilePendingEnvelopeIndices(ctx context.Context) error { + if f.db == nil { + return nil + } + roots, err := f.forkGraph.PendingEnvelopeIndexRoots() + if err != nil { + return err + } + for _, root := range roots { + envelope, err := f.forkGraph.ReadEnvelopeFromDisk(root) + if os.IsNotExist(err) { + if err := f.forkGraph.MarkEnvelopeIndicesCommitted(root); err != nil { + return err + } + continue + } + if err != nil { + return err + } + if envelope == nil || envelope.Message == nil || envelope.Message.Payload == nil || envelope.Message.BeaconBlockRoot != root { + return fmt.Errorf("invalid pending envelope for block %x", root) + } + if err := f.persistEnvelopeIndices(ctx, root, envelope.Message); err != nil { + return err + } + } + return nil +} + +func (f *ForkChoiceStore) prepareEnvelopeWithoutForkChoiceLock(blockRoot common.Hash, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) (func() error, error) { + f.mu.Unlock() + defer f.mu.Lock() + return f.forkGraph.PrepareEnvelopeOnDisk(blockRoot, signedEnvelope, true) +} + // applyEnvelope processes the envelope under f.mu: validates, verifies with CL and EL, // and persists the envelope to disk. No CL state transition is performed — the // execution effects are deferred to the next block's ProcessParentExecutionPayload. @@ -348,11 +560,23 @@ func (f *ForkChoiceStore) applyEnvelope(ctx context.Context, signedEnvelope *clt log.Warn("[applyEnvelope] received signed envelope with nil message") return false, errors.New("signed envelope has nil message") } + unlockOwner := f.lockEnvelopeOwner(signedEnvelope.Message.BeaconBlockRoot) + defer unlockOwner() + return f.applyEnvelopeOwned(ctx, signedEnvelope, checkBlobData, validatePayload) +} +func (f *ForkChoiceStore) applyEnvelopeOwned(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.applyEnvelopeLocked(ctx, signedEnvelope, checkBlobData, validatePayload) + return f.applyEnvelopeLocked(ctx, signedEnvelope, checkBlobData, validatePayload, false) +} + +func (f *ForkChoiceStore) validatePersistedEnvelopeOwned(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData bool) error { + f.mu.Lock() + defer f.mu.Unlock() + _, err := f.applyEnvelopeLocked(ctx, signedEnvelope, checkBlobData, true, true) + return err } // applyEnvelopeLocked is the lock-held implementation of applyEnvelope. @@ -360,16 +584,16 @@ func (f *ForkChoiceStore) applyEnvelope(ctx context.Context, signedEnvelope *clt // Returns (true, nil) if the envelope was applied, // (false, nil) if it was skipped (already processed or block not yet known), // or (false, err) on failure. -func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool) (bool, error) { +func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload, alreadyPersisted bool) (bool, error) { if signedEnvelope.Message == nil { log.Warn("[applyEnvelopeLocked] received signed envelope with nil message") - return false, errors.New("signed envelope has nil message") + return false, fmt.Errorf("%w: signed envelope has nil message", errExecutionPayloadInvalid) } envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot // Skip if envelope already processed and persisted - if f.forkGraph.HasEnvelope(beaconBlockRoot) { + if !alreadyPersisted && f.forkGraph.HasEnvelope(beaconBlockRoot) { return false, nil } @@ -400,7 +624,7 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop // Validate envelope against block (bid matching + signature verification) if validatePayload { if err := f.validateEnvelopeAgainstBlock(signedEnvelope, block, blockState); err != nil { - return false, fmt.Errorf("OnExecutionPayload: envelope validation failed: %w", err) + return false, fmt.Errorf("%w: OnExecutionPayload: envelope validation failed: %w", errExecutionPayloadInvalid, err) } } @@ -413,18 +637,23 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop // Validate payload with EL var elBehind bool + var payloadValidated bool if validatePayload { - if err := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)); err != nil { - if errors.Is(err, errELBehind) { + payloadStatus, validationErr := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)) + payloadValidated = payloadStatus == execution_client.PayloadStatusValidated + if validationErr != nil { + if errors.Is(validationErr, errELBehind) { // EL is behind (e.g. parent block not yet available after forward sync). // Proceed with persisting the envelope so HasEnvelope() returns true. // The execution block will be fed to EL via blockCollector on the next Flush(). elBehind = true + } else if payloadStatus == execution_client.PayloadStatusInvalidated { + return false, fmt.Errorf("%w: %w", errExecutionPayloadInvalid, validationErr) } else { - return false, err + return false, validationErr } } - if f.forkGraph.HasEnvelope(beaconBlockRoot) { + if !alreadyPersisted && f.forkGraph.HasEnvelope(beaconBlockRoot) { return false, nil } blockState, err = f.forkGraph.GetState(beaconBlockRoot, false) @@ -449,17 +678,24 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop // Always use ValidatingMachine so that signature verification and all spec checks run, // regardless of whether the EL-level validatePayload flag is set. if err := transition.ValidatingMachine.ProcessExecutionPayloadEnvelope(blockState, signedEnvelope); err != nil { - return false, fmt.Errorf("OnExecutionPayload: failed to verify execution payload: %w", err) + return false, fmt.Errorf("%w: OnExecutionPayload: failed to verify execution payload: %w", errExecutionPayloadInvalid, err) } - // Update eth2Roots mapping for FCU + // Persist envelope to disk — this marks the root as "has payload" in store.payloads + if !alreadyPersisted { + publish, err := f.prepareEnvelopeWithoutForkChoiceLock(beaconBlockRoot, signedEnvelope) + if err != nil { + return false, fmt.Errorf("OnExecutionPayload: failed to dump envelope: %w", err) + } + if err := publish(); err != nil { + return false, fmt.Errorf("OnExecutionPayload: failed to publish envelope: %w", err) + } + } if envelope.Payload != nil { f.eth2Roots.Add(beaconBlockRoot, envelope.Payload.BlockHash) } - - // Persist envelope to disk — this marks the root as "has payload" in store.payloads - if err := f.forkGraph.DumpEnvelopeOnDisk(beaconBlockRoot, signedEnvelope); err != nil { - return false, fmt.Errorf("OnExecutionPayload: failed to dump envelope: %w", err) + if payloadValidated { + f.markPayloadVerifiedLocked(beaconBlockRoot, envelope.Payload.BlockHash) } // Invalidate head cache — payload status may have changed from PENDING to FULL. @@ -488,22 +724,41 @@ func (f *ForkChoiceStore) StoreAnchorEnvelope(blockRoot common.Hash, signedEnvel if envelope.BeaconBlockRoot != blockRoot { return fmt.Errorf("StoreAnchorEnvelope: envelope root %v does not match block root %v", envelope.BeaconBlockRoot, blockRoot) } + unlockOwner := f.lockEnvelopeOwner(blockRoot) + defer unlockOwner() + + acceptedEnvelope := envelope + var publish func() error + if f.forkGraph.HasEnvelope(blockRoot) { + persistedEnvelope, err := f.forkGraph.ReadEnvelopeFromDisk(blockRoot) + if err != nil { + return fmt.Errorf("StoreAnchorEnvelope: failed to read persisted envelope: %w", err) + } + if persistedEnvelope == nil || persistedEnvelope.Message == nil || persistedEnvelope.Message.Payload == nil || persistedEnvelope.Message.BeaconBlockRoot != blockRoot { + return errors.New("StoreAnchorEnvelope: invalid persisted envelope") + } + acceptedEnvelope = persistedEnvelope.Message + } else { + var err error + publish, err = f.forkGraph.PrepareEnvelopeOnDisk(blockRoot, signedEnvelope, false) + if err != nil { + return fmt.Errorf("StoreAnchorEnvelope: failed to dump envelope: %w", err) + } + } f.mu.Lock() - if err := f.forkGraph.DumpEnvelopeOnDisk(blockRoot, signedEnvelope); err != nil { - f.mu.Unlock() - return fmt.Errorf("StoreAnchorEnvelope: failed to dump envelope: %w", err) + if publish != nil { + if err := publish(); err != nil { + f.mu.Unlock() + return fmt.Errorf("StoreAnchorEnvelope: failed to publish envelope: %w", err) + } } - f.eth2Roots.Add(blockRoot, envelope.Payload.BlockHash) + f.eth2Roots.Add(blockRoot, acceptedEnvelope.Payload.BlockHash) f.headHash = common.Hash{} f.headPayloadStatus = cltypes.PayloadStatusPending f.mu.Unlock() - if f.db != nil { - ctx := context.Background() - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, blockRoot, envelope) - }); err != nil { + if err := f.persistEnvelopeIndices(context.Background(), blockRoot, acceptedEnvelope); err != nil { return fmt.Errorf("StoreAnchorEnvelope: failed to write indices: %w", err) } } @@ -521,26 +776,63 @@ func (f *ForkChoiceStore) StoreAnchorEnvelope(blockRoot common.Hash, signedEnvel // - validatePayload: if true, call engine.NewPayload() to validate with EL before state transition func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool) error { if signedEnvelope == nil || signedEnvelope.Message == nil { - return errors.New("nil execution payload envelope") + return fmt.Errorf("%w: nil execution payload envelope", errExecutionPayloadInvalid) + } + if signedEnvelope.Message.Payload == nil || signedEnvelope.Message.ExecutionRequests == nil { + return fmt.Errorf("%w: incomplete execution payload envelope", errExecutionPayloadInvalid) } envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot + unlockOwner := f.lockEnvelopeOwner(beaconBlockRoot) + defer unlockOwner() // Process envelope under f.mu; DB index write happens after unlock to avoid // deadlock with postForkchoiceOperations (which holds MDBX tx then needs f.mu.RLock). - applied, err := f.applyEnvelope(ctx, signedEnvelope, checkBlobData, validatePayload) - if err != nil || !applied { + applied, err := f.applyEnvelopeOwned(ctx, signedEnvelope, checkBlobData, validatePayload) + if err != nil { + if !errors.Is(err, errExecutionPayloadInvalid) { + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, false) + } return err } + acceptedEnvelope := envelope + if !applied { + if !f.forkGraph.HasEnvelope(beaconBlockRoot) { + return nil + } + if !validatePayload && f.db == nil { + return nil + } + persistedEnvelope, readErr := f.forkGraph.ReadEnvelopeFromDisk(beaconBlockRoot) + if readErr != nil { + return fmt.Errorf("OnExecutionPayload: failed to read persisted envelope: %w", readErr) + } + if persistedEnvelope == nil || persistedEnvelope.Message == nil || persistedEnvelope.Message.Payload == nil || persistedEnvelope.Message.BeaconBlockRoot != beaconBlockRoot { + return fmt.Errorf("%w: OnExecutionPayload: invalid persisted envelope", errExecutionPayloadInvalid) + } + callerIdentity, identityErr := signedEnvelope.HashSSZ() + if identityErr != nil { + return fmt.Errorf("%w: OnExecutionPayload: failed to hash caller envelope: %w", errExecutionPayloadInvalid, identityErr) + } + persistedIdentity, identityErr := persistedEnvelope.HashSSZ() + if identityErr != nil { + return fmt.Errorf("%w: OnExecutionPayload: failed to hash persisted envelope: %w", errExecutionPayloadInvalid, identityErr) + } + if callerIdentity != persistedIdentity { + return fmt.Errorf("%w: OnExecutionPayload: caller does not match persisted envelope", errExecutionPayloadInvalid) + } + acceptedEnvelope = persistedEnvelope.Message + if validatePayload { + if err := f.validatePersistedEnvelopeOwned(ctx, persistedEnvelope, checkBlobData); err != nil { + return err + } + } + } // Write execution block indices outside f.mu. - if f.db != nil { - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, common.Hash(beaconBlockRoot), envelope) - }); err != nil { - return fmt.Errorf("OnExecutionPayload: failed to write execution payload indices: %w", err) - } + if err := f.writeEnvelopeIndices(ctx, common.Hash(beaconBlockRoot), acceptedEnvelope, signedEnvelope, false); err != nil { + return fmt.Errorf("OnExecutionPayload: failed to write execution payload indices: %w", err) } return nil @@ -561,23 +853,52 @@ func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope // [New in Gloas:EIP7732] func (f *ForkChoiceStore) ApplyLocalSelfBuildEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) error { if signedEnvelope == nil || signedEnvelope.Message == nil { - return errors.New("nil execution payload envelope") + return fmt.Errorf("%w: nil execution payload envelope", errExecutionPayloadInvalid) + } + if signedEnvelope.Message.Payload == nil || signedEnvelope.Message.ExecutionRequests == nil { + return fmt.Errorf("%w: incomplete execution payload envelope", errExecutionPayloadInvalid) } envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot + unlockOwner := f.lockEnvelopeOwner(beaconBlockRoot) + defer unlockOwner() - applied, err := f.applyLocalSelfBuildEnvelope(ctx, signedEnvelope) - if err != nil || !applied { + applied, err := f.applyLocalSelfBuildEnvelopeOwned(ctx, signedEnvelope) + if err != nil { + if !errors.Is(err, errExecutionPayloadInvalid) { + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, true) + } return err } - - if f.db != nil { - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, common.Hash(beaconBlockRoot), envelope) - }); err != nil { - return fmt.Errorf("ApplyLocalSelfBuildEnvelope: failed to write execution payload indices: %w", err) + acceptedEnvelope := envelope + if !applied { + if !f.forkGraph.HasEnvelope(beaconBlockRoot) || f.db == nil { + return nil + } + persistedEnvelope, readErr := f.forkGraph.ReadEnvelopeFromDisk(beaconBlockRoot) + if readErr != nil { + return fmt.Errorf("ApplyLocalSelfBuildEnvelope: failed to read persisted envelope: %w", readErr) } + if persistedEnvelope == nil || persistedEnvelope.Message == nil || persistedEnvelope.Message.Payload == nil || persistedEnvelope.Message.BeaconBlockRoot != beaconBlockRoot { + return fmt.Errorf("%w: ApplyLocalSelfBuildEnvelope: invalid persisted envelope", errExecutionPayloadInvalid) + } + callerIdentity, identityErr := signedEnvelope.HashSSZ() + if identityErr != nil { + return fmt.Errorf("%w: ApplyLocalSelfBuildEnvelope: failed to hash caller envelope: %w", errExecutionPayloadInvalid, identityErr) + } + persistedIdentity, identityErr := persistedEnvelope.HashSSZ() + if identityErr != nil { + return fmt.Errorf("%w: ApplyLocalSelfBuildEnvelope: failed to hash persisted envelope: %w", errExecutionPayloadInvalid, identityErr) + } + if callerIdentity != persistedIdentity { + return fmt.Errorf("%w: ApplyLocalSelfBuildEnvelope: caller does not match persisted envelope", errExecutionPayloadInvalid) + } + acceptedEnvelope = persistedEnvelope.Message + } + + if err := f.writeEnvelopeIndices(ctx, common.Hash(beaconBlockRoot), acceptedEnvelope, signedEnvelope, true); err != nil { + return fmt.Errorf("ApplyLocalSelfBuildEnvelope: failed to write execution payload indices: %w", err) } return nil @@ -588,7 +909,12 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelope(ctx context.Context, signe if signedEnvelope.Message == nil { return false, errors.New("signed envelope has nil message") } + unlockOwner := f.lockEnvelopeOwner(signedEnvelope.Message.BeaconBlockRoot) + defer unlockOwner() + return f.applyLocalSelfBuildEnvelopeOwned(ctx, signedEnvelope) +} +func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeOwned(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { f.mu.Lock() defer f.mu.Unlock() @@ -635,11 +961,14 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, // Validate payload with EL (NewPayload). var elBehind bool - if err := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)); err != nil { - if errors.Is(err, errELBehind) { + payloadStatus, validationErr := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)) + if validationErr != nil { + if errors.Is(validationErr, errELBehind) { elBehind = true + } else if payloadStatus == execution_client.PayloadStatusInvalidated { + return false, fmt.Errorf("%w: %w", errExecutionPayloadInvalid, validationErr) } else { - return false, err + return false, validationErr } } if f.forkGraph.HasEnvelope(beaconBlockRoot) { @@ -659,15 +988,21 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, // Use DefaultMachine (FullValidation=false) to skip BLS signature verification // in ProcessExecutionPayloadEnvelope while still running all other spec checks. if err := transition.DefaultMachine.ProcessExecutionPayloadEnvelope(blockState, signedEnvelope); err != nil { - return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to verify execution payload: %w", err) + return false, fmt.Errorf("%w: applyLocalSelfBuildEnvelopeLocked: failed to verify execution payload: %w", errExecutionPayloadInvalid, err) } + publish, err := f.prepareEnvelopeWithoutForkChoiceLock(beaconBlockRoot, signedEnvelope) + if err != nil { + return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to dump envelope: %w", err) + } + if err := publish(); err != nil { + return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to publish envelope: %w", err) + } if envelope.Payload != nil { f.eth2Roots.Add(beaconBlockRoot, envelope.Payload.BlockHash) } - - if err := f.forkGraph.DumpEnvelopeOnDisk(beaconBlockRoot, signedEnvelope); err != nil { - return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to dump envelope: %w", err) + if payloadStatus == execution_client.PayloadStatusValidated { + f.markPayloadVerifiedLocked(beaconBlockRoot, envelope.Payload.BlockHash) } f.headHash = common.Hash{} diff --git a/cl/phase1/forkchoice/on_execution_payload_test.go b/cl/phase1/forkchoice/on_execution_payload_test.go index 030151512cc..f2ace653823 100644 --- a/cl/phase1/forkchoice/on_execution_payload_test.go +++ b/cl/phase1/forkchoice/on_execution_payload_test.go @@ -24,17 +24,43 @@ import ( "time" "github.com/hashicorp/golang-lru/v2" + "github.com/spf13/afero" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "github.com/erigontech/erigon/cl/beacon/beacon_router_configuration" + "github.com/erigontech/erigon/cl/beacon/beaconevents" + "github.com/erigontech/erigon/cl/beacon/synced_data" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" + "github.com/erigontech/erigon/cl/persistence/beacon_indicies" + "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" + "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" + "github.com/erigontech/erigon/cl/phase1/forkchoice/optimistic" + "github.com/erigontech/erigon/cl/phase1/forkchoice/public_keys_registry" + "github.com/erigontech/erigon/cl/pool" + "github.com/erigontech/erigon/cl/validator/validator_params" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/hexutil" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/memdb" ) +type failFirstUpdateDB struct { + kv.RwDB + updates atomic.Int32 +} + +func (db *failFirstUpdateDB) Update(ctx context.Context, f func(kv.RwTx) error) error { + if db.updates.Add(1) == 1 { + return errors.New("injected index failure") + } + return db.RwDB.Update(ctx, f) +} + // TestValidateEnvelopeAgainstBlock_NoBid tests that validation fails when block has no bid func TestValidateEnvelopeAgainstBlock_NoBid(t *testing.T) { cfg := &clparams.MainnetBeaconConfig @@ -275,35 +301,539 @@ func TestValidatePayloadWithEL_NoEngine(t *testing.T) { }, } - err := f.validatePayloadWithEL(context.TODO(), envelope, block, common.Hash{}) + _, err := f.validatePayloadWithEL(context.TODO(), envelope, block, common.Hash{}) + require.NoError(t, err) +} + +func TestOnExecutionPayloadRepairsIndicesAfterPriorWriteFailure(t *testing.T) { + root := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + envelope := &cltypes.SignedExecutionPayloadEnvelope{ + Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig), + } + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload.BlockHash = executionHash + envelope.Message.Payload.BlockNumber = 42 + + db := &failFirstUpdateDB{RwDB: memdb.NewTestDB(t, dbcfg.ChainDB)} + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{hasEnvelope: true, envelope: envelope}, + db: db, + pendingEnvelopes: pending, + } + + require.ErrorContains(t, f.OnExecutionPayload(t.Context(), envelope, false, false), "injected index failure") + require.Equal(t, 1, pending.Len()) + require.NoError(t, f.OnExecutionPayload(t.Context(), envelope, false, false)) + + require.NoError(t, db.View(t.Context(), func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, root) + require.NoError(t, err) + require.NotNil(t, blockNumber) + require.Equal(t, uint64(42), *blockNumber) + blockHash, err := beacon_indicies.ReadExecutionBlockHash(tx, root) + require.NoError(t, err) + require.Equal(t, executionHash, blockHash) + return nil + })) +} + +func TestApplyLocalSelfBuildEnvelopeRepairsIndicesAfterPriorWriteFailure(t *testing.T) { + root := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + envelope := &cltypes.SignedExecutionPayloadEnvelope{ + Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig), + } + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload.BlockHash = executionHash + envelope.Message.Payload.BlockNumber = 42 + + db := &failFirstUpdateDB{RwDB: memdb.NewTestDB(t, dbcfg.ChainDB)} + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{hasEnvelope: true, envelope: envelope}, + db: db, + pendingLocalSelfBuildEnvelopes: pending, + } + + require.ErrorContains(t, f.ApplyLocalSelfBuildEnvelope(t.Context(), envelope), "injected index failure") + require.Equal(t, 1, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + require.Zero(t, pending.Len()) + + require.NoError(t, db.View(t.Context(), func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, root) + require.NoError(t, err) + require.NotNil(t, blockNumber) + require.Equal(t, uint64(42), *blockNumber) + return nil + })) +} + +func TestReconcilePendingEnvelopeIndicesAfterRestart(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + anchorState := state.New(cfg) + fs := afero.NewMemMapFs() + graph := fork_graph.NewForkGraphDisk(anchorState, nil, fs, beacon_router_configuration.RouterConfiguration{}) + root := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload.BlockHash = executionHash + envelope.Message.Payload.BlockNumber = 42 + require.NoError(t, graph.DumpEnvelopeOnDisk(root, envelope)) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + committedRoot := common.HexToHash("0x5678") + committedEnvelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + committedEnvelope.Message.BeaconBlockRoot = committedRoot + committedEnvelope.Message.Payload.BlockHash = common.HexToHash("0xcdef") + committedEnvelope.Message.Payload.BlockNumber = 43 + require.NoError(t, graph.DumpEnvelopeOnDisk(committedRoot, committedEnvelope)) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, committedRoot, committedEnvelope.Message) + })) + orphanRoot := common.HexToHash("0x9999") + orphanEnvelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + orphanEnvelope.Message.BeaconBlockRoot = orphanRoot + _, err := graph.PrepareEnvelopeOnDisk(orphanRoot, orphanEnvelope, false) + require.NoError(t, err) + + _, err = NewForkChoiceStore( + nil, + anchorState, + nil, + pool.NewOperationsPool(cfg), + graph, + beaconevents.NewEventEmitter(), + synced_data.NewSyncedDataManager(cfg, true), + nil, + public_keys_registry.NewInMemoryPublicKeysRegistry(), + validator_params.NewValidatorParams(), + false, + db, + ) + require.NoError(t, err) + + require.NoError(t, db.View(t.Context(), func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, root) + require.NoError(t, err) + require.NotNil(t, blockNumber) + require.Equal(t, uint64(42), *blockNumber) + blockHash, err := beacon_indicies.ReadExecutionBlockHash(tx, root) + require.NoError(t, err) + require.Equal(t, executionHash, blockHash) + committedNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, committedRoot) + require.NoError(t, err) + require.NotNil(t, committedNumber) + require.Equal(t, uint64(43), *committedNumber) + orphanNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, orphanRoot) + require.NoError(t, err) + require.Nil(t, orphanNumber) + return nil + })) + pendingRoots, err := graph.PendingEnvelopeIndexRoots() + require.NoError(t, err) + require.Empty(t, pendingRoots) +} + +func TestApplyLocalSelfBuildDoesNotPromoteELValidCLInvalidEnvelope(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + root := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + blockState := state.New(cfg) + blockState.SetVersion(clparams.GloasVersion) + blockState.SetSlot(1) + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{ + Message: &cltypes.ExecutionPayloadBid{ + BlockHash: executionHash, + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + }, + } + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 1, Body: body}} + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload.BlockHash = executionHash + envelope.Message.Payload.SlotNumber = 1 + + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + engine.EXPECT().NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(execution_client.PayloadStatusValidated, nil) + verified, err := lru.New[common.Hash, struct{}](16) + require.NoError(t, err) + executionStatus, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + statusByRoot, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + gasLimits, err := lru.New[common.Hash, uint64](16) + require.NoError(t, err) + eth2Roots, err := lru.New[common.Hash, common.Hash](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + beaconCfg: cfg, + engine: engine, + forkGraph: payloadVoteForkGraph{block: block, blockState: blockState}, + verifiedExecutionPayload: verified, + executionPayloadStatus: executionStatus, + payloadStatusByRoot: statusByRoot, + executionPayloadGasLimit: gasLimits, + eth2Roots: eth2Roots, + optimisticStore: optimistic.NewOptimisticStore(), + } + + err = f.ApplyLocalSelfBuildEnvelope(t.Context(), envelope) + require.ErrorContains(t, err, "beacon_block_root") + require.False(t, f.IsPayloadVerified(root)) + status, ok := f.GetRecentExecutionPayloadStatusByRoot(root) + require.False(t, ok) + require.Zero(t, status) +} + +func TestOnExecutionPayloadValidatesPersistedEnvelopeBeforeAcceptingRecovery(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + root := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + blockState := state.New(cfg) + blockState.SetVersion(clparams.GloasVersion) + blockState.SetSlot(1) + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{ + Message: &cltypes.ExecutionPayloadBid{ + BlockHash: executionHash, + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + }, + } + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 1, Body: body}} + persisted := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + persisted.Message.BeaconBlockRoot = root + persisted.Message.Payload.BlockHash = executionHash + persisted.Message.Payload.SlotNumber = 1 + requestsRoot, err := persisted.Message.ExecutionRequests.HashSSZ() + require.NoError(t, err) + body.SignedExecutionPayloadBid.Message.ExecutionRequestsRoot = requestsRoot + + f := &ForkChoiceStore{ + beaconCfg: cfg, + forkGraph: payloadVoteForkGraph{ + hasEnvelope: true, + envelope: persisted, + block: block, + blockState: blockState, + }, + } + + err = f.OnExecutionPayload(t.Context(), persisted, false, true) + require.ErrorContains(t, err, "invalid builder signature") +} + +func TestOnExecutionPayloadRejectsCallerDifferentFromPersistedEnvelope(t *testing.T) { + root := common.HexToHash("0x1234") + persisted := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + persisted.Message.BeaconBlockRoot = root + caller := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + caller.Message.BeaconBlockRoot = root + caller.Message.Payload.BlockHash = common.HexToHash("0xabcd") + f := &ForkChoiceStore{forkGraph: payloadVoteForkGraph{hasEnvelope: true, envelope: persisted}} + + err := f.OnExecutionPayload(t.Context(), caller, false, true) + require.ErrorContains(t, err, "does not match persisted envelope") +} + +func TestOnExecutionPayloadRejectsIncompleteCallerBeforePersistedFallback(t *testing.T) { + root := common.HexToHash("0x1234") + persisted := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + persisted.Message.BeaconBlockRoot = root + caller := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + caller.Message.BeaconBlockRoot = root + caller.Message.Payload = nil + f := &ForkChoiceStore{forkGraph: payloadVoteForkGraph{hasEnvelope: true, envelope: persisted}} + + err := f.OnExecutionPayload(t.Context(), caller, false, true) + require.ErrorContains(t, err, "incomplete execution payload envelope") +} + +func TestPendingLocalSelfBuildEnvelopeSurvivesCanceledApplyAndRetries(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + blockState := state.New(cfg) + blockState.SetVersion(clparams.GloasVersion) + blockState.SetSlot(1) + blockState.SetGenesisTime(0) + stateRoot := common.HexToHash("0x1111") + parentRoot := common.HexToHash("0x2222") + header := &cltypes.BeaconBlockHeader{Slot: 1, ParentRoot: parentRoot} + blockState.SetLatestBlockHeader(header) + blockState.SetPreviousStateRoot(stateRoot) + headerWithStateRoot := *header + headerWithStateRoot.Root = stateRoot + blockRoot, err := headerWithStateRoot.HashSSZ() + require.NoError(t, err) + + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + envelope.Message.BeaconBlockRoot = blockRoot + envelope.Message.ParentBeaconBlockRoot = parentRoot + envelope.Message.Payload.SlotNumber = 1 + envelope.Message.Payload.BlockHash = common.HexToHash("0xabcd") + envelope.Message.Payload.ParentHash = common.HexToHash("0x3333") + envelope.Message.Payload.Time = cfg.SecondsPerSlot + envelope.Message.Payload.Withdrawals = solid.NewStaticListSSZ[*cltypes.Withdrawal](int(cfg.MaxWithdrawalsPerPayload), 44) + requestsRoot, err := envelope.Message.ExecutionRequests.HashSSZ() + require.NoError(t, err) + bid := &cltypes.ExecutionPayloadBid{ + BuilderIndex: envelope.Message.BuilderIndex, + PrevRandao: envelope.Message.Payload.PrevRandao, + GasLimit: envelope.Message.Payload.GasLimit, + BlockHash: envelope.Message.Payload.BlockHash, + ExecutionRequestsRoot: requestsRoot, + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + } + blockState.SetLatestExecutionPayloadBid(bid) + blockState.SetLatestBlockHash(envelope.Message.Payload.ParentHash) + blockState.SetPayloadExpectedWithdrawals(envelope.Message.Payload.Withdrawals) + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{Message: bid} + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{ + Slot: 1, + ParentRoot: parentRoot, + StateRoot: stateRoot, + Body: body, + }} + + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + engine.EXPECT().NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(execution_client.PayloadStatusNone, context.Canceled) + engine.EXPECT().NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(execution_client.PayloadStatusValidated, nil).Times(2) + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + require.NoError(t, err) + verified, err := lru.New[common.Hash, struct{}](16) + require.NoError(t, err) + executionStatus, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + statusByRoot, err := lru.New[common.Hash, execution_client.PayloadStatus](16) require.NoError(t, err) + gasLimits, err := lru.New[common.Hash, uint64](16) + require.NoError(t, err) + eth2Roots, err := lru.New[common.Hash, common.Hash](16) + require.NoError(t, err) + dumpedRoot := common.Hash{} + var persisted atomic.Bool + var dumpCalls atomic.Int32 + dumpStarted := make(chan struct{}) + releaseDump := make(chan struct{}) + graph := &payloadVoteForkGraph{ + dumpedEnvelope: &dumpedRoot, + hasEnvelopeState: &persisted, + dumpCalls: &dumpCalls, + dumpStarted: dumpStarted, + releaseDump: releaseDump, + dumpErr: errors.New("injected persistence failure"), + } + f := &ForkChoiceStore{ + beaconCfg: cfg, + engine: engine, + forkGraph: graph, + pendingLocalSelfBuildEnvelopes: pending, + verifiedExecutionPayload: verified, + executionPayloadStatus: executionStatus, + payloadStatusByRoot: statusByRoot, + executionPayloadGasLimit: gasLimits, + eth2Roots: eth2Roots, + optimisticStore: optimistic.NewOptimisticStore(), + } + var childObservedHalfPublished atomic.Bool + graph.onEnvelopePublished = func() { + if !f.mu.TryLock() { + return + } + defer f.mu.Unlock() + if _, ok := f.eth2Roots.Peek(common.Hash(blockRoot)); !ok { + childObservedHalfPublished.Store(true) + } + } + + require.ErrorIs(t, f.ApplyLocalSelfBuildEnvelope(t.Context(), envelope), ErrIgnore) + _, ok := pending.Get(blockRoot) + require.True(t, ok) + graph.block = block + graph.blockState = blockState + require.Equal(t, 1, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + _, ok = pending.Get(blockRoot) + require.True(t, ok) + require.Equal(t, 1, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + require.False(t, f.IsPayloadVerified(blockRoot)) + _, ok = eth2Roots.Get(blockRoot) + require.False(t, ok) + _, ok = pending.Get(blockRoot) + require.True(t, ok) + firstDone := make(chan int, 1) + go func() { firstDone <- f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1) }() + <-dumpStarted + + lockAcquired := make(chan struct{}) + go func() { + f.mu.Lock() + close(lockAcquired) + f.mu.Unlock() + }() + select { + case <-lockAcquired: + case <-time.After(time.Second): + t.Fatal("forkchoice mutex stayed locked during envelope persistence") + } + + duplicateDone := make(chan error, 1) + go func() { duplicateDone <- f.ApplyLocalSelfBuildEnvelope(t.Context(), envelope) }() + select { + case err := <-duplicateDone: + t.Fatalf("same-root retry bypassed persistence ownership: %v", err) + case <-time.After(50 * time.Millisecond): + } + close(releaseDump) + require.Equal(t, 1, <-firstDone) + require.NoError(t, <-duplicateDone) + _, ok = pending.Get(blockRoot) + require.False(t, ok) + require.Equal(t, common.Hash(blockRoot), dumpedRoot) + require.Equal(t, int32(2), dumpCalls.Load()) + require.False(t, childObservedHalfPublished.Load(), "child import observed the envelope before its execution root was promoted") +} + +func TestRetryPendingExecutionPayloadEnvelopesIsBoundedAndFair(t *testing.T) { + gossip, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + pendingEnvelopes: gossip, + pendingLocalSelfBuildEnvelopes: local, + } + for i := byte(1); i <= 3; i++ { + gossip.Add(common.Hash{i}, &cltypes.SignedExecutionPayloadEnvelope{ + Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: common.Hash{i}}, + }) + local.Add(common.Hash{i + 3}, &cltypes.SignedExecutionPayloadEnvelope{ + Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: common.Hash{i + 3}}, + }) + } + + canceled, cancel := context.WithCancel(t.Context()) + cancel() + require.Zero(t, f.RetryPendingExecutionPayloadEnvelopes(canceled, 1)) + require.Equal(t, 1, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + require.Equal(t, 2, gossip.Len()) + require.Equal(t, 3, local.Len()) + + require.Equal(t, 3, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 3)) + require.Equal(t, 1, gossip.Len()) + require.Equal(t, 1, local.Len()) + + require.Equal(t, 1, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + require.Equal(t, 0, gossip.Len()) + require.Equal(t, 1, local.Len()) +} + +func TestRetryPendingExecutionPayloadEnvelopesRotatesDeferredWork(t *testing.T) { + gossip, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{}, + pendingEnvelopes: gossip, + } + for i := byte(1); i <= 3; i++ { + gossip.Add(common.Hash{i}, &cltypes.SignedExecutionPayloadEnvelope{ + Message: &cltypes.ExecutionPayloadEnvelope{ + BeaconBlockRoot: common.Hash{i}, + Payload: cltypes.NewEth1Block(clparams.GloasVersion, &clparams.MainnetBeaconConfig), + ExecutionRequests: cltypes.NewExecutionRequestsWithVersion(&clparams.MainnetBeaconConfig, clparams.GloasVersion), + }, + }) + } + + require.Equal(t, 2, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 2)) + require.Equal(t, []common.Hash{{3}, {1}, {2}}, gossip.Keys()) + require.Equal(t, 2, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 2)) + require.Equal(t, []common.Hash{{2}, {3}, {1}}, gossip.Keys()) +} + +func TestEnvelopeOwnershipAndPruneDoNotGloballyExcludeEachOther(t *testing.T) { + root := common.HexToHash("0x1234") + t.Run("active envelope owner does not block prune", func(t *testing.T) { + pruneStarted := make(chan struct{}) + releasePrune := make(chan struct{}) + f := &ForkChoiceStore{forkGraph: payloadVoteForkGraph{pruneStarted: pruneStarted, releasePrune: releasePrune}} + unlockOwner := f.lockEnvelopeOwner(root) + f.queuedPrunes = []uint64{1} + pruneDone := make(chan struct{}) + go func() { + f.drainQueuedWork() + close(pruneDone) + }() + select { + case <-pruneStarted: + case <-time.After(time.Second): + t.Fatal("active envelope owner blocked prune") + } + unlockOwner() + close(releasePrune) + <-pruneDone + }) + + t.Run("active prune does not block envelope owner", func(t *testing.T) { + pruneStarted := make(chan struct{}) + releasePrune := make(chan struct{}) + f := &ForkChoiceStore{forkGraph: payloadVoteForkGraph{pruneStarted: pruneStarted, releasePrune: releasePrune}} + f.queuedPrunes = []uint64{1} + pruneDone := make(chan struct{}) + go func() { + f.drainQueuedWork() + close(pruneDone) + }() + <-pruneStarted + ownerAcquired := make(chan func(), 1) + go func() { ownerAcquired <- f.lockEnvelopeOwner(root) }() + select { + case unlockOwner := <-ownerAcquired: + unlockOwner() + case <-time.After(time.Second): + t.Fatal("active prune blocked unrelated envelope owner") + } + close(releasePrune) + <-pruneDone + }) } func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { cfg := &clparams.MainnetBeaconConfig for _, tt := range []struct { - name string - status execution_client.PayloadStatus - wantErr bool - wantVerify bool + name string + status execution_client.PayloadStatus + engineErr error + wantErr bool }{ { - name: "validated", - status: execution_client.PayloadStatusValidated, - wantVerify: true, + name: "validated", + status: execution_client.PayloadStatusValidated, }, { name: "invalidated", status: execution_client.PayloadStatusInvalidated, wantErr: true, }, + { + name: "invalidated with validation error", + status: execution_client.PayloadStatusInvalidated, + engineErr: errors.New("invalid payload"), + wantErr: true, + }, } { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) engine := execution_client.NewMockExecutionEngine(ctrl) engine.EXPECT(). NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(tt.status, nil) + Return(tt.status, tt.engineErr) verifiedExecutionPayload, err := lru.New[common.Hash, struct{}](16) require.NoError(t, err) @@ -344,7 +874,8 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { go func() { f.mu.Lock() defer f.mu.Unlock() - done <- f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + _, err := f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + done <- err }() select { @@ -357,7 +888,7 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { case <-time.After(time.Second): t.Fatal("validatePayloadWithEL blocked while forkchoice mutex was already held") } - require.Equal(t, tt.wantVerify, f.IsPayloadVerified(blockRoot)) + require.False(t, f.IsPayloadVerified(blockRoot)) if tt.status == execution_client.PayloadStatusInvalidated { require.Equal(t, blockRoot, invalidatedHeader) } @@ -411,7 +942,8 @@ func TestValidatePayloadWithELReleasesForkChoiceMuDuringNewPayload(t *testing.T) go func() { f.mu.Lock() defer f.mu.Unlock() - validationDone <- f.validatePayloadWithEL(context.Background(), envelope, block, common.HexToHash("0x1234")) + _, err := f.validatePayloadWithEL(context.Background(), envelope, block, common.HexToHash("0x1234")) + validationDone <- err }() <-engineStarted @@ -488,7 +1020,8 @@ func TestValidatePayloadWithELDoesNotCoalesceDifferentPayloads(t *testing.T) { validate := func(envelope *cltypes.ExecutionPayloadEnvelope) { f.mu.Lock() defer f.mu.Unlock() - results <- f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + _, err := f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + results <- err } go validate(first) <-firstStarted diff --git a/cl/phase1/forkchoice/on_payload_attestation_message.go b/cl/phase1/forkchoice/on_payload_attestation_message.go index dfb11847d5f..87af0944d72 100644 --- a/cl/phase1/forkchoice/on_payload_attestation_message.go +++ b/cl/phase1/forkchoice/on_payload_attestation_message.go @@ -17,6 +17,7 @@ package forkchoice import ( + "context" "errors" "fmt" @@ -31,6 +32,7 @@ import ( // Caller should handle errors appropriately based on isFromBlock context. // [New in Gloas:EIP7732] func (f *ForkChoiceStore) OnPayloadAttestationMessage( + ctx context.Context, msg *cltypes.PayloadAttestationMessage, isFromBlock bool, ) error { @@ -49,7 +51,7 @@ func (f *ForkChoiceStore) OnPayloadAttestationMessage( } } - validationContext, err := f.payloadAttestationValidationContext(blockRoot, data.Slot) + validationContext, err := f.payloadAttestationValidationContext(ctx, blockRoot, data.Slot) if err != nil { return err } diff --git a/cl/phase1/forkchoice/payload_attestation_validation.go b/cl/phase1/forkchoice/payload_attestation_validation.go index e5627b2ad16..8539519896c 100644 --- a/cl/phase1/forkchoice/payload_attestation_validation.go +++ b/cl/phase1/forkchoice/payload_attestation_validation.go @@ -17,11 +17,10 @@ package forkchoice import ( + "context" "errors" "fmt" - "golang.org/x/sync/singleflight" - "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/fork" "github.com/erigontech/erigon/cl/phase1/core/state" @@ -44,7 +43,6 @@ type payloadAttestationValidationContext struct { type payloadAttestationValidationContexts struct { cache *lru.Cache[common.Hash, *payloadAttestationValidationContext] - buildGroup singleflight.Group buildSlots chan struct{} } @@ -63,36 +61,36 @@ func newPayloadAttestationValidationContexts() (*payloadAttestationValidationCon } func (c *payloadAttestationValidationContexts) get( + ctx context.Context, blockRoot common.Hash, build func() (*payloadAttestationValidationContext, error), ) (*payloadAttestationValidationContext, error) { if validationContext, ok := c.cache.Get(blockRoot); ok { return validationContext, nil } - value, err, _ := c.buildGroup.Do(string(blockRoot[:]), func() (any, error) { - if validationContext, ok := c.cache.Get(blockRoot); ok { - return validationContext, nil - } - c.buildSlots <- struct{}{} - defer func() { <-c.buildSlots }() - validationContext, err := build() - if err != nil { - return nil, err - } - c.cache.Add(blockRoot, validationContext) + select { + case c.buildSlots <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-c.buildSlots }() + if validationContext, ok := c.cache.Get(blockRoot); ok { return validationContext, nil - }) + } + validationContext, err := build() if err != nil { return nil, err } - return value.(*payloadAttestationValidationContext), nil + c.cache.Add(blockRoot, validationContext) + return validationContext, nil } func (f *ForkChoiceStore) payloadAttestationValidationContext( + ctx context.Context, blockRoot common.Hash, slot uint64, ) (*payloadAttestationValidationContext, error) { - return f.payloadAttestationContexts.get(blockRoot, func() (*payloadAttestationValidationContext, error) { + return f.payloadAttestationContexts.get(ctx, blockRoot, func() (*payloadAttestationValidationContext, error) { blockState, err := f.GetStateAtBlockRoot(blockRoot, true) if err != nil { return nil, err diff --git a/cl/phase1/forkchoice/payload_attestation_validation_test.go b/cl/phase1/forkchoice/payload_attestation_validation_test.go index 74070250554..dc5706ccf7f 100644 --- a/cl/phase1/forkchoice/payload_attestation_validation_test.go +++ b/cl/phase1/forkchoice/payload_attestation_validation_test.go @@ -17,6 +17,7 @@ package forkchoice import ( + "context" "errors" "sync" "sync/atomic" @@ -38,8 +39,8 @@ type payloadAttestationValidationContextResult struct { func TestOnPayloadAttestationMessageRejectsNil(t *testing.T) { f := &ForkChoiceStore{} - require.Error(t, f.OnPayloadAttestationMessage(nil, false)) - require.Error(t, f.OnPayloadAttestationMessage(&cltypes.PayloadAttestationMessage{}, false)) + require.Error(t, f.OnPayloadAttestationMessage(context.Background(), nil, false)) + require.Error(t, f.OnPayloadAttestationMessage(context.Background(), &cltypes.PayloadAttestationMessage{}, false)) } func TestPayloadAttestationValidationContextsCollapseConcurrentBuilds(t *testing.T) { @@ -64,7 +65,7 @@ func TestPayloadAttestationValidationContextsCollapseConcurrentBuilds(t *testing var wg sync.WaitGroup for range 16 { wg.Go(func() { - validationContext, getErr := contexts.get(root, build) + validationContext, getErr := contexts.get(context.Background(), root, build) results <- payloadAttestationValidationContextResult{validationContext, getErr} }) } @@ -78,7 +79,7 @@ func TestPayloadAttestationValidationContextsCollapseConcurrentBuilds(t *testing require.NoError(t, result.err) require.Same(t, expected, result.validationContext) } - _, err = contexts.get(root, build) + _, err = contexts.get(context.Background(), root, build) require.NoError(t, err) require.Equal(t, int32(1), builds.Load()) } @@ -89,14 +90,14 @@ func TestPayloadAttestationValidationContextsDoNotCacheBuildErrors(t *testing.T) root := common.HexToHash("0x1234") var builds atomic.Int32 - _, err = contexts.get(root, func() (*payloadAttestationValidationContext, error) { + _, err = contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { builds.Add(1) return nil, errors.New("state unavailable") }) require.ErrorContains(t, err, "state unavailable") expected := &payloadAttestationValidationContext{slot: 100} - actual, err := contexts.get(root, func() (*payloadAttestationValidationContext, error) { + actual, err := contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { builds.Add(1) return expected, nil }) @@ -116,7 +117,7 @@ func TestPayloadAttestationValidationContextsBoundDifferentRootBuilds(t *testing results := make(chan error, 3) for i := range 3 { go func() { - _, getErr := contexts.get(common.Hash{byte(i + 1)}, func() (*payloadAttestationValidationContext, error) { + _, getErr := contexts.get(context.Background(), common.Hash{byte(i + 1)}, func() (*payloadAttestationValidationContext, error) { current := active.Add(1) defer active.Add(-1) for { @@ -152,6 +153,21 @@ func TestPayloadAttestationValidationContextsBoundDifferentRootBuilds(t *testing require.Equal(t, int32(maxConcurrentValidationContextBuilds), maxActive.Load()) } +func TestPayloadAttestationValidationContextWaitHonorsCancellation(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + contexts.buildSlots <- struct{}{} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = contexts.get(ctx, common.Hash{1}, func() (*payloadAttestationValidationContext, error) { + return &payloadAttestationValidationContext{}, nil + }) + + require.ErrorIs(t, err, context.Canceled) + <-contexts.buildSlots +} + func TestPayloadAttestationValidationContextPositions(t *testing.T) { validationContext := &payloadAttestationValidationContext{ slot: 100, diff --git a/cl/phase1/forkchoice/payload_vote_test.go b/cl/phase1/forkchoice/payload_vote_test.go index 5443359aadd..8ad8823fc76 100644 --- a/cl/phase1/forkchoice/payload_vote_test.go +++ b/cl/phase1/forkchoice/payload_vote_test.go @@ -1,6 +1,7 @@ package forkchoice import ( + "sync/atomic" "testing" lru "github.com/hashicorp/golang-lru/v2" @@ -33,22 +34,92 @@ func (g ptcVoteForkGraph) GetBlock(root common.Hash) (*cltypes.SignedBeaconBlock type payloadVoteForkGraph struct { fork_graph.ForkGraph - hasEnvelope bool - dumpedEnvelope *common.Hash - invalidatedHeader *common.Hash + hasEnvelope bool + envelope *cltypes.SignedExecutionPayloadEnvelope + block *cltypes.SignedBeaconBlock + blockState *state2.CachingBeaconState + dumpedEnvelope *common.Hash + hasEnvelopeState *atomic.Bool + dumpCalls *atomic.Int32 + dumpStarted chan struct{} + releaseDump chan struct{} + dumpErr error + onEnvelopePublished func() + pruneStarted chan struct{} + releasePrune chan struct{} + invalidatedHeader *common.Hash +} + +func (g payloadVoteForkGraph) Prune(uint64) error { + if g.pruneStarted != nil { + close(g.pruneStarted) + } + if g.releasePrune != nil { + <-g.releasePrune + } + return nil } func (g payloadVoteForkGraph) HasEnvelope(common.Hash) bool { - return g.hasEnvelope + return g.hasEnvelope || g.hasEnvelopeState != nil && g.hasEnvelopeState.Load() } func (g payloadVoteForkGraph) DumpEnvelopeOnDisk(blockRoot common.Hash, _ *cltypes.SignedExecutionPayloadEnvelope) error { - if g.dumpedEnvelope != nil { - *g.dumpedEnvelope = blockRoot + publish, err := g.PrepareEnvelopeOnDisk(blockRoot, nil, false) + if err != nil { + return err + } + return publish() +} + +func (g payloadVoteForkGraph) PrepareEnvelopeOnDisk(blockRoot common.Hash, _ *cltypes.SignedExecutionPayloadEnvelope, _ bool) (func() error, error) { + var call int32 + if g.dumpCalls != nil { + call = g.dumpCalls.Add(1) + } + if g.dumpErr != nil && call == 1 { + return nil, g.dumpErr } + if g.dumpStarted != nil { + close(g.dumpStarted) + } + if g.releaseDump != nil { + <-g.releaseDump + } + return func() error { + if g.dumpedEnvelope != nil { + *g.dumpedEnvelope = blockRoot + } + if g.hasEnvelopeState != nil { + g.hasEnvelopeState.Store(true) + } + if g.onEnvelopePublished != nil { + g.onEnvelopePublished() + } + return nil + }, nil +} + +func (g payloadVoteForkGraph) PendingEnvelopeIndexRoots() ([]common.Hash, error) { + return nil, nil +} + +func (g payloadVoteForkGraph) MarkEnvelopeIndicesCommitted(common.Hash) error { return nil } +func (g payloadVoteForkGraph) ReadEnvelopeFromDisk(common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { + return g.envelope, nil +} + +func (g payloadVoteForkGraph) GetBlock(common.Hash) (*cltypes.SignedBeaconBlock, bool) { + return g.block, g.block != nil +} + +func (g payloadVoteForkGraph) GetState(common.Hash, bool) (*state2.CachingBeaconState, error) { + return g.blockState, nil +} + func (g payloadVoteForkGraph) MarkHeaderAsInvalid(blockRoot common.Hash) { if g.invalidatedHeader != nil { *g.invalidatedHeader = blockRoot diff --git a/cl/phase1/forkchoice/pending_el_payload_test.go b/cl/phase1/forkchoice/pending_el_payload_test.go index e4c5c9dd996..6d94e4a3610 100644 --- a/cl/phase1/forkchoice/pending_el_payload_test.go +++ b/cl/phase1/forkchoice/pending_el_payload_test.go @@ -51,3 +51,20 @@ func TestPendingELPayloadsDeduplicateByEnvelopeRoot(t *testing.T) { require.Len(t, payloads, 1) require.Equal(t, uint64(1), payloads[0].Block.Block.Slot) } + +func TestDrainPendingELPayloadsLimitLeavesRemainderQueued(t *testing.T) { + f := &ForkChoiceStore{} + for i := range 3 { + f.RequeuePendingELPayload(PendingELPayload{Envelope: &cltypes.SignedExecutionPayloadEnvelope{ + Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: common.Hash{byte(i + 1)}}, + }}) + } + + first := f.DrainPendingELPayloadsLimit(2) + require.Len(t, first, 2) + require.Equal(t, common.Hash{1}, first[0].Envelope.Message.BeaconBlockRoot) + require.Equal(t, common.Hash{2}, first[1].Envelope.Message.BeaconBlockRoot) + remaining := f.DrainPendingELPayloads() + require.Len(t, remaining, 1) + require.Equal(t, common.Hash{3}, remaining[0].Envelope.Message.BeaconBlockRoot) +} diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index 491a64e47e2..7b2aad40bcc 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -17,6 +17,7 @@ package network import ( + "bytes" "context" "errors" "fmt" @@ -34,13 +35,16 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/persistence/base_encoding" "github.com/erigontech/erigon/cl/persistence/beacon_indicies" + "github.com/erigontech/erigon/cl/persistence/format/snapshot_format" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/rpc" "github.com/erigontech/erigon/cl/sentinel/peers" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/snapshotsync/freezeblocks" + "github.com/klauspost/compress/zstd" ) // Whether the reverse downloader arrived at expected height or condition. @@ -52,6 +56,14 @@ type BlockChecker interface { HasBlock(blockNumber uint64) bool } +type blockRangeSource uint8 + +const ( + blockRangeSourceUnknown blockRangeSource = iota + blockRangeSourceP2P + blockRangeSourceHTTP +) + type BackwardBeaconDownloader struct { ctx context.Context slotToDownload atomic.Uint64 @@ -69,16 +81,20 @@ type BackwardBeaconDownloader struct { // [New in Gloas:EIP7732] highest block from the previous batch, used as lookahead // to determine FULL/EMPTY status of the highest block in the current batch. prevBatchTopBlock *cltypes.SignedBeaconBlock + prevBatchTopBlockUntrusted bool httpFallbackURL string // beacon API base URL for HTTP fallback when P2P fails httpPreferred atomic.Bool // set after first HTTP success; skips P2P probing consecutiveLookaheadFailures uint8 lookaheadSearchOffset uint64 lookaheadRescan bool + lookaheadRetryWindow bool lookaheadAnchorRoot common.Hash + blockRangeSource blockRangeSource - // Count consecutive batches where envelope fetch returned 0 for all FULL roots. + // Count consecutive batches where at least one required FULL envelope was unresolved. // After enough failures, skip envelope requirements and process blocks as EMPTY. consecutiveEnvelopeFailures int + consecutiveProbeFailures uint8 envelopesSkipped bool // set when we give up on envelopes // FULL blocks that were processed without envelopes due to envelopesSkipped. @@ -92,14 +108,21 @@ const ( gloasLookaheadWindow = uint64(64) maxSkippedFullBlocks = 65536 maxBeaconAPIResponseBytes = 64 << 20 + maxLookaheadFailures = uint8(6) ) +var errSkippedEnvelopeRecoveryCapacity = errors.New("skipped envelope recovery capacity exhausted") + // SkippedFullBlock records a GLOAS block that may need an envelope after degraded backward download. type SkippedFullBlock struct { Slot uint64 Root [32]byte } +type EnvelopeRecoveryResult struct { + Envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope +} + func NewBackwardBeaconDownloader(ctx context.Context, rpc *rpc.BeaconRpcP2P, sn *freezeblocks.CaplinSnapshots, engine execution_client.ExecutionEngine, db kv.RwDB, beaconCfg *clparams.BeaconChainConfig) *BackwardBeaconDownloader { return &BackwardBeaconDownloader{ ctx: ctx, @@ -219,6 +242,7 @@ func (b *BackwardBeaconDownloader) fetchBlockRange(ctx context.Context) ([]*clty if b.httpPreferred.Load() && b.httpFallbackURL != "" { blocks, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, count, b.beaconCfg) if err == nil && len(blocks) > 0 { + b.blockRangeSource = blockRangeSourceHTTP log.Debug("[BackwardBeaconDownloader] fetched blocks from beacon API", "fromSlot", start, "count", len(blocks)) return blocks, nil } @@ -245,6 +269,7 @@ func (b *BackwardBeaconDownloader) fetchBlockRange(ctx context.Context) ([]*clty go b.sendBlockRequest(ctx, start, count, received, &requestSent) case responses := <-received: + b.blockRangeSource = blockRangeSourceP2P return responses, nil case <-p2pDeadline.C: @@ -254,6 +279,7 @@ func (b *BackwardBeaconDownloader) fetchBlockRange(ctx context.Context) ([]*clty } blocks, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, count, b.beaconCfg) if err == nil && len(blocks) > 0 { + b.blockRangeSource = blockRangeSourceHTTP log.Debug("[BackwardBeaconDownloader] P2P failed, fetched blocks from beacon API", "fromSlot", start, "count", len(blocks)) b.httpPreferred.Store(true) return blocks, nil @@ -302,7 +328,10 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons if err := b.prepareFirstBatchLookahead(ctx, responses); err != nil { log.Warn("[BackwardBeaconDownloader] GLOAS lookahead unavailable", "err", err) b.waitBeforeLookaheadRetry(ctx) - return nil + if b.consecutiveLookaheadFailures < maxLookaheadFailures { + return nil + } + b.envelopesSkipped = true } // [New in Gloas:EIP7732] Fetch envelopes for GLOAS FULL blocks before processing. @@ -337,13 +366,24 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons matched = true var envelope *cltypes.SignedExecutionPayloadEnvelope + trackSkippedForRecovery := false if envelopes != nil { envelope = envelopes[common.Hash(blockRoot)] } if block.Version() >= clparams.GloasVersion { - if _, known := knownRootSet[common.Hash(blockRoot)]; !known { - log.Warn("[BackwardBeaconDownloader] GLOAS block availability unknown, will retry", "slot", block.Block.Slot) - return nil + _, availabilityKnown := knownRootSet[common.Hash(blockRoot)] + _, isFull := fullRootSet[common.Hash(blockRoot)] + trackSkippedForRecovery = isFull || !availabilityKnown + if !availabilityKnown && !b.envelopesSkipped { + b.recordEmptyProbeResult(1, 0) + if b.envelopesSkipped { + knownRootSet[common.Hash(blockRoot)] = struct{}{} + } else { + log.Warn("[BackwardBeaconDownloader] GLOAS block availability unknown, will retry", "slot", block.Block.Slot) + return nil + } + } else if availabilityKnown { + b.recordEmptyProbeResult(0, 0) } if envelope != nil { if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), envelope); err != nil { @@ -351,9 +391,8 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } } - if envelope == nil && b.envelopesSkipped && !b.canTrackSkippedFullBlock(block) { - log.Warn("[BackwardBeaconDownloader] skipped envelope recovery queue is full, will retry", "slot", block.Block.Slot) - return nil + if envelope == nil && b.envelopesSkipped && trackSkippedForRecovery && !b.canTrackSkippedFullBlock(block) { + return fmt.Errorf("%w at slot %d", errSkippedEnvelopeRecoveryCapacity, block.Block.Slot) } } @@ -374,16 +413,18 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons } // Record FULL blocks passing through without envelope for post-download recovery. - if _, isFull := fullRootSet[common.Hash(blockRoot)]; envelope == nil && (isFull || b.envelopesSkipped) { + if envelope == nil && trackSkippedForRecovery { b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) } advanced = true b.prevBatchTopBlock = block + b.prevBatchTopBlockUntrusted = false b.expectedRoot = block.Block.ParentRoot if block.Block.Slot == 0 { b.finished.Store(true) b.prevBatchTopBlock = firstCompleteBlock(responses) + b.prevBatchTopBlockUntrusted = false return nil } b.slotToDownload.Store(block.Block.Slot - 1) @@ -409,22 +450,36 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons var envelope *cltypes.SignedExecutionPayloadEnvelope isFull := false + availabilityKnown := block.Version() < clparams.GloasVersion if block.Version() >= clparams.GloasVersion { lookahead := b.prevBatchTopBlock - if _, known := gloasBlockAvailability(block, lookahead); !known { + if _, linked := gloasBlockAvailability(block, lookahead); !linked { lookahead, err = b.fetchGloasLookahead(ctx, block, common.Hash(blockRoot)) if err != nil { - log.Warn("[BackwardBeaconDownloader] root-fetched GLOAS lookahead unavailable, will retry", "slot", block.Block.Slot, "err", err) - return nil + if b.consecutiveLookaheadFailures < maxLookaheadFailures { + b.consecutiveLookaheadFailures++ + } + if b.consecutiveLookaheadFailures < maxLookaheadFailures { + log.Warn("[BackwardBeaconDownloader] root-fetched GLOAS lookahead unavailable, will retry", "slot", block.Block.Slot, "err", err) + return nil + } + b.envelopesSkipped = true + } else { + b.consecutiveLookaheadFailures = 0 } } - full, known := gloasBlockAvailability(block, lookahead) - if !known { - log.Warn("[BackwardBeaconDownloader] root-fetched GLOAS block availability unknown, will retry", "slot", block.Block.Slot) - return nil + full, known := b.gloasBlockAvailability(block, lookahead) + availabilityKnown = known + if !known && !b.envelopesSkipped { + b.recordEmptyProbeResult(1, 0) + if !b.envelopesSkipped { + log.Warn("[BackwardBeaconDownloader] root-fetched GLOAS block availability unknown, will retry", "slot", block.Block.Slot) + return nil + } } isFull = full if full && !b.envelopesSkipped { + b.recordEmptyProbeResult(0, 0) env, fetchErr := b.fetchSingleEnvelope(ctx, common.Hash(blockRoot)) if fetchErr == nil && env != nil { if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err == nil { @@ -442,8 +497,13 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons } else if !full && !b.envelopesSkipped { env, fetchErr := b.fetchSingleEnvelope(ctx, common.Hash(blockRoot)) if fetchErr != nil { - log.Warn("[BackwardBeaconDownloader] root-fetched EMPTY confirmation failed, will retry", "slot", block.Block.Slot, "err", fetchErr) - return nil + b.recordEmptyProbeResult(1, 0) + if !b.envelopesSkipped { + log.Warn("[BackwardBeaconDownloader] root-fetched EMPTY confirmation failed, will retry", "slot", block.Block.Slot, "err", fetchErr) + return nil + } + } else { + b.recordEmptyProbeResult(0, 0) } if env != nil { if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err != nil { @@ -454,9 +514,8 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons isFull = true } } - if envelope == nil && b.envelopesSkipped && !b.canTrackSkippedFullBlock(block) { - log.Warn("[BackwardBeaconDownloader] skipped envelope recovery queue is full, will retry", "slot", block.Block.Slot) - return nil + if envelope == nil && b.envelopesSkipped && (isFull || !availabilityKnown) && !b.canTrackSkippedFullBlock(block) { + return fmt.Errorf("%w at slot %d", errSkippedEnvelopeRecoveryCapacity, block.Block.Slot) } } @@ -465,10 +524,11 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons if err != nil { log.Warn("Error processing root-fetched block", "err", err) } else { - if envelope == nil && (isFull || b.envelopesSkipped) { + if envelope == nil && (isFull || !availabilityKnown) { b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) } b.prevBatchTopBlock = block + b.prevBatchTopBlockUntrusted = false b.expectedRoot = block.Block.ParentRoot if block.Block.Slot == 0 { b.finished.Store(true) @@ -540,9 +600,11 @@ func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Contex } if lookahead := selectGloasLookahead(anchor, anchorRoot, responses); lookahead != nil { b.prevBatchTopBlock = lookahead + b.prevBatchTopBlockUntrusted = b.blockRangeSource != blockRangeSourceHTTP b.consecutiveLookaheadFailures = 0 b.lookaheadSearchOffset = 0 b.lookaheadRescan = false + b.lookaheadRetryWindow = false return nil } @@ -554,6 +616,7 @@ func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Contex b.consecutiveLookaheadFailures = 0 b.lookaheadSearchOffset = 0 b.lookaheadRescan = false + b.lookaheadRetryWindow = false return nil } @@ -566,6 +629,7 @@ func (b *BackwardBeaconDownloader) fetchGloasLookahead( b.lookaheadAnchorRoot = anchorRoot b.lookaheadSearchOffset = 0 b.lookaheadRescan = false + b.lookaheadRetryWindow = false } if anchor.Block.Slot == math.MaxUint64 { return nil, errors.New("cannot fetch lookahead after max slot") @@ -579,27 +643,34 @@ func (b *BackwardBeaconDownloader) fetchGloasLookahead( offset = 0 } start := anchor.Block.Slot + 1 + offset - sources := make([]gloasLookaheadFetcher, 0, 2) + errs := make([]error, 0, 2) if b.httpFallbackURL != "" { - sources = append(sources, func(ctx context.Context, start, count uint64) ([]*cltypes.SignedBeaconBlock, error) { - return fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, count, b.beaconCfg) - }) + candidates, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, gloasLookaheadWindow, b.beaconCfg) + if err == nil { + if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { + b.prevBatchTopBlockUntrusted = false + return lookahead, nil + } + err = errors.New("HTTP lookahead source returned no direct child") + } + errs = append(errs, err) } if b.rpc != nil { - sources = append(sources, func(ctx context.Context, start, count uint64) ([]*cltypes.SignedBeaconBlock, error) { - blocks, _, err := b.rpc.SendBeaconBlocksByRangeReq(ctx, start, count) - return blocks, err - }) + candidates, _, err := b.rpc.SendBeaconBlocksByRangeReq(ctx, start, gloasLookaheadWindow) + if err == nil { + if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { + b.prevBatchTopBlockUntrusted = true + return lookahead, nil + } + err = errors.New("P2P lookahead source returned no direct child") + } + errs = append(errs, err) } - if len(sources) == 0 { + if len(errs) == 0 { return nil, errors.New("no GLOAS lookahead source configured") } - lookahead, err := fetchGloasLookaheadFromSources(ctx, anchor, anchorRoot, start, sources...) - if lookahead != nil { - return lookahead, nil - } b.advanceLookaheadSearch() - return nil, err + return nil, errors.Join(errs...) } type gloasLookaheadFetcher func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) @@ -623,6 +694,11 @@ func fetchGloasLookaheadFromSources(ctx context.Context, anchor *cltypes.SignedB func (b *BackwardBeaconDownloader) advanceLookaheadSearch() { if b.lookaheadRescan { b.lookaheadRescan = false + b.lookaheadRetryWindow = true + return + } + if b.lookaheadRetryWindow { + b.lookaheadRetryWindow = false b.lookaheadSearchOffset += gloasLookaheadWindow return } @@ -658,7 +734,7 @@ func selectGloasLookahead( } func (b *BackwardBeaconDownloader) waitBeforeLookaheadRetry(ctx context.Context) { - if b.consecutiveLookaheadFailures < 6 { + if b.consecutiveLookaheadFailures < maxLookaheadFailures { b.consecutiveLookaheadFailures++ } delay := time.Second << (b.consecutiveLookaheadFailures - 1) @@ -732,6 +808,14 @@ func gloasBlockAvailability(block, lookahead *cltypes.SignedBeaconBlock) (bool, return nextBid.Message.ParentBlockHash == bid.Message.BlockHash, true } +func (b *BackwardBeaconDownloader) gloasBlockAvailability(block, lookahead *cltypes.SignedBeaconBlock) (bool, bool) { + full, known := gloasBlockAvailability(block, lookahead) + if known && lookahead == b.prevBatchTopBlock && b.prevBatchTopBlockUntrusted { + return false, false + } + return full, known +} + // fetchGloasEnvelopes determines which GLOAS blocks in the batch are FULL and fetches their envelopes. // It returns the envelopes map and a set of block roots that were determined FULL by lookahead. // Callers must check: if a root is in fullRootSet but missing from envelopes, the fetch failed @@ -742,6 +826,19 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp } fullRoots, knownRootSet := determineGloasAvailability(responses, b.prevBatchTopBlock, b.expectedRoot) + var untrustedFullRoot common.Hash + untrustedFull := false + if b.prevBatchTopBlockUntrusted { + if anchor := blockByRoot(responses, b.expectedRoot); anchor != nil { + full, known := gloasBlockAvailability(anchor, b.prevBatchTopBlock) + if known && !full { + delete(knownRootSet, b.expectedRoot) + } else if known { + untrustedFullRoot = b.expectedRoot + untrustedFull = true + } + } + } // Build a set for O(1) lookup by callers. fullRootSet := make(map[common.Hash]struct{}, len(fullRoots)) @@ -750,6 +847,10 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp } if b.envelopesSkipped { + if untrustedFull { + delete(fullRootSet, untrustedFullRoot) + delete(knownRootSet, untrustedFullRoot) + } return nil, fullRootSet, knownRootSet } @@ -767,6 +868,10 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp } b.recordEnvelopeFetchResult(len(fullRoots), len(envelopes)) } + if untrustedFull && envelopes[untrustedFullRoot] == nil { + delete(fullRootSet, untrustedFullRoot) + delete(knownRootSet, untrustedFullRoot) + } inferredEmptyRoots := make(map[common.Hash]struct{}, len(knownRootSet)-len(fullRootSet)) for root := range knownRootSet { @@ -881,12 +986,12 @@ func validateFetchedEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks []*c } func (b *BackwardBeaconDownloader) recordEnvelopeFetchResult(requested, received int) { - if received < requested { + if requested > received { b.consecutiveEnvelopeFailures++ const maxConsecutiveFailures = 3 if b.consecutiveEnvelopeFailures >= maxConsecutiveFailures && !b.envelopesSkipped { b.envelopesSkipped = true - log.Warn("[BackwardBeaconDownloader] too many consecutive envelope failures, treating FULL blocks as EMPTY", + log.Warn("[BackwardBeaconDownloader] too many consecutive envelope failures, entering recovery mode", "consecutiveFailures", b.consecutiveEnvelopeFailures) } return @@ -895,11 +1000,28 @@ func (b *BackwardBeaconDownloader) recordEnvelopeFetchResult(requested, received b.envelopesSkipped = false } +func (b *BackwardBeaconDownloader) recordEmptyProbeResult(requested, resolved int) { + if requested > resolved { + if b.consecutiveProbeFailures < 3 { + b.consecutiveProbeFailures++ + } + if b.consecutiveProbeFailures >= 3 { + b.envelopesSkipped = true + } + return + } + b.consecutiveProbeFailures = 0 +} + // SkippedFullBlocks returns blocks that may still need envelopes after backward download. func (b *BackwardBeaconDownloader) SkippedFullBlocks() []SkippedFullBlock { return b.skippedFullBlocks } +func (b *BackwardBeaconDownloader) HasEnvelopeRecoverySource() bool { + return b.httpFallbackURL != "" || b.rpc != nil +} + func (b *BackwardBeaconDownloader) canTrackSkippedFullBlock(block *cltypes.SignedBeaconBlock) bool { return block != nil && len(b.skippedFullBlocks) < maxSkippedFullBlocks } @@ -944,12 +1066,10 @@ func ValidateFetchedEnvelope(beaconCfg *clparams.BeaconChainConfig, block *cltyp return nil } -// RecoverSkippedEnvelopes retries fetching envelopes for blocks that were -// skipped during backward download. Returns a map of successfully fetched -// envelopes keyed by beacon block root. -func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, skipped []SkippedFullBlock, blocks map[common.Hash]*cltypes.SignedBeaconBlock) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { +// RecoverSkippedEnvelopes retries fetching envelopes for blocks that were skipped during backward download. +func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, skipped []SkippedFullBlock, blocks map[common.Hash]*cltypes.SignedBeaconBlock) EnvelopeRecoveryResult { if len(skipped) == 0 { - return nil + return EnvelopeRecoveryResult{} } roots := make([][32]byte, len(skipped)) @@ -957,32 +1077,32 @@ func (b *BackwardBeaconDownloader) RecoverSkippedEnvelopes(ctx context.Context, roots[i] = s.Root } - sources := make([]func(context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, 0, 2) + sources := make([]func(context.Context) EnvelopeRecoveryResult, 0, 2) if b.httpFallbackURL != "" { - sources = append(sources, func(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { - envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(roots)) - b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped, envelopes) - return validateRecoveryEnvelopes(b.beaconCfg, blocks, envelopes) + sources = append(sources, func(ctx context.Context) EnvelopeRecoveryResult { + result := b.fetchSkippedEnvelopesFromBeaconAPI(ctx, skipped) + result.Envelopes = validateRecoveryEnvelopes(b.beaconCfg, blocks, result.Envelopes) + return result }) } if b.rpc != nil { - sources = append(sources, func(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { + sources = append(sources, func(ctx context.Context) EnvelopeRecoveryResult { envelopes, err := RequestEnvelopesFrantically(ctx, b.rpc, roots) if err != nil { log.Debug("[BackwardBeaconDownloader] envelope recovery: P2P failed", "err", err) } - return validateRecoveryEnvelopes(b.beaconCfg, blocks, envelopes) + return EnvelopeRecoveryResult{Envelopes: validateRecoveryEnvelopes(b.beaconCfg, blocks, envelopes)} }) } - fetched := fetchEnvelopeRecoverySources(ctx, sources...) - envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fetched)) - for root, envelope := range fetched { + fetched := fetchEnvelopeRecoveryResults(ctx, sources...) + envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(fetched.Envelopes)) + for root, envelope := range fetched.Envelopes { if ValidateFetchedEnvelope(b.beaconCfg, blocks[root], root, envelope) == nil { envelopes[root] = envelope } } - return envelopes + return EnvelopeRecoveryResult{Envelopes: envelopes} } func validateRecoveryEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks map[common.Hash]*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { @@ -1011,18 +1131,42 @@ func fetchEnvelopeRecoverySources(ctx context.Context, sources ...func(context.C return envelopes } -func (b *BackwardBeaconDownloader) fetchSkippedEnvelopesFromBeaconAPI(ctx context.Context, skipped []SkippedFullBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) { +func fetchEnvelopeRecoveryResults(ctx context.Context, sources ...func(context.Context) EnvelopeRecoveryResult) EnvelopeRecoveryResult { + results := make(chan EnvelopeRecoveryResult, len(sources)) + var wg sync.WaitGroup + for _, source := range sources { + wg.Go(func() { results <- source(ctx) }) + } + wg.Wait() + close(results) + + merged := EnvelopeRecoveryResult{ + Envelopes: make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope), + } + for result := range results { + maps.Copy(merged.Envelopes, result.Envelopes) + } + return merged +} + +func (b *BackwardBeaconDownloader) fetchSkippedEnvelopesFromBeaconAPI(ctx context.Context, skipped []SkippedFullBlock) EnvelopeRecoveryResult { + result := EnvelopeRecoveryResult{ + Envelopes: make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(skipped)), + } for _, item := range skipped { root := common.Hash(item.Root) - if _, ok := envelopes[root]; ok { + envelope, err := b.fetchSingleEnvelope(ctx, root) + if err != nil { continue } - envelope, err := b.fetchSingleEnvelope(ctx, root) - if err != nil || envelope == nil || envelope.Message == nil || envelope.Message.BeaconBlockRoot != root { + if envelope == nil { continue } - envelopes[root] = envelope + if envelope.Message != nil && envelope.Message.BeaconBlockRoot == root { + result.Envelopes[root] = envelope + } } + return result } // trySkipToExistingBlock attempts to skip ahead if the expected block already exists in the database. @@ -1070,6 +1214,16 @@ func (b *BackwardBeaconDownloader) trySkipToExistingBlock(ctx context.Context) e if !b.canSkipSlot(ctx, tx, elFrozenBlocks, clFrozenBlocks, *slot) { break } + var skippedBlock *cltypes.SignedBeaconBlock + if b.beaconCfg.GetCurrentStateVersion(*slot/b.beaconCfg.SlotsPerEpoch) >= clparams.GloasVersion { + skippedBlock, err = readStoredBeaconBlock(tx, *slot, b.expectedRoot, b.beaconCfg) + if err != nil { + return err + } + if skippedBlock == nil { + break + } + } b.slotToDownload.Store(*slot - 1) if err := beacon_indicies.MarkRootCanonical(b.ctx, tx, *slot, b.expectedRoot); err != nil { @@ -1080,6 +1234,10 @@ func (b *BackwardBeaconDownloader) trySkipToExistingBlock(ctx context.Context) e if err != nil { return err } + if skippedBlock != nil { + b.prevBatchTopBlock = skippedBlock + b.prevBatchTopBlockUntrusted = false + } // Clean up non-canonical slots newSlot, err := beacon_indicies.ReadBlockSlotByBlockRoot(tx, b.expectedRoot) @@ -1097,13 +1255,28 @@ func (b *BackwardBeaconDownloader) trySkipToExistingBlock(ctx context.Context) e return tx.Commit() } +func readStoredBeaconBlock(tx kv.Tx, slot uint64, root common.Hash, beaconCfg *clparams.BeaconChainConfig) (*cltypes.SignedBeaconBlock, error) { + encoded, err := tx.GetOne(kv.BeaconBlocks, dbutils.BlockBodyKey(slot, root)) + if err != nil || len(encoded) == 0 { + return nil, err + } + decoder, err := zstd.NewReader(bytes.NewReader(encoded)) + if err != nil { + return nil, err + } + defer decoder.Close() + return snapshot_format.ReadBlockFromSnapshot(decoder, nil, beaconCfg) +} + // canSkipSlot checks if we can skip to an existing block at the given slot. func (b *BackwardBeaconDownloader) canSkipSlot(ctx context.Context, tx kv.Tx, elFrozenBlocks, clFrozenBlocks, slot uint64) bool { if slot <= clFrozenBlocks { return false } - if b.engine == nil || !b.engine.SupportInsertion() { + isGloas := b.beaconCfg.GetCurrentStateVersion(slot/b.beaconCfg.SlotsPerEpoch) >= clparams.GloasVersion + supportsInsertion := b.engine != nil && b.engine.SupportInsertion() + if !supportsInsertion && !isGloas { return true } @@ -1113,10 +1286,10 @@ func (b *BackwardBeaconDownloader) canSkipSlot(ctx context.Context, tx kv.Tx, el return false } if blockHash == (common.Hash{}) { - // [New in Gloas:EIP7732] GLOAS EMPTY blocks have no execution hash (no payload delivered). - // If this slot is in the GLOAS era, no EL processing is needed, so we can skip. - epoch := slot / b.beaconCfg.SlotsPerEpoch - return b.beaconCfg.GetCurrentStateVersion(epoch) >= clparams.GloasVersion + return false + } + if !supportsInsertion { + return true } blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, b.expectedRoot) @@ -1177,8 +1350,7 @@ func fetchBlockFromBeaconAPIByRoot(ctx context.Context, baseURL string, root com } // fetchSingleEnvelope fetches the execution payload envelope for a single GLOAS block. -// Returns (envelope, nil) on success, (nil, nil) when the beacon API confirms the root -// has no envelope (HTTP 404 = genuinely EMPTY), or (nil, err) on fetch failure. +// Returns (envelope, nil) on success, (nil, nil) on HTTP 404, or (nil, err) on fetch failure. func (b *BackwardBeaconDownloader) fetchSingleEnvelope(ctx context.Context, blockRoot common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { if b.httpFallbackURL == "" { return nil, fmt.Errorf("no HTTP fallback URL configured") diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index 0b4807879c6..eaeca09d757 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -27,11 +27,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" + "github.com/erigontech/erigon/cl/persistence/beacon_indicies" + "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/memdb" ) // makeGloasBlock creates a GLOAS SignedBeaconBlock with the given bid hashes. @@ -403,6 +408,155 @@ func TestP2POnlyDownloadTracksLookaheadInferredEmptyBlock(t *testing.T) { assert.Empty(t, downloader.skippedFullBlocks) } +func TestUntrustedP2PLookaheadCannotProveEmpty(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + prevBatchTopBlockUntrusted: true, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) + require.False(t, processed) + require.False(t, downloader.Finished()) +} + +func TestUntrustedP2PLookaheadCannotProveFullWithoutEnvelope(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = blockRoot + + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + prevBatchTopBlockUntrusted: true, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + downloader.httpPreferred.Store(true) + + envelopes, fullRoots, knownRoots := downloader.fetchGloasEnvelopes(context.Background(), []*cltypes.SignedBeaconBlock{block}) + require.Empty(t, envelopes) + require.NotContains(t, fullRoots, common.Hash(blockRoot)) + require.NotContains(t, knownRoots, common.Hash(blockRoot)) +} + +func TestUntrustedP2PLookaheadCanProveFullWithMatchingEnvelope(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + lookahead := makeGloasBlock(101, hash(0xBB), block.Block.Body.GetSignedExecutionPayloadBid().Message.BlockHash) + lookahead.Block.ParentRoot = blockRoot + encoded, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(encoded) + })) + defer server.Close() + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + prevBatchTopBlockUntrusted: true, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + } + downloader.httpPreferred.Store(true) + + envelopes, fullRoots, knownRoots := downloader.fetchGloasEnvelopes(context.Background(), []*cltypes.SignedBeaconBlock{block}) + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), envelopes[common.Hash(blockRoot)])) + require.Contains(t, fullRoots, common.Hash(blockRoot)) + require.Contains(t, knownRoots, common.Hash(blockRoot)) +} + +func TestUntrustedP2PEmptyLookaheadEntersBoundedRecovery(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + prevBatchTopBlockUntrusted: true, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return true, nil + }, + } + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) + } + require.True(t, processed) + require.True(t, downloader.envelopesSkipped) + require.Len(t, downloader.skippedFullBlocks, 1) +} + +func TestDirectFirstBatchP2PLookaheadRemainsUntrusted(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + blockRangeSource: blockRangeSourceP2P, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block, lookahead})) + require.False(t, processed) + require.True(t, downloader.prevBatchTopBlockUntrusted) +} + +func TestUnavailableLookaheadEntersBoundedUnresolvedRecovery(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + beaconCfg: &clparams.MainnetBeaconConfig, + consecutiveLookaheadFailures: 5, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return true, nil + }, + } + + require.NoError(t, downloader.processResponses(ctx, []*cltypes.SignedBeaconBlock{block})) + require.True(t, processed) + require.True(t, downloader.envelopesSkipped) + require.Len(t, downloader.skippedFullBlocks, 1) +} + func TestProbeGloasEmptyCandidatesDeduplicatesRoots(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, err := block.Block.HashSSZ() @@ -437,6 +591,37 @@ func TestProbeGloasEmptyCandidatesDeduplicatesRoots(t *testing.T) { assert.Equal(t, int32(1), requests.Load()) } +func TestEmptyProbeServerFailuresEnterBoundedUnresolvedRecovery(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return true, nil + }, + } + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) + } + require.True(t, processed) + require.True(t, downloader.envelopesSkipped) + require.Len(t, downloader.skippedFullBlocks, 1) +} + func TestFetchEnvelopesFromBeaconAPIUsesBlockRoot(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, envelope := makeValidGloasEnvelope(t, block) @@ -494,7 +679,121 @@ func TestFetchGloasEnvelopesSkipsNetworkAfterFailureThreshold(t *testing.T) { assert.Zero(t, requests.Load()) } -func TestDegradedDownloadTracksLookaheadInferredEmptyBlock(t *testing.T) { +func TestPartialEnvelopeSuccessDoesNotResetDegradedModeFailures(t *testing.T) { + downloader := &BackwardBeaconDownloader{consecutiveEnvelopeFailures: 2} + + downloader.recordEnvelopeFetchResult(2, 1) + + require.Equal(t, 3, downloader.consecutiveEnvelopeFailures) + require.True(t, downloader.envelopesSkipped) +} + +func TestCompleteEnvelopeSuccessResetsDegradedModeFailures(t *testing.T) { + downloader := &BackwardBeaconDownloader{consecutiveEnvelopeFailures: 2, envelopesSkipped: true} + + downloader.recordEnvelopeFetchResult(2, 2) + + require.Zero(t, downloader.consecutiveEnvelopeFailures) + require.False(t, downloader.envelopesSkipped) +} + +func TestPartialEmptyProbeSuccessDoesNotResetExpectedRootFailure(t *testing.T) { + downloader := &BackwardBeaconDownloader{consecutiveProbeFailures: 2} + + downloader.recordEmptyProbeResult(2, 1) + + require.Equal(t, uint8(3), downloader.consecutiveProbeFailures) + require.True(t, downloader.envelopesSkipped) +} + +func TestResolvedExpectedRootResetsProbeFailures(t *testing.T) { + downloader := &BackwardBeaconDownloader{consecutiveProbeFailures: 2} + + downloader.recordEmptyProbeResult(1, 1) + + require.Zero(t, downloader.consecutiveProbeFailures) +} + +func TestPartialEnvelopeSuccessStillReachesRecoveryForMissingExpectedRoot(t *testing.T) { + older := makeGloasBlock(100, hash(0xAA), hash(0x10)) + olderRoot, olderEnvelope := makeValidGloasEnvelope(t, older) + newer := makeGloasBlock(101, hash(0xBB), older.Block.Body.GetSignedExecutionPayloadBid().Message.BlockHash) + newer.Block.ParentRoot = olderRoot + newerRoot, err := newer.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(102, hash(0xCC), newer.Block.Body.GetSignedExecutionPayloadBid().Message.BlockHash) + lookahead.Block.ParentRoot = newerRoot + encodedOlder, err := olderEnvelope.EncodeSSZ(nil) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(olderRoot).Hex() { + _, _ = w.Write(encodedOlder) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + var processed atomic.Int32 + downloader := &BackwardBeaconDownloader{ + expectedRoot: newerRoot, + prevBatchTopBlock: lookahead, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Add(1) + return false, nil + }, + } + downloader.httpPreferred.Store(true) + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{older, newer})) + } + + require.True(t, downloader.envelopesSkipped) + require.Equal(t, int32(2), processed.Load()) +} + +func TestResolvedEmptyProbeDoesNotResetUnknownExpectedRoot(t *testing.T) { + older := makeGloasBlock(100, hash(0xAA), hash(0x10)) + olderRoot, err := older.Block.HashSSZ() + require.NoError(t, err) + newer := makeGloasBlock(101, hash(0xBB), hash(0x20)) + newer.Block.ParentRoot = olderRoot + newerRoot, err := newer.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(102, hash(0xCC), hash(0x30)) + lookahead.Block.ParentRoot = newerRoot + + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + + var processed atomic.Int32 + downloader := &BackwardBeaconDownloader{ + expectedRoot: newerRoot, + prevBatchTopBlock: lookahead, + prevBatchTopBlockUntrusted: true, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + consecutiveEnvelopeFailures: 0, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Add(1) + return false, nil + }, + } + downloader.httpPreferred.Store(true) + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{older, newer})) + } + + require.True(t, downloader.envelopesSkipped) + require.Equal(t, int32(2), processed.Load()) +} + +func TestDegradedDownloadDoesNotTrackCanonicallyProvenEmptyBlock(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, err := block.Block.HashSSZ() require.NoError(t, err) @@ -513,8 +812,7 @@ func TestDegradedDownloadTracksLookaheadInferredEmptyBlock(t *testing.T) { } require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) - require.Len(t, downloader.skippedFullBlocks, 1) - assert.Equal(t, common.Hash(blockRoot), common.Hash(downloader.skippedFullBlocks[0].Root)) + require.Empty(t, downloader.skippedFullBlocks) } func TestSelectGloasLookaheadRejectsUnlinkedAndIncompleteBlocks(t *testing.T) { @@ -636,6 +934,21 @@ func TestFetchGloasLookaheadRescansEarlierWindow(t *testing.T) { require.Equal(t, uint64(101), got.Block.Slot) } +func TestLookaheadSearchRetriesTransientLaterWindowBeforeAdvancing(t *testing.T) { + downloader := &BackwardBeaconDownloader{} + + downloader.advanceLookaheadSearch() + require.Equal(t, gloasLookaheadWindow, downloader.lookaheadSearchOffset) + downloader.advanceLookaheadSearch() + require.True(t, downloader.lookaheadRescan) + downloader.advanceLookaheadSearch() + + require.False(t, downloader.lookaheadRescan) + require.Equal(t, gloasLookaheadWindow, downloader.lookaheadSearchOffset) + downloader.advanceLookaheadSearch() + require.Equal(t, 2*gloasLookaheadWindow, downloader.lookaheadSearchOffset) +} + func TestFetchGloasLookaheadFromSourcesFallsBackAfterMissOrError(t *testing.T) { anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) anchorRoot, err := anchor.Block.HashSSZ() @@ -749,6 +1062,120 @@ func TestRootFallbackProbesLookaheadInferredEmptyBlock(t *testing.T) { require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), processedEnvelope)) } +func TestRootFallbackLookaheadFailureEntersBoundedRecovery(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + encodedBlock, err := block.EncodeSSZ(nil) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v2/beacon/blocks/"+common.Hash(blockRoot).Hex() { + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedBlock) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + consecutiveLookaheadFailures: maxLookaheadFailures - 1, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return false, nil + }, + } + + require.NoError(t, downloader.processResponses(context.Background(), nil)) + require.True(t, processed) + require.True(t, downloader.envelopesSkipped) + require.Len(t, downloader.skippedFullBlocks, 1) +} + +func TestRootFallbackUntrustedP2PEmptyLookaheadEntersBoundedRecovery(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + encodedBlock, err := block.EncodeSSZ(nil) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v2/beacon/blocks/"+common.Hash(blockRoot).Hex() { + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedBlock) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + prevBatchTopBlockUntrusted: true, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return false, nil + }, + } + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), nil)) + } + require.True(t, processed) + require.True(t, downloader.envelopesSkipped) + require.Len(t, downloader.skippedFullBlocks, 1) +} + +func TestRootFallbackFailingProbeDoesNotTrackCanonicallyProvenEmptyBlock(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) + lookahead.Block.ParentRoot = blockRoot + encodedBlock, err := block.EncodeSSZ(nil) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/eth/v2/beacon/blocks/" + common.Hash(blockRoot).Hex(): + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedBlock) + case "/eth/v1/beacon/execution_payload_envelope/" + common.Hash(blockRoot).Hex(): + w.WriteHeader(http.StatusTooManyRequests) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + processed := false + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed = true + return false, nil + }, + } + + for range 3 { + require.NoError(t, downloader.processResponses(context.Background(), nil)) + } + require.True(t, processed) + require.True(t, downloader.envelopesSkipped) + require.Empty(t, downloader.skippedFullBlocks) +} + func TestValidateFetchedEnvelopeRejectsDifferentBeaconRoot(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, envelope := makeValidGloasEnvelope(t, block) @@ -816,7 +1243,48 @@ func TestMalformedHTTPRecoveryEnvelopeRemainsMissing(t *testing.T) { []SkippedFullBlock{{Slot: block.Block.Slot, Root: blockRoot}}, map[common.Hash]*cltypes.SignedBeaconBlock{common.Hash(blockRoot): block}, ) - require.Empty(t, got) + require.Empty(t, got.Envelopes) +} + +func TestHTTPRecoveryKeepsNotFoundUnresolved(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + + downloader := &BackwardBeaconDownloader{httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig} + got := downloader.RecoverSkippedEnvelopes(context.Background(), []SkippedFullBlock{{Slot: block.Block.Slot, Root: blockRoot}}, map[common.Hash]*cltypes.SignedBeaconBlock{common.Hash(blockRoot): block}) + + require.Empty(t, got.Envelopes) +} + +func TestHTTPRecoveryKeepsServerFailureUnresolved(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + downloader := &BackwardBeaconDownloader{httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig} + got := downloader.RecoverSkippedEnvelopes(context.Background(), []SkippedFullBlock{{Slot: block.Block.Slot, Root: blockRoot}}, map[common.Hash]*cltypes.SignedBeaconBlock{common.Hash(blockRoot): block}) + + require.Empty(t, got.Envelopes) +} + +func TestHTTPRecoveryDoesNotConfirmEmptyWithoutCanonicalBlock(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + + downloader := &BackwardBeaconDownloader{httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig} + got := downloader.RecoverSkippedEnvelopes(context.Background(), []SkippedFullBlock{{Slot: block.Block.Slot, Root: blockRoot}}, nil) + + require.Empty(t, got.Envelopes) } func TestHTTPRecoveryUsesBlockRoot(t *testing.T) { @@ -842,7 +1310,7 @@ func TestHTTPRecoveryUsesBlockRoot(t *testing.T) { map[common.Hash]*cltypes.SignedBeaconBlock{common.Hash(blockRoot): block}, ) - require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), got[common.Hash(blockRoot)])) + require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), got.Envelopes[common.Hash(blockRoot)])) assert.Equal(t, "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex(), requestedPath) } @@ -884,6 +1352,117 @@ func TestSkippedFullBlockMemoryBudget(t *testing.T) { require.False(t, downloader.canTrackSkippedFullBlock(block)) } +func TestBackwardBeaconDownloaderCapacityStopsInsteadOfWaitingForFinish(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + linkGloasBlocks(t, block, lookahead) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + beaconCfg: &clparams.MainnetBeaconConfig, + envelopesSkipped: true, + skippedFullBlocks: make([]SkippedFullBlock, maxSkippedFullBlocks), + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + t.Fatal("block processing must stop before unresolved recovery state is lost") + return false, nil + }, + } + + err = downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block}) + require.ErrorContains(t, err, "skipped envelope recovery capacity") + require.False(t, downloader.Finished()) +} + +func TestBackwardBeaconDownloaderRestartDoesNotSkipUnresolvedGloasBlock(t *testing.T) { + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + engine.EXPECT().SupportInsertion().Return(true) + beaconCfg := clparams.MainnetBeaconConfig + beaconCfg.AltairForkEpoch = 0 + beaconCfg.BellatrixForkEpoch = 0 + beaconCfg.CapellaForkEpoch = 0 + beaconCfg.DenebForkEpoch = 0 + beaconCfg.ElectraForkEpoch = 0 + beaconCfg.FuluForkEpoch = 0 + beaconCfg.GloasForkEpoch = 0 + db := memdb.NewTestDB(t, dbcfg.ChainDB) + tx, err := db.BeginRo(context.Background()) + require.NoError(t, err) + defer tx.Rollback() + + downloader := &BackwardBeaconDownloader{ + engine: engine, + beaconCfg: &beaconCfg, + expectedRoot: hash(0x42), + } + + require.False(t, downloader.canSkipSlot(context.Background(), tx, ^uint64(0), 0, 1)) +} + +func TestBackwardBeaconDownloaderRestartWithoutELDoesNotSkipUnresolvedGloasBlock(t *testing.T) { + beaconCfg := clparams.MainnetBeaconConfig + beaconCfg.AltairForkEpoch = 0 + beaconCfg.BellatrixForkEpoch = 0 + beaconCfg.CapellaForkEpoch = 0 + beaconCfg.DenebForkEpoch = 0 + beaconCfg.ElectraForkEpoch = 0 + beaconCfg.FuluForkEpoch = 0 + beaconCfg.GloasForkEpoch = 0 + db := memdb.NewTestDB(t, dbcfg.ChainDB) + tx, err := db.BeginRo(context.Background()) + require.NoError(t, err) + defer tx.Rollback() + + downloader := &BackwardBeaconDownloader{beaconCfg: &beaconCfg, expectedRoot: hash(0x42)} + + require.False(t, downloader.canSkipSlot(context.Background(), tx, ^uint64(0), 0, 1)) +} + +func TestTrySkipToExistingBlockRefreshesLookahead(t *testing.T) { + beaconCfg := clparams.MainnetBeaconConfig + beaconCfg.AltairForkEpoch = 0 + beaconCfg.BellatrixForkEpoch = 0 + beaconCfg.CapellaForkEpoch = 0 + beaconCfg.DenebForkEpoch = 0 + beaconCfg.ElectraForkEpoch = 0 + beaconCfg.FuluForkEpoch = 0 + beaconCfg.GloasForkEpoch = 0 + db := memdb.NewTestDB(t, dbcfg.ChainDB) + parent := makeGloasBlock(99, hash(0x10), hash(0x01)) + child := makeGloasBlock(100, hash(0x20), hash(0x10)) + linkGloasBlocks(t, parent, child) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + childRoot, err := child.Block.HashSSZ() + require.NoError(t, err) + + tx, err := db.BeginRw(context.Background()) + require.NoError(t, err) + defer tx.Rollback() + require.NoError(t, beacon_indicies.WriteBeaconBlockAndIndicies(context.Background(), tx, parent, false)) + require.NoError(t, beacon_indicies.WriteBeaconBlockAndIndicies(context.Background(), tx, child, false)) + require.NoError(t, beacon_indicies.WriteExecutionBlockHash(tx, childRoot, hash(0x20))) + require.NoError(t, tx.Commit()) + + downloader := &BackwardBeaconDownloader{ + ctx: context.Background(), + db: db, + beaconCfg: &beaconCfg, + expectedRoot: childRoot, + slotToDownload: atomic.Uint64{}, + } + require.NoError(t, downloader.trySkipToExistingBlock(context.Background())) + require.Equal(t, common.Hash(parentRoot), downloader.expectedRoot) + require.NotNil(t, downloader.prevBatchTopBlock) + gotRoot, err := downloader.prevBatchTopBlock.Block.HashSSZ() + require.NoError(t, err) + require.Equal(t, common.Hash(childRoot), common.Hash(gotRoot)) + require.False(t, downloader.prevBatchTopBlockUntrusted) +} + func TestReadBoundedBeaconAPIResponseRejectsOversize(t *testing.T) { _, err := readBoundedBeaconAPIResponse(bytes.NewReader(make([]byte, 9)), 8) require.Error(t, err) diff --git a/cl/phase1/network/services/execution_payload_service.go b/cl/phase1/network/services/execution_payload_service.go index 0b5c1f8e27a..5d14c72b09a 100644 --- a/cl/phase1/network/services/execution_payload_service.go +++ b/cl/phase1/network/services/execution_payload_service.go @@ -56,12 +56,16 @@ type pendingEnvelopeKey struct { type envelopeJob struct { envelope *cltypes.SignedExecutionPayloadEnvelope creationTime time.Time + nextAttempt time.Time + blockSeen atomic.Bool + resolving atomic.Bool } const ( seenEnvelopeCacheSize = 1000 - pendingEnvelopeExpiry = 30 * time.Second + pendingEnvelopeExpiry = 3 * time.Minute pendingEnvelopeCheckInterval = 100 * time.Millisecond + pendingEnvelopeRetryInterval = time.Second maxPendingEnvelopes = 1024 ) @@ -77,6 +81,7 @@ type executionPayloadService struct { pendingEnvelopes sync.Map // pendingEnvelopeKey -> *envelopeJob pendingCount atomic.Int32 pendingCond *sync.Cond + pendingMu sync.Mutex } // NewExecutionPayloadService creates a new execution payload service @@ -140,7 +145,7 @@ func (s *executionPayloadService) ProcessMessage(ctx context.Context, _ *uint64, block, ok := s.forkchoiceStore.GetBlock(beaconBlockRoot) if !ok || block == nil { // Block hasn't arrived yet, queue envelope for later processing - s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope) + s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope, false) // Also store in forkchoice's pendingEnvelopes so OnBlock can process it immediately // when the block arrives, instead of waiting for the 100ms polling loop. // validatePayload must be true: if the block arrives (via OnBlock) before this call @@ -174,8 +179,8 @@ func (s *executionPayloadService) ProcessMessage(ctx context.Context, _ *uint64, // Process the execution payload through forkchoice // Note: bid matching and signature verification are done in OnExecutionPayload.validateEnvelopeAgainstBlock if err := s.forkchoiceStore.OnExecutionPayload(ctx, signedEnvelope, true, true); err != nil { - if errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) { - s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope) + if isRetryableExecutionPayloadError(err) { + s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope, true) return fmt.Errorf("%w: %w", ErrIgnore, err) } if errors.Is(err, forkchoice.ErrIgnore) { @@ -202,17 +207,18 @@ func (s *executionPayloadService) ProcessMessage(ctx context.Context, _ *uint64, return nil } -// queuePendingEnvelope adds an envelope to the pending queue for later processing -func (s *executionPayloadService) queuePendingEnvelope(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) { - if s.pendingCount.Add(1) > maxPendingEnvelopes { - s.pendingCount.Add(-1) - return - } +func isRetryableExecutionPayloadError(err error) bool { + return errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) || + errors.Is(err, forkchoice.ErrELPayloadValidationUnavailable) || + errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) +} +// queuePendingEnvelope adds an envelope to the pending queue for later processing +func (s *executionPayloadService) queuePendingEnvelope(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, blockSeen bool) { // Compute envelope hash to allow multiple candidates per block envelopeHash, err := envelope.HashSSZ() if err != nil { - s.pendingCount.Add(-1) log.Warn("Failed to hash envelope for pending queue", "blockRoot", blockRoot, "err", err) return } @@ -221,19 +227,60 @@ func (s *executionPayloadService) queuePendingEnvelope(blockRoot common.Hash, en blockRoot: blockRoot, envelopeHash: envelopeHash, } + s.pendingMu.Lock() + defer s.pendingMu.Unlock() + if existing, loaded := s.pendingEnvelopes.Load(key); loaded { + if blockSeen { + existing.(*envelopeJob).blockSeen.Store(true) + } + return + } + for s.pendingCount.Load() >= maxPendingEnvelopes { + oldestKey, found := s.oldestPendingEnvelope(false) + if !found && blockSeen { + oldestKey, found = s.oldestPendingEnvelope(true) + } + if !found { + return + } + if _, loaded := s.pendingEnvelopes.LoadAndDelete(oldestKey); loaded { + s.pendingCount.Add(-1) + } + } - if _, loaded := s.pendingEnvelopes.LoadOrStore(key, &envelopeJob{ + job := &envelopeJob{ envelope: envelope, creationTime: time.Now(), - }); loaded { - s.pendingCount.Add(-1) + } + job.blockSeen.Store(blockSeen) + if _, loaded := s.pendingEnvelopes.LoadOrStore(key, job); loaded { } else { + s.pendingCount.Add(1) s.pendingCond.L.Lock() s.pendingCond.Signal() s.pendingCond.L.Unlock() } } +func (s *executionPayloadService) oldestPendingEnvelope(blockSeen bool) (pendingEnvelopeKey, bool) { + var oldestKey pendingEnvelopeKey + var oldestTime time.Time + found := false + s.pendingEnvelopes.Range(func(candidateKey, value any) bool { + candidate := value.(*envelopeJob) + if candidate.resolving.Load() || candidate.blockSeen.Load() != blockSeen { + return true + } + if !found || candidate.creationTime.Before(oldestTime) { + oldestKey = candidateKey.(pendingEnvelopeKey) + oldestTime = candidate.creationTime + found = true + } + return true + }) + return oldestKey, found +} + // loop is the background goroutine that processes pending envelopes func (s *executionPayloadService) loop(ctx context.Context) { // Wake any blocked Wait() on context cancellation to prevent deadlock. @@ -280,31 +327,71 @@ func (s *executionPayloadService) processPendingEnvelopes(ctx context.Context) { pendingKey := key.(pendingEnvelopeKey) job := value.(*envelopeJob) - // Check expiry - if time.Since(job.creationTime) > pendingEnvelopeExpiry { - if _, loaded := s.pendingEnvelopes.LoadAndDelete(pendingKey); loaded { - s.pendingCount.Add(-1) - log.Trace("Pending envelope expired", "blockRoot", pendingKey.blockRoot) - } + s.pendingMu.Lock() + current, stillPending := s.pendingEnvelopes.Load(pendingKey) + if !stillPending || current != job || !job.resolving.CompareAndSwap(false, true) { + s.pendingMu.Unlock() return true } - - // Check if block has arrived - block, ok := s.forkchoiceStore.GetBlock(pendingKey.blockRoot) - if !ok || block == nil { - return true // Block still not here, keep waiting + blockSeen := job.blockSeen.Load() + s.pendingMu.Unlock() + + if !blockSeen { + block, ok := s.forkchoiceStore.GetBlock(pendingKey.blockRoot) + if !ok || block == nil { + s.pendingMu.Lock() + current, stillPending = s.pendingEnvelopes.Load(pendingKey) + expired := stillPending && current == job && time.Since(job.creationTime) > pendingEnvelopeExpiry + if stillPending && current == job { + if expired { + s.pendingEnvelopes.Delete(pendingKey) + s.pendingCount.Add(-1) + } else { + job.resolving.Store(false) + } + } else { + job.resolving.Store(false) + } + s.pendingMu.Unlock() + if expired { + log.Trace("Pending envelope expired", "blockRoot", pendingKey.blockRoot) + } + return true + } + job.blockSeen.Store(true) } - - err := s.ProcessMessage(ctx, nil, job.envelope) - if errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) { + if time.Now().Before(job.nextAttempt) { + s.pendingMu.Lock() + current, stillPending = s.pendingEnvelopes.Load(pendingKey) + if stillPending && current == job { + job.resolving.Store(false) + } + s.pendingMu.Unlock() return true } - if _, loaded := s.pendingEnvelopes.LoadAndDelete(pendingKey); loaded { - s.pendingCount.Add(-1) - } + + err := s.ProcessMessage(ctx, nil, job.envelope) + s.finishPendingEnvelopeAttempt(pendingKey, job, err) if err != nil { log.Trace("Failed to process pending envelope", "blockRoot", pendingKey.blockRoot, "err", err) } return true }) } + +func (s *executionPayloadService) finishPendingEnvelopeAttempt(pendingKey pendingEnvelopeKey, job *envelopeJob, err error) { + s.pendingMu.Lock() + defer s.pendingMu.Unlock() + current, stillPending := s.pendingEnvelopes.Load(pendingKey) + if !stillPending || current != job { + job.resolving.Store(false) + return + } + if isRetryableExecutionPayloadError(err) { + job.nextAttempt = time.Now().Add(pendingEnvelopeRetryInterval) + job.resolving.Store(false) + return + } + s.pendingEnvelopes.Delete(pendingKey) + s.pendingCount.Add(-1) +} diff --git a/cl/phase1/network/services/execution_payload_service_test.go b/cl/phase1/network/services/execution_payload_service_test.go index 240101d4e10..815309cb67a 100644 --- a/cl/phase1/network/services/execution_payload_service_test.go +++ b/cl/phase1/network/services/execution_payload_service_test.go @@ -19,6 +19,7 @@ package services import ( "context" "errors" + "fmt" "sync" "testing" "time" @@ -239,6 +240,36 @@ func TestExecutionPayloadServicePendingEnvelopeExpiry(t *testing.T) { require.False(t, exists) } +func TestExecutionPayloadServiceRetainsEnvelopeAcrossColumnSyncInterval(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + } + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl.seenEnvelopesCache = seenCache + + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + key := pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash} + impl.pendingEnvelopes.Store(key, &envelopeJob{ + envelope: envelope, + creationTime: time.Now().Add(-time.Minute), + }) + impl.pendingCount.Store(1) + + impl.processPendingEnvelopes(t.Context()) + + require.Equal(t, int32(1), impl.pendingCount.Load()) + _, exists := impl.pendingEnvelopes.Load(key) + require.True(t, exists) +} + func TestExecutionPayloadServicePendingEnvelopeProcessing(t *testing.T) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) @@ -313,6 +344,62 @@ func TestExecutionPayloadServiceQueuesEnvelopeUntilDataAvailable(t *testing.T) { require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) } +func TestExecutionPayloadServiceQueuesInitialTemporaryELFailure(t *testing.T) { + service, fcu := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + fcu.OnExecutionPayloadErr = fmt.Errorf("%w: timeout", forkchoice.ErrELPayloadValidationUnavailable) + + err := service.ProcessMessage(t.Context(), nil, envelope) + require.ErrorIs(t, err, forkchoice.ErrELPayloadValidationUnavailable) + require.Equal(t, int32(1), impl.pendingCount.Load()) + + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash}) + require.True(t, exists) + + fcu.OnExecutionPayloadErr = nil + impl.processPendingEnvelopes(t.Context()) + require.Zero(t, impl.pendingCount.Load()) + require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) +} + +func TestExecutionPayloadServiceDoesNotExpireRetryableKnownBlock(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + } + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl.seenEnvelopesCache = seenCache + + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + key := pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash} + impl.pendingEnvelopes.Store(key, &envelopeJob{ + envelope: envelope, + creationTime: time.Now().Add(-pendingEnvelopeExpiry - time.Second), + }) + impl.pendingCount.Store(1) + forkchoiceMock.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + forkchoiceMock.OnExecutionPayloadErr = fmt.Errorf("%w: timeout", forkchoice.ErrELPayloadValidationUnavailable) + + impl.processPendingEnvelopes(t.Context()) + + require.Equal(t, int32(1), impl.pendingCount.Load()) + value, exists := impl.pendingEnvelopes.Load(key) + require.True(t, exists) + require.True(t, value.(*envelopeJob).nextAttempt.After(time.Now())) +} + func TestExecutionPayloadServiceRetainsPendingEnvelopeUntilDataAvailable(t *testing.T) { service, fcu := setupExecutionPayloadService(t) impl := service.(*executionPayloadService) @@ -337,8 +424,10 @@ func TestExecutionPayloadServiceRetainsPendingEnvelopeUntilDataAvailable(t *test value, ok = impl.pendingEnvelopes.Load(key) require.True(t, ok) require.Equal(t, creationTime, value.(*envelopeJob).creationTime) + require.True(t, value.(*envelopeJob).nextAttempt.After(time.Now())) fcu.OnExecutionPayloadErr = nil + value.(*envelopeJob).nextAttempt = time.Time{} impl.processPendingEnvelopes(t.Context()) require.Equal(t, int32(0), impl.pendingCount.Load()) @@ -363,6 +452,25 @@ func TestExecutionPayloadServiceDropsPendingEnvelopeAfterValidationFailure(t *te require.False(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) } +func TestExecutionPayloadServiceRetainsPendingEnvelopeAfterTemporaryELFailure(t *testing.T) { + service, fcu := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + blockRoot := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, blockRoot, 1) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + fcu.OnExecutionPayloadErr = fmt.Errorf("%w: timeout", forkchoice.ErrELPayloadValidationUnavailable) + + impl.processPendingEnvelopes(t.Context()) + + require.Equal(t, int32(1), impl.pendingCount.Load()) + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash}) + require.True(t, exists) +} + func TestExecutionPayloadServiceMultiplePendingForSameBlock(t *testing.T) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) @@ -432,7 +540,7 @@ func TestExecutionPayloadServicePendingQueueCap(t *testing.T) { blockRoot := common.HexToHash("0xffff") envelope := newTestSignedEnvelope(100, blockRoot, 999) - impl.queuePendingEnvelope(blockRoot, envelope) + impl.queuePendingEnvelope(blockRoot, envelope, false) require.Equal(t, int32(maxPendingEnvelopes), impl.pendingCount.Load()) envelopeHash, err := envelope.HashSSZ() @@ -441,6 +549,209 @@ func TestExecutionPayloadServicePendingQueueCap(t *testing.T) { require.False(t, exists) } +func TestExecutionPayloadServicePendingQueueRejectsUnknownWorkWhenAllJobsAreKnown(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + seenEnvelopesCache: seenCache, + pendingCond: sync.NewCond(&sync.Mutex{}), + } + + var oldestKey pendingEnvelopeKey + for i := range maxPendingEnvelopes { + blockRoot := common.Hash{byte(i), byte(i >> 8)} + envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) + envelopeHash, hashErr := envelope.HashSSZ() + require.NoError(t, hashErr) + key := pendingEnvelopeKey{blockRoot, envelopeHash} + if i == 0 { + oldestKey = key + } + job := &envelopeJob{ + envelope: envelope, + creationTime: time.Now().Add(-time.Hour + time.Duration(i)), + } + job.blockSeen.Store(true) + impl.pendingEnvelopes.Store(key, job) + } + impl.pendingCount.Store(maxPendingEnvelopes) + + blockRoot := common.HexToHash("0xffff") + envelope := newTestSignedEnvelope(100, blockRoot, 9999) + impl.queuePendingEnvelope(blockRoot, envelope, false) + + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + require.False(t, exists) + _, exists = impl.pendingEnvelopes.Load(oldestKey) + require.True(t, exists) + require.Equal(t, int32(maxPendingEnvelopes), impl.pendingCount.Load()) +} + +func TestExecutionPayloadServicePendingQueueEvictsUnknownBeforeOlderKnownWork(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + seenEnvelopesCache: seenCache, + pendingCond: sync.NewCond(&sync.Mutex{}), + } + + var oldestKnownKey, unknownKey pendingEnvelopeKey + for i := range maxPendingEnvelopes { + blockRoot := common.Hash{byte(i), byte(i >> 8)} + envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) + envelopeHash, hashErr := envelope.HashSSZ() + require.NoError(t, hashErr) + key := pendingEnvelopeKey{blockRoot, envelopeHash} + job := &envelopeJob{ + envelope: envelope, + creationTime: time.Now(), + } + job.blockSeen.Store(true) + switch i { + case 0: + oldestKnownKey = key + job.creationTime = time.Now().Add(-2 * time.Hour) + case 1: + unknownKey = key + job.creationTime = time.Now().Add(-time.Hour) + job.blockSeen.Store(false) + } + impl.pendingEnvelopes.Store(key, job) + } + impl.pendingCount.Store(maxPendingEnvelopes) + + blockRoot := common.HexToHash("0xffff") + envelope := newTestSignedEnvelope(100, blockRoot, 9999) + impl.queuePendingEnvelope(blockRoot, envelope, false) + + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + require.True(t, exists) + _, exists = impl.pendingEnvelopes.Load(oldestKnownKey) + require.True(t, exists) + _, exists = impl.pendingEnvelopes.Load(unknownKey) + require.False(t, exists) + require.Equal(t, int32(maxPendingEnvelopes), impl.pendingCount.Load()) +} + +func TestExecutionPayloadServicePendingQueueAdmitsKnownWorkWhenAllJobsAreKnown(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + seenEnvelopesCache: seenCache, + pendingCond: sync.NewCond(&sync.Mutex{}), + } + + var oldestKey pendingEnvelopeKey + for i := range maxPendingEnvelopes { + blockRoot := common.Hash{byte(i), byte(i >> 8)} + envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) + envelopeHash, hashErr := envelope.HashSSZ() + require.NoError(t, hashErr) + key := pendingEnvelopeKey{blockRoot, envelopeHash} + if i == 0 { + oldestKey = key + } + job := &envelopeJob{envelope: envelope, creationTime: time.Now().Add(-time.Hour + time.Duration(i))} + job.blockSeen.Store(true) + impl.pendingEnvelopes.Store(key, job) + } + impl.pendingCount.Store(maxPendingEnvelopes) + + blockRoot := common.HexToHash("0xffff") + envelope := newTestSignedEnvelope(100, blockRoot, 9999) + impl.queuePendingEnvelope(blockRoot, envelope, true) + + envelopeHash, err := envelope.HashSSZ() + require.NoError(t, err) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + require.True(t, exists) + _, exists = impl.pendingEnvelopes.Load(oldestKey) + require.False(t, exists) + require.Equal(t, int32(maxPendingEnvelopes), impl.pendingCount.Load()) +} + +func TestExecutionPayloadServicePendingQueueDoesNotEvictResolvingWork(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + seenEnvelopesCache: seenCache, + pendingCond: sync.NewCond(&sync.Mutex{}), + } + + var resolvingKey, unknownKey pendingEnvelopeKey + for i := range maxPendingEnvelopes { + blockRoot := common.Hash{byte(i), byte(i >> 8)} + envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) + envelopeHash, hashErr := envelope.HashSSZ() + require.NoError(t, hashErr) + key := pendingEnvelopeKey{blockRoot, envelopeHash} + job := &envelopeJob{envelope: envelope, creationTime: time.Now()} + job.blockSeen.Store(true) + switch i { + case 0: + resolvingKey = key + job.creationTime = time.Now().Add(-2 * time.Hour) + job.blockSeen.Store(false) + job.resolving.Store(true) + case 1: + unknownKey = key + job.creationTime = time.Now().Add(-time.Hour) + job.blockSeen.Store(false) + } + impl.pendingEnvelopes.Store(key, job) + } + impl.pendingCount.Store(maxPendingEnvelopes) + + blockRoot := common.HexToHash("0xffff") + envelope := newTestSignedEnvelope(100, blockRoot, 9999) + impl.queuePendingEnvelope(blockRoot, envelope, false) + + _, exists := impl.pendingEnvelopes.Load(resolvingKey) + require.True(t, exists) + _, exists = impl.pendingEnvelopes.Load(unknownKey) + require.False(t, exists) +} + +func TestExecutionPayloadServiceStaleCompletionDoesNotDeleteReplacement(t *testing.T) { + impl := &executionPayloadService{} + key := pendingEnvelopeKey{blockRoot: common.HexToHash("0x1234")} + stale := &envelopeJob{} + replacement := &envelopeJob{} + impl.pendingEnvelopes.Store(key, replacement) + impl.pendingCount.Store(1) + + impl.finishPendingEnvelopeAttempt(key, stale, nil) + + value, exists := impl.pendingEnvelopes.Load(key) + require.True(t, exists) + require.Same(t, replacement, value) + require.Equal(t, int32(1), impl.pendingCount.Load()) +} + func TestExecutionPayloadServicePendingQueueCapConcurrent(t *testing.T) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) @@ -462,7 +773,7 @@ func TestExecutionPayloadServicePendingQueueCapConcurrent(t *testing.T) { wg.Go(func() { blockRoot := common.Hash{byte(i), byte(i >> 8)} envelope := newTestSignedEnvelope(100, blockRoot, uint64(10000+i)) - impl.queuePendingEnvelope(blockRoot, envelope) + impl.queuePendingEnvelope(blockRoot, envelope, false) }) } wg.Wait() diff --git a/cl/phase1/network/services/payload_attestation_service.go b/cl/phase1/network/services/payload_attestation_service.go index 962f6e9d9ee..f50ed2641fd 100644 --- a/cl/phase1/network/services/payload_attestation_service.go +++ b/cl/phase1/network/services/payload_attestation_service.go @@ -185,7 +185,7 @@ func (s *payloadAttestationService) ProcessMessage(ctx context.Context, _ *uint6 // [IGNORE] block state not found // [REJECT] validator is not in PTC // [REJECT] signature verification - if err := s.forkchoiceStore.OnPayloadAttestationMessage(msg, false); err != nil { + if err := s.forkchoiceStore.OnPayloadAttestationMessage(ctx, msg, false); err != nil { // Preserve IGNORE vs REJECT distinction from forkchoice // forkchoice.ErrIgnore != services.ErrIgnore, so we need to convert if errors.Is(err, forkchoice.ErrIgnore) { @@ -218,6 +218,11 @@ func (s *payloadAttestationService) beginValidation(ctx context.Context, key see validation := &payloadAttestationValidation{done: make(chan struct{})} existing, loaded := s.validationsInFlight.LoadOrStore(key, validation) if !loaded { + if s.seenAttestationsCache.Contains(key) { + s.validationsInFlight.CompareAndDelete(key, validation) + close(validation.done) + return nil, true, nil + } return func() { s.validationsInFlight.CompareAndDelete(key, validation) close(validation.done) diff --git a/cl/phase1/network/services/payload_attestation_service_test.go b/cl/phase1/network/services/payload_attestation_service_test.go index 402f756f36e..b9e76757ad9 100644 --- a/cl/phase1/network/services/payload_attestation_service_test.go +++ b/cl/phase1/network/services/payload_attestation_service_test.go @@ -45,7 +45,7 @@ type blockingPayloadAttestationForkchoice struct { release chan struct{} } -func (f *blockingPayloadAttestationForkchoice) OnPayloadAttestationMessage(*cltypes.PayloadAttestationMessage, bool) error { +func (f *blockingPayloadAttestationForkchoice) OnPayloadAttestationMessage(context.Context, *cltypes.PayloadAttestationMessage, bool) error { active := f.active.Add(1) defer f.active.Add(-1) for { @@ -66,7 +66,7 @@ type retryPayloadAttestationForkchoice struct { releaseFirst chan struct{} } -func (f *retryPayloadAttestationForkchoice) OnPayloadAttestationMessage(*cltypes.PayloadAttestationMessage, bool) error { +func (f *retryPayloadAttestationForkchoice) OnPayloadAttestationMessage(context.Context, *cltypes.PayloadAttestationMessage, bool) error { if f.calls.Add(1) == 1 { close(f.firstStarted) <-f.releaseFirst diff --git a/cl/phase1/stages/chain_tip_sync.go b/cl/phase1/stages/chain_tip_sync.go index 38aea2134ca..bb6ca7da8ed 100644 --- a/cl/phase1/stages/chain_tip_sync.go +++ b/cl/phase1/stages/chain_tip_sync.go @@ -22,7 +22,12 @@ import ( "github.com/erigontech/erigon/common/log/v3" ) -const maxGloasVerificationSweepPerCycle = 32 +const ( + maxGloasVerificationSweepPerCycle = 32 + maxPendingGloasPayloadsPerCycle = 32 + maxPendingGloasEnvelopesPerCycle = 32 + pendingGloasEnvelopeRetryBudget = 2 * time.Second +) func gloasVersionedHashes(blobCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) ([]common.Hash, error) { if blobCommitments == nil || blobCommitments.Len() == 0 { @@ -540,7 +545,7 @@ func isGloasPayloadKnownInvalid(cfg *Cfg, envelope *cltypes.SignedExecutionPaylo } func drainPendingGloasPayloads(ctx context.Context, cfg *Cfg) { - for _, p := range cfg.forkChoice.DrainPendingELPayloads() { + for _, p := range cfg.forkChoice.DrainPendingELPayloadsLimit(maxPendingGloasPayloadsPerCycle) { if !validPendingGloasPayload(p) { continue } @@ -685,6 +690,10 @@ func retryUnverifiedAnchorPayload(ctx context.Context, cfg *Cfg) { // chainTipSync synchronizes the chain tip by fetching blocks from the highest seen block up to the target slot by listening to incoming blocks. // or by fetching blocks that might have been missed by gossip after a delay. func chainTipSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) error { + retryCtx, cancelRetry := context.WithTimeout(ctx, pendingGloasEnvelopeRetryBudget) + cfg.forkChoice.RetryPendingExecutionPayloadEnvelopes(retryCtx, maxPendingGloasEnvelopesPerCycle) + cancelRetry() + // [GLOAS] Recover any execution payload envelopes that were missed by gossip. // This runs every cycle (not just when caught-up) because seenSlot < targetSlot // is almost always true — by the time ChainTipSync finishes a slot, the next one diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index b5a504311d6..ce5a6fdb139 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -42,7 +42,7 @@ import ( type StageHistoryReconstructionCfg struct { beaconCfg *clparams.BeaconChainConfig - downloader *network.BackwardBeaconDownloader + downloader historyDownloader sn *freezeblocks.CaplinSnapshots startingRoot common.Hash caplinConfig clparams.CaplinConfig @@ -61,16 +61,32 @@ type StageHistoryReconstructionCfg struct { blobDownloader *network.BlobHistoryDownloader } +type historyDownloader interface { + SetSlotToDownload(uint64) + SetExpectedRoot(common.Hash) + SetBlockChecker(network.BlockChecker) + SetOnNewBlock(network.OnNewBlock) + Finished() bool + Progress() uint64 + RequestMore(context.Context) error + SkippedFullBlocks() []network.SkippedFullBlock + HasEnvelopeRecoverySource() bool + RecoverSkippedEnvelopes(context.Context, []network.SkippedFullBlock, map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult + SetThrottle(time.Duration) + SetNeverSkip(bool) +} + const logIntervalTime = 30 * time.Second const ( - skippedEnvelopeRecoveryMaxAttempts = 3 skippedEnvelopeRecoveryRetryInterval = 10 * time.Second skippedEnvelopeRecoveryBatchSize = 2 skippedEnvelopeRecoveryBatchTimeout = 5 * time.Second skippedEnvelopeRecoveryAttemptTimeout = 2 * time.Minute ) +var errSkippedEnvelopeRecoveryIncomplete = errors.New("skipped envelope recovery incomplete") + func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, antiquary *antiquary.Antiquary, sn *freezeblocks.CaplinSnapshots, indiciesDB kv.RwDB, engine execution_client.ExecutionEngine, beaconCfg *clparams.BeaconChainConfig, caplinConfig clparams.CaplinConfig, waitForAllRoutines bool, startingRoot common.Hash, startinSlot uint64, tmpdir string, backfillingThrottling time.Duration, executionBlocksCollector block_collector.BlockCollector, blockReader freezeblocks.BeaconSnapshotReader, blobStorage blob_storage.BlobStorage, logger log.Logger, forkchoiceStore forkchoice.ForkChoiceStorage, blobDownloader *network.BlobHistoryDownloader) StageHistoryReconstructionCfg { return StageHistoryReconstructionCfg{ beaconCfg: beaconCfg, @@ -287,6 +303,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co }) finishCh := make(chan struct{}) + workerResultCh := make(chan error, 1) // Start logging thread isBackfilling := atomic.Bool{} @@ -385,12 +402,23 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co go func() { defer close(finishCh) + publicResultSent := false + sendPublicResult := func(err error) { + if !publicResultSent { + workerResultCh <- err + publicResultSent = true + } + } for !cfg.downloader.Finished() { + if cfg.engine != nil && cfg.downloader.Progress() <= destinationSlotForEL { + sendPublicResult(nil) + } if err := cfg.downloader.RequestMore(ctx); err != nil { if !errors.Is(err, context.Canceled) { log.Warn("closing backfilling routine", "err", err) } + sendPublicResult(err) return } } @@ -398,6 +426,11 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co // Recover FULL blocks whose envelopes were skipped during backward download. if skipped := cfg.downloader.SkippedFullBlocks(); len(skipped) > 0 { if !recoverSkippedEnvelopesWithRetries(ctx, cfg, skipped) { + workerErr := ctx.Err() + if workerErr == nil { + workerErr = errSkippedEnvelopeRecoveryIncomplete + } + sendPublicResult(workerErr) return } } @@ -412,14 +445,16 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co cfg.blobDownloader.SetNotifyBlobBackfilled(cfg.antiquary.NotifyBlobBackfilled) cfg.blobDownloader.Start() } + sendPublicResult(nil) }() // We block until we are done with the EL side of the backfilling with 2000 blocks of safety margin. - for !cfg.downloader.Finished() && (cfg.engine == nil || cfg.downloader.Progress() > destinationSlotForEL) { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(5 * time.Second): + select { + case workerErr := <-workerResultCh: + if workerErr != nil { + return workerErr } + case <-ctx.Done(): + return ctx.Err() } cfg.downloader.SetThrottle(cfg.backfillingThrottling) // throttle to 0.6 second for backfilling cfg.downloader.SetNeverSkip(false) @@ -431,33 +466,45 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co } func recoverSkippedEnvelopesWithRetries(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock) bool { + return recoverSkippedEnvelopesWithRetryPolicy(ctx, cfg, skipped, + func(attemptCtx context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + return recoverSkippedEnvelopes(attemptCtx, cfg, pending) + }, skippedEnvelopeRecoveryRetryInterval) +} + +func recoverSkippedEnvelopesWithRetryPolicy(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock, recoverAttempt func(context.Context, []network.SkippedFullBlock) []network.SkippedFullBlock, retryInterval time.Duration) bool { + if cfg.downloader == nil || !cfg.downloader.HasEnvelopeRecoverySource() { + log.Warn("[BackwardBeaconDownloader] envelope recovery unavailable", "remaining", len(skipped)) + return false + } + return recoverSkippedEnvelopesUntilComplete(ctx, skipped, recoverAttempt, retryInterval) +} + +func recoverSkippedEnvelopesUntilComplete(ctx context.Context, skipped []network.SkippedFullBlock, recoverAttempt func(context.Context, []network.SkippedFullBlock) []network.SkippedFullBlock, retryInterval time.Duration) bool { pending := skipped - for attempt := 1; attempt <= skippedEnvelopeRecoveryMaxAttempts; attempt++ { + for attempt := 1; ; attempt++ { + if ctx.Err() != nil { + log.Warn("[BackwardBeaconDownloader] envelope recovery canceled", "remaining", len(pending), "err", ctx.Err()) + return false + } attemptCtx, cancel := context.WithTimeout(ctx, skippedEnvelopeRecoveryAttemptTimeout) - pending = recoverSkippedEnvelopes(attemptCtx, cfg, pending) + pending = recoverAttempt(attemptCtx, pending) cancel() if len(pending) == 0 { return true } - if attempt == skippedEnvelopeRecoveryMaxAttempts { - log.Warn("[BackwardBeaconDownloader] envelope recovery incomplete, proceeding with gap", - "recovered", len(skipped)-len(pending), "total", len(skipped), "remaining", len(pending)) - return true - } - log.Warn("[BackwardBeaconDownloader] envelope recovery incomplete, retrying", - "attempt", attempt, "maxAttempts", skippedEnvelopeRecoveryMaxAttempts, + "attempt", attempt, "recovered", len(skipped)-len(pending), "total", len(skipped), "remaining", len(pending)) select { case <-ctx.Done(): log.Warn("[BackwardBeaconDownloader] envelope recovery canceled", "remaining", len(pending), "err", ctx.Err()) return false - case <-time.After(skippedEnvelopeRecoveryRetryInterval): + case <-time.After(retryInterval): } } - return true } // recoverSkippedEnvelopes attempts to fetch execution payload envelopes for @@ -501,38 +548,38 @@ func recoverSkippedEnvelopeBatch(fetchCtx, persistCtx context.Context, cfg Stage return append([]network.SkippedFullBlock(nil), batch...) } blocks := readSkippedEnvelopeBlocks(persistCtx, cfg, batch) - envelopes := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, batch, blocks) + recovery := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, batch, blocks) tx, err := cfg.indiciesDB.BeginRo(persistCtx) if err != nil { return append([]network.SkippedFullBlock(nil), batch...) } defer tx.Rollback() - remaining := make([]network.SkippedFullBlock, 0, len(batch)) - for _, s := range batch { - env := envelopes[common.Hash(s.Root)] - if env == nil || cfg.blockReader == nil { - remaining = append(remaining, s) - continue - } + return unresolvedSkippedEnvelopes(batch, recovery, func(s network.SkippedFullBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { block, err := cfg.blockReader.ReadBlockByRoot(persistCtx, tx, common.Hash(s.Root)) if err != nil || block == nil || block.Block == nil || block.Block.Body == nil { log.Warn("[BackwardBeaconDownloader] skipped block unavailable during recovery", "slot", s.Slot, "root", common.Hash(s.Root), "err", err) - remaining = append(remaining, s) - continue + return false } root, err := block.Block.HashSSZ() if err != nil || root != s.Root { - remaining = append(remaining, s) - continue + return false } if err := network.ValidateFetchedEnvelope(cfg.beaconCfg, block, common.Hash(s.Root), env); err != nil { log.Warn("[BackwardBeaconDownloader] recovered envelope does not match block", "slot", s.Slot, "root", common.Hash(s.Root), "err", err) - remaining = append(remaining, s) - continue + return false } - if !recoverSkippedEnvelope(persistCtx, cfg, s, block, env) { - remaining = append(remaining, s) + return recoverSkippedEnvelope(persistCtx, cfg, s, block, env) + }) +} + +func unresolvedSkippedEnvelopes(batch []network.SkippedFullBlock, recovery network.EnvelopeRecoveryResult, persist func(network.SkippedFullBlock, *cltypes.SignedExecutionPayloadEnvelope) bool) []network.SkippedFullBlock { + remaining := make([]network.SkippedFullBlock, 0, len(batch)) + for _, item := range batch { + root := common.Hash(item.Root) + envelope := recovery.Envelopes[root] + if envelope == nil || !persist(item, envelope) { + remaining = append(remaining, item) } } return remaining diff --git a/cl/phase1/stages/stage_history_download_test.go b/cl/phase1/stages/stage_history_download_test.go index 53e1cfd036a..c6c6ca186f7 100644 --- a/cl/phase1/stages/stage_history_download_test.go +++ b/cl/phase1/stages/stage_history_download_test.go @@ -18,13 +18,117 @@ package stages import ( "context" + "errors" "math" "testing" "time" + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/phase1/network" + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/log/v3" ) +type historyDownloaderStub struct { + finished bool + progress uint64 + requestErr error + requestMore func() error + skipped []network.SkippedFullBlock + recoverySource bool +} + +func (d *historyDownloaderStub) SetSlotToDownload(uint64) {} +func (d *historyDownloaderStub) SetExpectedRoot(common.Hash) {} +func (d *historyDownloaderStub) SetBlockChecker(network.BlockChecker) {} +func (d *historyDownloaderStub) SetOnNewBlock(network.OnNewBlock) {} +func (d *historyDownloaderStub) Finished() bool { return d.finished } +func (d *historyDownloaderStub) Progress() uint64 { return d.progress } +func (d *historyDownloaderStub) RequestMore(context.Context) error { + if d.requestMore != nil { + return d.requestMore() + } + return d.requestErr +} +func (d *historyDownloaderStub) SkippedFullBlocks() []network.SkippedFullBlock { + return d.skipped +} +func (d *historyDownloaderStub) HasEnvelopeRecoverySource() bool { return d.recoverySource } +func (d *historyDownloaderStub) RecoverSkippedEnvelopes(context.Context, []network.SkippedFullBlock, map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult { + return network.EnvelopeRecoveryResult{} +} +func (d *historyDownloaderStub) SetThrottle(time.Duration) {} +func (d *historyDownloaderStub) SetNeverSkip(bool) {} + +func TestSpawnStageHistoryDownloadReturnsDownloaderFailure(t *testing.T) { + wantErr := errors.New("terminal downloader failure") + downloader := &historyDownloaderStub{progress: math.MaxUint64, requestErr: wantErr} + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + err := SpawnStageHistoryDownload(StageHistoryReconstructionCfg{ + beaconCfg: &clparams.MainnetBeaconConfig, + downloader: downloader, + }, ctx, log.New()) + if !errors.Is(err, wantErr) { + t.Fatalf("SpawnStageHistoryDownload() error = %v, want %v", err, wantErr) + } +} + +func TestSpawnStageHistoryDownloadReturnsFailureWhenRequestCrossesELFloor(t *testing.T) { + wantErr := errors.New("terminal downloader failure at EL floor") + destinationSlot := clparams.MainnetBeaconConfig.BellatrixForkEpoch * clparams.MainnetBeaconConfig.SlotsPerEpoch + downloader := &historyDownloaderStub{progress: destinationSlot + 1} + downloader.requestMore = func() error { + downloader.progress = destinationSlot + return wantErr + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + + err := SpawnStageHistoryDownload(StageHistoryReconstructionCfg{ + beaconCfg: &clparams.MainnetBeaconConfig, + downloader: downloader, + engine: &testExecutionEngine{supportInsertion: true}, + }, ctx, log.New()) + if !errors.Is(err, wantErr) { + t.Fatalf("SpawnStageHistoryDownload() error = %v, want %v", err, wantErr) + } +} + +func TestSpawnStageHistoryDownloadReturnsEnvelopeRecoveryFailure(t *testing.T) { + downloader := &historyDownloaderStub{ + finished: true, + skipped: []network.SkippedFullBlock{{Slot: 1}}, + } + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + err := SpawnStageHistoryDownload(StageHistoryReconstructionCfg{ + beaconCfg: &clparams.MainnetBeaconConfig, + downloader: downloader, + }, ctx, log.New()) + if err == nil { + t.Fatal("SpawnStageHistoryDownload() returned nil after envelope recovery failed") + } +} + +func TestUnresolvedSkippedEnvelopesRetriesEveryMissingEnvelope(t *testing.T) { + first := network.SkippedFullBlock{Slot: 1, Root: [32]byte{1}} + second := network.SkippedFullBlock{Slot: 2, Root: [32]byte{2}} + result := network.EnvelopeRecoveryResult{} + + remaining := unresolvedSkippedEnvelopes([]network.SkippedFullBlock{first, second}, result, func(network.SkippedFullBlock, *cltypes.SignedExecutionPayloadEnvelope) bool { + t.Fatal("missing envelopes must not be persisted") + return false + }) + + if len(remaining) != 2 || remaining[0] != first || remaining[1] != second { + t.Fatalf("remaining = %v, want both missing envelopes", remaining) + } +} + func TestRecoverSkippedEnvelopeBatchesDoesNotStarveLaterBatches(t *testing.T) { skipped := []network.SkippedFullBlock{{Slot: 1}, {Slot: 2}, {Slot: 3}, {Slot: 4}, {Slot: 5}, {Slot: 6}, {Slot: 7}, {Slot: 8}} attempted := make([]uint64, 0, len(skipped)) @@ -64,6 +168,66 @@ func TestRecoverSkippedEnvelopeBatchesKeepsPartialSuccess(t *testing.T) { } } +func TestRecoverSkippedEnvelopesWithoutSourcesDoesNotCompleteBackfill(t *testing.T) { + cfg := StageHistoryReconstructionCfg{downloader: &network.BackwardBeaconDownloader{}} + attempts := 0 + recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + attempts++ + return pending + } + + if recoverSkippedEnvelopesWithRetryPolicy(context.Background(), cfg, []network.SkippedFullBlock{{Slot: 1}}, recoverAttempt, 0) { + t.Fatal("recovery without an HTTP or P2P source must not report completion") + } + if attempts != 0 { + t.Fatalf("attempts = %d, want no recovery attempt without a source", attempts) + } +} + +func TestRecoverSkippedEnvelopesRetriesBeyondThreeAttemptCapacity(t *testing.T) { + const itemsPerAttempt = int(skippedEnvelopeRecoveryAttemptTimeout/skippedEnvelopeRecoveryBatchTimeout) * skippedEnvelopeRecoveryBatchSize + downloader := &network.BackwardBeaconDownloader{} + downloader.SetHTTPFallbackURL("http://recovery.test") + cfg := StageHistoryReconstructionCfg{downloader: downloader} + skipped := make([]network.SkippedFullBlock, itemsPerAttempt*3+1) + for i := range skipped { + skipped[i].Slot = uint64(i + 1) + } + + attemptStarts := make([]uint64, 0, 4) + recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + attemptStarts = append(attemptStarts, pending[0].Slot) + return pending[min(itemsPerAttempt, len(pending)):] + } + + if !recoverSkippedEnvelopesWithRetryPolicy(context.Background(), cfg, skipped, recoverAttempt, 0) { + t.Fatal("configured recovery stopped before all pending envelopes were recovered") + } + if len(attemptStarts) != 4 || attemptStarts[3] != uint64(itemsPerAttempt*3+1) { + t.Fatalf("attempt starts = %v, want a fourth attempt starting at slot %d", attemptStarts, itemsPerAttempt*3+1) + } +} + +func TestRecoverSkippedEnvelopesStopsWhenParentContextIsCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + downloader := &network.BackwardBeaconDownloader{} + downloader.SetHTTPFallbackURL("http://recovery.test") + cfg := StageHistoryReconstructionCfg{downloader: downloader} + attempts := 0 + recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + attempts++ + cancel() + return pending + } + + if recoverSkippedEnvelopesWithRetryPolicy(ctx, cfg, []network.SkippedFullBlock{{Slot: 1}}, recoverAttempt, time.Hour) { + t.Fatal("recovery reported completion with a pending envelope after parent cancellation") + } + if attempts != 1 { + t.Fatalf("attempts = %d, want 1", attempts) + } +} + // clampProgress must never report a total below processed nor underflow, even // when the floor and current counters drift past the frozen highestBlockSeen. // The last case mirrors the field report where the live EL head advanced past From 3ba378dc5024d9c611550cb2dc6a6909f1d22729 Mon Sep 17 00:00:00 2001 From: kewei Date: Wed, 12 Aug 2026 04:10:44 +0800 Subject: [PATCH 17/17] cl: harden Gloas envelope lifecycle --- .../encode_block_bench_test.go | 8 +- .../block_collector/encode_block_test.go | 9 +- .../block_collector/interface.go | 2 +- .../persistent_block_collector.go | 55 +- .../persistent_block_collector_test.go | 42 ++ .../payload_validation_coordinator.go | 4 + .../payload_validation_coordinator_test.go | 25 +- .../forkchoice/fork_graph/fork_graph_disk.go | 24 +- .../fork_graph/fork_graph_disk_fs.go | 88 +++- .../forkchoice/fork_graph/fork_graph_test.go | 300 +++++++++++ cl/phase1/forkchoice/fork_graph/interface.go | 6 +- cl/phase1/forkchoice/forkchoice.go | 23 +- cl/phase1/forkchoice/forkchoice_test.go | 70 ++- .../mock_services/forkchoice_mock.go | 10 +- cl/phase1/forkchoice/on_block.go | 40 +- .../on_block_fork_consistency_test.go | 35 -- cl/phase1/forkchoice/on_execution_payload.go | 289 ++++++++--- .../forkchoice/on_execution_payload_test.go | 313 +++++++++++- .../payload_attestation_validation.go | 85 +++- .../payload_attestation_validation_test.go | 172 ++++++- cl/phase1/forkchoice/payload_vote_test.go | 38 +- .../network/backward_beacon_downloader.go | 374 ++++++-------- .../backward_beacon_downloader_test.go | 229 +++++---- cl/phase1/network/beacon_downloader.go | 52 +- cl/phase1/network/beacon_downloader_test.go | 132 +++++ cl/phase1/network/services/block_service.go | 213 ++++++-- .../network/services/block_service_test.go | 349 ++++++++++++- cl/phase1/network/services/canonical_ssz.go | 21 + .../network/services/envelope_resolver.go | 138 +++++ .../services/envelope_resolver_test.go | 197 ++++++++ .../services/execution_payload_bid_service.go | 20 +- .../execution_payload_bid_service_test.go | 13 + .../services/execution_payload_service.go | 184 ++++--- .../execution_payload_service_test.go | 336 ++++++++---- .../services/payload_attestation_service.go | 3 + .../payload_attestation_service_test.go | 15 + cl/phase1/stages/forward_sync.go | 59 ++- cl/phase1/stages/gloas_payload_test.go | 90 ++++ cl/phase1/stages/stage_history_download.go | 271 +++++++++- .../stages/stage_history_download_test.go | 477 ++++++++++++++++-- cl/rpc/rpc.go | 29 +- cl/rpc/rpc_test.go | 53 ++ cmd/capcli/cli.go | 58 ++- cmd/capcli/cli_test.go | 43 ++ cmd/caplin/caplin1/run.go | 234 ++++++++- cmd/caplin/caplin1/run_test.go | 269 ++++++++++ 46 files changed, 4644 insertions(+), 853 deletions(-) create mode 100644 cl/phase1/network/services/canonical_ssz.go create mode 100644 cl/phase1/network/services/envelope_resolver.go create mode 100644 cl/phase1/network/services/envelope_resolver_test.go create mode 100644 cmd/capcli/cli_test.go create mode 100644 cmd/caplin/caplin1/run_test.go diff --git a/cl/phase1/execution_client/block_collector/encode_block_bench_test.go b/cl/phase1/execution_client/block_collector/encode_block_bench_test.go index 9af7aac72a6..e450eefc51e 100644 --- a/cl/phase1/execution_client/block_collector/encode_block_bench_test.go +++ b/cl/phase1/execution_client/block_collector/encode_block_bench_test.go @@ -52,9 +52,11 @@ func BenchmarkEncodeBlock(b *testing.B) { } { payload := benchPayload(tc.txCount, tc.txSize) b.Run(tc.name, func(b *testing.B) { - p := &PersistentBlockCollector{} - p.mu.Lock() - defer p.mu.Unlock() + p := &PersistentBlockCollector{operationSlot: make(chan struct{}, 1)} + if err := p.acquire(b.Context()); err != nil { + b.Fatal(err) + } + defer p.release() b.ReportAllocs() for b.Loop() { if _, err := p.encodeBlock(payload, parentRoot, nil); err != nil { diff --git a/cl/phase1/execution_client/block_collector/encode_block_test.go b/cl/phase1/execution_client/block_collector/encode_block_test.go index 683422b1a55..1d632b7b3bf 100644 --- a/cl/phase1/execution_client/block_collector/encode_block_test.go +++ b/cl/phase1/execution_client/block_collector/encode_block_test.go @@ -43,9 +43,12 @@ func signedTestTx(t *testing.T, nonce uint64) types.Transaction { // consecutive encodes on one collector reuse its scratch buffers, and each // result must decode back to the original execution block. func TestEncodeDecodeBlockRoundTrip(t *testing.T) { - c := &PersistentBlockCollector{beaconChainCfg: &clparams.MainnetBeaconConfig} - c.mu.Lock() - defer c.mu.Unlock() + c := &PersistentBlockCollector{ + beaconChainCfg: &clparams.MainnetBeaconConfig, + operationSlot: make(chan struct{}, 1), + } + require.NoError(t, c.acquire(t.Context())) + defer c.release() parent := common.HexToHash("0xaa") tx0, tx1, tx2 := signedTestTx(t, 0), signedTestTx(t, 1), signedTestTx(t, 2) diff --git a/cl/phase1/execution_client/block_collector/interface.go b/cl/phase1/execution_client/block_collector/interface.go index b2a46f39a70..59def1d3086 100644 --- a/cl/phase1/execution_client/block_collector/interface.go +++ b/cl/phase1/execution_client/block_collector/interface.go @@ -28,7 +28,7 @@ var batchSize = 1000 type BlockCollector interface { AddBlock(block *cltypes.BeaconBlock) error // AddGloasBlock adds a GLOAS (EIP-7732) FULL block using its execution payload envelope. - AddGloasBlock(block *cltypes.BeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) error + AddGloasBlock(ctx context.Context, block *cltypes.BeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) error Flush(ctx context.Context) error HasBlock(blockNumber uint64) bool } diff --git a/cl/phase1/execution_client/block_collector/persistent_block_collector.go b/cl/phase1/execution_client/block_collector/persistent_block_collector.go index f785247f1e1..103315f8797 100644 --- a/cl/phase1/execution_client/block_collector/persistent_block_collector.go +++ b/cl/phase1/execution_client/block_collector/persistent_block_collector.go @@ -20,7 +20,6 @@ import ( "context" "encoding/binary" "fmt" - "sync" "github.com/c2h5oh/datasize" "github.com/golang/snappy" @@ -54,8 +53,8 @@ type PersistentBlockCollector struct { logger log.Logger engine execution_client.ExecutionEngine - mu sync.Mutex - // encodeBlock scratch buffers; guarded by mu. + operationSlot chan struct{} + // encodeBlock scratch buffers; guarded by operationSlot. encodeBlockBuf []byte blockCompressBuf []byte } @@ -101,13 +100,29 @@ func NewPersistentBlockCollector( beaconChainCfg: beaconChainCfg, logger: logger, engine: engine, + operationSlot: make(chan struct{}, 1), } } +func (p *PersistentBlockCollector) acquire(ctx context.Context) error { + select { + case p.operationSlot <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (p *PersistentBlockCollector) release() { + <-p.operationSlot +} + // AddBlock adds a block to the collector, persisting it to the database func (p *PersistentBlockCollector) AddBlock(block *cltypes.BeaconBlock) error { - p.mu.Lock() - defer p.mu.Unlock() + if err := p.acquire(context.Background()); err != nil { + return err + } + defer p.release() if p.db == nil { return fmt.Errorf("database not initialized") @@ -126,9 +141,11 @@ func (p *PersistentBlockCollector) AddBlock(block *cltypes.BeaconBlock) error { // AddGloasBlock adds a GLOAS (EIP-7732) FULL block with its execution payload envelope to the collector. // The execution payload is extracted from the envelope, not the beacon block body. -func (p *PersistentBlockCollector) AddGloasBlock(block *cltypes.BeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) error { - p.mu.Lock() - defer p.mu.Unlock() +func (p *PersistentBlockCollector) AddGloasBlock(ctx context.Context, block *cltypes.BeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) error { + if err := p.acquire(ctx); err != nil { + return err + } + defer p.release() if p.db == nil { return fmt.Errorf("database not initialized") @@ -141,7 +158,7 @@ func (p *PersistentBlockCollector) AddGloasBlock(block *cltypes.BeaconBlock, env return fmt.Errorf("failed to encode gloas block: %w", err) } - return p.db.Update(context.Background(), func(tx kv.RwTx) error { + return p.db.Update(ctx, func(tx kv.RwTx) error { return tx.Put(kv.Headers, payloadKey(payload), encodedBlock) }) } @@ -156,7 +173,7 @@ const ( // encodeBlock serializes the block value: snappy(version + parentRoot + // [requestsHash +] SSZ(payload)). The result aliases p.blockCompressBuf and is // valid only until the next call, so callers must copy it or fully consume it -// before encoding again. Callers must hold p.mu. +// before encoding again. Callers must own operationSlot. func (p *PersistentBlockCollector) encodeBlock(payload *cltypes.Eth1Block, parentRoot common.Hash, executionRequestsList []hexutil.Bytes) ([]byte, error) { p.encodeBlockBuf = append(p.encodeBlockBuf[:0], byte(payload.Version())) p.encodeBlockBuf = append(p.encodeBlockBuf, parentRoot[:]...) @@ -196,8 +213,10 @@ func (p *PersistentBlockCollector) releaseOversizedScratch() { // If a real gap is detected, rows past the gap are kept so the next Flush can retry // once the missing range is re-downloaded. func (p *PersistentBlockCollector) Flush(ctx context.Context) error { - p.mu.Lock() - defer p.mu.Unlock() + if err := p.acquire(ctx); err != nil { + return err + } + defer p.release() defer p.releaseOversizedScratch() if p.db == nil { @@ -517,8 +536,10 @@ func (p *PersistentBlockCollector) doForkChoiceUpdate(ctx context.Context, lastB // HasBlock checks if a block with the given number is already in the collector func (p *PersistentBlockCollector) HasBlock(blockNumber uint64) bool { - p.mu.Lock() - defer p.mu.Unlock() + if err := p.acquire(context.Background()); err != nil { + return false + } + defer p.release() if p.db == nil { return false @@ -550,8 +571,10 @@ func (p *PersistentBlockCollector) HasBlock(blockNumber uint64) bool { // Close closes the database func (p *PersistentBlockCollector) Close() error { - p.mu.Lock() - defer p.mu.Unlock() + if err := p.acquire(context.Background()); err != nil { + return err + } + defer p.release() if p.db != nil { p.db.Close() diff --git a/cl/phase1/execution_client/block_collector/persistent_block_collector_test.go b/cl/phase1/execution_client/block_collector/persistent_block_collector_test.go index 2fc4c31936d..e2caa93cec9 100644 --- a/cl/phase1/execution_client/block_collector/persistent_block_collector_test.go +++ b/cl/phase1/execution_client/block_collector/persistent_block_collector_test.go @@ -145,6 +145,48 @@ func countRowsAtOrAbove(t *testing.T, db kv.RoDB, minNumber uint64) int { return count } +func TestAddGloasBlockHonorsCanceledContext(t *testing.T) { + h := newFlushTestHarness(t, 0) + block := makeBeaconBlock(t, 1, 1, common.Hash{}) + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.Payload = block.Body.ExecutionPayload + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + require.ErrorIs(t, h.collector.AddGloasBlock(ctx, block, envelope), context.Canceled) + require.Equal(t, 0, countRowsAtOrAbove(t, h.collector.db, 0)) +} + +func TestAddGloasBlockCancellationWhileFlushOwnsCollector(t *testing.T) { + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + flushStarted := make(chan struct{}) + releaseFlush := make(chan struct{}) + engine.EXPECT().FrozenBlocks(gomock.Any()).DoAndReturn(func(context.Context) uint64 { + close(flushStarted) + <-releaseFlush + return 0 + }) + collector := NewPersistentBlockCollector(log.New(), engine, &clparams.MainnetBeaconConfig, filepath.Join(t.TempDir(), "collector")) + require.NotNil(t, collector) + t.Cleanup(func() { _ = collector.Close() }) + + flushDone := make(chan error, 1) + go func() { flushDone <- collector.Flush(t.Context()) }() + <-flushStarted + + block := makeBeaconBlock(t, 1, 1, common.Hash{}) + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.Payload = block.Body.ExecutionPayload + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, collector.AddGloasBlock(ctx, block, envelope), context.Canceled) + + close(releaseFlush) + require.NoError(t, <-flushDone) +} + func TestDecodeBlockRejectsShortPersistentValue(t *testing.T) { c := &PersistentBlockCollector{} for name, raw := range map[string][]byte{ diff --git a/cl/phase1/execution_client/payload_validation_coordinator.go b/cl/phase1/execution_client/payload_validation_coordinator.go index d120fd57f75..9d52b271dc9 100644 --- a/cl/phase1/execution_client/payload_validation_coordinator.go +++ b/cl/phase1/execution_client/payload_validation_coordinator.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "sync" + "time" "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/monitor" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/hexutil" ) @@ -81,6 +83,8 @@ func (c *PayloadValidationCoordinator) NewPayload( panicValue any ) func() { + started := time.Now() + defer monitor.ObserveNewPayloadTime(started) defer func() { panicValue = recover() }() diff --git a/cl/phase1/execution_client/payload_validation_coordinator_test.go b/cl/phase1/execution_client/payload_validation_coordinator_test.go index a00a20cb998..5a22eeef7f6 100644 --- a/cl/phase1/execution_client/payload_validation_coordinator_test.go +++ b/cl/phase1/execution_client/payload_validation_coordinator_test.go @@ -3,6 +3,7 @@ package execution_client import ( "context" "errors" + "sync" "sync/atomic" "testing" "time" @@ -60,6 +61,21 @@ type payloadValidationContextResult struct { err error } +type payloadValidationObservedContext struct { + context.Context + observed chan struct{} + once sync.Once +} + +func newPayloadValidationObservedContext(parent context.Context) *payloadValidationObservedContext { + return &payloadValidationObservedContext{Context: parent, observed: make(chan struct{})} +} + +func (c *payloadValidationObservedContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.observed) }) + return c.Context.Done() +} + func TestPayloadValidationCoordinatorWaiterCancellationIsLocal(t *testing.T) { ctrl := gomock.NewController(t) engine := NewMockExecutionEngine(ctrl) @@ -153,11 +169,16 @@ func TestPayloadValidationCoordinatorReportsLeaderPanicToWaiter(t *testing.T) { }() <-started waiterDone := make(chan error, 1) + waiterCtx := newPayloadValidationObservedContext(context.Background()) go func() { - _, err := coordinator.NewPayload(context.Background(), key, nil, nil, nil, nil) + _, err := coordinator.NewPayload(waiterCtx, key, nil, nil, nil, nil) waiterDone <- err }() - time.Sleep(10 * time.Millisecond) + select { + case <-waiterCtx.observed: + case <-time.After(time.Second): + t.Fatal("payload validation waiter did not reach its wait point") + } close(release) require.Equal(t, "engine panic", <-leaderPanic) require.Error(t, <-waiterDone) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index 41db8714242..4ca4637330e 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -603,6 +603,22 @@ func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) { f.stateDumpLock.Unlock() continue } + temporaryFiles, globErr := afero.Glob(f.fs, getEnvelopeFilename(root)+".tmp-*") + if globErr != nil { + f.stateDumpLock.Unlock() + return globErr + } + filesToRemove := make([]string, 0, 3+len(temporaryFiles)) + filesToRemove = append(filesToRemove, + getBeaconStateFilename(root), + getEnvelopeFilename(root), + getEnvelopeIndexMarkerFilename(root), + ) + filesToRemove = append(filesToRemove, temporaryFiles...) + if removeErr := removeFilesAndSyncDirectory(f.fs, filesToRemove...); removeErr != nil { + f.stateDumpLock.Unlock() + return removeErr + } f.badBlocks.Delete(root) f.blocks.Delete(root) f.lightclientBootstraps.Delete(root) @@ -610,16 +626,8 @@ func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) { f.finalizedCheckpoints.Delete(root) f.headers.Delete(root) f.blockRewards.Delete(root) - f.fs.Remove(getBeaconStateFilename(root)) // [New in Gloas:EIP7732] Also remove envelope files f.envelopeExists.Delete(root) - f.fs.Remove(getEnvelopeFilename(root)) - f.fs.Remove(getEnvelopeIndexMarkerFilename(root)) - if temporaryFiles, err := afero.Glob(f.fs, getEnvelopeFilename(root)+".tmp-*"); err == nil { - for _, temporaryFile := range temporaryFiles { - f.fs.Remove(temporaryFile) - } - } f.stateDumpLock.Unlock() } log.Debug("Pruned old blocks", "pruneSlot", pruneSlot) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go index b508a50506d..bc42ec764ec 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go @@ -17,11 +17,14 @@ package fork_graph import ( + "bytes" "encoding/binary" "encoding/hex" + "errors" "fmt" "io" "os" + "runtime" "strings" "github.com/golang/snappy" @@ -254,6 +257,12 @@ func (f *forkGraphDisk) ReadEnvelopeFromDisk(blockRoot common.Hash) (envelope *c return nil, fmt.Errorf("failed to read snappy buffer: %w, root: %x", err, blockRoot) } f.sszBuffer = f.sszBuffer[:n] + var trailing [1]byte + trailingN, trailingErr := f.sszSnappyReader.Read(trailing[:]) + if trailingN != 0 || trailingErr != io.EOF { + return nil, fmt.Errorf("trailing envelope data, root: %x", blockRoot) + } + encoded := append([]byte(nil), f.sszBuffer...) envelope = &cltypes.SignedExecutionPayloadEnvelope{ Message: cltypes.NewExecutionPayloadEnvelope(f.beaconCfg), @@ -261,6 +270,13 @@ func (f *forkGraphDisk) ReadEnvelopeFromDisk(blockRoot common.Hash) (envelope *c if err = envelope.DecodeSSZ(f.sszBuffer, int(clparams.GloasVersion)); err != nil { return nil, fmt.Errorf("failed to decode envelope: %w, root: %x, len: %d", err, blockRoot, n) } + canonical, err := envelope.EncodeSSZ(nil) + if err != nil { + return nil, fmt.Errorf("failed to re-encode envelope: %w, root: %x", err, blockRoot) + } + if !bytes.Equal(encoded, canonical) { + return nil, fmt.Errorf("non-canonical envelope encoding, root: %x", blockRoot) + } return } @@ -290,7 +306,7 @@ func (f *forkGraphDisk) PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *c } filename := getEnvelopeFilename(blockRoot) - dumpedFile, err := afero.TempFile(f.fs, "", filename+".tmp-") + dumpedFile, err := afero.TempFile(f.fs, ".", filename+".tmp-") if err != nil { return nil, err } @@ -330,36 +346,54 @@ func (f *forkGraphDisk) PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *c log.Error("failed to sync dumped file", "err", err) return } - if err = dumpedFile.Close(); err != nil { + if err := dumpedFile.Close(); err != nil { return nil, err } markerFilename := getEnvelopeIndexMarkerFilename(blockRoot) - marker, err := f.fs.OpenFile(markerFilename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o644) + markerExisted, err := afero.Exists(f.fs, markerFilename) if err != nil { return nil, err } - if err = marker.Sync(); err != nil { - _ = marker.Close() - _ = f.fs.Remove(markerFilename) - return nil, err + if !markerExisted { + marker, err := f.fs.OpenFile(markerFilename, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + if err = marker.Sync(); err != nil { + _ = marker.Close() + _ = f.fs.Remove(markerFilename) + return nil, err + } + if err = marker.Close(); err != nil { + _ = f.fs.Remove(markerFilename) + return nil, err + } } - if err = marker.Close(); err != nil { - _ = f.fs.Remove(markerFilename) + if err := syncRootDirectory(f.fs); err != nil { return nil, err } keepTemporary = true + cleanupFiles := []string{temporaryFilename} + if !markerExisted { + cleanupFiles = append(cleanupFiles, markerFilename) + } return func() error { f.stateDumpLock.Lock() defer f.stateDumpLock.Unlock() if _, blockPresent := f.blocks.Load(blockRoot); requireBlock && !blockPresent { - _ = f.fs.Remove(temporaryFilename) - _ = f.fs.Remove(markerFilename) + if cleanupErr := removeFilesAndSyncDirectory(f.fs, cleanupFiles...); cleanupErr != nil { + return fmt.Errorf("cannot publish envelope for pruned block %x: %w", blockRoot, cleanupErr) + } return fmt.Errorf("cannot publish envelope for pruned block %x", blockRoot) } if err := f.fs.Rename(temporaryFilename, filename); err != nil { - _ = f.fs.Remove(temporaryFilename) - _ = f.fs.Remove(markerFilename) + if cleanupErr := removeFilesAndSyncDirectory(f.fs, cleanupFiles...); cleanupErr != nil { + return errors.Join(err, cleanupErr) + } + return err + } + if err := syncRootDirectory(f.fs); err != nil { return err } f.envelopeExists.Store(blockRoot, struct{}{}) @@ -403,8 +437,32 @@ func (f *forkGraphDisk) MarkEnvelopeIndicesCommitted(blockRoot common.Hash) erro } } err = f.fs.Remove(getEnvelopeIndexMarkerFilename(blockRoot)) - if os.IsNotExist(err) { + if err != nil && !os.IsNotExist(err) { + return err + } + return syncRootDirectory(f.fs) +} + +func removeFilesAndSyncDirectory(fs afero.Fs, filenames ...string) error { + for _, filename := range filenames { + if err := fs.Remove(filename); err != nil && !os.IsNotExist(err) { + return err + } + } + return syncRootDirectory(fs) +} + +func syncRootDirectory(fs afero.Fs) error { + if runtime.GOOS == "windows" { return nil } - return err + directory, err := fs.Open(".") + if err != nil { + return err + } + if err := directory.Sync(); err != nil { + _ = directory.Close() + return err + } + return directory.Close() } diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go index 65a7b00f59d..66a9f98a789 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go @@ -18,14 +18,18 @@ package fork_graph import ( _ "embed" + "encoding/binary" "errors" "os" + "path/filepath" + "strings" "sync" "testing" "time" "github.com/erigontech/erigon/cl/beacon/beacon_router_configuration" "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/golang/snappy" "github.com/spf13/afero" "github.com/erigontech/erigon/cl/clparams" @@ -51,6 +55,77 @@ type blockingStatFS struct { once sync.Once } +type directorySyncTrackingFS struct { + afero.Fs + mu sync.Mutex + events []string + failNext bool +} + +type directorySyncTrackingFile struct { + afero.File + fs *directorySyncTrackingFS +} + +func (f *directorySyncTrackingFS) Open(name string) (afero.File, error) { + file, err := f.Fs.Open(name) + if err != nil || name != "." { + return file, err + } + return &directorySyncTrackingFile{File: file, fs: f}, nil +} + +func (f *directorySyncTrackingFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if strings.HasSuffix(name, ".envelope.indices-pending") { + f.record("marker") + } + return f.Fs.OpenFile(name, flag, perm) +} + +func (f *directorySyncTrackingFS) Rename(oldname, newname string) error { + f.record("rename") + return f.Fs.Rename(oldname, newname) +} + +func (f *directorySyncTrackingFS) Remove(name string) error { + if strings.HasSuffix(name, ".envelope.indices-pending") { + f.record("remove marker") + } + return f.Fs.Remove(name) +} + +func (f *directorySyncTrackingFS) record(event string) { + f.mu.Lock() + f.events = append(f.events, event) + f.mu.Unlock() +} + +func (f *directorySyncTrackingFS) takeEvents() []string { + f.mu.Lock() + defer f.mu.Unlock() + events := append([]string(nil), f.events...) + f.events = nil + return events +} + +func (f *directorySyncTrackingFS) failNextSync() { + f.mu.Lock() + f.failNext = true + f.mu.Unlock() +} + +func (f *directorySyncTrackingFile) Sync() error { + f.fs.mu.Lock() + f.fs.events = append(f.fs.events, "sync directory") + fail := f.fs.failNext + f.fs.failNext = false + f.fs.mu.Unlock() + if fail { + return errors.New("injected directory sync failure") + } + return f.File.Sync() +} + func (f *blockingStatFS) Stat(name string) (os.FileInfo, error) { if name == f.target { f.once.Do(func() { close(f.started) }) @@ -168,6 +243,166 @@ func TestDumpEnvelopeOnDiskKeepsPreviousFileWhenRenameFails(t *testing.T) { require.False(t, exists) } +func TestDumpEnvelopeOnDiskWithBasePathFs(t *testing.T) { + baseDir := t.TempDir() + fs := afero.NewBasePathFs(afero.NewOsFs(), baseDir) + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + + require.NoError(t, graph.DumpEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}))) + require.FileExists(t, filepath.Join(baseDir, getEnvelopeFilename(root))) + require.FileExists(t, filepath.Join(baseDir, getEnvelopeIndexMarkerFilename(root))) + matches, err := filepath.Glob(filepath.Join(baseDir, getEnvelopeFilename(root)+".tmp-*")) + require.NoError(t, err) + require.Empty(t, matches) + + require.NoError(t, graph.MarkEnvelopeIndicesCommitted(root)) + require.NoFileExists(t, filepath.Join(baseDir, getEnvelopeIndexMarkerFilename(root))) +} + +func TestEnvelopeJournalSyncsDirectoryAtDurabilityBoundaries(t *testing.T) { + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + + publish, err := graph.PrepareEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}), false) + require.NoError(t, err) + require.Equal(t, []string{"marker", "sync directory"}, fs.takeEvents()) + + require.NoError(t, publish()) + require.Equal(t, []string{"rename", "sync directory"}, fs.takeEvents()) + + require.NoError(t, graph.MarkEnvelopeIndicesCommitted(root)) + require.Equal(t, []string{"remove marker", "sync directory"}, fs.takeEvents()) +} + +func TestEnvelopeJournalPropagatesDirectorySyncFailures(t *testing.T) { + t.Run("prepare marker", func(t *testing.T) { + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + fs.failNextSync() + + _, err := graph.PrepareEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}), false) + require.ErrorContains(t, err, "injected directory sync failure") + }) + + t.Run("publish rename", func(t *testing.T) { + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + publish, err := graph.PrepareEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}), false) + require.NoError(t, err) + fs.takeEvents() + fs.failNextSync() + + err = publish() + require.ErrorContains(t, err, "injected directory sync failure") + _, cached := graph.envelopeExists.Load(root) + require.False(t, cached) + }) + + t.Run("commit marker removal", func(t *testing.T) { + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + require.NoError(t, graph.DumpEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}))) + fs.takeEvents() + fs.failNextSync() + + err := graph.MarkEnvelopeIndicesCommitted(root) + require.ErrorContains(t, err, "injected directory sync failure") + }) +} + +func TestEnvelopeJournalSyncsAbortedPublishCleanup(t *testing.T) { + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + root := common.Hash{1} + block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + graph.blocks.Store(root, block) + publish, err := graph.PrepareEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}), true) + require.NoError(t, err) + fs.takeEvents() + graph.blocks.Delete(root) + + err = publish() + require.ErrorContains(t, err, "pruned block") + require.Equal(t, []string{"remove marker", "sync directory"}, fs.takeEvents()) + pendingRoots, pendingErr := graph.PendingEnvelopeIndexRoots() + require.NoError(t, pendingErr) + require.NotContains(t, pendingRoots, root) +} + +func TestReplacementPreservesPreexistingRecoveryMarker(t *testing.T) { + newGraph := func(t *testing.T) (*forkGraphDisk, *directorySyncTrackingFS, common.Hash) { + t.Helper() + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + root := common.Hash{1} + block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + graph.blocks.Store(root, block) + require.NoError(t, graph.DumpEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{2}))) + fs.takeEvents() + return graph, fs, root + } + assertMarker := func(t *testing.T, graph *forkGraphDisk, root common.Hash) { + t.Helper() + pendingRoots, err := graph.PendingEnvelopeIndexRoots() + require.NoError(t, err) + require.Contains(t, pendingRoots, root) + } + + t.Run("pruned before replacement publish", func(t *testing.T) { + graph, fs, root := newGraph(t) + publish, err := graph.PrepareEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{3}), true) + require.NoError(t, err) + graph.blocks.Delete(root) + fs.takeEvents() + + err = publish() + require.ErrorContains(t, err, "pruned block") + assertMarker(t, graph, root) + require.Equal(t, []string{"sync directory"}, fs.takeEvents()) + }) + + t.Run("replacement publish sync failure", func(t *testing.T) { + graph, fs, root := newGraph(t) + publish, err := graph.PrepareEnvelopeOnDisk(root, testExecutionPayloadEnvelope(root, common.Hash{3}), true) + require.NoError(t, err) + fs.takeEvents() + fs.failNextSync() + + err = publish() + require.ErrorContains(t, err, "injected directory sync failure") + assertMarker(t, graph, root) + }) +} + +func TestPrunePropagatesDirectorySyncFailureBeforeDroppingBlock(t *testing.T) { + fs := &directorySyncTrackingFS{Fs: afero.NewMemMapFs()} + oldRoot := common.Hash{1} + newRoot := common.Hash{2} + oldBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + oldBlock.Block.Slot = 100 + newBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + newBlock.Block.Slot = 200 + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + graph.blocks.Store(oldRoot, oldBlock) + graph.blocks.Store(newRoot, newBlock) + require.NoError(t, afero.WriteFile(fs, getBeaconStateFilename(oldRoot), []byte{1}, 0o644)) + require.NoError(t, afero.WriteFile(fs, getBeaconStateFilename(newRoot), []byte{1}, 0o644)) + fs.failNextSync() + + err := graph.Prune(150) + require.ErrorContains(t, err, "injected directory sync failure") + _, blockStillTracked := graph.blocks.Load(oldRoot) + require.True(t, blockStillTracked) + require.NoError(t, graph.Prune(150)) + _, blockStillTracked = graph.blocks.Load(oldRoot) + require.False(t, blockStillTracked) +} + func testExecutionPayloadEnvelope(root, executionHash common.Hash) *cltypes.SignedExecutionPayloadEnvelope { envelope := &cltypes.SignedExecutionPayloadEnvelope{ Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig), @@ -177,6 +412,51 @@ func testExecutionPayloadEnvelope(root, executionHash common.Hash) *cltypes.Sign return envelope } +func writeEncodedEnvelope(t *testing.T, fs afero.Fs, root common.Hash, encoded []byte) { + t.Helper() + file, err := fs.Create(getEnvelopeFilename(root)) + require.NoError(t, err) + writer := snappy.NewBufferedWriter(file) + var length [8]byte + binary.BigEndian.PutUint64(length[:], uint64(len(encoded))) + _, err = writer.Write(length[:]) + require.NoError(t, err) + _, err = writer.Write(encoded) + require.NoError(t, err) + require.NoError(t, writer.Close()) + require.NoError(t, file.Close()) +} + +func TestReadEnvelopeFromDiskRejectsTrailingSSZBytes(t *testing.T) { + fs := afero.NewMemMapFs() + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + encoded, err := testExecutionPayloadEnvelope(root, common.Hash{2}).EncodeSSZ(nil) + require.NoError(t, err) + writeEncodedEnvelope(t, fs, root, append(encoded, 0)) + + _, err = graph.ReadEnvelopeFromDisk(root) + require.Error(t, err) +} + +func TestReadEnvelopeFromDiskRejectsDynamicOffsetGap(t *testing.T) { + fs := afero.NewMemMapFs() + root := common.Hash{1} + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + encoded, err := testExecutionPayloadEnvelope(root, common.Hash{2}).EncodeSSZ(nil) + require.NoError(t, err) + messageOffset := binary.LittleEndian.Uint32(encoded[:4]) + mutated := make([]byte, 0, len(encoded)+4) + mutated = append(mutated, encoded[:messageOffset]...) + mutated = append(mutated, 0, 0, 0, 0) + mutated = append(mutated, encoded[messageOffset:]...) + binary.LittleEndian.PutUint32(mutated[:4], messageOffset+4) + writeEncodedEnvelope(t, fs, root, mutated) + + _, err = graph.ReadEnvelopeFromDisk(root) + require.ErrorContains(t, err, "non-canonical") +} + // A prune for an already-covered slot (e.g. from a concurrent lock-free drain) // must not move the lowest-available marker backward past deleted data. func TestPruneKeepsLowestAvailableBlockMonotonic(t *testing.T) { @@ -267,3 +547,23 @@ func TestPreparedEnvelopeCannotPublishAfterItsBlockIsPruned(t *testing.T) { require.NoError(t, err) require.NotContains(t, pendingRoots, oldRoot) } + +func TestEnvelopeCannotPrepareAfterItsBlockIsPruned(t *testing.T) { + fs := afero.NewMemMapFs() + oldRoot := common.Hash{1} + newRoot := common.Hash{2} + oldBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + oldBlock.Block.Slot = 100 + newBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion) + newBlock.Block.Slot = 200 + graph := &forkGraphDisk{fs: fs, beaconCfg: &clparams.MainnetBeaconConfig} + graph.blocks.Store(oldRoot, oldBlock) + graph.blocks.Store(newRoot, newBlock) + require.NoError(t, afero.WriteFile(fs, getBeaconStateFilename(oldRoot), []byte{1}, 0o644)) + require.NoError(t, afero.WriteFile(fs, getBeaconStateFilename(newRoot), []byte{1}, 0o644)) + + require.NoError(t, graph.Prune(150)) + _, err := graph.PrepareEnvelopeOnDisk(oldRoot, testExecutionPayloadEnvelope(oldRoot, common.Hash{3}), true) + require.ErrorContains(t, err, "missing block") + require.False(t, graph.HasEnvelope(oldRoot)) +} diff --git a/cl/phase1/forkchoice/fork_graph/interface.go b/cl/phase1/forkchoice/fork_graph/interface.go index 5819b6c639d..7e2218ee723 100644 --- a/cl/phase1/forkchoice/fork_graph/interface.go +++ b/cl/phase1/forkchoice/fork_graph/interface.go @@ -63,9 +63,13 @@ type ForkGraph interface { // and for the store.payloads membership check (HasEnvelope), but no separate // execution_payload_state is maintained. DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) error - PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, requireBlock bool) (publish func() error, err error) ReadEnvelopeFromDisk(blockRoot common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) HasEnvelope(blockRoot common.Hash) bool +} + +// EnvelopePersistence coordinates atomic envelope publication with index recovery. +type EnvelopePersistence interface { + PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, requireBlock bool) (publish func() error, err error) PendingEnvelopeIndexRoots() ([]common.Hash, error) MarkEnvelopeIndicesCommitted(blockRoot common.Hash) error } diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index 734c7704d7a..bd6ec6dcd1b 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -119,6 +119,7 @@ type ForkChoiceStore struct { // Use go map because this is actually an unordered set equivocatingIndicies []byte forkGraph fork_graph.ForkGraph + envelopePersistence fork_graph.EnvelopePersistence blobStorage blob_storage.BlobStorage peerDas das.PeerDas // Per-block unrealized checkpoints (spec: store.unrealized_justifications) @@ -185,19 +186,15 @@ type ForkChoiceStore struct { // Used by is_head_late and proposer boost reorg logic. blockTimeliness sync.Map // map[common.Hash][clparams.NumBlockTimelinessDeadlines]bool // [New in Gloas:EIP7732] - gloasWeightTree *gloasWeightTree - // [New in Gloas:EIP7732] Envelopes waiting for their corresponding block to arrive. - // In GLOAS, BeaconBlock and ExecutionPayloadEnvelope are gossiped separately. - // Due to network timing, the envelope may arrive before its corresponding block. - // When this happens, OnExecutionPayload queues the envelope here (keyed by beacon_block_root). - // Later, when OnBlock processes the block, it checks this cache and processes any pending envelope. - pendingEnvelopes *lru.Cache[common.Hash, *cltypes.SignedExecutionPayloadEnvelope] + gloasWeightTree *gloasWeightTree + pendingEnvelopes *lru.Cache[common.Hash, *pendingExecutionPayloadEnvelopeEntry] // [New in Gloas:EIP7732] Locally-produced self-build envelopes waiting for their block. // Separate from pendingEnvelopes so that OnBlock replay can distinguish local origin // (skip BLS) from gossip origin (full verification) without inspecting envelope contents. - pendingLocalSelfBuildEnvelopes *lru.Cache[common.Hash, *cltypes.SignedExecutionPayloadEnvelope] - pendingEnvelopeRetryMu sync.Mutex + pendingLocalSelfBuildEnvelopes *lru.Cache[common.Hash, *pendingExecutionPayloadEnvelopeEntry] + pendingEnvelopeRetryOnce sync.Once + pendingEnvelopeRetrySlot chan struct{} pendingEnvelopeRetryLocal bool // [New in Gloas:EIP7732] Execution blocks whose CL state transition succeeded but @@ -217,7 +214,7 @@ type ForkChoiceStore struct { } type envelopeOwner struct { - mu sync.Mutex + slot chan struct{} refs int } @@ -247,6 +244,7 @@ func NewForkChoiceStore( probabilisticHeadGetter bool, db kv.RwDB, ) (*ForkChoiceStore, error) { + envelopePersistence, _ := forkGraph.(fork_graph.EnvelopePersistence) anchorRoot, err := anchorState.BlockRoot() if err != nil { return nil, err @@ -322,13 +320,13 @@ func NewForkChoiceStore( } // [New in Gloas:EIP7732] LRU cache for pending envelopes waiting for their block - pendingEnvelopes, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](queueCacheSize) + pendingEnvelopes, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](queueCacheSize) if err != nil { return nil, err } // [New in Gloas:EIP7732] Separate queue for locally-produced self-build envelopes - pendingLocalSelfBuildEnvelopes, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](queueCacheSize) + pendingLocalSelfBuildEnvelopes, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](queueCacheSize) if err != nil { return nil, err } @@ -374,6 +372,7 @@ func NewForkChoiceStore( headSet[anchorRoot] = struct{}{} f := &ForkChoiceStore{ forkGraph: forkGraph, + envelopePersistence: envelopePersistence, equivocatingIndicies: make([]byte, anchorState.ValidatorLength(), anchorState.ValidatorLength()*2), latestMessages: newLatestMessagesStore(anchorState.ValidatorLength()), eth2Roots: eth2Roots, diff --git a/cl/phase1/forkchoice/forkchoice_test.go b/cl/phase1/forkchoice/forkchoice_test.go index 587e0e36510..a0606b82ef4 100644 --- a/cl/phase1/forkchoice/forkchoice_test.go +++ b/cl/phase1/forkchoice/forkchoice_test.go @@ -17,6 +17,7 @@ package forkchoice import ( + "context" "errors" "slices" "sync" @@ -25,18 +26,75 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/erigontech/erigon/cl/beacon/beaconevents" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" "github.com/erigontech/erigon/cl/pool" "github.com/erigontech/erigon/cl/transition/impl/eth2" + "github.com/erigontech/erigon/cl/utils/eth_clock" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/hexutil" ) +func TestOnBlockRejectsFinalizationChangedDuringPayloadValidation(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.AltairForkEpoch = 0 + cfg.BellatrixForkEpoch = 0 + parentRoot := common.Hash{1} + initialFinalized := solid.Checkpoint{Epoch: 2, Root: parentRoot} + + graph := &getFinalizedExecutionHashForkGraph{ + headers: map[common.Hash]*cltypes.BeaconBlockHeader{ + parentRoot: {Slot: 64}, + }, + } + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + started := make(chan struct{}) + release := make(chan struct{}) + engine.EXPECT().NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + close(started) + <-release + return execution_client.PayloadStatusValidated, nil + }, + ) + verified, err := lru.New[common.Hash, struct{}](16) + require.NoError(t, err) + store := &ForkChoiceStore{ + beaconCfg: &cfg, + forkGraph: graph, + engine: engine, + verifiedExecutionPayload: verified, + ethClock: eth_clock.NewEthereumClock(0, common.Hash{}, &cfg), + } + store.finalizedCheckpoint.Store(initialFinalized) + store.time.Store(cfg.SecondsPerSlot * 100) + + block := cltypes.NewSignedBeaconBlock(&cfg, clparams.BellatrixVersion) + block.Block.Slot = 65 + block.Block.ParentRoot = parentRoot + done := make(chan error, 1) + go func() { + done <- store.OnBlock(t.Context(), block, true, false, false) + }() + <-started + + store.mu.Lock() + store.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 3, Root: common.Hash{2}}) + store.mu.Unlock() + close(release) + + require.ErrorIs(t, <-done, ErrNotFinalizedDescendant) + require.False(t, graph.addChainSegmentCalled) +} + func TestGetFinalizedExecutionHash(t *testing.T) { cache, err := lru.New[common.Hash, common.Hash](16) require.NoError(t, err) @@ -514,18 +572,6 @@ func (g *getFinalizedExecutionHashForkGraph) DumpEnvelopeOnDisk(common.Hash, *cl panic("not used") } -func (g *getFinalizedExecutionHashForkGraph) PrepareEnvelopeOnDisk(common.Hash, *cltypes.SignedExecutionPayloadEnvelope, bool) (func() error, error) { - panic("not used") -} - -func (g *getFinalizedExecutionHashForkGraph) PendingEnvelopeIndexRoots() ([]common.Hash, error) { - panic("not used") -} - -func (g *getFinalizedExecutionHashForkGraph) MarkEnvelopeIndicesCommitted(common.Hash) error { - panic("not used") -} - func (g *getFinalizedExecutionHashForkGraph) ReadEnvelopeFromDisk(common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { panic("not used") } diff --git a/cl/phase1/forkchoice/mock_services/forkchoice_mock.go b/cl/phase1/forkchoice/mock_services/forkchoice_mock.go index 337b335a050..ee0d6b3a139 100644 --- a/cl/phase1/forkchoice/mock_services/forkchoice_mock.go +++ b/cl/phase1/forkchoice/mock_services/forkchoice_mock.go @@ -18,6 +18,7 @@ package mock_services import ( "context" + "sync/atomic" "testing" "go.uber.org/mock/gomock" @@ -77,6 +78,9 @@ type ForkChoiceStorageMock struct { Envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope VerifiedPayloads map[common.Hash]bool OnExecutionPayloadErr error + OnExecutionPayloadFunc func(*cltypes.SignedExecutionPayloadEnvelope) error + OnBlockErr error + OnBlockCalls atomic.Int32 GetBeaconCommitteeMock func(slot, committeeIndex uint64) ([]uint64, error) Pool pool.OperationsPool @@ -348,10 +352,14 @@ func (f *ForkChoiceStorageMock) OnBlock( fullValidation bool, checkDataAvaiability bool, ) error { - return nil + f.OnBlockCalls.Add(1) + return f.OnBlockErr } func (f *ForkChoiceStorageMock) OnExecutionPayload(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool) error { + if f.OnExecutionPayloadFunc != nil { + return f.OnExecutionPayloadFunc(signedEnvelope) + } return f.OnExecutionPayloadErr } diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go index 4044a850ff4..8c483f46fe6 100644 --- a/cl/phase1/forkchoice/on_block.go +++ b/cl/phase1/forkchoice/on_block.go @@ -227,9 +227,13 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac return fmt.Errorf("OnBlock: failed to process kzg commitments: %v", err) } } - timeStartExec := time.Now() - payloadStatus, err := f.engine.NewPayload(ctx, block.Block.Body.ExecutionPayload, &block.Block.ParentRoot, versionedHashes, executionRequestsList) - monitor.ObserveNewPayloadTime(timeStartExec) + payloadStatus, err := f.newPayloadLocked(ctx, common.Hash(blockRoot), block.Block.Body.ExecutionPayload, &block.Block.ParentRoot, versionedHashes, executionRequestsList) + currentFinalized := f.finalizedCheckpoint.Load().(solid.Checkpoint) + currentFinalizedSlot := f.computeStartSlotAtEpoch(currentFinalized.Epoch) + if currentFinalizedSlot >= f.forkGraph.AnchorSlot() && + (block.Block.Slot <= currentFinalizedSlot || f.Ancestor(block.Block.ParentRoot, currentFinalizedSlot).Root != currentFinalized.Root) { + return ErrNotFinalizedDescendant + } log.Trace("[OnBlock] NewPayload", "status", payloadStatus, "blockSlot", block.Block.Slot) // Track payload status and gas limit by execution block hash for GLOAS parent payload validation @@ -334,6 +338,7 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // [New in Gloas:EIP7732] GLOAS-specific on_block logic (post state transition) var appliedEnvelope *cltypes.ExecutionPayloadEnvelope var pendingEnvelope *cltypes.SignedExecutionPayloadEnvelope + var pendingEnvelopeEntry *pendingExecutionPayloadEnvelopeEntry var pendingLocalSelfBuild bool if blockVersion >= clparams.GloasVersion { // Initialize payload timeliness and data availability votes for this block @@ -358,11 +363,13 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // then the general gossip queue (full BLS verification). These are separate // queues so that origin is determined by which queue wrote the entry, not by // inspecting envelope contents (which an attacker could forge). - if pending, ok := f.pendingLocalSelfBuildEnvelopes.Get(common.Hash(blockRoot)); ok { - pendingEnvelope = pending + if pending, ok := f.pendingLocalSelfBuildEnvelopes.Peek(common.Hash(blockRoot)); ok && !f.pendingExecutionPayloadEnvelopeExpired(common.Hash(blockRoot), pending) { + pendingEnvelopeEntry = pending + pendingEnvelope = pending.envelope pendingLocalSelfBuild = true - } else if pending, ok := f.pendingEnvelopes.Get(common.Hash(blockRoot)); ok { - pendingEnvelope = pending + } else if pending, ok := f.pendingEnvelopes.Peek(common.Hash(blockRoot)); ok && !f.pendingExecutionPayloadEnvelopeExpired(common.Hash(blockRoot), pending) { + pendingEnvelopeEntry = pending + pendingEnvelope = pending.envelope } } if lastProcessedState.Slot()%f.beaconCfg.SlotsPerEpoch == 0 { @@ -473,10 +480,10 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac if applyErr != nil { log.Warn("OnBlock: failed to process pending envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) } else { - f.removePendingExecutionPayloadEnvelope(pendingExecutionPayloadEnvelope{ - root: common.Hash(blockRoot), - envelope: pendingEnvelope, - local: pendingLocalSelfBuild, + _ = f.removePendingExecutionPayloadEnvelope(ctx, pendingExecutionPayloadEnvelope{ + root: common.Hash(blockRoot), + entry: pendingEnvelopeEntry, + local: pendingLocalSelfBuild, }) if applied { appliedEnvelope = pendingEnvelope.Message @@ -494,17 +501,6 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac return nil } -func (f *ForkChoiceStore) refreshBlockStateAfterPayloadValidation(blockRoot common.Hash) (*state.CachingBeaconState, error) { - blockState, err := f.forkGraph.GetState(blockRoot, false) - if err != nil { - return nil, fmt.Errorf("OnBlock: failed to refresh block state after payload validation: %w", err) - } - if blockState == nil { - return nil, fmt.Errorf("OnBlock: block state disappeared after payload validation for block %v", blockRoot) - } - return blockState, nil -} - func (f *ForkChoiceStore) addChainSegmentAndQueueLightClientEvents(block *cltypes.SignedBeaconBlock, fullValidation bool) (*state.CachingBeaconState, fork_graph.ChainSegmentInsertionResult, error) { lcUpdateBefore := f.forkGraph.NewestLightClientUpdate() lastProcessedState, status, err := f.forkGraph.AddChainSegment(block, fullValidation) diff --git a/cl/phase1/forkchoice/on_block_fork_consistency_test.go b/cl/phase1/forkchoice/on_block_fork_consistency_test.go index ec2f76bc8ea..7a22f7e23e4 100644 --- a/cl/phase1/forkchoice/on_block_fork_consistency_test.go +++ b/cl/phase1/forkchoice/on_block_fork_consistency_test.go @@ -18,29 +18,15 @@ package forkchoice import ( "context" - "errors" "testing" "github.com/stretchr/testify/require" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" - "github.com/erigontech/erigon/cl/phase1/core/state" - "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" "github.com/erigontech/erigon/cl/utils" - "github.com/erigontech/erigon/common" ) -type refreshStateForkGraph struct { - fork_graph.ForkGraph - latest *state.CachingBeaconState - err error -} - -func (f *refreshStateForkGraph) GetState(common.Hash, bool) (*state.CachingBeaconState, error) { - return f.latest, f.err -} - // A response's decoded schema comes from the peer-chosen fork digest, so it is // independent of the slot the block claims. Gloas removed ExecutionPayload and // BlobKzgCommitments from BeaconBody, so a Gloas-decoded block whose slot maps @@ -69,24 +55,3 @@ func TestOnBlockRejectsForkSchemaSlotMismatch(t *testing.T) { err := store.OnBlock(context.Background(), mismatched, false, true, true) require.ErrorIs(t, err, ErrForkSchemaSlotMismatch) } - -func TestRefreshBlockStateAfterPayloadValidationUsesLatestState(t *testing.T) { - latest := state.New(&clparams.MainnetBeaconConfig) - latest.SetSlot(12) - store := &ForkChoiceStore{forkGraph: &refreshStateForkGraph{latest: latest}} - - got, err := store.refreshBlockStateAfterPayloadValidation(common.Hash{1}) - require.NoError(t, err) - require.Same(t, latest, got) -} - -func TestRefreshBlockStateAfterPayloadValidationRejectsMissingState(t *testing.T) { - store := &ForkChoiceStore{forkGraph: &refreshStateForkGraph{}} - _, err := store.refreshBlockStateAfterPayloadValidation(common.Hash{1}) - require.Error(t, err) - - expected := errors.New("read failed") - store.forkGraph = &refreshStateForkGraph{err: expected} - _, err = store.refreshBlockStateAfterPayloadValidation(common.Hash{1}) - require.ErrorIs(t, err, expected) -} diff --git a/cl/phase1/forkchoice/on_execution_payload.go b/cl/phase1/forkchoice/on_execution_payload.go index 49077375909..949abc0134b 100644 --- a/cl/phase1/forkchoice/on_execution_payload.go +++ b/cl/phase1/forkchoice/on_execution_payload.go @@ -28,10 +28,10 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/fork" - "github.com/erigontech/erigon/cl/monitor" "github.com/erigontech/erigon/cl/persistence/beacon_indicies" "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" + "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" "github.com/erigontech/erigon/cl/transition" "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/cl/utils/bls" @@ -41,7 +41,7 @@ import ( "github.com/erigontech/erigon/db/kv" ) -// errELBehind is returned by validatePayloadWithEL when the EL cannot process +// errELBehind is returned by validatePayloadWithELLocked when the EL cannot process // the payload because it hasn't caught up yet (e.g. parent block not available). // applyEnvelope treats this as non-fatal: it proceeds with persisting the envelope // and queues the execution block for later EL insertion. @@ -51,11 +51,18 @@ var ( ) type pendingExecutionPayloadEnvelope struct { - root common.Hash - envelope *cltypes.SignedExecutionPayloadEnvelope - local bool + root common.Hash + entry *pendingExecutionPayloadEnvelopeEntry + local bool } +type pendingExecutionPayloadEnvelopeEntry struct { + envelope *cltypes.SignedExecutionPayloadEnvelope + createdAt time.Time +} + +const pendingExecutionPayloadEnvelopeExpiry = 3 * time.Minute + // validateEnvelopeAgainstBlock validates the envelope against the block and state. // This includes: // - bid matching (slot, builder_index, block_hash) @@ -233,9 +240,9 @@ func (f *ForkChoiceStore) checkDataAvailability( return nil } -// validatePayloadWithEL validates the execution payload with the execution layer engine. +// validatePayloadWithELLocked validates the payload while preserving f.mu ownership. // Called before ProcessExecutionPayloadEnvelope verification. -func (f *ForkChoiceStore) validatePayloadWithEL( +func (f *ForkChoiceStore) validatePayloadWithELLocked( ctx context.Context, envelope *cltypes.ExecutionPayloadEnvelope, block *cltypes.SignedBeaconBlock, @@ -285,10 +292,8 @@ func (f *ForkChoiceStore) validatePayloadWithEL( } // Call NewPayload to validate execution payload with EL - timeStartExec := time.Now() parentBlockRoot := block.Block.ParentRoot - payloadStatus, err := f.newPayloadWithoutForkChoiceLock(ctx, common.Hash(validationKey), envelope.Payload, &parentBlockRoot, versionedHashes, executionRequestsList) - monitor.ObserveNewPayloadTime(timeStartExec) + payloadStatus, err := f.newPayloadLocked(ctx, common.Hash(validationKey), envelope.Payload, &parentBlockRoot, versionedHashes, executionRequestsList) log.Trace("[validatePayloadWithEL] NewPayload", "status", payloadStatus, "beaconBlockRoot", beaconBlockRoot) // Track payload status and gas limit by execution block hash for parent payload validation @@ -332,7 +337,8 @@ func (f *ForkChoiceStore) validatePayloadWithEL( return payloadStatus, nil } -func (f *ForkChoiceStore) newPayloadWithoutForkChoiceLock( +// newPayloadLocked requires f.mu and reacquires it after the engine call. +func (f *ForkChoiceStore) newPayloadLocked( ctx context.Context, beaconBlockRoot common.Hash, payload *cltypes.Eth1Block, @@ -350,28 +356,66 @@ func (f *ForkChoiceStore) newPayloadWithoutForkChoiceLock( } func (f *ForkChoiceStore) lockEnvelopeOwner(blockRoot common.Hash) func() { + unlock, err := f.lockEnvelopeOwnerContext(context.Background(), blockRoot) + if err != nil { + panic(err) + } + return unlock +} + +func (f *ForkChoiceStore) lockEnvelopeOwnerContext(ctx context.Context, blockRoot common.Hash) (func(), error) { f.envelopeOwnersMu.Lock() if f.envelopeOwners == nil { f.envelopeOwners = make(map[common.Hash]*envelopeOwner) } owner := f.envelopeOwners[blockRoot] if owner == nil { - owner = &envelopeOwner{} + owner = &envelopeOwner{slot: make(chan struct{}, 1)} f.envelopeOwners[blockRoot] = owner } owner.refs++ f.envelopeOwnersMu.Unlock() - owner.mu.Lock() + select { + case owner.slot <- struct{}{}: + if err := ctx.Err(); err != nil { + <-owner.slot + f.releaseEnvelopeOwnerReference(blockRoot, owner) + return nil, err + } + case <-ctx.Done(): + f.releaseEnvelopeOwnerReference(blockRoot, owner) + return nil, ctx.Err() + } return func() { - owner.mu.Unlock() - f.envelopeOwnersMu.Lock() - owner.refs-- - if owner.refs == 0 { - delete(f.envelopeOwners, blockRoot) + <-owner.slot + f.releaseEnvelopeOwnerReference(blockRoot, owner) + }, nil +} + +func (f *ForkChoiceStore) releaseEnvelopeOwnerReference(blockRoot common.Hash, owner *envelopeOwner) { + f.envelopeOwnersMu.Lock() + owner.refs-- + if owner.refs == 0 && f.envelopeOwners[blockRoot] == owner { + delete(f.envelopeOwners, blockRoot) + } + f.envelopeOwnersMu.Unlock() +} + +func (f *ForkChoiceStore) acquirePendingEnvelopeRetry(ctx context.Context) (func(), error) { + f.pendingEnvelopeRetryOnce.Do(func() { + f.pendingEnvelopeRetrySlot = make(chan struct{}, 1) + }) + select { + case f.pendingEnvelopeRetrySlot <- struct{}{}: + if err := ctx.Err(); err != nil { + <-f.pendingEnvelopeRetrySlot + return nil, err } - f.envelopeOwnersMu.Unlock() + case <-ctx.Done(): + return nil, ctx.Err() } + return func() { <-f.pendingEnvelopeRetrySlot }, nil } // RetryPendingExecutionPayloadEnvelopes retries a bounded, fair batch of envelopes retained by fork choice. @@ -380,10 +424,13 @@ func (f *ForkChoiceStore) RetryPendingExecutionPayloadEnvelopes(ctx context.Cont return 0 } - f.pendingEnvelopeRetryMu.Lock() - defer f.pendingEnvelopeRetryMu.Unlock() + releaseRetry, err := f.acquirePendingEnvelopeRetry(ctx) + if err != nil { + return 0 + } + defer releaseRetry() - candidates := f.pendingExecutionPayloadEnvelopeCandidates(limit) + candidates := f.pendingExecutionPayloadEnvelopeCandidates(ctx, limit) attempted := 0 for _, candidate := range candidates { if ctx.Err() != nil { @@ -393,21 +440,25 @@ func (f *ForkChoiceStore) RetryPendingExecutionPayloadEnvelopes(ctx context.Cont f.pendingEnvelopeRetryLocal = !candidate.local var err error if candidate.local { - err = f.ApplyLocalSelfBuildEnvelope(ctx, candidate.envelope) + err = f.ApplyLocalSelfBuildEnvelope(ctx, candidate.entry.envelope) } else { - err = f.OnExecutionPayload(ctx, candidate.envelope, true, true) + err = f.OnExecutionPayload(ctx, candidate.entry.envelope, true, true) } if err != nil && !errors.Is(err, errExecutionPayloadInvalid) { - f.rotatePendingExecutionPayloadEnvelope(candidate) + if err := f.rotatePendingExecutionPayloadEnvelope(ctx, candidate); err != nil { + break + } log.Debug("pending execution payload envelope retry deferred", "blockRoot", candidate.root, "local", candidate.local, "err", err) continue } - f.removePendingExecutionPayloadEnvelope(candidate) + if err := f.removePendingExecutionPayloadEnvelope(ctx, candidate); err != nil { + break + } } return attempted } -func (f *ForkChoiceStore) pendingExecutionPayloadEnvelopeCandidates(limit int) []pendingExecutionPayloadEnvelope { +func (f *ForkChoiceStore) pendingExecutionPayloadEnvelopeCandidates(ctx context.Context, limit int) []pendingExecutionPayloadEnvelope { var gossipRoots, localRoots []common.Hash if f.pendingEnvelopes != nil { gossipRoots = f.pendingEnvelopes.Keys() @@ -427,28 +478,51 @@ func (f *ForkChoiceStore) pendingExecutionPayloadEnvelopeCandidates(limit int) [ } var root common.Hash - var envelope *cltypes.SignedExecutionPayloadEnvelope + var entry *pendingExecutionPayloadEnvelopeEntry var ok bool if local { root = localRoots[localIndex] localIndex++ - envelope, ok = f.pendingLocalSelfBuildEnvelopes.Peek(root) + entry, ok = f.pendingLocalSelfBuildEnvelopes.Peek(root) } else { root = gossipRoots[gossipIndex] gossipIndex++ - envelope, ok = f.pendingEnvelopes.Peek(root) + entry, ok = f.pendingEnvelopes.Peek(root) } if !ok { continue } - candidates = append(candidates, pendingExecutionPayloadEnvelope{root: root, envelope: envelope, local: local}) + if f.pendingExecutionPayloadEnvelopeExpired(root, entry) { + if err := f.removePendingExecutionPayloadEnvelope(ctx, pendingExecutionPayloadEnvelope{root: root, entry: entry, local: local}); err != nil { + return candidates + } + continue + } + candidates = append(candidates, pendingExecutionPayloadEnvelope{root: root, entry: entry, local: local}) preferLocal = !local } return candidates } -func (f *ForkChoiceStore) removePendingExecutionPayloadEnvelope(candidate pendingExecutionPayloadEnvelope) { - unlockOwner := f.lockEnvelopeOwner(candidate.root) +func (f *ForkChoiceStore) pendingExecutionPayloadEnvelopeExpired(root common.Hash, entry *pendingExecutionPayloadEnvelopeEntry) bool { + if entry == nil || time.Since(entry.createdAt) > pendingExecutionPayloadEnvelopeExpiry { + return true + } + checkpoint := f.finalizedCheckpoint.Load() + if checkpoint == nil || f.beaconCfg == nil || f.beaconCfg.SlotsPerEpoch == 0 || f.forkGraph == nil { + return false + } + block, ok := f.GetBlock(root) + finalized := checkpoint.(solid.Checkpoint) + finalizedSlot := f.computeStartSlotAtEpoch(finalized.Epoch) + return ok && block != nil && block.Block != nil && block.Block.Slot <= finalizedSlot +} + +func (f *ForkChoiceStore) removePendingExecutionPayloadEnvelope(ctx context.Context, candidate pendingExecutionPayloadEnvelope) error { + unlockOwner, err := f.lockEnvelopeOwnerContext(ctx, candidate.root) + if err != nil { + return err + } defer unlockOwner() cache := f.pendingEnvelopes @@ -456,16 +530,20 @@ func (f *ForkChoiceStore) removePendingExecutionPayloadEnvelope(candidate pendin cache = f.pendingLocalSelfBuildEnvelopes } if cache == nil { - return + return nil } current, ok := cache.Peek(candidate.root) - if ok && current == candidate.envelope { + if ok && current == candidate.entry { cache.Remove(candidate.root) } + return nil } -func (f *ForkChoiceStore) rotatePendingExecutionPayloadEnvelope(candidate pendingExecutionPayloadEnvelope) { - unlockOwner := f.lockEnvelopeOwner(candidate.root) +func (f *ForkChoiceStore) rotatePendingExecutionPayloadEnvelope(ctx context.Context, candidate pendingExecutionPayloadEnvelope) error { + unlockOwner, err := f.lockEnvelopeOwnerContext(ctx, candidate.root) + if err != nil { + return err + } defer unlockOwner() cache := f.pendingEnvelopes @@ -473,12 +551,13 @@ func (f *ForkChoiceStore) rotatePendingExecutionPayloadEnvelope(candidate pendin cache = f.pendingLocalSelfBuildEnvelopes } if cache == nil { - return + return nil } current, ok := cache.Peek(candidate.root) - if ok && current == candidate.envelope { + if ok && current == candidate.entry { cache.Get(candidate.root) } + return nil } func (f *ForkChoiceStore) retainPendingExecutionPayloadEnvelope(signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, local bool) { @@ -490,7 +569,12 @@ func (f *ForkChoiceStore) retainPendingExecutionPayloadEnvelope(signedEnvelope * cache = f.pendingLocalSelfBuildEnvelopes } if cache != nil { - cache.Add(signedEnvelope.Message.BeaconBlockRoot, signedEnvelope) + root := signedEnvelope.Message.BeaconBlockRoot + createdAt := time.Now() + if current, ok := cache.Peek(root); ok && current != nil { + createdAt = current.createdAt + } + cache.Add(root, &pendingExecutionPayloadEnvelopeEntry{envelope: signedEnvelope, createdAt: createdAt}) } } @@ -511,21 +595,29 @@ func (f *ForkChoiceStore) persistEnvelopeIndices(ctx context.Context, blockRoot }); err != nil { return err } - return f.forkGraph.MarkEnvelopeIndicesCommitted(blockRoot) + persistence, err := f.envelopePersistenceStore() + if err != nil { + return err + } + return persistence.MarkEnvelopeIndicesCommitted(blockRoot) } func (f *ForkChoiceStore) reconcilePendingEnvelopeIndices(ctx context.Context) error { if f.db == nil { return nil } - roots, err := f.forkGraph.PendingEnvelopeIndexRoots() + persistence, err := f.envelopePersistenceStore() + if err != nil { + return err + } + roots, err := persistence.PendingEnvelopeIndexRoots() if err != nil { return err } for _, root := range roots { envelope, err := f.forkGraph.ReadEnvelopeFromDisk(root) if os.IsNotExist(err) { - if err := f.forkGraph.MarkEnvelopeIndicesCommitted(root); err != nil { + if err := persistence.MarkEnvelopeIndicesCommitted(root); err != nil { return err } continue @@ -546,7 +638,22 @@ func (f *ForkChoiceStore) reconcilePendingEnvelopeIndices(ctx context.Context) e func (f *ForkChoiceStore) prepareEnvelopeWithoutForkChoiceLock(blockRoot common.Hash, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) (func() error, error) { f.mu.Unlock() defer f.mu.Lock() - return f.forkGraph.PrepareEnvelopeOnDisk(blockRoot, signedEnvelope, true) + persistence, err := f.envelopePersistenceStore() + if err != nil { + return nil, err + } + return persistence.PrepareEnvelopeOnDisk(blockRoot, signedEnvelope, true) +} + +func (f *ForkChoiceStore) envelopePersistenceStore() (fork_graph.EnvelopePersistence, error) { + if f.envelopePersistence != nil { + return f.envelopePersistence, nil + } + persistence, ok := f.forkGraph.(fork_graph.EnvelopePersistence) + if !ok { + return nil, errors.New("fork graph does not support envelope persistence") + } + return persistence, nil } // applyEnvelope processes the envelope under f.mu: validates, verifies with CL and EL, @@ -560,7 +667,10 @@ func (f *ForkChoiceStore) applyEnvelope(ctx context.Context, signedEnvelope *clt log.Warn("[applyEnvelope] received signed envelope with nil message") return false, errors.New("signed envelope has nil message") } - unlockOwner := f.lockEnvelopeOwner(signedEnvelope.Message.BeaconBlockRoot) + unlockOwner, err := f.lockEnvelopeOwnerContext(ctx, signedEnvelope.Message.BeaconBlockRoot) + if err != nil { + return false, err + } defer unlockOwner() return f.applyEnvelopeOwned(ctx, signedEnvelope, checkBlobData, validatePayload) } @@ -608,7 +718,7 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop // Block hasn't arrived yet, queue envelope for later processing. // Per spec: assert envelope.beacon_block_root in store.block_states // Return an error so callers can distinguish "queued" from "applied". - f.pendingEnvelopes.Add(beaconBlockRoot, signedEnvelope) + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, false) log.Trace("OnExecutionPayload: block not found, queuing envelope for later", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block state not found for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } @@ -616,7 +726,7 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop // Get the block to verify it exists block, ok := f.forkGraph.GetBlock(beaconBlockRoot) if !ok || block == nil { - f.pendingEnvelopes.Add(beaconBlockRoot, signedEnvelope) + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, false) log.Trace("OnExecutionPayload: block not found in fork graph, queuing envelope", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block not found in fork graph for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } @@ -639,17 +749,18 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop var elBehind bool var payloadValidated bool if validatePayload { - payloadStatus, validationErr := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)) + payloadStatus, validationErr := f.validatePayloadWithELLocked(ctx, envelope, block, common.Hash(beaconBlockRoot)) payloadValidated = payloadStatus == execution_client.PayloadStatusValidated if validationErr != nil { - if errors.Is(validationErr, errELBehind) { + switch { + case errors.Is(validationErr, errELBehind): // EL is behind (e.g. parent block not yet available after forward sync). // Proceed with persisting the envelope so HasEnvelope() returns true. // The execution block will be fed to EL via blockCollector on the next Flush(). elBehind = true - } else if payloadStatus == execution_client.PayloadStatusInvalidated { + case payloadStatus == execution_client.PayloadStatusInvalidated: return false, fmt.Errorf("%w: %w", errExecutionPayloadInvalid, validationErr) - } else { + default: return false, validationErr } } @@ -740,7 +851,11 @@ func (f *ForkChoiceStore) StoreAnchorEnvelope(blockRoot common.Hash, signedEnvel acceptedEnvelope = persistedEnvelope.Message } else { var err error - publish, err = f.forkGraph.PrepareEnvelopeOnDisk(blockRoot, signedEnvelope, false) + persistence, persistenceErr := f.envelopePersistenceStore() + if persistenceErr != nil { + return persistenceErr + } + publish, err = persistence.PrepareEnvelopeOnDisk(blockRoot, signedEnvelope, false) if err != nil { return fmt.Errorf("StoreAnchorEnvelope: failed to dump envelope: %w", err) } @@ -781,10 +896,16 @@ func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope if signedEnvelope.Message.Payload == nil || signedEnvelope.Message.ExecutionRequests == nil { return fmt.Errorf("%w: incomplete execution payload envelope", errExecutionPayloadInvalid) } + if signedEnvelope.Signature == common.Bytes96(bls.InfiniteSignature) { + return fmt.Errorf("%w: unauthenticated execution payload envelope", errExecutionPayloadInvalid) + } envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot - unlockOwner := f.lockEnvelopeOwner(beaconBlockRoot) + unlockOwner, err := f.lockEnvelopeOwnerContext(ctx, beaconBlockRoot) + if err != nil { + return err + } defer unlockOwner() // Process envelope under f.mu; DB index write happens after unlock to avoid @@ -811,23 +932,50 @@ func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope if persistedEnvelope == nil || persistedEnvelope.Message == nil || persistedEnvelope.Message.Payload == nil || persistedEnvelope.Message.BeaconBlockRoot != beaconBlockRoot { return fmt.Errorf("%w: OnExecutionPayload: invalid persisted envelope", errExecutionPayloadInvalid) } - callerIdentity, identityErr := signedEnvelope.HashSSZ() + callerIdentity, identityErr := signedEnvelope.Message.HashSSZ() + if identityErr != nil { + return fmt.Errorf("%w: OnExecutionPayload: failed to hash caller envelope message: %w", errExecutionPayloadInvalid, identityErr) + } + persistedIdentity, identityErr := persistedEnvelope.Message.HashSSZ() + if identityErr != nil { + return fmt.Errorf("%w: OnExecutionPayload: failed to hash persisted envelope message: %w", errExecutionPayloadInvalid, identityErr) + } + if callerIdentity != persistedIdentity { + return fmt.Errorf("%w: OnExecutionPayload: caller does not match persisted envelope message", errExecutionPayloadInvalid) + } + callerWrapperIdentity, identityErr := signedEnvelope.HashSSZ() if identityErr != nil { return fmt.Errorf("%w: OnExecutionPayload: failed to hash caller envelope: %w", errExecutionPayloadInvalid, identityErr) } - persistedIdentity, identityErr := persistedEnvelope.HashSSZ() + persistedWrapperIdentity, identityErr := persistedEnvelope.HashSSZ() if identityErr != nil { return fmt.Errorf("%w: OnExecutionPayload: failed to hash persisted envelope: %w", errExecutionPayloadInvalid, identityErr) } - if callerIdentity != persistedIdentity { - return fmt.Errorf("%w: OnExecutionPayload: caller does not match persisted envelope", errExecutionPayloadInvalid) + wrapperChanged := callerWrapperIdentity != persistedWrapperIdentity + if wrapperChanged && !validatePayload { + return fmt.Errorf("%w: OnExecutionPayload: signature replacement requires validation", errExecutionPayloadInvalid) } - acceptedEnvelope = persistedEnvelope.Message if validatePayload { - if err := f.validatePersistedEnvelopeOwned(ctx, persistedEnvelope, checkBlobData); err != nil { + if err := f.validatePersistedEnvelopeOwned(ctx, signedEnvelope, checkBlobData); err != nil { return err } } + if wrapperChanged { + persistence, persistenceErr := f.envelopePersistenceStore() + if persistenceErr != nil { + return persistenceErr + } + publishReplacement, err := persistence.PrepareEnvelopeOnDisk(beaconBlockRoot, signedEnvelope, true) + if err != nil { + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, false) + return fmt.Errorf("OnExecutionPayload: failed to prepare authenticated envelope replacement: %w", err) + } + if err := publishReplacement(); err != nil { + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, false) + return fmt.Errorf("OnExecutionPayload: failed to publish authenticated envelope replacement: %w", err) + } + } + acceptedEnvelope = signedEnvelope.Message } // Write execution block indices outside f.mu. @@ -861,7 +1009,10 @@ func (f *ForkChoiceStore) ApplyLocalSelfBuildEnvelope(ctx context.Context, signe envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot - unlockOwner := f.lockEnvelopeOwner(beaconBlockRoot) + unlockOwner, err := f.lockEnvelopeOwnerContext(ctx, beaconBlockRoot) + if err != nil { + return err + } defer unlockOwner() applied, err := f.applyLocalSelfBuildEnvelopeOwned(ctx, signedEnvelope) @@ -909,7 +1060,10 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelope(ctx context.Context, signe if signedEnvelope.Message == nil { return false, errors.New("signed envelope has nil message") } - unlockOwner := f.lockEnvelopeOwner(signedEnvelope.Message.BeaconBlockRoot) + unlockOwner, err := f.lockEnvelopeOwnerContext(ctx, signedEnvelope.Message.BeaconBlockRoot) + if err != nil { + return false, err + } defer unlockOwner() return f.applyLocalSelfBuildEnvelopeOwned(ctx, signedEnvelope) } @@ -945,14 +1099,14 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to get block state: %w", err) } if blockState == nil { - f.pendingLocalSelfBuildEnvelopes.Add(beaconBlockRoot, signedEnvelope) + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, true) log.Trace("applyLocalSelfBuildEnvelopeLocked: block not found, queuing envelope for later", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block state not found for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } block, ok := f.forkGraph.GetBlock(beaconBlockRoot) if !ok || block == nil { - f.pendingLocalSelfBuildEnvelopes.Add(beaconBlockRoot, signedEnvelope) + f.retainPendingExecutionPayloadEnvelope(signedEnvelope, true) log.Trace("applyLocalSelfBuildEnvelopeLocked: block not found in fork graph, queuing envelope", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block not found in fork graph for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } @@ -961,13 +1115,14 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, // Validate payload with EL (NewPayload). var elBehind bool - payloadStatus, validationErr := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)) + payloadStatus, validationErr := f.validatePayloadWithELLocked(ctx, envelope, block, common.Hash(beaconBlockRoot)) if validationErr != nil { - if errors.Is(validationErr, errELBehind) { + switch { + case errors.Is(validationErr, errELBehind): elBehind = true - } else if payloadStatus == execution_client.PayloadStatusInvalidated { + case payloadStatus == execution_client.PayloadStatusInvalidated: return false, fmt.Errorf("%w: %w", errExecutionPayloadInvalid, validationErr) - } else { + default: return false, validationErr } } diff --git a/cl/phase1/forkchoice/on_execution_payload_test.go b/cl/phase1/forkchoice/on_execution_payload_test.go index f2ace653823..8cf968a54aa 100644 --- a/cl/phase1/forkchoice/on_execution_payload_test.go +++ b/cl/phase1/forkchoice/on_execution_payload_test.go @@ -19,6 +19,7 @@ package forkchoice import ( "context" "errors" + "sync" "sync/atomic" "testing" "time" @@ -34,6 +35,7 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" + "github.com/erigontech/erigon/cl/fork" "github.com/erigontech/erigon/cl/persistence/beacon_indicies" "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" @@ -41,6 +43,7 @@ import ( "github.com/erigontech/erigon/cl/phase1/forkchoice/optimistic" "github.com/erigontech/erigon/cl/phase1/forkchoice/public_keys_registry" "github.com/erigontech/erigon/cl/pool" + "github.com/erigontech/erigon/cl/utils/bls" "github.com/erigontech/erigon/cl/validator/validator_params" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/hexutil" @@ -54,6 +57,21 @@ type failFirstUpdateDB struct { updates atomic.Int32 } +type ownerAdmissionContext struct { + context.Context + observed chan struct{} + observeAt int32 + calls atomic.Int32 + once sync.Once +} + +func (c *ownerAdmissionContext) Done() <-chan struct{} { + if c.calls.Add(1) == c.observeAt { + c.once.Do(func() { close(c.observed) }) + } + return c.Context.Done() +} + func (db *failFirstUpdateDB) Update(ctx context.Context, f func(kv.RwTx) error) error { if db.updates.Add(1) == 1 { return errors.New("injected index failure") @@ -281,7 +299,7 @@ func TestCheckDataAvailability_NoBlobs(t *testing.T) { require.NoError(t, err) } -// TestValidatePayloadWithEL_NoEngine tests that validatePayloadWithEL returns nil when there's no engine +// TestValidatePayloadWithEL_NoEngine tests that payload validation returns nil when there's no engine. func TestValidatePayloadWithEL_NoEngine(t *testing.T) { cfg := &clparams.MainnetBeaconConfig f := &ForkChoiceStore{ @@ -301,7 +319,7 @@ func TestValidatePayloadWithEL_NoEngine(t *testing.T) { }, } - _, err := f.validatePayloadWithEL(context.TODO(), envelope, block, common.Hash{}) + _, err := f.validatePayloadWithELLocked(context.TODO(), envelope, block, common.Hash{}) require.NoError(t, err) } @@ -316,7 +334,7 @@ func TestOnExecutionPayloadRepairsIndicesAfterPriorWriteFailure(t *testing.T) { envelope.Message.Payload.BlockNumber = 42 db := &failFirstUpdateDB{RwDB: memdb.NewTestDB(t, dbcfg.ChainDB)} - pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + pending, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) require.NoError(t, err) f := &ForkChoiceStore{ forkGraph: payloadVoteForkGraph{hasEnvelope: true, envelope: envelope}, @@ -351,7 +369,7 @@ func TestApplyLocalSelfBuildEnvelopeRepairsIndicesAfterPriorWriteFailure(t *test envelope.Message.Payload.BlockNumber = 42 db := &failFirstUpdateDB{RwDB: memdb.NewTestDB(t, dbcfg.ChainDB)} - pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + pending, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) require.NoError(t, err) f := &ForkChoiceStore{ forkGraph: payloadVoteForkGraph{hasEnvelope: true, envelope: envelope}, @@ -375,8 +393,10 @@ func TestApplyLocalSelfBuildEnvelopeRepairsIndicesAfterPriorWriteFailure(t *test func TestReconcilePendingEnvelopeIndicesAfterRestart(t *testing.T) { cfg := &clparams.MainnetBeaconConfig anchorState := state.New(cfg) - fs := afero.NewMemMapFs() + baseDir := t.TempDir() + fs := afero.NewBasePathFs(afero.NewOsFs(), baseDir) graph := fork_graph.NewForkGraphDisk(anchorState, nil, fs, beacon_router_configuration.RouterConfiguration{}) + persistence := graph.(fork_graph.EnvelopePersistence) root := common.HexToHash("0x1234") executionHash := common.HexToHash("0xabcd") envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} @@ -398,9 +418,13 @@ func TestReconcilePendingEnvelopeIndicesAfterRestart(t *testing.T) { orphanRoot := common.HexToHash("0x9999") orphanEnvelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} orphanEnvelope.Message.BeaconBlockRoot = orphanRoot - _, err := graph.PrepareEnvelopeOnDisk(orphanRoot, orphanEnvelope, false) + _, err := persistence.PrepareEnvelopeOnDisk(orphanRoot, orphanEnvelope, false) require.NoError(t, err) + restartedFS := afero.NewBasePathFs(afero.NewOsFs(), baseDir) + graph = fork_graph.NewForkGraphDisk(anchorState, nil, restartedFS, beacon_router_configuration.RouterConfiguration{}) + persistence = graph.(fork_graph.EnvelopePersistence) + _, err = NewForkChoiceStore( nil, anchorState, @@ -434,7 +458,7 @@ func TestReconcilePendingEnvelopeIndicesAfterRestart(t *testing.T) { require.Nil(t, orphanNumber) return nil })) - pendingRoots, err := graph.PendingEnvelopeIndexRoots() + pendingRoots, err := persistence.PendingEnvelopeIndexRoots() require.NoError(t, err) require.Empty(t, pendingRoots) } @@ -555,6 +579,89 @@ func TestOnExecutionPayloadRejectsIncompleteCallerBeforePersistedFallback(t *tes require.ErrorContains(t, err, "incomplete execution payload envelope") } +func TestOnExecutionPayloadReplacesLocalSignatureWithAuthenticatedEnvelope(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + blockState := state.New(&cfg) + blockState.SetVersion(clparams.GloasVersion) + blockState.SetSlot(1) + parentRoot := common.HexToHash("0x2222") + stateRoot := common.HexToHash("0x1111") + header := &cltypes.BeaconBlockHeader{Slot: 1, ParentRoot: parentRoot, ProposerIndex: 0} + blockState.SetLatestBlockHeader(header) + blockState.SetPreviousStateRoot(stateRoot) + headerWithStateRoot := *header + headerWithStateRoot.Root = stateRoot + blockRoot, err := headerWithStateRoot.HashSSZ() + require.NoError(t, err) + + privateKey, err := bls.NewPrivateKeyFromIKM([]byte("01234567890123456789012345678901")) + require.NoError(t, err) + pubkey := common.Bytes48(bls.CompressPublicKey(privateKey.PublicKey())) + blockState.AddValidator(solid.NewValidatorFromParameters(pubkey, common.Hash{}, cfg.MaxEffectiveBalance, false, 0, 0, cfg.FarFutureEpoch, cfg.FarFutureEpoch), cfg.MaxEffectiveBalance) + + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&cfg)} + envelope.Message.BuilderIndex = clparams.BuilderIndexSelfBuild + envelope.Message.BeaconBlockRoot = blockRoot + envelope.Message.ParentBeaconBlockRoot = parentRoot + envelope.Message.Payload.SlotNumber = 1 + envelope.Message.Payload.BlockHash = common.HexToHash("0xabcd") + envelope.Message.Payload.ParentHash = common.HexToHash("0x3333") + envelope.Message.Payload.Time = cfg.SecondsPerSlot + envelope.Message.Payload.Withdrawals = solid.NewStaticListSSZ[*cltypes.Withdrawal](int(cfg.MaxWithdrawalsPerPayload), 44) + requestsRoot, err := envelope.Message.ExecutionRequests.HashSSZ() + require.NoError(t, err) + bid := &cltypes.ExecutionPayloadBid{ + BuilderIndex: envelope.Message.BuilderIndex, + PrevRandao: envelope.Message.Payload.PrevRandao, + GasLimit: envelope.Message.Payload.GasLimit, + BlockHash: envelope.Message.Payload.BlockHash, + ExecutionRequestsRoot: requestsRoot, + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + } + blockState.SetLatestExecutionPayloadBid(bid) + blockState.SetLatestBlockHash(envelope.Message.Payload.ParentHash) + blockState.SetPayloadExpectedWithdrawals(envelope.Message.Payload.Withdrawals) + body := cltypes.NewBeaconBody(&cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{Message: bid, Signature: common.Bytes96(bls.InfiniteSignature)} + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 1, ParentRoot: parentRoot, StateRoot: stateRoot, Body: body}} + domain, err := blockState.GetDomain(cfg.DomainBeaconBuilder, state.GetEpochAtSlot(&cfg, 1)) + require.NoError(t, err) + signingRoot, err := fork.ComputeSigningRoot(envelope.Message, domain) + require.NoError(t, err) + copy(envelope.Signature[:], privateKey.Sign(signingRoot[:]).Bytes()) + + localEnvelope := envelope.Clone().(*cltypes.SignedExecutionPayloadEnvelope) + localEnvelope.Signature = common.Bytes96(bls.InfiniteSignature) + var preparedEnvelope *cltypes.SignedExecutionPayloadEnvelope + var prepareRequiresBlock bool + eth2Roots, err := lru.New[common.Hash, common.Hash](16) + require.NoError(t, err) + f := &ForkChoiceStore{ + beaconCfg: &cfg, + forkGraph: payloadVoteForkGraph{ + hasEnvelope: true, + envelope: localEnvelope, + block: block, + blockState: blockState, + preparedEnvelope: &preparedEnvelope, + prepareRequiresBlock: &prepareRequiresBlock, + }, + eth2Roots: eth2Roots, + } + + require.NoError(t, f.OnExecutionPayload(t.Context(), envelope, false, true)) + require.Same(t, envelope, preparedEnvelope) + require.True(t, prepareRequiresBlock) +} + +func TestOnExecutionPayloadRejectsInfiniteSignature(t *testing.T) { + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Signature = common.Bytes96(bls.InfiniteSignature) + + err := (&ForkChoiceStore{}).OnExecutionPayload(t.Context(), envelope, false, false) + require.ErrorContains(t, err, "unauthenticated") +} + func TestPendingLocalSelfBuildEnvelopeSurvivesCanceledApplyAndRetries(t *testing.T) { cfg := &clparams.MainnetBeaconConfig blockState := state.New(cfg) @@ -605,7 +712,7 @@ func TestPendingLocalSelfBuildEnvelopeSurvivesCanceledApplyAndRetries(t *testing engine := execution_client.NewMockExecutionEngine(ctrl) engine.EXPECT().NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(execution_client.PayloadStatusNone, context.Canceled) engine.EXPECT().NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(execution_client.PayloadStatusValidated, nil).Times(2) - pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + pending, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) require.NoError(t, err) verified, err := lru.New[common.Hash, struct{}](16) require.NoError(t, err) @@ -701,21 +808,21 @@ func TestPendingLocalSelfBuildEnvelopeSurvivesCanceledApplyAndRetries(t *testing } func TestRetryPendingExecutionPayloadEnvelopesIsBoundedAndFair(t *testing.T) { - gossip, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + gossip, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) require.NoError(t, err) - local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + local, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) require.NoError(t, err) f := &ForkChoiceStore{ pendingEnvelopes: gossip, pendingLocalSelfBuildEnvelopes: local, } for i := byte(1); i <= 3; i++ { - gossip.Add(common.Hash{i}, &cltypes.SignedExecutionPayloadEnvelope{ + gossip.Add(common.Hash{i}, &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now(), envelope: &cltypes.SignedExecutionPayloadEnvelope{ Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: common.Hash{i}}, - }) - local.Add(common.Hash{i + 3}, &cltypes.SignedExecutionPayloadEnvelope{ + }}) + local.Add(common.Hash{i + 3}, &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now(), envelope: &cltypes.SignedExecutionPayloadEnvelope{ Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: common.Hash{i + 3}}, - }) + }}) } canceled, cancel := context.WithCancel(t.Context()) @@ -735,20 +842,20 @@ func TestRetryPendingExecutionPayloadEnvelopesIsBoundedAndFair(t *testing.T) { } func TestRetryPendingExecutionPayloadEnvelopesRotatesDeferredWork(t *testing.T) { - gossip, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](16) + gossip, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) require.NoError(t, err) f := &ForkChoiceStore{ forkGraph: payloadVoteForkGraph{}, pendingEnvelopes: gossip, } for i := byte(1); i <= 3; i++ { - gossip.Add(common.Hash{i}, &cltypes.SignedExecutionPayloadEnvelope{ + gossip.Add(common.Hash{i}, &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now(), envelope: &cltypes.SignedExecutionPayloadEnvelope{ Message: &cltypes.ExecutionPayloadEnvelope{ BeaconBlockRoot: common.Hash{i}, Payload: cltypes.NewEth1Block(clparams.GloasVersion, &clparams.MainnetBeaconConfig), ExecutionRequests: cltypes.NewExecutionRequestsWithVersion(&clparams.MainnetBeaconConfig, clparams.GloasVersion), }, - }) + }}) } require.Equal(t, 2, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 2)) @@ -757,6 +864,158 @@ func TestRetryPendingExecutionPayloadEnvelopesRotatesDeferredWork(t *testing.T) require.Equal(t, []common.Hash{{2}, {3}, {1}}, gossip.Keys()) } +func TestRetryPendingExecutionPayloadEnvelopesExpiredWorkDoesNotConsumeLimit(t *testing.T) { + gossip, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) + require.NoError(t, err) + expiredRoot := common.Hash{1} + freshRoot := common.Hash{2} + gossip.Add(expiredRoot, &pendingExecutionPayloadEnvelopeEntry{ + createdAt: time.Now().Add(-pendingExecutionPayloadEnvelopeExpiry - time.Second), + envelope: &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: expiredRoot}}, + }) + gossip.Add(freshRoot, &pendingExecutionPayloadEnvelopeEntry{ + createdAt: time.Now(), + envelope: &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: freshRoot}}, + }) + f := &ForkChoiceStore{pendingEnvelopes: gossip} + + require.Equal(t, 1, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + require.Zero(t, gossip.Len()) +} + +func TestPendingExecutionPayloadEnvelopeExpiryWithoutFinalizedCheckpoint(t *testing.T) { + f := &ForkChoiceStore{} + root := common.Hash{1} + require.False(t, f.pendingExecutionPayloadEnvelopeExpired(root, &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now()})) + require.True(t, f.pendingExecutionPayloadEnvelopeExpired(root, &pendingExecutionPayloadEnvelopeEntry{ + createdAt: time.Now().Add(-pendingExecutionPayloadEnvelopeExpiry - time.Second), + })) +} + +func TestPendingExecutionPayloadEnvelopeFinalizedBoundary(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + entry := &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now()} + f := &ForkChoiceStore{beaconCfg: cfg} + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 1}) + finalizedSlot := cfg.SlotsPerEpoch + + for _, tc := range []struct { + name string + slot uint64 + expired bool + }{ + {name: "before boundary", slot: finalizedSlot - 1, expired: true}, + {name: "exact boundary", slot: finalizedSlot, expired: true}, + {name: "after boundary", slot: finalizedSlot + 1, expired: false}, + } { + t.Run(tc.name, func(t *testing.T) { + f.forkGraph = payloadVoteForkGraph{block: &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: tc.slot}}} + require.Equal(t, tc.expired, f.pendingExecutionPayloadEnvelopeExpired(common.Hash{1}, entry)) + }) + } +} + +func TestRetryPendingExecutionPayloadEnvelopesCancellationWhileOwnerHeld(t *testing.T) { + gossip, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) + require.NoError(t, err) + root := common.Hash{1} + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = root + gossip.Add(root, &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now(), envelope: envelope}) + f := &ForkChoiceStore{pendingEnvelopes: gossip} + unlocksOwner := f.lockEnvelopeOwner(root) + + baseCtx, cancel := context.WithCancel(t.Context()) + ctx := &ownerAdmissionContext{Context: baseCtx, observed: make(chan struct{}), observeAt: 2} + retryDone := make(chan int, 1) + go func() { + retryDone <- f.RetryPendingExecutionPayloadEnvelopes(ctx, 1) + }() + select { + case <-ctx.observed: + case <-time.After(time.Second): + unlocksOwner() + t.Fatal("retry did not reach envelope owner admission") + } + cancel() + select { + case attempted := <-retryDone: + require.Equal(t, 1, attempted) + case <-time.After(time.Second): + unlocksOwner() + t.Fatal("retry ignored cancellation while waiting for envelope owner") + } + require.Equal(t, 1, gossip.Len()) + unlocksOwner() +} + +func TestRetryPendingExecutionPayloadEnvelopesCancellationWhileAnotherRetryActive(t *testing.T) { + gossip, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) + require.NoError(t, err) + root := common.Hash{1} + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = root + gossip.Add(root, &pendingExecutionPayloadEnvelopeEntry{createdAt: time.Now(), envelope: envelope}) + f := &ForkChoiceStore{pendingEnvelopes: gossip} + unlocksOwner := f.lockEnvelopeOwner(root) + + firstBaseCtx, cancelFirst := context.WithCancel(t.Context()) + firstCtx := &ownerAdmissionContext{Context: firstBaseCtx, observed: make(chan struct{}), observeAt: 2} + firstDone := make(chan int, 1) + go func() { firstDone <- f.RetryPendingExecutionPayloadEnvelopes(firstCtx, 1) }() + select { + case <-firstCtx.observed: + case <-time.After(time.Second): + unlocksOwner() + t.Fatal("first retry did not reach envelope owner admission") + } + + secondBaseCtx, cancelSecond := context.WithCancel(t.Context()) + secondCtx := &ownerAdmissionContext{Context: secondBaseCtx, observed: make(chan struct{}), observeAt: 1} + secondDone := make(chan int, 1) + go func() { secondDone <- f.RetryPendingExecutionPayloadEnvelopes(secondCtx, 1) }() + select { + case <-secondCtx.observed: + case <-time.After(time.Second): + cancelFirst() + unlocksOwner() + t.Fatal("second retry did not reach global retry admission") + } + cancelSecond() + select { + case attempted := <-secondDone: + require.Zero(t, attempted) + case <-time.After(time.Second): + cancelFirst() + unlocksOwner() + t.Fatal("second retry ignored cancellation while waiting for global retry admission") + } + require.Equal(t, 1, gossip.Len()) + + cancelFirst() + require.Equal(t, 1, <-firstDone) + unlocksOwner() +} + +func TestRetryPendingExecutionPayloadEnvelopesDropsFinalizedWork(t *testing.T) { + gossip, err := lru.New[common.Hash, *pendingExecutionPayloadEnvelopeEntry](16) + require.NoError(t, err) + root := common.Hash{1} + gossip.Add(root, &pendingExecutionPayloadEnvelopeEntry{ + createdAt: time.Now(), + envelope: &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: root}}, + }) + f := &ForkChoiceStore{ + beaconCfg: &clparams.MainnetBeaconConfig, + forkGraph: payloadVoteForkGraph{block: &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 1}}}, + pendingEnvelopes: gossip, + } + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 1}) + + require.Zero(t, f.RetryPendingExecutionPayloadEnvelopes(t.Context(), 1)) + require.Zero(t, gossip.Len()) +} + func TestEnvelopeOwnershipAndPruneDoNotGloballyExcludeEachOther(t *testing.T) { root := common.HexToHash("0x1234") t.Run("active envelope owner does not block prune", func(t *testing.T) { @@ -874,7 +1133,7 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { go func() { f.mu.Lock() defer f.mu.Unlock() - _, err := f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + _, err := f.validatePayloadWithELLocked(context.Background(), envelope, block, blockRoot) done <- err }() @@ -886,7 +1145,7 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { require.NoError(t, err) } case <-time.After(time.Second): - t.Fatal("validatePayloadWithEL blocked while forkchoice mutex was already held") + t.Fatal("validatePayloadWithELLocked blocked while forkchoice mutex was already held") } require.False(t, f.IsPayloadVerified(blockRoot)) if tt.status == execution_client.PayloadStatusInvalidated { @@ -942,7 +1201,7 @@ func TestValidatePayloadWithELReleasesForkChoiceMuDuringNewPayload(t *testing.T) go func() { f.mu.Lock() defer f.mu.Unlock() - _, err := f.validatePayloadWithEL(context.Background(), envelope, block, common.HexToHash("0x1234")) + _, err := f.validatePayloadWithELLocked(context.Background(), envelope, block, common.HexToHash("0x1234")) validationDone <- err }() <-engineStarted @@ -1020,7 +1279,7 @@ func TestValidatePayloadWithELDoesNotCoalesceDifferentPayloads(t *testing.T) { validate := func(envelope *cltypes.ExecutionPayloadEnvelope) { f.mu.Lock() defer f.mu.Unlock() - _, err := f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + _, err := f.validatePayloadWithELLocked(context.Background(), envelope, block, blockRoot) results <- err } go validate(first) @@ -1063,7 +1322,7 @@ func TestNewPayloadCoalescesSameKey(t *testing.T) { if acquired != nil { close(acquired) } - status, err := f.newPayloadWithoutForkChoiceLock(context.Background(), key, nil, nil, nil, nil) + status, err := f.newPayloadLocked(context.Background(), key, nil, nil, nil, nil) f.mu.Unlock() results <- result{status: status, err: err} } @@ -1101,7 +1360,7 @@ func TestNewPayloadCanceledWaiterDoesNotCancelLeader(t *testing.T) { leaderDone := make(chan error, 1) go func() { f.mu.Lock() - _, err := f.newPayloadWithoutForkChoiceLock(context.Background(), key, nil, nil, nil, nil) + _, err := f.newPayloadLocked(context.Background(), key, nil, nil, nil, nil) f.mu.Unlock() leaderDone <- err }() @@ -1110,7 +1369,7 @@ func TestNewPayloadCanceledWaiterDoesNotCancelLeader(t *testing.T) { waiterCtx, cancel := context.WithCancel(context.Background()) cancel() f.mu.Lock() - _, err := f.newPayloadWithoutForkChoiceLock(waiterCtx, key, nil, nil, nil, nil) + _, err := f.newPayloadLocked(waiterCtx, key, nil, nil, nil, nil) f.mu.Unlock() require.ErrorIs(t, err, context.Canceled) @@ -1142,7 +1401,7 @@ func TestNewPayloadConcurrencyIsBounded(t *testing.T) { for i := range 3 { go func(key byte) { f.mu.Lock() - _, _ = f.newPayloadWithoutForkChoiceLock(context.Background(), hashWithFirstByte(key), nil, nil, nil, nil) + _, _ = f.newPayloadLocked(context.Background(), hashWithFirstByte(key), nil, nil, nil, nil) f.mu.Unlock() done <- struct{}{} }(byte(i + 1)) @@ -1181,14 +1440,14 @@ func TestNewPayloadPanicRestoresForkChoiceState(t *testing.T) { defer func() { require.Equal(t, "engine panic", recover()) }() f.mu.Lock() defer f.mu.Unlock() - _, _ = f.newPayloadWithoutForkChoiceLock(context.Background(), key, nil, nil, nil, nil) + _, _ = f.newPayloadLocked(context.Background(), key, nil, nil, nil, nil) }() done := make(chan error, 2) for _, retryKey := range []common.Hash{key, hashWithFirstByte(2)} { go func() { f.mu.Lock() - _, err := f.newPayloadWithoutForkChoiceLock(context.Background(), retryKey, nil, nil, nil, nil) + _, err := f.newPayloadLocked(context.Background(), retryKey, nil, nil, nil, nil) f.mu.Unlock() done <- err }() diff --git a/cl/phase1/forkchoice/payload_attestation_validation.go b/cl/phase1/forkchoice/payload_attestation_validation.go index 8539519896c..89d30b1f4ee 100644 --- a/cl/phase1/forkchoice/payload_attestation_validation.go +++ b/cl/phase1/forkchoice/payload_attestation_validation.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "sync" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/fork" @@ -44,6 +45,15 @@ type payloadAttestationValidationContext struct { type payloadAttestationValidationContexts struct { cache *lru.Cache[common.Hash, *payloadAttestationValidationContext] buildSlots chan struct{} + mu sync.Mutex + builds map[common.Hash]*payloadAttestationValidationContextBuild +} + +type payloadAttestationValidationContextBuild struct { + done chan struct{} + validationContext *payloadAttestationValidationContext + err error + retry bool } func newPayloadAttestationValidationContexts() (*payloadAttestationValidationContexts, error) { @@ -57,6 +67,7 @@ func newPayloadAttestationValidationContexts() (*payloadAttestationValidationCon return &payloadAttestationValidationContexts{ cache: cache, buildSlots: make(chan struct{}, maxConcurrentValidationContextBuilds), + builds: make(map[common.Hash]*payloadAttestationValidationContextBuild), }, nil } @@ -65,24 +76,72 @@ func (c *payloadAttestationValidationContexts) get( blockRoot common.Hash, build func() (*payloadAttestationValidationContext, error), ) (*payloadAttestationValidationContext, error) { - if validationContext, ok := c.cache.Get(blockRoot); ok { - return validationContext, nil + for { + if validationContext, ok := c.cache.Get(blockRoot); ok { + return validationContext, nil + } + c.mu.Lock() + if current, ok := c.builds[blockRoot]; ok { + c.mu.Unlock() + validationContext, err, retry := waitForPayloadAttestationValidationContext(ctx, current) + if retry && ctx.Err() == nil { + continue + } + return validationContext, err + } + current := &payloadAttestationValidationContextBuild{done: make(chan struct{})} + c.builds[blockRoot] = current + c.mu.Unlock() + + select { + case c.buildSlots <- struct{}{}: + case <-ctx.Done(): + c.complete(blockRoot, current, nil, ctx.Err(), true) + return nil, ctx.Err() + } + var ( + validationContext *payloadAttestationValidationContext + err error + panicValue any + ) + func() { + defer func() { panicValue = recover() }() + validationContext, err = build() + }() + <-c.buildSlots + if panicValue != nil { + err = fmt.Errorf("payload attestation validation context build panicked: %v", panicValue) + } + if err == nil { + c.cache.Add(blockRoot, validationContext) + } + c.complete(blockRoot, current, validationContext, err, false) + if panicValue != nil { + panic(panicValue) + } + return validationContext, err } +} + +func waitForPayloadAttestationValidationContext(ctx context.Context, build *payloadAttestationValidationContextBuild) (*payloadAttestationValidationContext, error, bool) { select { - case c.buildSlots <- struct{}{}: + case <-build.done: + return build.validationContext, build.err, build.retry case <-ctx.Done(): - return nil, ctx.Err() - } - defer func() { <-c.buildSlots }() - if validationContext, ok := c.cache.Get(blockRoot); ok { - return validationContext, nil + return nil, ctx.Err(), false } - validationContext, err := build() - if err != nil { - return nil, err +} + +func (c *payloadAttestationValidationContexts) complete(blockRoot common.Hash, build *payloadAttestationValidationContextBuild, validationContext *payloadAttestationValidationContext, err error, retry bool) { + c.mu.Lock() + build.validationContext = validationContext + build.err = err + build.retry = retry + if c.builds[blockRoot] == build { + delete(c.builds, blockRoot) } - c.cache.Add(blockRoot, validationContext) - return validationContext, nil + close(build.done) + c.mu.Unlock() } func (f *ForkChoiceStore) payloadAttestationValidationContext( diff --git a/cl/phase1/forkchoice/payload_attestation_validation_test.go b/cl/phase1/forkchoice/payload_attestation_validation_test.go index dc5706ccf7f..63e7facebf7 100644 --- a/cl/phase1/forkchoice/payload_attestation_validation_test.go +++ b/cl/phase1/forkchoice/payload_attestation_validation_test.go @@ -37,6 +37,30 @@ type payloadAttestationValidationContextResult struct { err error } +type payloadAttestationObservedContext struct { + context.Context + observed chan struct{} + once sync.Once +} + +func newPayloadAttestationObservedContext(parent context.Context) *payloadAttestationObservedContext { + return &payloadAttestationObservedContext{Context: parent, observed: make(chan struct{})} +} + +func (c *payloadAttestationObservedContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.observed) }) + return c.Context.Done() +} + +func waitForPayloadAttestationWaiter(t *testing.T, ctx *payloadAttestationObservedContext) { + t.Helper() + select { + case <-ctx.observed: + case <-time.After(time.Second): + t.Fatal("payload attestation validation call did not reach its wait point") + } +} + func TestOnPayloadAttestationMessageRejectsNil(t *testing.T) { f := &ForkChoiceStore{} require.Error(t, f.OnPayloadAttestationMessage(context.Background(), nil, false)) @@ -62,20 +86,23 @@ func TestPayloadAttestationValidationContextsCollapseConcurrentBuilds(t *testing } results := make(chan payloadAttestationValidationContextResult, 16) - var wg sync.WaitGroup - for range 16 { - wg.Go(func() { - validationContext, getErr := contexts.get(context.Background(), root, build) + go func() { + validationContext, getErr := contexts.get(context.Background(), root, build) + results <- payloadAttestationValidationContextResult{validationContext, getErr} + }() + <-started + for range 15 { + waiterCtx := newPayloadAttestationObservedContext(context.Background()) + go func() { + validationContext, getErr := contexts.get(waiterCtx, root, build) results <- payloadAttestationValidationContextResult{validationContext, getErr} - }) + }() + waitForPayloadAttestationWaiter(t, waiterCtx) } - <-started require.Equal(t, int32(1), builds.Load()) close(release) - wg.Wait() - close(results) - - for result := range results { + for range 16 { + result := <-results require.NoError(t, result.err) require.Same(t, expected, result.validationContext) } @@ -106,6 +133,131 @@ func TestPayloadAttestationValidationContextsDoNotCacheBuildErrors(t *testing.T) require.Equal(t, int32(2), builds.Load()) } +func TestPayloadAttestationValidationContextsCollapseConcurrentBuildErrors(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + root := common.HexToHash("0x1234") + started := make(chan struct{}) + release := make(chan struct{}) + var builds atomic.Int32 + buildErr := errors.New("state unavailable") + build := func() (*payloadAttestationValidationContext, error) { + if builds.Add(1) == 1 { + close(started) + } + <-release + return nil, buildErr + } + + results := make(chan error, 16) + go func() { + _, getErr := contexts.get(context.Background(), root, build) + results <- getErr + }() + <-started + for range 15 { + waiterCtx := newPayloadAttestationObservedContext(context.Background()) + go func() { + _, getErr := contexts.get(waiterCtx, root, build) + results <- getErr + }() + waitForPayloadAttestationWaiter(t, waiterCtx) + } + require.Equal(t, int32(1), builds.Load()) + close(release) + for range 16 { + require.ErrorIs(t, <-results, buildErr) + } + require.Equal(t, int32(1), builds.Load()) +} + +func TestPayloadAttestationValidationContextsCanceledWaiterDoesNotCancelBuild(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + root := common.HexToHash("0x1234") + started := make(chan struct{}) + release := make(chan struct{}) + expected := &payloadAttestationValidationContext{slot: 100} + build := func() (*payloadAttestationValidationContext, error) { + close(started) + <-release + return expected, nil + } + leader := make(chan payloadAttestationValidationContextResult, 1) + go func() { + value, getErr := contexts.get(context.Background(), root, build) + leader <- payloadAttestationValidationContextResult{value, getErr} + }() + <-started + + waiterCtx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = contexts.get(waiterCtx, root, build) + require.ErrorIs(t, err, context.Canceled) + + close(release) + result := <-leader + require.NoError(t, result.err) + require.Same(t, expected, result.validationContext) +} + +func TestPayloadAttestationValidationContextsCanceledLeaderDoesNotPoisonWaiter(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + contexts.buildSlots <- struct{}{} + root := common.HexToHash("0x1234") + leaderParentCtx, cancelLeader := context.WithCancel(context.Background()) + leaderCtx := newPayloadAttestationObservedContext(leaderParentCtx) + leaderResult := make(chan error, 1) + go func() { + _, getErr := contexts.get(leaderCtx, root, func() (*payloadAttestationValidationContext, error) { + return nil, errors.New("leader build must not run") + }) + leaderResult <- getErr + }() + waitForPayloadAttestationWaiter(t, leaderCtx) + + expected := &payloadAttestationValidationContext{slot: 100} + waiterResult := make(chan payloadAttestationValidationContextResult, 1) + waiterCtx := newPayloadAttestationObservedContext(context.Background()) + go func() { + value, getErr := contexts.get(waiterCtx, root, func() (*payloadAttestationValidationContext, error) { + return expected, nil + }) + waiterResult <- payloadAttestationValidationContextResult{value, getErr} + }() + waitForPayloadAttestationWaiter(t, waiterCtx) + cancelLeader() + require.ErrorIs(t, <-leaderResult, context.Canceled) + <-contexts.buildSlots + + result := <-waiterResult + require.NoError(t, result.err) + require.Same(t, expected, result.validationContext) +} + +func TestPayloadAttestationValidationContextsPanicCleansInflightBuild(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + root := common.HexToHash("0x1234") + + func() { + defer func() { require.Equal(t, "boom", recover()) }() + _, _ = contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { + panic("boom") + }) + }() + + expected := &payloadAttestationValidationContext{slot: 100} + actual, err := contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { + return expected, nil + }) + require.NoError(t, err) + require.Same(t, expected, actual) +} + func TestPayloadAttestationValidationContextsBoundDifferentRootBuilds(t *testing.T) { contexts, err := newPayloadAttestationValidationContexts() require.NoError(t, err) diff --git a/cl/phase1/forkchoice/payload_vote_test.go b/cl/phase1/forkchoice/payload_vote_test.go index 8ad8823fc76..9d24cee6341 100644 --- a/cl/phase1/forkchoice/payload_vote_test.go +++ b/cl/phase1/forkchoice/payload_vote_test.go @@ -34,20 +34,22 @@ func (g ptcVoteForkGraph) GetBlock(root common.Hash) (*cltypes.SignedBeaconBlock type payloadVoteForkGraph struct { fork_graph.ForkGraph - hasEnvelope bool - envelope *cltypes.SignedExecutionPayloadEnvelope - block *cltypes.SignedBeaconBlock - blockState *state2.CachingBeaconState - dumpedEnvelope *common.Hash - hasEnvelopeState *atomic.Bool - dumpCalls *atomic.Int32 - dumpStarted chan struct{} - releaseDump chan struct{} - dumpErr error - onEnvelopePublished func() - pruneStarted chan struct{} - releasePrune chan struct{} - invalidatedHeader *common.Hash + hasEnvelope bool + envelope *cltypes.SignedExecutionPayloadEnvelope + block *cltypes.SignedBeaconBlock + blockState *state2.CachingBeaconState + dumpedEnvelope *common.Hash + preparedEnvelope **cltypes.SignedExecutionPayloadEnvelope + prepareRequiresBlock *bool + hasEnvelopeState *atomic.Bool + dumpCalls *atomic.Int32 + dumpStarted chan struct{} + releaseDump chan struct{} + dumpErr error + onEnvelopePublished func() + pruneStarted chan struct{} + releasePrune chan struct{} + invalidatedHeader *common.Hash } func (g payloadVoteForkGraph) Prune(uint64) error { @@ -72,7 +74,10 @@ func (g payloadVoteForkGraph) DumpEnvelopeOnDisk(blockRoot common.Hash, _ *cltyp return publish() } -func (g payloadVoteForkGraph) PrepareEnvelopeOnDisk(blockRoot common.Hash, _ *cltypes.SignedExecutionPayloadEnvelope, _ bool) (func() error, error) { +func (g payloadVoteForkGraph) PrepareEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, requireBlock bool) (func() error, error) { + if g.prepareRequiresBlock != nil { + *g.prepareRequiresBlock = requireBlock + } var call int32 if g.dumpCalls != nil { call = g.dumpCalls.Add(1) @@ -87,6 +92,9 @@ func (g payloadVoteForkGraph) PrepareEnvelopeOnDisk(blockRoot common.Hash, _ *cl <-g.releaseDump } return func() error { + if g.preparedEnvelope != nil { + *g.preparedEnvelope = envelope + } if g.dumpedEnvelope != nil { *g.dumpedEnvelope = blockRoot } diff --git a/cl/phase1/network/backward_beacon_downloader.go b/cl/phase1/network/backward_beacon_downloader.go index 7b2aad40bcc..0d2f707e943 100644 --- a/cl/phase1/network/backward_beacon_downloader.go +++ b/cl/phase1/network/backward_beacon_downloader.go @@ -51,33 +51,33 @@ import ( // [Modified in Gloas:EIP7732] envelope is non-nil for GLOAS FULL blocks, nil for EMPTY or pre-GLOAS. type OnNewBlock func(blk *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (finished bool, err error) +// ValidateBlockFn authenticates a root-committed block's wrapper signature. +type ValidateBlockFn func(*cltypes.SignedBeaconBlock) error + +// ValidateLookaheadFn authenticates a lookahead against its trusted anchor state. +type ValidateLookaheadFn func(*cltypes.SignedBeaconBlock, *cltypes.SignedBeaconBlock) error + // BlockChecker is an interface for checking if a block exists type BlockChecker interface { HasBlock(blockNumber uint64) bool } -type blockRangeSource uint8 - -const ( - blockRangeSourceUnknown blockRangeSource = iota - blockRangeSourceP2P - blockRangeSourceHTTP -) - type BackwardBeaconDownloader struct { - ctx context.Context - slotToDownload atomic.Uint64 - expectedRoot common.Hash - rpc *rpc.BeaconRpcP2P - engine execution_client.ExecutionEngine - onNewBlock OnNewBlock - finished atomic.Bool - reqInterval *time.Ticker - db kv.RwDB - sn *freezeblocks.CaplinSnapshots - neverSkip bool - blockChecker BlockChecker - beaconCfg *clparams.BeaconChainConfig + ctx context.Context + slotToDownload atomic.Uint64 + expectedRoot common.Hash + rpc *rpc.BeaconRpcP2P + engine execution_client.ExecutionEngine + onNewBlock OnNewBlock + validateBlock ValidateBlockFn + validateLookahead ValidateLookaheadFn + finished atomic.Bool + reqInterval *time.Ticker + db kv.RwDB + sn *freezeblocks.CaplinSnapshots + neverSkip bool + blockChecker BlockChecker + beaconCfg *clparams.BeaconChainConfig // [New in Gloas:EIP7732] highest block from the previous batch, used as lookahead // to determine FULL/EMPTY status of the highest block in the current batch. prevBatchTopBlock *cltypes.SignedBeaconBlock @@ -89,7 +89,6 @@ type BackwardBeaconDownloader struct { lookaheadRescan bool lookaheadRetryWindow bool lookaheadAnchorRoot common.Hash - blockRangeSource blockRangeSource // Count consecutive batches where at least one required FULL envelope was unresolved. // After enough failures, skip envelope requirements and process blocks as EMPTY. @@ -111,12 +110,15 @@ const ( maxLookaheadFailures = uint8(6) ) -var errSkippedEnvelopeRecoveryCapacity = errors.New("skipped envelope recovery capacity exhausted") +// ErrSkippedEnvelopeRecoveryCapacity indicates that pending recovery must drain before downloading continues. +var ErrSkippedEnvelopeRecoveryCapacity = errors.New("skipped envelope recovery capacity exhausted") // SkippedFullBlock records a GLOAS block that may need an envelope after degraded backward download. type SkippedFullBlock struct { - Slot uint64 - Root [32]byte + Slot uint64 + Root [32]byte + ChildSlot uint64 + ChildRoot [32]byte } type EnvelopeRecoveryResult struct { @@ -191,6 +193,13 @@ func (b *BackwardBeaconDownloader) SetOnNewBlock(onNewBlock OnNewBlock) { b.onNewBlock = onNewBlock } +func (b *BackwardBeaconDownloader) SetValidateFunctions(validateBlock ValidateBlockFn, validateLookahead ValidateLookaheadFn) { + b.mu.Lock() + defer b.mu.Unlock() + b.validateBlock = validateBlock + b.validateLookahead = validateLookahead +} + func (b *BackwardBeaconDownloader) RPC() *rpc.BeaconRpcP2P { return b.rpc } @@ -242,7 +251,6 @@ func (b *BackwardBeaconDownloader) fetchBlockRange(ctx context.Context) ([]*clty if b.httpPreferred.Load() && b.httpFallbackURL != "" { blocks, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, count, b.beaconCfg) if err == nil && len(blocks) > 0 { - b.blockRangeSource = blockRangeSourceHTTP log.Debug("[BackwardBeaconDownloader] fetched blocks from beacon API", "fromSlot", start, "count", len(blocks)) return blocks, nil } @@ -269,7 +277,6 @@ func (b *BackwardBeaconDownloader) fetchBlockRange(ctx context.Context) ([]*clty go b.sendBlockRequest(ctx, start, count, received, &requestSent) case responses := <-received: - b.blockRangeSource = blockRangeSourceP2P return responses, nil case <-p2pDeadline.C: @@ -279,7 +286,6 @@ func (b *BackwardBeaconDownloader) fetchBlockRange(ctx context.Context) ([]*clty } blocks, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, count, b.beaconCfg) if err == nil && len(blocks) > 0 { - b.blockRangeSource = blockRangeSourceHTTP log.Debug("[BackwardBeaconDownloader] P2P failed, fetched blocks from beacon API", "fromSlot", start, "count", len(blocks)) b.httpPreferred.Store(true) return blocks, nil @@ -364,6 +370,12 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons continue } matched = true + if block.Version() >= clparams.GloasVersion && b.validateBlock != nil { + if err := b.validateBlock(block); err != nil { + log.Warn("[BackwardBeaconDownloader] rejected unauthenticated block", "slot", block.Block.Slot, "err", err) + return nil + } + } var envelope *cltypes.SignedExecutionPayloadEnvelope trackSkippedForRecovery := false @@ -392,7 +404,7 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons } } if envelope == nil && b.envelopesSkipped && trackSkippedForRecovery && !b.canTrackSkippedFullBlock(block) { - return fmt.Errorf("%w at slot %d", errSkippedEnvelopeRecoveryCapacity, block.Block.Slot) + return fmt.Errorf("%w at slot %d", ErrSkippedEnvelopeRecoveryCapacity, block.Block.Slot) } } @@ -405,29 +417,19 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } - finished, err := b.onNewBlock(block, envelope) - b.finished.Store(finished) + var genesisTop *cltypes.SignedBeaconBlock + if block.Block.Slot == 0 { + genesisTop = firstCompleteBlock(responses) + } + reachedGenesis, err := b.acceptDownloadedBlock(block, envelope, trackSkippedForRecovery, b.prevBatchTopBlock, genesisTop) if err != nil { log.Warn("Error processing block", "err", err) continue } - - // Record FULL blocks passing through without envelope for post-download recovery. - if envelope == nil && trackSkippedForRecovery { - b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) - } - advanced = true - b.prevBatchTopBlock = block - b.prevBatchTopBlockUntrusted = false - b.expectedRoot = block.Block.ParentRoot - if block.Block.Slot == 0 { - b.finished.Store(true) - b.prevBatchTopBlock = firstCompleteBlock(responses) - b.prevBatchTopBlockUntrusted = false + if reachedGenesis { return nil } - b.slotToDownload.Store(block.Block.Slot - 1) } // Update prevBatchTopBlock only when at least one block was processed, @@ -446,13 +448,21 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons } else if block != nil { blockRoot, err := block.Block.HashSSZ() if err == nil && blockRoot == b.expectedRoot { + if block.Version() >= clparams.GloasVersion && b.validateBlock != nil { + if err := b.validateBlock(block); err != nil { + log.Warn("[BackwardBeaconDownloader] rejected unauthenticated root-fetched block", "slot", block.Block.Slot, "err", err) + return nil + } + } log.Debug("[BackwardBeaconDownloader] block matched via root lookup", "slot", block.Block.Slot, "root", common.Hash(blockRoot)) var envelope *cltypes.SignedExecutionPayloadEnvelope + var lookahead *cltypes.SignedBeaconBlock isFull := false availabilityKnown := block.Version() < clparams.GloasVersion if block.Version() >= clparams.GloasVersion { - lookahead := b.prevBatchTopBlock + lookahead = b.prevBatchTopBlock + lookaheadTrusted := lookahead != nil && !b.prevBatchTopBlockUntrusted if _, linked := gloasBlockAvailability(block, lookahead); !linked { lookahead, err = b.fetchGloasLookahead(ctx, block, common.Hash(blockRoot)) if err != nil { @@ -466,9 +476,13 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons b.envelopesSkipped = true } else { b.consecutiveLookaheadFailures = 0 + lookaheadTrusted = b.validateLookahead != nil && b.validateLookahead(block, lookahead) == nil } } - full, known := b.gloasBlockAvailability(block, lookahead) + full, known := false, false + if lookaheadTrusted { + full, known = gloasBlockAvailability(block, lookahead) + } availabilityKnown = known if !known && !b.envelopesSkipped { b.recordEmptyProbeResult(1, 0) @@ -488,53 +502,27 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons log.Warn("[BackwardBeaconDownloader] root-fetched envelope does not match block", "slot", block.Block.Slot, "err", err) } } - b.recordEnvelopeFetchResult(1, btoi(envelope != nil)) + received := 0 + if envelope != nil { + received = 1 + } + b.recordEnvelopeFetchResult(1, received) if envelope == nil && (!b.envelopesSkipped || !b.canTrackSkippedFullBlock(block)) { log.Warn("[BackwardBeaconDownloader] root-fetched FULL block envelope unavailable, will retry", "slot", block.Block.Slot, "err", fetchErr, "consecutiveFailures", b.consecutiveEnvelopeFailures) return nil } - } else if !full && !b.envelopesSkipped { - env, fetchErr := b.fetchSingleEnvelope(ctx, common.Hash(blockRoot)) - if fetchErr != nil { - b.recordEmptyProbeResult(1, 0) - if !b.envelopesSkipped { - log.Warn("[BackwardBeaconDownloader] root-fetched EMPTY confirmation failed, will retry", "slot", block.Block.Slot, "err", fetchErr) - return nil - } - } else { - b.recordEmptyProbeResult(0, 0) - } - if env != nil { - if err := ValidateFetchedEnvelope(b.beaconCfg, block, common.Hash(blockRoot), env); err != nil { - log.Warn("[BackwardBeaconDownloader] root-fetched envelope does not match block", "slot", block.Block.Slot, "err", err) - return nil - } - envelope = env - isFull = true - } } if envelope == nil && b.envelopesSkipped && (isFull || !availabilityKnown) && !b.canTrackSkippedFullBlock(block) { - return fmt.Errorf("%w at slot %d", errSkippedEnvelopeRecoveryCapacity, block.Block.Slot) + return fmt.Errorf("%w at slot %d", ErrSkippedEnvelopeRecoveryCapacity, block.Block.Slot) } } - finished, err := b.onNewBlock(block, envelope) - b.finished.Store(finished) + reachedGenesis, err := b.acceptDownloadedBlock(block, envelope, isFull || !availabilityKnown, lookahead, nil) if err != nil { log.Warn("Error processing root-fetched block", "err", err) - } else { - if envelope == nil && (isFull || !availabilityKnown) { - b.skippedFullBlocks = append(b.skippedFullBlocks, SkippedFullBlock{Slot: block.Block.Slot, Root: blockRoot}) - } - b.prevBatchTopBlock = block - b.prevBatchTopBlockUntrusted = false - b.expectedRoot = block.Block.ParentRoot - if block.Block.Slot == 0 { - b.finished.Store(true) - return nil - } - b.slotToDownload.Store(block.Block.Slot - 1) + } else if reachedGenesis { + return nil } } } @@ -543,11 +531,41 @@ func (b *BackwardBeaconDownloader) processResponses(ctx context.Context, respons return nil } -func btoi(value bool) int { - if value { - return 1 +func (b *BackwardBeaconDownloader) acceptDownloadedBlock(block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope, trackSkipped bool, recoveryChild, genesisTop *cltypes.SignedBeaconBlock) (bool, error) { + finished, err := b.onNewBlock(block, envelope) + b.finished.Store(finished) + if err != nil { + return false, err + } + if envelope == nil && trackSkipped { + b.skippedFullBlocks = append(b.skippedFullBlocks, skippedFullBlock(block, recoveryChild)) + } + b.prevBatchTopBlock = block + b.prevBatchTopBlockUntrusted = false + b.expectedRoot = block.Block.ParentRoot + if block.Block.Slot == 0 { + b.finished.Store(true) + if genesisTop != nil { + b.prevBatchTopBlock = genesisTop + } + return true, nil + } + b.slotToDownload.Store(block.Block.Slot - 1) + return false, nil +} + +func skippedFullBlock(block, child *cltypes.SignedBeaconBlock) SkippedFullBlock { + root, _ := block.Block.HashSSZ() + item := SkippedFullBlock{Slot: block.Block.Slot, Root: root} + if child == nil || child.Block == nil || child.Block.ParentRoot != root || child.Block.Slot <= block.Block.Slot { + return item + } + childRoot, err := child.Block.HashSSZ() + if err == nil { + item.ChildSlot = child.Block.Slot + item.ChildRoot = childRoot } - return 0 + return item } func firstCompleteBlock(responses []*cltypes.SignedBeaconBlock) *cltypes.SignedBeaconBlock { @@ -600,7 +618,7 @@ func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Contex } if lookahead := selectGloasLookahead(anchor, anchorRoot, responses); lookahead != nil { b.prevBatchTopBlock = lookahead - b.prevBatchTopBlockUntrusted = b.blockRangeSource != blockRangeSourceHTTP + b.prevBatchTopBlockUntrusted = b.validateLookahead == nil || b.validateLookahead(anchor, lookahead) != nil b.consecutiveLookaheadFailures = 0 b.lookaheadSearchOffset = 0 b.lookaheadRescan = false @@ -613,6 +631,7 @@ func (b *BackwardBeaconDownloader) prepareFirstBatchLookahead(ctx context.Contex return err } b.prevBatchTopBlock = lookahead + b.prevBatchTopBlockUntrusted = b.validateLookahead == nil || b.validateLookahead(anchor, lookahead) != nil b.consecutiveLookaheadFailures = 0 b.lookaheadSearchOffset = 0 b.lookaheadRescan = false @@ -648,7 +667,7 @@ func (b *BackwardBeaconDownloader) fetchGloasLookahead( candidates, err := fetchBlocksFromBeaconAPI(ctx, b.httpFallbackURL, start, gloasLookaheadWindow, b.beaconCfg) if err == nil { if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { - b.prevBatchTopBlockUntrusted = false + b.prevBatchTopBlockUntrusted = true return lookahead, nil } err = errors.New("HTTP lookahead source returned no direct child") @@ -673,24 +692,6 @@ func (b *BackwardBeaconDownloader) fetchGloasLookahead( return nil, errors.Join(errs...) } -type gloasLookaheadFetcher func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) - -func fetchGloasLookaheadFromSources(ctx context.Context, anchor *cltypes.SignedBeaconBlock, anchorRoot common.Hash, start uint64, sources ...gloasLookaheadFetcher) (*cltypes.SignedBeaconBlock, error) { - errs := make([]error, 0, len(sources)) - for _, fetch := range sources { - candidates, err := fetch(ctx, start, gloasLookaheadWindow) - if err != nil { - errs = append(errs, err) - continue - } - if lookahead := selectGloasLookahead(anchor, anchorRoot, candidates); lookahead != nil { - return lookahead, nil - } - errs = append(errs, errors.New("lookahead source returned no direct child")) - } - return nil, errors.Join(errs...) -} - func (b *BackwardBeaconDownloader) advanceLookaheadSearch() { if b.lookaheadRescan { b.lookaheadRescan = false @@ -746,19 +747,6 @@ func (b *BackwardBeaconDownloader) waitBeforeLookaheadRetry(ctx context.Context) } } -func determineGloasFullRoots(responses []*cltypes.SignedBeaconBlock, prevBatchTopBlock *cltypes.SignedBeaconBlock) [][32]byte { - anchor := lastCompleteBlock(responses) - if anchor == nil { - return nil - } - expectedRoot, err := anchor.Block.HashSSZ() - if err != nil { - return nil - } - fullRoots, _ := determineGloasAvailability(responses, prevBatchTopBlock, expectedRoot) - return fullRoots -} - func determineGloasAvailability( responses []*cltypes.SignedBeaconBlock, lookahead *cltypes.SignedBeaconBlock, @@ -825,11 +813,17 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp return nil, nil, nil } - fullRoots, knownRootSet := determineGloasAvailability(responses, b.prevBatchTopBlock, b.expectedRoot) + authenticatedResponses := b.authenticatedBackwardResponses(responses) + if b.prevBatchTopBlockUntrusted && b.validateLookahead != nil { + if anchor := blockByRoot(authenticatedResponses, b.expectedRoot); anchor != nil && b.validateLookahead(anchor, b.prevBatchTopBlock) == nil { + b.prevBatchTopBlockUntrusted = false + } + } + fullRoots, knownRootSet := determineGloasAvailability(authenticatedResponses, b.prevBatchTopBlock, b.expectedRoot) var untrustedFullRoot common.Hash untrustedFull := false if b.prevBatchTopBlockUntrusted { - if anchor := blockByRoot(responses, b.expectedRoot); anchor != nil { + if anchor := blockByRoot(authenticatedResponses, b.expectedRoot); anchor != nil { full, known := gloasBlockAvailability(anchor, b.prevBatchTopBlock) if known && !full { delete(knownRootSet, b.expectedRoot) @@ -839,6 +833,11 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp } } } + if untrustedFull { + fullRoots = slices.DeleteFunc(fullRoots, func(root [32]byte) bool { + return common.Hash(root) == untrustedFullRoot + }) + } // Build a set for O(1) lookup by callers. fullRootSet := make(map[common.Hash]struct{}, len(fullRoots)) @@ -873,98 +872,28 @@ func (b *BackwardBeaconDownloader) fetchGloasEnvelopes(ctx context.Context, resp delete(knownRootSet, untrustedFullRoot) } - inferredEmptyRoots := make(map[common.Hash]struct{}, len(knownRootSet)-len(fullRootSet)) - for root := range knownRootSet { - if _, full := fullRootSet[root]; !full { - if b.httpFallbackURL == "" { - continue - } - inferredEmptyRoots[root] = struct{}{} - delete(knownRootSet, root) - } - } - if len(inferredEmptyRoots) > 0 && b.httpFallbackURL != "" { - probed, confirmedEmpty := b.probeGloasEmptyCandidates(ctx, responses, inferredEmptyRoots) - if envelopes == nil && len(probed) > 0 { - envelopes = make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(probed)) - } - for root, envelope := range probed { - envelopes[root] = envelope - fullRootSet[root] = struct{}{} - knownRootSet[root] = struct{}{} - } - for root := range confirmedEmpty { - knownRootSet[root] = struct{}{} - } - } - return envelopes, fullRootSet, knownRootSet } -func (b *BackwardBeaconDownloader) probeGloasEmptyCandidates(ctx context.Context, blocks []*cltypes.SignedBeaconBlock, roots map[common.Hash]struct{}) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, map[common.Hash]struct{}) { - type result struct { - root common.Hash - envelope *cltypes.SignedExecutionPayloadEnvelope - confirmed bool - } - uniqueBlocks := make(map[common.Hash]*cltypes.SignedBeaconBlock, len(roots)) - for _, block := range blocks { +func (b *BackwardBeaconDownloader) authenticatedBackwardResponses(responses []*cltypes.SignedBeaconBlock) []*cltypes.SignedBeaconBlock { + authenticated := make([]*cltypes.SignedBeaconBlock, 0, len(responses)) + expectedRoot := b.expectedRoot + for _, block := range slices.Backward(responses) { if block == nil || block.Block == nil || block.Block.Body == nil { continue } root, err := block.Block.HashSSZ() - if err != nil { - continue - } - hash := common.Hash(root) - if _, ok := roots[hash]; !ok { + if err != nil || root != expectedRoot { continue } - if _, exists := uniqueBlocks[hash]; !exists { - uniqueBlocks[hash] = block - } - } - - results := make(chan result, len(uniqueBlocks)) - sem := make(chan struct{}, 8) - var wg sync.WaitGroup - for root, block := range uniqueBlocks { - wg.Go(func() { - select { - case sem <- struct{}{}: - case <-ctx.Done(): - return - } - defer func() { <-sem }() - envelope, err := b.fetchSingleEnvelope(ctx, root) - if err != nil { - results <- result{root: root} - return - } - if envelope == nil { - results <- result{root: root, confirmed: true} - return - } - if ValidateFetchedEnvelope(b.beaconCfg, block, root, envelope) != nil { - results <- result{root: root} - return - } - results <- result{root: root, envelope: envelope} - }) - } - wg.Wait() - close(results) - - envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) - confirmedEmpty := make(map[common.Hash]struct{}) - for result := range results { - if result.envelope != nil { - envelopes[result.root] = result.envelope - } else if result.confirmed { - confirmedEmpty[result.root] = struct{}{} + if block.Version() >= clparams.GloasVersion && b.validateBlock != nil && b.validateBlock(block) != nil { + return nil } + authenticated = append(authenticated, block) + expectedRoot = block.Block.ParentRoot } - return envelopes, confirmedEmpty + slices.Reverse(authenticated) + return authenticated } func validateFetchedEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks []*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { @@ -1018,6 +947,25 @@ func (b *BackwardBeaconDownloader) SkippedFullBlocks() []SkippedFullBlock { return b.skippedFullBlocks } +func (b *BackwardBeaconDownloader) AcknowledgeSkippedFullBlocks(recovered []SkippedFullBlock) { + if len(recovered) == 0 { + return + } + counts := make(map[SkippedFullBlock]int, len(recovered)) + for _, item := range recovered { + counts[item]++ + } + remaining := make([]SkippedFullBlock, 0, len(b.skippedFullBlocks)-min(len(recovered), len(b.skippedFullBlocks))) + for _, item := range b.skippedFullBlocks { + if counts[item] > 0 { + counts[item]-- + continue + } + remaining = append(remaining, item) + } + b.skippedFullBlocks = remaining +} + func (b *BackwardBeaconDownloader) HasEnvelopeRecoverySource() bool { return b.httpFallbackURL != "" || b.rpc != nil } @@ -1115,22 +1063,6 @@ func validateRecoveryEnvelopes(beaconCfg *clparams.BeaconChainConfig, blocks map return valid } -func fetchEnvelopeRecoverySources(ctx context.Context, sources ...func(context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { - results := make(chan map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, len(sources)) - var wg sync.WaitGroup - for _, source := range sources { - wg.Go(func() { results <- source(ctx) }) - } - wg.Wait() - close(results) - - envelopes := make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) - for result := range results { - maps.Copy(envelopes, result) - } - return envelopes -} - func fetchEnvelopeRecoveryResults(ctx context.Context, sources ...func(context.Context) EnvelopeRecoveryResult) EnvelopeRecoveryResult { results := make(chan EnvelopeRecoveryResult, len(sources)) var wg sync.WaitGroup @@ -1346,6 +1278,9 @@ func fetchBlockFromBeaconAPIByRoot(ctx context.Context, baseURL string, root com if err := block.DecodeSSZ(body, int(version)); err != nil { return nil, fmt.Errorf("block decode by root: %w", err) } + if err := requireCanonicalSSZ(body, block); err != nil { + return nil, fmt.Errorf("block decode by root: %w", err) + } return block, nil } @@ -1386,6 +1321,9 @@ func (b *BackwardBeaconDownloader) fetchSingleEnvelope(ctx context.Context, bloc if err := envelope.DecodeSSZ(body, int(clparams.GloasVersion)); err != nil { return nil, fmt.Errorf("envelope decode: %w", err) } + if err := requireCanonicalSSZ(body, envelope); err != nil { + return nil, fmt.Errorf("envelope decode: %w", err) + } return envelope, nil } diff --git a/cl/phase1/network/backward_beacon_downloader_test.go b/cl/phase1/network/backward_beacon_downloader_test.go index eaeca09d757..4dcd3067421 100644 --- a/cl/phase1/network/backward_beacon_downloader_test.go +++ b/cl/phase1/network/backward_beacon_downloader_test.go @@ -19,8 +19,10 @@ package network import ( "bytes" "context" + "errors" "net/http" "net/http/httptest" + "strings" "sync/atomic" "testing" "time" @@ -91,6 +93,12 @@ func makeValidGloasEnvelope(t *testing.T, block *cltypes.SignedBeaconBlock) ([32 envelope.Message.Payload.BlockHash = common.HexToHash("0x1da54a16ef5d8bd1d1559378bbdea3b084b58d1ff1e3db53c276a3ecd6c3ceb6") requestsHash := cltypes.ComputeExecutionRequestHash(cltypes.GetExecutionRequestsList(&clparams.MainnetBeaconConfig, envelope.Message.ExecutionRequests)) header, err := envelope.Message.Payload.RlpHeader(&envelope.Message.ParentBeaconBlockRoot, requestsHash) + if err != nil { + parts := strings.Fields(err.Error()) + require.Greater(t, len(parts), 6) + envelope.Message.Payload.BlockHash = common.HexToHash(parts[6]) + header, err = envelope.Message.Payload.RlpHeader(&envelope.Message.ParentBeaconBlockRoot, requestsHash) + } require.NoError(t, err) bid.BlockHash = header.Hash() blockRoot, err := block.Block.HashSSZ() @@ -100,6 +108,19 @@ func makeValidGloasEnvelope(t *testing.T, block *cltypes.SignedBeaconBlock) ([32 return blockRoot, envelope } +func determineGloasFullRoots(responses []*cltypes.SignedBeaconBlock, lookahead *cltypes.SignedBeaconBlock) [][32]byte { + anchor := lastCompleteBlock(responses) + if anchor == nil { + return nil + } + expectedRoot, err := anchor.Block.HashSSZ() + if err != nil { + return nil + } + fullRoots, _ := determineGloasAvailability(responses, lookahead, expectedRoot) + return fullRoots +} + // TestDetermineGloasFullRoots_EmptyBatch verifies that an empty batch returns no roots. func TestDetermineGloasFullRoots_EmptyBatch(t *testing.T) { roots := determineGloasFullRoots(nil, nil) @@ -246,7 +267,7 @@ func TestDetermineGloasFullRoots_MixedVersions(t *testing.T) { assert.Contains(t, roots, rootFull) } -func TestBackwardBeaconDownloaderFirstBatchUsesLookaheadAfterMissedSlot(t *testing.T) { +func TestBackwardBeaconDownloaderFirstBatchHTTPChildCannotProveEmpty(t *testing.T) { anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) anchorRoot, err := anchor.Block.HashSSZ() require.NoError(t, err) @@ -281,8 +302,9 @@ func TestBackwardBeaconDownloaderFirstBatchUsesLookaheadAfterMissedSlot(t *testi } require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{anchor})) - assert.True(t, processed.Load()) + assert.False(t, processed.Load()) assert.Equal(t, int32(1), lookaheadRequests.Load()) + assert.True(t, downloader.prevBatchTopBlockUntrusted) } func TestBackwardBeaconDownloaderFirstBatchFullBlockWaitsForEnvelope(t *testing.T) { @@ -310,6 +332,9 @@ func TestBackwardBeaconDownloaderFirstBatchFullBlockWaitsForEnvelope(t *testing. expectedRoot: anchorRoot, httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig, + validateLookahead: func(*cltypes.SignedBeaconBlock, *cltypes.SignedBeaconBlock) error { + return nil + }, onNewBlock: func(block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { processed.Store(true) return true, nil @@ -322,6 +347,50 @@ func TestBackwardBeaconDownloaderFirstBatchFullBlockWaitsForEnvelope(t *testing. assert.Equal(t, 1, downloader.consecutiveEnvelopeFailures) } +func TestBackwardBeaconDownloaderRejectsUnauthenticatedHTTPlookaheadBeforeEnvelopeFetch(t *testing.T) { + anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) + anchorRoot, err := anchor.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(102, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = anchorRoot + encodedLookahead, err := lookahead.EncodeSSZ(nil) + require.NoError(t, err) + var envelopeRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/eth/v2/beacon/blocks/102": + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = w.Write(encodedLookahead) + case "/eth/v1/beacon/execution_payload_envelope/" + common.Hash(anchorRoot).Hex(): + envelopeRequests.Add(1) + http.NotFound(w, r) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + var processed atomic.Bool + downloader := &BackwardBeaconDownloader{ + expectedRoot: anchorRoot, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + validateBlock: func(*cltypes.SignedBeaconBlock) error { return nil }, + validateLookahead: func(*cltypes.SignedBeaconBlock, *cltypes.SignedBeaconBlock) error { + return errors.New("invalid lookahead proposer") + }, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Store(true) + return true, nil + }, + } + downloader.httpPreferred.Store(true) + + require.NoError(t, downloader.processResponses(t.Context(), []*cltypes.SignedBeaconBlock{anchor})) + require.Zero(t, envelopeRequests.Load()) + require.False(t, processed.Load()) +} + func TestBackwardBeaconDownloaderHTTPPreferredMissingEnvelopeTracksFailure(t *testing.T) { server := httptest.NewServer(http.NotFoundHandler()) defer server.Close() @@ -348,7 +417,44 @@ func TestBackwardBeaconDownloaderHTTPPreferredMissingEnvelopeTracksFailure(t *te assert.Equal(t, 1, downloader.consecutiveEnvelopeFailures) } -func TestFetchGloasEnvelopesProbesLookaheadInferredEmptyBlock(t *testing.T) { +func TestBackwardBeaconDownloaderRejectsInvalidRootedSignatureBeforeEnvelopeFetch(t *testing.T) { + block := makeGloasBlock(100, hash(0xAA), hash(0x10)) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + lookahead := makeGloasBlock(101, hash(0xBB), hash(0xAA)) + lookahead.Block.ParentRoot = blockRoot + + var envelopeRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex() { + envelopeRequests.Add(1) + } + http.NotFound(w, r) + })) + defer server.Close() + + var processed atomic.Bool + downloader := &BackwardBeaconDownloader{ + expectedRoot: blockRoot, + prevBatchTopBlock: lookahead, + httpFallbackURL: server.URL, + beaconCfg: &clparams.MainnetBeaconConfig, + validateBlock: func(*cltypes.SignedBeaconBlock) error { + return errors.New("invalid proposer signature") + }, + onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { + processed.Store(true) + return true, nil + }, + } + downloader.httpPreferred.Store(true) + + require.NoError(t, downloader.processResponses(t.Context(), []*cltypes.SignedBeaconBlock{block})) + require.Zero(t, envelopeRequests.Load()) + require.False(t, processed.Load()) +} + +func TestFetchGloasEnvelopesCanonicalEmptyRejectsLateEnvelope(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, envelope := makeValidGloasEnvelope(t, block) lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) @@ -379,8 +485,8 @@ func TestFetchGloasEnvelopesProbesLookaheadInferredEmptyBlock(t *testing.T) { []*cltypes.SignedBeaconBlock{block, lookahead}, ) - require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), envelopes[common.Hash(blockRoot)])) - assert.Contains(t, fullRoots, common.Hash(blockRoot)) + assert.NotContains(t, envelopes, common.Hash(blockRoot)) + assert.NotContains(t, fullRoots, common.Hash(blockRoot)) assert.Contains(t, knownRoots, common.Hash(blockRoot)) } @@ -474,6 +580,9 @@ func TestUntrustedP2PLookaheadCanProveFullWithMatchingEnvelope(t *testing.T) { prevBatchTopBlockUntrusted: true, httpFallbackURL: server.URL, beaconCfg: &clparams.MainnetBeaconConfig, + validateLookahead: func(*cltypes.SignedBeaconBlock, *cltypes.SignedBeaconBlock) error { + return nil + }, } downloader.httpPreferred.Store(true) @@ -508,6 +617,10 @@ func TestUntrustedP2PEmptyLookaheadEntersBoundedRecovery(t *testing.T) { require.True(t, processed) require.True(t, downloader.envelopesSkipped) require.Len(t, downloader.skippedFullBlocks, 1) + lookaheadRoot, err := lookahead.Block.HashSSZ() + require.NoError(t, err) + require.Equal(t, lookahead.Block.Slot, downloader.skippedFullBlocks[0].ChildSlot) + require.Equal(t, lookaheadRoot, downloader.skippedFullBlocks[0].ChildRoot) } func TestDirectFirstBatchP2PLookaheadRemainsUntrusted(t *testing.T) { @@ -519,9 +632,8 @@ func TestDirectFirstBatchP2PLookaheadRemainsUntrusted(t *testing.T) { processed := false downloader := &BackwardBeaconDownloader{ - expectedRoot: blockRoot, - blockRangeSource: blockRangeSourceP2P, - beaconCfg: &clparams.MainnetBeaconConfig, + expectedRoot: blockRoot, + beaconCfg: &clparams.MainnetBeaconConfig, onNewBlock: func(*cltypes.SignedBeaconBlock, *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { processed = true return true, nil @@ -557,47 +669,15 @@ func TestUnavailableLookaheadEntersBoundedUnresolvedRecovery(t *testing.T) { require.Len(t, downloader.skippedFullBlocks, 1) } -func TestProbeGloasEmptyCandidatesDeduplicatesRoots(t *testing.T) { - block := makeGloasBlock(100, hash(0xAA), hash(0x10)) - blockRoot, err := block.Block.HashSSZ() - require.NoError(t, err) - - var requests atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - http.NotFound(w, r) - })) - defer server.Close() - - downloader := &BackwardBeaconDownloader{ - httpFallbackURL: server.URL, - beaconCfg: &clparams.MainnetBeaconConfig, - } - done := make(chan struct{}) - go func() { - downloader.probeGloasEmptyCandidates( - context.Background(), - []*cltypes.SignedBeaconBlock{block, block}, - map[common.Hash]struct{}{common.Hash(blockRoot): {}}, - ) - close(done) - }() - - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("duplicate root deadlocked envelope probing") - } - assert.Equal(t, int32(1), requests.Load()) -} - -func TestEmptyProbeServerFailuresEnterBoundedUnresolvedRecovery(t *testing.T) { +func TestCanonicalEmptyDoesNotDependOnEnvelopeSource(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, err := block.Block.HashSSZ() require.NoError(t, err) lookahead := makeGloasBlock(101, hash(0xBB), hash(0xCC)) lookahead.Block.ParentRoot = blockRoot + requests := atomic.Int32{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) w.WriteHeader(http.StatusTooManyRequests) })) defer server.Close() @@ -614,12 +694,11 @@ func TestEmptyProbeServerFailuresEnterBoundedUnresolvedRecovery(t *testing.T) { }, } - for range 3 { - require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) - } + require.NoError(t, downloader.processResponses(context.Background(), []*cltypes.SignedBeaconBlock{block})) require.True(t, processed) - require.True(t, downloader.envelopesSkipped) - require.Len(t, downloader.skippedFullBlocks, 1) + require.False(t, downloader.envelopesSkipped) + require.Empty(t, downloader.skippedFullBlocks) + require.Zero(t, requests.Load()) } func TestFetchEnvelopesFromBeaconAPIUsesBlockRoot(t *testing.T) { @@ -949,31 +1028,6 @@ func TestLookaheadSearchRetriesTransientLaterWindowBeforeAdvancing(t *testing.T) require.Equal(t, 2*gloasLookaheadWindow, downloader.lookaheadSearchOffset) } -func TestFetchGloasLookaheadFromSourcesFallsBackAfterMissOrError(t *testing.T) { - anchor := makeGloasBlock(100, hash(0xAA), hash(0x10)) - anchorRoot, err := anchor.Block.HashSSZ() - require.NoError(t, err) - child := makeGloasBlock(101, hash(0xBB), hash(0xAA)) - child.Block.ParentRoot = anchorRoot - - for _, first := range []gloasLookaheadFetcher{ - func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) { return nil, nil }, - func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) { - return nil, assert.AnError - }, - } { - secondCalled := false - got, fetchErr := fetchGloasLookaheadFromSources(context.Background(), anchor, anchorRoot, 101, first, - func(context.Context, uint64, uint64) ([]*cltypes.SignedBeaconBlock, error) { - secondCalled = true - return []*cltypes.SignedBeaconBlock{child}, nil - }) - require.NoError(t, fetchErr) - assert.True(t, secondCalled) - assert.Same(t, child, got) - } -} - func TestBlockByRootFindsExpectedBlockInMiddle(t *testing.T) { first := makeGloasBlock(100, hash(0x10), hash(0x00)) expected := makeGloasBlock(101, hash(0x20), hash(0x10)) @@ -1023,7 +1077,7 @@ func TestRootFallbackMissingEnvelopeEntersBoundedRecovery(t *testing.T) { assert.Equal(t, common.Hash(blockRoot), common.Hash(downloader.skippedFullBlocks[0].Root)) } -func TestRootFallbackProbesLookaheadInferredEmptyBlock(t *testing.T) { +func TestRootFallbackCanonicalEmptyRejectsLateEnvelope(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, envelope := makeValidGloasEnvelope(t, block) child := makeGloasBlock(101, hash(0xBB), hash(0xCC)) @@ -1059,7 +1113,7 @@ func TestRootFallbackProbesLookaheadInferredEmptyBlock(t *testing.T) { } require.NoError(t, downloader.processResponses(context.Background(), nil)) - require.NoError(t, ValidateFetchedEnvelope(&clparams.MainnetBeaconConfig, block, common.Hash(blockRoot), processedEnvelope)) + require.Nil(t, processedEnvelope) } func TestRootFallbackLookaheadFailureEntersBoundedRecovery(t *testing.T) { @@ -1168,11 +1222,9 @@ func TestRootFallbackFailingProbeDoesNotTrackCanonicallyProvenEmptyBlock(t *test }, } - for range 3 { - require.NoError(t, downloader.processResponses(context.Background(), nil)) - } + require.NoError(t, downloader.processResponses(context.Background(), nil)) require.True(t, processed) - require.True(t, downloader.envelopesSkipped) + require.False(t, downloader.envelopesSkipped) require.Empty(t, downloader.skippedFullBlocks) } @@ -1314,25 +1366,6 @@ func TestHTTPRecoveryUsesBlockRoot(t *testing.T) { assert.Equal(t, "/eth/v1/beacon/execution_payload_envelope/"+common.Hash(blockRoot).Hex(), requestedPath) } -func TestFetchEnvelopeRecoverySourcesStartsAllSourcesBeforeDeadline(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - wantRoot := hash(0x42) - wantEnvelope := &cltypes.SignedExecutionPayloadEnvelope{} - - got := fetchEnvelopeRecoverySources(ctx, - func(ctx context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { - <-ctx.Done() - return nil - }, - func(context.Context) map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope { - return map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{wantRoot: wantEnvelope} - }, - ) - - require.Same(t, wantEnvelope, got[wantRoot]) -} - func TestValidateFetchedEnvelopesDropsMalformedSameRoot(t *testing.T) { block := makeGloasBlock(100, hash(0xAA), hash(0x10)) blockRoot, err := block.Block.HashSSZ() diff --git a/cl/phase1/network/beacon_downloader.go b/cl/phase1/network/beacon_downloader.go index 5483f56c8e0..b48a542e5fc 100644 --- a/cl/phase1/network/beacon_downloader.go +++ b/cl/phase1/network/beacon_downloader.go @@ -17,6 +17,7 @@ package network import ( + "bytes" "cmp" "context" "errors" @@ -47,6 +48,9 @@ type ProcessFn func( newHighestSlotProcessed uint64, err error) +// ValidateBlocksFn returns the authenticated prefix available for envelope classification. +type ValidateBlocksFn func([]*cltypes.SignedBeaconBlock) (int, error) + type ForwardBeaconDownloader struct { ctx context.Context highestSlotProcessed uint64 @@ -54,6 +58,7 @@ type ForwardBeaconDownloader struct { minSlot uint64 // earliest requestable slot (e.g. checkpoint anchor) rpc *rpc.BeaconRpcP2P process ProcessFn + validate ValidateBlocksFn beaconCfg *clparams.BeaconChainConfig httpFallbackURL string // beacon API base URL for HTTP fallback when P2P fails httpPreferred atomic.Bool // set after first HTTP fallback success; skips P2P probing @@ -76,6 +81,12 @@ func (f *ForwardBeaconDownloader) SetProcessFunction(fn ProcessFn) { f.process = fn } +func (f *ForwardBeaconDownloader) SetValidateFunction(fn ValidateBlocksFn) { + f.mu.Lock() + defer f.mu.Unlock() + f.validate = fn +} + // SetHTTPFallbackURL sets the beacon API base URL for HTTP-based block fetching // when P2P blocks_by_range requests fail. Derived from the checkpoint sync URL. func (f *ForwardBeaconDownloader) SetHTTPFallbackURL(checkpointSyncURL string) { @@ -280,14 +291,28 @@ Process: // then trim to `count` before processing. var envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope if anyGloasBlock(processBlocks) { + if f.validate == nil { + return + } + authenticatedCount, err := f.validate(processBlocks) + if err != nil { + if pid != "http-fallback" && f.rpc != nil { + f.rpc.BanPeer(pid) + } + return + } + if authenticatedCount < 2 || authenticatedCount > len(processBlocks) { + return + } + processBlocks = processBlocks[:authenticatedCount] // Always keep at least 1 block as lookahead so the last processed // block's FULL/EMPTY status is determined from the actual next block // rather than guessed as EMPTY. Without this, a FULL block at the // batch boundary has its envelope skipped, and the next batch's first // block fails with ErrParentEnvelopePending. processCount := min(int(count), len(processBlocks)-1) - if processCount < 1 { - processCount = len(processBlocks) // single block: process it (best-effort) + if processCount == 0 { + return } fullRoots := determineFullGloasRoots(processBlocks, processCount) processBlocks = processBlocks[:processCount] @@ -517,6 +542,10 @@ func fetchBlocksFromBeaconAPI(ctx context.Context, baseURL string, startSlot, co results[idx].err = fmt.Errorf("HTTP block decode slot %d: %w", slot, err) return } + if err := requireCanonicalSSZ(body, block); err != nil { + results[idx].err = fmt.Errorf("HTTP block decode slot %d: %w", slot, err) + return + } results[idx].block = block }) } @@ -640,6 +669,10 @@ func fetchEnvelopesFromBeaconAPI( log.Debug("[ForwardBeaconDownloader] HTTP envelope decode failed", "root", common.Hash(root), "err", err) return } + if err := requireCanonicalSSZ(body, envelope); err != nil { + log.Debug("[ForwardBeaconDownloader] HTTP envelope decode failed", "root", common.Hash(root), "err", err) + return + } block := blockByRoot(blocks, common.Hash(root)) if err := ValidateFetchedEnvelope(beaconCfg, block, common.Hash(root), envelope); err != nil { log.Debug("[ForwardBeaconDownloader] HTTP envelope mismatch", "root", common.Hash(root), "err", err) @@ -660,6 +693,21 @@ func fetchEnvelopesFromBeaconAPI( return fetched } +type sszCanonicalEncoder interface { + EncodeSSZ([]byte) ([]byte, error) +} + +func requireCanonicalSSZ(input []byte, value sszCanonicalEncoder) error { + encoded, err := value.EncodeSSZ(nil) + if err != nil { + return err + } + if !bytes.Equal(input, encoded) { + return errors.New("non-canonical SSZ encoding") + } + return nil +} + // GetHighestProcessedSlot retrieve the highest processed slot we accumulated. func (f *ForwardBeaconDownloader) GetHighestProcessedSlot() uint64 { f.mu.Lock() diff --git a/cl/phase1/network/beacon_downloader_test.go b/cl/phase1/network/beacon_downloader_test.go index 0f4f14b9b64..78566c7c48e 100644 --- a/cl/phase1/network/beacon_downloader_test.go +++ b/cl/phase1/network/beacon_downloader_test.go @@ -1,9 +1,18 @@ package network import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" "testing" "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/common" ) func TestShouldBanIncompleteBlockResponse(t *testing.T) { @@ -13,3 +22,126 @@ func TestShouldBanIncompleteBlockResponse(t *testing.T) { require.False(t, shouldBanIncompleteBlockResponse("", 1, 0)) require.False(t, shouldBanIncompleteBlockResponse("http-fallback", 1, 0)) } + +func TestForwardBeaconDownloaderHTTPRetainsSingleGloasTipUntilLookahead(t *testing.T) { + block := makeGloasBlock(100, hash(0xaa), common.Hash{}) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + child := makeGloasBlock(101, hash(0xbb), block.Block.Body.GetSignedExecutionPayloadBid().Message.BlockHash) + linkGloasBlocks(t, block, child) + blockBytes, err := block.EncodeSSZ(nil) + require.NoError(t, err) + childBytes, err := child.EncodeSSZ(nil) + require.NoError(t, err) + envelopeBytes, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + var childAvailable atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Eth-Consensus-Version", "gloas") + switch r.URL.Path { + case "/eth/v2/beacon/blocks/100": + _, _ = w.Write(blockBytes) + case "/eth/v2/beacon/blocks/101": + if !childAvailable.Load() { + http.NotFound(w, r) + return + } + _, _ = w.Write(childBytes) + case "/eth/v1/beacon/execution_payload_envelope/" + common.Hash(blockRoot).Hex(): + _, _ = w.Write(envelopeBytes) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + downloader := NewForwardBeaconDownloader(t.Context(), nil, &clparams.MainnetBeaconConfig) + downloader.SetHighestProcessedSlot(99) + downloader.SetHTTPFallbackURL(server.URL) + downloader.httpPreferred.Store(true) + downloader.SetValidateFunction(func(blocks []*cltypes.SignedBeaconBlock) (int, error) { return len(blocks), nil }) + var processCalls atomic.Int32 + downloader.SetProcessFunction(func(_ uint64, blocks []*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) (uint64, error) { + processCalls.Add(1) + require.Len(t, blocks, 1) + require.Equal(t, uint64(100), blocks[0].Block.Slot) + require.Contains(t, envelopes, common.Hash(blockRoot)) + return blocks[0].Block.Slot, nil + }) + + downloader.RequestMore(context.Background()) + require.Zero(t, processCalls.Load()) + require.Equal(t, uint64(99), downloader.GetHighestProcessedSlot()) + + childAvailable.Store(true) + downloader.RequestMore(context.Background()) + require.Equal(t, int32(1), processCalls.Load()) + require.Equal(t, uint64(100), downloader.GetHighestProcessedSlot()) +} + +func TestForwardBeaconDownloaderRejectsUnauthenticatedLookaheadBeforeHTTPEnvelopeFetch(t *testing.T) { + block := makeGloasBlock(100, hash(0xaa), common.Hash{}) + blockRoot, envelope := makeValidGloasEnvelope(t, block) + child := makeGloasBlock(101, hash(0xbb), block.Block.Body.GetSignedExecutionPayloadBid().Message.BlockHash) + linkGloasBlocks(t, block, child) + blockBytes, err := block.EncodeSSZ(nil) + require.NoError(t, err) + childBytes, err := child.EncodeSSZ(nil) + require.NoError(t, err) + envelopeBytes, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + var envelopeRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Eth-Consensus-Version", "gloas") + switch r.URL.Path { + case "/eth/v2/beacon/blocks/100": + _, _ = w.Write(blockBytes) + case "/eth/v2/beacon/blocks/101": + _, _ = w.Write(childBytes) + case "/eth/v1/beacon/execution_payload_envelope/" + common.Hash(blockRoot).Hex(): + envelopeRequests.Add(1) + _, _ = w.Write(envelopeBytes) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + downloader := NewForwardBeaconDownloader(t.Context(), nil, &clparams.MainnetBeaconConfig) + downloader.SetHighestProcessedSlot(99) + downloader.SetHTTPFallbackURL(server.URL) + downloader.httpPreferred.Store(true) + downloader.SetValidateFunction(func([]*cltypes.SignedBeaconBlock) (int, error) { + return 0, errors.New("invalid proposer signature") + }) + downloader.SetProcessFunction(func(uint64, []*cltypes.SignedBeaconBlock, map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) (uint64, error) { + t.Fatal("rejected response must not be processed") + return 0, nil + }) + + downloader.RequestMore(t.Context()) + + require.Zero(t, envelopeRequests.Load()) + require.Equal(t, uint64(99), downloader.GetHighestProcessedSlot()) +} + +func TestFetchBlocksFromBeaconAPIRejectsNonCanonicalSSZ(t *testing.T) { + block := makeGloasBlock(100, hash(0xaa), hash(0x01)) + encoded, err := block.EncodeSSZ(nil) + require.NoError(t, err) + encoded = append(encoded, 0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Eth-Consensus-Version", "gloas") + if r.URL.Path != "/eth/v2/beacon/blocks/100" { + http.NotFound(w, r) + return + } + _, _ = w.Write(encoded) + })) + defer server.Close() + + _, err = fetchBlocksFromBeaconAPI(t.Context(), server.URL, 100, 1, &clparams.MainnetBeaconConfig) + require.Error(t, err) +} diff --git a/cl/phase1/network/services/block_service.go b/cl/phase1/network/services/block_service.go index 990a11a95ff..f7c9f1db126 100644 --- a/cl/phase1/network/services/block_service.go +++ b/cl/phase1/network/services/block_service.go @@ -36,6 +36,7 @@ import ( "github.com/erigontech/erigon/cl/phase1/core/state/lru" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice" + "github.com/erigontech/erigon/cl/transition" "github.com/erigontech/erigon/cl/transition/impl/eth2" "github.com/erigontech/erigon/cl/utils/eth_clock" "github.com/erigontech/erigon/common" @@ -51,8 +52,9 @@ type proposerIndexAndSlot struct { } type blockJob struct { - block *cltypes.SignedBeaconBlock - creationTime time.Time + block *cltypes.SignedBeaconBlock + creationTime time.Time + resolveParentEnvelope bool } type blockService struct { @@ -68,7 +70,8 @@ type blockService struct { emitter *beaconevents.EventEmitter blocksScheduledForLaterExecution sync.Map // store the block in db - db kv.RwDB + db kv.RwDB + envelopeResolver executionPayloadEnvelopeResolver } // NewBlockService creates a new block service @@ -80,19 +83,21 @@ func NewBlockService( ethClock eth_clock.EthereumClock, beaconCfg *clparams.BeaconChainConfig, emitter *beaconevents.EventEmitter, + envelopeResolver executionPayloadEnvelopeResolver, ) BlockService { seenBlocksCache, err := lru.New[proposerIndexAndSlot, struct{}]("seenblocks", seenBlockCacheSize) if err != nil { panic(err) } b := &blockService{ - forkchoiceStore: forkchoiceStore, - syncedData: syncedData, - ethClock: ethClock, - beaconCfg: beaconCfg, - seenBlocksCache: seenBlocksCache, - emitter: emitter, - db: db, + forkchoiceStore: forkchoiceStore, + syncedData: syncedData, + ethClock: ethClock, + beaconCfg: beaconCfg, + seenBlocksCache: seenBlocksCache, + emitter: emitter, + db: db, + envelopeResolver: envelopeResolver, } go b.loop(ctx) return b @@ -155,7 +160,7 @@ func (b *blockService) ProcessMessage(ctx context.Context, _ *uint64, msg *cltyp return nil }); err != nil { if errors.Is(err, ErrIgnore) { - b.scheduleBlockForLaterProcessing(msg) + b.scheduleBlockForLaterProcessing(msg, false) } return err } @@ -163,7 +168,7 @@ func (b *blockService) ProcessMessage(ctx context.Context, _ *uint64, msg *cltyp // [IGNORE] The block's parent (defined by block.parent_root) has been seen (via both gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is retrieved). parentHeader, ok := b.forkchoiceStore.GetHeader(msg.Block.ParentRoot) if !ok { - b.scheduleBlockForLaterProcessing(msg) + b.scheduleBlockForLaterProcessing(msg, false) return fmt.Errorf("%w: parent header not found: %v", ErrIgnore, msg.Block.ParentRoot) } if parentHeader.Slot >= msg.Block.Slot { @@ -197,6 +202,10 @@ func (b *blockService) ProcessMessage(ctx context.Context, _ *uint64, msg *cltyp if bid.Message.ParentBlockRoot != msg.Block.ParentRoot { return errors.New("bid.parent_block_root does not match block.parent_root") } + if err := b.authenticateEnvelopeResolutionTrigger(msg); err != nil { + return err + } + resolveParentEnvelope := b.gloasChildProvesFull(msg) // [IGNORE] The block's parent execution payload (defined by bid.parent_block_hash) has been seen // (via gossip or non-gossip sources). A client MAY queue blocks for processing once the parent payload is retrieved. @@ -205,8 +214,10 @@ func (b *blockService) ProcessMessage(ctx context.Context, _ *uint64, msg *cltyp parentBlockHash := bid.Message.ParentBlockHash status, seen := b.forkchoiceStore.GetRecentExecutionPayloadStatus(parentBlockHash) if !seen { - // Parent execution payload not seen yet, queue for later - b.scheduleBlockForLaterProcessing(msg) + if resolveParentEnvelope && b.envelopeResolver != nil { + b.envelopeResolver.ResolveExecutionPayloadEnvelope(msg.Block.ParentRoot) + } + b.scheduleBlockForLaterProcessing(msg, resolveParentEnvelope) return fmt.Errorf("%w: parent execution payload not seen: %v", ErrIgnore, parentBlockHash) } if status == execution_client.PayloadStatusInvalidated { @@ -220,8 +231,18 @@ func (b *blockService) ProcessMessage(ctx context.Context, _ *uint64, msg *cltyp b.publishBlockGossipEvent(msg) // the rest of the validation is done in the forkchoice store if err := b.processAndStoreBlock(ctx, msg); err != nil { - if errors.Is(err, forkchoice.ErrEIP4844DataNotAvailable) || errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) || errors.Is(err, forkchoice.ErrParentEnvelopePending) { - b.scheduleBlockForLaterProcessing(msg) + if errors.Is(err, forkchoice.ErrParentEnvelopePending) { + if blockVersion < clparams.GloasVersion { + return err + } + if b.envelopeResolver != nil { + b.envelopeResolver.ResolveExecutionPayloadEnvelope(msg.Block.ParentRoot) + } + b.scheduleBlockForLaterProcessing(msg, true) + return nil + } + if errors.Is(err, forkchoice.ErrEIP4844DataNotAvailable) || errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) { + b.scheduleBlockForLaterProcessing(msg, false) return nil } return err @@ -229,6 +250,94 @@ func (b *blockService) ProcessMessage(ctx context.Context, _ *uint64, msg *cltyp return nil } +func (b *blockService) gloasChildProvesFull(block *cltypes.SignedBeaconBlock) bool { + parent, ok := b.forkchoiceStore.GetBlock(block.Block.ParentRoot) + if !ok || parent == nil || parent.Block == nil || parent.Block.Body == nil { + return false + } + parentBid := parent.Block.Body.GetSignedExecutionPayloadBid() + childBid := block.Block.Body.GetSignedExecutionPayloadBid() + return parentBid != nil && parentBid.Message != nil && childBid != nil && childBid.Message != nil && + childBid.Message.ParentBlockHash == parentBid.Message.BlockHash +} + +func (b *blockService) validateScheduledGloasBlock(block *cltypes.SignedBeaconBlock) error { + if block == nil || block.Block == nil || block.Block.Body == nil { + return errors.New("missing Gloas block body") + } + bid := block.Block.Body.GetSignedExecutionPayloadBid() + if bid == nil || bid.Message == nil { + return errors.New("missing signed_execution_payload_bid in Gloas block") + } + if bid.Message.ParentBlockRoot != block.Block.ParentRoot { + return errors.New("bid.parent_block_root does not match block.parent_root") + } + epoch := block.Block.Slot / b.beaconCfg.SlotsPerEpoch + if bid.Message.BlobKzgCommitments.Len() > int(b.beaconCfg.GetBlobParameters(epoch).MaxBlobsPerBlock) { + return ErrInvalidCommitmentsCount + } + return nil +} + +func (b *blockService) refreshScheduledResolverEligibility(key any, job *blockJob) (*blockJob, bool) { + if job.resolveParentEnvelope || b.beaconCfg.GetCurrentStateVersion(job.block.Block.Slot/b.beaconCfg.SlotsPerEpoch) < clparams.GloasVersion { + return job, true + } + if _, ok := b.forkchoiceStore.GetHeader(job.block.Block.ParentRoot); !ok { + return job, true + } + if err := b.validateScheduledGloasBlock(job.block); err != nil { + b.blocksScheduledForLaterExecution.CompareAndDelete(key, job) + return nil, false + } + if err := b.authenticateEnvelopeResolutionTrigger(job.block); err != nil { + if errors.Is(err, ErrIgnore) { + return job, true + } + b.blocksScheduledForLaterExecution.CompareAndDelete(key, job) + return nil, false + } + if !b.gloasChildProvesFull(job.block) { + return job, true + } + upgraded := *job + upgraded.resolveParentEnvelope = true + if !b.blocksScheduledForLaterExecution.CompareAndSwap(key, job, &upgraded) { + return nil, false + } + return &upgraded, true +} + +func (b *blockService) authenticateEnvelopeResolutionTrigger(block *cltypes.SignedBeaconBlock) error { + parentState, err := b.forkchoiceStore.GetStateAtBlockRoot(block.Block.ParentRoot, true) + if err != nil || parentState == nil { + return fmt.Errorf("%w: parent state unavailable", ErrIgnore) + } + if parentState.Slot() > block.Block.Slot { + return ErrBlockYoungerThanParent + } + if parentState.Slot() < block.Block.Slot { + if err := transition.DefaultMachine.ProcessSlots(parentState, block.Block.Slot); err != nil { + return err + } + } + expectedProposer, err := parentState.GetBeaconProposerIndex() + if err != nil { + return err + } + if block.Block.ProposerIndex != expectedProposer { + return ErrInvalidSignature + } + valid, err := eth2.VerifyBlockSignature(parentState, block) + if err != nil { + return err + } + if !valid { + return ErrInvalidSignature + } + return nil +} + // publishBlockGossipEvent publishes a block event which has not been processed yet func (b *blockService) publishBlockGossipEvent(block *cltypes.SignedBeaconBlock) { if b.emitter == nil { @@ -247,7 +356,7 @@ func (b *blockService) publishBlockGossipEvent(block *cltypes.SignedBeaconBlock) } // scheduleBlockForLaterProcessing schedules a block for later processing -func (b *blockService) scheduleBlockForLaterProcessing(block *cltypes.SignedBeaconBlock) { +func (b *blockService) scheduleBlockForLaterProcessing(block *cltypes.SignedBeaconBlock, resolveParentEnvelope bool) { // [Modified in Gloas:EIP7732] ExecutionPayload is not in block.body for GLOAS var blockNum uint64 if block.Block.Body.ExecutionPayload != nil { @@ -261,8 +370,9 @@ func (b *blockService) scheduleBlockForLaterProcessing(block *cltypes.SignedBeac } b.blocksScheduledForLaterExecution.Store(blockRoot, &blockJob{ - block: block, - creationTime: time.Now(), + block: block, + creationTime: time.Now(), + resolveParentEnvelope: resolveParentEnvelope, }) } @@ -295,6 +405,32 @@ func (b *blockService) processAndStoreBlock(ctx context.Context, block *cltypes. return nil } +func (b *blockService) deleteScheduledBlockJob(key any, expected *blockJob) { + b.blocksScheduledForLaterExecution.CompareAndDelete(key, expected) +} + +func (b *blockService) scheduledBlockJobExpired(job *blockJob) bool { + expired, _ := b.scheduledBlockJobExpiry(job) + return expired +} + +func (b *blockService) scheduledBlockJobExpiry(job *blockJob) (expired, envelopeGrace bool) { + if time.Since(job.creationTime) <= blockJobExpiry { + return false, false + } + if !job.resolveParentEnvelope || job.block == nil || job.block.Block == nil { + return true, false + } + root := job.block.Block.ParentRoot + if b.envelopeResolver != nil && b.envelopeResolver.HasPendingExecutionPayloadEnvelope(root) { + return false, false + } + if b.forkchoiceStore != nil && b.forkchoiceStore.HasEnvelope(root) { + return false, true + } + return true, false +} + // importBlockOperations imports block operations in parallel func (b *blockService) importBlockOperations(block *cltypes.SignedBeaconBlock) { defer func() { // Would prefer this not to crash but rather log the error @@ -330,19 +466,36 @@ func (b *blockService) loop(ctx context.Context) { return case <-ticker.C: } - b.blocksScheduledForLaterExecution.Range(func(key, value any) bool { - blockJob := value.(*blockJob) - // check if it has expired - if time.Since(blockJob.creationTime) > blockJobExpiry { - b.blocksScheduledForLaterExecution.Delete(key.([32]byte)) + b.processScheduledBlockJobs(ctx) + } +} + +func (b *blockService) processScheduledBlockJobs(ctx context.Context) { + b.blocksScheduledForLaterExecution.Range(func(key, value any) bool { + blockJob := value.(*blockJob) + // check if it has expired + expired, envelopeGrace := b.scheduledBlockJobExpiry(blockJob) + if expired { + b.deleteScheduledBlockJob(key, blockJob) + return true + } + var current bool + blockJob, current = b.refreshScheduledResolverEligibility(key, blockJob) + if !current { + return true + } + if err := b.processAndStoreBlock(ctx, blockJob.block); err != nil { + if envelopeGrace && (b.envelopeResolver == nil || !b.envelopeResolver.HasPendingExecutionPayloadEnvelope(blockJob.block.Block.ParentRoot)) { + b.deleteScheduledBlockJob(key, blockJob) return true } - if err := b.processAndStoreBlock(ctx, blockJob.block); err != nil { - log.Trace("Failed to process and store block", "block", blockJob.block, "error", err) - return true + if errors.Is(err, forkchoice.ErrParentEnvelopePending) && blockJob.resolveParentEnvelope && b.envelopeResolver != nil { + b.envelopeResolver.ResolveExecutionPayloadEnvelope(blockJob.block.Block.ParentRoot) } - b.blocksScheduledForLaterExecution.Delete(key.([32]byte)) + log.Trace("Failed to process and store block", "block", blockJob.block, "error", err) return true - }) - } + } + b.deleteScheduledBlockJob(key, blockJob) + return true + }) } diff --git a/cl/phase1/network/services/block_service_test.go b/cl/phase1/network/services/block_service_test.go index 19604549cb2..9f75df2e771 100644 --- a/cl/phase1/network/services/block_service_test.go +++ b/cl/phase1/network/services/block_service_test.go @@ -21,18 +21,27 @@ import ( "context" "errors" "testing" + "time" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "github.com/erigontech/erigon/cl/antiquary/tests" + "github.com/erigontech/erigon/cl/beacon/beaconevents" "github.com/erigontech/erigon/cl/beacon/synced_data" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" + "github.com/erigontech/erigon/cl/fork" + state2 "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice" "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" + "github.com/erigontech/erigon/cl/transition/impl/eth2" + clutils "github.com/erigontech/erigon/cl/utils" + "github.com/erigontech/erigon/cl/utils/bls" "github.com/erigontech/erigon/cl/utils/eth_clock" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/db/kv/memdb" @@ -47,13 +56,351 @@ func (s attesterSlashingErrorStore) OnAttesterSlashing(*cltypes.AttesterSlashing return s.err } +func newResolverTriggerBlock(t *testing.T, cfg *clparams.BeaconChainConfig, state *state2.CachingBeaconState, keys []*bls.PrivateKey, proposer uint64) (*cltypes.SignedBeaconBlock, common.Hash) { + t.Helper() + parentRoot := common.HexToHash("0x1234") + block := cltypes.NewSignedBeaconBlock(cfg, clparams.GloasVersion) + block.Block.Slot = cfg.SlotsPerEpoch + block.Block.ParentRoot = parentRoot + block.Block.ProposerIndex = proposer + block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockRoot = parentRoot + block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash = common.HexToHash("0xabcd") + epoch := block.Block.Slot / cfg.SlotsPerEpoch + var domain []byte + var err error + if cfg.GetCurrentStateVersion(epoch) > state.Version() { + forkVersion := clutils.Uint32ToBytes4(cfg.GetForkVersionByVersion(clparams.GloasVersion)) + domain, err = fork.ComputeDomain(cfg.DomainBeaconProposer[:], forkVersion, state.GenesisValidatorsRoot()) + } else { + domain, err = state.GetDomain(cfg.DomainBeaconProposer, epoch) + } + require.NoError(t, err) + signingRoot, err := fork.ComputeSigningRoot(block.Block, domain) + require.NoError(t, err) + copy(block.Signature[:], keys[proposer].Sign(signingRoot[:]).Bytes()) + valid, err := eth2.VerifyBlockSignature(state, block) + require.NoError(t, err) + require.True(t, valid) + return block, parentRoot +} + +func newResolverTriggerState(t *testing.T, cfg *clparams.BeaconChainConfig) (*state2.CachingBeaconState, []*bls.PrivateKey) { + t.Helper() + st := state2.New(cfg) + st.SetSlot(cfg.SlotsPerEpoch) + keys := make([]*bls.PrivateKey, 2) + for i := range keys { + key, err := bls.NewPrivateKeyFromIKM(append(make([]byte, 31), byte(i+1))) + require.NoError(t, err) + keys[i] = key + pubkey := common.Bytes48(bls.CompressPublicKey(key.PublicKey())) + st.AddValidator(solid.NewValidatorFromParameters(pubkey, common.Hash{}, cfg.MaxEffectiveBalance, false, 0, 0, cfg.FarFutureEpoch, cfg.FarFutureEpoch), cfg.MaxEffectiveBalance) + } + return st, keys +} + +func TestBlockServiceAuthenticatedFullChildWithUnseenParentStatusTriggersResolver(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.AltairForkEpoch = 0 + cfg.BellatrixForkEpoch = 0 + cfg.CapellaForkEpoch = 0 + cfg.DenebForkEpoch = 0 + cfg.ElectraForkEpoch = 0 + cfg.FuluForkEpoch = 0 + cfg.GloasForkEpoch = 0 + st, keys := newResolverTriggerState(t, &cfg) + expected, err := st.GetBeaconProposerIndex() + require.NoError(t, err) + block, parentRoot := newResolverTriggerBlock(t, &cfg, st, keys, expected) + + ctrl := gomock.NewController(t) + db := memdb.NewTestDB(t, dbcfg.ChainDB) + synced := synced_data.NewSyncedDataManager(&cfg, true) + synced.OnHeadState(st) + clock := eth_clock.NewMockEthereumClock(ctrl) + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Headers[parentRoot] = &cltypes.BeaconBlockHeader{Slot: block.Block.Slot - 1} + fcu.StateAtBlockRootVal[parentRoot] = st + parent := cltypes.NewSignedBeaconBlock(&cfg, clparams.GloasVersion) + parent.Block.Body.SignedExecutionPayloadBid.Message.BlockHash = block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash + fcu.Blocks[parentRoot] = parent + fcu.OnBlockErr = forkchoice.ErrParentEnvelopePending + requester := &envelopeRequesterStub{started: make(chan struct{}, 1), release: make(chan struct{})} + payloads := NewExecutionPayloadService(t.Context(), fcu, &cfg, beaconevents.NewEventEmitter(), requester) + payloads.resolver.deadline = 20 * time.Millisecond + payloads.resolver.retry = time.Hour + service := NewBlockService(t.Context(), db, fcu, synced, clock, &cfg, nil, payloads).(*blockService) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, block), ErrIgnore) + select { + case <-requester.started: + case <-time.After(time.Second): + t.Fatal("resolver request was not started") + } + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + job, ok := service.blocksScheduledForLaterExecution.Load(blockRoot) + require.True(t, ok) + require.True(t, job.(*blockJob).resolveParentEnvelope) + require.Eventually(t, func() bool { return requester.calls.Load() >= 2 }, time.Second, time.Millisecond) + close(requester.release) +} + +func TestBlockServiceWrongScheduledProposerTriggersNoResolver(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.AltairForkEpoch = 0 + cfg.BellatrixForkEpoch = 0 + cfg.CapellaForkEpoch = 0 + cfg.DenebForkEpoch = 0 + cfg.ElectraForkEpoch = 0 + cfg.FuluForkEpoch = 0 + cfg.GloasForkEpoch = 0 + st, keys := newResolverTriggerState(t, &cfg) + expected, err := st.GetBeaconProposerIndex() + require.NoError(t, err) + wrong := (expected + 1) % uint64(len(keys)) + block, parentRoot := newResolverTriggerBlock(t, &cfg, st, keys, wrong) + + ctrl := gomock.NewController(t) + db := memdb.NewTestDB(t, dbcfg.ChainDB) + synced := synced_data.NewSyncedDataManager(&cfg, true) + synced.OnHeadState(st) + clock := eth_clock.NewMockEthereumClock(ctrl) + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Headers[parentRoot] = &cltypes.BeaconBlockHeader{Slot: block.Block.Slot - 1} + fcu.StateAtBlockRootVal[parentRoot] = st + parent := cltypes.NewSignedBeaconBlock(&cfg, clparams.GloasVersion) + parent.Block.Body.SignedExecutionPayloadBid.Message.BlockHash = block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash + fcu.Blocks[parentRoot] = parent + requester := &envelopeRequesterStub{} + payloads := NewExecutionPayloadService(t.Context(), fcu, &cfg, beaconevents.NewEventEmitter(), requester) + service := NewBlockService(t.Context(), db, fcu, synced, clock, &cfg, nil, payloads) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, block), ErrInvalidSignature) + require.Zero(t, requester.calls.Load()) +} + +func TestBlockServiceChildBeforeParentUpgradesScheduledResolverEligibility(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.AltairForkEpoch, cfg.BellatrixForkEpoch, cfg.CapellaForkEpoch = 0, 0, 0 + cfg.DenebForkEpoch, cfg.ElectraForkEpoch, cfg.FuluForkEpoch, cfg.GloasForkEpoch = 0, 0, 0, 0 + st, keys := newResolverTriggerState(t, &cfg) + expected, err := st.GetBeaconProposerIndex() + require.NoError(t, err) + block, parentRoot := newResolverTriggerBlock(t, &cfg, st, keys, expected) + + ctrl := gomock.NewController(t) + db := memdb.NewTestDB(t, dbcfg.ChainDB) + synced := synced_data.NewSyncedDataManager(&cfg, true) + synced.OnHeadState(st) + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.OnBlockErr = forkchoice.ErrParentEnvelopePending + requester := &envelopeRequesterStub{} + payloads := NewExecutionPayloadService(t.Context(), fcu, &cfg, beaconevents.NewEventEmitter(), requester) + payloads.resolver.deadline, payloads.resolver.retry = 20*time.Millisecond, time.Hour + service := NewBlockService(t.Context(), db, fcu, synced, eth_clock.NewMockEthereumClock(ctrl), &cfg, nil, payloads).(*blockService) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, block), ErrIgnore) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + job, ok := service.blocksScheduledForLaterExecution.Load(blockRoot) + require.True(t, ok) + require.False(t, job.(*blockJob).resolveParentEnvelope) + + fcu.Headers[parentRoot] = &cltypes.BeaconBlockHeader{Slot: block.Block.Slot - 1} + fcu.StateAtBlockRootVal[parentRoot] = st + parent := cltypes.NewSignedBeaconBlock(&cfg, clparams.GloasVersion) + parent.Block.Body.SignedExecutionPayloadBid.Message.BlockHash = block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash + fcu.Blocks[parentRoot] = parent + fcu.ExecutionPayloadStatusMap[block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash] = execution_client.PayloadStatusValidated + + require.Eventually(t, func() bool { return requester.calls.Load() >= 2 }, time.Second, time.Millisecond) + job, ok = service.blocksScheduledForLaterExecution.Load(blockRoot) + require.True(t, ok) + require.True(t, job.(*blockJob).resolveParentEnvelope) +} + +func TestBlockServiceChildBeforeParentWrongProposerNeverTriggersResolver(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.AltairForkEpoch, cfg.BellatrixForkEpoch, cfg.CapellaForkEpoch = 0, 0, 0 + cfg.DenebForkEpoch, cfg.ElectraForkEpoch, cfg.FuluForkEpoch, cfg.GloasForkEpoch = 0, 0, 0, 0 + st, keys := newResolverTriggerState(t, &cfg) + expected, err := st.GetBeaconProposerIndex() + require.NoError(t, err) + wrong := (expected + 1) % uint64(len(keys)) + block, parentRoot := newResolverTriggerBlock(t, &cfg, st, keys, wrong) + + ctrl := gomock.NewController(t) + db := memdb.NewTestDB(t, dbcfg.ChainDB) + synced := synced_data.NewSyncedDataManager(&cfg, true) + synced.OnHeadState(st) + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.OnBlockErr = forkchoice.ErrParentEnvelopePending + requester := &envelopeRequesterStub{} + payloads := NewExecutionPayloadService(t.Context(), fcu, &cfg, beaconevents.NewEventEmitter(), requester) + service := NewBlockService(t.Context(), db, fcu, synced, eth_clock.NewMockEthereumClock(ctrl), &cfg, nil, payloads).(*blockService) + + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, block), ErrIgnore) + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + fcu.Headers[parentRoot] = &cltypes.BeaconBlockHeader{Slot: block.Block.Slot - 1} + fcu.StateAtBlockRootVal[parentRoot] = st + parent := cltypes.NewSignedBeaconBlock(&cfg, clparams.GloasVersion) + parent.Block.Body.SignedExecutionPayloadBid.Message.BlockHash = block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash + fcu.Blocks[parentRoot] = parent + fcu.ExecutionPayloadStatusMap[block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash] = execution_client.PayloadStatusValidated + + require.Eventually(t, func() bool { + _, exists := service.blocksScheduledForLaterExecution.Load(blockRoot) + return !exists + }, time.Second, time.Millisecond) + require.Zero(t, requester.calls.Load()) +} + +func TestBlockServiceExpiredObservationDoesNotDeleteFreshReplacement(t *testing.T) { + service := &blockService{} + root := [32]byte{1} + oldJob := &blockJob{creationTime: time.Now().Add(-blockJobExpiry - time.Second)} + freshJob := &blockJob{creationTime: time.Now()} + service.blocksScheduledForLaterExecution.Store(root, oldJob) + observed, ok := service.blocksScheduledForLaterExecution.Load(root) + require.True(t, ok) + service.blocksScheduledForLaterExecution.Store(root, freshJob) + + service.deleteScheduledBlockJob(root, observed.(*blockJob)) + + current, ok := service.blocksScheduledForLaterExecution.Load(root) + require.True(t, ok) + require.Same(t, freshJob, current) +} + +func TestBlockServiceExpiredTrustedChildIsRetainedWhileEnvelopePending(t *testing.T) { + root := common.HexToHash("0x1234") + payloads := &executionPayloadService{pendingRootCounts: map[common.Hash]int{root: 1}} + now := time.Now() + service := &blockService{envelopeResolver: payloads} + job := &blockJob{ + block: &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{ParentRoot: root}}, + creationTime: now.Add(-blockJobExpiry - time.Second), + resolveParentEnvelope: true, + } + require.False(t, service.scheduledBlockJobExpired(job)) +} + +func TestBlockServiceExpiredTrustedChildProcessesAfterEnvelopeSuccess(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + root := common.HexToHash("0x1234") + payloads := &executionPayloadService{pendingRootCounts: map[common.Hash]int{}} + now := time.Now() + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Envelopes[root] = newTestSignedEnvelope(0, root, 0) + service := &blockService{ + beaconCfg: cfg, + db: memdb.NewTestDB(t, dbcfg.ChainDB), + forkchoiceStore: fcu, + envelopeResolver: payloads, + } + block := cltypes.NewSignedBeaconBlock(cfg, clparams.DenebVersion) + block.Block.ParentRoot = root + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + job := &blockJob{block: block, creationTime: now.Add(-blockJobExpiry - time.Second), resolveParentEnvelope: true} + service.blocksScheduledForLaterExecution.Store(blockRoot, job) + + service.processScheduledBlockJobs(t.Context()) + + _, ok := service.blocksScheduledForLaterExecution.Load(blockRoot) + require.False(t, ok) + require.Equal(t, int32(1), fcu.OnBlockCalls.Load()) +} + +func TestBlockServiceExpiredTrustedChildDeletesAfterPendingTerminates(t *testing.T) { + root := common.HexToHash("0x1234") + payloads := &executionPayloadService{pendingRootCounts: map[common.Hash]int{}} + now := time.Now() + service := &blockService{envelopeResolver: payloads} + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{ParentRoot: root}} + blockRoot := [32]byte{1} + job := &blockJob{block: block, creationTime: now.Add(-blockJobExpiry - time.Second), resolveParentEnvelope: true} + service.blocksScheduledForLaterExecution.Store(blockRoot, job) + + service.processScheduledBlockJobs(t.Context()) + + _, ok := service.blocksScheduledForLaterExecution.Load(blockRoot) + require.False(t, ok) +} + +func TestBlockServiceExpiredEnvelopeGraceTerminalErrorAttemptsOnce(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + root := common.HexToHash("0x1234") + payloads := &executionPayloadService{pendingRootCounts: map[common.Hash]int{}} + now := time.Now() + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Envelopes[root] = newTestSignedEnvelope(0, root, 0) + fcu.OnBlockErr = errors.New("terminal block error") + service := &blockService{ + beaconCfg: cfg, + db: memdb.NewTestDB(t, dbcfg.ChainDB), + forkchoiceStore: fcu, + envelopeResolver: payloads, + } + block := cltypes.NewSignedBeaconBlock(cfg, clparams.DenebVersion) + block.Block.ParentRoot = root + blockRoot, err := block.Block.HashSSZ() + require.NoError(t, err) + job := &blockJob{block: block, creationTime: now.Add(-blockJobExpiry - time.Second), resolveParentEnvelope: true} + service.blocksScheduledForLaterExecution.Store(blockRoot, job) + + service.processScheduledBlockJobs(t.Context()) + service.processScheduledBlockJobs(t.Context()) + + require.Equal(t, int32(1), fcu.OnBlockCalls.Load()) + _, ok := service.blocksScheduledForLaterExecution.Load(blockRoot) + require.False(t, ok) +} + +func TestEnvelopeResolutionTriggerRejectsValidSignatureFromWrongProposer(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.FuluForkEpoch = cfg.FarFutureEpoch + cfg.GloasForkEpoch = cfg.FarFutureEpoch + parentState := state2.New(&cfg) + parentState.SetSlot(0) + keys := make([]*bls.PrivateKey, 2) + for i := range keys { + key, err := bls.NewPrivateKeyFromIKM(append(make([]byte, 31), byte(i+1))) + require.NoError(t, err) + keys[i] = key + pubkey := common.Bytes48(bls.CompressPublicKey(key.PublicKey())) + parentState.AddValidator(solid.NewValidatorFromParameters(pubkey, common.Hash{}, cfg.MaxEffectiveBalance, false, 0, 0, cfg.FarFutureEpoch, cfg.FarFutureEpoch), cfg.MaxEffectiveBalance) + } + expected, err := parentState.GetBeaconProposerIndex() + require.NoError(t, err) + wrong := (expected + 1) % uint64(len(keys)) + block := cltypes.NewSignedBeaconBlock(&cfg, clparams.DenebVersion) + block.Block.Slot = 0 + block.Block.ProposerIndex = wrong + domain, err := parentState.GetDomain(cfg.DomainBeaconProposer, 0) + require.NoError(t, err) + signingRoot, err := fork.ComputeSigningRoot(block.Block, domain) + require.NoError(t, err) + copy(block.Signature[:], keys[wrong].Sign(signingRoot[:]).Bytes()) + valid, err := eth2.VerifyBlockSignature(parentState, block) + require.NoError(t, err) + require.True(t, valid) + + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.StateAtBlockRootVal[block.Block.ParentRoot] = parentState + service := &blockService{forkchoiceStore: fcu} + require.ErrorIs(t, service.authenticateEnvelopeResolutionTrigger(block), ErrInvalidSignature) +} + func setupBlockService(t *testing.T, ctrl *gomock.Controller) (BlockService, *synced_data.SyncedDataManager, *eth_clock.MockEthereumClock, *mock_services.ForkChoiceStorageMock) { db := memdb.NewTestDB(t, dbcfg.ChainDB) cfg := &clparams.MainnetBeaconConfig syncedDataManager := synced_data.NewSyncedDataManager(cfg, true) ethClock := eth_clock.NewMockEthereumClock(ctrl) forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) - blockService := NewBlockService(t.Context(), db, forkchoiceMock, syncedDataManager, ethClock, cfg, nil) + blockService := NewBlockService(t.Context(), db, forkchoiceMock, syncedDataManager, ethClock, cfg, nil, nil) return blockService, syncedDataManager, ethClock, forkchoiceMock } diff --git a/cl/phase1/network/services/canonical_ssz.go b/cl/phase1/network/services/canonical_ssz.go new file mode 100644 index 00000000000..3878171da24 --- /dev/null +++ b/cl/phase1/network/services/canonical_ssz.go @@ -0,0 +1,21 @@ +package services + +import ( + "bytes" + "errors" +) + +type sszCanonicalEncoder interface { + EncodeSSZ([]byte) ([]byte, error) +} + +func requireCanonicalSSZ(input []byte, value sszCanonicalEncoder) error { + encoded, err := value.EncodeSSZ(nil) + if err != nil { + return err + } + if !bytes.Equal(input, encoded) { + return errors.New("non-canonical SSZ encoding") + } + return nil +} diff --git a/cl/phase1/network/services/envelope_resolver.go b/cl/phase1/network/services/envelope_resolver.go new file mode 100644 index 00000000000..3bfa0bfd28e --- /dev/null +++ b/cl/phase1/network/services/envelope_resolver.go @@ -0,0 +1,138 @@ +package services + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/phase1/forkchoice" + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/log/v3" +) + +const ( + maxEnvelopeResolverJobs = 128 + maxConcurrentEnvelopeResolvers = 4 + envelopeResolverDeadline = 12 * time.Second + envelopeResolverRetryInterval = 500 * time.Millisecond +) + +type executionPayloadEnvelopeRequester interface { + SendExecutionPayloadEnvelopesByRootReq(context.Context, [][32]byte) ([]*cltypes.SignedExecutionPayloadEnvelope, string, error) + BanPeer(string) +} + +type executionPayloadEnvelopeResolver interface { + ResolveExecutionPayloadEnvelope(common.Hash) + HasPendingExecutionPayloadEnvelope(common.Hash) bool +} + +type envelopeResolver struct { + ctx context.Context + requester executionPayloadEnvelopeRequester + processor *executionPayloadService + sem chan struct{} + mu sync.Mutex + jobs map[common.Hash]struct{} + deadline time.Duration + retry time.Duration +} + +func newEnvelopeResolver(ctx context.Context, requester executionPayloadEnvelopeRequester, processor *executionPayloadService) *envelopeResolver { + if requester == nil { + return nil + } + return &envelopeResolver{ + ctx: ctx, + requester: requester, + processor: processor, + sem: make(chan struct{}, maxConcurrentEnvelopeResolvers), + jobs: make(map[common.Hash]struct{}), + deadline: envelopeResolverDeadline, + retry: envelopeResolverRetryInterval, + } +} + +func (r *envelopeResolver) ResolveExecutionPayloadEnvelope(root common.Hash) { + r.mu.Lock() + if len(r.jobs) >= maxEnvelopeResolverJobs { + r.mu.Unlock() + return + } + if _, exists := r.jobs[root]; exists { + r.mu.Unlock() + return + } + r.jobs[root] = struct{}{} + r.mu.Unlock() + + go r.resolve(root) +} + +func (r *envelopeResolver) resolve(root common.Hash) { + defer func() { + if recovered := recover(); recovered != nil { + log.Error("Execution payload envelope resolver recovered from panic", "err", recovered) + } + r.mu.Lock() + delete(r.jobs, root) + r.mu.Unlock() + }() + + select { + case r.sem <- struct{}{}: + defer func() { <-r.sem }() + case <-r.ctx.Done(): + return + } + + ctx, cancel := context.WithTimeout(r.ctx, r.deadline) + defer cancel() + ticker := time.NewTicker(r.retry) + defer ticker.Stop() + + for { + if r.processor.forkchoiceStore.HasEnvelope(root) { + return + } + if r.processor.hasPendingEnvelopeRoot(root) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + continue + } + } + envelopes, pid, err := r.requester.SendExecutionPayloadEnvelopesByRootReq(ctx, [][32]byte{root}) + if err == nil && r.processResponses(ctx, root, pid, envelopes) { + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (r *envelopeResolver) processResponses(ctx context.Context, root common.Hash, pid string, envelopes []*cltypes.SignedExecutionPayloadEnvelope) bool { + for _, envelope := range envelopes { + if envelope == nil || envelope.Message == nil || envelope.Message.BeaconBlockRoot != root { + r.requester.BanPeer(pid) + continue + } + err := r.processor.ProcessMessage(ctx, nil, envelope) + if err == nil || r.processor.forkchoiceStore.HasEnvelope(root) { + return true + } + if errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) || errors.Is(err, forkchoice.ErrELPayloadValidationUnavailable) { + return false + } + if !errors.Is(err, ErrIgnore) { + r.requester.BanPeer(pid) + } + } + return false +} diff --git a/cl/phase1/network/services/envelope_resolver_test.go b/cl/phase1/network/services/envelope_resolver_test.go new file mode 100644 index 00000000000..0c345124b52 --- /dev/null +++ b/cl/phase1/network/services/envelope_resolver_test.go @@ -0,0 +1,197 @@ +package services + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/cl/beacon/beaconevents" + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/phase1/forkchoice" + "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" + "github.com/erigontech/erigon/common" +) + +type envelopeRequesterStub struct { + mu sync.Mutex + responses [][]*cltypes.SignedExecutionPayloadEnvelope + calls atomic.Int32 + bans atomic.Int32 + started chan struct{} + release chan struct{} +} + +func (s *envelopeRequesterStub) SendExecutionPayloadEnvelopesByRootReq(ctx context.Context, _ [][32]byte) ([]*cltypes.SignedExecutionPayloadEnvelope, string, error) { + s.calls.Add(1) + if s.started != nil { + select { + case s.started <- struct{}{}: + default: + } + } + if s.release != nil { + select { + case <-s.release: + case <-ctx.Done(): + return nil, "peer", ctx.Err() + } + } + s.mu.Lock() + defer s.mu.Unlock() + if len(s.responses) == 0 { + return nil, "peer", nil + } + response := s.responses[0] + s.responses = s.responses[1:] + return response, "peer", nil +} + +func (s *envelopeRequesterStub) BanPeer(string) { s.bans.Add(1) } + +func TestEnvelopeResolverInvalidResponderThenValid(t *testing.T) { + root := common.HexToHash("0x1234") + wrong := newTestSignedEnvelope(100, common.HexToHash("0x5678"), 1) + valid := newTestSignedEnvelope(100, root, 2) + requester := &envelopeRequesterStub{responses: [][]*cltypes.SignedExecutionPayloadEnvelope{{wrong, valid}}} + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Blocks[root] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + processed := make(chan struct{}) + fcu.OnExecutionPayloadFunc = func(candidate *cltypes.SignedExecutionPayloadEnvelope) error { + fcu.Envelopes[root] = candidate + close(processed) + return nil + } + service := NewExecutionPayloadService(t.Context(), fcu, &clparams.MainnetBeaconConfig, beaconevents.NewEventEmitter(), requester) + service.ResolveExecutionPayloadEnvelope(root) + select { + case <-processed: + case <-time.After(time.Second): + t.Fatal("resolver did not process the valid response") + } + require.True(t, fcu.HasEnvelope(root)) + require.Equal(t, int32(1), requester.bans.Load()) + require.Equal(t, int32(1), requester.calls.Load()) +} + +func TestEnvelopeResolverSameRootTriggersCoalesce(t *testing.T) { + root := common.HexToHash("0x1234") + requester := &envelopeRequesterStub{started: make(chan struct{}, 1), release: make(chan struct{})} + fcu := mock_services.NewForkChoiceStorageMock(t) + service := NewExecutionPayloadService(t.Context(), fcu, &clparams.MainnetBeaconConfig, beaconevents.NewEventEmitter(), requester) + for range 100 { + service.ResolveExecutionPayloadEnvelope(root) + } + <-requester.started + require.Equal(t, int32(1), requester.calls.Load()) + close(requester.release) +} + +func TestEnvelopeResolverServiceCancellationStopsRequest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + requester := &envelopeRequesterStub{started: make(chan struct{}, 1), release: make(chan struct{})} + fcu := mock_services.NewForkChoiceStorageMock(t) + service := NewExecutionPayloadService(ctx, fcu, &clparams.MainnetBeaconConfig, beaconevents.NewEventEmitter(), requester) + service.ResolveExecutionPayloadEnvelope(common.HexToHash("0x1234")) + <-requester.started + cancel() + require.Eventually(t, func() bool { + service.resolver.mu.Lock() + defer service.resolver.mu.Unlock() + return len(service.resolver.jobs) == 0 + }, time.Second, time.Millisecond) +} + +func TestEnvelopeResolverDeadlineStopsRetries(t *testing.T) { + requester := &envelopeRequesterStub{} + fcu := mock_services.NewForkChoiceStorageMock(t) + service := NewExecutionPayloadService(t.Context(), fcu, &clparams.MainnetBeaconConfig, beaconevents.NewEventEmitter(), requester) + service.resolver.deadline = 20 * time.Millisecond + service.resolver.retry = time.Millisecond + service.ResolveExecutionPayloadEnvelope(common.HexToHash("0x1234")) + require.Eventually(t, func() bool { + service.resolver.mu.Lock() + defer service.resolver.mu.Unlock() + return requester.calls.Load() > 0 && len(service.resolver.jobs) == 0 + }, time.Second, time.Millisecond) +} + +func TestEnvelopeResolverGossipAndFetchRace(t *testing.T) { + root := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, root, 2) + requester := &envelopeRequesterStub{responses: [][]*cltypes.SignedExecutionPayloadEnvelope{{envelope}}, release: make(chan struct{}), started: make(chan struct{}, 1)} + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Blocks[root] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + var calls atomic.Int32 + fcu.OnExecutionPayloadFunc = func(candidate *cltypes.SignedExecutionPayloadEnvelope) error { + calls.Add(1) + fcu.Envelopes[root] = candidate + return nil + } + service := NewExecutionPayloadService(t.Context(), fcu, &clparams.MainnetBeaconConfig, beaconevents.NewEventEmitter(), requester) + service.ResolveExecutionPayloadEnvelope(root) + <-requester.started + require.NoError(t, service.ProcessMessage(t.Context(), nil, envelope)) + close(requester.release) + require.Eventually(t, func() bool { + service.resolver.mu.Lock() + defer service.resolver.mu.Unlock() + return len(service.resolver.jobs) == 0 + }, time.Second, time.Millisecond) + require.Equal(t, int32(1), calls.Load()) + require.Zero(t, service.pendingCount.Load()) +} + +func TestExecutionPayloadServiceUnknownRootFloodRetainsNothing(t *testing.T) { + service, _ := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + for i := range maxEnvelopeResolverJobs * 2 { + root := common.Hash{byte(i), byte(i >> 8)} + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, newTestSignedEnvelope(100, root, uint64(i))), ErrIgnore) + } + require.Zero(t, impl.pendingCount.Load()) +} + +func TestEnvelopeResolverDoesNotRefetchWhileAuthenticatedRetryOwnsRoot(t *testing.T) { + root := common.HexToHash("0x1234") + envelope := newTestSignedEnvelope(100, root, 2) + requester := &envelopeRequesterStub{responses: [][]*cltypes.SignedExecutionPayloadEnvelope{{envelope}}} + fcu := mock_services.NewForkChoiceStorageMock(t) + fcu.Blocks[root] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + var terminal atomic.Bool + fcu.OnExecutionPayloadFunc = func(*cltypes.SignedExecutionPayloadEnvelope) error { + if terminal.Load() { + return errors.New("terminal invalid envelope") + } + return forkchoice.ErrEIP7594ColumnDataNotAvailable + } + service := NewExecutionPayloadService(t.Context(), fcu, &clparams.MainnetBeaconConfig, beaconevents.NewEventEmitter(), requester) + service.resolver.deadline = time.Second + service.resolver.retry = 5 * time.Millisecond + service.ResolveExecutionPayloadEnvelope(root) + require.Eventually(t, func() bool { return service.pendingCount.Load() == 1 }, time.Second, time.Millisecond) + for range 100 { + service.ResolveExecutionPayloadEnvelope(root) + } + select { + case <-time.After(40 * time.Millisecond): + case <-t.Context().Done(): + t.Fatal(t.Context().Err()) + } + require.Equal(t, int32(1), requester.calls.Load()) + + terminal.Store(true) + key := pendingEnvelopeKeyForTest(t, envelope) + value, ok := service.pendingEnvelopes.Load(key) + require.True(t, ok) + service.pendingMu.Lock() + value.(*envelopeJob).nextAttempt = time.Time{} + service.pendingMu.Unlock() + service.processPendingEnvelopes(t.Context()) + require.Eventually(t, func() bool { return requester.calls.Load() >= 2 }, time.Second, time.Millisecond) +} diff --git a/cl/phase1/network/services/execution_payload_bid_service.go b/cl/phase1/network/services/execution_payload_bid_service.go index ab68dea8a0a..df3501f9bfe 100644 --- a/cl/phase1/network/services/execution_payload_bid_service.go +++ b/cl/phase1/network/services/execution_payload_bid_service.go @@ -148,6 +148,9 @@ func (s *executionPayloadBidService) DecodeGossipMessage(_ peer.ID, data []byte, if err := msg.DecodeSSZ(data, int(version)); err != nil { return nil, err } + if err := requireCanonicalSSZ(data, msg); err != nil { + return nil, err + } return msg, nil } @@ -410,18 +413,15 @@ func (s *executionPayloadBidService) bidValidationState(parentBlockRoot common.H } if parentState.Slot() > bidSlot { s.removeBidValidationState(cacheKey, entry) - return nil, fmt.Errorf("parent state slot %d is after bid slot %d", parentState.Slot(), bidSlot) - } - validationState := parentState - if parentState.Slot() == bidSlot { - entry.state = validationState - return entry, nil + return nil, fmt.Errorf("bid slot %d is before parent state slot %d", bidSlot, parentState.Slot()) } - if err := transition.DefaultMachine.ProcessSlots(validationState, bidSlot); err != nil { - s.removeBidValidationState(cacheKey, entry) - return nil, err + if parentState.Slot() < bidSlot { + if err := transition.DefaultMachine.ProcessSlots(parentState, bidSlot); err != nil { + s.removeBidValidationState(cacheKey, entry) + return nil, fmt.Errorf("failed to advance parent state to bid slot %d: %w", bidSlot, err) + } } - entry.state = validationState + entry.state = parentState return entry, nil } diff --git a/cl/phase1/network/services/execution_payload_bid_service_test.go b/cl/phase1/network/services/execution_payload_bid_service_test.go index b8e6f694f48..27a0b72da68 100644 --- a/cl/phase1/network/services/execution_payload_bid_service_test.go +++ b/cl/phase1/network/services/execution_payload_bid_service_test.go @@ -796,6 +796,19 @@ func TestExecutionPayloadBidServiceDecodeGossipMessageInvalid(t *testing.T) { require.Error(t, err) } +func TestExecutionPayloadBidServiceRejectsNonCanonicalSSZ(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + service, _, _, _, _ := setupExecutionPayloadBidService(t, ctrl) + original := newTestSignedExecutionPayloadBid(100, 1, 1000) + encoded, err := original.EncodeSSZ(nil) + require.NoError(t, err) + encoded = append(encoded, 0) + + _, err = service.DecodeGossipMessage("peer123", encoded, clparams.GloasVersion) + require.Error(t, err) +} + func TestExecutionPayloadBidServiceNonZeroExecutionPayment(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/cl/phase1/network/services/execution_payload_service.go b/cl/phase1/network/services/execution_payload_service.go index 5d14c72b09a..c15d1152662 100644 --- a/cl/phase1/network/services/execution_payload_service.go +++ b/cl/phase1/network/services/execution_payload_service.go @@ -41,15 +41,9 @@ type seenEnvelopeKey struct { builderIndex uint64 } -// pendingEnvelopeKey tracks envelopes waiting for their block or data columns. -// We use (blockRoot, envelopeHash) as key instead of just blockRoot because: -// - Multiple envelopes (including forged ones) may arrive before the block -// - Using only blockRoot would cause later arrivals to overwrite earlier ones -// - If a forged envelope overwrites the valid one, we lose the valid envelope -// - With envelopeHash, all candidates are kept and validated when block arrives type pendingEnvelopeKey struct { - blockRoot common.Hash - envelopeHash common.Hash + blockRoot common.Hash + messageHash common.Hash } // envelopeJob represents an envelope waiting for its dependencies. @@ -59,6 +53,7 @@ type envelopeJob struct { nextAttempt time.Time blockSeen atomic.Bool resolving atomic.Bool + validated bool } const ( @@ -67,6 +62,7 @@ const ( pendingEnvelopeCheckInterval = 100 * time.Millisecond pendingEnvelopeRetryInterval = time.Second maxPendingEnvelopes = 1024 + maxPendingCandidatesPerRoot = 4 ) type executionPayloadService struct { @@ -78,10 +74,12 @@ type executionPayloadService struct { seenEnvelopesCache *lru.Cache[seenEnvelopeKey, struct{}] // Pending envelopes waiting for their dependencies - pendingEnvelopes sync.Map // pendingEnvelopeKey -> *envelopeJob - pendingCount atomic.Int32 - pendingCond *sync.Cond - pendingMu sync.Mutex + pendingEnvelopes sync.Map // pendingEnvelopeKey -> *envelopeJob + pendingCount atomic.Int32 + pendingRootCounts map[common.Hash]int + pendingCond *sync.Cond + pendingMu sync.Mutex + resolver *envelopeResolver } // NewExecutionPayloadService creates a new execution payload service @@ -90,7 +88,8 @@ func NewExecutionPayloadService( forkchoiceStore forkchoice.ForkChoiceStorage, beaconCfg *clparams.BeaconChainConfig, emitters *beaconevents.EventEmitter, -) ExecutionPayloadService { + requester executionPayloadEnvelopeRequester, +) *executionPayloadService { seenEnvelopesCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) if err != nil { panic(err) @@ -102,10 +101,27 @@ func NewExecutionPayloadService( seenEnvelopesCache: seenEnvelopesCache, pendingCond: sync.NewCond(&sync.Mutex{}), } + s.resolver = newEnvelopeResolver(ctx, requester, s) go s.loop(ctx) return s } +func (s *executionPayloadService) ResolveExecutionPayloadEnvelope(root common.Hash) { + if s.resolver != nil { + s.resolver.ResolveExecutionPayloadEnvelope(root) + } +} + +func (s *executionPayloadService) HasPendingExecutionPayloadEnvelope(root common.Hash) bool { + return s.hasPendingEnvelopeRoot(root) +} + +func (s *executionPayloadService) hasPendingEnvelopeRoot(root common.Hash) bool { + s.pendingMu.Lock() + defer s.pendingMu.Unlock() + return s.pendingRootCounts[root] > 0 +} + func (s *executionPayloadService) Names() []string { return []string{gossip.TopicNameExecutionPayload} } @@ -121,6 +137,9 @@ func (s *executionPayloadService) DecodeGossipMessage(_ peer.ID, data []byte, ve if err := obj.DecodeSSZ(data, int(version)); err != nil { return nil, err } + if err := requireCanonicalSSZ(data, obj); err != nil { + return nil, err + } return obj, nil } @@ -144,20 +163,7 @@ func (s *executionPayloadService) ProcessMessage(ctx context.Context, _ *uint64, // A client MAY queue payload for processing once the block is retrieved. block, ok := s.forkchoiceStore.GetBlock(beaconBlockRoot) if !ok || block == nil { - // Block hasn't arrived yet, queue envelope for later processing - s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope, false) - // Also store in forkchoice's pendingEnvelopes so OnBlock can process it immediately - // when the block arrives, instead of waiting for the 100ms polling loop. - // validatePayload must be true: if the block arrives (via OnBlock) before this call - // acquires f.mu, the envelope will be applied with validatePayload — ensuring - // NewPayload is sent to the EL. With false, a mutex-contention race silently - // marks the envelope as processed without ever notifying the EL, permanently - // breaking the chain. - s.forkchoiceStore.OnExecutionPayload(ctx, signedEnvelope, false, true) - log.Trace("Queued execution payload envelope for later processing", - "beaconBlockRoot", beaconBlockRoot, - "builderIndex", builderIndex) - return ErrIgnore + return fmt.Errorf("%w: block %v not found", ErrIgnore, beaconBlockRoot) } // [IGNORE] The node has not seen another valid SignedExecutionPayloadEnvelope @@ -180,7 +186,9 @@ func (s *executionPayloadService) ProcessMessage(ctx context.Context, _ *uint64, // Note: bid matching and signature verification are done in OnExecutionPayload.validateEnvelopeAgainstBlock if err := s.forkchoiceStore.OnExecutionPayload(ctx, signedEnvelope, true, true); err != nil { if isRetryableExecutionPayloadError(err) { - s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope, true) + if errors.Is(err, forkchoice.ErrEIP7594ColumnDataNotAvailable) || errors.Is(err, forkchoice.ErrELPayloadValidationUnavailable) { + s.queuePendingEnvelope(beaconBlockRoot, signedEnvelope, true, true) + } return fmt.Errorf("%w: %w", ErrIgnore, err) } if errors.Is(err, forkchoice.ErrIgnore) { @@ -215,25 +223,45 @@ func isRetryableExecutionPayloadError(err error) bool { } // queuePendingEnvelope adds an envelope to the pending queue for later processing -func (s *executionPayloadService) queuePendingEnvelope(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, blockSeen bool) { - // Compute envelope hash to allow multiple candidates per block - envelopeHash, err := envelope.HashSSZ() +func (s *executionPayloadService) queuePendingEnvelope(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope, blockSeen, validated bool) bool { + messageHash, err := envelope.Message.HashSSZ() if err != nil { - log.Warn("Failed to hash envelope for pending queue", "blockRoot", blockRoot, "err", err) - return - } - - key := pendingEnvelopeKey{ - blockRoot: blockRoot, - envelopeHash: envelopeHash, + return false } + key := pendingEnvelopeKey{blockRoot: blockRoot, messageHash: messageHash} s.pendingMu.Lock() defer s.pendingMu.Unlock() + if s.pendingRootCounts == nil { + s.pendingRootCounts = make(map[common.Hash]int) + } if existing, loaded := s.pendingEnvelopes.Load(key); loaded { + job := existing.(*envelopeJob) if blockSeen { - existing.(*envelopeJob).blockSeen.Store(true) + job.blockSeen.Store(true) } - return + if !validated || job.validated { + return false + } + nextAttempt := job.nextAttempt + if retryAt := time.Now().Add(pendingEnvelopeRetryInterval); nextAttempt.Before(retryAt) { + nextAttempt = retryAt + } + replacement := &envelopeJob{ + envelope: envelope, + creationTime: job.creationTime, + nextAttempt: nextAttempt, + validated: true, + } + replacement.blockSeen.Store(true) + s.pendingEnvelopes.Store(key, replacement) + return true + } + if s.pendingRootCounts[blockRoot] >= maxPendingCandidatesPerRoot { + oldestKey, found := s.oldestPendingEnvelopeForRoot(blockRoot) + if !found { + return false + } + s.deletePendingEnvelopeLocked(oldestKey, nil) } for s.pendingCount.Load() >= maxPendingEnvelopes { oldestKey, found := s.oldestPendingEnvelope(false) @@ -241,25 +269,64 @@ func (s *executionPayloadService) queuePendingEnvelope(blockRoot common.Hash, en oldestKey, found = s.oldestPendingEnvelope(true) } if !found { - return - } - if _, loaded := s.pendingEnvelopes.LoadAndDelete(oldestKey); loaded { - s.pendingCount.Add(-1) + return false } + s.deletePendingEnvelopeLocked(oldestKey, nil) } job := &envelopeJob{ envelope: envelope, creationTime: time.Now(), + validated: validated, } job.blockSeen.Store(blockSeen) if _, loaded := s.pendingEnvelopes.LoadOrStore(key, job); loaded { + return false } else { s.pendingCount.Add(1) - s.pendingCond.L.Lock() - s.pendingCond.Signal() - s.pendingCond.L.Unlock() + s.pendingRootCounts[blockRoot]++ + if s.pendingCond != nil { + s.pendingCond.L.Lock() + s.pendingCond.Signal() + s.pendingCond.L.Unlock() + } } + return true +} + +func (s *executionPayloadService) oldestPendingEnvelopeForRoot(blockRoot common.Hash) (pendingEnvelopeKey, bool) { + var oldestKey pendingEnvelopeKey + var oldestTime time.Time + found := false + s.pendingEnvelopes.Range(func(candidateKey, value any) bool { + key := candidateKey.(pendingEnvelopeKey) + candidate := value.(*envelopeJob) + if key.blockRoot != blockRoot || candidate.validated || candidate.resolving.Load() { + return true + } + if !found || candidate.creationTime.Before(oldestTime) { + oldestKey = key + oldestTime = candidate.creationTime + found = true + } + return true + }) + return oldestKey, found +} + +func (s *executionPayloadService) deletePendingEnvelopeLocked(key pendingEnvelopeKey, expected *envelopeJob) bool { + current, loaded := s.pendingEnvelopes.Load(key) + if !loaded || expected != nil && current != expected { + return false + } + s.pendingEnvelopes.Delete(key) + s.pendingCount.Add(-1) + if count := s.pendingRootCounts[key.blockRoot]; count > 1 { + s.pendingRootCounts[key.blockRoot] = count - 1 + } else { + delete(s.pendingRootCounts, key.blockRoot) + } + return true } func (s *executionPayloadService) oldestPendingEnvelope(blockSeen bool) (pendingEnvelopeKey, bool) { @@ -335,27 +402,25 @@ func (s *executionPayloadService) processPendingEnvelopes(ctx context.Context) { } blockSeen := job.blockSeen.Load() s.pendingMu.Unlock() + if time.Since(job.creationTime) > pendingEnvelopeExpiry { + s.pendingMu.Lock() + if !s.deletePendingEnvelopeLocked(pendingKey, job) { + job.resolving.Store(false) + } + s.pendingMu.Unlock() + log.Trace("Pending envelope expired", "blockRoot", pendingKey.blockRoot) + return true + } if !blockSeen { block, ok := s.forkchoiceStore.GetBlock(pendingKey.blockRoot) if !ok || block == nil { s.pendingMu.Lock() current, stillPending = s.pendingEnvelopes.Load(pendingKey) - expired := stillPending && current == job && time.Since(job.creationTime) > pendingEnvelopeExpiry if stillPending && current == job { - if expired { - s.pendingEnvelopes.Delete(pendingKey) - s.pendingCount.Add(-1) - } else { - job.resolving.Store(false) - } - } else { job.resolving.Store(false) } s.pendingMu.Unlock() - if expired { - log.Trace("Pending envelope expired", "blockRoot", pendingKey.blockRoot) - } return true } job.blockSeen.Store(true) @@ -392,6 +457,5 @@ func (s *executionPayloadService) finishPendingEnvelopeAttempt(pendingKey pendin job.resolving.Store(false) return } - s.pendingEnvelopes.Delete(pendingKey) - s.pendingCount.Add(-1) + s.deletePendingEnvelopeLocked(pendingKey, job) } diff --git a/cl/phase1/network/services/execution_payload_service_test.go b/cl/phase1/network/services/execution_payload_service_test.go index 815309cb67a..c1050a02cbb 100644 --- a/cl/phase1/network/services/execution_payload_service_test.go +++ b/cl/phase1/network/services/execution_payload_service_test.go @@ -18,9 +18,11 @@ package services import ( "context" + "encoding/binary" "errors" "fmt" "sync" + "sync/atomic" "testing" "time" @@ -39,7 +41,7 @@ import ( func setupExecutionPayloadService(t *testing.T) (ExecutionPayloadService, *mock_services.ForkChoiceStorageMock) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) - service := NewExecutionPayloadService(t.Context(), forkchoiceMock, cfg, beaconevents.NewEventEmitter()) + service := NewExecutionPayloadService(t.Context(), forkchoiceMock, cfg, beaconevents.NewEventEmitter(), nil) return service, forkchoiceMock } @@ -58,6 +60,13 @@ func newTestSignedEnvelope(slot uint64, blockRoot common.Hash, builderIndex uint } } +func pendingEnvelopeKeyForTest(t *testing.T, envelope *cltypes.SignedExecutionPayloadEnvelope) pendingEnvelopeKey { + t.Helper() + messageHash, err := envelope.Message.HashSSZ() + require.NoError(t, err) + return pendingEnvelopeKey{blockRoot: envelope.Message.BeaconBlockRoot, messageHash: messageHash} +} + func TestExecutionPayloadServiceNilEnvelope(t *testing.T) { service, _ := setupExecutionPayloadService(t) @@ -72,19 +81,45 @@ func TestExecutionPayloadServiceNilEnvelope(t *testing.T) { require.Contains(t, err.Error(), "nil execution payload envelope") } +func TestExecutionPayloadServiceRejectsNonCanonicalSSZ(t *testing.T) { + service := &executionPayloadService{beaconCfg: &clparams.MainnetBeaconConfig} + original := newTestSignedEnvelope(100, common.HexToHash("0x1234"), 1) + encoded, err := original.EncodeSSZ(nil) + require.NoError(t, err) + + t.Run("trailing bytes", func(t *testing.T) { + mutated := append(append([]byte(nil), encoded...), 0) + _, decodeErr := service.DecodeGossipMessage("peer123", mutated, clparams.GloasVersion) + require.Error(t, decodeErr) + }) + + t.Run("dynamic offset gap", func(t *testing.T) { + const signedEnvelopeFixedSize = 4 + 96 + mutated := make([]byte, 0, len(encoded)+1) + mutated = append(mutated, encoded[:signedEnvelopeFixedSize]...) + mutated = append(mutated, 0) + mutated = append(mutated, encoded[signedEnvelopeFixedSize:]...) + binary.LittleEndian.PutUint32(mutated[:4], signedEnvelopeFixedSize+1) + lossy := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + require.NoError(t, lossy.DecodeSSZ(mutated, int(clparams.GloasVersion))) + _, decodeErr := service.DecodeGossipMessage("peer123", mutated, clparams.GloasVersion) + require.ErrorContains(t, decodeErr, "non-canonical SSZ") + }) +} + func TestExecutionPayloadServiceBlockNotFound(t *testing.T) { service, fcu := setupExecutionPayloadService(t) blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - // Block not in forkchoice - should queue and return ErrIgnore + // Block not in forkchoice - should not retain unauthenticated input. err := service.ProcessMessage(context.Background(), nil, envelope) require.ErrorIs(t, err, ErrIgnore) - // Verify envelope was queued (check internal state) + // Unknown-block envelopes are attacker-controlled and must not consume memory. impl := service.(*executionPayloadService) - require.Equal(t, int32(1), impl.pendingCount.Load()) + require.Zero(t, impl.pendingCount.Load()) // Now add block to forkchoice fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ @@ -218,14 +253,9 @@ func TestExecutionPayloadServicePendingEnvelopeExpiry(t *testing.T) { blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) // Add expired job directly - key := pendingEnvelopeKey{ - blockRoot: blockRoot, - envelopeHash: envelopeHash, - } + key := pendingEnvelopeKeyForTest(t, envelope) impl.pendingEnvelopes.Store(key, &envelopeJob{ envelope: envelope, creationTime: time.Now().Add(-pendingEnvelopeExpiry - time.Second), // expired @@ -254,9 +284,7 @@ func TestExecutionPayloadServiceRetainsEnvelopeAcrossColumnSyncInterval(t *testi blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - key := pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash} + key := pendingEnvelopeKeyForTest(t, envelope) impl.pendingEnvelopes.Store(key, &envelopeJob{ envelope: envelope, creationTime: time.Now().Add(-time.Minute), @@ -288,14 +316,9 @@ func TestExecutionPayloadServicePendingEnvelopeProcessing(t *testing.T) { blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) // Add pending job - key := pendingEnvelopeKey{ - blockRoot: blockRoot, - envelopeHash: envelopeHash, - } + key := pendingEnvelopeKeyForTest(t, envelope) impl.pendingEnvelopes.Store(key, &envelopeJob{ envelope: envelope, creationTime: time.Now(), @@ -356,9 +379,7 @@ func TestExecutionPayloadServiceQueuesInitialTemporaryELFailure(t *testing.T) { require.ErrorIs(t, err, forkchoice.ErrELPayloadValidationUnavailable) require.Equal(t, int32(1), impl.pendingCount.Load()) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash}) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, envelope)) require.True(t, exists) fcu.OnExecutionPayloadErr = nil @@ -367,7 +388,7 @@ func TestExecutionPayloadServiceQueuesInitialTemporaryELFailure(t *testing.T) { require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) } -func TestExecutionPayloadServiceDoesNotExpireRetryableKnownBlock(t *testing.T) { +func TestExecutionPayloadServiceExpiresRetryableKnownBlock(t *testing.T) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) impl := &executionPayloadService{ @@ -381,9 +402,7 @@ func TestExecutionPayloadServiceDoesNotExpireRetryableKnownBlock(t *testing.T) { blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - key := pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash} + key := pendingEnvelopeKeyForTest(t, envelope) impl.pendingEnvelopes.Store(key, &envelopeJob{ envelope: envelope, creationTime: time.Now().Add(-pendingEnvelopeExpiry - time.Second), @@ -394,10 +413,9 @@ func TestExecutionPayloadServiceDoesNotExpireRetryableKnownBlock(t *testing.T) { impl.processPendingEnvelopes(t.Context()) - require.Equal(t, int32(1), impl.pendingCount.Load()) - value, exists := impl.pendingEnvelopes.Load(key) - require.True(t, exists) - require.True(t, value.(*envelopeJob).nextAttempt.After(time.Now())) + require.Zero(t, impl.pendingCount.Load()) + _, exists := impl.pendingEnvelopes.Load(key) + require.False(t, exists) } func TestExecutionPayloadServiceRetainsPendingEnvelopeUntilDataAvailable(t *testing.T) { @@ -405,20 +423,17 @@ func TestExecutionPayloadServiceRetainsPendingEnvelopeUntilDataAvailable(t *test impl := service.(*executionPayloadService) blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Slot: 100}, + } + fcu.OnExecutionPayloadErr = forkchoice.ErrEIP7594ColumnDataNotAvailable require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - key := pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash} + key := pendingEnvelopeKeyForTest(t, envelope) value, ok := impl.pendingEnvelopes.Load(key) require.True(t, ok) creationTime := value.(*envelopeJob).creationTime - fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ - Block: &cltypes.BeaconBlock{Slot: 100}, - } - fcu.OnExecutionPayloadErr = forkchoice.ErrEIP7594ColumnDataNotAvailable - impl.processPendingEnvelopes(t.Context()) require.Equal(t, int32(1), impl.pendingCount.Load()) value, ok = impl.pendingEnvelopes.Load(key) @@ -439,14 +454,12 @@ func TestExecutionPayloadServiceDropsPendingEnvelopeAfterValidationFailure(t *te impl := service.(*executionPayloadService) blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - - require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ Block: &cltypes.BeaconBlock{Slot: 100}, } fcu.OnExecutionPayloadErr = errors.New("invalid envelope") - impl.processPendingEnvelopes(t.Context()) + require.Error(t, service.ProcessMessage(t.Context(), nil, envelope)) require.Equal(t, int32(0), impl.pendingCount.Load()) require.False(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) @@ -457,25 +470,19 @@ func TestExecutionPayloadServiceRetainsPendingEnvelopeAfterTemporaryELFailure(t impl := service.(*executionPayloadService) blockRoot := common.HexToHash("0x1234") envelope := newTestSignedEnvelope(100, blockRoot, 1) - - require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} fcu.OnExecutionPayloadErr = fmt.Errorf("%w: timeout", forkchoice.ErrELPayloadValidationUnavailable) - impl.processPendingEnvelopes(t.Context()) + require.ErrorIs(t, service.ProcessMessage(t.Context(), nil, envelope), ErrIgnore) require.Equal(t, int32(1), impl.pendingCount.Load()) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot: blockRoot, envelopeHash: envelopeHash}) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, envelope)) require.True(t, exists) } -func TestExecutionPayloadServiceMultiplePendingForSameBlock(t *testing.T) { +func TestExecutionPayloadServiceSignatureFloodUsesOnePendingRoot(t *testing.T) { cfg := &clparams.MainnetBeaconConfig forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) - ctx := t.Context() - impl := &executionPayloadService{ forkchoiceStore: forkchoiceMock, beaconCfg: cfg, @@ -487,38 +494,173 @@ func TestExecutionPayloadServiceMultiplePendingForSameBlock(t *testing.T) { impl.seenEnvelopesCache = seenCache blockRoot := common.HexToHash("0x1234") + first := newTestSignedEnvelope(100, blockRoot, 1) + require.True(t, impl.queuePendingEnvelope(blockRoot, first, false, false)) + for i := range maxPendingEnvelopes * 2 { + variant := newTestSignedEnvelope(100, blockRoot, 1) + variant.Signature[0] = byte(i) + variant.Signature[1] = byte(i >> 8) + require.False(t, impl.queuePendingEnvelope(blockRoot, variant, false, false)) + } - // Create two different envelopes for the same block (different builders) - envelope1 := newTestSignedEnvelope(100, blockRoot, 1) - envelope2 := newTestSignedEnvelope(100, blockRoot, 2) + require.Equal(t, int32(1), impl.pendingCount.Load()) + value, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, first)) + require.True(t, exists) + require.Same(t, first, value.(*envelopeJob).envelope) +} - hash1, _ := envelope1.HashSSZ() - hash2, _ := envelope2.HashSSZ() +func TestExecutionPayloadServiceForgedThenValidBeforeBlockProcessesValid(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + seenEnvelopesCache: seenCache, + } + blockRoot := common.HexToHash("0x1234") + forged := newTestSignedEnvelope(100, blockRoot, 1) + valid := newTestSignedEnvelope(100, blockRoot, 2) + var validationCalls atomic.Int32 + forkchoiceMock.OnExecutionPayloadFunc = func(candidate *cltypes.SignedExecutionPayloadEnvelope) error { + validationCalls.Add(1) + if candidate == forged { + return errors.New("forged envelope") + } + return nil + } - // Add both as pending - impl.pendingEnvelopes.Store(pendingEnvelopeKey{blockRoot, hash1}, &envelopeJob{ - envelope: envelope1, - creationTime: time.Now(), - }) - impl.pendingEnvelopes.Store(pendingEnvelopeKey{blockRoot, hash2}, &envelopeJob{ - envelope: envelope2, - creationTime: time.Now(), - }) - impl.pendingCount.Store(2) + require.ErrorIs(t, impl.ProcessMessage(t.Context(), nil, forged), ErrIgnore) + require.ErrorIs(t, impl.ProcessMessage(t.Context(), nil, valid), ErrIgnore) + require.Zero(t, validationCalls.Load()) + require.Zero(t, impl.pendingCount.Load()) - // Add block - forkchoiceMock.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{ - Block: &cltypes.BeaconBlock{ - Slot: 100, - }, + forkchoiceMock.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + require.Error(t, impl.ProcessMessage(t.Context(), nil, forged)) + require.NoError(t, impl.ProcessMessage(t.Context(), nil, valid)) + + require.Zero(t, impl.pendingCount.Load()) + require.NotContains(t, impl.pendingRootCounts, blockRoot) + require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 2})) + require.Equal(t, int32(2), validationCalls.Load()) +} + +func TestExecutionPayloadServiceSemanticCandidateFloodIsPerRootBounded(t *testing.T) { + impl := &executionPayloadService{} + blockRoot := common.HexToHash("0x1234") + for i := range maxPendingEnvelopes * 2 { + envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) + require.True(t, impl.queuePendingEnvelope(blockRoot, envelope, false, false)) } + require.Equal(t, int32(maxPendingCandidatesPerRoot), impl.pendingCount.Load()) +} - // Process - both should be processed - impl.processPendingEnvelopes(ctx) +func TestExecutionPayloadServiceValidCandidateAtPerRootCapReplacesOldestUnknown(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) + seenCache, err := lru.New[seenEnvelopeKey, struct{}]("seen_envelopes", seenEnvelopeCacheSize) + require.NoError(t, err) + impl := &executionPayloadService{ + forkchoiceStore: forkchoiceMock, + beaconCfg: cfg, + emitters: beaconevents.NewEventEmitter(), + seenEnvelopesCache: seenCache, + } + blockRoot := common.HexToHash("0x1234") + first := newTestSignedEnvelope(100, blockRoot, 1) + require.True(t, impl.queuePendingEnvelope(blockRoot, first, false, false)) + firstKey := pendingEnvelopeKeyForTest(t, first) + value, exists := impl.pendingEnvelopes.Load(firstKey) + require.True(t, exists) + value.(*envelopeJob).creationTime = time.Now().Add(-time.Hour) + for i := 1; i < maxPendingCandidatesPerRoot; i++ { + candidate := newTestSignedEnvelope(100, blockRoot, uint64(i+1)) + require.True(t, impl.queuePendingEnvelope(blockRoot, candidate, false, false)) + } - require.Equal(t, int32(0), impl.pendingCount.Load()) - require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 1})) - require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 2})) + newest := newTestSignedEnvelope(100, blockRoot, 999) + require.True(t, impl.queuePendingEnvelope(blockRoot, newest, false, false)) + require.Equal(t, int32(maxPendingCandidatesPerRoot), impl.pendingCount.Load()) + _, exists = impl.pendingEnvelopes.Load(firstKey) + require.False(t, exists) + _, exists = impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, newest)) + require.True(t, exists) + forkchoiceMock.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + forkchoiceMock.OnExecutionPayloadFunc = func(candidate *cltypes.SignedExecutionPayloadEnvelope) error { + if candidate != newest { + return errors.New("forged envelope") + } + return nil + } + impl.processPendingEnvelopes(t.Context()) + require.Zero(t, impl.pendingCount.Load()) + require.NotContains(t, impl.pendingRootCounts, blockRoot) + require.True(t, impl.seenEnvelopesCache.Contains(seenEnvelopeKey{blockRoot, 999})) +} + +func TestExecutionPayloadServiceValidatedCandidateReplacesUnknownCandidate(t *testing.T) { + impl := &executionPayloadService{pendingCond: sync.NewCond(&sync.Mutex{})} + blockRoot := common.HexToHash("0x1234") + unknown := newTestSignedEnvelope(100, blockRoot, 1) + require.True(t, impl.queuePendingEnvelope(blockRoot, unknown, false, false)) + key := pendingEnvelopeKeyForTest(t, unknown) + value, exists := impl.pendingEnvelopes.Load(key) + require.True(t, exists) + originalCreationTime := value.(*envelopeJob).creationTime + + validated := newTestSignedEnvelope(100, blockRoot, 1) + validated.Signature[0] = 1 + require.True(t, impl.queuePendingEnvelope(blockRoot, validated, true, true)) + value, exists = impl.pendingEnvelopes.Load(key) + require.True(t, exists) + job := value.(*envelopeJob) + require.Same(t, validated, job.envelope) + require.True(t, job.validated) + require.True(t, job.blockSeen.Load()) + require.Equal(t, originalCreationTime, job.creationTime) + require.Equal(t, int32(1), impl.pendingCount.Load()) + + for i := range maxPendingEnvelopes * 2 { + forged := newTestSignedEnvelope(100, blockRoot, 1) + forged.Signature[0] = byte(i) + forged.Signature[1] = byte(i >> 8) + require.False(t, impl.queuePendingEnvelope(blockRoot, forged, true, false)) + } + value, exists = impl.pendingEnvelopes.Load(key) + require.True(t, exists) + require.Same(t, validated, value.(*envelopeJob).envelope) +} + +func TestExecutionPayloadServiceCanceledCandidateDoesNotReplaceValidatedCandidate(t *testing.T) { + service, fcu := setupExecutionPayloadService(t) + impl := service.(*executionPayloadService) + blockRoot := common.HexToHash("0x1234") + validated := newTestSignedEnvelope(100, blockRoot, 1) + require.True(t, impl.queuePendingEnvelope(blockRoot, validated, true, true)) + fcu.Blocks[blockRoot] = &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Slot: 100}} + fcu.OnExecutionPayloadErr = context.Canceled + + canceled := newTestSignedEnvelope(100, blockRoot, 1) + canceled.Signature[0] = 2 + err := service.ProcessMessage(t.Context(), nil, canceled) + require.ErrorIs(t, err, context.Canceled) + value, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, validated)) + require.True(t, exists) + require.Same(t, validated, value.(*envelopeJob).envelope) + require.Equal(t, int32(1), impl.pendingCount.Load()) +} + +func TestExecutionPayloadServiceDifferentRootsRetainIndependentCapacity(t *testing.T) { + impl := &executionPayloadService{pendingCond: sync.NewCond(&sync.Mutex{})} + const roots = 128 + for i := range roots { + blockRoot := common.Hash{byte(i), byte(i >> 8)} + envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) + require.True(t, impl.queuePendingEnvelope(blockRoot, envelope, false, false)) + } + require.Equal(t, int32(roots), impl.pendingCount.Load()) } func TestExecutionPayloadServicePendingQueueCap(t *testing.T) { @@ -540,12 +682,10 @@ func TestExecutionPayloadServicePendingQueueCap(t *testing.T) { blockRoot := common.HexToHash("0xffff") envelope := newTestSignedEnvelope(100, blockRoot, 999) - impl.queuePendingEnvelope(blockRoot, envelope, false) + impl.queuePendingEnvelope(blockRoot, envelope, false, false) require.Equal(t, int32(maxPendingEnvelopes), impl.pendingCount.Load()) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, envelope)) require.False(t, exists) } @@ -566,9 +706,7 @@ func TestExecutionPayloadServicePendingQueueRejectsUnknownWorkWhenAllJobsAreKnow for i := range maxPendingEnvelopes { blockRoot := common.Hash{byte(i), byte(i >> 8)} envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) - envelopeHash, hashErr := envelope.HashSSZ() - require.NoError(t, hashErr) - key := pendingEnvelopeKey{blockRoot, envelopeHash} + key := pendingEnvelopeKey{blockRoot: blockRoot} if i == 0 { oldestKey = key } @@ -583,11 +721,9 @@ func TestExecutionPayloadServicePendingQueueRejectsUnknownWorkWhenAllJobsAreKnow blockRoot := common.HexToHash("0xffff") envelope := newTestSignedEnvelope(100, blockRoot, 9999) - impl.queuePendingEnvelope(blockRoot, envelope, false) + impl.queuePendingEnvelope(blockRoot, envelope, false, false) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, envelope)) require.False(t, exists) _, exists = impl.pendingEnvelopes.Load(oldestKey) require.True(t, exists) @@ -611,9 +747,7 @@ func TestExecutionPayloadServicePendingQueueEvictsUnknownBeforeOlderKnownWork(t for i := range maxPendingEnvelopes { blockRoot := common.Hash{byte(i), byte(i >> 8)} envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) - envelopeHash, hashErr := envelope.HashSSZ() - require.NoError(t, hashErr) - key := pendingEnvelopeKey{blockRoot, envelopeHash} + key := pendingEnvelopeKey{blockRoot: blockRoot} job := &envelopeJob{ envelope: envelope, creationTime: time.Now(), @@ -634,11 +768,9 @@ func TestExecutionPayloadServicePendingQueueEvictsUnknownBeforeOlderKnownWork(t blockRoot := common.HexToHash("0xffff") envelope := newTestSignedEnvelope(100, blockRoot, 9999) - impl.queuePendingEnvelope(blockRoot, envelope, false) + impl.queuePendingEnvelope(blockRoot, envelope, false, false) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, envelope)) require.True(t, exists) _, exists = impl.pendingEnvelopes.Load(oldestKnownKey) require.True(t, exists) @@ -664,9 +796,7 @@ func TestExecutionPayloadServicePendingQueueAdmitsKnownWorkWhenAllJobsAreKnown(t for i := range maxPendingEnvelopes { blockRoot := common.Hash{byte(i), byte(i >> 8)} envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) - envelopeHash, hashErr := envelope.HashSSZ() - require.NoError(t, hashErr) - key := pendingEnvelopeKey{blockRoot, envelopeHash} + key := pendingEnvelopeKey{blockRoot: blockRoot} if i == 0 { oldestKey = key } @@ -678,11 +808,9 @@ func TestExecutionPayloadServicePendingQueueAdmitsKnownWorkWhenAllJobsAreKnown(t blockRoot := common.HexToHash("0xffff") envelope := newTestSignedEnvelope(100, blockRoot, 9999) - impl.queuePendingEnvelope(blockRoot, envelope, true) + impl.queuePendingEnvelope(blockRoot, envelope, true, true) - envelopeHash, err := envelope.HashSSZ() - require.NoError(t, err) - _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKey{blockRoot, envelopeHash}) + _, exists := impl.pendingEnvelopes.Load(pendingEnvelopeKeyForTest(t, envelope)) require.True(t, exists) _, exists = impl.pendingEnvelopes.Load(oldestKey) require.False(t, exists) @@ -706,9 +834,7 @@ func TestExecutionPayloadServicePendingQueueDoesNotEvictResolvingWork(t *testing for i := range maxPendingEnvelopes { blockRoot := common.Hash{byte(i), byte(i >> 8)} envelope := newTestSignedEnvelope(100, blockRoot, uint64(i)) - envelopeHash, hashErr := envelope.HashSSZ() - require.NoError(t, hashErr) - key := pendingEnvelopeKey{blockRoot, envelopeHash} + key := pendingEnvelopeKey{blockRoot: blockRoot} job := &envelopeJob{envelope: envelope, creationTime: time.Now()} job.blockSeen.Store(true) switch i { @@ -728,7 +854,7 @@ func TestExecutionPayloadServicePendingQueueDoesNotEvictResolvingWork(t *testing blockRoot := common.HexToHash("0xffff") envelope := newTestSignedEnvelope(100, blockRoot, 9999) - impl.queuePendingEnvelope(blockRoot, envelope, false) + impl.queuePendingEnvelope(blockRoot, envelope, false, false) _, exists := impl.pendingEnvelopes.Load(resolvingKey) require.True(t, exists) @@ -773,7 +899,7 @@ func TestExecutionPayloadServicePendingQueueCapConcurrent(t *testing.T) { wg.Go(func() { blockRoot := common.Hash{byte(i), byte(i >> 8)} envelope := newTestSignedEnvelope(100, blockRoot, uint64(10000+i)) - impl.queuePendingEnvelope(blockRoot, envelope, false) + impl.queuePendingEnvelope(blockRoot, envelope, false, false) }) } wg.Wait() diff --git a/cl/phase1/network/services/payload_attestation_service.go b/cl/phase1/network/services/payload_attestation_service.go index f50ed2641fd..b4320d1d1b2 100644 --- a/cl/phase1/network/services/payload_attestation_service.go +++ b/cl/phase1/network/services/payload_attestation_service.go @@ -121,6 +121,9 @@ func (s *payloadAttestationService) DecodeGossipMessage(_ peer.ID, data []byte, if err := msg.DecodeSSZ(data, int(version)); err != nil { return nil, err } + if err := requireCanonicalSSZ(data, msg); err != nil { + return nil, err + } return msg, nil } diff --git a/cl/phase1/network/services/payload_attestation_service_test.go b/cl/phase1/network/services/payload_attestation_service_test.go index b9e76757ad9..082664bc626 100644 --- a/cl/phase1/network/services/payload_attestation_service_test.go +++ b/cl/phase1/network/services/payload_attestation_service_test.go @@ -647,3 +647,18 @@ func TestPayloadAttestationServiceDecodeGossipMessageInvalid(t *testing.T) { _, err := service.DecodeGossipMessage("peer123", []byte{0x00, 0x01, 0x02}, clparams.GloasVersion) require.Error(t, err) } + +func TestPayloadAttestationServiceRejectsNonCanonicalBoolean(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + service, _, _ := setupPayloadAttestationService(t, ctrl) + original := newTestPayloadAttestationMessage(100, 42, common.HexToHash("0x1234")) + encoded, err := original.EncodeSSZ(nil) + require.NoError(t, err) + encoded[48] = 2 + + lossy := new(cltypes.PayloadAttestationMessage) + require.NoError(t, lossy.DecodeSSZ(encoded, int(clparams.GloasVersion))) + _, err = service.DecodeGossipMessage("peer123", encoded, clparams.GloasVersion) + require.ErrorContains(t, err, "non-canonical SSZ") +} diff --git a/cl/phase1/stages/forward_sync.go b/cl/phase1/stages/forward_sync.go index 9542444a003..745aa84578a 100644 --- a/cl/phase1/stages/forward_sync.go +++ b/cl/phase1/stages/forward_sync.go @@ -20,6 +20,7 @@ import ( "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice" network2 "github.com/erigontech/erigon/cl/phase1/network" + eth2impl "github.com/erigontech/erigon/cl/transition/impl/eth2" "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/cl/utils/bls" "github.com/erigontech/erigon/common" @@ -213,7 +214,7 @@ func processDownloadedBlockBatches(ctx context.Context, logger log.Logger, cfg * if fceErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, canValidateGloasPayloads(cfg)); fceErr != nil { logger.Warn("[Caplin] forward sync: failed to process GLOAS envelope", "slot", block.Block.Slot, "err", fceErr) } else if shouldInsert { - if err = cfg.blockCollector.AddGloasBlock(block.Block, env); err != nil { + if err = cfg.blockCollector.AddGloasBlock(ctx, block.Block, env); err != nil { err = fmt.Errorf("failed to add gloas block to collector: %w", err) return } @@ -321,6 +322,9 @@ func forwardSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) er // Always start from the current finalized checkpoint downloader.SetHighestProcessedSlot(currentSlot.Load()) downloader.SetMinSlot(startSlot) + downloader.SetValidateFunction(func(blocks []*cltypes.SignedBeaconBlock) (int, error) { + return validateForwardEnvelopeEvidence(cfg.forkChoice, blocks) + }) // Set the function to process downloaded blocks downloader.SetProcessFunction(func(initialHighestSlotProcessed uint64, blocks []*cltypes.SignedBeaconBlock, envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope) (newHighestSlotProcessed uint64, err error) { @@ -404,6 +408,59 @@ func forwardSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) er return nil } +type forwardEnvelopeStateReader interface { + GetStateAtBlockRoot(common.Hash, bool) (*state.CachingBeaconState, error) +} + +func validateForwardEnvelopeEvidence(stateReader forwardEnvelopeStateReader, blocks []*cltypes.SignedBeaconBlock) (int, error) { + if len(blocks) == 0 || blocks[0] == nil || blocks[0].Block == nil { + return 0, errors.New("empty block admission batch") + } + validationState, err := stateReader.GetStateAtBlockRoot(blocks[0].Block.ParentRoot, true) + start := 0 + if err != nil || validationState == nil { + firstRoot, hashErr := blocks[0].Block.HashSSZ() + if hashErr != nil { + return 0, hashErr + } + validationState, err = stateReader.GetStateAtBlockRoot(firstRoot, true) + start = 1 + } + if err != nil { + return 0, fmt.Errorf("forward envelope admission state unavailable: %w", err) + } + if validationState == nil { + return 0, errors.New("forward envelope admission state unavailable") + } + for i, block := range blocks[start:] { + if block == nil || block.Block == nil || block.Block.Body == nil { + return 0, errors.New("incomplete block admission candidate") + } + if validationState.Version() >= clparams.FuluVersion { + stateEpoch := validationState.Slot() / validationState.BeaconConfig().SlotsPerEpoch + blockEpoch := block.Block.Slot / validationState.BeaconConfig().SlotsPerEpoch + if blockEpoch < stateEpoch || blockEpoch > stateEpoch+validationState.BeaconConfig().MinSeedLookahead { + return start + i, nil + } + } + expectedProposer, err := validationState.GetBeaconProposerIndexForSlot(block.Block.Slot) + if err != nil { + return start + i, nil + } + if expectedProposer != block.Block.ProposerIndex { + return 0, fmt.Errorf("unexpected proposer %d at slot %d, expected %d", block.Block.ProposerIndex, block.Block.Slot, expectedProposer) + } + valid, err := eth2impl.VerifyBlockSignature(validationState, block) + if err != nil { + return 0, err + } + if !valid { + return 0, fmt.Errorf("invalid proposer signature at slot %d", block.Block.Slot) + } + } + return len(blocks), nil +} + // setHeadStateFromForkChoice retrieves the current head state from the fork choice store // and sets it on syncedData. This clears the Syncing() flag so gossip beacon_block messages // are accepted by ChainTipSync instead of being ignored. diff --git a/cl/phase1/stages/gloas_payload_test.go b/cl/phase1/stages/gloas_payload_test.go index 609c28ba59f..05398440a97 100644 --- a/cl/phase1/stages/gloas_payload_test.go +++ b/cl/phase1/stages/gloas_payload_test.go @@ -18,6 +18,8 @@ import ( state2 "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice" + forkchoicemock "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" + eth2impl "github.com/erigontech/erigon/cl/transition/impl/eth2" "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/cl/utils/bls" "github.com/erigontech/erigon/common" @@ -127,6 +129,94 @@ func TestValidateAnchorEnvelope(t *testing.T) { } } +func TestValidateForwardEnvelopeEvidenceRejectsValidSignatureFromWrongProposer(t *testing.T) { + cfg, validationState, _, _, _ := validAnchorEnvelopeFixture(t, 0) + privateKey, err := bls.NewPrivateKeyFromIKM([]byte("01234567890123456789012345678901")) + require.NoError(t, err) + publicKey := common.Bytes48(bls.CompressPublicKey(privateKey.PublicKey())) + validationState.AddValidator( + solid.NewValidatorFromParameters(publicKey, common.Hash{}, cfg.MaxEffectiveBalance, false, 0, 0, cfg.FarFutureEpoch, cfg.FarFutureEpoch), + cfg.MaxEffectiveBalance, + ) + + parentRoot := common.HexToHash("0x1234") + block := cltypes.NewSignedBeaconBlock(cfg, clparams.GloasVersion) + block.Block.Slot = validationState.Slot() + 1 + block.Block.ParentRoot = parentRoot + expectedProposer, err := validationState.GetBeaconProposerIndexForSlot(block.Block.Slot) + require.NoError(t, err) + block.Block.ProposerIndex = 1 - expectedProposer + domain, err := validationState.GetDomain(cfg.DomainBeaconProposer, block.Block.Slot/cfg.SlotsPerEpoch) + require.NoError(t, err) + signingRoot, err := fork.ComputeSigningRoot(block.Block, domain) + require.NoError(t, err) + copy(block.Signature[:], privateKey.Sign(signingRoot[:]).Bytes()) + valid, err := eth2impl.VerifyBlockSignature(validationState, block) + require.NoError(t, err) + require.True(t, valid) + + fc := forkchoicemock.NewForkChoiceStorageMock(t) + fc.StateAtBlockRootVal[parentRoot] = validationState + + _, err = validateForwardEnvelopeEvidence(fc, []*cltypes.SignedBeaconBlock{block}) + require.ErrorContains(t, err, "unexpected proposer") +} + +func TestValidateForwardEnvelopeEvidenceAcceptsExpectedProposer(t *testing.T) { + cfg, validationState, _, _, _ := validAnchorEnvelopeFixture(t, 0) + privateKey, err := bls.NewPrivateKeyFromIKM([]byte("01234567890123456789012345678901")) + require.NoError(t, err) + parentRoot := common.HexToHash("0x1234") + block := cltypes.NewSignedBeaconBlock(cfg, clparams.GloasVersion) + block.Block.Slot = validationState.Slot() + 1 + block.Block.ParentRoot = parentRoot + block.Block.ProposerIndex, err = validationState.GetBeaconProposerIndexForSlot(block.Block.Slot) + require.NoError(t, err) + domain, err := validationState.GetDomain(cfg.DomainBeaconProposer, block.Block.Slot/cfg.SlotsPerEpoch) + require.NoError(t, err) + signingRoot, err := fork.ComputeSigningRoot(block.Block, domain) + require.NoError(t, err) + copy(block.Signature[:], privateKey.Sign(signingRoot[:]).Bytes()) + fc := forkchoicemock.NewForkChoiceStorageMock(t) + fc.StateAtBlockRootVal[parentRoot] = validationState + + authenticated, err := validateForwardEnvelopeEvidence(fc, []*cltypes.SignedBeaconBlock{block}) + + require.NoError(t, err) + require.Equal(t, 1, authenticated) +} + +func TestValidateForwardEnvelopeEvidenceFailsClosedWithoutState(t *testing.T) { + block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.GloasVersion) + block.Block.Slot = 1 + block.Block.ParentRoot = common.HexToHash("0x1234") + fc := forkchoicemock.NewForkChoiceStorageMock(t) + + authenticated, err := validateForwardEnvelopeEvidence(fc, []*cltypes.SignedBeaconBlock{block}) + + require.ErrorContains(t, err, "state unavailable") + require.Zero(t, authenticated) +} + +func TestValidateForwardEnvelopeEvidenceReturnsFuluLookaheadPrefix(t *testing.T) { + cfg, validationState, _, _, _ := validAnchorEnvelopeFixture(t, 0) + validationState.SetVersion(clparams.FuluVersion) + first := cltypes.NewSignedBeaconBlock(cfg, clparams.FuluVersion) + first.Block.Slot = validationState.Slot() + firstRoot, err := first.Block.HashSSZ() + require.NoError(t, err) + beyondLookahead := cltypes.NewSignedBeaconBlock(cfg, clparams.FuluVersion) + beyondLookahead.Block.Slot = validationState.Slot() + (cfg.MinSeedLookahead+1)*cfg.SlotsPerEpoch + beyondLookahead.Block.ParentRoot = firstRoot + fc := forkchoicemock.NewForkChoiceStorageMock(t) + fc.StateAtBlockRootVal[firstRoot] = validationState + + authenticated, err := validateForwardEnvelopeEvidence(fc, []*cltypes.SignedBeaconBlock{first, beyondLookahead}) + + require.NoError(t, err) + require.Equal(t, 1, authenticated) +} + func TestAnchorEnvelopeMatches(t *testing.T) { _, _, _, env, anchorRoot := validAnchorEnvelopeFixture(t, 1) diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go index ce5a6fdb139..72ea5c719ca 100644 --- a/cl/phase1/stages/stage_history_download.go +++ b/cl/phase1/stages/stage_history_download.go @@ -33,6 +33,7 @@ import ( "github.com/erigontech/erigon/cl/phase1/execution_client/block_collector" "github.com/erigontech/erigon/cl/phase1/forkchoice" "github.com/erigontech/erigon/cl/phase1/network" + eth2impl "github.com/erigontech/erigon/cl/transition/impl/eth2" "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" @@ -66,10 +67,12 @@ type historyDownloader interface { SetExpectedRoot(common.Hash) SetBlockChecker(network.BlockChecker) SetOnNewBlock(network.OnNewBlock) + SetValidateFunctions(network.ValidateBlockFn, network.ValidateLookaheadFn) Finished() bool Progress() uint64 RequestMore(context.Context) error SkippedFullBlocks() []network.SkippedFullBlock + AcknowledgeSkippedFullBlocks([]network.SkippedFullBlock) HasEnvelopeRecoverySource() bool RecoverSkippedEnvelopes(context.Context, []network.SkippedFullBlock, map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult SetThrottle(time.Duration) @@ -83,10 +86,65 @@ const ( skippedEnvelopeRecoveryBatchSize = 2 skippedEnvelopeRecoveryBatchTimeout = 5 * time.Second skippedEnvelopeRecoveryAttemptTimeout = 2 * time.Minute + skippedEnvelopeRecoverySweeps = 3 + skippedEnvelopeCanonicalFallbackSlots = 64 ) var errSkippedEnvelopeRecoveryIncomplete = errors.New("skipped envelope recovery incomplete") +func configureHistoryEnvelopeAdmission(cfg StageHistoryReconstructionCfg) { + if cfg.forkchoiceStore == nil { + return + } + cfg.downloader.SetValidateFunctions( + func(block *cltypes.SignedBeaconBlock) error { + validationState, err := cfg.forkchoiceStore.GetStateAtBlockRoot(cfg.forkchoiceStore.FinalizedCheckpoint().Root, true) + if err != nil { + return fmt.Errorf("historical block signature state unavailable: %w", err) + } + if validationState == nil { + return errors.New("historical block signature state unavailable") + } + valid, err := eth2impl.VerifyBlockSignature(validationState, block) + if err != nil { + return err + } + if !valid { + return fmt.Errorf("invalid historical block signature at slot %d", block.Block.Slot) + } + return nil + }, + func(anchor, child *cltypes.SignedBeaconBlock) error { + anchorRoot, err := anchor.Block.HashSSZ() + if err != nil { + return err + } + validationState, err := cfg.forkchoiceStore.GetStateAtBlockRoot(anchorRoot, true) + if err != nil { + return fmt.Errorf("lookahead admission state unavailable: %w", err) + } + if validationState == nil { + return errors.New("lookahead admission state unavailable") + } + expectedProposer, err := validationState.GetBeaconProposerIndexForSlot(child.Block.Slot) + if err != nil { + return err + } + if expectedProposer != child.Block.ProposerIndex { + return fmt.Errorf("unexpected lookahead proposer %d at slot %d, expected %d", child.Block.ProposerIndex, child.Block.Slot, expectedProposer) + } + valid, err := eth2impl.VerifyBlockSignature(validationState, child) + if err != nil { + return err + } + if !valid { + return fmt.Errorf("invalid lookahead signature at slot %d", child.Block.Slot) + } + return nil + }, + ) +} + func StageHistoryReconstruction(downloader *network.BackwardBeaconDownloader, antiquary *antiquary.Antiquary, sn *freezeblocks.CaplinSnapshots, indiciesDB kv.RwDB, engine execution_client.ExecutionEngine, beaconCfg *clparams.BeaconChainConfig, caplinConfig clparams.CaplinConfig, waitForAllRoutines bool, startingRoot common.Hash, startinSlot uint64, tmpdir string, backfillingThrottling time.Duration, executionBlocksCollector block_collector.BlockCollector, blockReader freezeblocks.BeaconSnapshotReader, blobStorage blob_storage.BlobStorage, logger log.Logger, forkchoiceStore forkchoice.ForkChoiceStorage, blobDownloader *network.BlobHistoryDownloader) StageHistoryReconstructionCfg { return StageHistoryReconstructionCfg{ beaconCfg: beaconCfg, @@ -168,6 +226,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co // EL block-number floor for snapshot-gap backfill, kept separate from the // beacon-slot destinationSlotForEL since the units must not be mixed. destinationBlockForEL := uint64(math.MaxUint64) + configureHistoryEnvelopeAdmission(cfg) // Set up onNewBlock callback // [Modified in Gloas:EIP7732] envelope is non-nil for GLOAS FULL blocks, nil for EMPTY or pre-GLOAS. cfg.downloader.SetOnNewBlock(func(blk *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) (finished bool, err error) { @@ -252,7 +311,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co if !hasELBlock { if blk.Version() >= clparams.GloasVersion { - if err := cfg.executionBlocksCollector.AddGloasBlock(blk.Block, envelope); err != nil { + if err := cfg.executionBlocksCollector.AddGloasBlock(ctx, blk.Block, envelope); err != nil { return false, fmt.Errorf("error adding gloas block to execution blocks collector: %s", err) } } else { @@ -414,7 +473,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co if cfg.engine != nil && cfg.downloader.Progress() <= destinationSlotForEL { sendPublicResult(nil) } - if err := cfg.downloader.RequestMore(ctx); err != nil { + if err := requestMoreWithEnvelopeRecovery(ctx, cfg); err != nil { if !errors.Is(err, context.Canceled) { log.Warn("closing backfilling routine", "err", err) } @@ -425,7 +484,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co // Recover FULL blocks whose envelopes were skipped during backward download. if skipped := cfg.downloader.SkippedFullBlocks(); len(skipped) > 0 { - if !recoverSkippedEnvelopesWithRetries(ctx, cfg, skipped) { + if pending := recoverSkippedEnvelopesWithRetries(ctx, cfg, skipped); len(pending) > 0 { workerErr := ctx.Err() if workerErr == nil { workerErr = errSkippedEnvelopeRecoveryIncomplete @@ -465,33 +524,73 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co return nil } -func recoverSkippedEnvelopesWithRetries(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock) bool { - return recoverSkippedEnvelopesWithRetryPolicy(ctx, cfg, skipped, - func(attemptCtx context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { - return recoverSkippedEnvelopes(attemptCtx, cfg, pending) - }, skippedEnvelopeRecoveryRetryInterval) +func requestMoreWithEnvelopeRecovery(ctx context.Context, cfg StageHistoryReconstructionCfg) error { + return requestMoreWithEnvelopeRecoveryPolicy(ctx, cfg, skippedEnvelopeRecoveryRetryInterval) } -func recoverSkippedEnvelopesWithRetryPolicy(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock, recoverAttempt func(context.Context, []network.SkippedFullBlock) []network.SkippedFullBlock, retryInterval time.Duration) bool { +func requestMoreWithEnvelopeRecoveryPolicy(ctx context.Context, cfg StageHistoryReconstructionCfg, retryInterval time.Duration) error { + err := cfg.downloader.RequestMore(ctx) + if !errors.Is(err, network.ErrSkippedEnvelopeRecoveryCapacity) { + return err + } + skipped := cfg.downloader.SkippedFullBlocks() + if len(skipped) == 0 { + return err + } + pending := recoverSkippedEnvelopesWithRetryInterval(ctx, cfg, skipped, retryInterval) + recovered := recoveredSkippedEnvelopes(skipped, pending) + cfg.downloader.AcknowledgeSkippedFullBlocks(recovered) + if len(recovered) == 0 { + return err + } + return nil +} + +func recoveredSkippedEnvelopes(skipped, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + counts := make(map[network.SkippedFullBlock]int, len(pending)) + for _, item := range pending { + counts[item]++ + } + recovered := make([]network.SkippedFullBlock, 0, len(skipped)-min(len(skipped), len(pending))) + for _, item := range skipped { + if counts[item] > 0 { + counts[item]-- + continue + } + recovered = append(recovered, item) + } + return recovered +} + +func recoverSkippedEnvelopesWithRetries(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock) []network.SkippedFullBlock { + return recoverSkippedEnvelopesWithRetryInterval(ctx, cfg, skipped, skippedEnvelopeRecoveryRetryInterval) +} + +func recoverSkippedEnvelopesWithRetryInterval(ctx context.Context, cfg StageHistoryReconstructionCfg, skipped []network.SkippedFullBlock, retryInterval time.Duration) []network.SkippedFullBlock { if cfg.downloader == nil || !cfg.downloader.HasEnvelopeRecoverySource() { log.Warn("[BackwardBeaconDownloader] envelope recovery unavailable", "remaining", len(skipped)) - return false + return append([]network.SkippedFullBlock(nil), skipped...) } - return recoverSkippedEnvelopesUntilComplete(ctx, skipped, recoverAttempt, retryInterval) + return recoverSkippedEnvelopesUntilComplete(ctx, skipped, + func(attemptCtx context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + return recoverSkippedEnvelopes(attemptCtx, cfg, pending) + }, retryInterval) } -func recoverSkippedEnvelopesUntilComplete(ctx context.Context, skipped []network.SkippedFullBlock, recoverAttempt func(context.Context, []network.SkippedFullBlock) []network.SkippedFullBlock, retryInterval time.Duration) bool { +func recoverSkippedEnvelopesUntilComplete(ctx context.Context, skipped []network.SkippedFullBlock, recoverAttempt func(context.Context, []network.SkippedFullBlock) []network.SkippedFullBlock, retryInterval time.Duration) []network.SkippedFullBlock { pending := skipped - for attempt := 1; ; attempt++ { + itemsPerAttempt := int(skippedEnvelopeRecoveryAttemptTimeout/skippedEnvelopeRecoveryBatchTimeout) * skippedEnvelopeRecoveryBatchSize + maxAttempts := max(1, (len(skipped)+itemsPerAttempt-1)/itemsPerAttempt) * skippedEnvelopeRecoverySweeps + for attempt := 1; attempt <= maxAttempts; attempt++ { if ctx.Err() != nil { log.Warn("[BackwardBeaconDownloader] envelope recovery canceled", "remaining", len(pending), "err", ctx.Err()) - return false + return pending } attemptCtx, cancel := context.WithTimeout(ctx, skippedEnvelopeRecoveryAttemptTimeout) pending = recoverAttempt(attemptCtx, pending) cancel() if len(pending) == 0 { - return true + return nil } log.Warn("[BackwardBeaconDownloader] envelope recovery incomplete, retrying", @@ -501,10 +600,13 @@ func recoverSkippedEnvelopesUntilComplete(ctx context.Context, skipped []network select { case <-ctx.Done(): log.Warn("[BackwardBeaconDownloader] envelope recovery canceled", "remaining", len(pending), "err", ctx.Err()) - return false + return pending case <-time.After(retryInterval): } } + log.Warn("[BackwardBeaconDownloader] envelope recovery exhausted", + "attempts", maxAttempts, "total", len(skipped), "remaining", len(pending)) + return pending } // recoverSkippedEnvelopes attempts to fetch execution payload envelopes for @@ -548,14 +650,18 @@ func recoverSkippedEnvelopeBatch(fetchCtx, persistCtx context.Context, cfg Stage return append([]network.SkippedFullBlock(nil), batch...) } blocks := readSkippedEnvelopeBlocks(persistCtx, cfg, batch) - recovery := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, batch, blocks) + recoverable, unknown := classifyCanonicalSkippedEnvelopes(persistCtx, cfg, batch, blocks) + if len(recoverable) == 0 { + return unknown + } + recovery := cfg.downloader.RecoverSkippedEnvelopes(fetchCtx, recoverable, blocks) tx, err := cfg.indiciesDB.BeginRo(persistCtx) if err != nil { - return append([]network.SkippedFullBlock(nil), batch...) + return append(unknown, recoverable...) } defer tx.Rollback() - return unresolvedSkippedEnvelopes(batch, recovery, func(s network.SkippedFullBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { + remaining := unresolvedSkippedEnvelopes(recoverable, recovery, func(s network.SkippedFullBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { block, err := cfg.blockReader.ReadBlockByRoot(persistCtx, tx, common.Hash(s.Root)) if err != nil || block == nil || block.Block == nil || block.Block.Body == nil { log.Warn("[BackwardBeaconDownloader] skipped block unavailable during recovery", "slot", s.Slot, "root", common.Hash(s.Root), "err", err) @@ -571,6 +677,131 @@ func recoverSkippedEnvelopeBatch(fetchCtx, persistCtx context.Context, cfg Stage } return recoverSkippedEnvelope(persistCtx, cfg, s, block, env) }) + if len(unknown) == 0 { + return remaining + } + pending := make(map[network.SkippedFullBlock]struct{}, len(unknown)+len(remaining)) + for _, item := range unknown { + pending[item] = struct{}{} + } + for _, item := range remaining { + pending[item] = struct{}{} + } + ordered := make([]network.SkippedFullBlock, 0, len(pending)) + for _, item := range batch { + if _, ok := pending[item]; ok { + ordered = append(ordered, item) + } + } + return ordered +} + +func classifyCanonicalSkippedEnvelopes(ctx context.Context, cfg StageHistoryReconstructionCfg, batch []network.SkippedFullBlock, blocks map[common.Hash]*cltypes.SignedBeaconBlock) (recoverable, unknown []network.SkippedFullBlock) { + recoverable = make([]network.SkippedFullBlock, 0, len(batch)) + unknown = make([]network.SkippedFullBlock, 0, len(batch)) + tx, err := cfg.indiciesDB.BeginRo(ctx) + if err != nil { + return recoverable, append(unknown, batch...) + } + defer tx.Rollback() + for _, item := range batch { + full, known, err := findCanonicalSkippedEnvelopeAvailability( + ctx, + item, + blocks[common.Hash(item.Root)], + func(ctx context.Context, slot uint64) (*cltypes.SignedBeaconBlock, bool, error) { + canonicalRoot, err := beacon_indicies.ReadCanonicalBlockRoot(tx, slot) + if err != nil || canonicalRoot == (common.Hash{}) { + return nil, false, err + } + child, err := cfg.blockReader.ReadBlockByRoot(ctx, tx, canonicalRoot) + if err != nil || child == nil || child.Block == nil { + return child, true, err + } + childRoot, err := child.Block.HashSSZ() + if err != nil || childRoot != canonicalRoot { + return nil, true, err + } + return child, true, nil + }, + ) + if err != nil || !known { + unknown = append(unknown, item) + } else if full { + recoverable = append(recoverable, item) + } + } + return recoverable, unknown +} + +func findCanonicalSkippedEnvelopeAvailability( + ctx context.Context, + item network.SkippedFullBlock, + block *cltypes.SignedBeaconBlock, + readCanonicalBlock func(context.Context, uint64) (*cltypes.SignedBeaconBlock, bool, error), +) (bool, bool, error) { + if block == nil || block.Block == nil || block.Block.Body == nil { + return false, false, nil + } + anchorSlot := block.Block.Slot + hintSlot := item.ChildSlot + hintAttempted := hintSlot > anchorSlot + var hintBlock *cltypes.SignedBeaconBlock + var hintFound bool + if hintAttempted { + if err := ctx.Err(); err != nil { + return false, false, err + } + var err error + hintBlock, hintFound, err = readCanonicalBlock(ctx, hintSlot) + if err != nil { + return false, false, err + } + if hintFound && hintBlock != nil && hintBlock.Block != nil && hintBlock.Block.Slot == hintSlot { + full, known := canonicalGloasBlockAvailability(block, hintBlock, common.Hash(item.Root)) + if known { + return full, true, nil + } + } + } + for offset := uint64(1); offset <= skippedEnvelopeCanonicalFallbackSlots; offset++ { + if err := ctx.Err(); err != nil { + return false, false, err + } + if anchorSlot > ^uint64(0)-offset { + break + } + slot := anchorSlot + offset + child, found := hintBlock, hintFound + var err error + if !hintAttempted || slot != hintSlot { + child, found, err = readCanonicalBlock(ctx, slot) + } + if err != nil { + return false, false, err + } + if !found { + continue + } + if child == nil || child.Block == nil || child.Block.Slot != slot { + return false, false, nil + } + full, known := canonicalGloasBlockAvailability(block, child, common.Hash(item.Root)) + return full, known, nil + } + return false, false, nil +} + +func canonicalGloasBlockAvailability(block, child *cltypes.SignedBeaconBlock, blockRoot common.Hash) (bool, bool) { + if block == nil || block.Block == nil || block.Block.Body == nil || child == nil || child.Block == nil || child.Block.Body == nil || child.Block.ParentRoot != blockRoot || child.Block.Slot <= block.Block.Slot { + return false, false + } + bid := block.Block.Body.GetSignedExecutionPayloadBid() + childBid := child.Block.Body.GetSignedExecutionPayloadBid() + if bid == nil || bid.Message == nil || childBid == nil || childBid.Message == nil { + return false, false + } + return childBid.Message.ParentBlockHash == bid.Message.BlockHash, true } func unresolvedSkippedEnvelopes(batch []network.SkippedFullBlock, recovery network.EnvelopeRecoveryResult, persist func(network.SkippedFullBlock, *cltypes.SignedExecutionPayloadEnvelope) bool) []network.SkippedFullBlock { @@ -608,7 +839,7 @@ func readSkippedEnvelopeBlocks(ctx context.Context, cfg StageHistoryReconstructi func recoverSkippedEnvelope(ctx context.Context, cfg StageHistoryReconstructionCfg, s network.SkippedFullBlock, block *cltypes.SignedBeaconBlock, env *cltypes.SignedExecutionPayloadEnvelope) bool { if cfg.executionBlocksCollector != nil { - if err := cfg.executionBlocksCollector.AddGloasBlock(block.Block, env); err != nil { + if err := cfg.executionBlocksCollector.AddGloasBlock(ctx, block.Block, env); err != nil { log.Warn("[BackwardBeaconDownloader] envelope recovery: add block failed", "err", err) return false } diff --git a/cl/phase1/stages/stage_history_download_test.go b/cl/phase1/stages/stage_history_download_test.go index c6c6ca186f7..afb00a010c9 100644 --- a/cl/phase1/stages/stage_history_download_test.go +++ b/cl/phase1/stages/stage_history_download_test.go @@ -20,31 +20,72 @@ import ( "context" "errors" "math" + "sync/atomic" "testing" "time" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/persistence/beacon_indicies" "github.com/erigontech/erigon/cl/phase1/network" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/memdb" + "github.com/erigontech/erigon/execution/types" + "github.com/stretchr/testify/require" ) type historyDownloaderStub struct { - finished bool - progress uint64 - requestErr error - requestMore func() error - skipped []network.SkippedFullBlock - recoverySource bool + finished bool + progress atomic.Uint64 + requestErr error + requestMore func() error + skipped []network.SkippedFullBlock + recoverySource bool + recover func(context.Context, []network.SkippedFullBlock, map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult + acknowledged int + validateBlock network.ValidateBlockFn + validateLookahead network.ValidateLookaheadFn +} + +type recoveryBlockReader map[common.Hash]*cltypes.SignedBeaconBlock + +func (r recoveryBlockReader) ReadBlockBySlot(context.Context, kv.Tx, uint64) (*cltypes.SignedBeaconBlock, error) { + return nil, nil +} +func (r recoveryBlockReader) ReadBlockByRoot(_ context.Context, _ kv.Tx, root common.Hash) (*cltypes.SignedBeaconBlock, error) { + return r[root], nil +} +func (r recoveryBlockReader) ReadHeaderByRoot(context.Context, kv.Tx, common.Hash) (*cltypes.SignedBeaconBlockHeader, error) { + return nil, nil +} +func (r recoveryBlockReader) ReadBeaconBlockBodyBySlot(context.Context, kv.Tx, uint64) (*cltypes.SignedBeaconBlock, error) { + return nil, nil +} +func (r recoveryBlockReader) FrozenSlots() uint64 { return 0 } +func (r recoveryBlockReader) CacheBlockBody(uint64, [][]byte, []*types.Withdrawal) {} + +func recoveryGloasBlock(slot uint64, parentRoot, blockHash, parentBlockHash common.Hash) *cltypes.SignedBeaconBlock { + block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.GloasVersion) + block.Block.Slot = slot + block.Block.ParentRoot = parentRoot + block.Block.Body.SignedExecutionPayloadBid.Message.BlockHash = blockHash + block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash = parentBlockHash + return block } func (d *historyDownloaderStub) SetSlotToDownload(uint64) {} func (d *historyDownloaderStub) SetExpectedRoot(common.Hash) {} func (d *historyDownloaderStub) SetBlockChecker(network.BlockChecker) {} func (d *historyDownloaderStub) SetOnNewBlock(network.OnNewBlock) {} -func (d *historyDownloaderStub) Finished() bool { return d.finished } -func (d *historyDownloaderStub) Progress() uint64 { return d.progress } +func (d *historyDownloaderStub) SetValidateFunctions(validateBlock network.ValidateBlockFn, validateLookahead network.ValidateLookaheadFn) { + d.validateBlock = validateBlock + d.validateLookahead = validateLookahead +} +func (d *historyDownloaderStub) Finished() bool { return d.finished } +func (d *historyDownloaderStub) Progress() uint64 { return d.progress.Load() } func (d *historyDownloaderStub) RequestMore(context.Context) error { if d.requestMore != nil { return d.requestMore() @@ -54,8 +95,25 @@ func (d *historyDownloaderStub) RequestMore(context.Context) error { func (d *historyDownloaderStub) SkippedFullBlocks() []network.SkippedFullBlock { return d.skipped } +func (d *historyDownloaderStub) AcknowledgeSkippedFullBlocks(recovered []network.SkippedFullBlock) { + d.acknowledged += len(recovered) + pending := make(map[network.SkippedFullBlock]struct{}, len(recovered)) + for _, item := range recovered { + pending[item] = struct{}{} + } + remaining := d.skipped[:0] + for _, item := range d.skipped { + if _, ok := pending[item]; !ok { + remaining = append(remaining, item) + } + } + d.skipped = remaining +} func (d *historyDownloaderStub) HasEnvelopeRecoverySource() bool { return d.recoverySource } -func (d *historyDownloaderStub) RecoverSkippedEnvelopes(context.Context, []network.SkippedFullBlock, map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult { +func (d *historyDownloaderStub) RecoverSkippedEnvelopes(ctx context.Context, skipped []network.SkippedFullBlock, blocks map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult { + if d.recover != nil { + return d.recover(ctx, skipped, blocks) + } return network.EnvelopeRecoveryResult{} } func (d *historyDownloaderStub) SetThrottle(time.Duration) {} @@ -63,7 +121,8 @@ func (d *historyDownloaderStub) SetNeverSkip(bool) {} func TestSpawnStageHistoryDownloadReturnsDownloaderFailure(t *testing.T) { wantErr := errors.New("terminal downloader failure") - downloader := &historyDownloaderStub{progress: math.MaxUint64, requestErr: wantErr} + downloader := &historyDownloaderStub{requestErr: wantErr} + downloader.progress.Store(math.MaxUint64) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() @@ -79,9 +138,10 @@ func TestSpawnStageHistoryDownloadReturnsDownloaderFailure(t *testing.T) { func TestSpawnStageHistoryDownloadReturnsFailureWhenRequestCrossesELFloor(t *testing.T) { wantErr := errors.New("terminal downloader failure at EL floor") destinationSlot := clparams.MainnetBeaconConfig.BellatrixForkEpoch * clparams.MainnetBeaconConfig.SlotsPerEpoch - downloader := &historyDownloaderStub{progress: destinationSlot + 1} + downloader := &historyDownloaderStub{} + downloader.progress.Store(destinationSlot + 1) downloader.requestMore = func() error { - downloader.progress = destinationSlot + downloader.progress.Store(destinationSlot) return wantErr } ctx, cancel := context.WithTimeout(t.Context(), time.Second) @@ -114,6 +174,29 @@ func TestSpawnStageHistoryDownloadReturnsEnvelopeRecoveryFailure(t *testing.T) { } } +func TestConfigureHistoryEnvelopeAdmissionPreservesPreinstalledValidatorsWithoutForkchoice(t *testing.T) { + downloader := &historyDownloaderStub{} + blockCalls := 0 + lookaheadCalls := 0 + downloader.SetValidateFunctions( + func(*cltypes.SignedBeaconBlock) error { + blockCalls++ + return nil + }, + func(*cltypes.SignedBeaconBlock, *cltypes.SignedBeaconBlock) error { + lookaheadCalls++ + return nil + }, + ) + + configureHistoryEnvelopeAdmission(StageHistoryReconstructionCfg{downloader: downloader}) + + require.NoError(t, downloader.validateBlock(nil)) + require.NoError(t, downloader.validateLookahead(nil, nil)) + require.Equal(t, 1, blockCalls) + require.Equal(t, 1, lookaheadCalls) +} + func TestUnresolvedSkippedEnvelopesRetriesEveryMissingEnvelope(t *testing.T) { first := network.SkippedFullBlock{Slot: 1, Root: [32]byte{1}} second := network.SkippedFullBlock{Slot: 2, Root: [32]byte{2}} @@ -129,6 +212,318 @@ func TestUnresolvedSkippedEnvelopesRetriesEveryMissingEnvelope(t *testing.T) { } } +func TestRecoverSkippedEnvelopeBatchCanonicalChildProvesEmptyWithoutEnvelope(t *testing.T) { + parent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + candidateChild := recoveryGloasBlock(101, parentRoot, common.Hash{4}, common.Hash{5}) + candidateChildRoot, err := candidateChild.Block.HashSSZ() + require.NoError(t, err) + canonicalChild := recoveryGloasBlock(parent.Block.Slot+65, parentRoot, common.Hash{6}, common.Hash{7}) + canonicalChildRoot, err := canonicalChild.Block.HashSSZ() + require.NoError(t, err) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.MarkRootCanonical(t.Context(), tx, canonicalChild.Block.Slot, canonicalChildRoot) + })) + recoveryCalled := false + downloader := &historyDownloaderStub{ + recoverySource: true, + recover: func(context.Context, []network.SkippedFullBlock, map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult { + recoveryCalled = true + return network.EnvelopeRecoveryResult{} + }, + } + item := network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: canonicalChild.Block.Slot, ChildRoot: candidateChildRoot} + reader := recoveryBlockReader{parentRoot: parent, candidateChildRoot: candidateChild, canonicalChildRoot: canonicalChild} + cfg := StageHistoryReconstructionCfg{indiciesDB: db, blockReader: reader, downloader: downloader} + + remaining := recoverSkippedEnvelopeBatch(t.Context(), t.Context(), cfg, []network.SkippedFullBlock{item}) + + require.Empty(t, remaining) + require.False(t, recoveryCalled) +} + +func TestRecoverSkippedEnvelopeBatchDistantCanonicalChildProvesFull(t *testing.T) { + beaconCfg, _, bid, envelope, _ := validAnchorEnvelopeFixture(t, 0) + parent := cltypes.NewSignedBeaconBlock(beaconCfg, clparams.GloasVersion) + parent.Block.Slot = bid.Slot + parent.Block.Body.SignedExecutionPayloadBid.Message = bid + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + envelope.Message.BeaconBlockRoot = parentRoot + canonicalChild := recoveryGloasBlock(parent.Block.Slot+65, parentRoot, common.Hash{4}, bid.BlockHash) + canonicalChildRoot, err := canonicalChild.Block.HashSSZ() + require.NoError(t, err) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.MarkRootCanonical(t.Context(), tx, canonicalChild.Block.Slot, canonicalChildRoot) + })) + recoveryCalled := false + downloader := &historyDownloaderStub{ + recoverySource: true, + recover: func(_ context.Context, skipped []network.SkippedFullBlock, _ map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult { + recoveryCalled = true + require.Len(t, skipped, 1) + return network.EnvelopeRecoveryResult{Envelopes: map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{parentRoot: envelope}} + }, + } + item := network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: canonicalChild.Block.Slot, ChildRoot: common.Hash{9}} + reader := recoveryBlockReader{parentRoot: parent, canonicalChildRoot: canonicalChild} + cfg := StageHistoryReconstructionCfg{beaconCfg: beaconCfg, indiciesDB: db, blockReader: reader, downloader: downloader} + + remaining := recoverSkippedEnvelopeBatch(t.Context(), t.Context(), cfg, []network.SkippedFullBlock{item}) + + require.Empty(t, remaining) + require.True(t, recoveryCalled) +} + +func TestCanonicalSkippedEnvelopeSearchRejectsNonDirectDescendant(t *testing.T) { + parent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + directChild := recoveryGloasBlock(101, parentRoot, common.Hash{4}, common.Hash{5}) + directChildRoot, err := directChild.Block.HashSSZ() + require.NoError(t, err) + descendant := recoveryGloasBlock(102, directChildRoot, common.Hash{6}, common.Hash{7}) + + full, known, err := findCanonicalSkippedEnvelopeAvailability( + t.Context(), + network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: descendant.Block.Slot}, + parent, + func(_ context.Context, slot uint64) (*cltypes.SignedBeaconBlock, bool, error) { + if slot == descendant.Block.Slot { + return descendant, true, nil + } + return nil, false, nil + }, + ) + + require.NoError(t, err) + require.False(t, full) + require.False(t, known) +} + +func TestCanonicalSkippedEnvelopeSearchFindsAlternateSlotFull(t *testing.T) { + parent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + canonicalChild := recoveryGloasBlock(102, parentRoot, common.Hash{4}, common.Hash{2}) + + full, known, err := findCanonicalSkippedEnvelopeAvailability( + t.Context(), + network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: parent.Block.Slot + 1}, + parent, + func(_ context.Context, slot uint64) (*cltypes.SignedBeaconBlock, bool, error) { + if slot == canonicalChild.Block.Slot { + return canonicalChild, true, nil + } + return nil, false, nil + }, + ) + + require.NoError(t, err) + require.True(t, full) + require.True(t, known) +} + +func TestCanonicalSkippedEnvelopeSearchInvalidHugeHintBoundsCanonicalReads(t *testing.T) { + parent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + reads := 0 + + full, known, err := findCanonicalSkippedEnvelopeAvailability( + t.Context(), + network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: math.MaxUint64}, + parent, + func(context.Context, uint64) (*cltypes.SignedBeaconBlock, bool, error) { + reads++ + return nil, false, nil + }, + ) + + require.NoError(t, err) + require.False(t, full) + require.False(t, known) + require.Equal(t, skippedEnvelopeCanonicalFallbackSlots+1, reads) +} + +func TestCanonicalSkippedEnvelopeSearchMaxAnchorDoesNotRead(t *testing.T) { + parent := recoveryGloasBlock(math.MaxUint64, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + reads := 0 + + full, known, err := findCanonicalSkippedEnvelopeAvailability( + t.Context(), + network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot}, + parent, + func(context.Context, uint64) (*cltypes.SignedBeaconBlock, bool, error) { + reads++ + return nil, false, nil + }, + ) + + require.NoError(t, err) + require.False(t, full) + require.False(t, known) + require.Zero(t, reads) +} + +func TestCanonicalSkippedEnvelopeSearchNearMaxAnchorBoundsReads(t *testing.T) { + parent := recoveryGloasBlock(math.MaxUint64-63, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + reads := 0 + + full, known, err := findCanonicalSkippedEnvelopeAvailability( + t.Context(), + network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot}, + parent, + func(context.Context, uint64) (*cltypes.SignedBeaconBlock, bool, error) { + reads++ + return nil, false, nil + }, + ) + + require.NoError(t, err) + require.False(t, full) + require.False(t, known) + require.Equal(t, 63, reads) +} + +func TestCanonicalSkippedEnvelopeSearchStopsOnCancellation(t *testing.T) { + parent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, _, err = findCanonicalSkippedEnvelopeAvailability( + ctx, + network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot}, + parent, + func(context.Context, uint64) (*cltypes.SignedBeaconBlock, bool, error) { + t.Fatal("canceled search must not read canonical slots") + return nil, false, nil + }, + ) + + require.ErrorIs(t, err, context.Canceled) +} + +func TestRecoverSkippedEnvelopeBatchUnknownCannotPersistLateEnvelope(t *testing.T) { + beaconCfg, _, bid, lateEnvelope, _ := validAnchorEnvelopeFixture(t, 0) + parent := cltypes.NewSignedBeaconBlock(beaconCfg, clparams.GloasVersion) + parent.Block.Slot = bid.Slot + parent.Block.Body.SignedExecutionPayloadBid.Message = bid + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + lateEnvelope.Message.BeaconBlockRoot = parentRoot + require.NoError(t, network.ValidateFetchedEnvelope(beaconCfg, parent, parentRoot, lateEnvelope)) + sideChild := recoveryGloasBlock(parent.Block.Slot+2, parentRoot, common.Hash{4}, common.Hash{5}) + sideChildRoot, err := sideChild.Block.HashSSZ() + require.NoError(t, err) + canonicalChild := recoveryGloasBlock(parent.Block.Slot+2, common.Hash{9}, common.Hash{6}, common.Hash{7}) + canonicalChildRoot, err := canonicalChild.Block.HashSSZ() + require.NoError(t, err) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.MarkRootCanonical(t.Context(), tx, canonicalChild.Block.Slot, canonicalChildRoot) + })) + recoveryCalled := false + downloader := &historyDownloaderStub{ + recoverySource: true, + recover: func(_ context.Context, skipped []network.SkippedFullBlock, _ map[common.Hash]*cltypes.SignedBeaconBlock) network.EnvelopeRecoveryResult { + recoveryCalled = true + return network.EnvelopeRecoveryResult{Envelopes: map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{ + parentRoot: lateEnvelope, + }} + }, + } + item := network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: sideChild.Block.Slot, ChildRoot: sideChildRoot} + reader := recoveryBlockReader{parentRoot: parent, sideChildRoot: sideChild, canonicalChildRoot: canonicalChild} + cfg := StageHistoryReconstructionCfg{beaconCfg: beaconCfg, indiciesDB: db, blockReader: reader, downloader: downloader} + + remaining := recoverSkippedEnvelopeBatch(t.Context(), t.Context(), cfg, []network.SkippedFullBlock{item}) + + require.Equal(t, []network.SkippedFullBlock{item}, remaining) + require.False(t, recoveryCalled) +} + +func TestRequestMoreRecoversCapacityBeforeDownloaderFinishes(t *testing.T) { + parent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + parentRoot, err := parent.Block.HashSSZ() + require.NoError(t, err) + child := recoveryGloasBlock(102, parentRoot, common.Hash{4}, common.Hash{5}) + childRoot, err := child.Block.HashSSZ() + require.NoError(t, err) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.MarkRootCanonical(t.Context(), tx, child.Block.Slot, childRoot) + })) + item := network.SkippedFullBlock{Slot: parent.Block.Slot, Root: parentRoot, ChildSlot: child.Block.Slot, ChildRoot: childRoot} + downloader := &historyDownloaderStub{ + requestErr: network.ErrSkippedEnvelopeRecoveryCapacity, + skipped: []network.SkippedFullBlock{item}, + recoverySource: true, + } + cfg := StageHistoryReconstructionCfg{ + downloader: downloader, + indiciesDB: db, + blockReader: recoveryBlockReader{parentRoot: parent, childRoot: child}, + } + + require.NoError(t, requestMoreWithEnvelopeRecovery(t.Context(), cfg)) + require.Equal(t, 1, downloader.acknowledged) + require.Empty(t, downloader.skipped) +} + +func TestRequestMoreAcknowledgesPartialCapacityRecovery(t *testing.T) { + resolvedParent := recoveryGloasBlock(100, common.Hash{1}, common.Hash{2}, common.Hash{3}) + resolvedRoot, err := resolvedParent.Block.HashSSZ() + require.NoError(t, err) + resolvedChild := recoveryGloasBlock(102, resolvedRoot, common.Hash{4}, common.Hash{5}) + resolvedChildRoot, err := resolvedChild.Block.HashSSZ() + require.NoError(t, err) + unresolvedParent := recoveryGloasBlock(99, common.Hash{6}, common.Hash{7}, common.Hash{8}) + unresolvedRoot, err := unresolvedParent.Block.HashSSZ() + require.NoError(t, err) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.MarkRootCanonical(t.Context(), tx, resolvedChild.Block.Slot, resolvedChildRoot) + })) + resolved := network.SkippedFullBlock{Slot: resolvedParent.Block.Slot, Root: resolvedRoot, ChildSlot: resolvedChild.Block.Slot, ChildRoot: common.Hash{9}} + unresolved := network.SkippedFullBlock{Slot: unresolvedParent.Block.Slot, Root: unresolvedRoot, ChildSlot: 101, ChildRoot: common.Hash{10}} + downloader := &historyDownloaderStub{ + requestErr: network.ErrSkippedEnvelopeRecoveryCapacity, + skipped: []network.SkippedFullBlock{resolved, unresolved}, + recoverySource: true, + } + cfg := StageHistoryReconstructionCfg{ + downloader: downloader, + indiciesDB: db, + blockReader: recoveryBlockReader{ + resolvedRoot: resolvedParent, + resolvedChildRoot: resolvedChild, + unresolvedRoot: unresolvedParent, + }, + } + + require.NoError(t, requestMoreWithEnvelopeRecoveryPolicy(t.Context(), cfg, 0)) + require.Equal(t, 1, downloader.acknowledged) + require.Equal(t, []network.SkippedFullBlock{unresolved}, downloader.skipped) + + downloader.requestErr = nil + require.NoError(t, requestMoreWithEnvelopeRecoveryPolicy(t.Context(), cfg, 0)) +} + func TestRecoverSkippedEnvelopeBatchesDoesNotStarveLaterBatches(t *testing.T) { skipped := []network.SkippedFullBlock{{Slot: 1}, {Slot: 2}, {Slot: 3}, {Slot: 4}, {Slot: 5}, {Slot: 6}, {Slot: 7}, {Slot: 8}} attempted := make([]uint64, 0, len(skipped)) @@ -170,25 +565,14 @@ func TestRecoverSkippedEnvelopeBatchesKeepsPartialSuccess(t *testing.T) { func TestRecoverSkippedEnvelopesWithoutSourcesDoesNotCompleteBackfill(t *testing.T) { cfg := StageHistoryReconstructionCfg{downloader: &network.BackwardBeaconDownloader{}} - attempts := 0 - recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { - attempts++ - return pending - } - - if recoverSkippedEnvelopesWithRetryPolicy(context.Background(), cfg, []network.SkippedFullBlock{{Slot: 1}}, recoverAttempt, 0) { + pending := recoverSkippedEnvelopesWithRetryInterval(context.Background(), cfg, []network.SkippedFullBlock{{Slot: 1}}, 0) + if len(pending) == 0 { t.Fatal("recovery without an HTTP or P2P source must not report completion") } - if attempts != 0 { - t.Fatalf("attempts = %d, want no recovery attempt without a source", attempts) - } } func TestRecoverSkippedEnvelopesRetriesBeyondThreeAttemptCapacity(t *testing.T) { const itemsPerAttempt = int(skippedEnvelopeRecoveryAttemptTimeout/skippedEnvelopeRecoveryBatchTimeout) * skippedEnvelopeRecoveryBatchSize - downloader := &network.BackwardBeaconDownloader{} - downloader.SetHTTPFallbackURL("http://recovery.test") - cfg := StageHistoryReconstructionCfg{downloader: downloader} skipped := make([]network.SkippedFullBlock, itemsPerAttempt*3+1) for i := range skipped { skipped[i].Slot = uint64(i + 1) @@ -200,8 +584,8 @@ func TestRecoverSkippedEnvelopesRetriesBeyondThreeAttemptCapacity(t *testing.T) return pending[min(itemsPerAttempt, len(pending)):] } - if !recoverSkippedEnvelopesWithRetryPolicy(context.Background(), cfg, skipped, recoverAttempt, 0) { - t.Fatal("configured recovery stopped before all pending envelopes were recovered") + if pending := recoverSkippedEnvelopesUntilComplete(context.Background(), skipped, recoverAttempt, 0); len(pending) != 0 { + t.Fatal("recovery stopped before all pending envelopes were recovered") } if len(attemptStarts) != 4 || attemptStarts[3] != uint64(itemsPerAttempt*3+1) { t.Fatalf("attempt starts = %v, want a fourth attempt starting at slot %d", attemptStarts, itemsPerAttempt*3+1) @@ -210,9 +594,6 @@ func TestRecoverSkippedEnvelopesRetriesBeyondThreeAttemptCapacity(t *testing.T) func TestRecoverSkippedEnvelopesStopsWhenParentContextIsCanceled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - downloader := &network.BackwardBeaconDownloader{} - downloader.SetHTTPFallbackURL("http://recovery.test") - cfg := StageHistoryReconstructionCfg{downloader: downloader} attempts := 0 recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { attempts++ @@ -220,7 +601,7 @@ func TestRecoverSkippedEnvelopesStopsWhenParentContextIsCanceled(t *testing.T) { return pending } - if recoverSkippedEnvelopesWithRetryPolicy(ctx, cfg, []network.SkippedFullBlock{{Slot: 1}}, recoverAttempt, time.Hour) { + if pending := recoverSkippedEnvelopesUntilComplete(ctx, []network.SkippedFullBlock{{Slot: 1}}, recoverAttempt, time.Hour); len(pending) == 0 { t.Fatal("recovery reported completion with a pending envelope after parent cancellation") } if attempts != 1 { @@ -228,6 +609,40 @@ func TestRecoverSkippedEnvelopesStopsWhenParentContextIsCanceled(t *testing.T) { } } +func TestRecoverSkippedEnvelopesStopsAfterBoundedZeroProgress(t *testing.T) { + attempts := 0 + recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + attempts++ + return pending + } + + if pending := recoverSkippedEnvelopesUntilComplete(t.Context(), []network.SkippedFullBlock{{Slot: 1}}, recoverAttempt, 0); len(pending) == 0 { + t.Fatal("zero-progress recovery must return an explicit incomplete result") + } + if attempts != skippedEnvelopeRecoverySweeps { + t.Fatalf("attempts = %d, want %d bounded attempts", attempts, skippedEnvelopeRecoverySweeps) + } +} + +func TestRecoverSkippedEnvelopesPartialProgressDoesNotResetBudget(t *testing.T) { + skipped := []network.SkippedFullBlock{{Slot: 1}, {Slot: 2}} + attempts := 0 + recoverAttempt := func(_ context.Context, pending []network.SkippedFullBlock) []network.SkippedFullBlock { + attempts++ + if attempts == 1 { + return pending[1:] + } + return pending + } + + if pending := recoverSkippedEnvelopesUntilComplete(t.Context(), skipped, recoverAttempt, 0); len(pending) == 0 { + t.Fatal("one recovered item must not turn an unrelated permanently missing item into success") + } + if attempts != skippedEnvelopeRecoverySweeps { + t.Fatalf("attempts = %d, want %d fixed attempts", attempts, skippedEnvelopeRecoverySweeps) + } +} + // clampProgress must never report a total below processed nor underflow, even // when the floor and current counters drift past the frozen highestBlockSeen. // The last case mirrors the field report where the live EL head advanced past diff --git a/cl/rpc/rpc.go b/cl/rpc/rpc.go index 9dac6a11221..0b1f8e1b192 100644 --- a/cl/rpc/rpc.go +++ b/cl/rpc/rpc.go @@ -192,10 +192,8 @@ func (b *BeaconRpcP2P) SendExecutionPayloadEnvelopesByRangeReq(ctx context.Conte envelopes := make([]*cltypes.SignedExecutionPayloadEnvelope, 0, len(responsePacket)) for _, data := range responsePacket { - envelope := &cltypes.SignedExecutionPayloadEnvelope{ - Message: cltypes.NewExecutionPayloadEnvelope(b.beaconConfig), - } - if err := envelope.DecodeSSZ(data.raw, int(data.version)); err != nil { + envelope, err := decodeExecutionPayloadEnvelope(data, b.beaconConfig) + if err != nil { return nil, pid, err } envelopes = append(envelopes, envelope) @@ -229,10 +227,8 @@ func (b *BeaconRpcP2P) SendExecutionPayloadEnvelopesByRootReq(ctx context.Contex envelopes := make([]*cltypes.SignedExecutionPayloadEnvelope, 0, len(responsePacket)) for _, data := range responsePacket { - envelope := &cltypes.SignedExecutionPayloadEnvelope{ - Message: cltypes.NewExecutionPayloadEnvelope(b.beaconConfig), - } - if err := envelope.DecodeSSZ(data.raw, int(data.version)); err != nil { + envelope, err := decodeExecutionPayloadEnvelope(data, b.beaconConfig) + if err != nil { return nil, pid, err } envelopes = append(envelopes, envelope) @@ -240,6 +236,23 @@ func (b *BeaconRpcP2P) SendExecutionPayloadEnvelopesByRootReq(ctx context.Contex return envelopes, pid, nil } +func decodeExecutionPayloadEnvelope(data responseData, beaconConfig *clparams.BeaconChainConfig) (*cltypes.SignedExecutionPayloadEnvelope, error) { + envelope := &cltypes.SignedExecutionPayloadEnvelope{ + Message: cltypes.NewExecutionPayloadEnvelope(beaconConfig), + } + if err := envelope.DecodeSSZ(data.raw, int(data.version)); err != nil { + return nil, err + } + canonical, err := envelope.EncodeSSZ(nil) + if err != nil { + return nil, err + } + if !bytes.Equal(data.raw, canonical) { + return nil, errors.New("non-canonical SSZ encoding") + } + return envelope, nil +} + // SendBeaconBlocksByRangeReq retrieves blocks range from beacon chain. func (b *BeaconRpcP2P) SendBlobsSidecarByIdentifierReq(ctx context.Context, req *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) { var buffer buffer.Buffer diff --git a/cl/rpc/rpc_test.go b/cl/rpc/rpc_test.go index 46ab29211ce..db72afa9710 100644 --- a/cl/rpc/rpc_test.go +++ b/cl/rpc/rpc_test.go @@ -3,6 +3,7 @@ package rpc import ( "bytes" "context" + "encoding/binary" "testing" "github.com/stretchr/testify/require" @@ -16,6 +17,11 @@ import ( "github.com/erigontech/erigon/node/gointerfaces/sentinelproto" ) +type rawSSZ []byte + +func (r rawSSZ) EncodeSSZ(dst []byte) ([]byte, error) { return append(dst, r...), nil } +func (r rawSSZ) EncodingSizeSSZ() int { return len(r) } + type blockResponseSentinel struct { sentinelproto.SentinelClient response []byte @@ -90,3 +96,50 @@ func TestSendBeaconBlocksByRangeReqRejectsForkSchemaSlotMismatch(t *testing.T) { require.Equal(t, "malicious-peer", pid) require.Equal(t, "malicious-peer", sentinel.bannedPeer) } + +func TestExecutionPayloadEnvelopeRequestsRejectNonCanonicalSSZ(t *testing.T) { + cfg := clparams.MainnetBeaconConfig + cfg.InitializeForkSchedule() + clock := eth_clock.NewEthereumClock(0, common.Hash{}, &cfg) + gloasDigest, err := clock.ComputeForkDigest(cfg.GloasForkEpoch) + require.NoError(t, err) + + envelope := &cltypes.SignedExecutionPayloadEnvelope{ + Message: cltypes.NewExecutionPayloadEnvelope(&cfg), + } + encoded, err := envelope.EncodeSSZ(nil) + require.NoError(t, err) + + const signedEnvelopeFixedSize = 4 + 96 + dynamicGap := make([]byte, 0, len(encoded)+1) + dynamicGap = append(dynamicGap, encoded[:signedEnvelopeFixedSize]...) + dynamicGap = append(dynamicGap, 0) + dynamicGap = append(dynamicGap, encoded[signedEnvelopeFixedSize:]...) + binary.LittleEndian.PutUint32(dynamicGap[:4], signedEnvelopeFixedSize+1) + + for _, test := range []struct { + name string + ssz []byte + }{ + {name: "dynamic offset gap", ssz: dynamicGap}, + {name: "trailing byte", ssz: append(append([]byte(nil), encoded...), 0)}, + } { + t.Run(test.name, func(t *testing.T) { + var response bytes.Buffer + require.NoError(t, ssz_snappy.EncodeAndWrite(&response, rawSSZ(test.ssz), gloasDigest[:]...)) + sentinel := &blockResponseSentinel{response: response.Bytes()} + client := &BeaconRpcP2P{ + ctx: context.Background(), + sentinel: sentinel, + beaconConfig: &cfg, + ethClock: clock, + } + + _, _, rangeErr := client.SendExecutionPayloadEnvelopesByRangeReq(context.Background(), 10, 1) + require.Error(t, rangeErr) + + _, _, rootErr := client.SendExecutionPayloadEnvelopesByRootReq(context.Background(), [][32]byte{{1}}) + require.Error(t, rootErr) + }) + } +} diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index 47b55dd035a..6b851e4ff81 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -51,6 +51,7 @@ import ( "github.com/erigontech/erigon/cl/phase1/network" "github.com/erigontech/erigon/cl/phase1/stages" "github.com/erigontech/erigon/cl/rpc" + eth2impl "github.com/erigontech/erigon/cl/transition/impl/eth2" "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/cl/utils/bls" "github.com/erigontech/erigon/cl/utils/eth_clock" @@ -144,6 +145,10 @@ func (c *Chain) Run(ctx *Context) error { freezingCfg := ethconfig.Defaults.Snapshot freezingCfg.ChainName = c.Chain csn := freezeblocks.NewCaplinSnapshots(freezingCfg, beaconConfig, dirs, log.Root()) + if err := csn.OpenFolder(); err != nil { + return err + } + defer csn.Close() bs, err := checkpoint_sync.NewRemoteCheckpointSync(beaconConfig, networkType).GetLatestBeaconState(ctx) if err != nil { return err @@ -179,10 +184,61 @@ func (c *Chain) Run(ctx *Context) error { } downloader := network.NewBackwardBeaconDownloader(ctx, beacon, nil, nil, db, beaconConfig) - cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, clparams.CaplinConfig{}, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root(), nil, nil) + admissionState, err := bs.Copy() + if err != nil { + return err + } + downloader.SetValidateFunctions(checkpointBackwardAdmissionValidators(admissionState, bRoot)) + snr := freezeblocks.NewBeaconSnapshotReader(csn, nil, beaconConfig) + cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, clparams.CaplinConfig{}, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, snr, blobStorage, log.Root(), nil, nil) return stages.SpawnStageHistoryDownload(cfg, ctx, log.Root()) } +func checkpointBackwardAdmissionValidators(admissionState *state.CachingBeaconState, startingRoot common.Hash) (network.ValidateBlockFn, network.ValidateLookaheadFn) { + validateBlock := func(block *cltypes.SignedBeaconBlock) error { + valid, err := eth2impl.VerifyBlockSignature(admissionState, block) + if err != nil { + return err + } + if !valid { + return fmt.Errorf("invalid historical block signature at slot %d", block.Block.Slot) + } + return nil + } + validateLookahead := func(anchor, child *cltypes.SignedBeaconBlock) error { + anchorRoot, err := anchor.Block.HashSSZ() + if err != nil { + return err + } + if anchorRoot != startingRoot { + return errors.New("lookahead anchor state unavailable") + } + if admissionState.Version() >= clparams.FuluVersion { + stateEpoch := admissionState.Slot() / admissionState.BeaconConfig().SlotsPerEpoch + childEpoch := child.Block.Slot / admissionState.BeaconConfig().SlotsPerEpoch + if childEpoch < stateEpoch || childEpoch > stateEpoch+admissionState.BeaconConfig().MinSeedLookahead { + return errors.New("lookahead proposer schedule unavailable") + } + } + expectedProposer, err := admissionState.GetBeaconProposerIndexForSlot(child.Block.Slot) + if err != nil { + return err + } + if expectedProposer != child.Block.ProposerIndex { + return fmt.Errorf("unexpected lookahead proposer %d at slot %d, expected %d", child.Block.ProposerIndex, child.Block.Slot, expectedProposer) + } + valid, err := eth2impl.VerifyBlockSignature(admissionState, child) + if err != nil { + return err + } + if !valid { + return fmt.Errorf("invalid lookahead signature at slot %d", child.Block.Slot) + } + return nil + } + return validateBlock, validateLookahead +} + type ChainEndpoint struct { Endpoint string `help:"endpoint" default:""` Blobs bool `help:"also download blobs" default:"false"` diff --git a/cmd/capcli/cli_test.go b/cmd/capcli/cli_test.go new file mode 100644 index 00000000000..13807a4f55a --- /dev/null +++ b/cmd/capcli/cli_test.go @@ -0,0 +1,43 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" + state2 "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/common" +) + +func TestCheckpointBackwardAdmissionRejectsHistoricalLookahead(t *testing.T) { + admissionState := state2.New(&clparams.MainnetBeaconConfig) + admissionState.SetVersion(clparams.GloasVersion) + admissionState.SetSlot(100) + anchor := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.GloasVersion) + anchor.Block.Slot = 99 + child := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.GloasVersion) + child.Block.Slot = 101 + _, validateLookahead := checkpointBackwardAdmissionValidators(admissionState, common.HexToHash("0x1234")) + + err := validateLookahead(anchor, child) + + require.ErrorContains(t, err, "anchor state unavailable") +} diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index 87f8d17fee9..e6805655aed 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -18,11 +18,16 @@ package caplin1 import ( "context" + "encoding/hex" "errors" "fmt" "math" "os" "path" + "path/filepath" + "runtime" + "sort" + "strings" "time" "github.com/spf13/afero" @@ -343,9 +348,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi attestationProducer := attestation_producer.New(ctx, beaconConfig) caplinFcuPath := path.Join(dirs.Tmp, "caplin-forkchoice") - dir.RemoveAll(caplinFcuPath) - err = os.MkdirAll(caplinFcuPath, 0o755) - if err != nil { + if err := prepareForkChoiceDirectory(caplinFcuPath); err != nil { return err } fcuFs := afero.NewBasePathFs(afero.NewOsFs(), caplinFcuPath) @@ -466,7 +469,8 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi committeeSub := committee_subscription.NewCommitteeSubscribeManagement(ctx, beaconConfig, networkConfig, ethClock, aggregationPool, syncedDataManager, gossipManager) batchSignatureVerifier := services.NewBatchSignatureVerifier(ctx, sentinel) // Define gossip services - blockService := services.NewBlockService(ctx, indexDB, forkChoice, syncedDataManager, ethClock, beaconConfig, emitters) + executionPayloadService := services.NewExecutionPayloadService(ctx, forkChoice, beaconConfig, emitters, beaconRpc) + blockService := services.NewBlockService(ctx, indexDB, forkChoice, syncedDataManager, ethClock, beaconConfig, emitters, executionPayloadService) blobService := services.NewBlobSidecarService(ctx, beaconConfig, forkChoice, syncedDataManager, ethClock, emitters, false) dataColumnSidecarService := services.NewDataColumnSidecarService(ctx, beaconConfig, ethClock, forkChoice, syncedDataManager, columnStorage, emitters) syncCommitteeMessagesService := services.NewSyncCommitteeMessagesService(beaconConfig, ethClock, syncedDataManager, syncContributionPool, batchSignatureVerifier, false) @@ -477,7 +481,6 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi blsToExecutionChangeService := services.NewBLSToExecutionChangeService(pool, emitters, syncedDataManager, beaconConfig, batchSignatureVerifier) proposerSlashingService := services.NewProposerSlashingService(pool, syncedDataManager, beaconConfig, ethClock, emitters) attesterSlashingService := services.NewAttesterSlashingService(forkChoice) - executionPayloadService := services.NewExecutionPayloadService(ctx, forkChoice, beaconConfig, emitters) payloadAttestationService := services.NewPayloadAttestationService(ctx, forkChoice, ethClock, networkConfig, emitters) proposerPreferencesService := services.NewProposerPreferencesService(syncedDataManager, forkChoice, ethClock, beaconConfig, epbsPool) executionPayloadBidService := services.NewExecutionPayloadBidService(ctx, syncedDataManager, forkChoice, ethClock, beaconConfig, epbsPool, emitters) @@ -661,3 +664,224 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi } return err } + +func prepareForkChoiceDirectory(forkChoicePath string) error { + return prepareForkChoiceDirectoryWithHook(forkChoicePath, nil) +} + +func prepareForkChoiceDirectoryWithHook(forkChoicePath string, hook func(string) error) error { + recoveryPath := forkChoicePath + "-envelope-recovery" + if err := os.MkdirAll(forkChoicePath, 0o755); err != nil { + return err + } + if err := os.MkdirAll(recoveryPath, 0o755); err != nil { + return err + } + if err := syncDirectory(filepath.Dir(forkChoicePath)); err != nil { + return err + } + if err := movePendingEnvelopeArtifacts(forkChoicePath, recoveryPath, hook); err != nil { + return err + } + if err := syncDirectory(forkChoicePath); err != nil { + return err + } + if err := syncDirectory(recoveryPath); err != nil { + return err + } + if err := dir.RemoveAll(forkChoicePath); err != nil { + return err + } + if err := runDirectoryBoundary(hook, "clear transient forkchoice"); err != nil { + return err + } + if err := os.MkdirAll(forkChoicePath, 0o755); err != nil { + return err + } + if err := restorePendingEnvelopeArtifacts(recoveryPath, forkChoicePath, hook); err != nil { + return err + } + if err := syncDirectory(forkChoicePath); err != nil { + return err + } + if err := syncDirectory(filepath.Dir(forkChoicePath)); err != nil { + return err + } + if err := dir.RemoveFile(recoveryPath); err != nil && !os.IsNotExist(err) { + return err + } + if err := runDirectoryBoundary(hook, "remove recovery directory"); err != nil { + return err + } + if err := syncDirectory(filepath.Dir(forkChoicePath)); err != nil { + return err + } + return nil +} + +func movePendingEnvelopeArtifacts(sourcePath, recoveryPath string, hook func(string) error) error { + const markerSuffix = ".envelope.indices-pending" + roots := make(map[string]struct{}) + for _, directory := range []string{sourcePath, recoveryPath} { + entries, err := os.ReadDir(directory) + if err != nil { + return err + } + for _, entry := range entries { + if directory == recoveryPath && (entry.IsDir() || !isEnvelopeRecoveryArtifact(entry.Name())) { + return fmt.Errorf("unexpected recovery artifact %q", entry.Name()) + } + if entry.IsDir() { + continue + } + rootHex, ok := envelopeArtifactRoot(entry.Name(), markerSuffix) + if ok { + roots[rootHex] = struct{}{} + } + } + } + recoveryEntries, err := os.ReadDir(recoveryPath) + if err != nil { + return err + } + for _, entry := range recoveryEntries { + rootHex, ok := envelopeRecoveryArtifactRoot(entry.Name()) + if !ok { + return fmt.Errorf("unexpected recovery artifact %q", entry.Name()) + } + if _, pending := roots[rootHex]; !pending { + return fmt.Errorf("recovery artifact has no pending marker %q", entry.Name()) + } + } + orderedRoots := make([]string, 0, len(roots)) + for rootHex := range roots { + orderedRoots = append(orderedRoots, rootHex) + } + sort.Strings(orderedRoots) + for _, rootHex := range orderedRoots { + filename := rootHex + ".envelope.snappy_ssz" + artifacts := []string{filename} + temporaryFiles, err := filepath.Glob(filepath.Join(sourcePath, filename+".tmp-*")) + if err != nil { + return err + } + for _, temporaryFile := range temporaryFiles { + artifacts = append(artifacts, filepath.Base(temporaryFile)) + } + sort.Strings(artifacts) + artifacts = append(artifacts, rootHex+markerSuffix) + for _, artifact := range artifacts { + if _, err := moveFileIfPresent(sourcePath, recoveryPath, artifact, hook, "preserve "); err != nil { + return err + } + } + } + return nil +} + +func restorePendingEnvelopeArtifacts(recoveryPath, destinationPath string, hook func(string) error) error { + entries, err := os.ReadDir(recoveryPath) + if err != nil { + return err + } + sort.Slice(entries, func(i, j int) bool { + iMarker := strings.HasSuffix(entries[i].Name(), ".envelope.indices-pending") + jMarker := strings.HasSuffix(entries[j].Name(), ".envelope.indices-pending") + if iMarker != jMarker { + return !iMarker + } + return entries[i].Name() < entries[j].Name() + }) + for _, entry := range entries { + if entry.IsDir() { + return fmt.Errorf("unexpected recovery directory %q", entry.Name()) + } + if !isEnvelopeRecoveryArtifact(entry.Name()) { + return fmt.Errorf("unexpected recovery artifact %q", entry.Name()) + } + if _, err := moveFileIfPresent(recoveryPath, destinationPath, entry.Name(), hook, "restore "); err != nil { + return err + } + } + return nil +} + +func moveFileIfPresent(sourcePath, destinationPath, name string, hook func(string) error, boundaryPrefix string) (bool, error) { + source := filepath.Join(sourcePath, name) + if _, err := os.Stat(source); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + destination := filepath.Join(destinationPath, name) + if _, err := os.Stat(destination); err == nil { + return false, fmt.Errorf("recovery artifact already exists: %s", destination) + } else if !os.IsNotExist(err) { + return false, err + } + if err := os.Rename(source, destination); err != nil { + return false, err + } + if err := syncDirectory(sourcePath); err != nil { + return true, err + } + if err := syncDirectory(destinationPath); err != nil { + return true, err + } + if err := runDirectoryBoundary(hook, boundaryPrefix+name); err != nil { + return true, err + } + return true, nil +} + +func runDirectoryBoundary(hook func(string) error, boundary string) error { + if hook == nil { + return nil + } + return hook(boundary) +} + +func syncDirectory(path string) error { + if runtime.GOOS == "windows" { + return nil + } + directory, err := os.Open(path) + if err != nil { + return err + } + if err := directory.Sync(); err != nil { + _ = directory.Close() + return err + } + return directory.Close() +} + +func envelopeArtifactRoot(name, suffix string) (string, bool) { + if !strings.HasSuffix(name, suffix) { + return "", false + } + rootHex := strings.TrimSuffix(name, suffix) + root, err := hex.DecodeString(rootHex) + return rootHex, err == nil && len(root) == len(common.Hash{}) +} + +func isEnvelopeRecoveryArtifact(name string) bool { + _, ok := envelopeRecoveryArtifactRoot(name) + return ok +} + +func envelopeRecoveryArtifactRoot(name string) (string, bool) { + for _, suffix := range []string{".envelope.indices-pending", ".envelope.snappy_ssz"} { + if rootHex, ok := envelopeArtifactRoot(name, suffix); ok { + return rootHex, true + } + } + const temporarySeparator = ".envelope.snappy_ssz.tmp-" + parts := strings.SplitN(name, temporarySeparator, 2) + if len(parts) != 2 || parts[1] == "" { + return "", false + } + root, err := hex.DecodeString(parts[0]) + return parts[0], err == nil && len(root) == len(common.Hash{}) +} diff --git a/cmd/caplin/caplin1/run_test.go b/cmd/caplin/caplin1/run_test.go new file mode 100644 index 00000000000..1e2955ac732 --- /dev/null +++ b/cmd/caplin/caplin1/run_test.go @@ -0,0 +1,269 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package caplin1 + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/cl/beacon/beacon_router_configuration" + "github.com/erigontech/erigon/cl/beacon/beaconevents" + "github.com/erigontech/erigon/cl/beacon/synced_data" + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/persistence/beacon_indicies" + "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/cl/phase1/forkchoice" + "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" + "github.com/erigontech/erigon/cl/phase1/forkchoice/public_keys_registry" + "github.com/erigontech/erigon/cl/pool" + "github.com/erigontech/erigon/cl/validator/validator_params" + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/memdb" +) + +func TestPrepareForkChoiceDirectoryResumesAfterEveryBoundary(t *testing.T) { + root := common.HexToHash("0x1234") + rootHex := fmt.Sprintf("%x", root) + filename := rootHex + ".envelope.snappy_ssz" + marker := rootHex + ".envelope.indices-pending" + boundaries := []string{ + "preserve " + filename, + "preserve " + marker, + "clear transient forkchoice", + "restore " + filename, + "restore " + marker, + "remove recovery directory", + } + for _, boundary := range boundaries { + t.Run(boundary, func(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + anchorState := state.New(cfg) + forkChoicePath := t.TempDir() + "/caplin-forkchoice" + require.NoError(t, prepareForkChoiceDirectory(forkChoicePath)) + fs := afero.NewBasePathFs(afero.NewOsFs(), forkChoicePath) + graph := fork_graph.NewForkGraphDisk(anchorState, nil, fs, beacon_router_configuration.RouterConfiguration{}) + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload.BlockHash = common.HexToHash("0xabcd") + envelope.Message.Payload.BlockNumber = 42 + require.NoError(t, graph.DumpEnvelopeOnDisk(root, envelope)) + + errInterrupted := errors.New("interrupted") + err := prepareForkChoiceDirectoryWithHook(forkChoicePath, func(current string) error { + if current == boundary { + return errInterrupted + } + return nil + }) + require.ErrorIs(t, err, errInterrupted) + require.NoError(t, prepareForkChoiceDirectory(forkChoicePath)) + + restartedFS := afero.NewBasePathFs(afero.NewOsFs(), forkChoicePath) + graph = fork_graph.NewForkGraphDisk(anchorState, nil, restartedFS, beacon_router_configuration.RouterConfiguration{}) + db := memdb.NewTestDB(t, dbcfg.ChainDB) + _, err = forkchoice.NewForkChoiceStore( + nil, + anchorState, + nil, + pool.NewOperationsPool(cfg), + graph, + beaconevents.NewEventEmitter(), + synced_data.NewSyncedDataManager(cfg, true), + nil, + public_keys_registry.NewInMemoryPublicKeysRegistry(), + validator_params.NewValidatorParams(), + false, + db, + ) + require.NoError(t, err) + require.NoError(t, db.View(t.Context(), func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, root) + require.NoError(t, err) + require.Equal(t, uint64(42), *blockNumber) + return nil + })) + }) + } +} + +func TestPrepareForkChoiceDirectoryFailsClosedOnArtifactConflict(t *testing.T) { + root := common.HexToHash("0x1234") + rootHex := fmt.Sprintf("%x", root) + filename := rootHex + ".envelope.snappy_ssz" + marker := rootHex + ".envelope.indices-pending" + baseDir := t.TempDir() + forkChoicePath := baseDir + "/caplin-forkchoice" + recoveryPath := forkChoicePath + "-envelope-recovery" + require.NoError(t, os.MkdirAll(forkChoicePath, 0o755)) + require.NoError(t, os.MkdirAll(recoveryPath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(forkChoicePath, filename), []byte("source"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(recoveryPath, filename), []byte("recovery"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(recoveryPath, marker), nil, 0o644)) + + err := prepareForkChoiceDirectory(forkChoicePath) + require.ErrorContains(t, err, "already exists") + source, readErr := os.ReadFile(filepath.Join(forkChoicePath, filename)) + require.NoError(t, readErr) + require.Equal(t, []byte("source"), source) +} + +func TestPrepareForkChoiceDirectoryResumesTemporaryEnvelopeMoves(t *testing.T) { + for _, phase := range []string{"preserve ", "restore "} { + t.Run(phase, func(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + anchorState := state.New(cfg) + forkChoicePath := t.TempDir() + "/caplin-forkchoice" + require.NoError(t, prepareForkChoiceDirectory(forkChoicePath)) + fs := afero.NewBasePathFs(afero.NewOsFs(), forkChoicePath) + graph := fork_graph.NewForkGraphDisk(anchorState, nil, fs, beacon_router_configuration.RouterConfiguration{}) + persistence := graph.(fork_graph.EnvelopePersistence) + root := common.HexToHash("0x9999") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + envelope.Message.BeaconBlockRoot = root + _, err := persistence.PrepareEnvelopeOnDisk(root, envelope, false) + require.NoError(t, err) + rootHex := fmt.Sprintf("%x", root) + temporaryFiles, err := filepath.Glob(filepath.Join(forkChoicePath, rootHex+".envelope.snappy_ssz.tmp-*")) + require.NoError(t, err) + require.Len(t, temporaryFiles, 1) + boundary := phase + filepath.Base(temporaryFiles[0]) + + errInterrupted := errors.New("interrupted") + err = prepareForkChoiceDirectoryWithHook(forkChoicePath, func(current string) error { + if current == boundary { + return errInterrupted + } + return nil + }) + require.ErrorIs(t, err, errInterrupted) + require.NoError(t, prepareForkChoiceDirectory(forkChoicePath)) + + restartedFS := afero.NewBasePathFs(afero.NewOsFs(), forkChoicePath) + graph = fork_graph.NewForkGraphDisk(anchorState, nil, restartedFS, beacon_router_configuration.RouterConfiguration{}) + persistence = graph.(fork_graph.EnvelopePersistence) + db := memdb.NewTestDB(t, dbcfg.ChainDB) + _, err = forkchoice.NewForkChoiceStore( + nil, + anchorState, + nil, + pool.NewOperationsPool(cfg), + graph, + beaconevents.NewEventEmitter(), + synced_data.NewSyncedDataManager(cfg, true), + nil, + public_keys_registry.NewInMemoryPublicKeysRegistry(), + validator_params.NewValidatorParams(), + false, + db, + ) + require.NoError(t, err) + pendingRoots, err := persistence.PendingEnvelopeIndexRoots() + require.NoError(t, err) + require.NotContains(t, pendingRoots, root) + require.NoError(t, db.View(t.Context(), func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, root) + require.NoError(t, err) + require.Nil(t, blockNumber) + return nil + })) + temporaryFiles, err = filepath.Glob(filepath.Join(forkChoicePath, rootHex+".envelope.snappy_ssz.tmp-*")) + require.NoError(t, err) + require.Empty(t, temporaryFiles) + }) + } +} + +func TestPrepareForkChoiceDirectoryPreservesPendingIndexRecovery(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + anchorState := state.New(cfg) + baseDir := t.TempDir() + forkChoicePath := baseDir + "/caplin-forkchoice" + require.NoError(t, prepareForkChoiceDirectory(forkChoicePath)) + fs := afero.NewBasePathFs(afero.NewOsFs(), forkChoicePath) + graph := fork_graph.NewForkGraphDisk(anchorState, nil, fs, beacon_router_configuration.RouterConfiguration{}) + persistence := graph.(fork_graph.EnvelopePersistence) + + pendingRoot := common.HexToHash("0x1234") + pendingEnvelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + pendingEnvelope.Message.BeaconBlockRoot = pendingRoot + pendingEnvelope.Message.Payload.BlockHash = common.HexToHash("0xabcd") + pendingEnvelope.Message.Payload.BlockNumber = 42 + require.NoError(t, graph.DumpEnvelopeOnDisk(pendingRoot, pendingEnvelope)) + + committedRoot := common.HexToHash("0x5678") + committedEnvelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + committedEnvelope.Message.BeaconBlockRoot = committedRoot + committedEnvelope.Message.Payload.BlockHash = common.HexToHash("0xcdef") + committedEnvelope.Message.Payload.BlockNumber = 43 + require.NoError(t, graph.DumpEnvelopeOnDisk(committedRoot, committedEnvelope)) + + orphanRoot := common.HexToHash("0x9999") + orphanEnvelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + orphanEnvelope.Message.BeaconBlockRoot = orphanRoot + _, err := persistence.PrepareEnvelopeOnDisk(orphanRoot, orphanEnvelope, false) + require.NoError(t, err) + + db := memdb.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, committedRoot, committedEnvelope.Message) + })) + require.NoError(t, prepareForkChoiceDirectory(forkChoicePath)) + restartedFS := afero.NewBasePathFs(afero.NewOsFs(), forkChoicePath) + graph = fork_graph.NewForkGraphDisk(anchorState, nil, restartedFS, beacon_router_configuration.RouterConfiguration{}) + persistence = graph.(fork_graph.EnvelopePersistence) + + _, err = forkchoice.NewForkChoiceStore( + nil, + anchorState, + nil, + pool.NewOperationsPool(cfg), + graph, + beaconevents.NewEventEmitter(), + synced_data.NewSyncedDataManager(cfg, true), + nil, + public_keys_registry.NewInMemoryPublicKeysRegistry(), + validator_params.NewValidatorParams(), + false, + db, + ) + require.NoError(t, err) + + require.NoError(t, db.View(t.Context(), func(tx kv.Tx) error { + pendingNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, pendingRoot) + require.NoError(t, err) + require.Equal(t, uint64(42), *pendingNumber) + committedNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, committedRoot) + require.NoError(t, err) + require.Equal(t, uint64(43), *committedNumber) + orphanNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, orphanRoot) + require.NoError(t, err) + require.Nil(t, orphanNumber) + return nil + })) + pendingRoots, err := persistence.PendingEnvelopeIndexRoots() + require.NoError(t, err) + require.Empty(t, pendingRoots) +}