diff --git a/cl/das/mock_services/peer_das_mock.go b/cl/das/mock_services/peer_das_mock.go index 090c5007d76..ab065a443d9 100644 --- a/cl/das/mock_services/peer_das_mock.go +++ b/cl/das/mock_services/peer_das_mock.go @@ -347,6 +347,42 @@ func (c *MockPeerDasSetForkChoiceCall) DoAndReturn(f func(das.BlockGetter)) *Moc return c } +// Start mocks base method. +func (m *MockPeerDas) Start(ctx context.Context) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start", ctx) +} + +// Start indicates an expected call of Start. +func (mr *MockPeerDasMockRecorder) Start(ctx any) *MockPeerDasStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockPeerDas)(nil).Start), ctx) + return &MockPeerDasStartCall{Call: call} +} + +// MockPeerDasStartCall wrap *gomock.Call +type MockPeerDasStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockPeerDasStartCall) Return() *MockPeerDasStartCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockPeerDasStartCall) Do(f func(context.Context)) *MockPeerDasStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockPeerDasStartCall) DoAndReturn(f func(context.Context)) *MockPeerDasStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // StateReader mocks base method. func (m *MockPeerDas) StateReader() peerdasstate.PeerDasStateReader { m.ctrl.T.Helper() diff --git a/cl/das/peer_das.go b/cl/das/peer_das.go index 6e625e43f2e..2575200c57d 100644 --- a/cl/das/peer_das.go +++ b/cl/das/peer_das.go @@ -48,6 +48,7 @@ type gloasBlockData struct { //go:generate mockgen -typed=true -destination=mock_services/peer_das_mock.go -package=mock_services . PeerDas type PeerDas interface { + Start(ctx context.Context) // [Modified in Gloas:EIP7732] Changed from []*SignedBlindedBeaconBlock to []ColumnSyncableSignedBlock // to support both pre-GLOAS (blinded) and GLOAS (non-blinded) blocks DownloadColumnsAndRecoverBlobs(ctx context.Context, blocks []cltypes.ColumnSyncableSignedBlock) error @@ -88,10 +89,10 @@ type peerdas struct { blockReader freezeblocks.BeaconSnapshotReader indiciesDB kv.RoDB gloasDataCache *lru.Cache[common.Hash, *gloasBlockData] // cache for GLOAS block data (~1KB per entry) + startOnce sync.Once } func NewPeerDas( - ctx context.Context, rpc *rpc.BeaconRpcP2P, beaconConfig *clparams.BeaconChainConfig, caplinConfig *clparams.CaplinConfig, @@ -128,14 +129,19 @@ func NewPeerDas( indiciesDB: indiciesDB, gloasDataCache: gloasDataCache, } - p.resubscribeGossip() - for range numOfBlobRecoveryWorkers { - go p.blobsRecoverWorker(ctx) - } - go p.syncColumnDataWorker(ctx) return p } +func (d *peerdas) Start(ctx context.Context) { + d.startOnce.Do(func() { + d.resubscribeGossip() + for range numOfBlobRecoveryWorkers { + go d.blobsRecoverWorker(ctx) + } + go d.syncColumnDataWorker(ctx) + }) +} + func (d *peerdas) StateReader() peerdasstate.PeerDasStateReader { return d.state } diff --git a/cl/das/peer_das_start_test.go b/cl/das/peer_das_start_test.go new file mode 100644 index 00000000000..3f13477ef45 --- /dev/null +++ b/cl/das/peer_das_start_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package das + +import ( + "context" + "testing" + + "go.uber.org/mock/gomock" + + "github.com/erigontech/erigon/cl/clparams" + peerdasstate "github.com/erigontech/erigon/cl/das/state" + gossipmock "github.com/erigontech/erigon/cl/phase1/network/gossip/mock_services" +) + +func TestPeerDasSubscribesOnlyAfterStart(t *testing.T) { + ctrl := gomock.NewController(t) + gossipManager := gossipmock.NewMockGossip(ctrl) + beaconConfig := clparams.MainnetBeaconConfig + beaconConfig.DataColumnSidecarSubnetCount = 2 + caplinConfig := clparams.CaplinConfig{ArchiveBlobs: true} + peerDasState := peerdasstate.NewPeerDasState(&beaconConfig, &clparams.NetworkConfig{}) + + peerDas := NewPeerDas(nil, &beaconConfig, &caplinConfig, nil, nil, nil, [32]byte{}, nil, peerDasState, gossipManager, nil, nil) + + gossipManager.EXPECT().SubscribeWithExpiry(gomock.Any(), gomock.Any()).Times(2) + ctx, cancel := context.WithCancel(context.Background()) + peerDas.Start(ctx) + peerDas.Start(ctx) + cancel() +} diff --git a/cl/phase1/core/checkpoint_sync/finalized_state_root.go b/cl/phase1/core/checkpoint_sync/finalized_state_root.go new file mode 100644 index 00000000000..0c8075c9291 --- /dev/null +++ b/cl/phase1/core/checkpoint_sync/finalized_state_root.go @@ -0,0 +1,83 @@ +package checkpoint_sync + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path/filepath" + + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/dir" + "github.com/spf13/afero" +) + +const finalizedStateRootPrefix = ".finalized-state-root-" + +var ErrFinalizedGloasStateRootMissing = errors.New("finalized Gloas state is missing its authoritative state root") + +func FinalizedStateRootFileName(snappyState []byte) string { + digest := sha256.Sum256(snappyState) + return finalizedStateRootPrefix + hex.EncodeToString(digest[:]) +} + +func RemoveObsoleteFinalizedStateRoots(directory, keepPath string) error { + rootFiles, err := filepath.Glob(filepath.Join(directory, finalizedStateRootPrefix+"*")) + if err != nil { + return err + } + for _, rootPath := range rootFiles { + if rootPath == keepPath { + continue + } + if err := dir.RemoveFile(rootPath); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + return nil +} + +func RestoreFinalizedStateRoot(storage afero.Fs, snappyState []byte, st *state.CachingBeaconState) error { + if st.Version() < clparams.GloasVersion { + return nil + } + record, err := afero.ReadFile(storage, FinalizedStateRootFileName(snappyState)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + header := st.LatestBlockHeader() + if st.Version() >= clparams.GloasVersion && header.Slot == st.Slot() && header.Root == (common.Hash{}) { + return ErrFinalizedGloasStateRootMissing + } + return nil + } + return fmt.Errorf("read finalized state root: %w", err) + } + if len(record) != 2*len(common.Hash{}) { + return fmt.Errorf("invalid finalized state root record length %d", len(record)) + } + digest := sha256.Sum256(snappyState) + checksumInput := make([]byte, 0, len(digest)+len(common.Hash{})) + checksumInput = append(checksumInput, digest[:]...) + checksumInput = append(checksumInput, record[:len(common.Hash{})]...) + wantChecksum := sha256.Sum256(checksumInput) + if !bytes.Equal(wantChecksum[:], record[len(common.Hash{}):]) { + return errors.New("invalid finalized state root checksum") + } + st.SetPreviousStateRoot(common.BytesToHash(record[:len(common.Hash{})])) + return nil +} + +func EncodeFinalizedStateRoot(snappyState []byte, root common.Hash) []byte { + digest := sha256.Sum256(snappyState) + checksumInput := make([]byte, 0, len(digest)+len(root)) + checksumInput = append(checksumInput, digest[:]...) + checksumInput = append(checksumInput, root[:]...) + checksum := sha256.Sum256(checksumInput) + record := make([]byte, 0, 2*len(root)) + record = append(record, root[:]...) + return append(record, checksum[:]...) +} diff --git a/cl/phase1/core/checkpoint_sync/local_checkpoint_syncer.go b/cl/phase1/core/checkpoint_sync/local_checkpoint_syncer.go index afdaaaff24d..57be3824c30 100644 --- a/cl/phase1/core/checkpoint_sync/local_checkpoint_syncer.go +++ b/cl/phase1/core/checkpoint_sync/local_checkpoint_syncer.go @@ -2,6 +2,7 @@ package checkpoint_sync import ( "context" + "errors" "fmt" "github.com/erigontech/erigon/cl/clparams" @@ -45,6 +46,13 @@ func (l *LocalCheckpointSyncer) GetLatestBeaconState(ctx context.Context) (*stat if err := bs.DecodeSSZ(decompressedSnappy, int(beaconCfg.GetCurrentStateVersion(slot/beaconCfg.SlotsPerEpoch))); err != nil { return nil, fmt.Errorf("could not deserialize state: %w", err) } + if err := RestoreFinalizedStateRoot(l.dir, snappyEncoded, bs); err != nil { + if errors.Is(err, ErrFinalizedGloasStateRootMissing) { + log.Warn("Local finalized Gloas state predates state-root persistence, starting sync from genesis.") + return l.genesisState.Copy() + } + return nil, err + } // Same-network gate as the remote-sync resume paths: a file left by another chain must never // anchor the node. Staleness is not gated here — there is no remote to fall back to, and a stale // same-network finalized anchor beats replaying from genesis. diff --git a/cl/phase1/core/checkpoint_sync/util.go b/cl/phase1/core/checkpoint_sync/util.go index be75cf9296e..fb418f56cde 100644 --- a/cl/phase1/core/checkpoint_sync/util.go +++ b/cl/phase1/core/checkpoint_sync/util.go @@ -147,6 +147,11 @@ func readLocalStateFile(dirs datadir.Dirs, beaconCfg *clparams.BeaconChainConfig if err := bs.DecodeSSZ(decompressed, int(beaconCfg.GetCurrentStateVersion(epoch))); err != nil { return nil, fmt.Errorf("could not decode local %s state: %w", kind, err) } + if fileName == clparams.LatestFinalizedStateFileName { + if err := RestoreFinalizedStateRoot(afero.NewBasePathFs(afero.NewOsFs(), dirs.CaplinLatest), snappyEncoded, bs); err != nil { + return nil, err + } + } return bs, nil } diff --git a/cl/phase1/execution_client/execution_client_engine_test.go b/cl/phase1/execution_client/execution_client_engine_test.go index 2af65f5e8b6..2de71c07375 100644 --- a/cl/phase1/execution_client/execution_client_engine_test.go +++ b/cl/phase1/execution_client/execution_client_engine_test.go @@ -114,6 +114,37 @@ func TestExecutionPayloadFromSSZBlock_BlockAccessListGloasOnly(t *testing.T) { } } +func TestExecutionPayloadFromSSZBlock_TransactionsAreJSONArray(t *testing.T) { + beaconCfg := clparams.MainnetBeaconConfig + tests := []struct { + name string + version clparams.StateVersion + json string + want []any + }{ + {name: "pre-Gloas empty", version: clparams.ElectraVersion, json: "[]", want: []any{}}, + {name: "pre-Gloas non-empty", version: clparams.ElectraVersion, json: `["0x0102"]`, want: []any{"0x0102"}}, + {name: "Gloas empty", version: clparams.GloasVersion, json: "[]", want: []any{}}, + {name: "Gloas non-empty", version: clparams.GloasVersion, json: `["0x0102"]`, want: []any{"0x0102"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload := cltypes.NewEth1Block(tt.version, &beaconCfg) + payload.Extra = solid.NewExtraData() + payload.Transactions = &solid.TransactionsSSZ{} + payload.Withdrawals = solid.NewStaticListSSZ[*cltypes.Withdrawal](int(beaconCfg.MaxWithdrawalsPerPayload), 44) + require.NoError(t, payload.Transactions.UnmarshalJSON([]byte(tt.json))) + + raw, err := json.Marshal(engine_types.ExecutionPayloadFromSSZBlock(payload, tt.version)) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, tt.want, decoded["transactions"]) + }) + } +} + func gloas(cfg *clparams.BeaconChainConfig, balData []byte) *cltypes.Eth1Block { block := cltypes.NewEth1Block(clparams.GloasVersion, cfg) block.Extra = solid.NewExtraData() diff --git a/cl/phase1/forkchoice/CLAUDE.md b/cl/phase1/forkchoice/CLAUDE.md index be75064ec31..6e9c2c107c9 100644 --- a/cl/phase1/forkchoice/CLAUDE.md +++ b/cl/phase1/forkchoice/CLAUDE.md @@ -40,7 +40,7 @@ verified against the Go code in this repository. | `on_block.go`: `verifyKzgCommitmentsAgainstTransactions` | Deneb execution payload blob versioned-hash checks; Electra/Fulu maximum blob count plumbing | | `on_block.go`: `isDataAvailable` | Deneb `is_data_available` for blob sidecars; pre-Gloas local blob storage path | | `on_block.go`: PeerDAS `IsDataAvailable` and `SyncColumnDataLater` branch inside `OnBlock` | Fulu modified `is_data_available`: data availability is checked by block root through data column sidecars, without passing `blob_kzg_commitments`; modified Fulu `on_block` calls it as `is_data_available(hash_tree_root(block))` | -| `on_execution_payload.go`: `OnExecutionPayload`, `applyEnvelope`, `applyEnvelopeLocked`, `ApplyLocalSelfBuildEnvelope`, `StoreAnchorEnvelope` | Gloas `on_execution_payload_envelope`, `Store.payloads`, `Store.payload_timeliness_vote`, `Store.payload_data_availability_vote` | +| `on_execution_payload.go`: `OnExecutionPayload`, `applyEnvelope`, `applyEnvelopeCoordinated`, `ApplyLocalSelfBuildEnvelope`, `StoreAnchorEnvelope` | Gloas `on_execution_payload_envelope`, `Store.payloads`, `Store.payload_timeliness_vote`, `Store.payload_data_availability_vote` | | `on_execution_payload.go`: `validateEnvelopeAgainstBlock`, `verifyEnvelopeBuilderSignature`, `checkDataAvailability`, `validatePayloadWithEL` | Gloas `on_execution_payload_envelope`; Gloas/Fulu data availability for committed bid blob data; Bellatrix `ExecutionEngine.notify_forkchoice_updated`/payload validation context | | `on_payload_attestation_message.go`: `OnPayloadAttestationMessage` | Gloas `on_payload_attestation_message`, PTC membership/signature/current-slot checks | | `on_attestation.go`: `OnAttestation`, `ProcessAttestingIndicies`, `ValidateOnAttestation`, `validateTargetEpochAgainstCurrentTime` | Phase0 `on_attestation`, `validate_on_attestation`, `validate_target_epoch_against_current_time`; Gloas modified `validate_on_attestation` | diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index 065b0e5f944..a017397829a 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -150,30 +150,21 @@ func NewForkGraphDisk(anchorState *state.CachingBeaconState, syncedData synced_d } anchorHeader := anchorState.LatestBlockHeader() if anchorState.Version() >= clparams.GloasVersion && anchorState.Slot() > 0 { - // GLOAS checkpoint/anchor sync fix: the first transitionSlot for this - // anchor needs to record the correct state root (computed with - // LatestBlockHeader.Root == zero) into stateRoots. Two cases arise: - // - // Fresh checkpoint sync: Root is zero per spec (process_block_header - // zeroes it). We compute HashSSZ with Root=0 (the correct value), - // fill in Root, and cache it as PreviousStateRoot. - // - // Restart from disk: a previous run already filled in Root and - // serialized the state. Root is now that same correct hash (the one - // originally computed with Root=0). HashSSZ would return a different - // (wrong) value because Root is non-zero, so we must NOT recompute; - // instead we use the stored Root directly as PreviousStateRoot. - if anchorHeader.Root == [32]byte{} { - stateHash, err := anchorState.HashSSZ() - if err != nil { - panic(err) + stateHash := anchorState.PeekPreviousStateRoot() + if stateHash == (common.Hash{}) { + if anchorHeader.Slot == anchorState.Slot() && anchorHeader.Root != (common.Hash{}) { + stateHash = anchorHeader.Root + } else { + stateHash, err = anchorState.HashSSZ() + if err != nil { + panic(err) + } } + } + if anchorHeader.Root == (common.Hash{}) { anchorHeader.Root = stateHash - anchorState.SetLatestBlockHeader(&anchorHeader) - anchorState.SetPreviousStateRoot(stateHash) - } else { - anchorState.SetPreviousStateRoot(anchorHeader.Root) } + anchorState.SetPreviousStateRoot(stateHash) } else { if anchorHeader.Root, err = anchorState.HashSSZ(); err != nil { panic(err) diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go index f8b728db142..6eb756f07c1 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go @@ -156,13 +156,11 @@ func (f *forkGraphDisk) DumpBeaconStateOnDisk(blockRoot common.Hash, bs *state.C log.Error("failed to write ssz buffer", "err", err) return err } - // Write the authoritative state root so it can be restored on load. - // Use the stored block header's Root (set from block.StateRoot in AddChainSegment) - // rather than the state's PreviousStateRoot cache field, which can be stale if - // a concurrent block arrival modified f.currentState between GetStateAtBlockRoot - // and the copy in OnHeadStateWithBlockRoot. + // A skipped-slot state root differs from the latest block header's state root. var stateRootToWrite common.Hash - if hdr, ok := f.GetHeader(blockRoot); ok { + if bs.Version() >= clparams.GloasVersion && bs.LatestBlockHeader().Slot < bs.Slot() { + stateRootToWrite = bs.PeekPreviousStateRoot() + } else if hdr, ok := f.GetHeader(blockRoot); ok { stateRootToWrite = hdr.Root } else { // Fallback for anchor state or cases where header isn't stored yet diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go index 55cbbd11d83..35b2f98125d 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_test.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_test.go @@ -68,6 +68,47 @@ func TestForkGraphInDisk(t *testing.T) { require.Equal(t, PreValidated, status) } +func TestNewForkGraphDiskCachesAnchorStateRoot(t *testing.T) { + for _, tc := range []struct { + name string + stateSlot uint64 + headerSlot uint64 + headerRoot common.Hash + cachedRoot common.Hash + }{ + {name: "skipped slot", stateSlot: 64, headerSlot: 63, headerRoot: common.Hash{1}}, + {name: "block slot", stateSlot: 64, headerSlot: 64}, + {name: "restored block slot", stateSlot: 64, headerSlot: 64, cachedRoot: common.Hash{2}}, + {name: "legacy block slot", stateSlot: 64, headerSlot: 64, headerRoot: common.Hash{1}}, + } { + t.Run(tc.name, func(t *testing.T) { + anchorState := state.New(&clparams.MainnetBeaconConfig) + anchorState.SetVersion(clparams.GloasVersion) + anchorState.SetSlot(tc.stateSlot) + header := &cltypes.BeaconBlockHeader{Slot: tc.headerSlot, Root: tc.headerRoot} + anchorState.SetLatestBlockHeader(header) + expectedStateRoot, err := anchorState.HashSSZ() + require.NoError(t, err) + if tc.cachedRoot != (common.Hash{}) { + expectedStateRoot = tc.cachedRoot + anchorState.SetPreviousStateRoot(tc.cachedRoot) + } else if tc.headerSlot == tc.stateSlot && tc.headerRoot != (common.Hash{}) { + expectedStateRoot = tc.headerRoot + } + anchorRoot, err := anchorState.BlockRoot() + require.NoError(t, err) + + graph := NewForkGraphDisk(anchorState, nil, afero.NewMemMapFs(), beacon_router_configuration.RouterConfiguration{}).(*forkGraphDisk) + + require.Equal(t, common.Hash(expectedStateRoot), anchorState.PeekPreviousStateRoot()) + require.Equal(t, header.Root, anchorState.LatestBlockHeader().Root) + persistedState, err := graph.readBeaconStateFromDisk(anchorRoot) + require.NoError(t, err) + require.Equal(t, common.Hash(expectedStateRoot), persistedState.PeekPreviousStateRoot()) + }) + } +} + // A prune for an already-covered slot (e.g. from a concurrent lock-free drain) // must not move the lowest-available marker backward past deleted data. func TestPruneKeepsLowestAvailableBlockMonotonic(t *testing.T) { diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index 2e3f8cbdce8..b8e16d967e1 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -173,9 +173,12 @@ type ForkChoiceStore struct { probabilisticHeadGetter bool // [New in Gloas:EIP7732] - ptcVoteMu sync.Mutex // protects read-modify-write on payloadTimelinessVote and payloadDataAvailabilityVote + ptcVoteMu sync.Mutex // protects payload vote updates and first-valid gossip tracking payloadTimelinessVote sync.Map // map[common.Hash][clparams.PtcSize]int8 (0=unvoted, 1=true, -1=false) payloadDataAvailabilityVote sync.Map // map[common.Hash][clparams.PtcSize]int8 (0=unvoted, 1=true, -1=false) + payloadAttestationSeenSlot uint64 + payloadAttestationSeen map[uint64]struct{} + payloadAttestationContexts *payloadAttestationValidationContexts // [New in Gloas:EIP7732] Block timeliness tracking. // Pre-GLOAS: stores [block_timely, false] (only index 0 is meaningful). // Post-GLOAS: stores [block_timely, payload_timely] — two independent booleans. @@ -199,8 +202,11 @@ type ForkChoiceStore struct { // whose EL newPayload failed (e.g. because EL hasn't caught up after forward sync). // The stages layer drains these into blockCollector before each Flush() so EL // eventually receives the blocks. - pendingELPayloadsMu sync.Mutex - pendingELPayloads []PendingELPayload + pendingELPayloadsMu sync.Mutex + pendingELPayloads []PendingELPayload + payloadValidationOnce sync.Once + payloadValidationAdmission chan struct{} + envelopeIndexWrites sync.Map // db is used to persist execution payload indices (block number/hash) when an envelope // is accepted in OnExecutionPayload. May be nil (e.g. in tests), in which case the @@ -208,6 +214,12 @@ type ForkChoiceStore struct { db kv.RwDB } +type envelopeIndexWrite struct { + done chan struct{} + err error + envelope *cltypes.SignedExecutionPayloadEnvelope +} + // PendingELPayload holds a block+envelope pair that needs to be fed to the EL. type PendingELPayload struct { Block *cltypes.SignedBeaconBlock @@ -335,6 +347,10 @@ func NewForkChoiceStore( if err != nil { return nil, err } + payloadAttestationContexts, err := newPayloadAttestationValidationContexts() + if err != nil { + return nil, err + } publicKeysRegistry.ResetAnchor(anchorState) participation.Add(state.Epoch(anchorState.BeaconState), anchorState.CurrentEpochParticipation().Copy()) @@ -392,6 +408,7 @@ func NewForkChoiceStore( executionPayloadStatus: executionPayloadStatus, payloadStatusByRoot: payloadStatusByRoot, executionPayloadGasLimit: executionPayloadGasLimit, + payloadAttestationContexts: payloadAttestationContexts, db: db, } f.justifiedCheckpoint.Store(anchorCheckpoint) @@ -1083,11 +1100,26 @@ func (f *ForkChoiceStore) RequeuePendingELPayload(p PendingELPayload) { // DrainPendingELPayloads returns and clears all queued EL payloads. // The stages layer calls this before Flush() to retry them with engine.NewPayload. func (f *ForkChoiceStore) DrainPendingELPayloads() []PendingELPayload { + return f.DrainPendingELPayloadsLimit(maxPendingELPayloads) +} + +func (f *ForkChoiceStore) DrainPendingELPayloadsLimit(limit int) []PendingELPayload { + if limit <= 0 { + return nil + } f.pendingELPayloadsMu.Lock() defer f.pendingELPayloadsMu.Unlock() if len(f.pendingELPayloads) == 0 { return nil } + if len(f.pendingELPayloads) > limit { + result := make([]PendingELPayload, limit) + copy(result, f.pendingELPayloads[:limit]) + copy(f.pendingELPayloads, f.pendingELPayloads[limit:]) + clear(f.pendingELPayloads[len(f.pendingELPayloads)-limit:]) + f.pendingELPayloads = f.pendingELPayloads[:len(f.pendingELPayloads)-limit] + return result + } if cap(f.pendingELPayloads) > pendingELPayloadsShrinkCap { result := f.pendingELPayloads f.pendingELPayloads = nil diff --git a/cl/phase1/forkchoice/interface.go b/cl/phase1/forkchoice/interface.go index e94aa336463..9211048f700 100644 --- a/cl/phase1/forkchoice/interface.go +++ b/cl/phase1/forkchoice/interface.go @@ -143,7 +143,7 @@ type ForkChoiceStorageWriter interface { ApplyLocalSelfBuildEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) error // [New in Gloas:EIP7732] OnPayloadAttestationMessage processes a PTC attestation message from gossip. // Returns error if validation fails (REJECT), nil if accepted or ignored. - OnPayloadAttestationMessage(msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error + OnPayloadAttestationMessage(ctx context.Context, msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error // [New in Gloas:EIP7732] StoreAnchorEnvelope persists an envelope to disk and updates // eth2Roots without running state transition. Used during checkpoint sync where the // finalized state already incorporates the envelope's effects but subsequent blocks diff --git a/cl/phase1/forkchoice/mock_services/forkchoice_mock.go b/cl/phase1/forkchoice/mock_services/forkchoice_mock.go index 5c283b4a6a8..50b89d65d55 100644 --- a/cl/phase1/forkchoice/mock_services/forkchoice_mock.go +++ b/cl/phase1/forkchoice/mock_services/forkchoice_mock.go @@ -363,7 +363,7 @@ func (f *ForkChoiceStorageMock) StoreAnchorEnvelope(blockRoot common.Hash, signe return nil } -func (f *ForkChoiceStorageMock) OnPayloadAttestationMessage(msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error { +func (f *ForkChoiceStorageMock) OnPayloadAttestationMessage(ctx context.Context, msg *cltypes.PayloadAttestationMessage, isFromBlock bool) error { return nil } diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go index d32b9904b08..daeb1c9c341 100644 --- a/cl/phase1/forkchoice/on_block.go +++ b/cl/phase1/forkchoice/on_block.go @@ -29,7 +29,6 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/monitor" - "github.com/erigontech/erigon/cl/persistence/beacon_indicies" "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" @@ -39,7 +38,6 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/execution/protocol/misc" "github.com/erigontech/erigon/execution/types" ) @@ -294,6 +292,9 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac } switch status { case fork_graph.PreValidated: + unlocked = true + f.mu.Unlock() + f.processPendingEnvelopeAfterBlock(ctx, common.Hash(blockRoot), checkDataAvaiability) return nil case fork_graph.Success: f.updateChildren(block.Block.Slot-1, block.Block.ParentRoot, blockRoot) // parent slot can be innacurate @@ -331,7 +332,9 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac f.updateProposerBoostRoot(headBeforeBlock, common.Hash(blockRoot)) // [New in Gloas:EIP7732] GLOAS-specific on_block logic (post state transition) - var appliedEnvelope *cltypes.ExecutionPayloadEnvelope + var pendingEnvelope *cltypes.SignedExecutionPayloadEnvelope + var pendingEnvelopeLocal bool + var pendingEnvelopeFound bool if blockVersion >= clparams.GloasVersion { // Initialize payload timeliness and data availability votes for this block f.payloadTimelinessVote.Store(common.Hash(blockRoot), [clparams.PtcSize]int8{}) @@ -344,38 +347,14 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac f.notifyPtcMessages(lastProcessedState, block.Block.Body.PayloadAttestations) } - // [New in Gloas:EIP7732] Check if there's a pending envelope waiting for this block. - // This handles the case where envelope arrives before the block via gossip. - // IMPORTANT: must run BEFORE ProcessJustificationBitsAndFinality below, which - // temporarily mutates the state for unrealized justification. The envelope's - // ProcessExecutionPayloadEnvelope spec check requires state.HashSSZ() to match - // the block's state_root, and the mutation+restoration cycle can cause the - // incremental hash cache to diverge. - // Check for pending envelopes: first the local self-build queue (skip BLS), - // then the general gossip queue (full BLS verification). These are separate - // queues so that origin is determined by which queue wrote the entry, not by - // inspecting envelope contents (which an attacker could forge). + // Preserve envelope origin so the post-commit path applies the correct BLS policy. if pending, ok := f.pendingLocalSelfBuildEnvelopes.Get(common.Hash(blockRoot)); ok { - f.pendingLocalSelfBuildEnvelopes.Remove(common.Hash(blockRoot)) - log.Trace("OnBlock: processing pending local self-build envelope", "blockRoot", common.Hash(blockRoot)) - applied, applyErr := f.applyLocalSelfBuildEnvelopeLocked(ctx, pending) - if applyErr != nil { - log.Warn("OnBlock: failed to process pending local self-build envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) - } else if applied { - appliedEnvelope = pending.Message - } + pendingEnvelope = pending + pendingEnvelopeLocal = true + pendingEnvelopeFound = true } else if pending, ok := f.pendingEnvelopes.Get(common.Hash(blockRoot)); ok { - f.pendingEnvelopes.Remove(common.Hash(blockRoot)) - log.Trace("OnBlock: processing pending envelope", "blockRoot", common.Hash(blockRoot)) - // Always validate payload with EL for pending envelopes, regardless of the caller's newPayload flag. - // During forward sync newPayload is false, but the envelope still needs to reach the EL; - // otherwise the EL never learns about this block and the chain stalls. - applied, applyErr := f.applyEnvelopeLocked(ctx, pending, checkDataAvaiability, true) - if applyErr != nil { - log.Warn("OnBlock: failed to process pending envelope", "blockRoot", common.Hash(blockRoot), "err", applyErr) - } else if applied { - appliedEnvelope = pending.Message - } + pendingEnvelope = pending + pendingEnvelopeFound = true } } if lastProcessedState.Slot()%f.beaconCfg.SlotsPerEpoch == 0 { @@ -452,13 +431,6 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac if blockEpoch < currentEpoch { f.updateCheckpoints(postPullupJustified, postPullupFinalized) } - blockData := &beaconevents.BlockData{ - Slot: block.Block.Slot, - Block: blockRoot, - ExecutionOptimistic: f.optimisticStore.IsOptimistic(blockRoot), - } - f.queueEmit(func() { f.emitters.State().SendBlock(blockData) }) - if !isVerifiedExecutionPayload { log.Debug("OnBlock", "elapsed", time.Since(start), "slot", block.Block.Slot) } @@ -474,22 +446,196 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac // Release lock (via defer) before writing DB indices for the applied envelope. unlocked = true f.mu.Unlock() + + var appliedEnvelope *cltypes.ExecutionPayloadEnvelope + if pendingEnvelopeFound { + appliedEnvelope = f.applyPendingEnvelope(ctx, common.Hash(blockRoot), pendingEnvelope, pendingEnvelopeLocal, checkDataAvaiability) + } + f.mu.Lock() + blockData := &beaconevents.BlockData{ + Slot: block.Block.Slot, + Block: blockRoot, + ExecutionOptimistic: f.optimisticStore.IsOptimistic(blockRoot), + } + f.queueEmit(func() { f.emitters.State().SendBlock(blockData) }) + f.mu.Unlock() f.drainQueuedWork() // Write execution payload envelope indices outside f.mu to avoid deadlock // with postForkchoiceOperations (which holds MDBX tx then needs f.mu.RLock). - if appliedEnvelope != nil && f.db != nil { - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, common.Hash(blockRoot), appliedEnvelope) - }); err != nil { - log.Warn("OnBlock: failed to write execution payload indices for pending envelope", - "blockRoot", common.Hash(blockRoot), "err", err) - } + if appliedEnvelope != nil { + f.writePendingEnvelopeIndices(ctx, common.Hash(blockRoot), pendingEnvelope, appliedEnvelope, pendingEnvelopeLocal) } return nil } +func (f *ForkChoiceStore) processPendingEnvelopeAfterBlock(ctx context.Context, blockRoot common.Hash, checkDataAvailability bool) { + var pending *cltypes.SignedExecutionPayloadEnvelope + local := false + found := false + if candidate, ok := f.pendingLocalSelfBuildEnvelopes.Peek(blockRoot); ok { + pending = candidate + local = true + found = true + } else if candidate, ok := f.pendingEnvelopes.Peek(blockRoot); ok { + pending = candidate + found = true + } + if !found { + return + } + appliedEnvelope := f.applyPendingEnvelope(ctx, blockRoot, pending, local, checkDataAvailability) + if appliedEnvelope != nil { + f.writePendingEnvelopeIndices(ctx, blockRoot, pending, appliedEnvelope, local) + } +} + +func (f *ForkChoiceStore) writePendingEnvelopeIndices(ctx context.Context, blockRoot common.Hash, pending *cltypes.SignedExecutionPayloadEnvelope, appliedEnvelope *cltypes.ExecutionPayloadEnvelope, local bool) { + if f.db == nil { + return + } + if pending == nil || pending.Message != appliedEnvelope { + pending = &cltypes.SignedExecutionPayloadEnvelope{Message: appliedEnvelope} + } + indexEnvelope, err := f.ensureExecutionPayloadEnvelopeIndices(ctx, blockRoot, pending, true) + if err == nil { + return + } + if local { + f.pendingLocalSelfBuildEnvelopes.Add(blockRoot, indexEnvelope) + } else { + f.pendingEnvelopes.Add(blockRoot, indexEnvelope) + } + log.Warn("OnBlock: failed to write execution payload indices for pending envelope", "blockRoot", blockRoot, "err", err) +} + +func (f *ForkChoiceStore) RetryPendingExecutionPayloadEnvelopes(ctx context.Context, limit int) { + if limit <= 0 || f.pendingLocalSelfBuildEnvelopes == nil || f.pendingEnvelopes == nil { + return + } + localRoots := f.pendingLocalSelfBuildEnvelopes.Keys() + gossipRoots := f.pendingEnvelopes.Keys() + seen := make(map[common.Hash]struct{}, len(localRoots)+len(gossipRoots)) + for i := 0; limit > 0 && (i < len(localRoots) || i < len(gossipRoots)); i++ { + for _, roots := range [][]common.Hash{localRoots, gossipRoots} { + if i >= len(roots) { + continue + } + root := roots[i] + if _, ok := seen[root]; ok { + continue + } + seen[root] = struct{}{} + f.processPendingEnvelopeAfterBlock(ctx, root, true) + if f.pendingLocalSelfBuildEnvelopes.Contains(root) { + f.pendingLocalSelfBuildEnvelopes.Get(root) + } + if f.pendingEnvelopes.Contains(root) { + f.pendingEnvelopes.Get(root) + } + limit-- + if limit == 0 || ctx.Err() != nil { + return + } + } + } +} + +func (f *ForkChoiceStore) applyPendingEnvelope(ctx context.Context, blockRoot common.Hash, pending *cltypes.SignedExecutionPayloadEnvelope, local, checkDataAvailability bool) *cltypes.ExecutionPayloadEnvelope { + if pending == nil { + if !f.forkGraph.HasEnvelope(blockRoot) { + return nil + } + persisted, err := f.forkGraph.ReadEnvelopeFromDisk(blockRoot) + if err != nil || persisted == nil || persisted.Message == nil { + return nil + } + if local { + if current, ok := f.pendingLocalSelfBuildEnvelopes.Peek(blockRoot); ok && current == nil { + f.pendingLocalSelfBuildEnvelopes.Remove(blockRoot) + } + } else if current, ok := f.pendingEnvelopes.Peek(blockRoot); ok && current == nil { + f.pendingEnvelopes.Remove(blockRoot) + } + return persisted.Message + } + var applied bool + var err error + if local { + applied, err = f.applyLocalSelfBuildEnvelope(ctx, pending, retryQueuedEnvelope) + } else { + applied, err = f.applyEnvelope(ctx, pending, checkDataAvailability, true, retryQueuedEnvelope) + } + if err != nil { + log.Warn("OnBlock: failed to process pending envelope", "blockRoot", blockRoot, "local", local, "err", err) + if !f.retryPendingEnvelopeError(err, pending) { + if local { + if current, ok := f.pendingLocalSelfBuildEnvelopes.Peek(blockRoot); ok && current == pending { + f.pendingLocalSelfBuildEnvelopes.Remove(blockRoot) + } + } else if current, ok := f.pendingEnvelopes.Peek(blockRoot); ok && current == pending { + f.pendingEnvelopes.Remove(blockRoot) + } + } + return nil + } + completedByAnother := !applied && f.forkGraph.HasEnvelope(blockRoot) + if !applied && !completedByAnother { + return nil + } + completedSameEnvelope := applied + if completedByAnother { + persisted, err := f.forkGraph.ReadEnvelopeFromDisk(blockRoot) + if err != nil { + return nil + } + if persisted != nil && persisted.Message != nil { + persistedRoot, persistedErr := persisted.Message.HashSSZ() + pendingRoot, pendingErr := pending.Message.HashSSZ() + completedSameEnvelope = persistedErr == nil && pendingErr == nil && persistedRoot == pendingRoot + } + } + if local { + if current, ok := f.pendingLocalSelfBuildEnvelopes.Peek(blockRoot); ok && current == pending { + f.pendingLocalSelfBuildEnvelopes.Remove(blockRoot) + } + } else if current, ok := f.pendingEnvelopes.Peek(blockRoot); ok && current == pending { + f.pendingEnvelopes.Remove(blockRoot) + } + if !completedSameEnvelope { + return nil + } + return pending.Message +} + +func (f *ForkChoiceStore) retryPendingEnvelopeError(err error, pending *cltypes.SignedExecutionPayloadEnvelope) bool { + if errors.Is(err, errInvalidExecutionPayloadEnvelope) { + return false + } + if !errors.Is(err, ErrIgnore) { + return true + } + if pending == nil || pending.Message == nil || pending.Message.Payload == nil { + return false + } + checkpoint := f.finalizedCheckpoint.Load() + if checkpoint == nil || f.beaconCfg == nil { + return true + } + payloadSlot := pending.Message.Payload.SlotNumber + if f.ethClock != nil { + currentSlot := f.ethClock.GetCurrentSlot() + if payloadSlot > currentSlot { + if payloadSlot-currentSlot > 1 || !f.ethClock.IsSlotCurrentSlotWithMaximumClockDisparity(payloadSlot) { + return false + } + } + } + finalizedSlot := f.computeStartSlotAtEpoch(checkpoint.(solid.Checkpoint).Epoch) + return payloadSlot >= finalizedSlot +} + func (f *ForkChoiceStore) addChainSegmentAndQueueLightClientEvents(block *cltypes.SignedBeaconBlock, fullValidation bool) (*state.CachingBeaconState, fork_graph.ChainSegmentInsertionResult, error) { lcUpdateBefore := f.forkGraph.NewestLightClientUpdate() lastProcessedState, status, err := f.forkGraph.AddChainSegment(block, fullValidation) diff --git a/cl/phase1/forkchoice/on_execution_payload.go b/cl/phase1/forkchoice/on_execution_payload.go index 6de1ef11431..79452c7d4cb 100644 --- a/cl/phase1/forkchoice/on_execution_payload.go +++ b/cl/phase1/forkchoice/on_execution_payload.go @@ -46,6 +46,10 @@ import ( // and queues the execution block for later EL insertion. var errELBehind = errors.New("EL behind: payload not processable yet") +var errPayloadValidationAdmission = errors.New("payload validation admission canceled") + +var errInvalidExecutionPayloadEnvelope = errors.New("invalid execution payload envelope") + // validateEnvelopeAgainstBlock validates the envelope against the block and state. // This includes: // - bid matching (slot, builder_index, block_hash) @@ -230,15 +234,15 @@ func (f *ForkChoiceStore) validatePayloadWithEL( envelope *cltypes.ExecutionPayloadEnvelope, block *cltypes.SignedBeaconBlock, beaconBlockRoot common.Hash, -) error { +) (execution_client.PayloadStatus, error) { if f.engine == nil { - return nil + return execution_client.PayloadStatusNone, nil } // Get committed bid from the block (not from state, since state transition hasn't happened yet) committedBid := block.Block.Body.GetSignedExecutionPayloadBid() if committedBid == nil || committedBid.Message == nil { - return errors.New("validatePayloadWithEL: block missing execution payload bid") + return execution_client.PayloadStatusNone, errors.New("validatePayloadWithEL: block missing execution payload bid") } // Calculate versioned hashes from committed bid's blob_kzg_commitments @@ -254,7 +258,7 @@ func (f *ForkChoiceStore) validatePayloadWithEL( versionedHashes = append(versionedHashes, versionedHash) return nil }); err != nil { - return fmt.Errorf("validatePayloadWithEL: failed to compute versioned hashes: %w", err) + return execution_client.PayloadStatusNone, fmt.Errorf("validatePayloadWithEL: failed to compute versioned hashes: %w", err) } } @@ -268,11 +272,51 @@ func (f *ForkChoiceStore) validatePayloadWithEL( } // Call NewPayload to validate execution payload with EL - timeStartExec := time.Now() parentBlockRoot := block.Block.ParentRoot - payloadStatus, err := f.engine.NewPayload(ctx, envelope.Payload, &parentBlockRoot, versionedHashes, executionRequestsList) - monitor.ObserveNewPayloadTime(timeStartExec) + payloadStatus, err := f.newPayloadWhileYieldingForkChoiceLock(ctx, beaconBlockRoot, envelope.Payload, &parentBlockRoot, versionedHashes, executionRequestsList) log.Trace("[validatePayloadWithEL] NewPayload", "status", payloadStatus, "beaconBlockRoot", beaconBlockRoot) + return payloadStatus, err +} + +func (f *ForkChoiceStore) newPayloadWhileYieldingForkChoiceLock( + ctx context.Context, + beaconBlockRoot common.Hash, + payload *cltypes.Eth1Block, + parentBlockRoot *common.Hash, + versionedHashes []common.Hash, + executionRequestsList []hexutil.Bytes, +) (execution_client.PayloadStatus, error) { + f.mu.Unlock() + defer f.mu.Lock() + + f.payloadValidationOnce.Do(func() { + f.payloadValidationAdmission = make(chan struct{}, 1) + }) + select { + case f.payloadValidationAdmission <- struct{}{}: + defer func() { <-f.payloadValidationAdmission }() + case <-ctx.Done(): + return execution_client.PayloadStatusNone, fmt.Errorf("%w: %w", errPayloadValidationAdmission, ctx.Err()) + } + f.mu.Lock() + alreadyApplied := f.forkGraph.HasEnvelope(beaconBlockRoot) + f.mu.Unlock() + if alreadyApplied { + return execution_client.PayloadStatusValidated, nil + } + + timeStartExec := time.Now() + defer monitor.ObserveNewPayloadTime(timeStartExec) + return f.engine.NewPayload(ctx, payload, parentBlockRoot, versionedHashes, executionRequestsList) +} + +func (f *ForkChoiceStore) applyPayloadValidationResultLocked( + payloadStatus execution_client.PayloadStatus, + validationErr error, + envelope *cltypes.ExecutionPayloadEnvelope, + block *cltypes.SignedBeaconBlock, + beaconBlockRoot common.Hash, +) error { // Track payload status and gas limit by execution block hash for parent payload validation executionBlockHash := envelope.Payload.BlockHash @@ -286,7 +330,7 @@ func (f *ForkChoiceStore) validatePayloadWithEL( // applyEnvelope can persist the envelope and queue the execution block // for later insertion into EL. log.Warn("validatePayloadWithEL: EL could not process payload (EL behind)", - "beaconBlockRoot", beaconBlockRoot, "blockHash", executionBlockHash, "err", err) + "beaconBlockRoot", beaconBlockRoot, "blockHash", executionBlockHash, "err", validationErr) if optErr := f.optimisticStore.AddOptimisticCandidate(beaconBlockRoot, block.Block); optErr != nil { return fmt.Errorf("failed to add block to optimistic store: %w", optErr) } @@ -298,48 +342,66 @@ func (f *ForkChoiceStore) validatePayloadWithEL( return fmt.Errorf("failed to add block to optimistic store: %w", err) } case execution_client.PayloadStatusInvalidated: - log.Warn("validatePayloadWithEL: payload is invalid", "beaconBlockRoot", beaconBlockRoot, "err", err) + log.Warn("validatePayloadWithEL: payload is invalid", "beaconBlockRoot", beaconBlockRoot, "err", validationErr) f.markPayloadInvalidLocked(beaconBlockRoot, executionBlockHash) - return errors.New("execution payload is invalid") + return fmt.Errorf("%w: execution payload is invalid", errInvalidExecutionPayloadEnvelope) case execution_client.PayloadStatusValidated: log.Trace("validatePayloadWithEL: payload is validated", "beaconBlockRoot", beaconBlockRoot) f.markPayloadVerifiedLocked(beaconBlockRoot, executionBlockHash) } - if err != nil { - return fmt.Errorf("validatePayloadWithEL: newPayload failed: %w", err) + if validationErr != nil { + return fmt.Errorf("validatePayloadWithEL: newPayload failed: %w", validationErr) } return nil } +func (f *ForkChoiceStore) refreshEnvelopeBlockLocked(beaconBlockRoot common.Hash) (*state.CachingBeaconState, *cltypes.SignedBeaconBlock, error) { + blockState, err := f.forkGraph.GetState(beaconBlockRoot, false) + if err != nil { + return nil, nil, err + } + block, ok := f.forkGraph.GetBlock(beaconBlockRoot) + if blockState == nil || !ok || block == nil { + return nil, nil, fmt.Errorf("%w: block disappeared during payload validation for beacon_block_root %v", ErrIgnore, beaconBlockRoot) + } + return blockState, block, nil +} + +type missingEnvelopeMode bool + +const ( + retryQueuedEnvelope missingEnvelopeMode = false + queueMissingEnvelope missingEnvelopeMode = true +) + // applyEnvelope processes the envelope under f.mu: validates, verifies with CL and EL, // and persists the envelope to disk. No CL state transition is performed — the // execution effects are deferred to the next block's ProcessParentExecutionPayload. // Returns (true, nil) if the envelope was applied, // (false, nil) if it was skipped (already processed or block not yet known), // or (false, err) on failure. -func (f *ForkChoiceStore) applyEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool) (bool, error) { +func (f *ForkChoiceStore) applyEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool, missingMode missingEnvelopeMode) (bool, error) { if signedEnvelope.Message == nil { log.Warn("[applyEnvelope] received signed envelope with nil message") - return false, errors.New("signed envelope has nil message") + return false, fmt.Errorf("%w: signed envelope has nil message", errInvalidExecutionPayloadEnvelope) } f.mu.Lock() defer f.mu.Unlock() - return f.applyEnvelopeLocked(ctx, signedEnvelope, checkBlobData, validatePayload) + return f.applyEnvelopeCoordinated(ctx, signedEnvelope, checkBlobData, validatePayload, missingMode) } -// applyEnvelopeLocked is the lock-held implementation of applyEnvelope. -// The caller MUST hold f.mu before calling this method. +// applyEnvelopeCoordinated temporarily yields the caller-held fork-choice lock during EL validation. // Returns (true, nil) if the envelope was applied, // (false, nil) if it was skipped (already processed or block not yet known), // or (false, err) on failure. -func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool) (bool, error) { +func (f *ForkChoiceStore) applyEnvelopeCoordinated(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, checkBlobData, validatePayload bool, missingMode missingEnvelopeMode) (bool, error) { if signedEnvelope.Message == nil { - log.Warn("[applyEnvelopeLocked] received signed envelope with nil message") - return false, errors.New("signed envelope has nil message") + log.Warn("[applyEnvelopeCoordinated] received signed envelope with nil message") + return false, fmt.Errorf("%w: signed envelope has nil message", errInvalidExecutionPayloadEnvelope) } envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot @@ -357,26 +419,27 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop return false, fmt.Errorf("OnExecutionPayload: failed to get block state: %w", err) } if blockState == nil { - // Block hasn't arrived yet, queue envelope for later processing. - // Per spec: assert envelope.beacon_block_root in store.block_states - // Return an error so callers can distinguish "queued" from "applied". - f.pendingEnvelopes.Add(beaconBlockRoot, signedEnvelope) - log.Trace("OnExecutionPayload: block not found, queuing envelope for later", "beaconBlockRoot", common.Hash(beaconBlockRoot)) + if missingMode == queueMissingEnvelope { + f.pendingEnvelopes.Add(beaconBlockRoot, signedEnvelope) + } + log.Trace("OnExecutionPayload: block state not found", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block state not found for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } // Get the block to verify it exists block, ok := f.forkGraph.GetBlock(beaconBlockRoot) if !ok || block == nil { - f.pendingEnvelopes.Add(beaconBlockRoot, signedEnvelope) - log.Trace("OnExecutionPayload: block not found in fork graph, queuing envelope", "beaconBlockRoot", common.Hash(beaconBlockRoot)) + if missingMode == queueMissingEnvelope { + f.pendingEnvelopes.Add(beaconBlockRoot, signedEnvelope) + } + log.Trace("OnExecutionPayload: block not found in fork graph", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block not found in fork graph for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } // Validate envelope against block (bid matching + signature verification) if validatePayload { if err := f.validateEnvelopeAgainstBlock(signedEnvelope, block, blockState); err != nil { - return false, fmt.Errorf("OnExecutionPayload: envelope validation failed: %w", err) + return false, fmt.Errorf("%w: OnExecutionPayload: envelope validation failed: %w", errInvalidExecutionPayloadEnvelope, err) } } @@ -386,11 +449,26 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop return false, err } } + blockState.SetPreviousStateRoot(block.Block.StateRoot) + if err := transition.ValidatingMachine.ProcessExecutionPayloadEnvelope(blockState, signedEnvelope); err != nil { + return false, fmt.Errorf("%w: OnExecutionPayload: failed to verify execution payload: %w", errInvalidExecutionPayloadEnvelope, err) + } // Validate payload with EL var elBehind bool - if validatePayload { - if err := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)); err != nil { + if validatePayload && f.engine != nil { + payloadStatus, validationErr := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)) + if errors.Is(validationErr, errPayloadValidationAdmission) { + return false, validationErr + } + if f.forkGraph.HasEnvelope(beaconBlockRoot) { + return false, nil + } + blockState, block, err = f.refreshEnvelopeBlockLocked(beaconBlockRoot) + if err != nil { + return false, fmt.Errorf("OnExecutionPayload: failed to refresh block state: %w", err) + } + if err := f.applyPayloadValidationResultLocked(payloadStatus, validationErr, envelope, block, common.Hash(beaconBlockRoot)); err != nil { if errors.Is(err, errELBehind) { // EL is behind (e.g. parent block not yet available after forward sync). // Proceed with persisting the envelope so HasEnvelope() returns true. @@ -402,21 +480,6 @@ func (f *ForkChoiceStore) applyEnvelopeLocked(ctx context.Context, signedEnvelop } } - // Ensure the correct state root is available for the beacon_block_root check - // inside ProcessExecutionPayloadEnvelope. PreviousStateRoot() is consumptive - // (cleared on read) and may have already been consumed by transitionSlot during - // TransitionState, or by a replay in GetState. Re-setting it from the block's - // known-correct StateRoot guarantees ProcessExecutionPayloadEnvelope can - // reconstruct the block header root without relying on the incremental hash cache. - blockState.SetPreviousStateRoot(block.Block.StateRoot) - - // Run ProcessExecutionPayloadEnvelope for CL-level verification (no state mutation). - // Always use ValidatingMachine so that signature verification and all spec checks run, - // regardless of whether the EL-level validatePayload flag is set. - if err := transition.ValidatingMachine.ProcessExecutionPayloadEnvelope(blockState, signedEnvelope); err != nil { - return false, fmt.Errorf("OnExecutionPayload: failed to verify execution payload: %w", err) - } - // Update eth2Roots mapping for FCU if envelope.Payload != nil { f.eth2Roots.Add(beaconBlockRoot, envelope.Payload.BlockHash) @@ -469,6 +532,7 @@ func (f *ForkChoiceStore) StoreAnchorEnvelope(blockRoot common.Hash, signedEnvel if err := f.db.Update(ctx, func(tx kv.RwTx) error { return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, blockRoot, envelope) }); err != nil { + f.pendingEnvelopes.Add(blockRoot, signedEnvelope) return fmt.Errorf("StoreAnchorEnvelope: failed to write indices: %w", err) } } @@ -494,18 +558,14 @@ func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope // Process envelope under f.mu; DB index write happens after unlock to avoid // deadlock with postForkchoiceOperations (which holds MDBX tx then needs f.mu.RLock). - applied, err := f.applyEnvelope(ctx, signedEnvelope, checkBlobData, validatePayload) - if err != nil || !applied { + applied, err := f.applyEnvelope(ctx, signedEnvelope, checkBlobData, validatePayload, queueMissingEnvelope) + if err != nil { return err } - - // Write execution block indices outside f.mu. - if f.db != nil { - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, common.Hash(beaconBlockRoot), envelope) - }); err != nil { - return fmt.Errorf("OnExecutionPayload: failed to write execution payload indices: %w", err) - } + indexEnvelope, err := f.ensureExecutionPayloadEnvelopeIndices(ctx, common.Hash(beaconBlockRoot), signedEnvelope, applied) + if err != nil { + f.pendingEnvelopes.Add(common.Hash(beaconBlockRoot), indexEnvelope) + return fmt.Errorf("OnExecutionPayload: failed to write execution payload indices: %w", err) } return nil @@ -525,49 +585,116 @@ func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope // verifies BLS signatures. // [New in Gloas:EIP7732] func (f *ForkChoiceStore) ApplyLocalSelfBuildEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) error { - if signedEnvelope == nil || signedEnvelope.Message == nil { - return errors.New("nil execution payload envelope") + if signedEnvelope == nil || signedEnvelope.Message == nil || signedEnvelope.Message.Payload == nil { + return errors.New("execution payload envelope has nil payload") } envelope := signedEnvelope.Message beaconBlockRoot := envelope.BeaconBlockRoot - applied, err := f.applyLocalSelfBuildEnvelope(ctx, signedEnvelope) - if err != nil || !applied { + applied, err := f.applyLocalSelfBuildEnvelope(ctx, signedEnvelope, queueMissingEnvelope) + if err != nil { return err } + indexEnvelope, err := f.ensureExecutionPayloadEnvelopeIndices(ctx, common.Hash(beaconBlockRoot), signedEnvelope, applied) + if err != nil { + f.pendingLocalSelfBuildEnvelopes.Add(common.Hash(beaconBlockRoot), indexEnvelope) + return fmt.Errorf("ApplyLocalSelfBuildEnvelope: failed to write execution payload indices: %w", err) + } - if f.db != nil { - if err := f.db.Update(ctx, func(tx kv.RwTx) error { - return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, common.Hash(beaconBlockRoot), envelope) - }); err != nil { - return fmt.Errorf("ApplyLocalSelfBuildEnvelope: failed to write execution payload indices: %w", err) + return nil +} + +func (f *ForkChoiceStore) ensureExecutionPayloadEnvelopeIndices(ctx context.Context, blockRoot common.Hash, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, applied bool) (*cltypes.SignedExecutionPayloadEnvelope, error) { + if f.db == nil || (!applied && !f.forkGraph.HasEnvelope(blockRoot)) { + return signedEnvelope, nil + } + retried := false + for { + write := &envelopeIndexWrite{done: make(chan struct{})} + existing, loaded := f.envelopeIndexWrites.LoadOrStore(blockRoot, write) + if loaded { + current := existing.(*envelopeIndexWrite) + select { + case <-current.done: + if !retried && ctx.Err() == nil && (errors.Is(current.err, context.Canceled) || errors.Is(current.err, context.DeadlineExceeded) || errors.Is(current.err, errExecutionPayloadIndexWritePanicked)) { + retried = true + continue + } + return current.envelope, current.err + case <-ctx.Done(): + return signedEnvelope, ctx.Err() + } } + return f.runExecutionPayloadEnvelopeIndexWrite(ctx, blockRoot, signedEnvelope, applied, write) } +} - return nil +var errExecutionPayloadIndexWritePanicked = errors.New("execution payload index write panicked") + +func (f *ForkChoiceStore) runExecutionPayloadEnvelopeIndexWrite(ctx context.Context, blockRoot common.Hash, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, applied bool, write *envelopeIndexWrite) (envelope *cltypes.SignedExecutionPayloadEnvelope, err error) { + defer func() { + if recovered := recover(); recovered != nil { + write.envelope = signedEnvelope + write.err = fmt.Errorf("%w: %v", errExecutionPayloadIndexWritePanicked, recovered) + f.envelopeIndexWrites.CompareAndDelete(blockRoot, write) + close(write.done) + panic(recovered) + } + write.envelope, write.err = envelope, err + f.envelopeIndexWrites.CompareAndDelete(blockRoot, write) + close(write.done) + }() + return f.writeExecutionPayloadEnvelopeIndices(ctx, blockRoot, signedEnvelope, applied) +} + +func (f *ForkChoiceStore) writeExecutionPayloadEnvelopeIndices(ctx context.Context, blockRoot common.Hash, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, applied bool) (*cltypes.SignedExecutionPayloadEnvelope, error) { + if !applied { + persisted, err := f.forkGraph.ReadEnvelopeFromDisk(blockRoot) + if err != nil { + return nil, err + } + signedEnvelope = persisted + } + if signedEnvelope == nil || signedEnvelope.Message == nil || signedEnvelope.Message.Payload == nil { + return signedEnvelope, errors.New("persisted execution payload envelope is incomplete") + } + indexed := false + err := f.db.View(ctx, func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, blockRoot) + if err != nil { + return err + } + blockHash, err := beacon_indicies.ReadExecutionBlockHash(tx, blockRoot) + if err != nil { + return err + } + indexed = blockNumber != nil && *blockNumber == signedEnvelope.Message.Payload.BlockNumber && blockHash == signedEnvelope.Message.Payload.BlockHash + return nil + }) + if err != nil || indexed { + return signedEnvelope, err + } + err = f.db.Update(ctx, func(tx kv.RwTx) error { + return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, blockRoot, signedEnvelope.Message) + }) + return signedEnvelope, err } -// applyLocalSelfBuildEnvelope acquires f.mu and delegates to the lock-held implementation. -func (f *ForkChoiceStore) applyLocalSelfBuildEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { +// applyLocalSelfBuildEnvelope coordinates fork-choice ownership around local envelope processing. +func (f *ForkChoiceStore) applyLocalSelfBuildEnvelope(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, missingMode missingEnvelopeMode) (bool, error) { if signedEnvelope.Message == nil { - return false, errors.New("signed envelope has nil message") + return false, fmt.Errorf("%w: signed envelope has nil message", errInvalidExecutionPayloadEnvelope) } f.mu.Lock() defer f.mu.Unlock() - return f.applyLocalSelfBuildEnvelopeLocked(ctx, signedEnvelope) + return f.applyLocalSelfBuildEnvelopeCoordinated(ctx, signedEnvelope, missingMode) } -// applyLocalSelfBuildEnvelopeLocked is the lock-held implementation for local self-build envelopes. -// It mirrors applyEnvelopeLocked but skips signature verification by: -// - Not calling validateEnvelopeAgainstBlock -// - Using transition.DefaultMachine (FullValidation=false) instead of ValidatingMachine -// -// The caller MUST hold f.mu before calling this method. -// EL validation via NewPayload still runs. -func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope) (bool, error) { +// applyLocalSelfBuildEnvelopeCoordinated skips only BLS verification for locally produced envelopes. +func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeCoordinated(ctx context.Context, signedEnvelope *cltypes.SignedExecutionPayloadEnvelope, missingMode missingEnvelopeMode) (bool, error) { if signedEnvelope.Message == nil { return false, errors.New("signed envelope has nil message") } @@ -581,39 +708,52 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, blockState, err := f.forkGraph.GetState(beaconBlockRoot, false) if err != nil { - return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to get block state: %w", err) + return false, fmt.Errorf("applyLocalSelfBuildEnvelopeCoordinated: failed to get block state: %w", err) } if blockState == nil { - f.pendingLocalSelfBuildEnvelopes.Add(beaconBlockRoot, signedEnvelope) - log.Trace("applyLocalSelfBuildEnvelopeLocked: block not found, queuing envelope for later", "beaconBlockRoot", common.Hash(beaconBlockRoot)) + if missingMode == queueMissingEnvelope { + f.pendingLocalSelfBuildEnvelopes.Add(beaconBlockRoot, signedEnvelope) + } + log.Trace("applyLocalSelfBuildEnvelopeCoordinated: block state not found", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block state not found for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } block, ok := f.forkGraph.GetBlock(beaconBlockRoot) if !ok || block == nil { - f.pendingLocalSelfBuildEnvelopes.Add(beaconBlockRoot, signedEnvelope) - log.Trace("applyLocalSelfBuildEnvelopeLocked: block not found in fork graph, queuing envelope", "beaconBlockRoot", common.Hash(beaconBlockRoot)) + if missingMode == queueMissingEnvelope { + f.pendingLocalSelfBuildEnvelopes.Add(beaconBlockRoot, signedEnvelope) + } + log.Trace("applyLocalSelfBuildEnvelopeCoordinated: block not found in fork graph", "beaconBlockRoot", common.Hash(beaconBlockRoot)) return false, fmt.Errorf("%w: block not found in fork graph for beacon_block_root %v", ErrIgnore, common.Hash(beaconBlockRoot)) } // Skip validateEnvelopeAgainstBlock — we produced this envelope locally. + blockState.SetPreviousStateRoot(block.Block.StateRoot) + if err := transition.DefaultMachine.ProcessExecutionPayloadEnvelope(blockState, signedEnvelope); err != nil { + return false, fmt.Errorf("%w: applyLocalSelfBuildEnvelopeCoordinated: failed to verify execution payload: %w", errInvalidExecutionPayloadEnvelope, err) + } // Validate payload with EL (NewPayload). var elBehind bool - if err := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)); err != nil { - if errors.Is(err, errELBehind) { - elBehind = true - } else { - return false, err + if f.engine != nil { + payloadStatus, validationErr := f.validatePayloadWithEL(ctx, envelope, block, common.Hash(beaconBlockRoot)) + if errors.Is(validationErr, errPayloadValidationAdmission) { + return false, validationErr + } + if f.forkGraph.HasEnvelope(beaconBlockRoot) { + return false, nil + } + blockState, block, err = f.refreshEnvelopeBlockLocked(beaconBlockRoot) + if err != nil { + return false, fmt.Errorf("applyLocalSelfBuildEnvelopeCoordinated: failed to refresh block state: %w", err) + } + if err := f.applyPayloadValidationResultLocked(payloadStatus, validationErr, envelope, block, common.Hash(beaconBlockRoot)); err != nil { + if errors.Is(err, errELBehind) { + elBehind = true + } else { + return false, err + } } - } - - blockState.SetPreviousStateRoot(block.Block.StateRoot) - - // Use DefaultMachine (FullValidation=false) to skip BLS signature verification - // in ProcessExecutionPayloadEnvelope while still running all other spec checks. - if err := transition.DefaultMachine.ProcessExecutionPayloadEnvelope(blockState, signedEnvelope); err != nil { - return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to verify execution payload: %w", err) } if envelope.Payload != nil { @@ -621,7 +761,7 @@ func (f *ForkChoiceStore) applyLocalSelfBuildEnvelopeLocked(ctx context.Context, } if err := f.forkGraph.DumpEnvelopeOnDisk(beaconBlockRoot, signedEnvelope); err != nil { - return false, fmt.Errorf("applyLocalSelfBuildEnvelopeLocked: failed to dump envelope: %w", err) + return false, fmt.Errorf("applyLocalSelfBuildEnvelopeCoordinated: failed to dump envelope: %w", err) } f.headHash = common.Hash{} diff --git a/cl/phase1/forkchoice/on_execution_payload_test.go b/cl/phase1/forkchoice/on_execution_payload_test.go index 85dd3524a94..37737e1608f 100644 --- a/cl/phase1/forkchoice/on_execution_payload_test.go +++ b/cl/phase1/forkchoice/on_execution_payload_test.go @@ -18,6 +18,10 @@ package forkchoice import ( "context" + "errors" + "fmt" + "sync" + "sync/atomic" "testing" "time" @@ -28,10 +32,673 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" + "github.com/erigontech/erigon/cl/persistence/beacon_indicies" + state2 "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" + "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph" + "github.com/erigontech/erigon/cl/utils/eth_clock" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/hexutil" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/mdbx/mdbxtest" ) +type failingUpdateDB struct { + kv.RwDB + fail bool + calls int +} + +type blockingUpdateDB struct { + kv.RwDB + calls atomic.Int32 + started chan struct{} + release chan struct{} +} + +type panickingUpdateDB struct { + kv.RwDB + started chan struct{} + release chan struct{} + calls atomic.Int32 +} + +func (db *panickingUpdateDB) Update(ctx context.Context, f func(kv.RwTx) error) error { + if db.calls.Add(1) == 1 { + close(db.started) + <-db.release + panic("injected update panic") + } + return db.RwDB.Update(ctx, f) +} + +type observedContext struct { + context.Context + doneObserved chan struct{} + once sync.Once +} + +func (ctx *observedContext) Done() <-chan struct{} { + ctx.once.Do(func() { close(ctx.doneObserved) }) + return ctx.Context.Done() +} + +func (db *blockingUpdateDB) Update(ctx context.Context, f func(kv.RwTx) error) error { + if db.calls.Add(1) == 1 { + close(db.started) + select { + case <-db.release: + case <-ctx.Done(): + return ctx.Err() + } + } + return db.RwDB.Update(ctx, f) +} + +func (db *failingUpdateDB) Update(ctx context.Context, f func(kv.RwTx) error) error { + db.calls++ + if db.fail { + return errors.New("injected update failure") + } + return db.RwDB.Update(ctx, f) +} + +type pendingRetryForkGraph struct { + fork_graph.ForkGraph + completed common.Hash + completedEnvelope *cltypes.SignedExecutionPayloadEnvelope +} + +type transientEnvelopeReadForkGraph struct { + pendingRetryForkGraph + fail atomic.Bool +} + +type replacingPendingForkGraph struct { + fork_graph.ForkGraph + replace func() +} + +type missingBlockForkGraph struct { + pendingRetryForkGraph + state *state2.CachingBeaconState +} + +func (g replacingPendingForkGraph) GetState(common.Hash, bool) (*state2.CachingBeaconState, error) { + g.replace() + return nil, nil +} + +func (g replacingPendingForkGraph) HasEnvelope(common.Hash) bool { return false } + +func (g missingBlockForkGraph) GetState(common.Hash, bool) (*state2.CachingBeaconState, error) { + return g.state, nil +} + +func (g missingBlockForkGraph) GetBlock(common.Hash) (*cltypes.SignedBeaconBlock, bool) { + return nil, false +} + +func (g *transientEnvelopeReadForkGraph) ReadEnvelopeFromDisk(root common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { + if g.fail.Load() { + return nil, errors.New("injected envelope read failure") + } + return g.pendingRetryForkGraph.ReadEnvelopeFromDisk(root) +} + +func TestApplyLocalSelfBuildEnvelopeRejectsNilPayloadAtIngress(t *testing.T) { + f := &ForkChoiceStore{} + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{}} + + require.ErrorContains(t, f.ApplyLocalSelfBuildEnvelope(context.Background(), envelope), "nil payload") +} + +func (g pendingRetryForkGraph) HasEnvelope(root common.Hash) bool { return root == g.completed } +func (g pendingRetryForkGraph) ReadEnvelopeFromDisk(root common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { + if root != g.completed { + return nil, nil + } + return g.completedEnvelope, nil +} +func (g pendingRetryForkGraph) GetState(common.Hash, bool) (*state2.CachingBeaconState, error) { + return nil, nil +} + +func TestApplyPendingEnvelopeDoesNotIndexConcurrentWinner(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: blockRoot}} + pending.Add(blockRoot, envelope) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{hasEnvelope: true}, + pendingEnvelopes: pending, + } + + appliedEnvelope := f.applyPendingEnvelope(context.Background(), blockRoot, envelope, false, false) + require.Nil(t, appliedEnvelope) + require.False(t, pending.Contains(blockRoot)) +} + +func TestApplyPendingEnvelopeDropsHardFailure(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{} + pending.Add(blockRoot, envelope) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + + require.Nil(t, f.applyPendingEnvelope(context.Background(), blockRoot, envelope, false, false)) + require.False(t, pending.Contains(blockRoot)) +} + +func TestPendingEnvelopeErrorClassification(t *testing.T) { + f := &ForkChoiceStore{} + require.True(t, f.retryPendingEnvelopeError(errors.New("temporary disk failure"), nil)) + require.True(t, f.retryPendingEnvelopeError(ErrEIP7594ColumnDataNotAvailable, nil)) + require.False(t, f.retryPendingEnvelopeError(fmt.Errorf("%w: bad signature", errInvalidExecutionPayloadEnvelope), nil)) +} + +func TestRetryPendingExecutionPayloadEnvelopesCleansCompletedWork(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](2) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + otherRoot := common.HexToHash("0x5678") + pending.Add(blockRoot, &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: blockRoot}}) + pending.Add(otherRoot, &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: otherRoot}}) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{hasEnvelope: true}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.Equal(t, 1, pending.Len()) +} + +func TestRetryPendingExecutionPayloadEnvelopesSharesBudgetAcrossOrigins(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](2) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](2) + require.NoError(t, err) + localRoot := common.HexToHash("0x1234") + gossipRoot := common.HexToHash("0x5678") + local.Add(localRoot, &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: localRoot}}) + pending.Add(gossipRoot, &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: gossipRoot}}) + f := &ForkChoiceStore{ + forkGraph: payloadVoteForkGraph{hasEnvelope: true}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 2) + require.Zero(t, local.Len()) + require.Zero(t, pending.Len()) +} + +func TestRetryPendingExecutionPayloadEnvelopesRotatesFailures(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](3) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + roots := []common.Hash{common.HexToHash("0x1"), common.HexToHash("0x2"), common.HexToHash("0x3")} + for _, root := range roots { + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = root + pending.Add(root, envelope) + } + completedEnvelope, ok := pending.Peek(roots[2]) + require.True(t, ok) + f := &ForkChoiceStore{ + forkGraph: pendingRetryForkGraph{completed: roots[2], completedEnvelope: completedEnvelope}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 2) + require.True(t, pending.Contains(roots[2])) + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 2) + require.False(t, pending.Contains(roots[2])) +} + +func TestRetryPendingExecutionPayloadEnvelopesDropsMissingBlockOlderThanFinality(t *testing.T) { + for _, localOrigin := range []bool{false, true} { + t.Run(fmt.Sprintf("local=%t", localOrigin), func(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](3) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](3) + require.NoError(t, err) + staleRoot := common.HexToHash("0x1234") + boundaryRoot := common.HexToHash("0x3456") + recentRoot := common.HexToHash("0x5678") + stale := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + stale.Message.BeaconBlockRoot = staleRoot + stale.Message.Payload.SlotNumber = 63 + boundary := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + boundary.Message.BeaconBlockRoot = boundaryRoot + boundary.Message.Payload.SlotNumber = 64 + recent := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + recent.Message.BeaconBlockRoot = recentRoot + recent.Message.Payload.SlotNumber = 94 + origin := pending + if localOrigin { + origin = local + } + origin.Add(staleRoot, stale) + origin.Add(boundaryRoot, boundary) + origin.Add(recentRoot, recent) + f := &ForkChoiceStore{ + beaconCfg: &clparams.MainnetBeaconConfig, + forkGraph: pendingRetryForkGraph{}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 2}) + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.False(t, origin.Contains(staleRoot)) + require.True(t, origin.Contains(boundaryRoot)) + require.True(t, origin.Contains(recentRoot)) + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 2) + require.True(t, origin.Contains(boundaryRoot)) + require.True(t, origin.Contains(recentRoot)) + + f.forkGraph = pendingRetryForkGraph{completed: recentRoot, completedEnvelope: recent} + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 2) + require.True(t, origin.Contains(boundaryRoot)) + require.False(t, origin.Contains(recentRoot)) + }) + } +} + +func TestRetryPendingExecutionPayloadEnvelopesDropsFarFutureSlot(t *testing.T) { + for _, localOrigin := range []bool{false, true} { + t.Run(fmt.Sprintf("local=%t", localOrigin), func(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + clock := eth_clock.NewEthereumClock(0, common.Hash{}, &clparams.MainnetBeaconConfig) + envelope.Message.Payload.SlotNumber = clock.GetCurrentSlot() + (uint64(1) << 62) + origin := pending + if localOrigin { + origin = local + } + origin.Add(blockRoot, envelope) + f := &ForkChoiceStore{ + beaconCfg: &clparams.MainnetBeaconConfig, + ethClock: clock, + forkGraph: pendingRetryForkGraph{}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 2}) + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.False(t, origin.Contains(blockRoot)) + }) + } +} + +func TestRetryPendingExecutionPayloadEnvelopesKeepsNextSlotWithinClockDisparity(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + envelope.Message.Payload.SlotNumber = 101 + pending.Add(blockRoot, envelope) + clock := eth_clock.NewMockEthereumClock(gomock.NewController(t)) + clock.EXPECT().GetCurrentSlot().Return(uint64(100)) + clock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(101)).Return(true) + f := &ForkChoiceStore{ + beaconCfg: &clparams.MainnetBeaconConfig, + ethClock: clock, + forkGraph: pendingRetryForkGraph{}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 2}) + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.True(t, pending.Contains(blockRoot)) +} + +func TestMissingBlockExecutionPayloadEnvelopeQueuesAtIngress(t *testing.T) { + for _, missingState := range []bool{false, true} { + for _, localOrigin := range []bool{false, true} { + t.Run(fmt.Sprintf("missing-state=%t/local=%t", missingState, localOrigin), func(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + var graph fork_graph.ForkGraph = missingBlockForkGraph{state: state2.New(&clparams.MainnetBeaconConfig)} + if missingState { + graph = pendingRetryForkGraph{} + } + f := &ForkChoiceStore{ + forkGraph: graph, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + + if localOrigin { + require.ErrorIs(t, f.ApplyLocalSelfBuildEnvelope(context.Background(), envelope), ErrIgnore) + require.True(t, local.Contains(blockRoot)) + } else { + require.ErrorIs(t, f.OnExecutionPayload(context.Background(), envelope, false, true), ErrIgnore) + require.True(t, pending.Contains(blockRoot)) + } + + f.forkGraph = pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope} + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.False(t, local.Contains(blockRoot)) + require.False(t, pending.Contains(blockRoot)) + }) + } + } +} + +func TestRetryPendingExecutionPayloadEnvelopesKeepsConcurrentReplacement(t *testing.T) { + for _, localOrigin := range []bool{false, true} { + t.Run(fmt.Sprintf("local=%t", localOrigin), func(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + stale := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + stale.Message.BeaconBlockRoot = blockRoot + stale.Message.Payload.SlotNumber = 63 + replacement := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + replacement.Message.BeaconBlockRoot = blockRoot + replacement.Message.Payload.SlotNumber = 64 + origin := pending + if localOrigin { + origin = local + } + origin.Add(blockRoot, stale) + f := &ForkChoiceStore{ + beaconCfg: &clparams.MainnetBeaconConfig, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + f.forkGraph = replacingPendingForkGraph{replace: func() { origin.Add(blockRoot, replacement) }} + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 2}) + + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + queued, ok := origin.Peek(blockRoot) + require.True(t, ok) + require.Same(t, replacement, queued) + }) + } +} + +func TestRetryPendingExecutionPayloadEnvelopesHandlesMalformedEnvelope(t *testing.T) { + for name, envelope := range map[string]*cltypes.SignedExecutionPayloadEnvelope{ + "nil message": {}, + "nil payload": {Message: &cltypes.ExecutionPayloadEnvelope{}}, + } { + t.Run(name, func(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + if envelope.Message != nil { + envelope.Message.BeaconBlockRoot = blockRoot + } + pending.Add(blockRoot, envelope) + f := &ForkChoiceStore{ + beaconCfg: &clparams.MainnetBeaconConfig, + forkGraph: pendingRetryForkGraph{}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + } + f.finalizedCheckpoint.Store(solid.Checkpoint{Epoch: 2}) + + require.NotPanics(t, func() { + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + }) + require.False(t, pending.Contains(blockRoot)) + }) + } +} + +func TestPendingEnvelopeIndexWriteRetriesThroughOriginQueue(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + envelope.Message.Payload.BlockNumber = 42 + envelope.Message.Payload.BlockHash = common.HexToHash("0xabcd") + pending.Add(blockRoot, envelope) + db := &failingUpdateDB{RwDB: mdbxtest.NewTestDB(t, dbcfg.ChainDB), fail: true} + f := &ForkChoiceStore{ + forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope}, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + db: db, + } + + f.processPendingEnvelopeAfterBlock(context.Background(), blockRoot, false) + require.True(t, pending.Contains(blockRoot)) + require.Equal(t, 1, db.calls) + db.fail = false + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.False(t, pending.Contains(blockRoot)) + require.Equal(t, 2, db.calls) +} + +func TestIndexRepairFailureQueuesPersistedEnvelope(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + persisted := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + persisted.Message.BeaconBlockRoot = blockRoot + persisted.Message.Payload.BlockHash = common.HexToHash("0xaaaa") + redelivered := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + redelivered.Message.BeaconBlockRoot = blockRoot + redelivered.Message.Payload.BlockHash = common.HexToHash("0xbbbb") + f := &ForkChoiceStore{ + forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: persisted}, + pendingEnvelopes: pending, + db: &failingUpdateDB{RwDB: mdbxtest.NewTestDB(t, dbcfg.ChainDB), fail: true}, + } + + require.Error(t, f.OnExecutionPayload(context.Background(), redelivered, false, false)) + queued, ok := pending.Get(blockRoot) + require.True(t, ok) + require.Same(t, persisted, queued) +} + +func TestIndexRepairReadFailureQueuesRootRepair(t *testing.T) { + pending, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + local, err := lru.New[common.Hash, *cltypes.SignedExecutionPayloadEnvelope](1) + require.NoError(t, err) + blockRoot := common.HexToHash("0x1234") + persisted := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + persisted.Message.BeaconBlockRoot = blockRoot + persisted.Message.Payload.BlockHash = common.HexToHash("0xaaaa") + redelivered := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + redelivered.Message.BeaconBlockRoot = blockRoot + redelivered.Message.Payload.BlockHash = common.HexToHash("0xbbbb") + graph := &transientEnvelopeReadForkGraph{pendingRetryForkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: persisted}} + graph.fail.Store(true) + db := mdbxtest.NewTestDB(t, dbcfg.ChainDB) + f := &ForkChoiceStore{ + forkGraph: graph, + pendingEnvelopes: pending, + pendingLocalSelfBuildEnvelopes: local, + db: db, + } + + require.Error(t, f.OnExecutionPayload(context.Background(), redelivered, false, false)) + queued, ok := pending.Get(blockRoot) + require.True(t, ok) + require.Nil(t, queued) + graph.fail.Store(false) + f.RetryPendingExecutionPayloadEnvelopes(context.Background(), 1) + require.False(t, pending.Contains(blockRoot)) + require.NoError(t, db.View(context.Background(), func(tx kv.Tx) error { + blockHash, err := beacon_indicies.ReadExecutionBlockHash(tx, blockRoot) + require.NoError(t, err) + require.Equal(t, persisted.Message.Payload.BlockHash, blockHash) + return nil + })) +} + +func TestOnExecutionPayloadRedeliveryRepairsMissingIndices(t *testing.T) { + blockRoot := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + envelope.Message.Payload.BlockNumber = 42 + envelope.Message.Payload.BlockHash = executionHash + db := mdbxtest.NewTestDB(t, dbcfg.ChainDB) + f := &ForkChoiceStore{ + forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope}, + db: db, + } + + require.NoError(t, f.OnExecutionPayload(context.Background(), envelope, false, true)) + require.NoError(t, db.View(context.Background(), func(tx kv.Tx) error { + blockNumber, err := beacon_indicies.ReadExecutionBlockNumber(tx, blockRoot) + require.NoError(t, err) + require.Equal(t, uint64(42), *blockNumber) + blockHash, err := beacon_indicies.ReadExecutionBlockHash(tx, blockRoot) + require.NoError(t, err) + require.Equal(t, executionHash, blockHash) + return nil + })) +} + +func TestOnExecutionPayloadRedeliverySkipsExistingIndices(t *testing.T) { + blockRoot := common.HexToHash("0x1234") + executionHash := common.HexToHash("0xabcd") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + envelope.Message.Payload.BlockNumber = 42 + envelope.Message.Payload.BlockHash = executionHash + rwdb := mdbxtest.NewTestDB(t, dbcfg.ChainDB) + require.NoError(t, rwdb.Update(context.Background(), func(tx kv.RwTx) error { + return beacon_indicies.WriteExecutionPayloadEnvelopeIndicies(tx, blockRoot, envelope.Message) + })) + db := &failingUpdateDB{RwDB: rwdb} + f := &ForkChoiceStore{ + forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope}, + db: db, + } + + require.NoError(t, f.OnExecutionPayload(context.Background(), envelope, false, true)) + require.Zero(t, db.calls) +} + +func TestExecutionPayloadIndexWritesCollapseByRoot(t *testing.T) { + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + db := &blockingUpdateDB{ + RwDB: mdbxtest.NewTestDB(t, dbcfg.ChainDB), + started: make(chan struct{}), + release: make(chan struct{}), + } + f := &ForkChoiceStore{ + forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope}, + db: db, + } + + results := make(chan error, 1) + go func() { + _, err := f.ensureExecutionPayloadEnvelopeIndices(context.Background(), blockRoot, envelope, true) + results <- err + }() + <-db.started + waiterCtx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := f.ensureExecutionPayloadEnvelopeIndices(waiterCtx, blockRoot, envelope, false) + require.ErrorIs(t, err, context.Canceled) + close(db.release) + require.NoError(t, <-results) + require.Equal(t, int32(1), db.calls.Load()) +} + +func TestExecutionPayloadIndexWriteCanceledLeaderDoesNotPoisonWaiter(t *testing.T) { + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + db := &blockingUpdateDB{ + RwDB: mdbxtest.NewTestDB(t, dbcfg.ChainDB), + started: make(chan struct{}), + release: make(chan struct{}), + } + f := &ForkChoiceStore{forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope}, db: db} + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderDone := make(chan error, 1) + go func() { + _, err := f.ensureExecutionPayloadEnvelopeIndices(leaderCtx, blockRoot, envelope, true) + leaderDone <- err + }() + <-db.started + waiterCtx := &observedContext{Context: context.Background(), doneObserved: make(chan struct{})} + waiterDone := make(chan error, 1) + go func() { + _, err := f.ensureExecutionPayloadEnvelopeIndices(waiterCtx, blockRoot, envelope, false) + waiterDone <- err + }() + <-waiterCtx.doneObserved + cancelLeader() + require.ErrorIs(t, <-leaderDone, context.Canceled) + require.NoError(t, <-waiterDone) + require.Equal(t, int32(2), db.calls.Load()) +} + +func TestExecutionPayloadIndexWritePanicDoesNotReportSuccessToWaiter(t *testing.T) { + blockRoot := common.HexToHash("0x1234") + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(&clparams.MainnetBeaconConfig)} + envelope.Message.BeaconBlockRoot = blockRoot + db := &panickingUpdateDB{RwDB: mdbxtest.NewTestDB(t, dbcfg.ChainDB), started: make(chan struct{}), release: make(chan struct{})} + f := &ForkChoiceStore{forkGraph: pendingRetryForkGraph{completed: blockRoot, completedEnvelope: envelope}, db: db} + leaderDone := make(chan any, 1) + go func() { + defer func() { leaderDone <- recover() }() + _, _ = f.ensureExecutionPayloadEnvelopeIndices(context.Background(), blockRoot, envelope, true) + }() + <-db.started + waiterCtx := &observedContext{Context: context.Background(), doneObserved: make(chan struct{})} + waiterDone := make(chan error, 1) + go func() { + _, err := f.ensureExecutionPayloadEnvelopeIndices(waiterCtx, blockRoot, envelope, false) + waiterDone <- err + }() + <-waiterCtx.doneObserved + close(db.release) + require.Equal(t, "injected update panic", <-leaderDone) + require.NoError(t, <-waiterDone) + require.Equal(t, int32(2), db.calls.Load()) +} + // TestValidateEnvelopeAgainstBlock_NoBid tests that validation fails when block has no bid func TestValidateEnvelopeAgainstBlock_NoBid(t *testing.T) { cfg := &clparams.MainnetBeaconConfig @@ -272,7 +939,7 @@ func TestValidatePayloadWithEL_NoEngine(t *testing.T) { }, } - err := f.validatePayloadWithEL(context.TODO(), envelope, block, common.Hash{}) + _, err := f.validatePayloadWithEL(context.TODO(), envelope, block, common.Hash{}) require.NoError(t, err) } @@ -342,7 +1009,8 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { go func() { f.mu.Lock() defer f.mu.Unlock() - done <- f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + status, validationErr := f.validatePayloadWithEL(context.Background(), envelope, block, blockRoot) + done <- f.applyPayloadValidationResultLocked(status, validationErr, envelope, block, blockRoot) }() select { @@ -362,3 +1030,87 @@ func TestValidatePayloadWithELDoesNotRelockForkChoiceMu(t *testing.T) { }) } } + +func TestValidatePayloadWithELReleasesForkChoiceMuDuringNewPayload(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + ctrl := gomock.NewController(t) + engine := execution_client.NewMockExecutionEngine(ctrl) + engineStarted := make(chan struct{}) + releaseEngine := make(chan struct{}) + engine.EXPECT(). + NewPayload(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { + close(engineStarted) + <-releaseEngine + return execution_client.PayloadStatusValidated, nil + }) + + verifiedExecutionPayload, err := lru.New[common.Hash, struct{}](16) + require.NoError(t, err) + executionPayloadStatus, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + payloadStatusByRoot, err := lru.New[common.Hash, execution_client.PayloadStatus](16) + require.NoError(t, err) + executionPayloadGasLimit, err := lru.New[common.Hash, uint64](16) + require.NoError(t, err) + + f := &ForkChoiceStore{ + beaconCfg: cfg, + engine: engine, + forkGraph: payloadVoteForkGraph{}, + verifiedExecutionPayload: verifiedExecutionPayload, + executionPayloadStatus: executionPayloadStatus, + payloadStatusByRoot: payloadStatusByRoot, + executionPayloadGasLimit: executionPayloadGasLimit, + } + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{Message: &cltypes.ExecutionPayloadBid{ + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + }} + block := &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Body: body}} + envelope := cltypes.NewExecutionPayloadEnvelope(cfg) + envelope.Payload.BlockHash = common.HexToHash("0xabcd") + + done := make(chan error, 1) + go func() { + f.mu.Lock() + defer f.mu.Unlock() + status, validationErr := f.validatePayloadWithEL(context.Background(), envelope, block, common.HexToHash("0x1234")) + done <- f.applyPayloadValidationResultLocked(status, validationErr, envelope, block, common.HexToHash("0x1234")) + }() + <-engineStarted + + lockAcquired := make(chan struct{}) + go func() { + f.mu.Lock() + close(lockAcquired) + f.mu.Unlock() + }() + select { + case <-lockAcquired: + case <-time.After(time.Second): + close(releaseEngine) + t.Fatal("forkchoice mutex stayed locked during NewPayload") + } + close(releaseEngine) + require.NoError(t, <-done) +} + +func TestValidatePayloadWithELAdmissionCancellationIsNotELBehind(t *testing.T) { + f := &ForkChoiceStore{ + engine: execution_client.NewMockExecutionEngine(gomock.NewController(t)), + payloadValidationAdmission: make(chan struct{}, 1), + } + f.payloadValidationOnce.Do(func() {}) + f.payloadValidationAdmission <- struct{}{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + f.mu.Lock() + status, err := f.newPayloadWhileYieldingForkChoiceLock(ctx, common.Hash{}, nil, nil, nil, nil) + f.mu.Unlock() + + require.EqualValues(t, execution_client.PayloadStatusNone, status) + require.ErrorIs(t, err, errPayloadValidationAdmission) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/cl/phase1/forkchoice/on_payload_attestation_message.go b/cl/phase1/forkchoice/on_payload_attestation_message.go index 3b36675881d..58f93e10df5 100644 --- a/cl/phase1/forkchoice/on_payload_attestation_message.go +++ b/cl/phase1/forkchoice/on_payload_attestation_message.go @@ -17,13 +17,13 @@ package forkchoice import ( + "context" "errors" "fmt" "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" - "github.com/erigontech/erigon/cl/cltypes/solid" - "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/common" ) // OnPayloadAttestationMessage processes a payload attestation message and updates @@ -33,75 +33,53 @@ import ( // Caller should handle errors appropriately based on isFromBlock context. // [New in Gloas:EIP7732] func (f *ForkChoiceStore) OnPayloadAttestationMessage( + ctx context.Context, msg *cltypes.PayloadAttestationMessage, isFromBlock bool, ) error { - if msg.Data == nil { + if msg == nil || msg.Data == nil { return errors.New("nil payload attestation data") } data := msg.Data blockRoot := data.BeaconBlockRoot - blockState, err := f.GetStateAtBlockRoot(blockRoot, true) - if err != nil { - return err - } - if blockState == nil { - return fmt.Errorf("%w: block state not found for root %v", ErrIgnore, blockRoot) + if !isFromBlock { + // Wall-clock time is authoritative for gossip because store time can lag OnTick. + currentSlot := f.ethClock.GetCurrentSlot() + if data.Slot != currentSlot { + return fmt.Errorf("%w: attestation slot %d is not current slot %d", ErrIgnore, data.Slot, currentSlot) + } } - // Get the PTC for the attestation slot - ptc, err := blockState.GetPTC(data.Slot) + validationContext, err := f.payloadAttestationValidationContext(ctx, blockRoot, data.Slot) if err != nil { return err } - - // PTC votes can only change the vote for their assigned beacon block - if data.Slot != blockState.Slot() { - return fmt.Errorf("%w: attestation slot %d does not match block slot %d", ErrIgnore, data.Slot, blockState.Slot()) - } - - // [REJECT] Check that the attester is from the PTC - var ptcIndices []int - for i, idx := range ptc { - if idx == msg.ValidatorIndex { - ptcIndices = append(ptcIndices, i) - } - } - if len(ptcIndices) == 0 { - return fmt.Errorf("validator %d is not in PTC for slot %d", msg.ValidatorIndex, data.Slot) + ptcIndices, err := validationContext.ptcPositions(msg) + if err != nil { + return err } // Verify the signature and check that it's for the current slot if coming from wire if !isFromBlock { - // [IGNORE] Check that the attestation is for the current slot. - // Use ethClock.GetCurrentSlot() (wall-clock based) instead of f.Slot() - // (forkchoice-store time based) because f.Slot() depends on f.time which - // is only updated by OnTick and can be stale or uninitialized, causing - // uint64 underflow and an absurdly large slot number. - currentSlot := f.ethClock.GetCurrentSlot() - if data.Slot != currentSlot { - return fmt.Errorf("%w: attestation slot %d is not current slot %d", ErrIgnore, data.Slot, currentSlot) - } - // [REJECT] Verify the signature - indexedAttestation := &cltypes.IndexedPayloadAttestation{ - AttestingIndices: solid.NewRawUint64List(1, []uint64{msg.ValidatorIndex}), - Data: data, - Signature: msg.Signature, - } - valid, err := state.IsValidIndexedPayloadAttestation(blockState, indexedAttestation) - if err != nil { + if err := validationContext.validateSignature(msg); err != nil { return err } - if !valid { - return errors.New("invalid payload attestation signature") - } } - // Atomically update PTC vote arrays under mutex to prevent concurrent - // Load→modify→Store from losing votes. See also applyPayloadAttestationVote. + return f.applyValidatedPayloadAttestation(msg.ValidatorIndex, ptcIndices, data, blockRoot, isFromBlock) +} + +func (f *ForkChoiceStore) applyValidatedPayloadAttestation( + validatorIndex uint64, + ptcIndices []int, + data *cltypes.PayloadAttestationData, + blockRoot common.Hash, + isFromBlock bool, +) error { f.ptcVoteMu.Lock() + defer f.ptcVoteMu.Unlock() var timelinessVotes [clparams.PtcSize]int8 if existing, ok := f.payloadTimelinessVote.Load(blockRoot); ok { @@ -111,14 +89,24 @@ func (f *ForkChoiceStore) OnPayloadAttestationMessage( if existing, ok := f.payloadDataAvailabilityVote.Load(blockRoot); ok { dataAvailabilityVotes = existing.([clparams.PtcSize]int8) } + if !isFromBlock { + if f.payloadAttestationSeen != nil && data.Slot < f.payloadAttestationSeenSlot { + return fmt.Errorf("%w: payload attestation validation completed after a newer slot", ErrIgnore) + } + if f.payloadAttestationSeen == nil || f.payloadAttestationSeenSlot < data.Slot { + f.payloadAttestationSeenSlot = data.Slot + f.payloadAttestationSeen = make(map[uint64]struct{}, clparams.PtcSize) + } + if _, seen := f.payloadAttestationSeen[validatorIndex]; seen { + return fmt.Errorf("%w: already processed a valid payload attestation", ErrIgnore) + } + f.payloadAttestationSeen[validatorIndex] = struct{}{} + } for _, idx := range ptcIndices { timelinessVotes[idx] = boolToVote(data.PayloadPresent) dataAvailabilityVotes[idx] = boolToVote(data.BlobDataAvailable) } f.payloadTimelinessVote.Store(blockRoot, timelinessVotes) f.payloadDataAvailabilityVote.Store(blockRoot, dataAvailabilityVotes) - - f.ptcVoteMu.Unlock() - return nil } diff --git a/cl/phase1/forkchoice/payload_attestation_validation.go b/cl/phase1/forkchoice/payload_attestation_validation.go new file mode 100644 index 00000000000..06c04ae8d0f --- /dev/null +++ b/cl/phase1/forkchoice/payload_attestation_validation.go @@ -0,0 +1,257 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package forkchoice + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/fork" + "github.com/erigontech/erigon/cl/phase1/core/state" + "github.com/erigontech/erigon/cl/phase1/core/state/lru" + "github.com/erigontech/erigon/cl/utils/bls" + "github.com/erigontech/erigon/common" +) + +const ( + payloadAttestationValidationContextCacheSize = 128 + maxConcurrentValidationContextBuilds = 1 + maxPendingValidationContextBuilds = int(clparams.PtcSize) +) + +type payloadAttestationValidationContext struct { + slot uint64 + domain common.Hash + positions map[uint64][]int + publicKeys map[uint64]common.Bytes48 +} + +type payloadAttestationValidationContexts struct { + cache *lru.Cache[common.Hash, *payloadAttestationValidationContext] + buildAdmission chan struct{} + pendingBuilds chan struct{} + mu sync.Mutex + builds map[common.Hash]*payloadAttestationValidationContextBuild +} + +type payloadAttestationValidationContextBuild struct { + done chan struct{} + validationContext *payloadAttestationValidationContext + err error + retry bool +} + +func newPayloadAttestationValidationContexts() (*payloadAttestationValidationContexts, error) { + cache, err := lru.New[common.Hash, *payloadAttestationValidationContext]( + "payload_attestation_validation_contexts", + payloadAttestationValidationContextCacheSize, + ) + if err != nil { + return nil, err + } + return &payloadAttestationValidationContexts{ + cache: cache, + buildAdmission: make(chan struct{}, maxConcurrentValidationContextBuilds), + pendingBuilds: make(chan struct{}, maxPendingValidationContextBuilds), + builds: make(map[common.Hash]*payloadAttestationValidationContextBuild), + }, nil +} + +func (c *payloadAttestationValidationContexts) get( + ctx context.Context, + blockRoot common.Hash, + build func() (*payloadAttestationValidationContext, error), +) (*payloadAttestationValidationContext, error) { + for { + if validationContext, ok := c.cache.Get(blockRoot); ok { + return validationContext, nil + } + c.mu.Lock() + if current, ok := c.builds[blockRoot]; ok { + c.mu.Unlock() + validationContext, err, retry := waitForPayloadAttestationValidationContext(ctx, current) + if retry && ctx.Err() == nil { + continue + } + return validationContext, err + } + c.mu.Unlock() + select { + case c.pendingBuilds <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + default: + return nil, fmt.Errorf("%w: payload attestation validation backlog full", ErrIgnore) + } + c.mu.Lock() + if current, ok := c.builds[blockRoot]; ok { + c.mu.Unlock() + <-c.pendingBuilds + validationContext, err, retry := waitForPayloadAttestationValidationContext(ctx, current) + if retry && ctx.Err() == nil { + continue + } + return validationContext, err + } + current := &payloadAttestationValidationContextBuild{done: make(chan struct{})} + c.builds[blockRoot] = current + c.mu.Unlock() + return c.build(ctx, blockRoot, current, build) + } +} + +func (c *payloadAttestationValidationContexts) build(ctx context.Context, blockRoot common.Hash, current *payloadAttestationValidationContextBuild, build func() (*payloadAttestationValidationContext, error)) (*payloadAttestationValidationContext, error) { + defer func() { <-c.pendingBuilds }() + select { + case c.buildAdmission <- struct{}{}: + case <-ctx.Done(): + c.complete(blockRoot, current, nil, ctx.Err(), true) + return nil, ctx.Err() + } + var validationContext *payloadAttestationValidationContext + var err error + var panicValue any + func() { + defer func() { panicValue = recover() }() + validationContext, err = build() + }() + <-c.buildAdmission + if panicValue != nil { + err = fmt.Errorf("payload attestation validation context build panicked: %v", panicValue) + } + if err == nil { + c.cache.Add(blockRoot, validationContext) + } + c.complete(blockRoot, current, validationContext, err, false) + if panicValue != nil { + panic(panicValue) + } + return validationContext, err +} + +func waitForPayloadAttestationValidationContext(ctx context.Context, build *payloadAttestationValidationContextBuild) (*payloadAttestationValidationContext, error, bool) { + select { + case <-build.done: + return build.validationContext, build.err, build.retry + case <-ctx.Done(): + return nil, ctx.Err(), false + } +} + +func (c *payloadAttestationValidationContexts) complete(blockRoot common.Hash, build *payloadAttestationValidationContextBuild, validationContext *payloadAttestationValidationContext, err error, retry bool) { + c.mu.Lock() + build.validationContext = validationContext + build.err = err + build.retry = retry + if c.builds[blockRoot] == build { + delete(c.builds, blockRoot) + } + close(build.done) + c.mu.Unlock() +} + +func (f *ForkChoiceStore) payloadAttestationValidationContext( + ctx context.Context, + blockRoot common.Hash, + slot uint64, +) (*payloadAttestationValidationContext, error) { + return f.payloadAttestationContexts.get(ctx, blockRoot, func() (*payloadAttestationValidationContext, error) { + blockState, err := f.GetStateAtBlockRoot(blockRoot, true) + if err != nil { + return nil, err + } + if blockState == nil { + return nil, fmt.Errorf("%w: block state not found for root %v", ErrIgnore, blockRoot) + } + if slot != blockState.Slot() { + return nil, fmt.Errorf("%w: attestation slot %d does not match block slot %d", ErrIgnore, slot, blockState.Slot()) + } + + ptc, err := blockState.GetPTC(slot) + if err != nil { + return nil, err + } + if len(ptc) != int(blockState.BeaconConfig().PtcSize) { + return nil, fmt.Errorf("invalid PTC length %d, expected %d", len(ptc), blockState.BeaconConfig().PtcSize) + } + domain, err := blockState.GetDomain(blockState.BeaconConfig().DomainPtcAttester, state.GetEpochAtSlot(blockState.BeaconConfig(), slot)) + if err != nil { + return nil, fmt.Errorf("unable to get the domain: %w", err) + } + if len(domain) != len(common.Hash{}) { + return nil, fmt.Errorf("invalid PTC attester domain length %d", len(domain)) + } + + validationContext := &payloadAttestationValidationContext{ + slot: slot, + positions: make(map[uint64][]int, len(ptc)), + publicKeys: make(map[uint64]common.Bytes48, len(ptc)), + } + copy(validationContext.domain[:], domain) + for position, validatorIndex := range ptc { + if validatorIndex >= uint64(blockState.ValidatorLength()) { + return nil, fmt.Errorf("PTC validator %d is out of range", validatorIndex) + } + validationContext.positions[validatorIndex] = append(validationContext.positions[validatorIndex], position) + if _, ok := validationContext.publicKeys[validatorIndex]; ok { + continue + } + validator, err := blockState.ValidatorForValidatorIndex(int(validatorIndex)) + if err != nil { + return nil, fmt.Errorf("failed to get PTC validator %d: %w", validatorIndex, err) + } + if len(validator.PublicKeyBytes()) != len(common.Bytes48{}) { + return nil, fmt.Errorf("invalid public key length for PTC validator %d", validatorIndex) + } + var publicKey common.Bytes48 + copy(publicKey[:], validator.PublicKeyBytes()) + validationContext.publicKeys[validatorIndex] = publicKey + } + return validationContext, nil + }) +} + +func (c *payloadAttestationValidationContext) ptcPositions(msg *cltypes.PayloadAttestationMessage) ([]int, error) { + if msg.Data.Slot != c.slot { + return nil, fmt.Errorf("%w: attestation slot %d does not match block slot %d", ErrIgnore, msg.Data.Slot, c.slot) + } + positions, ok := c.positions[msg.ValidatorIndex] + if !ok { + return nil, fmt.Errorf("validator %d is not in PTC for slot %d", msg.ValidatorIndex, msg.Data.Slot) + } + return positions, nil +} + +func (c *payloadAttestationValidationContext) validateSignature(msg *cltypes.PayloadAttestationMessage) error { + signingRoot, err := fork.ComputeSigningRoot(msg.Data, c.domain[:]) + if err != nil { + return fmt.Errorf("unable to get signing root: %w", err) + } + publicKey := c.publicKeys[msg.ValidatorIndex] + valid, err := bls.VerifyAggregate(msg.Signature[:], signingRoot[:], [][]byte{publicKey[:]}) + if err != nil { + return fmt.Errorf("error while validating signature: %w", err) + } + if !valid { + return errors.New("invalid payload attestation signature") + } + return nil +} diff --git a/cl/phase1/forkchoice/payload_attestation_validation_test.go b/cl/phase1/forkchoice/payload_attestation_validation_test.go new file mode 100644 index 00000000000..e87a37b6eaf --- /dev/null +++ b/cl/phase1/forkchoice/payload_attestation_validation_test.go @@ -0,0 +1,422 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package forkchoice + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" + "github.com/erigontech/erigon/cl/fork" + "github.com/erigontech/erigon/cl/utils/bls" + "github.com/erigontech/erigon/common" +) + +type payloadAttestationValidationContextResult struct { + validationContext *payloadAttestationValidationContext + err error +} + +type payloadAttestationObservedContext struct { + context.Context + observed chan struct{} + once sync.Once +} + +func newPayloadAttestationObservedContext(parent context.Context) *payloadAttestationObservedContext { + return &payloadAttestationObservedContext{Context: parent, observed: make(chan struct{})} +} + +func (c *payloadAttestationObservedContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.observed) }) + return c.Context.Done() +} + +func waitForPayloadAttestationWaiter(t *testing.T, ctx *payloadAttestationObservedContext) { + t.Helper() + select { + case <-ctx.observed: + case <-time.After(time.Second): + t.Fatal("payload attestation validation call did not reach its wait point") + } +} + +func TestOnPayloadAttestationMessageRejectsNil(t *testing.T) { + f := &ForkChoiceStore{} + require.Error(t, f.OnPayloadAttestationMessage(context.Background(), nil, false)) + require.Error(t, f.OnPayloadAttestationMessage(context.Background(), &cltypes.PayloadAttestationMessage{}, false)) +} + +func TestApplyValidatedPayloadAttestationAcceptsOnlyFirstGossipVote(t *testing.T) { + f := &ForkChoiceStore{} + root := common.HexToHash("0x1234") + first := &cltypes.PayloadAttestationData{PayloadPresent: true, BlobDataAvailable: true} + second := &cltypes.PayloadAttestationData{PayloadPresent: false, BlobDataAvailable: false} + + require.NoError(t, f.applyValidatedPayloadAttestation(42, []int{7}, first, root, false)) + require.ErrorIs(t, f.applyValidatedPayloadAttestation(42, []int{7}, second, common.HexToHash("0x5678"), false), ErrIgnore) + + timeliness := f.payloadTimelinessVoteValue(root) + availability := f.payloadDataAvailabilityVoteValue(root) + require.Equal(t, int8(1), timeliness[7]) + require.Equal(t, int8(1), availability[7]) +} + +func TestApplyValidatedPayloadAttestationFromBlockOverwritesGossipVote(t *testing.T) { + f := &ForkChoiceStore{} + root := common.HexToHash("0x1234") + + require.NoError(t, f.applyValidatedPayloadAttestation(42, []int{7}, &cltypes.PayloadAttestationData{ + PayloadPresent: true, BlobDataAvailable: true, + }, root, false)) + require.NoError(t, f.applyValidatedPayloadAttestation(42, []int{7}, &cltypes.PayloadAttestationData{}, root, true)) + + timeliness := f.payloadTimelinessVoteValue(root) + availability := f.payloadDataAvailabilityVoteValue(root) + require.Equal(t, int8(-1), timeliness[7]) + require.Equal(t, int8(-1), availability[7]) +} + +func TestApplyValidatedPayloadAttestationResetsFirstValidAtNextSlot(t *testing.T) { + f := &ForkChoiceStore{} + data := &cltypes.PayloadAttestationData{Slot: 100, PayloadPresent: true} + + require.NoError(t, f.applyValidatedPayloadAttestation(42, []int{7}, data, common.HexToHash("0x1234"), false)) + data.Slot++ + require.NoError(t, f.applyValidatedPayloadAttestation(42, []int{7}, data, common.HexToHash("0x5678"), false)) + require.Len(t, f.payloadAttestationSeen, 1) +} + +func TestApplyValidatedPayloadAttestationDoesNotRegressSeenSlot(t *testing.T) { + f := &ForkChoiceStore{} + root := common.HexToHash("0x1234") + + require.NoError(t, f.applyValidatedPayloadAttestation(42, []int{7}, &cltypes.PayloadAttestationData{Slot: 101}, root, false)) + require.ErrorIs(t, f.applyValidatedPayloadAttestation(43, []int{8}, &cltypes.PayloadAttestationData{Slot: 100}, root, false), ErrIgnore) + require.Equal(t, uint64(101), f.payloadAttestationSeenSlot) +} + +func TestApplyValidatedPayloadAttestationConcurrentCandidatesHaveOneWinner(t *testing.T) { + f := &ForkChoiceStore{} + start := make(chan struct{}) + results := make(chan error, 16) + + for i := range 16 { + go func() { + <-start + data := &cltypes.PayloadAttestationData{Slot: 100, PayloadPresent: i%2 == 0} + results <- f.applyValidatedPayloadAttestation(42, []int{7}, data, common.Hash{byte(i + 1)}, false) + }() + } + close(start) + + var successes int + for range 16 { + err := <-results + if err == nil { + successes++ + } else { + require.ErrorIs(t, err, ErrIgnore) + } + } + require.Equal(t, 1, successes) +} + +func (f *ForkChoiceStore) payloadTimelinessVoteValue(root common.Hash) [clparams.PtcSize]int8 { + votes, _ := f.payloadTimelinessVote.Load(root) + return votes.([clparams.PtcSize]int8) +} + +func (f *ForkChoiceStore) payloadDataAvailabilityVoteValue(root common.Hash) [clparams.PtcSize]int8 { + votes, _ := f.payloadDataAvailabilityVote.Load(root) + return votes.([clparams.PtcSize]int8) +} + +func TestPayloadAttestationValidationContextsCollapseConcurrentBuilds(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + root := common.HexToHash("0x1234") + expected := &payloadAttestationValidationContext{slot: 100} + started := make(chan struct{}) + release := make(chan struct{}) + var builds atomic.Int32 + + build := func() (*payloadAttestationValidationContext, error) { + if builds.Add(1) == 1 { + close(started) + } + <-release + return expected, nil + } + + results := make(chan payloadAttestationValidationContextResult, 16) + go func() { + validationContext, getErr := contexts.get(context.Background(), root, build) + results <- payloadAttestationValidationContextResult{validationContext, getErr} + }() + <-started + for range 15 { + waiterCtx := newPayloadAttestationObservedContext(context.Background()) + go func() { + validationContext, getErr := contexts.get(waiterCtx, root, build) + results <- payloadAttestationValidationContextResult{validationContext, getErr} + }() + waitForPayloadAttestationWaiter(t, waiterCtx) + } + require.Equal(t, int32(1), builds.Load()) + close(release) + for range 16 { + result := <-results + require.NoError(t, result.err) + require.Same(t, expected, result.validationContext) + } + _, err = contexts.get(context.Background(), root, build) + require.NoError(t, err) + require.Equal(t, int32(1), builds.Load()) +} + +func TestPayloadAttestationValidationContextsCanceledWaiterDoesNotCancelBuild(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + root := common.HexToHash("0x1234") + started := make(chan struct{}) + release := make(chan struct{}) + expected := &payloadAttestationValidationContext{slot: 100} + leader := make(chan payloadAttestationValidationContextResult, 1) + go func() { + value, getErr := contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { + close(started) + <-release + return expected, nil + }) + leader <- payloadAttestationValidationContextResult{value, getErr} + }() + <-started + waiterCtx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = contexts.get(waiterCtx, root, func() (*payloadAttestationValidationContext, error) { return nil, nil }) + require.ErrorIs(t, err, context.Canceled) + close(release) + result := <-leader + require.NoError(t, result.err) + require.Same(t, expected, result.validationContext) +} + +func TestPayloadAttestationValidationContextsCanceledLeaderDoesNotPoisonWaiter(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + contexts.buildAdmission <- struct{}{} + root := common.HexToHash("0x1234") + leaderParentCtx, cancelLeader := context.WithCancel(context.Background()) + leaderCtx := newPayloadAttestationObservedContext(leaderParentCtx) + leaderResult := make(chan error, 1) + go func() { + _, getErr := contexts.get(leaderCtx, root, func() (*payloadAttestationValidationContext, error) { + return nil, errors.New("leader build must not run") + }) + leaderResult <- getErr + }() + waitForPayloadAttestationWaiter(t, leaderCtx) + + expected := &payloadAttestationValidationContext{slot: 100} + waiterResult := make(chan payloadAttestationValidationContextResult, 1) + waiterCtx := newPayloadAttestationObservedContext(context.Background()) + go func() { + value, getErr := contexts.get(waiterCtx, root, func() (*payloadAttestationValidationContext, error) { return expected, nil }) + waiterResult <- payloadAttestationValidationContextResult{value, getErr} + }() + waitForPayloadAttestationWaiter(t, waiterCtx) + cancelLeader() + require.ErrorIs(t, <-leaderResult, context.Canceled) + <-contexts.buildAdmission + result := <-waiterResult + require.NoError(t, result.err) + require.Same(t, expected, result.validationContext) +} + +func TestPayloadAttestationValidationContextsPanicCleansInflightBuild(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + root := common.HexToHash("0x1234") + func() { + defer func() { require.Equal(t, "boom", recover()) }() + _, _ = contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { panic("boom") }) + }() + expected := &payloadAttestationValidationContext{slot: 100} + actual, err := contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { return expected, nil }) + require.NoError(t, err) + require.Same(t, expected, actual) +} + +func TestPayloadAttestationValidationContextsDoNotCacheBuildErrors(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + root := common.HexToHash("0x1234") + var builds atomic.Int32 + _, err = contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { + builds.Add(1) + return nil, errors.New("state unavailable") + }) + require.ErrorContains(t, err, "state unavailable") + + expected := &payloadAttestationValidationContext{slot: 100} + actual, err := contexts.get(context.Background(), root, func() (*payloadAttestationValidationContext, error) { + builds.Add(1) + return expected, nil + }) + require.NoError(t, err) + require.Same(t, expected, actual) + require.Equal(t, int32(2), builds.Load()) +} + +func TestPayloadAttestationValidationContextsBoundDifferentRootBuilds(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + + var active atomic.Int32 + var maxActive atomic.Int32 + started := make(chan struct{}, 3) + release := make(chan struct{}) + results := make(chan error, 3) + for i := range 3 { + go func() { + _, getErr := contexts.get(context.Background(), common.Hash{byte(i + 1)}, func() (*payloadAttestationValidationContext, error) { + current := active.Add(1) + defer active.Add(-1) + for { + maximum := maxActive.Load() + if current <= maximum || maxActive.CompareAndSwap(maximum, current) { + break + } + } + started <- struct{}{} + <-release + return &payloadAttestationValidationContext{}, nil + }) + results <- getErr + }() + } + + for range maxConcurrentValidationContextBuilds { + select { + case <-started: + case <-time.After(time.Second): + require.FailNow(t, "validation context build did not start") + } + } + select { + case <-started: + require.FailNow(t, "too many validation contexts built concurrently") + case <-time.After(100 * time.Millisecond): + } + close(release) + for range 3 { + require.NoError(t, <-results) + } + require.Equal(t, int32(maxConcurrentValidationContextBuilds), maxActive.Load()) +} + +func TestPayloadAttestationValidationContextsBoundPendingRoots(t *testing.T) { + contexts, err := newPayloadAttestationValidationContexts() + require.NoError(t, err) + contexts.buildAdmission <- struct{}{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + results := make(chan error, maxPendingValidationContextBuilds+1) + for i := range maxPendingValidationContextBuilds + 1 { + go func() { + _, getErr := contexts.get(ctx, common.Hash{byte(i), byte(i >> 8)}, func() (*payloadAttestationValidationContext, error) { + return &payloadAttestationValidationContext{}, nil + }) + results <- getErr + }() + } + + require.Eventually(t, func() bool { + contexts.mu.Lock() + defer contexts.mu.Unlock() + return len(contexts.builds) == maxPendingValidationContextBuilds + }, time.Second, time.Millisecond) + require.ErrorIs(t, <-results, ErrIgnore) + cancel() + <-contexts.buildAdmission + for range maxPendingValidationContextBuilds { + <-results + } +} + +func TestPayloadAttestationValidationContextPositions(t *testing.T) { + validationContext := &payloadAttestationValidationContext{ + slot: 100, + positions: map[uint64][]int{42: {1, 7}}, + } + msg := &cltypes.PayloadAttestationMessage{ + ValidatorIndex: 42, + Data: &cltypes.PayloadAttestationData{Slot: 100}, + } + + positions, err := validationContext.ptcPositions(msg) + require.NoError(t, err) + require.Equal(t, []int{1, 7}, positions) + + msg.Data.Slot = 99 + _, err = validationContext.ptcPositions(msg) + require.ErrorIs(t, err, ErrIgnore) +} + +func TestPayloadAttestationValidationContextSignature(t *testing.T) { + privateKey, err := bls.GenerateKey() + require.NoError(t, err) + + data := &cltypes.PayloadAttestationData{ + BeaconBlockRoot: common.HexToHash("0x1234"), + Slot: 100, + PayloadPresent: true, + BlobDataAvailable: true, + } + domain := common.HexToHash("0xabcd") + signingRoot, err := fork.ComputeSigningRoot(data, domain[:]) + require.NoError(t, err) + + var publicKey common.Bytes48 + copy(publicKey[:], bls.CompressPublicKey(privateKey.PublicKey())) + msg := &cltypes.PayloadAttestationMessage{ + ValidatorIndex: 42, + Data: data, + } + copy(msg.Signature[:], privateKey.Sign(signingRoot[:]).Bytes()) + validationContext := &payloadAttestationValidationContext{ + slot: 100, + domain: domain, + positions: map[uint64][]int{42: {1}}, + publicKeys: map[uint64]common.Bytes48{42: publicKey}, + } + + require.NoError(t, validationContext.validateSignature(msg)) + msg.Data.PayloadPresent = false + require.ErrorContains(t, validationContext.validateSignature(msg), "invalid payload attestation signature") +} diff --git a/cl/phase1/forkchoice/payload_vote_test.go b/cl/phase1/forkchoice/payload_vote_test.go index 5443359aadd..c8a9adffb14 100644 --- a/cl/phase1/forkchoice/payload_vote_test.go +++ b/cl/phase1/forkchoice/payload_vote_test.go @@ -42,6 +42,10 @@ func (g payloadVoteForkGraph) HasEnvelope(common.Hash) bool { return g.hasEnvelope } +func (g payloadVoteForkGraph) ReadEnvelopeFromDisk(common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error) { + return nil, nil +} + func (g payloadVoteForkGraph) DumpEnvelopeOnDisk(blockRoot common.Hash, _ *cltypes.SignedExecutionPayloadEnvelope) error { if g.dumpedEnvelope != nil { *g.dumpedEnvelope = blockRoot diff --git a/cl/phase1/forkchoice/pending_el_payload_test.go b/cl/phase1/forkchoice/pending_el_payload_test.go index e4c5c9dd996..ec55fe6d27c 100644 --- a/cl/phase1/forkchoice/pending_el_payload_test.go +++ b/cl/phase1/forkchoice/pending_el_payload_test.go @@ -35,6 +35,24 @@ func TestDrainPendingELPayloadsReleasesLargeBackingArray(t *testing.T) { require.Nil(t, f.pendingELPayloads) } +func TestDrainPendingELPayloadsLimitPreservesRemainingOrder(t *testing.T) { + f := &ForkChoiceStore{} + for i := range 5 { + f.addPendingELPayload(&cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Slot: uint64(i + 1)}, + }, nil) + } + + first := f.DrainPendingELPayloadsLimit(2) + require.Equal(t, []uint64{1, 2}, []uint64{first[0].Block.Block.Slot, first[1].Block.Block.Slot}) + remaining := f.DrainPendingELPayloads() + require.Equal(t, []uint64{3, 4, 5}, []uint64{ + remaining[0].Block.Block.Slot, + remaining[1].Block.Block.Slot, + remaining[2].Block.Block.Slot, + }) +} + func TestPendingELPayloadsDeduplicateByEnvelopeRoot(t *testing.T) { f := &ForkChoiceStore{} root := common.HexToHash("0x1234") diff --git a/cl/phase1/network/services/payload_attestation_service.go b/cl/phase1/network/services/payload_attestation_service.go index 50b4134fb17..20296ea4ef7 100644 --- a/cl/phase1/network/services/payload_attestation_service.go +++ b/cl/phase1/network/services/payload_attestation_service.go @@ -61,10 +61,11 @@ const ( // seenPayloadAttestationCacheSize: PTC has 512 validators per slot. // With clock disparity, we may see attestations for ~2 slots. // 512 * 4 = 2048 provides safety margin. - seenPayloadAttestationCacheSize = 2048 - pendingPayloadAttestationExpiry = 30 * time.Second - pendingPayloadAttestationCheckInterval = 100 * time.Millisecond - maxPendingAttestations = 2048 + seenPayloadAttestationCacheSize = 2048 + pendingPayloadAttestationExpiry = 30 * time.Second + pendingPayloadAttestationCheckInterval = 100 * time.Millisecond + maxPendingAttestations = 2048 + maxConcurrentPayloadAttestationValidations = clparams.PtcSize ) type payloadAttestationService struct { @@ -80,6 +81,7 @@ type payloadAttestationService struct { pendingAttestations sync.Map // pendingPayloadAttestationKey -> *pendingPayloadAttestationJob pendingCount atomic.Int32 pendingCond *sync.Cond + validationAdmission chan struct{} } // NewPayloadAttestationService creates a new payload attestation service. @@ -102,6 +104,7 @@ func NewPayloadAttestationService( emitters: emitters, seenAttestationsCache: seenCache, pendingCond: sync.NewCond(&sync.Mutex{}), + validationAdmission: make(chan struct{}, maxConcurrentPayloadAttestationValidations), } go s.loop(ctx) return s @@ -166,15 +169,21 @@ func (s *payloadAttestationService) ProcessMessage(ctx context.Context, _ *uint6 if blockHeader.Slot != slot { return fmt.Errorf("%w: payload attestation slot %d does not match referenced block slot %d", ErrIgnore, slot, blockHeader.Slot) } + select { + case s.validationAdmission <- struct{}{}: + defer func() { <-s.validationAdmission }() + case <-ctx.Done(): + return fmt.Errorf("%w: payload attestation validation canceled: %v", ErrIgnore, ctx.Err()) //nolint:errorlint // converting cancellation to IGNORE + } // Process through forkchoice which handles: // [IGNORE] block state not found // [REJECT] validator is not in PTC // [REJECT] signature verification - if err := s.forkchoiceStore.OnPayloadAttestationMessage(msg, false); err != nil { + if err := s.forkchoiceStore.OnPayloadAttestationMessage(ctx, msg, false); err != nil { // Preserve IGNORE vs REJECT distinction from forkchoice // forkchoice.ErrIgnore != services.ErrIgnore, so we need to convert - if errors.Is(err, forkchoice.ErrIgnore) { + if errors.Is(err, forkchoice.ErrIgnore) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return fmt.Errorf("%w: %v", ErrIgnore, err) //nolint:errorlint // converting, not wrapping: forkchoice.ErrIgnore must not stay matchable } return fmt.Errorf("forkchoice rejected payload attestation: %w", err) diff --git a/cl/phase1/network/services/payload_attestation_service_test.go b/cl/phase1/network/services/payload_attestation_service_test.go index 1b487d3bd5e..3b6384aa246 100644 --- a/cl/phase1/network/services/payload_attestation_service_test.go +++ b/cl/phase1/network/services/payload_attestation_service_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "sync" + "sync/atomic" "testing" "time" @@ -30,11 +31,70 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/phase1/core/state/lru" + "github.com/erigontech/erigon/cl/phase1/forkchoice" "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" "github.com/erigontech/erigon/cl/utils/eth_clock" "github.com/erigontech/erigon/common" ) +type blockingPayloadAttestationForkchoice struct { + forkchoice.ForkChoiceStorage + active atomic.Int32 + max atomic.Int32 + started chan struct{} + release chan struct{} +} + +func (f *blockingPayloadAttestationForkchoice) OnPayloadAttestationMessage(ctx context.Context, _ *cltypes.PayloadAttestationMessage, _ bool) error { + active := f.active.Add(1) + defer f.active.Add(-1) + for { + maxActive := f.max.Load() + if active <= maxActive || f.max.CompareAndSwap(maxActive, active) { + break + } + } + f.started <- struct{}{} + select { + case <-f.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +type retryPayloadAttestationForkchoice struct { + forkchoice.ForkChoiceStorage + calls atomic.Int32 + firstStarted chan struct{} + releaseFirst chan struct{} +} + +type candidatePayloadAttestationForkchoice struct { + forkchoice.ForkChoiceStorage + started chan byte + release chan struct{} +} + +func (f *candidatePayloadAttestationForkchoice) OnPayloadAttestationMessage(_ context.Context, msg *cltypes.PayloadAttestationMessage, _ bool) error { + candidate := msg.Signature[0] + f.started <- candidate + <-f.release + if candidate != 3 { + return errors.New("invalid signature") + } + return nil +} + +func (f *retryPayloadAttestationForkchoice) OnPayloadAttestationMessage(context.Context, *cltypes.PayloadAttestationMessage, bool) error { + if f.calls.Add(1) == 1 { + close(f.firstStarted) + <-f.releaseFirst + return errors.New("invalid signature") + } + return nil +} + func setupPayloadAttestationService(t *testing.T, ctrl *gomock.Controller) (*payloadAttestationService, *mock_services.ForkChoiceStorageMock, *eth_clock.MockEthereumClock) { forkchoiceMock := mock_services.NewForkChoiceStorageMock(t) ethClockMock := eth_clock.NewMockEthereumClock(ctrl) @@ -49,11 +109,276 @@ func setupPayloadAttestationService(t *testing.T, ctrl *gomock.Controller) (*pay seenAttestationsCache: seenCache, emitters: beaconevents.NewEventEmitter(), pendingCond: sync.NewCond(&sync.Mutex{}), // Needed for queuePendingAttestation + validationAdmission: make(chan struct{}, maxConcurrentPayloadAttestationValidations), } return service, forkchoiceMock, ethClockMock } +func TestPayloadAttestationServiceBoundsKnownBlockValidation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + service.validationAdmission = make(chan struct{}, 1) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + blockingForkchoice := &blockingPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + firstDone := make(chan error, 1) + go func() { + firstDone <- service.ProcessMessage(context.Background(), nil, &cltypes.PayloadAttestationMessage{ + ValidatorIndex: 1, + Data: &cltypes.PayloadAttestationData{Slot: 100, BeaconBlockRoot: blockRoot}, + }) + }() + <-blockingForkchoice.started + + secondCtx, cancelSecond := context.WithCancel(context.Background()) + cancelSecond() + err := service.ProcessMessage(secondCtx, nil, &cltypes.PayloadAttestationMessage{ + ValidatorIndex: 2, + Data: &cltypes.PayloadAttestationData{Slot: 100, BeaconBlockRoot: blockRoot}, + }) + require.ErrorIs(t, err, ErrIgnore) + close(blockingForkchoice.release) + require.NoError(t, <-firstDone) +} + +func TestPayloadAttestationServiceBackpressuresInsteadOfDroppingValidCandidate(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + service.validationAdmission = make(chan struct{}, 1) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + validations := &candidatePayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan byte, 2), + release: make(chan struct{}), + } + service.forkchoiceStore = validations + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + invalid := newTestPayloadAttestationMessage(100, 1, blockRoot) + invalid.Signature[0] = 1 + firstResult := make(chan error, 1) + go func() { firstResult <- service.ProcessMessage(context.Background(), nil, invalid) }() + require.Equal(t, byte(1), <-validations.started) + + valid := newTestPayloadAttestationMessage(100, 1, blockRoot) + valid.Signature[0] = 3 + secondResult := make(chan error, 1) + go func() { secondResult <- service.ProcessMessage(context.Background(), nil, valid) }() + select { + case err := <-secondResult: + close(validations.release) + t.Fatalf("valid candidate returned before admission was released: %v", err) + case <-time.After(20 * time.Millisecond): + } + + close(validations.release) + require.Error(t, <-firstResult) + require.NoError(t, <-secondResult) +} + +func TestPayloadAttestationServiceDoesNotDropValidCandidateBehindInvalidCandidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + blockingForkchoice := &candidatePayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan byte, 3), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(3) + + results := make(chan error, 3) + for candidate := byte(1); candidate <= 3; candidate++ { + msg := newTestPayloadAttestationMessage(100, 1, blockRoot) + msg.Signature[0] = candidate + go func() { results <- service.ProcessMessage(context.Background(), nil, msg) }() + if candidate == 1 { + require.Equal(t, byte(1), <-blockingForkchoice.started) + } + } + for range 2 { + select { + case <-blockingForkchoice.started: + case <-time.After(time.Second): + close(blockingForkchoice.release) + t.Fatal("candidate was dropped before validation") + } + } + close(blockingForkchoice.release) + + var successes int + for range 3 { + if <-results == nil { + successes++ + } + } + require.Equal(t, 1, successes) +} + +func TestPayloadAttestationServiceAllowsConcurrentValidationForDifferentValidators(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + + blockingForkchoice := &blockingPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan struct{}, 8), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(8) + + var wg sync.WaitGroup + results := make(chan error, 8) + for i := range 8 { + wg.Go(func() { + msg := newTestPayloadAttestationMessage(100, uint64(i), blockRoot) + results <- service.ProcessMessage(context.Background(), nil, msg) + }) + } + + for range 8 { + select { + case <-blockingForkchoice.started: + case <-time.After(time.Second): + close(blockingForkchoice.release) + wg.Wait() + require.FailNow(t, "validation was throttled") + } + } + close(blockingForkchoice.release) + wg.Wait() + close(results) + for err := range results { + require.NoError(t, err) + } + require.Equal(t, int32(8), blockingForkchoice.max.Load()) +} + +func TestPayloadAttestationServiceAllowsConcurrentValidationForSameValidator(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + + blockingForkchoice := &blockingPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan struct{}, 2), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + results := make(chan error, 2) + for range 2 { + go func() { + results <- service.ProcessMessage(context.Background(), nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + }() + } + + <-blockingForkchoice.started + select { + case <-blockingForkchoice.started: + case <-time.After(time.Second): + close(blockingForkchoice.release) + t.Fatal("second candidate did not reach validation") + } + close(blockingForkchoice.release) + + require.NoError(t, <-results) + require.NoError(t, <-results) + require.Equal(t, int32(2), blockingForkchoice.max.Load()) +} + +func TestPayloadAttestationServiceRetriesAfterInvalidDuplicate(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + + retryForkchoice := &retryPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + firstStarted: make(chan struct{}), + releaseFirst: make(chan struct{}), + } + service.forkchoiceStore = retryForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + firstResult := make(chan error, 1) + go func() { + firstResult <- service.ProcessMessage(context.Background(), nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + }() + <-retryForkchoice.firstStarted + + secondResult := make(chan error, 1) + go func() { + second := newTestPayloadAttestationMessage(100, 42, blockRoot) + second.Signature[0] = 1 + secondResult <- service.ProcessMessage(context.Background(), nil, second) + }() + + require.Eventually(t, func() bool { return retryForkchoice.calls.Load() == 2 }, time.Second, time.Millisecond) + close(retryForkchoice.releaseFirst) + + require.ErrorContains(t, <-firstResult, "invalid signature") + require.NoError(t, <-secondResult) + require.Equal(t, int32(2), retryForkchoice.calls.Load()) +} + +func TestPayloadAttestationServiceIgnoresCanceledDuplicateWaiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + service, fcu, ethClockMock := setupPayloadAttestationService(t, ctrl) + blockRoot := common.HexToHash("0x1234") + fcu.Headers[blockRoot] = &cltypes.BeaconBlockHeader{Slot: 100} + blockingForkchoice := &blockingPayloadAttestationForkchoice{ + ForkChoiceStorage: fcu, + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + service.forkchoiceStore = blockingForkchoice + ethClockMock.EXPECT().IsSlotCurrentSlotWithMaximumClockDisparity(uint64(100)).Return(true).Times(2) + + firstResult := make(chan error, 1) + go func() { + firstResult <- service.ProcessMessage(context.Background(), nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + }() + <-blockingForkchoice.started + + waiterCtx, cancel := context.WithCancel(context.Background()) + cancel() + err := service.ProcessMessage(waiterCtx, nil, newTestPayloadAttestationMessage(100, 42, blockRoot)) + require.ErrorIs(t, err, ErrIgnore) + + close(blockingForkchoice.release) + require.NoError(t, <-firstResult) +} + func newTestPayloadAttestationMessage(slot uint64, validatorIndex uint64, blockRoot common.Hash) *cltypes.PayloadAttestationMessage { return &cltypes.PayloadAttestationMessage{ ValidatorIndex: validatorIndex, diff --git a/cl/phase1/stages/chain_tip_sync.go b/cl/phase1/stages/chain_tip_sync.go index 02dffddf278..ee9b2ad2e61 100644 --- a/cl/phase1/stages/chain_tip_sync.go +++ b/cl/phase1/stages/chain_tip_sync.go @@ -12,6 +12,7 @@ import ( "github.com/erigontech/erigon/cl/clparams" "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" + "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice" network "github.com/erigontech/erigon/cl/phase1/network" @@ -22,7 +23,12 @@ import ( "github.com/erigontech/erigon/common/log/v3" ) -const maxGloasVerificationSweepPerCycle = 32 +const ( + maxGloasVerificationSweepPerCycle = 32 + maxGloasAncestorVisitsPerCycle = 32 + maxPendingGloasPayloadsPerCycle = 32 + gloasPayloadRetryBudget = 2 * time.Second +) func gloasVersionedHashes(blobCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) ([]common.Hash, error) { if blobCommitments == nil || blobCommitments.Len() == 0 { @@ -60,8 +66,8 @@ func gloasEnvelopePayloadHash(envelope *cltypes.SignedExecutionPayloadEnvelope) return envelope.Message.Payload.BlockHash, true } -func canRetryGloasPayloads(cfg *Cfg) bool { - return cfg.executionClient != nil && cfg.executionClient.SupportInsertion() +func canValidateGloasPayloads(cfg *Cfg) bool { + return cfg.executionClient != nil } // waitForExecutionEngineToBeFinished checks if the execution engine is ready within a specified timeout. @@ -259,7 +265,7 @@ MainLoop: if block.Version() >= clparams.GloasVersion && len(envelopes) > 0 { parentRoot := block.Block.ParentRoot if env, ok := envelopes[common.Hash(parentRoot)]; ok { - if envErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, canRetryGloasPayloads(cfg)); envErr != nil { + if envErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, canValidateGloasPayloads(cfg)); envErr != nil { log.Debug("[chainTipSync] failed to apply parent envelope", "slot", block.Block.Slot, "err", envErr) } } @@ -296,7 +302,7 @@ func fetchAndApplyEnvelopes(ctx context.Context, cfg *Cfg, roots [][32]byte) { return } for _, env := range envelopes { - if err := cfg.forkChoice.OnExecutionPayload(ctx, env, true, canRetryGloasPayloads(cfg)); err != nil { + if err := cfg.forkChoice.OnExecutionPayload(ctx, env, true, canValidateGloasPayloads(cfg)); err != nil { log.Debug("[chainTipSync] failed to apply recovered GLOAS envelope", "beaconBlockRoot", env.Message.BeaconBlockRoot, "err", err) } } @@ -392,17 +398,14 @@ func fetchParentEnvelopes(ctx context.Context, cfg *Cfg, roots [][32]byte) map[c return envelopes } -// recoverMissingEnvelopes walks backwards from the highest-seen block root +// recoverMissingEnvelopes walks backwards from the selected fork-choice head // and fetches execution payload envelopes for FULL GLOAS blocks that are missing them. // It walks all the way back to the finalized slot (or the GLOAS boundary) so that // non-contiguous gaps are recovered — a nearer FULL-with-envelope does not guarantee // that all earlier parents also have their envelopes. -// -// Uses HighestSeenRoot (O(1) gossip-tracked tip) instead of GetHead to avoid -// running the full fork choice traversal, which is expensive for large trees. func recoverMissingEnvelopes(ctx context.Context, cfg *Cfg) { - headRoot := cfg.forkChoice.HighestSeenRoot() - if headRoot == (common.Hash{}) { + headRoot, err := gloasVerificationHeadRoot(cfg.forkChoice) + if err != nil || headRoot == (common.Hash{}) { return } @@ -417,36 +420,70 @@ func recoverMissingEnvelopes(ctx context.Context, cfg *Cfg) { return } - var missingRoots [][32]byte - - // Also check the head block itself — if its envelope is missing, fork choice - // can only offer EMPTY, causing the next builder to build on the wrong path. - if !cfg.forkChoice.HasEnvelope(headRoot) { - missingRoots = append(missingRoots, [32]byte(headRoot)) - } + missingRoots := make([][32]byte, 0, maxGloasAncestorVisitsPerCycle) finalizedSlot := cfg.forkChoice.FinalizedSlot() + var directExtensionParent common.Hash + if cfg.gloasEnvelopeRecoveryHead != (common.Hash{}) && cfg.gloasEnvelopeRecoveryHead != headRoot { + newHeadBlock, ok := cfg.forkChoice.GetBlock(headRoot) + directExtension := ok && common.Hash(newHeadBlock.Block.ParentRoot) == cfg.gloasEnvelopeRecoveryHead + if directExtension { + directExtensionParent = cfg.gloasEnvelopeRecoveryHead + oldHeadBlock, oldOK := cfg.forkChoice.GetBlock(cfg.gloasEnvelopeRecoveryHead) + newBid := newHeadBlock.Block.Body.GetSignedExecutionPayloadBid() + if oldOK && oldHeadBlock != nil && newBid != nil && newBid.Message != nil { + oldBid := oldHeadBlock.Block.Body.GetSignedExecutionPayloadBid() + if oldBid != nil && oldBid.Message != nil && newBid.Message.ParentBlockHash == oldBid.Message.BlockHash && !cfg.forkChoice.HasEnvelope(cfg.gloasEnvelopeRecoveryHead) { + missingRoots = append(missingRoots, [32]byte(cfg.gloasEnvelopeRecoveryHead)) + } + } + } + if !directExtension { + cfg.gloasEnvelopeRecoveryCursor = common.Hash{} + } + } + cfg.gloasEnvelopeRecoveryHead = headRoot + scanRoot := cfg.gloasEnvelopeRecoveryCursor + if scanRoot == (common.Hash{}) { + if directExtensionParent != (common.Hash{}) { + scanRoot = directExtensionParent + if cursorBlock, ok := cfg.forkChoice.GetBlock(scanRoot); ok { + childBlock = cursorBlock + } + } else { + scanRoot = headRoot + } + } else if cursorBlock, cursorOK := cfg.forkChoice.GetBlock(scanRoot); cursorOK { + childBlock = cursorBlock + } else { + scanRoot = headRoot + } - for { + completedScan := false + for visited := 1; visited < maxGloasAncestorVisitsPerCycle; visited++ { parentRoot := childBlock.Block.ParentRoot parentBlock, ok := cfg.forkChoice.GetBlock(parentRoot) if !ok { + completedScan = true break } if parentBlock.Block.Slot <= finalizedSlot { + completedScan = true break } parentEpoch := parentBlock.Block.Slot / cfg.beaconCfg.SlotsPerEpoch if cfg.beaconCfg.GetCurrentStateVersion(parentEpoch) < clparams.GloasVersion { + completedScan = true break } childBid := childBlock.Block.Body.GetSignedExecutionPayloadBid() parentBid := parentBlock.Block.Body.GetSignedExecutionPayloadBid() + childBlock = parentBlock + scanRoot = common.Hash(parentRoot) if childBid == nil || childBid.Message == nil || parentBid == nil || parentBid.Message == nil { - childBlock = parentBlock continue } @@ -456,19 +493,62 @@ func recoverMissingEnvelopes(ctx context.Context, cfg *Cfg) { missingRoots = append(missingRoots, parentRoot) } } - childBlock = parentBlock } - if len(missingRoots) > 0 { log.Info("[chainTipSync] envelope recovery: fetching missing envelopes", "count", len(missingRoots)) fetchAndApplyEnvelopes(ctx, cfg, missingRoots) } + advanceGloasEnvelopeRecoveryCursor(cfg, scanRoot, completedScan) +} + +func advanceGloasEnvelopeRecoveryCursor(cfg *Cfg, scanRoot common.Hash, completedScan bool) { + if completedScan { + cfg.gloasEnvelopeRecoveryCursor = common.Hash{} + } else { + cfg.gloasEnvelopeRecoveryCursor = scanRoot + } } -// pollForEnvelope polls HasEnvelope until the envelope arrives or the timeout expires. -func pollForEnvelope(ctx context.Context, cfg *Cfg, headRoot common.Hash, timeout time.Duration) { +type selectedHeadEnvelopeStore interface { + HasEnvelope(common.Hash) bool + OnExecutionPayload(context.Context, *cltypes.SignedExecutionPayloadEnvelope, bool, bool) error +} + +type envelopeRequestFunc func(context.Context, [][32]byte) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, error) + +func waitForSelectedHeadEnvelope( + ctx context.Context, + store selectedHeadEnvelopeStore, + requestEnvelopes envelopeRequestFunc, + headRoot common.Hash, + timeout time.Duration, + requestFromPeer bool, + validatePayload bool, +) { pollCtx, pollCancel := context.WithTimeout(ctx, timeout) defer pollCancel() + if store.HasEnvelope(headRoot) { + return + } + if requestFromPeer { + go func() { + envelopes, err := requestEnvelopes(pollCtx, [][32]byte{headRoot}) + if err != nil { + log.Debug("[chainTipSync] failed to request selected head envelope", "headRoot", headRoot, "err", err) + return + } + envelope := envelopes[headRoot] + if envelope == nil { + return + } + if pollCtx.Err() != nil { + return + } + if err := store.OnExecutionPayload(pollCtx, envelope, true, validatePayload); err != nil { + log.Debug("[chainTipSync] failed to apply selected head envelope", "headRoot", headRoot, "err", err) + } + }() + } ticker := time.NewTicker(50 * time.Millisecond) defer ticker.Stop() @@ -478,13 +558,45 @@ func pollForEnvelope(ctx context.Context, cfg *Cfg, headRoot common.Hash, timeou case <-pollCtx.Done(): return case <-ticker.C: - if cfg.forkChoice.HasEnvelope(headRoot) { + if store.HasEnvelope(headRoot) { return } } } } +type selectedHeadEnvelopeRequestClaim struct { + root common.Hash + id uint64 +} + +func claimSelectedHeadEnvelopeRequest(cfg *Cfg, headRoot common.Hash) (selectedHeadEnvelopeRequestClaim, bool) { + cfg.gloasHeadEnvelopeRequestMu.Lock() + defer cfg.gloasHeadEnvelopeRequestMu.Unlock() + if _, ok := cfg.gloasHeadEnvelopeRequests[headRoot]; ok { + return selectedHeadEnvelopeRequestClaim{}, false + } + if cfg.gloasHeadEnvelopeRequests == nil { + cfg.gloasHeadEnvelopeRequests = make(map[common.Hash]uint64) + } + cfg.gloasHeadEnvelopeRequestID++ + id := cfg.gloasHeadEnvelopeRequestID + cfg.gloasHeadEnvelopeRequests[headRoot] = id + return selectedHeadEnvelopeRequestClaim{root: headRoot, id: id}, true +} + +func releaseSelectedHeadEnvelopeRequest(cfg *Cfg, claim selectedHeadEnvelopeRequestClaim) { + cfg.gloasHeadEnvelopeRequestMu.Lock() + defer cfg.gloasHeadEnvelopeRequestMu.Unlock() + if cfg.gloasHeadEnvelopeRequests[claim.root] == claim.id { + delete(cfg.gloasHeadEnvelopeRequests, claim.root) + } +} + +func blockSupportsExecutionPayloadEnvelope(block *cltypes.SignedBeaconBlock) bool { + return block != nil && block.Block != nil && block.Version() >= clparams.GloasVersion +} + func buildGloasNewPayloadArgs(cfg *Cfg, block *cltypes.SignedBeaconBlock, envelope *cltypes.SignedExecutionPayloadEnvelope) ([]common.Hash, []hexutil.Bytes, error) { if block == nil || block.Block == nil { return nil, nil, errors.New("missing beacon block") @@ -524,7 +636,14 @@ func isGloasPayloadKnownInvalid(cfg *Cfg, envelope *cltypes.SignedExecutionPaylo } func drainPendingGloasPayloads(ctx context.Context, cfg *Cfg) { - for _, p := range cfg.forkChoice.DrainPendingELPayloads() { + pending := cfg.forkChoice.DrainPendingELPayloadsLimit(maxPendingGloasPayloadsPerCycle) + for i, p := range pending { + if ctx.Err() != nil { + for _, deferred := range pending[i:] { + cfg.forkChoice.RequeuePendingELPayload(deferred) + } + return + } if !validPendingGloasPayload(p) { continue } @@ -546,8 +665,21 @@ func drainPendingGloasPayloads(ctx context.Context, cfg *Cfg) { } } +type gloasHeadReader interface { + GetHead(*state.CachingBeaconState) (common.Hash, uint64, error) +} + +func gloasVerificationHeadRoot(forkChoice gloasHeadReader) (common.Hash, error) { + headRoot, _, err := forkChoice.GetHead(nil) + return headRoot, err +} + func verifyUnverifiedGloasPayloads(ctx context.Context, cfg *Cfg) { - headRoot := cfg.forkChoice.HighestSeenRoot() + headRoot, err := gloasVerificationHeadRoot(cfg.forkChoice) + if err != nil { + log.Debug("[chainTipSync] failed to select GLOAS verification head", "err", err) + return + } if headRoot == (common.Hash{}) { return } @@ -557,17 +689,77 @@ func verifyUnverifiedGloasPayloads(ctx context.Context, cfg *Cfg) { root common.Hash block *cltypes.SignedBeaconBlock } + var immediateHead *struct { + root common.Hash + block *cltypes.SignedBeaconBlock + } - for root := headRoot; root != (common.Hash{}); { + root := cfg.gloasVerificationCursor + var directExtensionParentRoot common.Hash + if cfg.gloasVerificationHead != (common.Hash{}) && cfg.gloasVerificationHead != headRoot { + newHeadBlock, ok := cfg.forkChoice.GetBlock(headRoot) + directExtension := ok && common.Hash(newHeadBlock.Block.ParentRoot) == cfg.gloasVerificationHead + if directExtension { + directExtensionParentRoot = cfg.gloasVerificationHead + oldHeadBlock, oldOK := cfg.forkChoice.GetBlock(cfg.gloasVerificationHead) + if oldOK && oldHeadBlock != nil && cfg.forkChoice.HasEnvelope(cfg.gloasVerificationHead) && !cfg.forkChoice.IsPayloadVerified(cfg.gloasVerificationHead) { + immediateHead = &struct { + root common.Hash + block *cltypes.SignedBeaconBlock + }{root: cfg.gloasVerificationHead, block: oldHeadBlock} + } + } + if !directExtension { + root = headRoot + cfg.gloasVerificationCursor = common.Hash{} + } + } + cfg.gloasVerificationHead = headRoot + visitLimit := maxGloasAncestorVisitsPerCycle + if immediateHead != nil { + visitLimit-- + } + var selectedHead *struct { + root common.Hash + block *cltypes.SignedBeaconBlock + } + if root == (common.Hash{}) { + if directExtensionParentRoot != (common.Hash{}) { + oldHeadBlock, ok := cfg.forkChoice.GetBlock(directExtensionParentRoot) + if ok && oldHeadBlock != nil { + root = common.Hash(oldHeadBlock.Block.ParentRoot) + } else { + root = headRoot + } + } else { + root = headRoot + } + } else if _, ok := cfg.forkChoice.GetBlock(root); !ok { + root = headRoot + } else if root != headRoot { + headBlock, headOK := cfg.forkChoice.GetBlock(headRoot) + if headOK && headBlock != nil && cfg.forkChoice.HasEnvelope(headRoot) && !cfg.forkChoice.IsPayloadVerified(headRoot) { + selectedHead = &struct { + root common.Hash + block *cltypes.SignedBeaconBlock + }{root: headRoot, block: headBlock} + } + visitLimit-- + } + completedScan := false + for visited := 0; root != (common.Hash{}) && visited < visitLimit; visited++ { block, ok := cfg.forkChoice.GetBlock(root) if !ok || block == nil { + completedScan = true break } if block.Block.Slot <= finalizedSlot { + completedScan = true break } epoch := block.Block.Slot / cfg.beaconCfg.SlotsPerEpoch if cfg.beaconCfg.GetCurrentStateVersion(epoch) < clparams.GloasVersion { + completedScan = true break } if cfg.forkChoice.HasEnvelope(root) && !cfg.forkChoice.IsPayloadVerified(root) { @@ -575,31 +767,34 @@ func verifyUnverifiedGloasPayloads(ctx context.Context, cfg *Cfg) { root common.Hash block *cltypes.SignedBeaconBlock }{root: root, block: block}) - if len(blocks) >= maxGloasVerificationSweepPerCycle { - break - } } root = common.Hash(block.Block.ParentRoot) } - swept := 0 - for _, item := range slices.Backward(blocks) { + completeBatch := true + processItem := func(item struct { + root common.Hash + block *cltypes.SignedBeaconBlock + }) bool { + if ctx.Err() != nil { + return false + } if cfg.forkChoice.IsPayloadVerified(item.root) { - continue + return true } envelope, err := cfg.forkChoice.ReadEnvelopeFromDisk(item.root) if err != nil { log.Debug("[chainTipSync] failed to read GLOAS envelope for verification sweep", "slot", item.block.Block.Slot, "blockRoot", item.root, "err", err) - continue + return false } execHash, ok := gloasEnvelopePayloadHash(envelope) if !ok { log.Warn("[chainTipSync] missing GLOAS envelope payload during verification sweep", "slot", item.block.Block.Slot, "blockRoot", item.root) - continue + return false } if isGloasPayloadKnownInvalid(cfg, envelope) { cfg.forkChoice.MarkPayloadInvalid(item.root, execHash) - continue + return true } status, err := retryGloasPayloadWithEL(ctx, cfg, item.block, envelope) if err != nil { @@ -615,6 +810,26 @@ func verifyUnverifiedGloasPayloads(ctx context.Context, cfg *Cfg) { log.Warn("[chainTipSync] GLOAS verification sweep found invalid payload", "slot", item.block.Block.Slot, "blockRoot", item.root) } swept++ + return true + } + if selectedHead != nil { + completeBatch = processItem(*selectedHead) + } + if completeBatch && immediateHead != nil { + completeBatch = processItem(*immediateHead) + } + for _, item := range slices.Backward(blocks) { + if !processItem(item) { + completeBatch = false + break + } + } + if completeBatch { + if completedScan || root == (common.Hash{}) { + cfg.gloasVerificationCursor = common.Hash{} + } else { + cfg.gloasVerificationCursor = root + } } if swept > 0 || len(blocks) >= maxGloasVerificationSweepPerCycle { log.Info("[chainTipSync] GLOAS verification sweep", "swept", swept, "queued", len(blocks), "limit", maxGloasVerificationSweepPerCycle) @@ -622,6 +837,9 @@ func verifyUnverifiedGloasPayloads(ctx context.Context, cfg *Cfg) { } func retryUnverifiedAnchorPayload(ctx context.Context, cfg *Cfg) { + if ctx.Err() != nil { + return + } anchorSlot := cfg.forkChoice.AnchorSlot() epoch := anchorSlot / cfg.beaconCfg.SlotsPerEpoch if cfg.beaconCfg.GetCurrentStateVersion(epoch) < clparams.GloasVersion { @@ -666,6 +884,20 @@ func retryUnverifiedAnchorPayload(ctx context.Context, cfg *Cfg) { } } +func runGloasPayloadRetryPhases(ctx context.Context, budget time.Duration, offset uint32, phases ...func(context.Context)) { + if len(phases) == 0 { + return + } + parentCtx, cancelParent := context.WithTimeout(ctx, budget) + defer cancelParent() + for i := range phases { + if parentCtx.Err() != nil { + return + } + phases[(int(offset)+i)%len(phases)](parentCtx) + } +} + // chainTipSync synchronizes the chain tip by fetching blocks from the highest seen block up to the target slot by listening to incoming blocks. // or by fetching blocks that might have been missed by gossip after a delay. func chainTipSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) error { @@ -676,13 +908,23 @@ func chainTipSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) e // insertion — so it must run regardless of SupportInsertion(). recoverMissingEnvelopes(ctx, cfg) - if canRetryGloasPayloads(cfg) { - // [New in Gloas:EIP7732] Drain execution blocks whose CL transition succeeded - // but whose EL newPayload previously returned SYNCING/ACCEPTED. - drainPendingGloasPayloads(ctx, cfg) - retryUnverifiedAnchorPayload(ctx, cfg) - if err := cfg.blockCollector.Flush(context.Background()); err != nil { - log.Warn("[chainTipSync] blockCollector.Flush failed (EL may still be catching up)", "err", err) + if canValidateGloasPayloads(cfg) { + offset := cfg.gloasPayloadRetryOffset.Add(1) - 1 + runGloasPayloadRetryPhases(ctx, gloasPayloadRetryBudget, offset, + func(retryCtx context.Context) { + cfg.forkChoice.RetryPendingExecutionPayloadEnvelopes(retryCtx, maxPendingGloasPayloadsPerCycle) + }, + func(retryCtx context.Context) { + drainPendingGloasPayloads(retryCtx, cfg) + }, + func(retryCtx context.Context) { + retryUnverifiedAnchorPayload(retryCtx, cfg) + }, + ) + if cfg.executionClient.SupportInsertion() { + if err := cfg.blockCollector.Flush(context.Background()); err != nil { + log.Warn("[chainTipSync] blockCollector.Flush failed (EL may still be catching up)", "err", err) + } } } @@ -692,12 +934,22 @@ func chainTipSync(ctx context.Context, logger log.Logger, cfg *Cfg, args Args) e // may still be in-flight. Without this, FCU sends the parent's execution hash. headEpoch := args.targetSlot / cfg.beaconCfg.SlotsPerEpoch if cfg.beaconCfg.GetCurrentStateVersion(headEpoch) >= clparams.GloasVersion { - headRoot := cfg.forkChoice.HighestSeenRoot() - if headRoot != (common.Hash{}) && !cfg.forkChoice.HasEnvelope(headRoot) { - pollForEnvelope(ctx, cfg, headRoot, 2*time.Second) + headRoot, _, err := cfg.forkChoice.GetHead(nil) + if err != nil { + return err + } + headBlock, _ := cfg.forkChoice.GetBlock(headRoot) + if headRoot != (common.Hash{}) && blockSupportsExecutionPayloadEnvelope(headBlock) && !cfg.forkChoice.HasEnvelope(headRoot) { + requestClaim, requestFromPeer := claimSelectedHeadEnvelopeRequest(cfg, headRoot) + waitForSelectedHeadEnvelope(ctx, cfg.forkChoice, func(requestCtx context.Context, roots [][32]byte) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, error) { + defer releaseSelectedHeadEnvelopeRequest(cfg, requestClaim) + return network.RequestEnvelopesFrantically(requestCtx, cfg.rpc, roots) + }, headRoot, 2*time.Second, requestFromPeer, canValidateGloasPayloads(cfg)) } - if canRetryGloasPayloads(cfg) { - verifyUnverifiedGloasPayloads(ctx, cfg) + if canValidateGloasPayloads(cfg) { + verifyCtx, cancelVerify := context.WithTimeout(ctx, gloasPayloadRetryBudget) + verifyUnverifiedGloasPayloads(verifyCtx, cfg) + cancelVerify() } // NOTE: recoverMissingEnvelopes runs unconditionally above (before // SupportInsertion check), so it covers every cycle. diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index 84746b66d00..9badd0b6ab0 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -20,6 +20,8 @@ import ( "context" "errors" "fmt" + "sync" + "sync/atomic" "time" "github.com/erigontech/erigon/cl/antiquary" @@ -48,26 +50,34 @@ import ( ) type Cfg struct { - rpc *rpc.BeaconRpcP2P - ethClock eth_clock.EthereumClock - beaconCfg *clparams.BeaconChainConfig - executionClient execution_client.ExecutionEngine - state *state.CachingBeaconState - forkChoice *forkchoice.ForkChoiceStore - indiciesDB kv.RwDB - dirs datadir.Dirs - blockReader freezeblocks.BeaconSnapshotReader - antiquary *antiquary.Antiquary - syncedData *synced_data.SyncedDataManager - emitter *beaconevents.EventEmitter - blockCollector block_collector.BlockCollector - sn *freezeblocks.CaplinSnapshots - blobStore blob_storage.BlobStorage - peerDas das.PeerDas - blobDownloader *network2.BlobHistoryDownloader - attestationDataProducer attestation_producer.AttestationDataProducer - caplinConfig clparams.CaplinConfig - hasDownloaded bool + rpc *rpc.BeaconRpcP2P + ethClock eth_clock.EthereumClock + beaconCfg *clparams.BeaconChainConfig + executionClient execution_client.ExecutionEngine + state *state.CachingBeaconState + forkChoice *forkchoice.ForkChoiceStore + indiciesDB kv.RwDB + dirs datadir.Dirs + blockReader freezeblocks.BeaconSnapshotReader + antiquary *antiquary.Antiquary + syncedData *synced_data.SyncedDataManager + emitter *beaconevents.EventEmitter + blockCollector block_collector.BlockCollector + sn *freezeblocks.CaplinSnapshots + blobStore blob_storage.BlobStorage + peerDas das.PeerDas + blobDownloader *network2.BlobHistoryDownloader + attestationDataProducer attestation_producer.AttestationDataProducer + caplinConfig clparams.CaplinConfig + hasDownloaded bool + gloasPayloadRetryOffset atomic.Uint32 + gloasEnvelopeRecoveryCursor common.Hash + gloasEnvelopeRecoveryHead common.Hash + gloasHeadEnvelopeRequestMu sync.Mutex + gloasHeadEnvelopeRequestID uint64 + gloasHeadEnvelopeRequests map[common.Hash]uint64 + gloasVerificationCursor common.Hash + gloasVerificationHead common.Hash } type Args struct { diff --git a/cl/phase1/stages/finalized_state_save_test.go b/cl/phase1/stages/finalized_state_save_test.go index 09c3738e3fe..4766b3c0f28 100644 --- a/cl/phase1/stages/finalized_state_save_test.go +++ b/cl/phase1/stages/finalized_state_save_test.go @@ -17,6 +17,8 @@ package stages import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -24,11 +26,15 @@ import ( "github.com/erigontech/erigon/cl/antiquary/tests" "github.com/erigontech/erigon/cl/clparams" + "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/solid" "github.com/erigontech/erigon/cl/phase1/core/checkpoint_sync" + "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" + "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/datadir" + "github.com/spf13/afero" ) func TestWriteFinalizedStateFile_RoundTrip(t *testing.T) { @@ -36,6 +42,9 @@ func TestWriteFinalizedStateFile_RoundTrip(t *testing.T) { dirs := datadir.New(t.TempDir()) require.NoError(t, writeFinalizedStateFile(dirs, st)) + rootFiles, err := filepath.Glob(filepath.Join(dirs.CaplinLatest, ".finalized-state-root-*")) + require.NoError(t, err) + require.Empty(t, rootFiles) got, err := checkpoint_sync.ReadLocalFinalizedState(dirs, &clparams.MainnetBeaconConfig) require.NoError(t, err) @@ -48,6 +57,92 @@ func TestWriteFinalizedStateFile_RoundTrip(t *testing.T) { assert.Equal(t, wantRoot, gotRoot) } +func TestWriteFinalizedStateFile_RestoresGloasStateRoot(t *testing.T) { + beaconCfg, st := newGloasFinalizedState() + authoritativeRoot := common.Hash{0xaa} + st.SetPreviousStateRoot(authoritativeRoot) + wantBlockRoot, err := st.BlockRoot() + require.NoError(t, err) + dirs := datadir.New(t.TempDir()) + + require.NoError(t, writeFinalizedStateFile(dirs, st)) + got, err := checkpoint_sync.ReadLocalFinalizedState(dirs, beaconCfg) + require.NoError(t, err) + require.Equal(t, authoritativeRoot, got.PeekPreviousStateRoot()) + gotBlockRoot, err := got.BlockRoot() + require.NoError(t, err) + require.Equal(t, wantBlockRoot, gotBlockRoot) +} + +func TestReadFinalizedGloasStateWithoutRootRejectsUnsafeResume(t *testing.T) { + beaconCfg, st := newGloasFinalizedState() + dirs := datadir.New(t.TempDir()) + require.NoError(t, os.MkdirAll(dirs.CaplinLatest, 0o755)) + encoded, err := utils.EncodeSSZSnappy(st) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dirs.CaplinLatest, clparams.LatestFinalizedStateFileName), encoded, 0o644)) + + genesis := state.New(beaconCfg) + got, err := checkpoint_sync.NewLocalCheckpointSyncer(genesis, afero.NewBasePathFs(afero.NewOsFs(), dirs.CaplinLatest)).GetLatestBeaconState(t.Context()) + require.NoError(t, err) + require.Equal(t, genesis.Slot(), got.Slot()) +} + +func TestReadFinalizedGloasStateRejectsCorruptRootRecord(t *testing.T) { + beaconCfg, st := newGloasFinalizedState() + dirs := datadir.New(t.TempDir()) + require.NoError(t, writeFinalizedStateFile(dirs, st)) + encoded, err := os.ReadFile(filepath.Join(dirs.CaplinLatest, clparams.LatestFinalizedStateFileName)) + require.NoError(t, err) + rootPath := filepath.Join(dirs.CaplinLatest, checkpoint_sync.FinalizedStateRootFileName(encoded)) + record, err := os.ReadFile(rootPath) + require.NoError(t, err) + record[0] ^= 1 + require.NoError(t, os.WriteFile(rootPath, record, 0o644)) + + _, err = checkpoint_sync.ReadLocalFinalizedState(dirs, beaconCfg) + require.ErrorContains(t, err, "invalid finalized state root checksum") +} + +func TestReadFinalizedStateRejectsRootRecordFromDifferentState(t *testing.T) { + beaconCfg, stateA := newGloasFinalizedState() + dirsA := datadir.New(t.TempDir()) + dirsB := datadir.New(t.TempDir()) + stateB, err := stateA.Copy() + require.NoError(t, err) + stateB.SetSlot(stateA.Slot() + 1) + stateA.SetPreviousStateRoot(common.Hash{1}) + stateB.SetPreviousStateRoot(common.Hash{2}) + require.NoError(t, writeFinalizedStateFile(dirsA, stateA)) + require.NoError(t, writeFinalizedStateFile(dirsB, stateB)) + encodedA, err := os.ReadFile(filepath.Join(dirsA.CaplinLatest, clparams.LatestFinalizedStateFileName)) + require.NoError(t, err) + encodedB, err := os.ReadFile(filepath.Join(dirsB.CaplinLatest, clparams.LatestFinalizedStateFileName)) + require.NoError(t, err) + recordB, err := os.ReadFile(filepath.Join(dirsB.CaplinLatest, checkpoint_sync.FinalizedStateRootFileName(encodedB))) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dirsA.CaplinLatest, checkpoint_sync.FinalizedStateRootFileName(encodedA)), recordB, 0o644)) + + _, err = checkpoint_sync.ReadLocalFinalizedState(dirsA, beaconCfg) + require.ErrorContains(t, err, "invalid finalized state root checksum") +} + +func newGloasFinalizedState() (*clparams.BeaconChainConfig, *state.CachingBeaconState) { + beaconCfg := clparams.MainnetBeaconConfig + beaconCfg.AltairForkEpoch = 0 + beaconCfg.BellatrixForkEpoch = 0 + beaconCfg.CapellaForkEpoch = 0 + beaconCfg.DenebForkEpoch = 0 + beaconCfg.ElectraForkEpoch = 0 + beaconCfg.FuluForkEpoch = 0 + beaconCfg.GloasForkEpoch = 0 + st := state.New(&beaconCfg) + st.SetVersion(clparams.GloasVersion) + st.SetSlot(64) + st.SetLatestBlockHeader(&cltypes.BeaconBlockHeader{Slot: 64}) + return &beaconCfg, st +} + func TestSaveFinalizedStateOnDisk(t *testing.T) { _, st, _ := tests.GetPhase0Random() beaconCfg := &clparams.MainnetBeaconConfig diff --git a/cl/phase1/stages/forkchoice.go b/cl/phase1/stages/forkchoice.go index b8651cbe1e2..e263243f1dc 100644 --- a/cl/phase1/stages/forkchoice.go +++ b/cl/phase1/stages/forkchoice.go @@ -19,6 +19,7 @@ import ( "github.com/erigontech/erigon/cl/monitor/shuffling_metrics" "github.com/erigontech/erigon/cl/persistence/beacon_indicies" "github.com/erigontech/erigon/cl/phase1/core/caches" + "github.com/erigontech/erigon/cl/phase1/core/checkpoint_sync" "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/core/state/shuffling" "github.com/erigontech/erigon/cl/phase1/forkchoice" @@ -297,27 +298,71 @@ func writeFinalizedStateFile(dirs datadir.Dirs, st *state.CachingBeaconState) er if err := os.MkdirAll(dirs.CaplinLatest, 0o755); err != nil { return fmt.Errorf("failed to create directory: %w", err) } + var rootPath string + if st.Version() >= clparams.GloasVersion { + rootFileName := checkpoint_sync.FinalizedStateRootFileName(dat) + rootPath = filepath.Join(dirs.CaplinLatest, rootFileName) + rootTmpPath := rootPath + ".tmp" + stateRoot := st.PeekPreviousStateRoot() + if err := replaceDurableFile(rootTmpPath, rootPath, checkpoint_sync.EncodeFinalizedStateRoot(dat, stateRoot)); err != nil { + return fmt.Errorf("failed to replace finalized state root: %w", err) + } + } tmpName := filepath.Join(dirs.CaplinLatest, clparams.LatestFinalizedStateFileName+".tmp") - tmp, err := os.OpenFile(tmpName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + statePath := filepath.Join(dirs.CaplinLatest, clparams.LatestFinalizedStateFileName) + if err := replaceDurableFile(tmpName, statePath, dat); err != nil { + return fmt.Errorf("failed to replace finalized state: %w", err) + } + if rootPath != "" { + if err := checkpoint_sync.RemoveObsoleteFinalizedStateRoots(dirs.CaplinLatest, rootPath); err != nil { + return fmt.Errorf("failed to remove obsolete finalized state roots: %w", err) + } + if err := syncDirectory(dirs.CaplinLatest); err != nil { + return fmt.Errorf("failed to sync finalized state root cleanup: %w", err) + } + } + return nil +} + +func replaceDurableFile(tmpPath, finalPath string, data []byte) (err error) { + tmp, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { - return fmt.Errorf("failed to create temp finalized state file: %w", err) + return err + } + closed := false + defer func() { + if !closed { + err = errors.Join(err, tmp.Close()) + } + _ = dir.RemoveFile(tmpPath) + }() + if _, err = tmp.Write(data); err != nil { + return err } - defer func() { _ = dir.RemoveFile(tmpName) }() - if _, err := tmp.Write(dat); err != nil { - tmp.Close() - return fmt.Errorf("failed to write finalized state to disk: %w", err) + if syncErr := tmp.Sync(); syncErr != nil { + return syncErr } - if err := tmp.Sync(); err != nil { - tmp.Close() - return fmt.Errorf("failed to sync finalized state temp file: %w", err) + closeErr := tmp.Close() + closed = true + if closeErr != nil { + return closeErr } - if err := tmp.Close(); err != nil { - return fmt.Errorf("failed to close finalized state temp file: %w", err) + if renameErr := os.Rename(tmpPath, finalPath); renameErr != nil { + return renameErr } - if err := os.Rename(tmpName, filepath.Join(dirs.CaplinLatest, clparams.LatestFinalizedStateFileName)); err != nil { - return fmt.Errorf("failed to rename finalized state file: %w", err) + return syncDirectory(filepath.Dir(finalPath)) +} + +func syncDirectory(path string) error { + if runtime.GOOS == "windows" { + return nil } - return nil + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() } // saveFinalizedStateOnDiskIfNeeded persists the node's own most-recently-finalized state so a diff --git a/cl/phase1/stages/forward_sync.go b/cl/phase1/stages/forward_sync.go index d9a22b0ae09..e6e1f8c8c0b 100644 --- a/cl/phase1/stages/forward_sync.go +++ b/cl/phase1/stages/forward_sync.go @@ -119,7 +119,7 @@ func processDownloadedBlockBatches(ctx context.Context, logger log.Logger, cfg * if block.Version() >= clparams.GloasVersion { if env, ok := envelopes[blockRoot]; ok { // FULL block: update forkchoice with the envelope (updates eth2Roots, persists to disk). - if fceErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, false); fceErr != nil { + if fceErr := cfg.forkChoice.OnExecutionPayload(ctx, env, false, shouldValidateForwardSyncPayload(cfg, shouldInsert)); fceErr != nil { logger.Warn("[Caplin] forward sync: failed to process GLOAS envelope", "slot", block.Block.Slot, "err", fceErr) } else if shouldInsert { if err = cfg.blockCollector.AddGloasBlock(block.Block, env); err != nil { @@ -176,6 +176,10 @@ func processDownloadedBlockBatches(ctx context.Context, logger log.Logger, cfg * return } +func shouldValidateForwardSyncPayload(cfg *Cfg, shouldInsert bool) bool { + return !shouldInsert && canValidateGloasPayloads(cfg) +} + // forwardSyncProgress returns the slots still to sync and the observed sync rate // in slots/sec. currentSlot can overshoot chainTipSlot and dip below prevProgress // on reorgs, so both differences are clamped to keep the unsigned math from @@ -424,7 +428,7 @@ func ensureAnchorEnvelopeOnce(ctx context.Context, cfg *Cfg) error { if err := cfg.forkChoice.StoreAnchorEnvelope(anchorRoot, env); err != nil { return fmt.Errorf("failed to store anchor envelope: %w", err) } - if err := validateAnchorPayloadIfLocalEL(ctx, cfg, anchorRoot, bid, env); err != nil { + if err := validateAnchorPayloadWithExecutionClient(ctx, cfg, anchorRoot, bid, env); err != nil { return err } @@ -433,8 +437,8 @@ func ensureAnchorEnvelopeOnce(ctx context.Context, cfg *Cfg) error { return nil } -func validateAnchorPayloadIfLocalEL(ctx context.Context, cfg *Cfg, anchorRoot common.Hash, bid *cltypes.ExecutionPayloadBid, env *cltypes.SignedExecutionPayloadEnvelope) error { - if !canRetryGloasPayloads(cfg) { +func validateAnchorPayloadWithExecutionClient(ctx context.Context, cfg *Cfg, anchorRoot common.Hash, bid *cltypes.ExecutionPayloadBid, env *cltypes.SignedExecutionPayloadEnvelope) error { + if !canValidateGloasPayloads(cfg) { return nil } status, err := validateAnchorPayloadWithEL(ctx, cfg, bid, env) diff --git a/cl/phase1/stages/gloas_payload_test.go b/cl/phase1/stages/gloas_payload_test.go index 46f1bd88b74..73c6d6ec459 100644 --- a/cl/phase1/stages/gloas_payload_test.go +++ b/cl/phase1/stages/gloas_payload_test.go @@ -3,7 +3,9 @@ package stages import ( "context" "math/big" + "sync" "testing" + "time" "github.com/holiman/uint256" "github.com/stretchr/testify/require" @@ -15,6 +17,7 @@ import ( state2 "github.com/erigontech/erigon/cl/phase1/core/state" "github.com/erigontech/erigon/cl/phase1/execution_client" "github.com/erigontech/erigon/cl/phase1/forkchoice" + "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services" "github.com/erigontech/erigon/cl/utils" "github.com/erigontech/erigon/cl/utils/bls" "github.com/erigontech/erigon/common" @@ -26,6 +29,140 @@ import ( "github.com/erigontech/erigon/node/gointerfaces/typesproto" ) +type selectedHeadEnvelopeTestStore struct { + mu sync.Mutex + envelopes map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope +} + +func (s *selectedHeadEnvelopeTestStore) HasEnvelope(root common.Hash) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.envelopes[root] != nil +} + +func (s *selectedHeadEnvelopeTestStore) OnExecutionPayload(_ context.Context, envelope *cltypes.SignedExecutionPayloadEnvelope, _, _ bool) error { + s.mu.Lock() + defer s.mu.Unlock() + s.envelopes[envelope.Message.BeaconBlockRoot] = envelope + return nil +} + +func TestWaitForSelectedHeadEnvelopeRequestsAndAppliesPeerEnvelope(t *testing.T) { + headRoot := common.HexToHash("0x1234") + store := &selectedHeadEnvelopeTestStore{envelopes: make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope)} + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: headRoot}} + requests := make(chan [][32]byte, 1) + + waitForSelectedHeadEnvelope(context.Background(), store, func(_ context.Context, roots [][32]byte) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, error) { + requests <- roots + return map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{headRoot: envelope}, nil + }, headRoot, time.Second, true, false) + + require.Equal(t, [][32]byte{headRoot}, <-requests) + require.True(t, store.HasEnvelope(headRoot)) +} + +func TestSelectedHeadEnvelopeRequestCanRetryAfterCompletion(t *testing.T) { + cfg := &Cfg{} + firstHead := common.HexToHash("0x1234") + secondHead := common.HexToHash("0x5678") + + firstAttempt, ok := claimSelectedHeadEnvelopeRequest(cfg, firstHead) + require.True(t, ok) + _, ok = claimSelectedHeadEnvelopeRequest(cfg, firstHead) + require.False(t, ok) + releaseSelectedHeadEnvelopeRequest(cfg, firstAttempt) + secondAttempt, ok := claimSelectedHeadEnvelopeRequest(cfg, firstHead) + require.True(t, ok) + secondHeadAttempt, ok := claimSelectedHeadEnvelopeRequest(cfg, secondHead) + require.True(t, ok) + releaseSelectedHeadEnvelopeRequest(cfg, firstAttempt) + _, ok = claimSelectedHeadEnvelopeRequest(cfg, firstHead) + require.False(t, ok) + releaseSelectedHeadEnvelopeRequest(cfg, secondAttempt) + _, ok = claimSelectedHeadEnvelopeRequest(cfg, firstHead) + require.True(t, ok) + releaseSelectedHeadEnvelopeRequest(cfg, secondHeadAttempt) +} + +func TestGloasRecoveryCursorAdvancesAfterIncompleteFetch(t *testing.T) { + cfg := &Cfg{} + scanRoot := common.HexToHash("0x1234") + + advanceGloasEnvelopeRecoveryCursor(cfg, scanRoot, false) + require.Equal(t, scanRoot, cfg.gloasEnvelopeRecoveryCursor) + + advanceGloasEnvelopeRecoveryCursor(cfg, scanRoot, true) + require.Equal(t, common.Hash{}, cfg.gloasEnvelopeRecoveryCursor) +} + +func TestBlockSupportsExecutionPayloadEnvelopeUsesBlockVersion(t *testing.T) { + require.False(t, blockSupportsExecutionPayloadEnvelope(nil)) + require.False(t, blockSupportsExecutionPayloadEnvelope(&cltypes.SignedBeaconBlock{})) + require.False(t, blockSupportsExecutionPayloadEnvelope(&cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Body: &cltypes.BeaconBody{Version: clparams.FuluVersion}}, + })) + require.True(t, blockSupportsExecutionPayloadEnvelope(&cltypes.SignedBeaconBlock{ + Block: &cltypes.BeaconBlock{Body: &cltypes.BeaconBody{Version: clparams.GloasVersion}}, + })) +} + +func TestWaitForSelectedHeadEnvelopeDoesNotRequestUnclaimedHead(t *testing.T) { + store := &selectedHeadEnvelopeTestStore{envelopes: make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope)} + requests := make(chan struct{}, 1) + waitForSelectedHeadEnvelope(context.Background(), store, func(context.Context, [][32]byte) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, error) { + requests <- struct{}{} + return nil, nil + }, common.HexToHash("0x1234"), time.Millisecond, false, false) + + select { + case <-requests: + t.Fatal("unclaimed head triggered a peer request") + default: + } +} + +func TestWaitForSelectedHeadEnvelopeBoundsEmptyPeerResponse(t *testing.T) { + store := &selectedHeadEnvelopeTestStore{envelopes: make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope)} + requests := make(chan struct{}, 1) + waitForSelectedHeadEnvelope(context.Background(), store, func(requestCtx context.Context, _ [][32]byte) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, error) { + requests <- struct{}{} + <-requestCtx.Done() + return nil, requestCtx.Err() + }, common.HexToHash("0x1234"), 10*time.Millisecond, true, false) + + require.Len(t, requests, 1) +} + +func TestWaitForSelectedHeadEnvelopeDoesNotJoinLateRequester(t *testing.T) { + headRoot := common.HexToHash("0x1234") + store := &selectedHeadEnvelopeTestStore{envelopes: make(map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope)} + release := make(chan struct{}) + requestDone := make(chan struct{}) + returned := make(chan struct{}) + go func() { + waitForSelectedHeadEnvelope(context.Background(), store, func(context.Context, [][32]byte) (map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope, error) { + defer close(requestDone) + <-release + return map[common.Hash]*cltypes.SignedExecutionPayloadEnvelope{ + headRoot: {Message: &cltypes.ExecutionPayloadEnvelope{BeaconBlockRoot: headRoot}}, + }, nil + }, headRoot, 10*time.Millisecond, true, false) + close(returned) + }() + + select { + case <-returned: + case <-time.After(100 * time.Millisecond): + close(release) + <-returned + t.Fatal("selected-head wait joined a late requester") + } + close(release) + <-requestDone + require.False(t, store.HasEnvelope(headRoot)) +} + func TestValidateAnchorEnvelope(t *testing.T) { cfg, st, bid, env, anchorRoot := validAnchorEnvelopeFixture(t, 1) @@ -181,31 +318,49 @@ func TestGloasPayloadHelpers(t *testing.T) { require.Equal(t, want, hash) } -func TestStandaloneExecutionClientDoesNotRunLocalGloasRetry(t *testing.T) { - require.False(t, canRetryGloasPayloads(&Cfg{})) - require.False(t, canRetryGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: false}})) - require.True(t, canRetryGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: true}})) +func TestGloasVerificationHeadUsesForkChoiceHead(t *testing.T) { + want := common.HexToHash("0x22") + fc := &mock_services.ForkChoiceStorageMock{HeadVal: want} + + got, err := gloasVerificationHeadRoot(fc) + require.NoError(t, err) + require.Equal(t, want, got) +} + +func TestGloasPayloadValidationRequiresExecutionClient(t *testing.T) { + require.False(t, canValidateGloasPayloads(&Cfg{})) + require.True(t, canValidateGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: false}})) + require.True(t, canValidateGloasPayloads(&Cfg{executionClient: &testExecutionEngine{supportInsertion: true}})) } -func TestValidateAnchorPayloadIfLocalELFollowsSupportInsertion(t *testing.T) { +func TestForwardSyncValidatesOnlyWhenInsertionIsUnavailable(t *testing.T) { + localEL := &Cfg{executionClient: &testExecutionEngine{supportInsertion: true}} + remoteEL := &Cfg{executionClient: &testExecutionEngine{supportInsertion: false}} + + require.False(t, shouldValidateForwardSyncPayload(localEL, true)) + require.True(t, shouldValidateForwardSyncPayload(remoteEL, false)) + require.False(t, shouldValidateForwardSyncPayload(&Cfg{}, false)) +} + +func TestValidateAnchorPayloadWithAnyExecutionClient(t *testing.T) { cfg, _, bid, env, anchorRoot := validAnchorEnvelopeFixture(t, 1) remoteEL := &testExecutionEngine{ supportInsertion: false, - payloadStatus: execution_client.PayloadStatusInvalidated, + payloadStatus: execution_client.PayloadStatusValidated, } - require.NoError(t, validateAnchorPayloadIfLocalEL(context.Background(), &Cfg{ + require.NoError(t, validateAnchorPayloadWithExecutionClient(context.Background(), &Cfg{ beaconCfg: cfg, executionClient: remoteEL, forkChoice: &forkchoice.ForkChoiceStore{}, }, anchorRoot, bid, env)) - require.Equal(t, 0, remoteEL.newPayloadCalls) + require.Equal(t, 1, remoteEL.newPayloadCalls) localEL := &testExecutionEngine{ supportInsertion: true, payloadStatus: execution_client.PayloadStatusValidated, } - require.NoError(t, validateAnchorPayloadIfLocalEL(context.Background(), &Cfg{ + require.NoError(t, validateAnchorPayloadWithExecutionClient(context.Background(), &Cfg{ beaconCfg: cfg, executionClient: localEL, forkChoice: &forkchoice.ForkChoiceStore{}, @@ -262,6 +417,55 @@ func TestDrainPendingGloasPayloadsRequeuesNotValidatedPayload(t *testing.T) { require.Equal(t, blockRoot, queued[0].Envelope.Message.BeaconBlockRoot) } +func TestDrainPendingGloasPayloadsStopsAfterCancellation(t *testing.T) { + cfg := &clparams.MainnetBeaconConfig + engine := &testExecutionEngine{supportInsertion: false} + engine.newPayloadFn = func(ctx context.Context) (execution_client.PayloadStatus, error) { + <-ctx.Done() + return execution_client.PayloadStatusNone, ctx.Err() + } + fc := &forkchoice.ForkChoiceStore{} + for i := byte(1); i <= 3; i++ { + root := common.Hash{i} + payload := cltypes.NewEth1Block(clparams.GloasVersion, cfg) + payload.BlockHash = common.Hash{i + 10} + envelope := &cltypes.SignedExecutionPayloadEnvelope{Message: cltypes.NewExecutionPayloadEnvelope(cfg)} + envelope.Message.BeaconBlockRoot = root + envelope.Message.Payload = payload + body := cltypes.NewBeaconBody(cfg, clparams.GloasVersion) + body.SignedExecutionPayloadBid = &cltypes.SignedExecutionPayloadBid{Message: &cltypes.ExecutionPayloadBid{ + BlobKzgCommitments: *solid.NewStaticListSSZ[*cltypes.KZGCommitment](0, 48), + }} + fc.RequeuePendingELPayload(forkchoice.PendingELPayload{ + Block: &cltypes.SignedBeaconBlock{Block: &cltypes.BeaconBlock{Body: body}}, + Envelope: envelope, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + drainPendingGloasPayloads(ctx, &Cfg{beaconCfg: cfg, executionClient: engine, forkChoice: fc}) + + require.Equal(t, 1, engine.newPayloadCalls) + require.Len(t, fc.DrainPendingELPayloads(), 3) +} + +func TestGloasPayloadRetryPhasesRotateFirstClass(t *testing.T) { + calls := [3]int{} + for offset := range uint32(3) { + phases := make([]func(context.Context), 3) + for i := range phases { + index := i + phases[i] = func(ctx context.Context) { + calls[index]++ + <-ctx.Done() + } + } + runGloasPayloadRetryPhases(context.Background(), 20*time.Millisecond, offset, phases...) + } + require.Equal(t, [3]int{1, 1, 1}, calls) +} + func validAnchorEnvelopeFixture(t *testing.T, builderIndex uint64) (*clparams.BeaconChainConfig, *state2.CachingBeaconState, *cltypes.ExecutionPayloadBid, *cltypes.SignedExecutionPayloadEnvelope, common.Hash) { t.Helper() @@ -392,10 +596,14 @@ type testExecutionEngine struct { supportInsertion bool payloadStatus execution_client.PayloadStatus newPayloadCalls int + newPayloadFn func(context.Context) (execution_client.PayloadStatus, error) } -func (t *testExecutionEngine) NewPayload(context.Context, *cltypes.Eth1Block, *common.Hash, []common.Hash, []hexutil.Bytes) (execution_client.PayloadStatus, error) { +func (t *testExecutionEngine) NewPayload(ctx context.Context, _ *cltypes.Eth1Block, _ *common.Hash, _ []common.Hash, _ []hexutil.Bytes) (execution_client.PayloadStatus, error) { t.newPayloadCalls++ + if t.newPayloadFn != nil { + return t.newPayloadFn(ctx) + } return t.payloadStatus, nil } diff --git a/cl/spectest/consensus_tests/fork_choice.go b/cl/spectest/consensus_tests/fork_choice.go index 705748d3df6..e750bc2757f 100644 --- a/cl/spectest/consensus_tests/fork_choice.go +++ b/cl/spectest/consensus_tests/fork_choice.go @@ -312,7 +312,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err blobStorage := blob_storage.NewBlobStore(mdbxtest.New(t, "/tmp", dbcfg.ChainDB), afero.NewMemMapFs(), math.MaxUint64, &clparams.MainnetBeaconConfig, ethClock) columnStorage := blob_storage.NewDataColumnStore(afero.NewMemMapFs(), 1000, &clparams.MainnetBeaconConfig, ethClock, emitters) peerDasState := peerdasstate.NewPeerDasState(&clparams.MainnetBeaconConfig, &clparams.NetworkConfig{}) - peerDas := das.NewPeerDas(ctx, nil, &clparams.MainnetBeaconConfig, &clparams.CaplinConfig{}, columnStorage, blobStorage, nil, enode.ID{}, ethClock, peerDasState, nil, nil, nil) + peerDas := das.NewPeerDas(nil, &clparams.MainnetBeaconConfig, &clparams.CaplinConfig{}, columnStorage, blobStorage, nil, enode.ID{}, ethClock, peerDasState, nil, nil, nil) localValidators := validator_params.NewValidatorParams() forkStore, err := forkchoice.NewForkChoiceStore( diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index 6423b05788a..a232fb83590 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -466,7 +466,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi peerDasState.SetLocalNodeID(localNode) beaconRpc := rpc.NewBeaconRpcP2P(ctx, sentinel, beaconConfig, ethClock, state) gossipManager.SetPeerBanner(beaconRpc) - peerDas := das.NewPeerDas(ctx, beaconRpc, beaconConfig, &config, columnStorage, blobStorage, sentinel, localNode.ID(), ethClock, peerDasState, gossipManager, rcsn, indexDB) + peerDas := das.NewPeerDas(beaconRpc, beaconConfig, &config, columnStorage, blobStorage, sentinel, localNode.ID(), ethClock, peerDasState, gossipManager, rcsn, indexDB) forkChoice.InitPeerDas(peerDas) // hack init peerDas.SetForkChoice(forkChoice) // [New in Gloas:EIP7732] Set forkChoice for GLOAS kzg_commitments lookup committeeSub := committee_subscription.NewCommitteeSubscribeManagement(ctx, beaconConfig, networkConfig, ethClock, aggregationPool, syncedDataManager, gossipManager) @@ -507,6 +507,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi proposerPreferencesService, executionPayloadBidService, ) + peerDas.Start(ctx) { go batchSignatureVerifier.Start() diff --git a/execution/engineapi/engine_types/ssz.go b/execution/engineapi/engine_types/ssz.go index 3de43bfbf3d..34e92f3783d 100644 --- a/execution/engineapi/engine_types/ssz.go +++ b/execution/engineapi/engine_types/ssz.go @@ -162,6 +162,7 @@ func ExecutionPayloadFromSSZBlock(block *cltypes.Eth1Block, version clparams.Sta ExtraData: block.Extra.Bytes(), BaseFeePerGas: (*hexutil.Big)(baseFee.ToBig()), BlockHash: block.BlockHash, + Transactions: make([]hexutil.Bytes, 0, len(body.Transactions)), Withdrawals: body.Withdrawals, SSZVersion: version, }