diff --git a/cl/beacon/beaconhttp/api.go b/cl/beacon/beaconhttp/api.go
index 6213800b915..eb640bfcbcf 100644
--- a/cl/beacon/beaconhttp/api.go
+++ b/cl/beacon/beaconhttp/api.go
@@ -83,6 +83,7 @@ func (e EndpointError) Unwrap() error {
}
func (e *EndpointError) WriteTo(w http.ResponseWriter) {
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(e.Code)
encErr := json.NewEncoder(w).Encode(e)
if encErr != nil {
diff --git a/cl/beacon/beaconhttp/api_test.go b/cl/beacon/beaconhttp/api_test.go
index c224c25edfc..d998040bdb3 100644
--- a/cl/beacon/beaconhttp/api_test.go
+++ b/cl/beacon/beaconhttp/api_test.go
@@ -3,6 +3,7 @@ package beaconhttp
import (
"bytes"
"encoding/json"
+ "errors"
"net/http"
"net/http/httptest"
"testing"
@@ -109,6 +110,16 @@ func TestHandleEndpoint_RejectsSSZOnlyAcceptWhenResponseDoesNotSupportSSZ(t *tes
}
}
+func TestEndpointErrorWritesJSONContentType(t *testing.T) {
+ rr := httptest.NewRecorder()
+
+ NewEndpointError(http.StatusServiceUnavailable, errors.New("temporarily unavailable")).WriteTo(rr)
+
+ if got := rr.Header().Get("Content-Type"); got != "application/json" {
+ t.Fatalf("Content-Type = %q, want %q", got, "application/json")
+ }
+}
+
func TestHandleEndpoint_UsesSSZWhenExactTypeBeatsWildcard(t *testing.T) {
h := HandleEndpointFunc(func(w http.ResponseWriter, r *http.Request) (*BeaconResponse, error) {
return NewBeaconResponse(testSSZResponse{}), nil
diff --git a/cl/beacon/handler/blobs.go b/cl/beacon/handler/blobs.go
index f2f1e93d929..434148ac942 100644
--- a/cl/beacon/handler/blobs.go
+++ b/cl/beacon/handler/blobs.go
@@ -19,6 +19,7 @@ package handler
import (
"context"
"errors"
+ "fmt"
"net/http"
"strconv"
@@ -29,7 +30,7 @@ import (
"github.com/erigontech/erigon/cl/persistence/beacon_indicies"
"github.com/erigontech/erigon/cl/utils"
"github.com/erigontech/erigon/common"
- "github.com/erigontech/erigon/common/log/v3"
+ "github.com/erigontech/erigon/db/kv"
)
var blobSidecarSSZLenght = (*cltypes.BlobSidecar)(nil).EncodingSizeSSZ()
@@ -39,6 +40,11 @@ type caplinBlobSnapshotReader interface {
ReadBlobSidecars(slot uint64) ([]*cltypes.BlobSidecar, error)
}
+// BlobBackfillStatus reports whether canonical blob history is still incomplete at a slot.
+type BlobBackfillStatus interface {
+ BlobBackfillPending(slot uint64) bool
+}
+
func (a *ApiHandler) GetEthV1BeaconBlobSidecars(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
ctx := r.Context()
tx, err := a.indiciesDB.BeginRo(ctx)
@@ -68,11 +74,23 @@ func (a *ApiHandler) GetEthV1BeaconBlobSidecars(w http.ResponseWriter, r *http.R
return nil, err
}
- out, found, err := a.readBlobSidecars(ctx, *slot, blockRoot, canonicalRoot)
+ strIdxs, err := beaconhttp.StringListFromQueryParams(r, "indices")
if err != nil {
return nil, err
}
- strIdxs, err := beaconhttp.StringListFromQueryParams(r, "indices")
+ included := make(map[uint64]struct{}, len(strIdxs))
+ for _, idx := range strIdxs {
+ i, err := strconv.ParseUint(idx, 10, 64)
+ if err != nil {
+ return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err)
+ }
+ if _, duplicate := included[i]; duplicate {
+ return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, fmt.Errorf("duplicate blob index %d", i))
+ }
+ included[i] = struct{}{}
+ }
+
+ out, _, pending, err := a.readBlobSidecarsWithBackfillStatus(ctx, *slot, blockRoot, canonicalRoot)
if err != nil {
return nil, err
}
@@ -82,31 +100,27 @@ func (a *ApiHandler) GetEthV1BeaconBlobSidecars(w http.ResponseWriter, r *http.R
isFinalized := canonicalRoot == blockRoot && *slot <= a.forkchoiceStore.FinalizedSlot()
resp := solid.NewStaticListSSZ[*cltypes.BlobSidecar](696969, blobSidecarSSZLenght)
- if !found {
- return beaconhttp.NewBeaconResponse(resp).
- WithFinalized(isFinalized).
- WithVersion(version).
- WithOptimistic(isOptimistic), nil
- }
- if len(strIdxs) == 0 {
+ includeAll := len(strIdxs) == 0
+ if includeAll {
for _, v := range out {
resp.Append(v)
}
} else {
- included := make(map[uint64]struct{})
- for _, idx := range strIdxs {
- i, err := strconv.ParseUint(idx, 10, 64)
- if err != nil {
- return nil, err
- }
- included[i] = struct{}{}
- }
for _, v := range out {
if _, ok := included[v.Index]; ok {
resp.Append(v)
}
}
}
+ if pending {
+ missingExpected, err := a.requestedBlobSidecarsMissing(ctx, tx, *slot, included, includeAll, out)
+ if err != nil {
+ return nil, err
+ }
+ if missingExpected {
+ return nil, beaconhttp.NewEndpointError(http.StatusServiceUnavailable, errors.New("blob sidecars are still being backfilled"))
+ }
+ }
return beaconhttp.NewBeaconResponse(resp).
WithFinalized(isFinalized).
@@ -114,6 +128,54 @@ func (a *ApiHandler) GetEthV1BeaconBlobSidecars(w http.ResponseWriter, r *http.R
WithOptimistic(isOptimistic), nil
}
+func (a *ApiHandler) blobBackfillPending(slot uint64, blockRoot, canonicalRoot common.Hash) bool {
+ if a.blobBackfillStatus == nil || !a.blobBackfillStatus.BlobBackfillPending(slot) || blockRoot != canonicalRoot {
+ return false
+ }
+ if a.caplinSnapshots != nil && slot < a.caplinSnapshots.FrozenBlobs() {
+ return false
+ }
+ return true
+}
+
+func (a *ApiHandler) requestedBlobSidecarsMissing(ctx context.Context, tx kv.Tx, slot uint64, included map[uint64]struct{}, includeAll bool, available []*cltypes.BlobSidecar) (bool, error) {
+ block, err := a.blockReader.ReadBeaconBlockBodyBySlot(ctx, tx, slot)
+ if err != nil {
+ return false, err
+ }
+ if block == nil {
+ canonicalRoot, err := beacon_indicies.ReadCanonicalBlockRoot(tx, slot)
+ if err != nil {
+ return false, err
+ }
+ if canonicalRoot != (common.Hash{}) {
+ return false, errors.New("canonical block body is unavailable")
+ }
+ return false, nil
+ }
+ if block.Block == nil || block.Block.Body == nil {
+ return false, errors.New("block body is missing")
+ }
+ commitments := block.Block.Body.GetBlobKzgCommitments()
+ if commitments == nil || commitments.Len() == 0 {
+ return false, nil
+ }
+ availableIndices := make(map[uint64]struct{}, len(available))
+ for _, sidecar := range available {
+ availableIndices[sidecar.Index] = struct{}{}
+ }
+ if includeAll {
+ return len(availableIndices) < commitments.Len(), nil
+ }
+ for index := range included {
+ _, available := availableIndices[index]
+ if index < uint64(commitments.Len()) && !available {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
func (a *ApiHandler) readBlobSidecars(ctx context.Context, slot uint64, blockRoot, canonicalRoot common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
if blockRoot == canonicalRoot && a.caplinSnapshots != nil && slot < a.caplinSnapshots.FrozenBlobs() {
sidecars, err := a.caplinSnapshots.ReadBlobSidecars(slot)
@@ -122,7 +184,46 @@ func (a *ApiHandler) readBlobSidecars(ctx context.Context, slot uint64, blockRoo
}
return sidecars, len(sidecars) != 0, nil
}
- return a.blobStoage.ReadBlobSidecars(ctx, slot, blockRoot)
+ sidecars, complete, err := a.blobStoage.ReadBlobSidecars(ctx, slot, blockRoot)
+ if err != nil || complete || blockRoot != canonicalRoot || a.caplinSnapshots == nil || slot >= a.caplinSnapshots.FrozenBlobs() {
+ return sidecars, complete, err
+ }
+ snapshotSidecars, err := a.caplinSnapshots.ReadBlobSidecars(slot)
+ if err != nil || len(snapshotSidecars) == 0 {
+ return sidecars, complete, err
+ }
+ return snapshotSidecars, true, nil
+}
+
+func (a *ApiHandler) readBlobSidecarsWithBackfillStatus(ctx context.Context, slot uint64, blockRoot, canonicalRoot common.Hash) ([]*cltypes.BlobSidecar, bool, bool, error) {
+ pendingBefore := a.blobBackfillPending(slot, blockRoot, canonicalRoot)
+ sidecars, complete, err := a.readBlobSidecars(ctx, slot, blockRoot, canonicalRoot)
+ if err != nil {
+ return sidecars, complete, false, err
+ }
+ pendingAfter := a.blobBackfillPending(slot, blockRoot, canonicalRoot)
+ if complete || pendingBefore == pendingAfter || pendingAfter {
+ return sidecars, complete, pendingAfter, nil
+ }
+ reread, complete, err := a.readBlobSidecars(ctx, slot, blockRoot, canonicalRoot)
+ if err != nil || complete {
+ return reread, complete, false, err
+ }
+ byIndex := make(map[uint64]struct{}, len(sidecars)+len(reread))
+ merged := make([]*cltypes.BlobSidecar, 0, len(sidecars)+len(reread))
+ for _, source := range [][]*cltypes.BlobSidecar{sidecars, reread} {
+ for _, sidecar := range source {
+ if sidecar == nil {
+ continue
+ }
+ if _, exists := byIndex[sidecar.Index]; exists {
+ continue
+ }
+ byIndex[sidecar.Index] = struct{}{}
+ merged = append(merged, sidecar)
+ }
+ }
+ return merged, false, true, nil
}
func (a *ApiHandler) GetEthV1DebugBeaconDataColumnSidecars(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {
@@ -262,41 +363,46 @@ func (a *ApiHandler) GetEthV1BeaconBlobs(w http.ResponseWriter, r *http.Request)
indicies[i] = uint64(i)
}
} else {
- // take the blobs by the versioned hashes
- versionedHashesToIndex := make(map[common.Hash]uint64)
+ selected := make(map[common.Hash]struct{}, len(versionedHashes))
+ for _, encodedHash := range versionedHashes {
+ var hash common.Hash
+ if err := hash.UnmarshalText([]byte(encodedHash)); err != nil {
+ return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, fmt.Errorf("invalid versioned hash %q: %w", encodedHash, err))
+ }
+ if _, duplicate := selected[hash]; duplicate {
+ return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, fmt.Errorf("duplicate versioned hash %s", hash))
+ }
+ selected[hash] = struct{}{}
+ }
commitments.Range(func(index int, value *cltypes.KZGCommitment, length int) bool {
hash, err := utils.KzgCommitmentToVersionedHash(common.Bytes48(*value))
if err != nil {
return false
}
- versionedHashesToIndex[hash] = uint64(index)
+ if _, ok := selected[hash]; ok {
+ indicies = append(indicies, uint64(index))
+ }
return true
})
- for _, hash := range versionedHashes {
- index, ok := versionedHashesToIndex[common.HexToHash(hash)]
- if ok {
- indicies = append(indicies, index)
- }
- }
}
// collect the blobs
blobs := solid.NewStaticListSSZ[*cltypes.Blob](int(a.beaconChainCfg.MaxBlobCommittmentsPerBlock), int(cltypes.BYTES_PER_BLOB))
- blobSidecars, found, err := a.readBlobSidecars(ctx, *slot, blockRoot, canonicalRoot)
+ blobSidecars, _, pending, err := a.readBlobSidecarsWithBackfillStatus(ctx, *slot, blockRoot, canonicalRoot)
if err != nil {
return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, err)
}
- if !found {
- return beaconhttp.NewBeaconResponse(blobs).
- WithFinalized(canonicalRoot == blockRoot && *slot <= a.forkchoiceStore.FinalizedSlot()).
- WithOptimistic(a.forkchoiceStore.IsRootOptimistic(blockRoot)), nil
+ byIndex := make(map[uint64]*cltypes.BlobSidecar, len(blobSidecars))
+ for _, sidecar := range blobSidecars {
+ byIndex[sidecar.Index] = sidecar
}
for _, index := range indicies {
- if index >= uint64(len(blobSidecars)) {
- log.Warn("blob index out of range", "index", index, "len", len(blobSidecars))
- return nil, beaconhttp.NewEndpointError(http.StatusInternalServerError, errors.New("blob index out of range"))
+ if sidecar := byIndex[index]; sidecar != nil {
+ blobs.Append(&sidecar.Blob)
}
- blobs.Append(&blobSidecars[index].Blob)
+ }
+ if pending && blobs.Len() < len(indicies) {
+ return nil, beaconhttp.NewEndpointError(http.StatusServiceUnavailable, errors.New("blobs are still being backfilled"))
}
return beaconhttp.NewBeaconResponse(blobs).
diff --git a/cl/beacon/handler/blobs_test.go b/cl/beacon/handler/blobs_test.go
index 7417caf14e9..52e1cc73aa7 100644
--- a/cl/beacon/handler/blobs_test.go
+++ b/cl/beacon/handler/blobs_test.go
@@ -17,23 +17,29 @@
package handler
import (
+ "context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strconv"
"strings"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/require"
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/cltypes/solid"
"github.com/erigontech/erigon/cl/persistence/beacon_indicies"
+ "github.com/erigontech/erigon/cl/persistence/blob_storage"
forkchoicemock "github.com/erigontech/erigon/cl/phase1/forkchoice/mock_services"
"github.com/erigontech/erigon/cl/utils"
"github.com/erigontech/erigon/common"
"github.com/erigontech/erigon/common/log/v3"
+ "github.com/erigontech/erigon/db/kv"
+ "github.com/erigontech/erigon/db/snapshotsync/freezeblocks"
)
type frozenBlobSnapshotReader struct {
@@ -42,6 +48,83 @@ type frozenBlobSnapshotReader struct {
err error
}
+type blobBackfillStatusStub bool
+
+func (s blobBackfillStatusStub) BlobBackfillPending(uint64) bool { return bool(s) }
+
+type changingBlobBackfillStatus struct{ calls atomic.Int64 }
+
+func (s *changingBlobBackfillStatus) BlobBackfillPending(uint64) bool {
+ return s.calls.Add(1) > 1
+}
+
+type completingBlobBackfillStatus struct{ calls atomic.Int64 }
+
+func (s *completingBlobBackfillStatus) BlobBackfillPending(uint64) bool {
+ return s.calls.Add(1) == 1
+}
+
+type partialBlobStorage struct {
+ blob_storage.BlobStorage
+ sidecars []*cltypes.BlobSidecar
+ complete bool
+}
+
+type completingBlobStorage struct {
+ blob_storage.BlobStorage
+ reads atomic.Int64
+ sidecars []*cltypes.BlobSidecar
+}
+
+func (s *completingBlobStorage) ReadBlobSidecars(context.Context, uint64, common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
+ if s.reads.Add(1) == 1 {
+ return nil, false, nil
+ }
+ return s.sidecars, true, nil
+}
+
+type advancingFrozenBlobSnapshots struct {
+ frozen atomic.Uint64
+ sidecars []*cltypes.BlobSidecar
+}
+
+func (s *advancingFrozenBlobSnapshots) FrozenBlobs() uint64 { return s.frozen.Load() }
+func (s *advancingFrozenBlobSnapshots) ReadBlobSidecars(uint64) ([]*cltypes.BlobSidecar, error) {
+ return s.sidecars, nil
+}
+
+type freezingBlobStorage struct {
+ blob_storage.BlobStorage
+ snapshots *advancingFrozenBlobSnapshots
+ freezeAt uint64
+ storageSidecars []*cltypes.BlobSidecar
+}
+
+func (s freezingBlobStorage) ReadBlobSidecars(context.Context, uint64, common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
+ s.snapshots.frozen.Store(s.freezeAt)
+ return s.storageSidecars, false, nil
+}
+
+func (s partialBlobStorage) ReadBlobSidecars(context.Context, uint64, common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
+ return s.sidecars, s.complete, nil
+}
+
+type bodyOnlyBlobBlockReader struct {
+ freezeblocks.BeaconSnapshotReader
+}
+
+func (r bodyOnlyBlobBlockReader) ReadBlockByRoot(context.Context, kv.Tx, common.Hash) (*cltypes.SignedBeaconBlock, error) {
+ return nil, errors.New("full block unavailable")
+}
+
+type unavailableBlobBodyReader struct {
+ freezeblocks.BeaconSnapshotReader
+}
+
+func (r unavailableBlobBodyReader) ReadBeaconBlockBodyBySlot(context.Context, kv.Tx, uint64) (*cltypes.SignedBeaconBlock, error) {
+ return nil, nil
+}
+
func (r frozenBlobSnapshotReader) FrozenBlobs() uint64 { return r.frozenBlobsExclusive }
func (r frozenBlobSnapshotReader) ReadBlobSidecars(slot uint64) ([]*cltypes.BlobSidecar, error) {
if r.err != nil {
@@ -54,7 +137,9 @@ type blobsTestFixture struct {
handler *ApiHandler
fcu *forkchoicemock.ForkChoiceStorageMock
slot uint64
+ blockRoot common.Hash
versionedHash common.Hash
+ sidecars []*cltypes.BlobSidecar
}
func TestGetBlobsFromFrozenSnapshots(t *testing.T) {
@@ -84,6 +169,334 @@ func TestGetBlobsEmptyWhenFrozenSidecarsMissing(t *testing.T) {
require.Empty(t, out.Data)
}
+func TestGetBlobsUnavailableWhileBackfillInProgress(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+
+ resp := requestBeaconBlobs(t, server.URL, f)
+ defer resp.Body.Close()
+ require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+ require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+}
+
+func TestGetBlobsUnavailableAtFirstUnfrozenSlotDuringBackfill(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.caplinSnapshots = frozenBlobSnapshotReader{frozenBlobsExclusive: f.slot}
+
+ statusCode := getBeaconBlobsStatus(t, f)
+
+ require.Equal(t, http.StatusServiceUnavailable, statusCode)
+}
+
+func TestBlobSidecarsBackfillNotPendingForNonCanonicalBlock(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ pending := f.handler.blobBackfillPending(f.slot, f.blockRoot, common.HexToHash("0x01"))
+
+ require.False(t, pending)
+}
+
+func TestGetBlobSidecarsUnavailableWhileBackfillInProgress(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blob_sidecars/" + strconv.FormatUint(f.slot, 10))
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+ require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+
+ var endpointError struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ }
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&endpointError))
+ require.Equal(t, http.StatusServiceUnavailable, endpointError.Code)
+ require.Equal(t, "blob sidecars are still being backfilled", endpointError.Message)
+}
+
+func TestGetBlobSidecarsReturnsRequestedStoredIndexFromIncompleteBlock(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars[:1]}
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blob_sidecars/" + strconv.FormatUint(f.slot, 10) + "?indices=0")
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ require.Equal(t, http.StatusOK, resp.StatusCode)
+ var envelope struct {
+ Data []struct {
+ Index string `json:"index"`
+ } `json:"data"`
+ }
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&envelope))
+ require.Len(t, envelope.Data, 1)
+ require.Equal(t, "0", envelope.Data[0].Index)
+}
+
+func TestGetBlobSidecarsDoesNotTrustTooSmallCompleteSet(t *testing.T) {
+ testCases := []struct {
+ name string
+ query string
+ statusCode int
+ }{
+ {name: "all", statusCode: http.StatusServiceUnavailable},
+ {name: "missing index", query: "?indices=1", statusCode: http.StatusServiceUnavailable},
+ {name: "available index", query: "?indices=0", statusCode: http.StatusOK},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars[:1], complete: true}
+
+ require.Equal(t, tc.statusCode, getBlobSidecarsStatus(t, f, tc.query))
+ })
+ }
+}
+
+func TestGetBlobSidecarsUnavailableForRequestedMissingIndex(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars[:1]}
+
+ require.Equal(t, http.StatusServiceUnavailable, getBlobSidecarsStatus(t, f, "?indices=1"))
+}
+
+func TestGetBlobSidecarsUnavailableWhenAnyRequestedIndexIsMissing(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars[:1]}
+
+ require.Equal(t, http.StatusServiceUnavailable, getBlobSidecarsStatus(t, f, "?indices=0&indices=1"))
+}
+
+func TestGetBlobSidecarsBackfillStatusDoesNotRequireFullBlock(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blockReader = bodyOnlyBlobBlockReader{BeaconSnapshotReader: f.handler.blockReader}
+
+ require.Equal(t, http.StatusServiceUnavailable, getBlobSidecarsStatus(t, f, ""))
+}
+
+func TestGetBlobSidecarsErrorsWhenCanonicalBodyIsUnavailable(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blockReader = unavailableBlobBodyReader{BeaconSnapshotReader: f.handler.blockReader}
+
+ require.Equal(t, http.StatusInternalServerError, getBlobSidecarsStatus(t, f, ""))
+}
+
+func TestRequestedBlobSidecarsMissingTreatsZeroCanonicalRootAsEmptySlot(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blockReader = unavailableBlobBodyReader{BeaconSnapshotReader: f.handler.blockReader}
+ tx, err := f.handler.indiciesDB.BeginRo(t.Context())
+ require.NoError(t, err)
+ defer tx.Rollback()
+
+ missing, err := f.handler.requestedBlobSidecarsMissing(t.Context(), tx, f.slot+1, nil, true, nil)
+
+ require.NoError(t, err)
+ require.False(t, missing)
+}
+
+func TestGetBlobSidecarsEmptyForBlockWithoutCommitmentsDuringBackfill(t *testing.T) {
+ f := setupBlobsTestWithoutCommitments(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ require.Equal(t, http.StatusOK, getBlobSidecarsStatus(t, f, ""))
+}
+
+func TestGetBlobSidecarsEmptyForUnmatchedIndexDuringBackfill(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ require.Equal(t, http.StatusOK, getBlobSidecarsStatus(t, f, "?indices=99"))
+}
+
+func TestGetBlobSidecarsRejectsMalformedIndexWhileSidecarsMissing(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ require.Equal(t, http.StatusBadRequest, getBlobSidecarsStatus(t, f, "?indices=abc"))
+}
+
+func TestGetBlobSidecarsRejectsEmptyIndex(t *testing.T) {
+ f := setupBlobsTest(t)
+
+ require.Equal(t, http.StatusBadRequest, getBlobSidecarsStatus(t, f, "?indices="))
+}
+
+func TestGetBlobSidecarsRejectsDuplicateIndices(t *testing.T) {
+ f := setupBlobsTest(t)
+
+ require.Equal(t, http.StatusBadRequest, getBlobSidecarsStatus(t, f, "?indices=0&indices=0"))
+}
+
+func TestReadBlobSidecarsRechecksPendingAfterIncompleteReread(t *testing.T) {
+ f := setupBlobsTest(t)
+ status := &changingBlobBackfillStatus{}
+ f.handler.blobBackfillStatus = status
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage}
+
+ _, complete, pending, err := f.handler.readBlobSidecarsWithBackfillStatus(t.Context(), f.slot, f.blockRoot, f.blockRoot)
+ require.NoError(t, err)
+ require.False(t, complete)
+ require.True(t, pending)
+ require.Equal(t, int64(2), status.calls.Load())
+}
+
+func TestGetBlobSidecarsDoesNotRereadWithoutBackfillTransition(t *testing.T) {
+ f := setupBlobsTest(t)
+ storage := &completingBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars}
+ f.handler.blobStoage = storage
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(false)
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blob_sidecars/" + strconv.FormatUint(f.slot, 10) + "?indices=0")
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ require.Equal(t, http.StatusOK, resp.StatusCode)
+ var envelope struct {
+ Data []json.RawMessage `json:"data"`
+ }
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&envelope))
+ require.Empty(t, envelope.Data)
+ require.Equal(t, int64(1), storage.reads.Load())
+}
+
+func TestReadBlobSidecarsRereadsWhenBackfillCompletesDuringRead(t *testing.T) {
+ f := setupBlobsTest(t)
+ status := &completingBlobBackfillStatus{}
+ storage := &completingBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars}
+ f.handler.blobBackfillStatus = status
+ f.handler.blobStoage = storage
+
+ out, complete, pending, err := f.handler.readBlobSidecarsWithBackfillStatus(t.Context(), f.slot, f.blockRoot, f.blockRoot)
+
+ require.NoError(t, err)
+ require.True(t, complete)
+ require.False(t, pending)
+ require.Equal(t, f.sidecars, out)
+ require.Equal(t, int64(2), storage.reads.Load())
+}
+
+func TestGetBlobSidecarsFrozenMissingDuringBackfillKeepsExistingResponse(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.caplinSnapshots = frozenBlobSnapshotReader{frozenBlobsExclusive: f.slot + 1}
+
+ require.Equal(t, http.StatusOK, getBlobSidecarsStatus(t, f, ""))
+}
+
+func TestGetBlobsEmptyAfterBackfillCompleted(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(false)
+
+ out := getBeaconBlobs(t, f)
+
+ require.Empty(t, out.Data)
+}
+
+func TestGetBlobsReturnsStoredDataDuringBackfill(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ require.NoError(t, f.handler.blobStoage.WriteBlobSidecars(t.Context(), f.blockRoot, f.sidecars))
+
+ out := getBeaconBlobs(t, f)
+
+ require.Len(t, out.Data, 1)
+}
+
+func TestGetBlobsReturnsRequestedStoredDataFromIncompleteBlock(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars[1:]}
+
+ out := getBeaconBlobs(t, f)
+
+ require.Len(t, out.Data, 1)
+}
+
+func TestGetBlobsUnavailableWhenAnyRequestedHashIsMissing(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.handler.blobStoage = partialBlobStorage{BlobStorage: f.handler.blobStoage, sidecars: f.sidecars[1:]}
+ firstHash, err := utils.KzgCommitmentToVersionedHash(f.sidecars[0].KzgCommitment)
+ require.NoError(t, err)
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+ url := server.URL + "/eth/v1/beacon/blobs/" + strconv.FormatUint(f.slot, 10) + "?versioned_hashes=" + firstHash.Hex() + "&versioned_hashes=" + f.versionedHash.Hex()
+ resp, err := http.Get(url)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+}
+
+func TestGetBlobsEmptyForUnmatchedHashDuringBackfill(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+ f.versionedHash = common.HexToHash("0xffff")
+
+ out := getBeaconBlobs(t, f)
+
+ require.Empty(t, out.Data)
+}
+
+func TestGetBlobsRejectsMalformedVersionedHash(t *testing.T) {
+ f := setupBlobsTest(t)
+
+ require.Equal(t, http.StatusBadRequest, getBeaconBlobsStatusWithQuery(t, f, "?versioned_hashes=0x01"))
+}
+
+func TestGetBlobsRejectsDuplicateVersionedHash(t *testing.T) {
+ f := setupBlobsTest(t)
+ hash := f.versionedHash.Hex()
+
+ require.Equal(t, http.StatusBadRequest, getBeaconBlobsStatusWithQuery(t, f, "?versioned_hashes="+hash+"&versioned_hashes="+hash))
+}
+
+func TestGetBlobsOrdersResponseByBlockCommitments(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.caplinSnapshots = frozenBlobSnapshotReader{frozenBlobsExclusive: f.slot + 1, sidecars: f.sidecars}
+ firstHash, err := utils.KzgCommitmentToVersionedHash(f.sidecars[0].KzgCommitment)
+ require.NoError(t, err)
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+ query := "?versioned_hashes=" + f.versionedHash.Hex() + "&versioned_hashes=" + firstHash.Hex()
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blobs/" + strconv.FormatUint(f.slot, 10) + query)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ require.Equal(t, http.StatusOK, resp.StatusCode)
+ var out beaconBlobsResponse
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
+ require.Len(t, out.Data, 2)
+ require.True(t, strings.HasPrefix(out.Data[0], "0x01"))
+ require.True(t, strings.HasPrefix(out.Data[1], "0x02"))
+}
+
+func TestGetBlobsEmptyForBlockWithoutCommitmentsDuringBackfill(t *testing.T) {
+ f := setupBlobsTestWithoutCommitments(t)
+ f.handler.blobBackfillStatus = blobBackfillStatusStub(true)
+
+ out := getBeaconBlobs(t, f)
+
+ require.Empty(t, out.Data)
+}
+
func TestGetBlobsErrorsWhenFrozenSnapshotReadFails(t *testing.T) {
f := setupBlobsTest(t)
@@ -97,6 +510,94 @@ func TestGetBlobsErrorsWhenFrozenSnapshotReadFails(t *testing.T) {
require.Equal(t, http.StatusInternalServerError, statusCode)
}
+func TestReadBlobSidecarsAtFirstUnfrozenSlotFromStorage(t *testing.T) {
+ f := setupBlobsTest(t)
+ f.handler.caplinSnapshots = frozenBlobSnapshotReader{
+ frozenBlobsExclusive: f.slot,
+ sidecars: []*cltypes.BlobSidecar{{Index: 99}},
+ }
+ require.NoError(t, f.handler.blobStoage.WriteBlobSidecars(t.Context(), f.blockRoot, f.sidecars))
+
+ out, found, err := f.handler.readBlobSidecars(t.Context(), f.slot, f.blockRoot, f.blockRoot)
+
+ require.NoError(t, err)
+ require.True(t, found)
+ require.Equal(t, f.sidecars, out)
+}
+
+func TestReadBlobSidecarsRetriesSnapshotWhenFrozenBoundaryAdvances(t *testing.T) {
+ f := setupBlobsTest(t)
+ snapshots := &advancingFrozenBlobSnapshots{sidecars: f.sidecars}
+ snapshots.frozen.Store(f.slot)
+ f.handler.caplinSnapshots = snapshots
+ f.handler.blobStoage = freezingBlobStorage{BlobStorage: f.handler.blobStoage, snapshots: snapshots, freezeAt: f.slot + 1}
+
+ out, found, err := f.handler.readBlobSidecars(t.Context(), f.slot, f.blockRoot, f.blockRoot)
+
+ require.NoError(t, err)
+ require.True(t, found)
+ require.Equal(t, f.sidecars, out)
+}
+
+func TestReadBlobSidecarsRetainsPartialStorageWhenAdvancedSnapshotIsEmpty(t *testing.T) {
+ f := setupBlobsTest(t)
+ snapshots := &advancingFrozenBlobSnapshots{}
+ snapshots.frozen.Store(f.slot)
+ f.handler.caplinSnapshots = snapshots
+ f.handler.blobStoage = freezingBlobStorage{
+ BlobStorage: f.handler.blobStoage,
+ snapshots: snapshots,
+ freezeAt: f.slot + 1,
+ storageSidecars: f.sidecars[:1],
+ }
+
+ out, found, err := f.handler.readBlobSidecars(t.Context(), f.slot, f.blockRoot, f.blockRoot)
+
+ require.NoError(t, err)
+ require.False(t, found)
+ require.Equal(t, f.sidecars[:1], out)
+}
+
+func TestGetBlobSidecarsRetainsPartialStorageWhenBackfillCompletesIntoEmptySnapshot(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ query string
+ statusCode int
+ dataLen int
+ }{
+ {name: "stored member", query: "?indices=0", statusCode: http.StatusOK, dataLen: 1},
+ {name: "missing member", query: "?indices=1", statusCode: http.StatusServiceUnavailable},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := setupBlobsTest(t)
+ snapshots := &advancingFrozenBlobSnapshots{}
+ snapshots.frozen.Store(f.slot)
+ f.handler.caplinSnapshots = snapshots
+ f.handler.blobBackfillStatus = &completingBlobBackfillStatus{}
+ f.handler.blobStoage = freezingBlobStorage{
+ BlobStorage: f.handler.blobStoage,
+ snapshots: snapshots,
+ freezeAt: f.slot + 1,
+ storageSidecars: f.sidecars[:1],
+ }
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blob_sidecars/" + strconv.FormatUint(f.slot, 10) + tc.query)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ require.Equal(t, tc.statusCode, resp.StatusCode)
+ if tc.statusCode == http.StatusOK {
+ var envelope struct {
+ Data []json.RawMessage `json:"data"`
+ }
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&envelope))
+ require.Len(t, envelope.Data, tc.dataLen)
+ }
+ })
+ }
+}
+
type beaconBlobsResponse struct {
Data []string `json:"data"`
}
@@ -127,6 +628,16 @@ func getBeaconBlobsStatus(t *testing.T, f blobsTestFixture) int {
return resp.StatusCode
}
+func getBeaconBlobsStatusWithQuery(t *testing.T, f blobsTestFixture, query string) int {
+ t.Helper()
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blobs/" + strconv.FormatUint(f.slot, 10) + query)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ return resp.StatusCode
+}
+
func requestBeaconBlobs(t *testing.T, baseURL string, f blobsTestFixture) *http.Response {
t.Helper()
@@ -135,21 +646,57 @@ func requestBeaconBlobs(t *testing.T, baseURL string, f blobsTestFixture) *http.
return resp
}
+func getBlobSidecarsStatus(t *testing.T, f blobsTestFixture, query string) int {
+ t.Helper()
+
+ server := httptest.NewServer(f.handler.mux)
+ defer server.Close()
+
+ resp, err := http.Get(server.URL + "/eth/v1/beacon/blob_sidecars/" + strconv.FormatUint(f.slot, 10) + query)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ return resp.StatusCode
+}
+
func setupBlobsTest(t *testing.T) blobsTestFixture {
t.Helper()
+ return setupBlobsTestWithCommitments(t, []cltypes.KZGCommitment{{69}, {1}})
+}
+
+func setupBlobsTestWithoutCommitments(t *testing.T) blobsTestFixture {
+ t.Helper()
+ return setupBlobsTestWithCommitments(t, nil)
+}
+
+func setupBlobsTestWithCommitments(t *testing.T, commitments []cltypes.KZGCommitment) blobsTestFixture {
+ t.Helper()
db, blocks, _, _, _, handler, _, _, fcu, _ := setupTestingHandler(t, clparams.ElectraVersion, log.Root(), false)
block := blocks[0]
slot := block.Block.Slot
- commitments := []cltypes.KZGCommitment{{69}, {1}}
block.Block.Body.BlobKzgCommitments.Clear()
- block.Block.Body.BlobKzgCommitments.Append(&commitments[0])
- block.Block.Body.BlobKzgCommitments.Append(&commitments[1])
+ for i := range commitments {
+ block.Block.Body.BlobKzgCommitments.Append(&commitments[i])
+ }
blockRoot, err := block.Block.HashSSZ()
require.NoError(t, err)
- versionedHash, err := utils.KzgCommitmentToVersionedHash(common.Bytes48(commitments[1]))
- require.NoError(t, err)
+ var versionedHash common.Hash
+ if len(commitments) > 1 {
+ versionedHash, err = utils.KzgCommitmentToVersionedHash(common.Bytes48(commitments[1]))
+ require.NoError(t, err)
+ }
+ sidecars := make([]*cltypes.BlobSidecar, len(commitments))
+ for i := range commitments {
+ sidecars[i] = cltypes.NewBlobSidecar(
+ uint64(i),
+ &cltypes.Blob{byte(i + 1)},
+ common.Bytes48(commitments[i]),
+ common.Bytes48{},
+ block.SignedBeaconBlockHeader(),
+ solid.NewHashVector(cltypes.CommitmentBranchSize),
+ )
+ }
tx, err := db.BeginRw(t.Context())
require.NoError(t, err)
@@ -162,11 +709,12 @@ func setupBlobsTest(t *testing.T) blobsTestFixture {
handler: handler,
fcu: fcu,
slot: slot,
+ blockRoot: blockRoot,
versionedHash: versionedHash,
+ sidecars: sidecars,
}
}
-// blobSidecarsEnvelope captures the full response shape for blob_sidecars.
type blobSidecarsEnvelope struct {
Version *string `json:"version"`
ExecutionOptimistic *bool `json:"execution_optimistic"`
@@ -174,17 +722,12 @@ type blobSidecarsEnvelope struct {
Data json.RawMessage `json:"data"`
}
-// TestBlobSidecarsResponseEnvelope verifies that GET /eth/v1/beacon/blob_sidecars/{block_id}
-// returns the required envelope fields (version, execution_optimistic, finalized) per the
-// Beacon API specification. This test checks the non-finalized and optimistic state.
func TestBlobSidecarsResponseEnvelope(t *testing.T) {
f := setupBlobsTest(t)
- // Set up forkchoice mock to simulate non-finalized and optimistic block.
f.fcu.IsRootOptimisticVal = true
f.fcu.FinalizedSlotVal = f.slot - 1
- // Set up frozen snapshots with actual blob data.
f.handler.caplinSnapshots = frozenBlobSnapshotReader{
frozenBlobsExclusive: f.slot + 1,
sidecars: []*cltypes.BlobSidecar{
@@ -203,28 +746,22 @@ func TestBlobSidecarsResponseEnvelope(t *testing.T) {
var envelope blobSidecarsEnvelope
require.NoError(t, json.NewDecoder(resp.Body).Decode(&envelope))
- // All three envelope fields must be present (non-nil pointers).
require.NotNil(t, envelope.Version, "response must include 'version'")
require.NotNil(t, envelope.ExecutionOptimistic, "response must include 'execution_optimistic'")
require.NotNil(t, envelope.Finalized, "response must include 'finalized'")
require.NotNil(t, envelope.Data, "response must include 'data'")
- // Verify values.
require.Equal(t, "electra", *envelope.Version)
require.True(t, *envelope.ExecutionOptimistic, "execution_optimistic must be true")
require.False(t, *envelope.Finalized, "finalized must be false")
}
-// TestBlobSidecarsEmptyResponseEnvelope verifies that even when no blobs are found,
-// the response envelope still includes all required fields. This test checks the finalized and non-optimistic state.
func TestBlobSidecarsEmptyResponseEnvelope(t *testing.T) {
f := setupBlobsTest(t)
- // Set up forkchoice mock to simulate finalized and non-optimistic block.
f.fcu.IsRootOptimisticVal = false
f.fcu.FinalizedSlotVal = f.slot + 1
- // No snapshots, no blob storage data for this block → empty response.
f.handler.caplinSnapshots = frozenBlobSnapshotReader{frozenBlobsExclusive: f.slot + 1}
server := httptest.NewServer(f.handler.mux)
@@ -238,13 +775,11 @@ func TestBlobSidecarsEmptyResponseEnvelope(t *testing.T) {
var envelope blobSidecarsEnvelope
require.NoError(t, json.NewDecoder(resp.Body).Decode(&envelope))
- // All envelope fields must still be present even with empty data.
require.NotNil(t, envelope.Version, "empty response must include 'version'")
require.NotNil(t, envelope.ExecutionOptimistic, "empty response must include 'execution_optimistic'")
require.NotNil(t, envelope.Finalized, "empty response must include 'finalized'")
require.NotNil(t, envelope.Data, "empty response must include 'data'")
- // Verify values.
require.Equal(t, "electra", *envelope.Version)
require.False(t, *envelope.ExecutionOptimistic, "execution_optimistic must be false")
require.True(t, *envelope.Finalized, "finalized must be true")
diff --git a/cl/beacon/handler/handler.go b/cl/beacon/handler/handler.go
index 1f15ca474cb..908a4f46fed 100644
--- a/cl/beacon/handler/handler.go
+++ b/cl/beacon/handler/handler.go
@@ -91,6 +91,7 @@ type ApiHandler struct {
blobStoage blob_storage.BlobStorage
columnStorage blob_storage.DataColumnStorage
caplinSnapshots caplinBlobSnapshotReader
+ blobBackfillStatus BlobBackfillStatus
caplinStateSnapshots *snapshotsync.CaplinStateSnapshots
peerDas das.PeerDas
@@ -149,6 +150,14 @@ type ApiHandler struct {
selfBuildEnvelopes *lru.Cache[uint64, *cltypes.ExecutionPayloadEnvelope]
}
+// BlobDataDependencies groups storage and backfill availability dependencies for blob endpoints.
+type BlobDataDependencies struct {
+ Storage blob_storage.BlobStorage
+ ColumnStorage blob_storage.DataColumnStorage
+ Snapshots *freezeblocks.CaplinSnapshots
+ BackfillStatus BlobBackfillStatus
+}
+
func NewApiHandler(
logger log.Logger,
netConfig *clparams.NetworkConfig,
@@ -164,9 +173,7 @@ func NewApiHandler(
version string,
routerCfg *beacon_router_configuration.RouterConfiguration,
emitters *beaconevents.EventEmitter,
- blobStoage blob_storage.BlobStorage,
- columnStorage blob_storage.DataColumnStorage,
- caplinSnapshots *freezeblocks.CaplinSnapshots,
+ blobData BlobDataDependencies,
validatorParams *validator_params.ValidatorParams,
attestationProducer attestation_producer.AttestationDataProducer,
engine execution_client.ExecutionEngine,
@@ -195,8 +202,8 @@ func NewApiHandler(
panic(err)
}
var blobSnapshots caplinBlobSnapshotReader
- if caplinSnapshots != nil {
- blobSnapshots = caplinSnapshots
+ if blobData.Snapshots != nil {
+ blobSnapshots = blobData.Snapshots
}
slotWaitedForAttestationProduction, err := lru.New[uint64, struct{}]("slotWaitedForAttestationProduction", 1024)
@@ -234,9 +241,10 @@ func NewApiHandler(
version: version,
routerCfg: routerCfg,
emitters: emitters,
- blobStoage: blobStoage,
- columnStorage: columnStorage,
+ blobStoage: blobData.Storage,
+ columnStorage: blobData.ColumnStorage,
caplinSnapshots: blobSnapshots,
+ blobBackfillStatus: blobData.BackfillStatus,
attestationProducer: attestationProducer,
blobBundles: blobBundles,
engine: engine,
diff --git a/cl/beacon/handler/utils_test.go b/cl/beacon/handler/utils_test.go
index 71657e54917..f73325e89a7 100644
--- a/cl/beacon/handler/utils_test.go
+++ b/cl/beacon/handler/utils_test.go
@@ -180,7 +180,7 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge
Events: true,
Validator: true,
Lighthouse: true,
- }, nil, blobStorage, columnStorage, nil, vp, nil, nil, fcu.SyncContributionPool, nil, nil,
+ }, nil, BlobDataDependencies{Storage: blobStorage, ColumnStorage: columnStorage}, vp, nil, nil, fcu.SyncContributionPool, nil, nil,
syncCommitteeMessagesService,
syncContributionService,
aggregateAndProofsService,
diff --git a/cl/beacon/handler/validator_test.go b/cl/beacon/handler/validator_test.go
index 0a457074878..fad3dd3f2fc 100644
--- a/cl/beacon/handler/validator_test.go
+++ b/cl/beacon/handler/validator_test.go
@@ -58,9 +58,7 @@ func (t *validatorTestSuite) SetupTest() {
"0",
&beacon_router_configuration.RouterConfiguration{Validator: true},
nil,
- nil,
- nil,
- nil,
+ BlobDataDependencies{},
nil,
nil,
nil,
diff --git a/cl/das/peer_das.go b/cl/das/peer_das.go
index 6e625e43f2e..39b261b166f 100644
--- a/cl/das/peer_das.go
+++ b/cl/das/peer_das.go
@@ -66,6 +66,11 @@ type PeerDas interface {
var numOfBlobRecoveryWorkers = 8
+const (
+ maxBlobRecoveryWaiters = 128
+ blobRecoveryTimeout = 2 * time.Minute
+)
+
type peerdas struct {
state *peerdasstate.PeerDasState
nodeID enode.ID
@@ -80,7 +85,7 @@ type peerdas struct {
recoverBlobsQueue chan recoverBlobsRequest
recoveringMutex sync.Mutex
- isRecovering map[common.Hash]bool
+ isRecovering map[common.Hash]*blobRecovery
blocksToCheckSync sync.Map // blockRoot -> ColumnSyncableSignedBlock (SignedBeaconBlock or SignedBlindedBeaconBlock)
// [New in Gloas:EIP7732] For fetching blocks to get kzg_commitments
@@ -121,7 +126,7 @@ func NewPeerDas(
recoverBlobsQueue: make(chan recoverBlobsRequest, 128),
recoveringMutex: sync.Mutex{},
- isRecovering: make(map[common.Hash]bool),
+ isRecovering: make(map[common.Hash]*blobRecovery),
blocksToCheckSync: sync.Map{},
blockReader: blockReader,
@@ -347,24 +352,35 @@ func (d *peerdas) Prune(keepSlotDistance uint64) error {
}
type recoverBlobsRequest struct {
- slot uint64
- blockRoot common.Hash
+ slot uint64
+ blockRoot common.Hash
+ expectedBlobs uint64
+ force bool
+ ctx context.Context
+ result chan error
+ retryOwner *blobRecovery
+}
+
+type blobRecovery struct {
+ waiters []chan error
+ retryRequested bool
+ retrySlot uint64
}
func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
- recover := func(toRecover recoverBlobsRequest) {
+ recover := func(recoveryCtx context.Context, toRecover recoverBlobsRequest) error {
begin := time.Now()
log.Debug("[blobsRecover] recovering blobs", "slot", toRecover.slot, "blockRoot", toRecover.blockRoot)
- ctx := context.Background()
+ ctx := recoveryCtx
slot, blockRoot := toRecover.slot, toRecover.blockRoot
existingColumns, err := d.columnStorage.GetSavedColumnIndex(ctx, slot, blockRoot)
if err != nil {
log.Warn("[blobsRecover] failed to get saved column index", "err", err)
- return
+ return err
}
if len(existingColumns) < int(d.beaconConfig.NumberOfColumns+1)/2 {
log.Debug("[blobsRecover] not enough columns to recover", "slot", slot, "blockRoot", blockRoot, "existingColumns", len(existingColumns))
- return
+ return errors.New("not enough columns to recover blobs")
}
// [Modified in Gloas:EIP7732] For GLOAS, kzg_commitments and SignedBlockHeader come from block
@@ -376,12 +392,12 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
kzgCommitmentsFromBlock, err = d.getKzgCommitmentsForGloas(slot, blockRoot)
if err != nil {
log.Warn("[blobsRecover] failed to get kzg commitments for GLOAS", "err", err, "slot", slot, "blockRoot", blockRoot)
- return
+ return err
}
signedBlockHeaderFromBlock, err = d.getSignedBlockHeaderForGloas(blockRoot)
if err != nil {
log.Warn("[blobsRecover] failed to get signed block header for GLOAS", "err", err, "slot", slot, "blockRoot", blockRoot)
- return
+ return err
}
}
@@ -393,11 +409,11 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
if err != nil {
log.Debug("[blobsRecover] failed to read column sidecar", "err", err)
d.columnStorage.RemoveColumnSidecars(ctx, slot, blockRoot, int64(columnIndex))
- return
+ return err
}
if sidecar.Column.Len() > int(d.beaconConfig.MaxBlobCommittmentsPerBlock) {
log.Warn("[blobsRecover] invalid column sidecar", "slot", slot, "blockRoot", blockRoot, "columnIndex", columnIndex, "columnLen", sidecar.Column.Len())
- return
+ return errors.New("column sidecar exceeds blob commitment limit")
}
for i := 0; i < sidecar.Column.Len(); i++ {
matrixEntries = append(matrixEntries, cltypes.MatrixEntry{
@@ -417,7 +433,7 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
blobMatrix, err := peerdasutils.RecoverMatrix(matrixEntries, numberOfBlobs)
if err != nil {
log.Warn("[blobsRecover] failed to recover matrix", "err", err, "slot", slot, "blockRoot", blockRoot, "numberOfBlobs", numberOfBlobs)
- return
+ return err
}
timeRecoverMatrix := time.Since(beginRecoverMatrix)
log.Trace("[blobsRecover] recovered matrix", "slot", slot, "blockRoot", blockRoot, "numberOfBlobs", numberOfBlobs)
@@ -436,12 +452,12 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
// blob
if len(blobEntries) != int(d.beaconConfig.NumberOfColumns) {
log.Warn("[blobsRecover] invalid blob entries", "blobIndex", blobIndex, "slot", slot, "blockRoot", blockRoot, "blobEntries", len(blobEntries))
- return
+ return errors.New("recovered blob has incomplete matrix entries")
}
for i := range len(blobEntries) / 2 {
if copied := copy(blob[i*cltypes.BytesPerCell:], blobEntries[i].Cell[:]); copied != cltypes.BytesPerCell {
log.Warn("[blobsRecover] failed to copy cell", "blobIndex", blobIndex, "slot", slot, "blockRoot", blockRoot)
- return
+ return errors.New("recovered blob cell has invalid size")
}
}
// kzg commitment
@@ -456,7 +472,7 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
proof, err := kzg.Ctx().ComputeBlobKZGProof(&ckzgBlob, goethkzg.KZGCommitment(kzgCommitment), 0 /* numGoRoutines */)
if err != nil {
log.Warn("[blobsRecover] failed to compute blob kzg proof", "blobIndex", blobIndex, "slot", slot, "blockRoot", blockRoot)
- return
+ return err
}
copy(kzgProof[:], proof[:])
// [Modified in Gloas:EIP7732] Use SignedBlockHeader from block for GLOAS
@@ -496,7 +512,7 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
// Save blobs
if err := d.blobStorage.WriteBlobSidecars(ctx, blockRoot, blobSidecars); err != nil {
log.Warn("[blobsRecover] failed to write blob sidecars", "err", err, "slot", slot, "blockRoot", blockRoot)
- return
+ return err
}
log.Trace("[blobsRecover] saved blobs", "slot", slot, "blockRoot", blockRoot, "numberOfBlobs", numberOfBlobs)
@@ -504,7 +520,7 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
custodyColumns, err := d.state.GetMyCustodyColumns()
if err != nil {
log.Warn("[blobsRecover] failed to get my custody columns", "err", err, "slot", slot, "blockRoot", blockRoot)
- return
+ return err
}
beginRemoveColumns := time.Now()
toRemove := []int64{}
@@ -577,6 +593,7 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
timeAddColumns := time.Since(beginAddColumns)
log.Debug("[blobsRecover] recovering done", "slot", slot, "blockRoot", blockRoot, "numberOfBlobs", numberOfBlobs, "elapsedTime", time.Since(begin),
"timeRecoverMatrix", timeRecoverMatrix, "timeRecoverBlobs", timeRecoverBlobs, "timeRemoveColumns", timeRemoveColumns, "timeAddColumns", timeAddColumns)
+ return nil
}
// main loop
@@ -585,25 +602,178 @@ func (d *peerdas) blobsRecoverWorker(ctx context.Context) {
case <-ctx.Done():
return
case toRecover := <-d.recoverBlobsQueue:
- d.recoveringMutex.Lock()
- if _, ok := d.isRecovering[toRecover.blockRoot]; ok {
- // recovering, skip
+ d.handleRecoverBlobsRequest(ctx, toRecover, recover)
+ }
+ }
+}
+
+func (d *peerdas) handleRecoverBlobsRequest(workerCtx context.Context, request recoverBlobsRequest, recover func(context.Context, recoverBlobsRequest) error) {
+ requestCtx := request.ctx
+ if requestCtx == nil {
+ requestCtx = workerCtx
+ }
+ if err := requestCtx.Err(); err != nil {
+ d.completeRecoveryRequest(request.result, err)
+ return
+ }
+
+ d.recoveringMutex.Lock()
+ active := d.isRecovering[request.blockRoot]
+ if request.retryOwner != nil && active != request.retryOwner {
+ d.recoveringMutex.Unlock()
+ return
+ }
+ if active != nil && request.retryOwner == nil {
+ if !request.force {
+ active.retryRequested = true
+ active.retrySlot = request.slot
+ }
+ if request.result != nil {
+ if len(active.waiters) >= maxBlobRecoveryWaiters {
d.recoveringMutex.Unlock()
- continue
+ d.completeRecoveryRequest(request.result, errors.New("too many callers waiting for blob recovery"))
+ return
}
- d.isRecovering[toRecover.blockRoot] = true
- d.recoveringMutex.Unlock()
+ active.waiters = append(active.waiters, request.result)
+ }
+ d.recoveringMutex.Unlock()
+ return
+ }
+ if active == nil {
+ active = &blobRecovery{}
+ if request.result != nil {
+ active.waiters = append(active.waiters, request.result)
+ }
+ d.isRecovering[request.blockRoot] = active
+ }
+ d.recoveringMutex.Unlock()
+ ownerCtx, cancelOwner := context.WithTimeout(workerCtx, blobRecoveryTimeout)
+ defer cancelOwner()
- // check if the blobs are already recovered
- if !d.IsBlobAlreadyRecovered(toRecover.blockRoot) {
- // recover the blobs
- recover(toRecover)
- }
- // remove the block from the recovering map
- d.recoveringMutex.Lock()
- delete(d.isRecovering, toRecover.blockRoot)
- d.recoveringMutex.Unlock()
+ if request.force {
+ if complete, err := d.blobRecoveryComplete(ownerCtx, request.slot, request.blockRoot, request.expectedBlobs); err != nil {
+ d.finishBlobRecovery(request.blockRoot, err)
+ return
+ } else if complete {
+ d.finishBlobRecovery(request.blockRoot, nil)
+ return
+ }
+ } else if d.IsBlobAlreadyRecovered(request.blockRoot) {
+ d.finishBlobRecovery(request.blockRoot, nil)
+ return
+ }
+ err := recover(ownerCtx, request)
+ switch {
+ case request.force:
+ var complete bool
+ complete, completeErr := d.blobRecoveryComplete(ownerCtx, request.slot, request.blockRoot, request.expectedBlobs)
+ switch {
+ case completeErr != nil:
+ err = completeErr
+ case complete:
+ err = nil
+ case err == nil:
+ err = errors.New("blob recovery did not complete")
}
+ case d.IsBlobAlreadyRecovered(request.blockRoot):
+ err = nil
+ case err == nil:
+ err = errors.New("blob recovery did not complete")
+ }
+ d.finishBlobRecovery(request.blockRoot, err)
+}
+
+func (d *peerdas) blobRecoveryComplete(ctx context.Context, slot uint64, blockRoot common.Hash, expectedBlobs uint64) (bool, error) {
+ sidecars, complete, err := d.blobStorage.ReadBlobSidecars(ctx, slot, blockRoot)
+ return complete && uint64(len(sidecars)) == expectedBlobs, err
+}
+
+func (d *peerdas) completeRecoveryRequest(result chan error, err error) {
+ if result == nil {
+ return
+ }
+ select {
+ case result <- err:
+ default:
+ }
+}
+
+func (d *peerdas) finishBlobRecovery(blockRoot common.Hash, err error) {
+ d.recoveringMutex.Lock()
+ active := d.isRecovering[blockRoot]
+ if active == nil {
+ d.recoveringMutex.Unlock()
+ return
+ }
+ waiters := active.waiters
+ active.waiters = nil
+ retryRequested := err != nil && active.retryRequested
+ retrySlot := active.retrySlot
+ active.retryRequested = false
+ if !retryRequested {
+ delete(d.isRecovering, blockRoot)
+ }
+ d.recoveringMutex.Unlock()
+ for _, waiter := range waiters {
+ d.completeRecoveryRequest(waiter, err)
+ }
+ if retryRequested {
+ go d.scheduleRecoveryRetry(retrySlot, blockRoot, active)
+ }
+}
+
+func (d *peerdas) scheduleRecoveryRetry(slot uint64, blockRoot common.Hash, active *blobRecovery) {
+ timer := time.NewTimer(3 * time.Second)
+ defer timer.Stop()
+ select {
+ case d.recoverBlobsQueue <- recoverBlobsRequest{slot: slot, blockRoot: blockRoot, retryOwner: active}:
+ return
+ case <-timer.C:
+ }
+
+ d.recoveringMutex.Lock()
+ if d.isRecovering[blockRoot] != active {
+ d.recoveringMutex.Unlock()
+ return
+ }
+ delete(d.isRecovering, blockRoot)
+ waiters := active.waiters
+ d.recoveringMutex.Unlock()
+ for _, waiter := range waiters {
+ d.completeRecoveryRequest(waiter, errors.New("failed to schedule recover: timeout"))
+ }
+}
+
+func (d *peerdas) ForceScheduleRecover(ctx context.Context, slot uint64, blockRoot common.Hash, expectedBlobs uint64) error {
+ if complete, err := d.blobRecoveryComplete(ctx, slot, blockRoot, expectedBlobs); err != nil {
+ return err
+ } else if complete {
+ return nil
+ }
+ if !d.IsColumnOverHalf(slot, blockRoot) {
+ return errors.New("not enough columns to force blob recovery")
+ }
+ result := make(chan error, 1)
+ select {
+ case d.recoverBlobsQueue <- recoverBlobsRequest{slot: slot, blockRoot: blockRoot, expectedBlobs: expectedBlobs, force: true, ctx: ctx, result: result}:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ select {
+ case err := <-result:
+ if err != nil {
+ return err
+ }
+ complete, err := d.blobRecoveryComplete(ctx, slot, blockRoot, expectedBlobs)
+ if err != nil {
+ return err
+ }
+ if !complete {
+ return errors.New("blob recovery did not complete")
+ }
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
}
}
@@ -619,7 +789,9 @@ func (d *peerdas) TryScheduleRecover(slot uint64, blockRoot common.Hash) error {
// early check if the blobs are recovering
d.recoveringMutex.Lock()
- if _, ok := d.isRecovering[blockRoot]; ok {
+ if active := d.isRecovering[blockRoot]; active != nil {
+ active.retryRequested = true
+ active.retrySlot = slot
d.recoveringMutex.Unlock()
return nil
}
diff --git a/cl/das/peer_das_recovery_test.go b/cl/das/peer_das_recovery_test.go
new file mode 100644
index 00000000000..cf6e235aeaf
--- /dev/null
+++ b/cl/das/peer_das_recovery_test.go
@@ -0,0 +1,321 @@
+// Copyright 2026 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see .
+
+package das
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/mock/gomock"
+
+ "github.com/erigontech/erigon/cl/clparams"
+ "github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/persistence/blob_storage/mock_services"
+ "github.com/erigontech/erigon/common"
+)
+
+func newRecoveryTestPeerDas(t *testing.T) (*peerdas, *mock_services.MockDataColumnStorage, *mock_services.MockBlobStorage) {
+ t.Helper()
+ ctrl := gomock.NewController(t)
+ columns := mock_services.NewMockDataColumnStorage(ctrl)
+ blobs := mock_services.NewMockBlobStorage(ctrl)
+ cfg := clparams.MainnetBeaconConfig
+ cfg.NumberOfColumns = 4
+ return &peerdas{
+ beaconConfig: &cfg,
+ columnStorage: columns,
+ blobStorage: blobs,
+ recoverBlobsQueue: make(chan recoverBlobsRequest, 1),
+ isRecovering: make(map[common.Hash]*blobRecovery),
+ }, columns, blobs
+}
+
+func TestCanceledForcedRecoveryIsDroppedBeforeWork(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x01")
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(10), root).Return(nil, false, nil)
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), uint64(10), root).Return([]uint64{0, 1}, nil)
+
+ ctx, cancel := context.WithCancel(t.Context())
+ result := make(chan error, 1)
+ go func() { result <- d.ForceScheduleRecover(ctx, 10, root, 2) }()
+ request := <-d.recoverBlobsQueue
+ cancel()
+ require.ErrorIs(t, <-result, context.Canceled)
+
+ called := false
+ d.handleRecoverBlobsRequest(t.Context(), request, func(context.Context, recoverBlobsRequest) error {
+ called = true
+ return nil
+ })
+ require.False(t, called)
+}
+
+func TestAdmittedRecoverySurvivesCallerCancellation(t *testing.T) {
+ d, _, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x05")
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(50), root).Return(nil, false, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(50), root).Return([]*cltypes.BlobSidecar{{}}, true, nil)
+ callerCtx, cancelCaller := context.WithCancel(t.Context())
+ result := make(chan error, 1)
+ request := recoverBlobsRequest{slot: 50, blockRoot: root, expectedBlobs: 1, force: true, ctx: callerCtx, result: result}
+ d.handleRecoverBlobsRequest(t.Context(), request, func(ownerCtx context.Context, _ recoverBlobsRequest) error {
+ cancelCaller()
+ require.NoError(t, ownerCtx.Err())
+ return nil
+ })
+ require.NoError(t, <-result)
+}
+
+func TestLiveRecoveryCoalescesRetryAfterOwnerFailure(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ d.caplinConfig = &clparams.CaplinConfig{ArchiveBlobs: true}
+ d.recoverBlobsQueue = make(chan recoverBlobsRequest, maxBlobRecoveryWaiters+2)
+ root := common.HexToHash("0x06")
+ otherRoot := common.HexToHash("0x16")
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), gomock.Any(), gomock.Any()).Return([]uint64{0, 1}, nil).AnyTimes()
+ blobs.EXPECT().KzgCommitmentsCount(gomock.Any(), gomock.Any()).Return(uint32(0), nil).AnyTimes()
+
+ started := make(chan struct{})
+ release := make(chan struct{})
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ d.handleRecoverBlobsRequest(t.Context(), recoverBlobsRequest{slot: 60, blockRoot: root}, func(context.Context, recoverBlobsRequest) error {
+ close(started)
+ <-release
+ return errors.New("owner failed")
+ })
+ }()
+ <-started
+ for range maxBlobRecoveryWaiters {
+ require.NoError(t, d.TryScheduleRecover(60, root))
+ }
+ d.recoveringMutex.Lock()
+ waiterCount := len(d.isRecovering[root].waiters)
+ d.recoveringMutex.Unlock()
+ close(release)
+ <-done
+ require.Zero(t, waiterCount)
+
+ require.NoError(t, d.TryScheduleRecover(61, otherRoot))
+ requests := []recoverBlobsRequest{<-d.recoverBlobsQueue, <-d.recoverBlobsQueue}
+ counts := map[common.Hash]int{}
+ for _, request := range requests {
+ counts[request.blockRoot]++
+ }
+ require.Equal(t, 1, counts[root])
+ require.Equal(t, 1, counts[otherRoot])
+ require.Empty(t, d.recoverBlobsQueue)
+}
+
+func TestDequeuedDuplicateRecoveryRetriesOnceAfterOwnerFailure(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ d.caplinConfig = &clparams.CaplinConfig{ArchiveBlobs: true}
+ d.recoverBlobsQueue = make(chan recoverBlobsRequest, 3)
+ root := common.HexToHash("0x17")
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), uint64(71), root).Return([]uint64{0, 1}, nil).AnyTimes()
+ blobs.EXPECT().KzgCommitmentsCount(gomock.Any(), root).Return(uint32(0), nil).AnyTimes()
+
+ require.NoError(t, d.TryScheduleRecover(71, root))
+ require.NoError(t, d.TryScheduleRecover(71, root))
+ first := <-d.recoverBlobsQueue
+ duplicate := <-d.recoverBlobsQueue
+
+ started := make(chan struct{})
+ release := make(chan struct{})
+ done := make(chan struct{})
+ callbacks := 0
+ go func() {
+ defer close(done)
+ d.handleRecoverBlobsRequest(t.Context(), first, func(context.Context, recoverBlobsRequest) error {
+ callbacks++
+ close(started)
+ <-release
+ return errors.New("owner failed")
+ })
+ }()
+ <-started
+ d.handleRecoverBlobsRequest(t.Context(), duplicate, func(context.Context, recoverBlobsRequest) error {
+ t.Fatal("dequeued duplicate started concurrent recovery")
+ return nil
+ })
+ close(release)
+ <-done
+
+ select {
+ case retry := <-d.recoverBlobsQueue:
+ d.handleRecoverBlobsRequest(t.Context(), retry, func(context.Context, recoverBlobsRequest) error {
+ callbacks++
+ return errors.New("retry remained incomplete")
+ })
+ case <-time.After(time.Second):
+ t.Fatal("dequeued duplicate did not preserve a retry after owner failure")
+ }
+ require.Equal(t, 2, callbacks)
+ require.Empty(t, d.recoverBlobsQueue)
+}
+
+func TestRecoveryOwnerStopsWhenWorkerIsCanceled(t *testing.T) {
+ d, _, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x26")
+ blobs.EXPECT().KzgCommitmentsCount(gomock.Any(), root).Return(uint32(0), nil).AnyTimes()
+ workerCtx, cancelWorker := context.WithCancel(t.Context())
+ entered := make(chan struct{})
+ release := make(chan struct{})
+ callbackResult := make(chan error, 1)
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ d.handleRecoverBlobsRequest(workerCtx, recoverBlobsRequest{slot: 62, blockRoot: root}, func(ownerCtx context.Context, _ recoverBlobsRequest) error {
+ close(entered)
+ select {
+ case <-ownerCtx.Done():
+ callbackResult <- ownerCtx.Err()
+ return ownerCtx.Err()
+ case <-release:
+ return errors.New("test released callback")
+ }
+ })
+ }()
+ <-entered
+ cancelWorker()
+
+ select {
+ case err := <-callbackResult:
+ require.ErrorIs(t, err, context.Canceled)
+ case <-time.After(time.Second):
+ close(release)
+ <-done
+ t.Fatal("worker cancellation did not reach recovery callback")
+ }
+ <-done
+}
+
+func TestLiveRecoveryRequeuesWhenCoalescedOwnerRemainsIncomplete(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ d.caplinConfig = &clparams.CaplinConfig{ArchiveBlobs: true}
+ root := common.HexToHash("0x07")
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), uint64(70), root).Return([]uint64{0, 1}, nil).AnyTimes()
+ blobs.EXPECT().KzgCommitmentsCount(gomock.Any(), root).Return(uint32(0), nil).AnyTimes()
+ started := make(chan struct{})
+ release := make(chan struct{})
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ d.handleRecoverBlobsRequest(t.Context(), recoverBlobsRequest{slot: 70, blockRoot: root}, func(context.Context, recoverBlobsRequest) error {
+ close(started)
+ <-release
+ return nil
+ })
+ }()
+ <-started
+ require.NoError(t, d.TryScheduleRecover(70, root))
+ close(release)
+ <-done
+
+ select {
+ case request := <-d.recoverBlobsQueue:
+ require.Equal(t, uint64(70), request.slot)
+ require.Equal(t, root, request.blockRoot)
+ case <-time.After(time.Second):
+ t.Fatal("incomplete coalesced recovery was not requeued")
+ }
+}
+
+func TestNonForcedRecoveryBroadcastsCallbackFailure(t *testing.T) {
+ d, _, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x08")
+ wantErr := errors.New("column recovery failed")
+ blobs.EXPECT().KzgCommitmentsCount(gomock.Any(), root).Return(uint32(0), nil).Times(2)
+ result := make(chan error, 1)
+
+ d.handleRecoverBlobsRequest(t.Context(), recoverBlobsRequest{slot: 80, blockRoot: root, result: result}, func(context.Context, recoverBlobsRequest) error {
+ return wantErr
+ })
+
+ require.ErrorIs(t, <-result, wantErr)
+}
+
+func TestForcedRecoveryCoalescesOntoActiveResult(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x02")
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(20), root).Return(nil, false, nil)
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), uint64(20), root).Return([]uint64{0, 1}, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(20), root).Return([]*cltypes.BlobSidecar{{}}, true, nil)
+ d.isRecovering[root] = &blobRecovery{}
+
+ result := make(chan error, 1)
+ go func() { result <- d.ForceScheduleRecover(t.Context(), 20, root, 2) }()
+ request := <-d.recoverBlobsQueue
+ d.handleRecoverBlobsRequest(t.Context(), request, func(context.Context, recoverBlobsRequest) error {
+ t.Fatal("coalesced request started duplicate recovery")
+ return nil
+ })
+
+ select {
+ case err := <-result:
+ t.Fatalf("forced recovery returned before active recovery completed: %v", err)
+ default:
+ }
+ d.finishBlobRecovery(root, nil)
+ require.ErrorContains(t, <-result, "blob recovery did not complete")
+}
+
+func TestForcedRecoveryRechecksCompletedDataAfterQueueDelay(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x03")
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(30), root).Return(nil, false, nil)
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), uint64(30), root).Return([]uint64{0, 1}, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(30), root).Return([]*cltypes.BlobSidecar{{}, {}}, true, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(30), root).Return([]*cltypes.BlobSidecar{{}, {}}, true, nil)
+
+ result := make(chan error, 1)
+ go func() { result <- d.ForceScheduleRecover(t.Context(), 30, root, 2) }()
+ request := <-d.recoverBlobsQueue
+ d.handleRecoverBlobsRequest(t.Context(), request, func(context.Context, recoverBlobsRequest) error {
+ t.Fatal("completed queued request started stale forced recovery")
+ return nil
+ })
+ require.NoError(t, <-result)
+}
+
+func TestForcedRecoveryRejectsUnderreportedBlobMetadata(t *testing.T) {
+ d, columns, blobs := newRecoveryTestPeerDas(t)
+ root := common.HexToHash("0x04")
+ oneBlob := []*cltypes.BlobSidecar{{}}
+ twoBlobs := []*cltypes.BlobSidecar{{}, {}}
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(40), root).Return(oneBlob, true, nil)
+ columns.EXPECT().GetSavedColumnIndex(gomock.Any(), uint64(40), root).Return([]uint64{0, 1}, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(40), root).Return(oneBlob, true, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(40), root).Return(twoBlobs, true, nil)
+ blobs.EXPECT().ReadBlobSidecars(gomock.Any(), uint64(40), root).Return(twoBlobs, true, nil)
+
+ result := make(chan error, 1)
+ go func() { result <- d.ForceScheduleRecover(t.Context(), 40, root, 2) }()
+ request := <-d.recoverBlobsQueue
+ called := false
+ d.handleRecoverBlobsRequest(t.Context(), request, func(context.Context, recoverBlobsRequest) error {
+ called = true
+ return nil
+ })
+ require.True(t, called)
+ require.NoError(t, <-result)
+}
diff --git a/cl/persistence/blob_storage/blob_db.go b/cl/persistence/blob_storage/blob_db.go
index adce814c1bd..881a480a5f7 100644
--- a/cl/persistence/blob_storage/blob_db.go
+++ b/cl/persistence/blob_storage/blob_db.go
@@ -49,7 +49,8 @@ const (
type BlobStorage interface {
WriteBlobSidecars(ctx context.Context, blockRoot common.Hash, blobSidecars []*cltypes.BlobSidecar) error
RemoveBlobSidecars(ctx context.Context, slot uint64, blockRoot common.Hash) error
- ReadBlobSidecars(ctx context.Context, slot uint64, blockRoot common.Hash) (out []*cltypes.BlobSidecar, found bool, err error)
+ // ReadBlobSidecars returns all available sidecars and complete reports whether every indexed sidecar was read.
+ ReadBlobSidecars(ctx context.Context, slot uint64, blockRoot common.Hash) (out []*cltypes.BlobSidecar, complete bool, err error)
BlobSidecarExists(ctx context.Context, slot uint64, blockRoot common.Hash, idx uint64) (bool, error)
WriteStream(w io.Writer, slot uint64, blockRoot common.Hash, idx uint64) error // Used for P2P networking
KzgCommitmentsCount(ctx context.Context, blockRoot common.Hash) (uint32, error)
@@ -119,7 +120,7 @@ func (bs *BlobStore) WriteBlobSidecars(ctx context.Context, blockRoot common.Has
return tx.Commit()
}
-// ReadBlobSidecars reads the sidecars from the database. it assumes that all blobSidecars are for the same blockRoot and we have all of them.
+// ReadBlobSidecars returns every available sidecar and reports whether the indexed block set is complete.
func (bs *BlobStore) ReadBlobSidecars(ctx context.Context, slot uint64, blockRoot common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
tx, err := bs.db.BeginRo(ctx)
if err != nil {
@@ -134,15 +135,24 @@ func (bs *BlobStore) ReadBlobSidecars(ctx context.Context, slot uint64, blockRoo
if len(val) == 0 {
return nil, false, nil
}
+ if len(val) != 4 {
+ return nil, false, fmt.Errorf("invalid blob commitment count encoding length %d", len(val))
+ }
kzgCommitmentsLength := binary.LittleEndian.Uint32(val)
+ maxCommitments := bs.beaconChainConfig.MaxBlobsPerBlockUpperBound()
+ if uint64(kzgCommitmentsLength) > maxCommitments {
+ return nil, false, fmt.Errorf("blob commitment count %d exceeds maximum %d", kzgCommitmentsLength, maxCommitments)
+ }
var blobSidecars []*cltypes.BlobSidecar
+ complete := true
for i := range kzgCommitmentsLength {
_, filePath := blobSidecarFilePath(slot, uint64(i), blockRoot)
file, err := bs.fs.Open(filePath)
if err != nil {
if errors.Is(err, afero.ErrFileNotFound) {
- return nil, false, nil
+ complete = false
+ continue
}
return nil, false, err
}
@@ -159,7 +169,7 @@ func (bs *BlobStore) ReadBlobSidecars(ctx context.Context, slot uint64, blockRoo
}
blobSidecars = append(blobSidecars, blobSidecar)
}
- return blobSidecars, true, nil
+ return blobSidecars, complete, nil
}
// Do a bit of pruning
@@ -250,9 +260,38 @@ type sidecarsPayload struct {
type verifyHeaderSignatureFn func(header *cltypes.SignedBeaconBlockHeader) error
+// VerifyBlobSidecars validates sidecar proofs and optionally their signed headers.
+func VerifyBlobSidecars(sidecars []*cltypes.BlobSidecar, verifySignatureFn verifyHeaderSignatureFn) error {
+ if len(sidecars) == 0 {
+ return nil
+ }
+ blobs := make([]*goethkzg.Blob, len(sidecars))
+ commitments := make([]goethkzg.KZGCommitment, len(sidecars))
+ proofs := make([]goethkzg.KZGProof, len(sidecars))
+ for i, sidecar := range sidecars {
+ if sidecar == nil || sidecar.SignedBlockHeader == nil || sidecar.SignedBlockHeader.Header == nil {
+ return errors.New("blob response contains incomplete sidecar")
+ }
+ if !cltypes.VerifyCommitmentInclusionProof(sidecar.KzgCommitment, sidecar.CommitmentInclusionProof, sidecar.Index, clparams.DenebVersion, sidecar.SignedBlockHeader.Header.BodyRoot) {
+ return errors.New("could not verify blob's inclusion proof")
+ }
+ if verifySignatureFn != nil {
+ if err := verifySignatureFn(sidecar.SignedBlockHeader); err != nil {
+ return err
+ }
+ }
+ blobs[i] = (*goethkzg.Blob)(&sidecar.Blob)
+ commitments[i] = goethkzg.KZGCommitment(sidecar.KzgCommitment)
+ proofs[i] = goethkzg.KZGProof(sidecar.KzgProof)
+ }
+ if err := kzg.Ctx().VerifyBlobKZGProofBatch(blobs, commitments, proofs); err != nil {
+ return errors.New("sidecar is wrong")
+ }
+ return nil
+}
+
// VerifyAgainstIdentifiersAndInsertIntoTheBlobStore does all due verification for blobs before database insertion. it also returns the latest correctly return blob.
func VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(ctx context.Context, storage BlobStorage, identifiers *solid.ListSSZ[*cltypes.BlobIdentifier], sidecars []*cltypes.BlobSidecar, verifySignatureFn verifyHeaderSignatureFn) (uint64, uint64, error) {
- kzgCtx := kzg.Ctx()
inserted := atomic.Uint64{}
if identifiers.Len() == 0 || len(sidecars) == 0 {
return 0, 0, nil
@@ -260,6 +299,9 @@ func VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(ctx context.Context, stor
if len(sidecars) > identifiers.Len() {
return 0, 0, errors.New("sidecars length is greater than identifiers length")
}
+ if sidecars[0] == nil || sidecars[0].SignedBlockHeader == nil || sidecars[0].SignedBlockHeader.Header == nil {
+ return 0, 0, errors.New("blob response contains incomplete sidecar")
+ }
prevBlockRoot := identifiers.Get(0).BlockRoot
totalProcessed := 0
@@ -268,6 +310,9 @@ func VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(ctx context.Context, stor
lastProcessed := sidecars[0].SignedBlockHeader.Header.Slot
// Some will be stored, truncate when validation goes to shit
for i, sidecar := range sidecars {
+ if sidecar == nil || sidecar.SignedBlockHeader == nil || sidecar.SignedBlockHeader.Header == nil {
+ return 0, 0, errors.New("blob response contains incomplete sidecar")
+ }
identifier := identifiers.Get(i)
// check if the root of the block matches the identifier
sidecarBlockRoot, err := sidecar.SignedBlockHeader.Header.HashSSZ()
@@ -282,15 +327,6 @@ func VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(ctx context.Context, stor
break
}
- if !cltypes.VerifyCommitmentInclusionProof(sidecar.KzgCommitment, sidecar.CommitmentInclusionProof, sidecar.Index, clparams.DenebVersion, sidecar.SignedBlockHeader.Header.BodyRoot) {
- return 0, 0, errors.New("could not verify blob's inclusion proof")
- }
- if verifySignatureFn != nil {
- // verify the signature of the sidecar head, we leave this step up to the caller to define
- if err := verifySignatureFn(sidecar.SignedBlockHeader); err != nil {
- return 0, 0, err
- }
- }
// if the sidecar is valid, add it to the current payload of sidecars being built.
if identifier.BlockRoot != prevBlockRoot {
storableSidecars = append(storableSidecars, currentSidecarsPayload)
@@ -312,20 +348,8 @@ func VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(ctx context.Context, stor
var wg sync.WaitGroup
for _, sds := range storableSidecars {
wg.Go(func() {
- blobs := make([]*goethkzg.Blob, len(sds.sidecars))
- for i, sidecar := range sds.sidecars {
- blobs[i] = (*goethkzg.Blob)(&sidecar.Blob)
- }
- kzgCommitments := make([]goethkzg.KZGCommitment, len(sds.sidecars))
- for i, sidecar := range sds.sidecars {
- kzgCommitments[i] = goethkzg.KZGCommitment(sidecar.KzgCommitment)
- }
- kzgProofs := make([]goethkzg.KZGProof, len(sds.sidecars))
- for i, sidecar := range sds.sidecars {
- kzgProofs[i] = goethkzg.KZGProof(sidecar.KzgProof)
- }
- if err := kzgCtx.VerifyBlobKZGProofBatch(blobs, kzgCommitments, kzgProofs); err != nil {
- errAtomic.Store(errors.New("sidecar is wrong"))
+ if err := VerifyBlobSidecars(sds.sidecars, verifySignatureFn); err != nil {
+ errAtomic.Store(err)
return
}
if err := storage.WriteBlobSidecars(ctx, sds.blockRoot, sds.sidecars); err != nil {
diff --git a/cl/persistence/blob_storage/blob_db_test.go b/cl/persistence/blob_storage/blob_db_test.go
index badbc8f0747..858fc120fb0 100644
--- a/cl/persistence/blob_storage/blob_db_test.go
+++ b/cl/persistence/blob_storage/blob_db_test.go
@@ -18,6 +18,8 @@ package blob_storage
import (
"context"
+ "encoding/binary"
+ "strconv"
"testing"
"github.com/spf13/afero"
@@ -37,6 +39,40 @@ func setupTestDB(t *testing.T) kv.RwDB {
return db
}
+func TestReadBlobSidecarsRejectsOversizedCommitmentCount(t *testing.T) {
+ db := setupTestDB(t)
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.MaxBlobsPerBlock = 1
+ beaconCfg.MaxBlobsPerBlockElectra = 1
+ beaconCfg.BlobSchedule = nil
+ store := NewBlobStore(db, afero.NewMemMapFs(), 12, &beaconCfg, nil)
+ blockRoot := common.Hash{1}
+ count := make([]byte, 4)
+ binary.LittleEndian.PutUint32(count, 2)
+ require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error {
+ return tx.Put(kv.BlockRootToKzgCommitments, blockRoot[:], count)
+ }))
+
+ _, _, err := store.ReadBlobSidecars(t.Context(), 1, blockRoot)
+ require.ErrorContains(t, err, "commitment count 2 exceeds maximum 1")
+}
+
+func TestReadBlobSidecarsRejectsInvalidCommitmentCountEncoding(t *testing.T) {
+ for _, size := range []int{1, 2, 3, 5} {
+ t.Run(strconv.Itoa(size), func(t *testing.T) {
+ db := setupTestDB(t)
+ store := NewBlobStore(db, afero.NewMemMapFs(), 12, &clparams.MainnetBeaconConfig, nil)
+ blockRoot := common.Hash{byte(size)}
+ require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error {
+ return tx.Put(kv.BlockRootToKzgCommitments, blockRoot[:], make([]byte, size))
+ }))
+
+ _, _, err := store.ReadBlobSidecars(t.Context(), 1, blockRoot)
+ require.ErrorContains(t, err, "invalid blob commitment count encoding")
+ })
+ }
+}
+
func TestBlobDB(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
@@ -50,9 +86,9 @@ func TestBlobDB(t *testing.T) {
err := bs.WriteBlobSidecars(context.Background(), blockRoot, []*cltypes.BlobSidecar{s1, s2})
require.NoError(t, err)
- sidecars, found, err := bs.ReadBlobSidecars(context.Background(), 1, blockRoot)
+ sidecars, complete, err := bs.ReadBlobSidecars(context.Background(), 1, blockRoot)
require.NoError(t, err)
- require.True(t, found)
+ require.True(t, complete)
require.Len(t, sidecars, 2)
require.Equal(t, s1.Blob, sidecars[0].Blob)
@@ -68,3 +104,23 @@ func TestBlobDB(t *testing.T) {
require.Equal(t, s1.SignedBlockHeader, sidecars[0].SignedBlockHeader)
require.Equal(t, s2.SignedBlockHeader, sidecars[1].SignedBlockHeader)
}
+
+func TestReadBlobSidecarsReturnsAvailableSidecarsWhenBlockIsIncomplete(t *testing.T) {
+ db := setupTestDB(t)
+ fs := afero.NewMemMapFs()
+ store := NewBlobStore(db, fs, 12, &clparams.MainnetBeaconConfig, nil)
+ blockRoot := common.Hash{1}
+ sidecars := []*cltypes.BlobSidecar{
+ cltypes.NewBlobSidecar(0, &cltypes.Blob{1}, common.Bytes48{2}, common.Bytes48{3}, &cltypes.SignedBeaconBlockHeader{Header: &cltypes.BeaconBlockHeader{Slot: 1}}, solid.NewHashVector(cltypes.CommitmentBranchSize)),
+ cltypes.NewBlobSidecar(1, &cltypes.Blob{4}, common.Bytes48{5}, common.Bytes48{6}, &cltypes.SignedBeaconBlockHeader{Header: &cltypes.BeaconBlockHeader{Slot: 1}}, solid.NewHashVector(cltypes.CommitmentBranchSize)),
+ }
+ require.NoError(t, store.WriteBlobSidecars(t.Context(), blockRoot, sidecars))
+ _, missingPath := blobSidecarFilePath(1, 1, blockRoot)
+ require.NoError(t, fs.Remove(missingPath))
+
+ available, complete, err := store.ReadBlobSidecars(t.Context(), 1, blockRoot)
+ require.NoError(t, err)
+ require.False(t, complete)
+ require.Len(t, available, 1)
+ require.Equal(t, uint64(0), available[0].Index)
+}
diff --git a/cl/persistence/blob_storage/mock_services/blob_storage_mock.go b/cl/persistence/blob_storage/mock_services/blob_storage_mock.go
index 5068263e0eb..60e27762a2d 100644
--- a/cl/persistence/blob_storage/mock_services/blob_storage_mock.go
+++ b/cl/persistence/blob_storage/mock_services/blob_storage_mock.go
@@ -182,8 +182,8 @@ type MockBlobStorageReadBlobSidecarsCall struct {
}
// Return rewrite *gomock.Call.Return
-func (c *MockBlobStorageReadBlobSidecarsCall) Return(out []*cltypes.BlobSidecar, found bool, err error) *MockBlobStorageReadBlobSidecarsCall {
- c.Call = c.Call.Return(out, found, err)
+func (c *MockBlobStorageReadBlobSidecarsCall) Return(out []*cltypes.BlobSidecar, complete bool, err error) *MockBlobStorageReadBlobSidecarsCall {
+ c.Call = c.Call.Return(out, complete, err)
return c
}
diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go
index a31e3b4cff1..716c0a9c345 100644
--- a/cl/phase1/forkchoice/on_block.go
+++ b/cl/phase1/forkchoice/on_block.go
@@ -543,12 +543,20 @@ func (f *ForkChoiceStore) isDataAvailable(ctx context.Context, slot uint64, bloc
return true
})
// Blobs are preverified so we skip verification, we just need to check if commitments checks out.
- sidecars, foundOnDisk, err := f.blobStorage.ReadBlobSidecars(ctx, slot, blockRoot)
+ sidecars, completeOnDisk, err := f.blobStorage.ReadBlobSidecars(ctx, slot, blockRoot)
if err != nil {
return fmt.Errorf("cannot check data avaiability. failed to read blob sidecars: %v", err)
}
- if !foundOnDisk {
- sidecars = f.hotSidecars[blockRoot] // take it from memory
+ if !completeOnDisk {
+ byIndex := make(map[uint64]struct{}, len(sidecars))
+ for _, sidecar := range sidecars {
+ byIndex[sidecar.Index] = struct{}{}
+ }
+ for _, sidecar := range f.hotSidecars[blockRoot] {
+ if _, exists := byIndex[sidecar.Index]; !exists {
+ sidecars = append(sidecars, sidecar)
+ }
+ }
}
if blobKzgCommitments.Len() != len(sidecars) {
@@ -560,8 +568,7 @@ func (f *ForkChoiceStore) isDataAvailable(ctx context.Context, slot uint64, bloc
if len(commitmentsLeftToCheck) > 0 {
return ErrEIP4844DataNotAvailable // This should then schedule the block for reprocessing
}
- if !foundOnDisk {
- // If we didn't find the sidecars on disk, we should write them to disk now
+ if !completeOnDisk {
slices.SortFunc(sidecars, func(a, b *cltypes.BlobSidecar) int {
return cmp.Compare(a.Index, b.Index)
})
diff --git a/cl/phase1/network/blob_downloader.go b/cl/phase1/network/blob_downloader.go
index f8039bfa736..8fb7e1c541c 100644
--- a/cl/phase1/network/blob_downloader.go
+++ b/cl/phase1/network/blob_downloader.go
@@ -17,19 +17,24 @@
package network
import (
+ "cmp"
"context"
"errors"
"fmt"
+ "math"
+ "slices"
"sync"
"sync/atomic"
"time"
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/cltypes/solid"
"github.com/erigontech/erigon/cl/das"
+ "github.com/erigontech/erigon/cl/persistence/beacon_indicies"
"github.com/erigontech/erigon/cl/persistence/blob_storage"
- "github.com/erigontech/erigon/cl/rpc"
"github.com/erigontech/erigon/cl/utils"
+ "github.com/erigontech/erigon/common"
"github.com/erigontech/erigon/common/log/v3"
"github.com/erigontech/erigon/db/kv"
"github.com/erigontech/erigon/db/snapshotsync/freezeblocks"
@@ -55,30 +60,36 @@ type PeerDasGetter interface {
GetPeerDas() das.PeerDas
}
-type blobPeerCounter interface {
- Peers() (uint64, error)
+type forcedBlobRecoverer interface {
+ ForceScheduleRecover(context.Context, uint64, common.Hash, uint64) error
+}
+
+type blobBackfillRequester interface {
+ SendBlobsSidecarByIdentifierReqForBackfill(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error)
}
type blobSnapshotReader interface {
FrozenBlobs() uint64
}
+type blobRequestFn func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error)
+
// BlobHistoryDownloader downloads blob history backwards from a head slot
type BlobHistoryDownloader struct {
ctx context.Context
- beaconCfg *clparams.BeaconChainConfig
- rpc *rpc.BeaconRpcP2P
- peerCounter blobPeerCounter
- indiciesDB kv.RoDB
- blobStorage blob_storage.BlobStorage
- blockReader freezeblocks.BeaconSnapshotReader
- sn blobSnapshotReader
+ beaconCfg *clparams.BeaconChainConfig
+ rpc BlobPeerClient
+ indiciesDB kv.RoDB
+ blobStorage blob_storage.BlobStorage
+ blockReader freezeblocks.BeaconSnapshotReader
+ sn blobSnapshotReader
+ requestBlobs blobRequestFn
syncedChecker SyncedChecker
peerDasGetter PeerDasGetter
- // headSlot is the slot we start downloading from (currentSlot + 1)
+ // headSlot is the inclusive upper bound of the range to download.
headSlot atomic.Uint64
// highestBackfilledSlot is the highest slot we've successfully backfilled to
highestBackfilledSlot atomic.Uint64
@@ -88,11 +99,16 @@ type BlobHistoryDownloader struct {
archiveBlobs bool
// immediateBlobsBackfilling indicates whether to backfill blobs immediately
immediateBlobsBackfilling bool
- // columnBackfillTimeout bounds each fulu block's PeerDAS column recovery
- columnBackfillTimeout time.Duration
+ columnBackfillTimeout time.Duration
running atomic.Bool
backfillCompleted atomic.Bool
+ completedRanges []backfillRange
+ headRoot common.Hash
+ headGeneration uint64
+ activePasses map[*backfillPass]struct{}
+ transitionCeiling uint64
+ transitionActive bool
logger log.Logger
// notifyBlobBackfilled is called when blob backfilling is complete
@@ -105,7 +121,7 @@ type BlobHistoryDownloader struct {
func NewBlobHistoryDownloader(
ctx context.Context,
beaconCfg *clparams.BeaconChainConfig,
- rpc *rpc.BeaconRpcP2P,
+ rpc BlobPeerClient,
indiciesDB kv.RoDB,
blobStorage blob_storage.BlobStorage,
blockReader freezeblocks.BeaconSnapshotReader,
@@ -116,16 +132,21 @@ func NewBlobHistoryDownloader(
immediateBlobsBackfilling bool,
logger log.Logger,
) *BlobHistoryDownloader {
- targetSlot := beaconCfg.DenebForkEpoch * beaconCfg.SlotsPerEpoch
+ targetSlot, _ := denebStartSlot(beaconCfg)
return &BlobHistoryDownloader{
- ctx: ctx,
- beaconCfg: beaconCfg,
- rpc: rpc,
- peerCounter: rpc,
- indiciesDB: indiciesDB,
- blobStorage: blobStorage,
- blockReader: blockReader,
- sn: sn,
+ ctx: ctx,
+ beaconCfg: beaconCfg,
+ rpc: rpc,
+ indiciesDB: indiciesDB,
+ blobStorage: blobStorage,
+ blockReader: blockReader,
+ sn: sn,
+ requestBlobs: func(ctx context.Context, client BlobPeerClient, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ if backfillClient, ok := client.(blobBackfillRequester); ok {
+ return requestBlobsFranticallyForBackfill(ctx, backfillClient, req)
+ }
+ return RequestBlobsFrantically(ctx, client, req)
+ },
syncedChecker: syncedChecker,
peerDasGetter: peerDasGetter,
targetSlot: targetSlot,
@@ -136,11 +157,54 @@ func NewBlobHistoryDownloader(
}
}
-// SetHeadSlot sets the head slot to download from (should be currentSlot + 1)
+// SetHead sets the inclusive upper bound and preserves completion only through safeThrough.
+func (b *BlobHistoryDownloader) SetHead(slot uint64, root common.Hash, safeThrough uint64) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.constrainActivePasses(safeThrough)
+ b.headGeneration++
+ b.headSlot.Store(slot)
+ b.trimCompletedRanges(safeThrough)
+ b.headRoot = root
+ b.transitionActive = false
+ if !b.completedRangesContain(b.backfillRanges(slot)) {
+ b.backfillCompleted.Store(false)
+ }
+}
+
+// SetHeadSlot initializes the scan head without changing canonical completion metadata.
func (b *BlobHistoryDownloader) SetHeadSlot(slot uint64) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.headGeneration++
b.headSlot.Store(slot)
}
+// InvalidateCompletionAbove conservatively trims completed coverage before a canonical-head change is committed.
+func (b *BlobHistoryDownloader) InvalidateCompletionAbove(safeThrough uint64) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if b.transitionActive {
+ b.transitionCeiling = min(b.transitionCeiling, safeThrough)
+ } else {
+ b.transitionCeiling = safeThrough
+ b.transitionActive = true
+ }
+ b.constrainActivePasses(b.transitionCeiling)
+ b.headGeneration++
+ b.trimCompletedRanges(b.transitionCeiling)
+ if !b.completedRangesContain(b.backfillRanges(b.headSlot.Load())) {
+ b.backfillCompleted.Store(false)
+ }
+}
+
+// AbortHeadUpdate ends a failed canonical-head transition without restoring invalidated completion.
+func (b *BlobHistoryDownloader) AbortHeadUpdate() {
+ b.mu.Lock()
+ b.transitionActive = false
+ b.mu.Unlock()
+}
+
// SetNotifyBlobBackfilled sets the callback for when blob backfilling is complete
func (b *BlobHistoryDownloader) SetNotifyBlobBackfilled(notify func()) {
b.mu.Lock()
@@ -163,6 +227,193 @@ func (b *BlobHistoryDownloader) Running() bool {
return b.running.Load()
}
+// BlobBackfillPending reports whether a slot remains outside durable completed coverage.
+func (b *BlobHistoryDownloader) BlobBackfillPending(slot uint64) bool {
+ if !b.archiveBlobs && !b.immediateBlobsBackfilling {
+ return false
+ }
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+ headSlot := b.headSlot.Load()
+ if headSlot == 0 {
+ denebSlot, denebScheduled := forkStartSlot(b.beaconCfg.DenebForkEpoch, b.beaconCfg.SlotsPerEpoch)
+ return denebScheduled && slot >= denebSlot
+ }
+ if headSlot != 0 && slot > headSlot {
+ denebStart, scheduled := denebStartSlot(b.beaconCfg)
+ return scheduled && slot >= denebStart
+ }
+ if !b.slotWithinBackfillRange(slot, headSlot) {
+ return false
+ }
+ if b.transitionActive && slot > b.transitionCeiling {
+ return true
+ }
+ completed := b.completedSlot(slot)
+ return !completed
+}
+
+func denebStartSlot(beaconCfg *clparams.BeaconChainConfig) (uint64, bool) {
+ return forkStartSlot(beaconCfg.DenebForkEpoch, beaconCfg.SlotsPerEpoch)
+}
+
+func forkStartSlot(forkEpoch, slotsPerEpoch uint64) (uint64, bool) {
+ if forkEpoch == math.MaxUint64 || slotsPerEpoch == 0 || forkEpoch > math.MaxUint64/slotsPerEpoch {
+ return math.MaxUint64, false
+ }
+ return forkEpoch * slotsPerEpoch, true
+}
+
+func epochRetentionFloor(head, forkEpoch, retentionEpochs, slotsPerEpoch uint64) (uint64, bool) {
+ if _, ok := forkStartSlot(forkEpoch, slotsPerEpoch); !ok {
+ return math.MaxUint64, false
+ }
+ currentEpoch := head / slotsPerEpoch
+ earliestEpoch := currentEpoch - min(currentEpoch, retentionEpochs)
+ earliestEpoch = max(earliestEpoch, forkEpoch)
+ return earliestEpoch * slotsPerEpoch, true
+}
+
+type backfillRange struct{ start, end uint64 }
+
+type backfillPass struct {
+ safeThrough uint64
+}
+
+func (b *BlobHistoryDownloader) constrainActivePasses(safeThrough uint64) {
+ for pass := range b.activePasses {
+ pass.safeThrough = min(pass.safeThrough, safeThrough)
+ }
+}
+
+func (b *BlobHistoryDownloader) addPassCompletedRanges(pass *backfillPass, ranges []backfillRange) {
+ accepted := make([]backfillRange, 0, len(ranges))
+ for _, completed := range ranges {
+ if completed.start > pass.safeThrough {
+ continue
+ }
+ completed.end = min(completed.end, pass.safeThrough)
+ accepted = append(accepted, completed)
+ }
+ b.addCompletedRanges(accepted)
+}
+
+func (b *BlobHistoryDownloader) backfillRanges(head uint64) []backfillRange {
+ denebStart, denebScheduled := denebStartSlot(b.beaconCfg)
+ if b.archiveBlobs && b.targetSlot != 0 {
+ denebStart, denebScheduled = b.targetSlot, true
+ }
+ if !denebScheduled || head < denebStart {
+ return nil
+ }
+ fuluStart, fuluScheduled := forkStartSlot(b.beaconCfg.FuluForkEpoch, b.beaconCfg.SlotsPerEpoch)
+ ranges := make([]backfillRange, 0, 2)
+ if !fuluScheduled || denebStart < fuluStart {
+ denebFloor := denebStart
+ if b.archiveBlobs && b.targetSlot != 0 {
+ denebFloor = b.targetSlot
+ } else if !b.archiveBlobs {
+ denebFloor, _ = epochRetentionFloor(head, b.beaconCfg.DenebForkEpoch, b.beaconCfg.MinEpochsForBlobSidecarsRequests, b.beaconCfg.SlotsPerEpoch)
+ }
+ if !fuluScheduled || denebFloor < fuluStart {
+ end := head
+ if fuluScheduled && fuluStart > 0 {
+ end = min(end, fuluStart-1)
+ }
+ if denebFloor <= end {
+ ranges = append(ranges, backfillRange{denebFloor, end})
+ }
+ }
+ }
+ if fuluScheduled && head >= fuluStart {
+ fuluFloor, _ := epochRetentionFloor(head, b.beaconCfg.FuluForkEpoch, b.beaconCfg.MinEpochsForDataColumnSidecarsRequests, b.beaconCfg.SlotsPerEpoch)
+ ranges = append(ranges, backfillRange{fuluFloor, head})
+ }
+ return ranges
+}
+
+func (b *BlobHistoryDownloader) slotWithinBackfillRange(slot, head uint64) bool {
+ for _, r := range b.backfillRanges(head) {
+ if slot >= r.start && slot <= r.end {
+ return true
+ }
+ }
+ return false
+}
+
+func (b *BlobHistoryDownloader) completedRangeContains(start, end uint64) bool {
+ for _, completed := range b.completedRanges {
+ if start >= completed.start && end <= completed.end {
+ return true
+ }
+ }
+ return false
+}
+
+func (b *BlobHistoryDownloader) completedSlot(slot uint64) bool {
+ return b.completedRangeContains(slot, slot)
+}
+
+func (b *BlobHistoryDownloader) completedRangesContain(ranges []backfillRange) bool {
+ for _, r := range ranges {
+ if !b.completedRangeContains(r.start, r.end) {
+ return false
+ }
+ }
+ return len(ranges) > 0
+}
+
+func (b *BlobHistoryDownloader) trimCompletedRanges(safeThrough uint64) {
+ kept := b.completedRanges[:0]
+ for _, r := range b.completedRanges {
+ if r.start > safeThrough {
+ break
+ }
+ r.end = min(r.end, safeThrough)
+ kept = append(kept, r)
+ }
+ b.completedRanges = kept
+}
+
+func (b *BlobHistoryDownloader) addCompletedRanges(ranges []backfillRange) {
+ all := append(append(make([]backfillRange, 0, len(b.completedRanges)+len(ranges)), b.completedRanges...), ranges...)
+ slices.SortFunc(all, func(a, c backfillRange) int { return cmp.Compare(a.start, c.start) })
+ b.completedRanges = b.completedRanges[:0]
+ for _, r := range all {
+ last := len(b.completedRanges) - 1
+ if last < 0 || !intervalsTouch(r.start, r.end, b.completedRanges[last].start, b.completedRanges[last].end) {
+ b.completedRanges = append(b.completedRanges, r)
+ continue
+ }
+ b.completedRanges[last].end = max(b.completedRanges[last].end, r.end)
+ }
+}
+
+func (b *BlobHistoryDownloader) incompleteRanges(desired []backfillRange) []backfillRange {
+ pending := make([]backfillRange, 0, len(desired))
+ for _, want := range desired {
+ cursor := want.start
+ coveredThroughEnd := false
+ for _, done := range b.completedRanges {
+ if done.end < cursor || done.start > want.end {
+ continue
+ }
+ if done.start > cursor {
+ pending = append(pending, backfillRange{cursor, done.start - 1})
+ }
+ if done.end >= want.end {
+ coveredThroughEnd = true
+ break
+ }
+ cursor = done.end + 1
+ }
+ if !coveredThroughEnd && cursor <= want.end {
+ pending = append(pending, backfillRange{cursor, want.end})
+ }
+ }
+ return pending
+}
+
// Start begins the blob history download loop, querying every 12 seconds
func (b *BlobHistoryDownloader) Start() {
if !b.archiveBlobs && !b.immediateBlobsBackfilling {
@@ -209,14 +460,42 @@ func (b *BlobHistoryDownloader) run() {
// downloadOnce performs a single download pass
func (b *BlobHistoryDownloader) downloadOnce(shouldLog bool) error {
- currentSlot := b.headSlot.Load()
- if currentSlot == 0 {
+ b.mu.Lock()
+ headSlot := b.headSlot.Load()
+ desiredRanges := b.backfillRanges(headSlot)
+ if len(desiredRanges) == 0 {
+ b.backfillCompleted.Store(true)
+ b.mu.Unlock()
+ return nil
+ }
+ pendingRanges := b.incompleteRanges(desiredRanges)
+ if len(pendingRanges) == 0 {
+ b.backfillCompleted.Store(true)
+ b.mu.Unlock()
+ return nil
+ }
+ passCeiling := uint64(math.MaxUint64)
+ if b.transitionActive {
+ passCeiling = b.transitionCeiling
+ }
+ pass := &backfillPass{safeThrough: passCeiling}
+ if b.activePasses == nil {
+ b.activePasses = make(map[*backfillPass]struct{})
+ }
+ b.activePasses[pass] = struct{}{}
+ b.mu.Unlock()
+ defer func() {
+ b.mu.Lock()
+ delete(b.activePasses, pass)
+ b.mu.Unlock()
+ }()
+ if headSlot == 0 {
return nil // not initialized yet
}
- startSlot := currentSlot
+ b.backfillCompleted.Store(false)
// Check peer count before proceeding
- peers, err := b.peerCounter.Peers()
+ peers, err := b.rpc.Peers()
if err != nil {
b.logger.Warn("[BlobHistoryDownloader] Failed to get peer count", "err", err)
return nil
@@ -229,93 +508,111 @@ func (b *BlobHistoryDownloader) downloadOnce(shouldLog bool) error {
logInterval := time.NewTicker(blobLogInterval)
defer logInterval.Stop()
- prevLogSlot := currentSlot
+ prevLogSlot := pendingRanges[len(pendingRanges)-1].end
prevTime := time.Now()
- targetSlot := b.targetSlot
- // in case of non-archive mode, we only backfill the last relevant epochs
- if !b.archiveBlobs {
- targetSlot = currentSlot - min(currentSlot, b.beaconCfg.MinSlotsForBlobsSidecarsRequest())
- }
-
- defer func() {
- // set target slot back in case it was modified
- b.targetSlot = currentSlot - b.beaconCfg.SlotsPerEpoch*2
- }()
-
if shouldLog {
- b.logger.Info("[BlobHistoryDownloader] Downloading blobs backwards", "slot", currentSlot)
+ b.logger.Info("[BlobHistoryDownloader] Downloading blobs backwards", "slot", prevLogSlot)
}
- for currentSlot >= targetSlot {
- firstUnfrozenSlot := max(targetSlot, b.sn.FrozenBlobs())
- if currentSlot < firstUnfrozenSlot {
- break
- }
- if !b.syncedChecker.Synced() {
- time.Sleep(5 * time.Second)
- continue
- }
+ var passErr error
+ for _, work := range slices.Backward(pendingRanges) {
+ currentSlot := work.end
+ targetSlot := work.start
+ for currentSlot >= targetSlot {
+ firstUnfrozenSlot := max(targetSlot, b.sn.FrozenBlobs())
+ if currentSlot < firstUnfrozenSlot {
+ break
+ }
+ if !b.syncedChecker.Synced() {
+ time.Sleep(5 * time.Second)
+ continue
+ }
- batch, visited, err := b.collectIncompleteBlocks(currentSlot, firstUnfrozenSlot)
- if err != nil {
- return err
- }
+ batch, visited, err := b.collectIncompleteBlocks(currentSlot, firstUnfrozenSlot, headSlot)
+ if err != nil {
+ return err
+ }
- if len(batch) > 0 {
- select {
- case <-b.ctx.Done():
- return b.ctx.Err()
- case <-logInterval.C:
- if shouldLog {
- slotSec := float64(prevLogSlot-currentSlot) / time.Since(prevTime).Seconds()
- prevLogSlot = currentSlot
- prevTime = time.Now()
- progress := 0.0
- if startSlot > targetSlot {
- progress = float64(startSlot-currentSlot) / float64(startSlot-targetSlot) * 100
+ if len(batch) > 0 {
+ select {
+ case <-b.ctx.Done():
+ return b.ctx.Err()
+ case <-logInterval.C:
+ if shouldLog {
+ slotSec := float64(prevLogSlot-currentSlot) / time.Since(prevTime).Seconds()
+ prevLogSlot = currentSlot
+ prevTime = time.Now()
+ b.logger.Info("[BlobHistoryDownloader] Downloading blobs backwards",
+ "slot", currentSlot, "to", targetSlot,
+ "slots/sec", fmt.Sprintf("%.2f", slotSec),
+ "eta", utils.ETA(currentSlot-targetSlot, slotSec))
+ }
+ default:
+ }
+ if err := b.processBatch(batch); err != nil {
+ passErr = errors.Join(passErr, err)
+ } else {
+ b.highestBackfilledSlot.Store(currentSlot)
+ }
+ for _, block := range batch {
+ root, err := block.Block.HashSSZ()
+ if err != nil {
+ passErr = errors.Join(passErr, err)
+ continue
+ }
+ complete, err := b.actualBlobSetComplete(block, root)
+ if err != nil {
+ passErr = errors.Join(passErr, err)
+ continue
+ }
+ if complete {
+ b.mu.Lock()
+ b.addPassCompletedRanges(pass, []backfillRange{{start: block.GetSlot(), end: block.GetSlot()}})
+ b.mu.Unlock()
}
- b.logger.Info("[BlobHistoryDownloader] Downloading blobs backwards",
- "slot", currentSlot, "to", targetSlot,
- "slots/sec", fmt.Sprintf("%.2f", slotSec),
- "progress", fmt.Sprintf("%.1f%%", progress),
- "eta", utils.ETA(currentSlot-targetSlot, slotSec))
}
- default:
}
- b.processBatch(batch)
- b.highestBackfilledSlot.Store(currentSlot)
- }
- // Always advance so an uncompletable batch can't rebuild at the same slot forever.
- // step>=1 guarantees progress; stop once the distance left to the floor is below one
- // step. The loop guard keeps currentSlot>=targetSlot, so neither subtraction underflows.
- step := max(visited, 1)
- if currentSlot-targetSlot < step {
- break
+ step := max(visited, 1)
+ if currentSlot-targetSlot < step {
+ break
+ }
+ currentSlot -= step
}
- currentSlot -= step
+ }
+ if passErr != nil {
+ b.backfillCompleted.Store(false)
+ return passErr
}
if shouldLog {
b.logger.Info("[BlobHistoryDownloader] Blob history download finished successfully")
}
- b.backfillCompleted.Store(true)
-
- b.mu.RLock()
+ b.mu.Lock()
+ b.addPassCompletedRanges(pass, pendingRanges)
+ currentDesiredRanges := b.backfillRanges(b.headSlot.Load())
+ complete := b.completedRangesContain(currentDesiredRanges)
+ b.backfillCompleted.Store(complete)
notify := b.notifyBlobBackfilled
- b.mu.RUnlock()
- if notify != nil {
+ b.mu.Unlock()
+ if complete && notify != nil {
notify()
}
return nil
}
+func intervalsTouch(firstStart, firstEnd, secondStart, secondEnd uint64) bool {
+ firstBeforeSecond := firstEnd < secondStart && (firstEnd == math.MaxUint64 || secondStart-firstEnd > 1)
+ secondBeforeFirst := secondEnd < firstStart && (secondEnd == math.MaxUint64 || firstStart-secondEnd > 1)
+ return !firstBeforeSecond && !secondBeforeFirst
+}
+
// collectIncompleteBlocks scans backwards from currentSlot for Deneb+ blocks still
// missing blobs. Its read tx is released before the caller's network download.
-func (b *BlobHistoryDownloader) collectIncompleteBlocks(currentSlot, targetSlot uint64) (batch []*cltypes.SignedBeaconBlock, visited uint64, err error) {
+func (b *BlobHistoryDownloader) collectIncompleteBlocks(currentSlot, targetSlot, rangeHead uint64) (batch []*cltypes.SignedBeaconBlock, visited uint64, err error) {
tx, err := b.indiciesDB.BeginRo(b.ctx)
if err != nil {
return nil, 0, err
@@ -332,16 +629,22 @@ func (b *BlobHistoryDownloader) collectIncompleteBlocks(currentSlot, targetSlot
return nil, 0, err
}
if block == nil {
+ canonicalRoot, err := beacon_indicies.ReadCanonicalBlockRoot(tx, currentSlot-visited)
+ if err != nil {
+ return nil, 0, err
+ }
+ if canonicalRoot != (common.Hash{}) {
+ return nil, 0, fmt.Errorf("canonical block body is unavailable at slot %d", currentSlot-visited)
+ }
continue
}
if block.Version() < clparams.DenebVersion {
break
}
- blockRoot, err := block.Block.HashSSZ()
- if err != nil {
- return nil, 0, err
+ if !b.slotWithinBackfillRange(currentSlot-visited, rangeHead) {
+ continue
}
- blobsCount, err := b.blobStorage.KzgCommitmentsCount(b.ctx, blockRoot)
+ blockRoot, err := block.Block.HashSSZ()
if err != nil {
return nil, 0, err
}
@@ -352,7 +655,11 @@ func (b *BlobHistoryDownloader) collectIncompleteBlocks(currentSlot, targetSlot
b.logger.Warn("[BlobHistoryDownloader] skipping block with nil kzg commitments", "slot", block.Block.Slot, "version", block.Version())
continue
}
- if commitments.Len() == int(blobsCount) {
+ complete, err := b.actualBlobSetComplete(block, blockRoot)
+ if err != nil {
+ return nil, 0, err
+ }
+ if complete {
continue
}
batch = append(batch, block)
@@ -360,9 +667,47 @@ func (b *BlobHistoryDownloader) collectIncompleteBlocks(currentSlot, targetSlot
return batch, visited, nil
}
-// processBatch best-effort recovers each block's blobs: Deneb by-root, Fulu from
-// PeerDAS columns.
-func (b *BlobHistoryDownloader) processBatch(batch []*cltypes.SignedBeaconBlock) {
+func (b *BlobHistoryDownloader) actualBlobSetComplete(block *cltypes.SignedBeaconBlock, blockRoot common.Hash) (bool, error) {
+ commitments := blobCommitments(block)
+ if commitments == nil {
+ return false, nil
+ }
+ if commitments.Len() == 0 {
+ return true, nil
+ }
+ count, err := b.blobStorage.KzgCommitmentsCount(b.ctx, blockRoot)
+ if err != nil || count != uint32(commitments.Len()) {
+ return false, err
+ }
+ sidecars, complete, err := b.blobStorage.ReadBlobSidecars(b.ctx, block.GetSlot(), blockRoot)
+ if err != nil || !complete || len(sidecars) != commitments.Len() {
+ return false, err
+ }
+ for index, sidecar := range sidecars {
+ if sidecar == nil || sidecar.SignedBlockHeader == nil || sidecar.SignedBlockHeader.Header == nil ||
+ sidecar.Index != uint64(index) || sidecar.SignedBlockHeader.Header.Slot != block.GetSlot() ||
+ commitments.Get(index) == nil || sidecar.KzgCommitment != common.Bytes48(*commitments.Get(index)) {
+ return false, nil
+ }
+ sidecarRoot, err := sidecar.SignedBlockHeader.Header.HashSSZ()
+ if err != nil {
+ return false, err
+ }
+ if sidecarRoot != blockRoot {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+func blobCommitments(block *cltypes.SignedBeaconBlock) *solid.ListSSZ[*cltypes.KZGCommitment] {
+ if block == nil || block.Block == nil || block.Block.Body == nil {
+ return nil
+ }
+ return block.Block.Body.GetBlobKzgCommitments()
+}
+
+func (b *BlobHistoryDownloader) processBatch(batch []*cltypes.SignedBeaconBlock) error {
fuluBlocks := make([]*cltypes.SignedBeaconBlock, 0, len(batch))
denebBlocks := make([]*cltypes.SignedBeaconBlock, 0, len(batch))
for _, block := range batch {
@@ -372,56 +717,225 @@ func (b *BlobHistoryDownloader) processBatch(batch []*cltypes.SignedBeaconBlock)
denebBlocks = append(denebBlocks, block)
}
}
+ var denebErr error
if len(denebBlocks) > 0 {
- b.recoverDenebBlobs(denebBlocks)
+ denebErr = b.recoverDenebBlobs(denebBlocks)
}
+ var fuluErr error
if len(fuluBlocks) > 0 {
- b.recoverFuluColumns(fuluBlocks)
+ fuluErr = b.recoverFuluColumns(fuluBlocks)
}
+ return errors.Join(denebErr, fuluErr)
}
-func (b *BlobHistoryDownloader) recoverDenebBlobs(blocks []*cltypes.SignedBeaconBlock) {
+func (b *BlobHistoryDownloader) recoverDenebBlobs(blocks []*cltypes.SignedBeaconBlock) error {
req, err := BlobsIdentifiersFromBlocks(blocks, b.beaconCfg)
if err != nil {
- b.logger.Debug("[BlobHistoryDownloader] Error generating blob identifiers", "err", err)
- return
+ return fmt.Errorf("generate blob identifiers: %w", err)
}
- blobs, err := RequestBlobsFrantically(b.ctx, b.rpc, req)
+ batch, err := newDenebRecoveryBatch(blocks, req)
if err != nil {
- b.logger.Debug("[BlobHistoryDownloader] Error requesting blobs", "err", err)
- return
+ return err
}
- _, _, err = blob_storage.VerifyAgainstIdentifiersAndInsertIntoTheBlobStore(b.ctx, b.blobStorage, req, blobs.Responses, func(header *cltypes.SignedBeaconBlockHeader) error {
- // The block is preverified so just check that the signature is correct against the block
- for _, block := range blocks {
- if block.Block.Slot != header.Header.Slot {
- continue
- }
- if block.Signature != header.Signature {
- return errors.New("signature mismatch between blob and stored block")
- }
- return nil
+ remaining := req
+ for remaining.Len() > 0 {
+ blobs, err := b.requestBlobs(b.ctx, b.rpc, remaining)
+ if err != nil {
+ return fmt.Errorf("request blobs: %w", err)
}
- return errors.New("block not in batch")
- })
- if err != nil {
- // Best-effort backfill: log and move on rather than banning a peer from the
- // shared live-sync pool for a historical-blob verification miss.
- b.logger.Warn("[BlobHistoryDownloader] Error verifying blobs", "err", err)
+ if blobs == nil {
+ return errors.New("request blobs: empty result")
+ }
+ progress, err := batch.validate(remaining, blobs.Responses)
+ if err != nil {
+ return err
+ }
+ if progress == 0 {
+ return errors.New("request blobs: response made no progress")
+ }
+ if err := batch.store(b.ctx, b.blobStorage); err != nil {
+ return err
+ }
+ remaining = batch.remaining()
}
+ return nil
}
-// recoverFuluColumns recovers blobs from PeerDAS columns, bounding each attempt so
-// columns no peer still serves can't block the backfill indefinitely.
-func (b *BlobHistoryDownloader) recoverFuluColumns(blocks []*cltypes.SignedBeaconBlock) {
+func (b *BlobHistoryDownloader) recoverFuluColumns(blocks []*cltypes.SignedBeaconBlock) error {
peerDas := b.peerDasGetter.GetPeerDas()
+ var recoveryErr error
for _, block := range blocks {
+ commitments := blobCommitments(block)
+ if commitments == nil {
+ recoveryErr = errors.Join(recoveryErr, errors.New("cannot recover Fulu block without kzg commitments"))
+ continue
+ }
// [Modified in Gloas:EIP7732] Use ColumnSyncableSignedBlock interface
ctx, cancel := context.WithTimeout(b.ctx, b.columnBackfillTimeout)
err := peerDas.DownloadColumnsAndRecoverBlobs(ctx, []cltypes.ColumnSyncableSignedBlock{block})
cancel()
if err != nil {
b.logger.Warn("[BlobHistoryDownloader] Error recovering blobs from block", "err", err, "slot", block.GetSlot())
+ recoveryErr = errors.Join(recoveryErr, err)
+ continue
+ }
+ blockRoot, err := block.Block.HashSSZ()
+ if err != nil {
+ recoveryErr = errors.Join(recoveryErr, err)
+ continue
+ }
+ complete, err := b.actualBlobSetComplete(block, blockRoot)
+ if err != nil {
+ recoveryErr = errors.Join(recoveryErr, err)
+ continue
+ }
+ if !complete {
+ forced, ok := peerDas.(forcedBlobRecoverer)
+ if !ok {
+ recoveryErr = errors.Join(recoveryErr, fmt.Errorf("incomplete Fulu blob recovery at slot %d", block.GetSlot()))
+ continue
+ }
+ ctx, cancel := context.WithTimeout(b.ctx, b.columnBackfillTimeout)
+ err = forced.ForceScheduleRecover(ctx, block.GetSlot(), blockRoot, uint64(commitments.Len()))
+ cancel()
+ if err != nil {
+ recoveryErr = errors.Join(recoveryErr, err)
+ continue
+ }
+ complete, err = b.actualBlobSetComplete(block, blockRoot)
+ if err != nil {
+ recoveryErr = errors.Join(recoveryErr, err)
+ continue
+ }
+ if !complete {
+ recoveryErr = errors.Join(recoveryErr, fmt.Errorf("incomplete Fulu blob recovery at slot %d", block.GetSlot()))
+ }
+ }
+ }
+ return recoveryErr
+}
+
+type blobIdentifierKey struct {
+ root common.Hash
+ index uint64
+}
+
+type requestedBlobBlock struct {
+ block *cltypes.SignedBeaconBlock
+ ids []*cltypes.BlobIdentifier
+ sidecars map[uint64]*cltypes.BlobSidecar
+ stored bool
+}
+
+type denebRecoveryBatch struct {
+ groups map[common.Hash]*requestedBlobBlock
+ order []common.Hash
+}
+
+func newDenebRecoveryBatch(blocks []*cltypes.SignedBeaconBlock, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*denebRecoveryBatch, error) {
+ batch := &denebRecoveryBatch{
+ groups: make(map[common.Hash]*requestedBlobBlock, len(blocks)),
+ order: make([]common.Hash, 0, len(blocks)),
+ }
+ blocksByRoot := make(map[common.Hash]*cltypes.SignedBeaconBlock, len(blocks))
+ for _, block := range blocks {
+ root, err := block.Block.HashSSZ()
+ if err != nil {
+ return nil, err
+ }
+ blocksByRoot[root] = block
+ }
+ for i := range req.Len() {
+ id := req.Get(i)
+ group := batch.groups[id.BlockRoot]
+ if group == nil {
+ group = &requestedBlobBlock{block: blocksByRoot[id.BlockRoot], sidecars: make(map[uint64]*cltypes.BlobSidecar)}
+ batch.groups[id.BlockRoot] = group
+ batch.order = append(batch.order, id.BlockRoot)
+ }
+ group.ids = append(group.ids, id)
+ }
+ return batch, nil
+}
+
+func (b *denebRecoveryBatch) validate(req *solid.ListSSZ[*cltypes.BlobIdentifier], sidecars []*cltypes.BlobSidecar) (int, error) {
+ requested := make(map[blobIdentifierKey]struct{}, req.Len())
+ for i := range req.Len() {
+ id := req.Get(i)
+ requested[blobIdentifierKey{root: id.BlockRoot, index: id.Index}] = struct{}{}
+ }
+ seen := make(map[blobIdentifierKey]struct{}, len(sidecars))
+ for _, sidecar := range sidecars {
+ if sidecar == nil || sidecar.SignedBlockHeader == nil || sidecar.SignedBlockHeader.Header == nil {
+ return 0, errors.New("blob response contains incomplete sidecar")
+ }
+ root, err := sidecar.SignedBlockHeader.Header.HashSSZ()
+ if err != nil {
+ return 0, err
+ }
+ key := blobIdentifierKey{root: root, index: sidecar.Index}
+ if _, ok := requested[key]; !ok {
+ return 0, fmt.Errorf("unrequested blob sidecar %x:%d", root, sidecar.Index)
+ }
+ group := b.groups[root]
+ if group == nil || group.block == nil {
+ return 0, fmt.Errorf("blob response block %x is not in batch", root)
+ }
+ if _, duplicate := group.sidecars[sidecar.Index]; duplicate {
+ return 0, fmt.Errorf("duplicate blob sidecar %x:%d", root, sidecar.Index)
+ }
+ if _, duplicate := seen[key]; duplicate {
+ return 0, fmt.Errorf("duplicate blob sidecar %x:%d", root, sidecar.Index)
+ }
+ if sidecar.SignedBlockHeader.Signature != group.block.Signature {
+ return 0, errors.New("signature mismatch between blob and stored block")
+ }
+ seen[key] = struct{}{}
+ }
+ if err := blob_storage.VerifyBlobSidecars(sidecars, nil); err != nil {
+ return 0, err
+ }
+ for _, sidecar := range sidecars {
+ root, err := sidecar.SignedBlockHeader.Header.HashSSZ()
+ if err != nil {
+ return 0, err
+ }
+ group := b.groups[root]
+ group.sidecars[sidecar.Index] = sidecar
+ }
+ return len(sidecars), nil
+}
+
+func (b *denebRecoveryBatch) store(ctx context.Context, storage blob_storage.BlobStorage) error {
+ for _, root := range b.order {
+ group := b.groups[root]
+ if group.stored || len(group.sidecars) != len(group.ids) {
+ continue
+ }
+ orderedSidecars := make([]*cltypes.BlobSidecar, 0, len(group.ids))
+ for _, id := range group.ids {
+ orderedSidecars = append(orderedSidecars, group.sidecars[id.Index])
+ }
+ if err := storage.WriteBlobSidecars(ctx, root, orderedSidecars); err != nil {
+ return fmt.Errorf("store blobs: %w", err)
+ }
+ group.stored = true
+ }
+ return nil
+}
+
+func (b *denebRecoveryBatch) remaining() *solid.ListSSZ[*cltypes.BlobIdentifier] {
+ remaining := solid.NewStaticListSSZ[*cltypes.BlobIdentifier](0, 40)
+ for _, root := range b.order {
+ group := b.groups[root]
+ if group.stored {
+ continue
+ }
+ for _, id := range group.ids {
+ if group.sidecars[id.Index] == nil {
+ remaining.Append(id)
+ }
}
}
+ return remaining
}
diff --git a/cl/phase1/network/blob_downloader_boundary_test.go b/cl/phase1/network/blob_downloader_boundary_test.go
index 4d27f906d83..15823fc24a3 100644
--- a/cl/phase1/network/blob_downloader_boundary_test.go
+++ b/cl/phase1/network/blob_downloader_boundary_test.go
@@ -19,6 +19,7 @@ package network
import (
"context"
"errors"
+ "math"
"sync/atomic"
"testing"
@@ -26,6 +27,9 @@ import (
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/cltypes/solid"
+ "github.com/erigontech/erigon/cl/persistence/beacon_indicies"
+ "github.com/erigontech/erigon/common"
"github.com/erigontech/erigon/common/log/v3"
"github.com/erigontech/erigon/db/kv"
"github.com/erigontech/erigon/db/kv/dbcfg"
@@ -83,12 +87,35 @@ func TestBlobHistoryDownloaderWaitsWithoutPeers(t *testing.T) {
require.Empty(t, reader.slots)
}
+func TestBlobHistoryDownloaderErrorsWhenCanonicalBodyIsUnavailable(t *testing.T) {
+ const slot = uint64(100)
+ downloader := newBoundaryDownloader(t, slot, 0, slot, 1, &boundaryBlockReader{})
+ tx, err := downloader.indiciesDB.(kv.RwDB).BeginRw(t.Context())
+ require.NoError(t, err)
+ defer tx.Rollback()
+ require.NoError(t, beacon_indicies.MarkRootCanonical(t.Context(), tx, slot, common.HexToHash("0x01")))
+ require.NoError(t, tx.Commit())
+
+ require.ErrorContains(t, downloader.downloadOnce(false), "canonical block body is unavailable")
+}
+
+func TestBlobHistoryDownloaderTreatsZeroCanonicalRootAsEmptySlot(t *testing.T) {
+ const slot = uint64(100)
+ downloader := newBoundaryDownloader(t, slot, 0, slot, 1, &boundaryBlockReader{})
+
+ require.NoError(t, downloader.downloadOnce(false))
+ require.False(t, downloader.BlobBackfillPending(slot))
+}
+
func newBoundaryDownloader(t *testing.T, headSlot, frozenBlobs, targetSlot, peers uint64, reader freezeblocks.BeaconSnapshotReader) *BlobHistoryDownloader {
t.Helper()
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = math.MaxUint64
downloader := &BlobHistoryDownloader{
ctx: t.Context(),
- beaconCfg: &clparams.MainnetBeaconConfig,
- peerCounter: boundaryPeerCounter(peers),
+ beaconCfg: &beaconCfg,
+ rpc: boundaryPeerClient(peers),
indiciesDB: memdb.NewTestDB(t, dbcfg.ChainDB),
blockReader: reader,
sn: boundarySnapshot(frozenBlobs),
@@ -116,9 +143,13 @@ func (r *boundaryBlockReader) ReadBeaconBlockBodyBySlot(_ context.Context, _ kv.
return nil, r.err
}
-type boundaryPeerCounter uint64
+type boundaryPeerClient uint64
-func (p boundaryPeerCounter) Peers() (uint64, error) { return uint64(p), nil }
+func (p boundaryPeerClient) Peers() (uint64, error) { return uint64(p), nil }
+
+func (boundaryPeerClient) SendBlobsSidecarByIdentifierReq(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) {
+ return nil, "", nil
+}
type boundarySnapshot uint64
diff --git a/cl/phase1/network/blob_downloader_test.go b/cl/phase1/network/blob_downloader_test.go
index 51771c4c91b..dd618844cef 100644
--- a/cl/phase1/network/blob_downloader_test.go
+++ b/cl/phase1/network/blob_downloader_test.go
@@ -18,26 +18,45 @@ package network
import (
"context"
+ "errors"
+ "math"
+ "sync/atomic"
"testing"
"time"
+ goethkzg "github.com/crate-crypto/go-eth-kzg"
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/cltypes/solid"
"github.com/erigontech/erigon/cl/das"
"github.com/erigontech/erigon/cl/das/mock_services"
+ "github.com/erigontech/erigon/cl/persistence/blob_storage"
+ "github.com/erigontech/erigon/common"
+ "github.com/erigontech/erigon/common/crypto/kzg"
"github.com/erigontech/erigon/common/log/v3"
+ "github.com/erigontech/erigon/db/kv"
+ "github.com/erigontech/erigon/db/kv/dbcfg"
+ "github.com/erigontech/erigon/db/kv/memdb"
+ "github.com/erigontech/erigon/db/snapshotsync/freezeblocks"
)
type staticPeerDasGetter struct{ pd das.PeerDas }
func (s staticPeerDasGetter) GetPeerDas() das.PeerDas { return s.pd }
-// A historical fulu block whose PeerDAS data columns are served by no peer (older
-// than the network custody window) makes DownloadColumnsAndRecoverBlobs block until
-// its context is cancelled. Column recovery must be bounded per block so the archive
-// blob backfill cannot hang forever holding the index read tx.
+type forcedRecoveryPeerDas struct {
+ das.PeerDas
+ force func(context.Context, uint64, common.Hash) error
+}
+
+func (p forcedRecoveryPeerDas) ForceScheduleRecover(ctx context.Context, slot uint64, root common.Hash, expectedBlobs uint64) error {
+ return p.force(ctx, slot, root)
+}
+
func TestBlobHistoryDownloaderFuluColumnRecoveryIsBounded(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -72,3 +91,952 @@ func TestBlobHistoryDownloaderFuluColumnRecoveryIsBounded(t *testing.T) {
t.Fatal("recoverFuluColumns hung — unbounded PeerDAS column recovery")
}
}
+
+func TestBlobHistoryDownloaderReportsPendingSlots(t *testing.T) {
+ t.Run("archive", func(t *testing.T) {
+ denebSlot := clparams.MainnetBeaconConfig.DenebForkEpoch * clparams.MainnetBeaconConfig.SlotsPerEpoch
+ b := &BlobHistoryDownloader{archiveBlobs: true, beaconCfg: &clparams.MainnetBeaconConfig}
+ b.headSlot.Store(denebSlot + 100)
+ require.True(t, b.BlobBackfillPending(denebSlot))
+ require.False(t, b.BlobBackfillPending(denebSlot-1))
+ require.True(t, b.BlobBackfillPending(b.headSlot.Load()))
+ require.True(t, b.BlobBackfillPending(b.headSlot.Load()+1))
+ b.completedRanges = []backfillRange{{denebSlot, b.headSlot.Load()}}
+ b.backfillCompleted.Store(true)
+ require.False(t, b.BlobBackfillPending(denebSlot))
+ })
+
+ t.Run("immediate", func(t *testing.T) {
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = math.MaxUint64
+ window := beaconCfg.MinSlotsForBlobsSidecarsRequest()
+ b := &BlobHistoryDownloader{immediateBlobsBackfilling: true, beaconCfg: &beaconCfg}
+ b.headSlot.Store(window + 100)
+ oldestEpoch := b.headSlot.Load()/b.beaconCfg.SlotsPerEpoch - b.beaconCfg.MinEpochsForBlobSidecarsRequests
+ oldestSlot := oldestEpoch * b.beaconCfg.SlotsPerEpoch
+ require.True(t, b.BlobBackfillPending(b.headSlot.Load()-1))
+ require.True(t, b.BlobBackfillPending(oldestSlot))
+ require.False(t, b.BlobBackfillPending(oldestSlot-1))
+ require.True(t, b.BlobBackfillPending(b.headSlot.Load()+1))
+ })
+
+ t.Run("head not initialized", func(t *testing.T) {
+ denebSlot := clparams.MainnetBeaconConfig.DenebForkEpoch * clparams.MainnetBeaconConfig.SlotsPerEpoch
+ for _, b := range []*BlobHistoryDownloader{
+ {archiveBlobs: true, beaconCfg: &clparams.MainnetBeaconConfig},
+ {immediateBlobsBackfilling: true, beaconCfg: &clparams.MainnetBeaconConfig},
+ } {
+ require.True(t, b.BlobBackfillPending(denebSlot))
+ require.False(t, b.BlobBackfillPending(denebSlot-1))
+ }
+ })
+
+ t.Run("disabled", func(t *testing.T) {
+ b := &BlobHistoryDownloader{}
+ require.False(t, b.BlobBackfillPending(1))
+ })
+}
+
+func TestBlobHistoryDownloaderHandlesUnscheduledDeneb(t *testing.T) {
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = math.MaxUint64
+ d := NewBlobHistoryDownloader(t.Context(), &beaconCfg, nil, nil, nil, nil, nil, nil, nil, true, false, log.Root())
+
+ require.Equal(t, uint64(math.MaxUint64), d.targetSlot)
+ require.False(t, d.BlobBackfillPending(math.MaxUint64))
+}
+
+func TestBlobHistoryDownloaderPreservesArchiveTargetAfterFailure(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(1)
+ )
+ wantErr := errors.New("read failed")
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, 0, targetSlot, 16, &recordingBlobBlockReader{err: wantErr})
+
+ require.ErrorIs(t, d.downloadOnce(false), wantErr)
+ if d.targetSlot != targetSlot {
+ t.Fatalf("target slot %d, want %d", d.targetSlot, targetSlot)
+ }
+}
+
+func TestBlobHistoryDownloaderPreservesArchiveTargetAfterDenebFailure(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(1)
+ )
+ block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, clparams.DenebVersion)
+ block.Block.Slot = headSlot
+ block.Block.Body.SyncAggregate = cltypes.NewSyncAggregate()
+ block.Block.Body.BlobKzgCommitments.Append(&cltypes.KZGCommitment{})
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, targetSlot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: block},
+ })
+ d.blobStorage = emptyBlobStorage{}
+ wantErr := errors.New("Deneb request failed")
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return nil, wantErr
+ }
+
+ require.ErrorIs(t, d.downloadOnce(false), wantErr)
+ if d.targetSlot != targetSlot {
+ t.Fatalf("target slot %d, want %d", d.targetSlot, targetSlot)
+ }
+ if d.backfillCompleted.Load() {
+ t.Fatal("backfill marked complete after Deneb failure")
+ }
+}
+
+func TestBlobHistoryDownloaderContinuesPastFailedDenebBatch(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ olderSlot = headSlot - blocksBatchSize
+ targetSlot = uint64(1)
+ )
+ failedBlock, _ := makeBlobBoundaryObjects(t, headSlot, 1)
+ olderBlock, olderSidecars := makeBlobBoundaryObjects(t, olderSlot, 1)
+ failedRoot, err := failedBlock.Block.HashSSZ()
+ require.NoError(t, err)
+ olderRoot, err := olderBlock.Block.HashSSZ()
+ require.NoError(t, err)
+ storage := newBlobBoundaryStorage(t)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, olderSlot, targetSlot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{
+ headSlot: failedBlock,
+ olderSlot: olderBlock,
+ },
+ })
+ d.blobStorage = storage
+ wantErr := errors.New("newest batch unavailable")
+ d.requestBlobs = func(_ context.Context, _ BlobPeerClient, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ if req.Get(0).BlockRoot == failedRoot {
+ return nil, wantErr
+ }
+ return &PeerAndSidecars{Peer: "peer", Responses: olderSidecars}, nil
+ }
+
+ require.ErrorIs(t, d.downloadOnce(false), wantErr)
+ stored, complete, err := storage.ReadBlobSidecars(t.Context(), olderSlot, olderRoot)
+ require.NoError(t, err)
+ require.True(t, complete)
+ require.Len(t, stored, 1)
+ require.False(t, d.backfillCompleted.Load())
+ require.False(t, d.BlobBackfillPending(olderSlot))
+ require.True(t, d.BlobBackfillPending(headSlot))
+}
+
+func TestBlobHistoryDownloaderFuluFailureDoesNotCompleteBackfill(t *testing.T) {
+ const slot = uint64(100)
+ ctrl := gomock.NewController(t)
+ peerDas := mock_services.NewMockPeerDas(ctrl)
+ wantErr := errors.New("columns unavailable")
+ peerDas.EXPECT().DownloadColumnsAndRecoverBlobs(gomock.Any(), gomock.Any()).Return(wantErr)
+ fulu, _ := makeBlobBoundaryObjectsVersion(t, slot, 1, clparams.FuluVersion)
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, 1, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{slot: fulu},
+ })
+ d.blobStorage = newBlobBoundaryStorage(t)
+ d.peerDasGetter = staticPeerDasGetter{pd: peerDas}
+ d.columnBackfillTimeout = time.Second
+
+ require.ErrorIs(t, d.downloadOnce(false), wantErr)
+ require.False(t, d.backfillCompleted.Load())
+}
+
+func TestBlobHistoryDownloaderFuluSuccessRequiresStoredBlobs(t *testing.T) {
+ const slot = uint64(100)
+ ctrl := gomock.NewController(t)
+ peerDas := mock_services.NewMockPeerDas(ctrl)
+ peerDas.EXPECT().DownloadColumnsAndRecoverBlobs(gomock.Any(), gomock.Any()).Return(nil)
+ fulu, _ := makeBlobBoundaryObjectsVersion(t, slot, 1, clparams.FuluVersion)
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, 1, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{slot: fulu},
+ })
+ d.blobStorage = newBlobBoundaryStorage(t)
+ d.peerDasGetter = staticPeerDasGetter{pd: peerDas}
+ d.columnBackfillTimeout = time.Second
+
+ require.ErrorContains(t, d.downloadOnce(false), "incomplete Fulu blob recovery")
+ require.False(t, d.backfillCompleted.Load())
+}
+
+func TestBlobHistoryDownloaderForcesFuluRecoveryWhenIndexExistsButBlobIsMissing(t *testing.T) {
+ const slot = uint64(100)
+ ctrl := gomock.NewController(t)
+ peerDas := mock_services.NewMockPeerDas(ctrl)
+ peerDas.EXPECT().DownloadColumnsAndRecoverBlobs(gomock.Any(), gomock.Any()).Return(nil)
+ fulu, sidecars := makeBlobBoundaryObjectsVersion(t, slot, 1, clparams.FuluVersion)
+ root, err := fulu.Block.HashSSZ()
+ require.NoError(t, err)
+ storage := newBlobBoundaryStorage(t)
+ forced := false
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, slot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{slot: fulu},
+ })
+ d.blobStorage = storage
+ d.peerDasGetter = staticPeerDasGetter{pd: forcedRecoveryPeerDas{
+ PeerDas: peerDas,
+ force: func(ctx context.Context, gotSlot uint64, gotRoot common.Hash) error {
+ forced = true
+ require.Equal(t, slot, gotSlot)
+ require.Equal(t, common.Hash(root), gotRoot)
+ return storage.WriteBlobSidecars(ctx, root, sidecars)
+ },
+ }}
+ d.columnBackfillTimeout = time.Second
+
+ require.NoError(t, d.downloadOnce(false))
+ require.True(t, forced)
+ require.True(t, d.backfillCompleted.Load())
+}
+
+func TestBlobHistoryDownloaderRejectsInvalidStoredBlobSet(t *testing.T) {
+ testCases := []struct {
+ name string
+ mutate func(*cltypes.BlobSidecar)
+ }{
+ {
+ name: "wrong block root",
+ mutate: func(sidecar *cltypes.BlobSidecar) {
+ sidecar.SignedBlockHeader.Header.ParentRoot[0]++
+ },
+ },
+ {
+ name: "wrong commitment",
+ mutate: func(sidecar *cltypes.BlobSidecar) {
+ sidecar.KzgCommitment[0]++
+ },
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ block, sidecars := makeBlobBoundaryObjects(t, 100, 1)
+ root, err := block.Block.HashSSZ()
+ require.NoError(t, err)
+ tc.mutate(sidecars[0])
+ d := &BlobHistoryDownloader{ctx: t.Context(), blobStorage: completeBlobStorage{sidecars: sidecars}}
+
+ complete, err := d.actualBlobSetComplete(block, root)
+ require.NoError(t, err)
+ require.False(t, complete)
+ })
+ }
+}
+
+func TestBlobHistoryDownloaderRunsFuluRecoveryAfterDenebFailure(t *testing.T) {
+ const headSlot = uint64(100)
+ ctrl := gomock.NewController(t)
+ peerDas := mock_services.NewMockPeerDas(ctrl)
+ deneb, _ := makeBlobBoundaryObjects(t, headSlot, 1)
+ fulu, fuluSidecars := makeBlobBoundaryObjectsVersion(t, headSlot-1, 1, clparams.FuluVersion)
+ fuluRoot, err := fulu.Block.HashSSZ()
+ require.NoError(t, err)
+ storage := newBlobBoundaryStorage(t)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot-1, 1, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: deneb, headSlot - 1: fulu},
+ })
+ d.blobStorage = storage
+ d.peerDasGetter = staticPeerDasGetter{pd: peerDas}
+ d.columnBackfillTimeout = time.Second
+ wantErr := errors.New("deneb unavailable")
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return nil, wantErr
+ }
+ peerDas.EXPECT().DownloadColumnsAndRecoverBlobs(gomock.Any(), gomock.Any()).DoAndReturn(
+ func(context.Context, []cltypes.ColumnSyncableSignedBlock) error {
+ return storage.WriteBlobSidecars(t.Context(), fuluRoot, fuluSidecars)
+ },
+ )
+
+ require.ErrorIs(t, d.downloadOnce(false), wantErr)
+ _, complete, err := storage.ReadBlobSidecars(t.Context(), headSlot-1, fuluRoot)
+ require.NoError(t, err)
+ require.True(t, complete)
+}
+
+func TestBlobHistoryDownloaderHeadAdvanceOnlyMarksNewRangePending(t *testing.T) {
+ const headSlot = uint64(100)
+ reader := &mappedBlobBlockReader{blocks: map[uint64]*cltypes.SignedBeaconBlock{}}
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, 1, 16, reader)
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ d.beaconCfg = &beaconCfg
+ d.blobStorage = newBlobBoundaryStorage(t)
+ require.NoError(t, d.downloadOnce(false))
+ require.False(t, d.BlobBackfillPending(headSlot))
+
+ newBlock, _ := makeBlobBoundaryObjects(t, headSlot+1, 1)
+ reader.blocks[headSlot+1] = newBlock
+ d.SetHead(headSlot+1, common.Hash{}, headSlot)
+ wantErr := errors.New("new head unavailable")
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return nil, wantErr
+ }
+
+ require.ErrorIs(t, d.downloadOnce(false), wantErr)
+ require.False(t, d.BlobBackfillPending(headSlot))
+ require.True(t, d.BlobBackfillPending(headSlot+1))
+}
+
+func TestBlobHistoryDownloaderDoesNotMarkSkippedFuluGapComplete(t *testing.T) {
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.SlotsPerEpoch = 1
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = 100
+ beaconCfg.MinEpochsForDataColumnSidecarsRequests = 10
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, 200, 0, 0, 16, reader)
+ d.beaconCfg = &beaconCfg
+ d.blobStorage = newBlobBoundaryStorage(t)
+
+ require.NoError(t, d.downloadOnce(false))
+ reader.slots = nil
+ d.SetHead(105, common.Hash{}, 105)
+ require.True(t, d.BlobBackfillPending(100))
+ require.NoError(t, d.downloadOnce(false))
+ require.Contains(t, reader.slots, uint64(100))
+ require.Contains(t, reader.slots, uint64(105))
+}
+
+func TestBlobHistoryDownloaderDoesNotRescanCompletedHead(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(90)
+ )
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, targetSlot, targetSlot, 16, reader)
+
+ require.NoError(t, d.downloadOnce(false))
+ reader.slots = nil
+ require.NoError(t, d.downloadOnce(false))
+ require.Empty(t, reader.slots)
+}
+
+func TestBlobHistoryDownloaderHeadAdvanceScansOnlyUncoveredSuffix(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(90)
+ )
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, targetSlot, targetSlot, 16, reader)
+ require.NoError(t, d.downloadOnce(false))
+
+ reader.slots = nil
+ d.SetHead(headSlot+2, common.Hash{}, headSlot)
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, []uint64{headSlot + 2, headSlot + 1}, reader.slots)
+}
+
+func TestBlobHistoryDownloaderHeadRegressionExpandsEpochAlignedLowerEdge(t *testing.T) {
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = math.MaxUint64
+ beaconCfg.SlotsPerEpoch = 8
+ beaconCfg.MinEpochsForBlobSidecarsRequests = 2
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, 100, 0, 0, 16, reader)
+ d.archiveBlobs = false
+ d.immediateBlobsBackfilling = true
+ d.beaconCfg = &beaconCfg
+ require.NoError(t, d.downloadOnce(false))
+ require.False(t, d.BlobBackfillPending(80))
+
+ reader.slots = nil
+ d.SetHead(79, common.Hash{}, 79)
+ require.True(t, d.BlobBackfillPending(56))
+ require.False(t, d.BlobBackfillPending(55))
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, uint64(79), reader.slots[0])
+ require.Equal(t, uint64(56), reader.slots[len(reader.slots)-1])
+
+ reader.slots = nil
+ d.SetHead(100, common.Hash{}, 79)
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, uint64(100), reader.slots[0])
+ require.Equal(t, uint64(80), reader.slots[len(reader.slots)-1])
+}
+
+func TestBlobHistoryDownloaderUsesForkSpecificRetentionWindows(t *testing.T) {
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = 10
+ beaconCfg.SlotsPerEpoch = 8
+ beaconCfg.MinEpochsForBlobSidecarsRequests = 4
+ beaconCfg.MinEpochsForDataColumnSidecarsRequests = 2
+ b := &BlobHistoryDownloader{immediateBlobsBackfilling: true, beaconCfg: &beaconCfg}
+ b.headSlot.Store(104)
+
+ require.True(t, b.BlobBackfillPending(72))
+ require.True(t, b.BlobBackfillPending(79))
+ require.False(t, b.BlobBackfillPending(80))
+ require.False(t, b.BlobBackfillPending(87))
+ require.True(t, b.BlobBackfillPending(88))
+ require.True(t, b.BlobBackfillPending(104))
+}
+
+func TestBlobHistoryDownloaderSameSlotReorgInvalidatesCompletion(t *testing.T) {
+ const headSlot = uint64(100)
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, headSlot, 16, reader)
+ d.SetHead(headSlot, common.HexToHash("0x01"), headSlot)
+ require.NoError(t, d.downloadOnce(false))
+ require.False(t, d.BlobBackfillPending(headSlot))
+
+ reader.slots = nil
+ d.SetHead(headSlot, common.HexToHash("0x02"), headSlot-1)
+ require.True(t, d.BlobBackfillPending(headSlot))
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, []uint64{headSlot}, reader.slots)
+}
+
+func TestBlobHistoryDownloaderInvalidateCompletionAboveMakesReorgRangePending(t *testing.T) {
+ d := newBlobDownloaderForBoundaryTest(t, 100, 100, 64, 16, &recordingBlobBlockReader{})
+ d.SetHead(100, common.HexToHash("0x01"), 100)
+ d.mu.Lock()
+ d.completedRanges = []backfillRange{{start: 64, end: 100}}
+ d.backfillCompleted.Store(true)
+ d.mu.Unlock()
+ require.False(t, d.BlobBackfillPending(100))
+
+ d.InvalidateCompletionAbove(99)
+
+ require.True(t, d.BlobBackfillPending(100))
+ require.Equal(t, uint64(100), d.HeadSlot())
+}
+
+func TestBlobHistoryDownloaderTransitionCeilingBlocksOldCanonicalPass(t *testing.T) {
+ const slot = uint64(100)
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, slot, 16, &recordingBlobBlockReader{})
+ d.SetHead(slot, common.HexToHash("0x01"), slot)
+ d.mu.Lock()
+ d.completedRanges = []backfillRange{{start: slot, end: slot}}
+ d.backfillCompleted.Store(true)
+ d.mu.Unlock()
+
+ d.InvalidateCompletionAbove(slot - 1)
+ require.NoError(t, d.downloadOnce(false))
+ require.True(t, d.BlobBackfillPending(slot))
+
+ d.SetHead(slot, common.HexToHash("0x02"), slot-1)
+ require.True(t, d.BlobBackfillPending(slot))
+}
+
+func TestBlobHistoryDownloaderAbortHeadUpdateAllowsOldCanonicalRetry(t *testing.T) {
+ const slot = uint64(100)
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, slot, 16, &recordingBlobBlockReader{})
+ d.SetHead(slot, common.HexToHash("0x01"), slot)
+ d.mu.Lock()
+ d.completedRanges = []backfillRange{{start: slot, end: slot}}
+ d.backfillCompleted.Store(true)
+ d.mu.Unlock()
+
+ d.InvalidateCompletionAbove(slot - 1)
+ d.AbortHeadUpdate()
+ require.True(t, d.BlobBackfillPending(slot))
+ require.NoError(t, d.downloadOnce(false))
+ require.False(t, d.BlobBackfillPending(slot))
+}
+
+func TestBlobHistoryDownloaderHigherHeadReorgRescansChangedSuffix(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(90)
+ forkSlot = uint64(98)
+ )
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, targetSlot, targetSlot, 16, reader)
+ d.SetHead(headSlot, common.HexToHash("0x01"), headSlot)
+ require.NoError(t, d.downloadOnce(false))
+
+ reader.slots = nil
+ d.SetHead(headSlot+2, common.HexToHash("0x02"), forkSlot)
+ require.True(t, d.BlobBackfillPending(forkSlot+1))
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, []uint64{102, 101, 100, 99}, reader.slots)
+}
+
+func TestBlobHistoryDownloaderStalePassCannotRestoreReorgedSuffix(t *testing.T) {
+ const slot = uint64(100)
+ oldBlock, oldSidecars := makeBlobBoundaryObjects(t, slot, 1)
+ newBlock, newSidecars := makeBlobBoundaryObjects(t, slot, 2)
+ oldRoot, err := oldBlock.Block.HashSSZ()
+ require.NoError(t, err)
+ newRoot, err := newBlock.Block.HashSSZ()
+ require.NoError(t, err)
+ require.NotEqual(t, oldRoot, newRoot)
+
+ reader := &mappedBlobBlockReader{blocks: map[uint64]*cltypes.SignedBeaconBlock{slot: oldBlock}}
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, slot, 1, reader)
+ d.blobStorage = newBlobBoundaryStorage(t)
+ d.SetHead(slot, oldRoot, slot-1)
+ collected := make(chan struct{})
+ resume := make(chan struct{})
+ requestedRoots := make(chan common.Hash, 2)
+ var requests atomic.Int64
+ d.requestBlobs = func(_ context.Context, _ BlobPeerClient, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ requestedRoots <- req.Get(0).BlockRoot
+ if requests.Add(1) == 1 {
+ close(collected)
+ <-resume
+ return &PeerAndSidecars{Peer: "peer", Responses: oldSidecars}, nil
+ }
+ return &PeerAndSidecars{Peer: "peer", Responses: newSidecars}, nil
+ }
+
+ firstPass := make(chan error, 1)
+ go func() { firstPass <- d.downloadOnce(false) }()
+ <-collected
+ d.InvalidateCompletionAbove(slot - 1)
+ reader.blocks[slot] = newBlock
+ d.SetHead(slot, newRoot, slot-1)
+ close(resume)
+ require.NoError(t, <-firstPass)
+ require.True(t, d.BlobBackfillPending(slot))
+
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, common.Hash(oldRoot), <-requestedRoots)
+ require.Equal(t, common.Hash(newRoot), <-requestedRoots)
+ require.False(t, d.BlobBackfillPending(slot))
+}
+
+func TestBlobHistoryDownloaderStalePassPreservesSafeHeadExtensionPrefix(t *testing.T) {
+ const slot = uint64(100)
+ block, sidecars := makeBlobBoundaryObjects(t, slot, 1)
+ root, err := block.Block.HashSSZ()
+ require.NoError(t, err)
+
+ reader := &mappedBlobBlockReader{blocks: map[uint64]*cltypes.SignedBeaconBlock{slot: block}}
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, slot, 1, reader)
+ d.blobStorage = newBlobBoundaryStorage(t)
+ d.SetHead(slot, root, slot-1)
+ collected := make(chan struct{})
+ resume := make(chan struct{})
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ close(collected)
+ <-resume
+ return &PeerAndSidecars{Peer: "peer", Responses: sidecars}, nil
+ }
+
+ firstPass := make(chan error, 1)
+ go func() { firstPass <- d.downloadOnce(false) }()
+ <-collected
+ d.SetHead(slot+1, common.HexToHash("0x101"), slot)
+ close(resume)
+ require.NoError(t, <-firstPass)
+
+ require.False(t, d.BlobBackfillPending(slot))
+ require.True(t, d.BlobBackfillPending(slot+1))
+}
+
+func TestBlobHistoryDownloaderHeadJumpStartsAtCurrentRetentionFloor(t *testing.T) {
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = math.MaxUint64
+ beaconCfg.SlotsPerEpoch = 8
+ beaconCfg.MinEpochsForBlobSidecarsRequests = 2
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, 100, 0, 0, 16, reader)
+ d.archiveBlobs = false
+ d.immediateBlobsBackfilling = true
+ d.beaconCfg = &beaconCfg
+ require.NoError(t, d.downloadOnce(false))
+
+ reader.slots = nil
+ d.SetHead(1000, common.Hash{}, 100)
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, uint64(1000), reader.slots[0])
+ require.Equal(t, uint64(984), reader.slots[len(reader.slots)-1])
+ require.Len(t, reader.slots, 17)
+}
+
+func TestBlobHistoryDownloaderAcceptsReorderedDenebResponse(t *testing.T) {
+ const headSlot = uint64(100)
+ first, firstSidecars := makeBlobBoundaryObjects(t, headSlot, 1)
+ second, secondSidecars := makeBlobBoundaryObjects(t, headSlot-1, 1)
+ storage := newBlobBoundaryStorage(t)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot-1, 1, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: first, headSlot - 1: second},
+ })
+ d.blobStorage = storage
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return &PeerAndSidecars{Peer: "peer", Responses: []*cltypes.BlobSidecar{secondSidecars[0], firstSidecars[0]}}, nil
+ }
+
+ require.NoError(t, d.downloadOnce(false))
+ for slot, block := range map[uint64]*cltypes.SignedBeaconBlock{headSlot: first, headSlot - 1: second} {
+ root, err := block.Block.HashSSZ()
+ require.NoError(t, err)
+ _, complete, err := storage.ReadBlobSidecars(t.Context(), slot, root)
+ require.NoError(t, err)
+ require.True(t, complete)
+ }
+}
+
+func TestBlobHistoryDownloaderPersistsCompleteBlocksFromShortResponse(t *testing.T) {
+ const headSlot = uint64(100)
+ first, firstSidecars := makeBlobBoundaryObjects(t, headSlot, 1)
+ second, _ := makeBlobBoundaryObjects(t, headSlot-1, 1)
+ firstRoot, err := first.Block.HashSSZ()
+ require.NoError(t, err)
+ storage := newBlobBoundaryStorage(t)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot-1, 1, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: first, headSlot - 1: second},
+ })
+ d.blobStorage = storage
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return &PeerAndSidecars{Peer: "peer", Responses: firstSidecars}, nil
+ }
+
+ require.Error(t, d.downloadOnce(false))
+ _, complete, err := storage.ReadBlobSidecars(t.Context(), headSlot, firstRoot)
+ require.NoError(t, err)
+ require.True(t, complete)
+ require.False(t, d.backfillCompleted.Load())
+}
+
+func TestBlobHistoryDownloaderDoesNotTrustIndexedCountWhenSidecarIsMissing(t *testing.T) {
+ const slot = uint64(100)
+ block, _ := makeBlobBoundaryObjects(t, slot, 1)
+ d := newBlobDownloaderForBoundaryTest(t, slot, slot, slot, 1, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{slot: block},
+ })
+ d.blobStorage = indexedButMissingBlobStorage{}
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return nil, errors.New("recovery required")
+ }
+
+ require.ErrorContains(t, d.downloadOnce(false), "recovery required")
+ require.False(t, d.backfillCompleted.Load())
+ require.True(t, d.BlobBackfillPending(slot))
+}
+
+func TestBlobHistoryDownloaderAccumulatesPartialBlockAcrossRequests(t *testing.T) {
+ const headSlot = uint64(100)
+ block, sidecars := makeBlobBoundaryObjects(t, headSlot, 2)
+ blockRoot, err := block.Block.HashSSZ()
+ require.NoError(t, err)
+ storage := newBlobBoundaryStorage(t)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, headSlot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: block},
+ })
+ d.blobStorage = storage
+ requests := 0
+ d.requestBlobs = func(_ context.Context, _ BlobPeerClient, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ requests++
+ require.Equal(t, 3-requests, req.Len())
+ return &PeerAndSidecars{Peer: "peer", Responses: []*cltypes.BlobSidecar{sidecars[requests-1]}}, nil
+ }
+
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, 2, requests)
+ stored, complete, err := storage.ReadBlobSidecars(t.Context(), headSlot, blockRoot)
+ require.NoError(t, err)
+ require.True(t, complete)
+ require.Len(t, stored, 2)
+}
+
+func TestBlobHistoryDownloaderRejectsRepeatedPartialResponse(t *testing.T) {
+ const headSlot = uint64(100)
+ block, sidecars := makeBlobBoundaryObjects(t, headSlot, 2)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, headSlot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: block},
+ })
+ d.blobStorage = newBlobBoundaryStorage(t)
+ requests := 0
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ requests++
+ return &PeerAndSidecars{Peer: "peer", Responses: sidecars[:1]}, nil
+ }
+
+ require.ErrorContains(t, d.downloadOnce(false), "unrequested blob sidecar")
+ require.Equal(t, 2, requests)
+}
+
+func TestBlobHistoryDownloaderRejectsInvalidOrIncompleteDenebResponse(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(1)
+ )
+ testCases := []struct {
+ name string
+ commitments int
+ alterResponse func([]*cltypes.BlobSidecar) *PeerAndSidecars
+ }{
+ {
+ name: "nil result",
+ commitments: 1,
+ alterResponse: func([]*cltypes.BlobSidecar) *PeerAndSidecars {
+ return nil
+ },
+ },
+ {
+ name: "short response",
+ commitments: 2,
+ alterResponse: func(sidecars []*cltypes.BlobSidecar) *PeerAndSidecars {
+ return &PeerAndSidecars{Peer: "peer", Responses: sidecars[:1]}
+ },
+ },
+ {
+ name: "extra response",
+ commitments: 1,
+ alterResponse: func(sidecars []*cltypes.BlobSidecar) *PeerAndSidecars {
+ return &PeerAndSidecars{Peer: "peer", Responses: append(sidecars, sidecars[0])}
+ },
+ },
+ {
+ name: "duplicate index",
+ commitments: 2,
+ alterResponse: func(sidecars []*cltypes.BlobSidecar) *PeerAndSidecars {
+ sidecars[1] = sidecars[0]
+ return &PeerAndSidecars{Peer: "peer", Responses: sidecars}
+ },
+ },
+ {
+ name: "wrong root",
+ commitments: 1,
+ alterResponse: func(sidecars []*cltypes.BlobSidecar) *PeerAndSidecars {
+ sidecars[0].SignedBlockHeader.Header.ParentRoot[0]++
+ return &PeerAndSidecars{Peer: "peer", Responses: sidecars}
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ block, sidecars := makeBlobBoundaryObjects(t, headSlot, tc.commitments)
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, targetSlot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: block},
+ })
+ d.blobStorage = blob_storage.NewBlobStore(
+ memdb.NewTestDB(t, dbcfg.ChainDB),
+ afero.NewMemMapFs(),
+ math.MaxUint64,
+ &clparams.MainnetBeaconConfig,
+ nil,
+ )
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return tc.alterResponse(sidecars), nil
+ }
+
+ require.Error(t, d.downloadOnce(false))
+ require.Equal(t, targetSlot, d.targetSlot)
+ require.False(t, d.backfillCompleted.Load())
+ })
+ }
+}
+
+func TestBlobHistoryDownloaderPersistsBoundaryBlobAcrossRestart(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(1)
+ )
+ block, sidecars := makeBlobBoundaryObjects(t, headSlot, 1)
+ blockRoot, err := block.Block.HashSSZ()
+ require.NoError(t, err)
+ storage := blob_storage.NewBlobStore(
+ memdb.NewTestDB(t, dbcfg.ChainDB),
+ afero.NewMemMapFs(),
+ math.MaxUint64,
+ &clparams.MainnetBeaconConfig,
+ nil,
+ )
+ newDownloader := func() *BlobHistoryDownloader {
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, headSlot, targetSlot, 16, &mappedBlobBlockReader{
+ blocks: map[uint64]*cltypes.SignedBeaconBlock{headSlot: block},
+ })
+ d.blobStorage = storage
+ return d
+ }
+ d := newDownloader()
+ requests := 0
+ d.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ requests++
+ return &PeerAndSidecars{Peer: "peer", Responses: sidecars}, nil
+ }
+
+ require.NoError(t, d.downloadOnce(false))
+ require.Equal(t, 1, requests)
+ stored, complete, err := storage.ReadBlobSidecars(t.Context(), headSlot, blockRoot)
+ require.NoError(t, err)
+ require.True(t, complete)
+ require.Len(t, stored, 1)
+
+ restarted := newDownloader()
+ restarted.requestBlobs = func(context.Context, BlobPeerClient, *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return nil, errors.New("persisted boundary blob requested after restart")
+ }
+ require.NoError(t, restarted.downloadOnce(false))
+}
+
+func TestBlobHistoryDownloaderPreservesArchiveTargetAfterSuccess(t *testing.T) {
+ const (
+ headSlot = uint64(100)
+ targetSlot = uint64(1)
+ )
+ reader := &recordingBlobBlockReader{}
+ d := newBlobDownloaderForBoundaryTest(t, headSlot, 90, targetSlot, 16, reader)
+
+ if err := d.downloadOnce(false); err != nil {
+ t.Fatal(err)
+ }
+ if len(reader.slots) <= int(blocksBatchSize) {
+ t.Fatalf("read only %d slots, want more than one batch", len(reader.slots))
+ }
+ if d.targetSlot != targetSlot {
+ t.Fatalf("target slot %d, want %d", d.targetSlot, targetSlot)
+ }
+
+ d.headSlot.Store(headSlot + 10)
+ if err := d.downloadOnce(false); err != nil {
+ t.Fatal(err)
+ }
+ if d.targetSlot != targetSlot {
+ t.Fatalf("target slot after head advance %d, want %d", d.targetSlot, targetSlot)
+ }
+}
+
+func TestBlobHistoryDownloaderDoesNotPersistNonArchiveTarget(t *testing.T) {
+ const targetSlot = uint64(1)
+ d := newBlobDownloaderForBoundaryTest(t, 100, 100, targetSlot, 16, &recordingBlobBlockReader{})
+ d.archiveBlobs = false
+
+ if err := d.downloadOnce(false); err != nil {
+ t.Fatal(err)
+ }
+ if d.targetSlot != targetSlot {
+ t.Fatalf("target slot %d, want %d", d.targetSlot, targetSlot)
+ }
+}
+
+func makeBlobBoundaryObjects(t *testing.T, slot uint64, count int) (*cltypes.SignedBeaconBlock, []*cltypes.BlobSidecar) {
+ return makeBlobBoundaryObjectsVersion(t, slot, count, clparams.DenebVersion)
+}
+
+func makeBlobBoundaryObjectsVersion(t *testing.T, slot uint64, count int, version clparams.StateVersion) (*cltypes.SignedBeaconBlock, []*cltypes.BlobSidecar) {
+ t.Helper()
+ blob := cltypes.Blob{}
+ commitment, err := kzg.Ctx().BlobToKZGCommitment((*goethkzg.Blob)(&blob), 0)
+ require.NoError(t, err)
+ proof, err := kzg.Ctx().ComputeBlobKZGProof((*goethkzg.Blob)(&blob), commitment, 0)
+ require.NoError(t, err)
+ block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig, version)
+ block.Block.Slot = slot
+ block.Block.Body.SyncAggregate = cltypes.NewSyncAggregate()
+ blockCommitment := cltypes.KZGCommitment(commitment)
+ for range count {
+ block.Block.Body.BlobKzgCommitments.Append(&blockCommitment)
+ }
+ sidecars := make([]*cltypes.BlobSidecar, count)
+ for index := range count {
+ proofBranch, err := block.Block.Body.KzgCommitmentMerkleProof(index)
+ require.NoError(t, err)
+ inclusionProof := solid.NewHashVector(cltypes.CommitmentBranchSize)
+ for i, hash := range proofBranch {
+ inclusionProof.Set(i, hash)
+ }
+ sidecars[index] = cltypes.NewBlobSidecar(
+ uint64(index),
+ &blob,
+ common.Bytes48(commitment),
+ common.Bytes48(proof),
+ block.SignedBeaconBlockHeader(),
+ inclusionProof,
+ )
+ }
+ return block, sidecars
+}
+
+func newBlobBoundaryStorage(t *testing.T) blob_storage.BlobStorage {
+ t.Helper()
+ return blob_storage.NewBlobStore(
+ memdb.NewTestDB(t, dbcfg.ChainDB),
+ afero.NewMemMapFs(),
+ math.MaxUint64,
+ &clparams.MainnetBeaconConfig,
+ nil,
+ )
+}
+
+func newBlobDownloaderForBoundaryTest(t *testing.T, headSlot, frozenBlobs, targetSlot, peers uint64, reader freezeblocks.BeaconSnapshotReader) *BlobHistoryDownloader {
+ t.Helper()
+ ctx := t.Context()
+ beaconCfg := clparams.MainnetBeaconConfig
+ beaconCfg.DenebForkEpoch = 0
+ beaconCfg.FuluForkEpoch = math.MaxUint64
+ d := &BlobHistoryDownloader{
+ ctx: ctx,
+ beaconCfg: &beaconCfg,
+ rpc: peerCountClient{active: peers},
+ indiciesDB: memdb.NewTestDB(t, dbcfg.ChainDB),
+ blockReader: reader,
+ sn: frozenBlobSnapshot{exclusiveEnd: frozenBlobs},
+ syncedChecker: syncedChecker(true),
+ targetSlot: targetSlot,
+ archiveBlobs: true,
+ logger: log.Root(),
+ }
+ d.headSlot.Store(headSlot)
+ return d
+}
+
+type recordingBlobBlockReader struct {
+ freezeblocks.BeaconSnapshotReader
+ slots []uint64
+ err error
+}
+
+type mappedBlobBlockReader struct {
+ freezeblocks.BeaconSnapshotReader
+ blocks map[uint64]*cltypes.SignedBeaconBlock
+}
+
+func (r *mappedBlobBlockReader) ReadBeaconBlockBodyBySlot(_ context.Context, _ kv.Tx, slot uint64) (*cltypes.SignedBeaconBlock, error) {
+ return r.blocks[slot], nil
+}
+
+type emptyBlobStorage struct{ blob_storage.BlobStorage }
+
+func (emptyBlobStorage) KzgCommitmentsCount(context.Context, common.Hash) (uint32, error) {
+ return 0, nil
+}
+
+func (emptyBlobStorage) ReadBlobSidecars(context.Context, uint64, common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
+ return nil, false, nil
+}
+
+type indexedButMissingBlobStorage struct{ emptyBlobStorage }
+
+func (indexedButMissingBlobStorage) KzgCommitmentsCount(context.Context, common.Hash) (uint32, error) {
+ return 1, nil
+}
+
+type completeBlobStorage struct {
+ emptyBlobStorage
+ sidecars []*cltypes.BlobSidecar
+}
+
+func (s completeBlobStorage) ReadBlobSidecars(context.Context, uint64, common.Hash) ([]*cltypes.BlobSidecar, bool, error) {
+ return s.sidecars, true, nil
+}
+
+func (r *recordingBlobBlockReader) ReadBeaconBlockBodyBySlot(_ context.Context, _ kv.Tx, slot uint64) (*cltypes.SignedBeaconBlock, error) {
+ r.slots = append(r.slots, slot)
+ return nil, r.err
+}
+
+type frozenBlobSnapshot struct{ exclusiveEnd uint64 }
+
+func (s frozenBlobSnapshot) FrozenBlobs() uint64 { return s.exclusiveEnd }
+
+type syncedChecker bool
+
+func (s syncedChecker) Synced() bool { return bool(s) }
+
+type peerCountClient struct{ active uint64 }
+
+func (p peerCountClient) Peers() (uint64, error) { return p.active, nil }
+
+func (peerCountClient) SendBlobsSidecarByIdentifierReq(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) {
+ return nil, "", errors.New("unexpected blob request")
+}
diff --git a/cl/phase1/network/blobs.go b/cl/phase1/network/blobs.go
index d9b45c70712..4eb693bb60e 100644
--- a/cl/phase1/network/blobs.go
+++ b/cl/phase1/network/blobs.go
@@ -19,13 +19,11 @@ package network
import (
"context"
"errors"
- "sync/atomic"
"time"
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/cltypes"
"github.com/erigontech/erigon/cl/cltypes/solid"
- "github.com/erigontech/erigon/cl/rpc"
"github.com/erigontech/erigon/common/log/v3"
)
@@ -33,6 +31,12 @@ var ErrTimeout = errors.New("timeout")
var requestBlobBatchExpiration = 15 * time.Second
+const (
+ initialBlobRequestBackoff = 100 * time.Millisecond
+ maxBlobRequestBackoff = 2 * time.Second
+ maxConcurrentBlobRequests = 2
+)
+
// This is just a bunch of functions to handle blobs
// BlobsIdentifiersFromBlocks returns a list of blob identifiers from a list of blocks, which should then be forwarded to the network.
@@ -71,52 +75,83 @@ type PeerAndSidecars struct {
Responses []*cltypes.BlobSidecar
}
+type BlobPeerClient interface {
+ Peers() (uint64, error)
+ SendBlobsSidecarByIdentifierReq(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error)
+}
+
+type blobBackfillPeerClient interface {
+ SendBlobsSidecarByIdentifierReqForBackfill(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error)
+}
+
// RequestBlobsFrantically requests blobs from the network frantically.
-func RequestBlobsFrantically(ctx context.Context, r *rpc.BeaconRpcP2P, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
- var atomicResp atomic.Value
+func RequestBlobsFrantically(ctx context.Context, r BlobPeerClient, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return requestBlobsFrantically(ctx, req, r.SendBlobsSidecarByIdentifierReq)
+}
+
+func requestBlobsFranticallyForBackfill(ctx context.Context, r blobBackfillPeerClient, req *solid.ListSSZ[*cltypes.BlobIdentifier]) (*PeerAndSidecars, error) {
+ return requestBlobsFrantically(ctx, req, r.SendBlobsSidecarByIdentifierReqForBackfill)
+}
- atomicResp.Store(&PeerAndSidecars{})
- timer := time.NewTimer(requestBlobBatchExpiration)
- defer timer.Stop()
- reqInterval := time.NewTicker(100 * time.Millisecond)
- defer reqInterval.Stop()
-Loop:
+func requestBlobsFrantically(ctx context.Context, req *solid.ListSSZ[*cltypes.BlobIdentifier], send func(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error)) (*PeerAndSidecars, error) {
+ type requestResult struct {
+ responses []*cltypes.BlobSidecar
+ peer string
+ err error
+ }
+ attemptCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ expiration := time.NewTimer(requestBlobBatchExpiration)
+ defer expiration.Stop()
+ retry := time.NewTimer(0)
+ defer retry.Stop()
+ retryC := retry.C
+ results := make(chan requestResult, maxConcurrentBlobRequests)
+ inFlight := 0
+ backoff := initialBlobRequestBackoff
+ resetRetry := func(delay time.Duration) {
+ if !retry.Stop() {
+ select {
+ case <-retry.C:
+ default:
+ }
+ }
+ retry.Reset(delay)
+ retryC = retry.C
+ }
+ launch := func() {
+ inFlight++
+ go func() {
+ responses, peer, err := send(attemptCtx, req)
+ results <- requestResult{responses: responses, peer: peer, err: err}
+ }()
+ }
for {
select {
- case <-reqInterval.C:
- go func() {
- if len(atomicResp.Load().(*PeerAndSidecars).Responses) > 0 {
- return
- }
- // this is so we do not get stuck on a side-fork
- responses, pid, err := r.SendBlobsSidecarByIdentifierReq(ctx, req)
- if err != nil {
- log.Trace("RequestBlobsFrantically: error", "err", err, "peer", pid)
- return
- }
- if responses == nil {
- log.Trace("RequestBlobsFrantically: response is nil", "peer", pid)
- return
- }
- if len(atomicResp.Load().(*PeerAndSidecars).Responses) > 0 {
- return
- }
- atomicResp.Store(&PeerAndSidecars{
- Peer: pid,
- Responses: responses,
- })
- }()
+ case <-retryC:
+ launch()
+ if inFlight < maxConcurrentBlobRequests {
+ resetRetry(initialBlobRequestBackoff)
+ } else {
+ retryC = nil
+ }
+ case result := <-results:
+ inFlight--
+ if result.err == nil && len(result.responses) > 0 {
+ return &PeerAndSidecars{Peer: result.peer, Responses: result.responses}, nil
+ }
+ if result.err != nil {
+ log.Trace("RequestBlobsFrantically: error", "err", result.err, "peer", result.peer)
+ } else {
+ log.Trace("RequestBlobsFrantically: response is empty", "peer", result.peer)
+ }
+ backoff = min(backoff*2, maxBlobRequestBackoff)
+ resetRetry(backoff)
case <-ctx.Done():
return nil, ctx.Err()
- case <-timer.C:
+ case <-expiration.C:
log.Trace("RequestBlobsFrantically: timeout")
return nil, ErrTimeout
- default:
- if len(atomicResp.Load().(*PeerAndSidecars).Responses) > 0 {
- break Loop
- }
- time.Sleep(10 * time.Millisecond)
}
}
- return atomicResp.Load().(*PeerAndSidecars), nil
}
diff --git a/cl/phase1/network/blobs_test.go b/cl/phase1/network/blobs_test.go
new file mode 100644
index 00000000000..2586d4f521a
--- /dev/null
+++ b/cl/phase1/network/blobs_test.go
@@ -0,0 +1,86 @@
+// Copyright 2026 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see .
+
+package network
+
+import (
+ "context"
+ "errors"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/cltypes/solid"
+)
+
+type blockingBlobPeerClient struct {
+ calls atomic.Int64
+ inFlight atomic.Int64
+ maxFlight atomic.Int64
+}
+
+type failingBlobPeerClient struct{ calls atomic.Int64 }
+
+func (*failingBlobPeerClient) Peers() (uint64, error) { return 1, nil }
+
+func (c *failingBlobPeerClient) SendBlobsSidecarByIdentifierReq(context.Context, *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) {
+ c.calls.Add(1)
+ return nil, "peer", errors.New("resource unavailable")
+}
+
+func (*blockingBlobPeerClient) Peers() (uint64, error) { return 1, nil }
+
+func (c *blockingBlobPeerClient) SendBlobsSidecarByIdentifierReq(ctx context.Context, _ *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) {
+ c.calls.Add(1)
+ inFlight := c.inFlight.Add(1)
+ defer c.inFlight.Add(-1)
+ for {
+ maxFlight := c.maxFlight.Load()
+ if inFlight <= maxFlight || c.maxFlight.CompareAndSwap(maxFlight, inFlight) {
+ break
+ }
+ }
+ <-ctx.Done()
+ return nil, "peer", ctx.Err()
+}
+
+func TestRequestBlobsFranticallyBoundsConcurrentRequests(t *testing.T) {
+ client := &blockingBlobPeerClient{}
+ req := solid.NewStaticListSSZ[*cltypes.BlobIdentifier](0, 40)
+ req.Append(&cltypes.BlobIdentifier{})
+ ctx, cancel := context.WithTimeout(t.Context(), 350*time.Millisecond)
+ defer cancel()
+
+ _, err := RequestBlobsFrantically(ctx, client, req)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ require.LessOrEqual(t, client.maxFlight.Load(), int64(2))
+ require.LessOrEqual(t, client.calls.Load(), int64(2))
+}
+
+func TestRequestBlobsFranticallyBacksOffAfterFailures(t *testing.T) {
+ client := &failingBlobPeerClient{}
+ req := solid.NewStaticListSSZ[*cltypes.BlobIdentifier](0, 40)
+ req.Append(&cltypes.BlobIdentifier{})
+ ctx, cancel := context.WithTimeout(t.Context(), 750*time.Millisecond)
+ defer cancel()
+
+ _, err := RequestBlobsFrantically(ctx, client, req)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ require.LessOrEqual(t, client.calls.Load(), int64(4))
+}
diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go
index 84746b66d00..5b317c78d5d 100644
--- a/cl/phase1/stages/clstages.go
+++ b/cl/phase1/stages/clstages.go
@@ -79,6 +79,11 @@ type Args struct {
hasDownloaded bool
}
+// BlobDownloader returns the canonical blob-history availability tracker.
+func (c *Cfg) BlobDownloader() *network2.BlobHistoryDownloader {
+ return c.blobDownloader
+}
+
func ClStagesCfg(
ctx context.Context,
rpc *rpc.BeaconRpcP2P,
diff --git a/cl/phase1/stages/forkchoice.go b/cl/phase1/stages/forkchoice.go
index abfd0a9b6f7..27835754736 100644
--- a/cl/phase1/stages/forkchoice.go
+++ b/cl/phase1/stages/forkchoice.go
@@ -99,7 +99,12 @@ func computeAndNotifyServicesOfNewForkChoice(ctx context.Context, logger log.Log
// updateCanonicalChainInTheDatabase updates the canonical chain in the database by marking the given head slot and root as canonical.
// It traces back through parent block roots to find the common ancestor with the existing canonical chain, truncates the chain,
// and then marks the new chain segments as canonical.
-func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot uint64, headRoot common.Hash, cfg *Cfg) error {
+type canonicalUpdate struct {
+ commonAncestorSlot uint64
+ reorgEvent *beaconevents.ChainReorgData
+}
+
+func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot uint64, headRoot common.Hash, cfg *Cfg) (canonicalUpdate, error) {
type canonicalEntry struct {
slot uint64
root common.Hash
@@ -109,13 +114,13 @@ func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot
// Read the current canonical block root for the given slot
currentCanonical, err := beacon_indicies.ReadCanonicalBlockRoot(tx, currentSlot)
if err != nil {
- return fmt.Errorf("failed to read canonical block root: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to read canonical block root: %w", err)
}
// Capture the actual old canonical tip before any mutations.
oldHeadSlot, oldHeadRoot, err := beacon_indicies.ReadCanonicalHead(tx)
if err != nil {
- return fmt.Errorf("failed to read canonical head: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to read canonical head: %w", err)
}
// List of new canonical chain entries
@@ -127,15 +132,15 @@ func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot
// Read the parent block root
if currentRoot, err = beacon_indicies.ReadParentBlockRoot(ctx, tx, currentRoot); err != nil {
- return fmt.Errorf("failed to read parent block root: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to read parent block root: %w", err)
}
// Read the slot for the current block root
if newFoundSlot, err = beacon_indicies.ReadBlockSlotByBlockRoot(tx, currentRoot); err != nil {
- return fmt.Errorf("failed to read block slot by block root: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to read block slot by block root: %w", err)
}
if newFoundSlot == nil {
- break
+ return canonicalUpdate{}, fmt.Errorf("common ancestor is unavailable for block root %x", currentRoot)
}
currentSlot = *newFoundSlot
@@ -143,29 +148,34 @@ func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot
// Read the canonical block root for the new slot
currentCanonical, err = beacon_indicies.ReadCanonicalBlockRoot(tx, currentSlot)
if err != nil {
- return fmt.Errorf("failed to read canonical block root: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to read canonical block root: %w", err)
}
// Append the current slot and root to the list of reconnection roots
reconnectionRoots = append(reconnectionRoots, canonicalEntry{currentSlot, currentRoot})
}
+ commonAncestorSlot := uint64(0)
+ if currentRoot == currentCanonical {
+ commonAncestorSlot = currentSlot
+ }
// Truncate the canonical chain at the current slot
if err := beacon_indicies.TruncateCanonicalChain(ctx, tx, currentSlot); err != nil {
- return fmt.Errorf("failed to truncate canonical chain: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to truncate canonical chain: %w", err)
}
// Mark the new canonical chain segments in reverse order
for _, reconnectionRoot := range slices.Backward(reconnectionRoots) {
if err := beacon_indicies.MarkRootCanonical(ctx, tx, reconnectionRoot.slot, reconnectionRoot.root); err != nil {
- return fmt.Errorf("failed to mark root canonical: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to mark root canonical: %w", err)
}
}
// Mark the head slot and root as canonical
if err := beacon_indicies.MarkRootCanonical(ctx, tx, headSlot, headRoot); err != nil {
- return fmt.Errorf("failed to mark root canonical: %w", err)
+ return canonicalUpdate{}, fmt.Errorf("failed to mark root canonical: %w", err)
}
+ update := canonicalUpdate{commonAncestorSlot: commonAncestorSlot}
// A reorg occurred if the fork point (currentSlot) is strictly below the
// old canonical tip. Normal chain extension lands exactly at oldHeadSlot.
@@ -174,12 +184,12 @@ func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot
oldStateRoot, err := beacon_indicies.ReadStateRootByBlockRoot(ctx, tx, oldHeadRoot)
if err != nil {
log.Warn("failed to read state root by block root", "err", err, "block_root", oldHeadRoot)
- return nil
+ return update, nil
}
newStateRoot, err := beacon_indicies.ReadStateRootByBlockRoot(ctx, tx, headRoot)
if err != nil {
log.Warn("failed to read state root by block root", "err", err, "block_root", headRoot)
- return nil
+ return update, nil
}
reorgDepth := uint64(0)
if oldHeadSlot > currentSlot {
@@ -189,7 +199,7 @@ func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot
if cfg.forkChoice != nil {
executionOptimistic = cfg.forkChoice.IsRootOptimistic(headRoot)
}
- reorgEvent := &beaconevents.ChainReorgData{
+ update.reorgEvent = &beaconevents.ChainReorgData{
Slot: headSlot,
Depth: reorgDepth,
OldHeadBlock: oldHeadRoot,
@@ -199,10 +209,9 @@ func updateCanonicalChainInTheDatabase(ctx context.Context, tx kv.RwTx, headSlot
Epoch: headSlot / cfg.beaconCfg.SlotsPerEpoch,
ExecutionOptimistic: executionOptimistic,
}
- cfg.emitter.State().SendChainReorg(reorgEvent)
}
- return nil
+ return update, nil
}
// emitHeadEvent emits the head event with the given head slot, head root, and head state.
@@ -349,29 +358,33 @@ func saveFinalizedStateOnDiskIfNeeded(fc forkchoice.ForkChoiceStorageReader, bea
// postForkchoiceOperations performs the post fork choice operations such as updating the head state, producing and caching attestation data,
// these sets of operations can take as long as they need to run, as by-now we are already synced.
-func postForkchoiceOperations(ctx context.Context, tx kv.RwTx, logger log.Logger, cfg *Cfg, headSlot uint64, headRoot common.Hash) error {
+func prepareForkchoiceCommit(tx kv.RwTx, cfg *Cfg, headRoot common.Hash) (*state.CachingBeaconState, error) {
// Retrieve the head state.
var headState *state.CachingBeaconState
var err error
headState, err = cfg.forkChoice.GetStateAtBlockRoot(headRoot, false)
if err != nil {
- return fmt.Errorf("failed to get state at block root: %w", err)
+ return nil, fmt.Errorf("failed to get state at block root: %w", err)
}
// fail-safe check§
if headState == nil {
- return nil
+ return nil, nil
}
- cfg.blobDownloader.SetHeadSlot(headSlot)
- // First emit events that depend on the head state.
- emitHeadEvent(cfg, headSlot, headRoot, headState)
- emitNextPaylodAttributesEvent(cfg, headSlot, headRoot, headState)
-
if _, err = cfg.attestationDataProducer.ProduceAndCacheAttestationData(tx, headState, headRoot, headState.Slot()); err != nil {
- logger.Warn("failed to produce and cache attestation data", "err", err)
+ log.Warn("failed to produce and cache attestation data", "err", err)
}
if err := beacon_indicies.WriteHighestFinalized(tx, cfg.forkChoice.FinalizedSlot()); err != nil {
- return err
+ return nil, err
}
+ return headState, nil
+}
+
+func postForkchoiceOperations(ctx context.Context, logger log.Logger, cfg *Cfg, headSlot uint64, headRoot common.Hash, headState *state.CachingBeaconState) error {
+ if headState == nil {
+ return nil
+ }
+ emitHeadEvent(cfg, headSlot, headRoot, headState)
+ emitNextPaylodAttributesEvent(cfg, headSlot, headRoot, headState)
start := time.Now()
cfg.forkChoice.SetSynced(true) // Now we are synced
// Update the head state with the new head state
@@ -414,12 +427,14 @@ func doForkchoiceRoutine(ctx context.Context, logger log.Logger, cfg *Cfg, args
return fmt.Errorf("failed to begin transaction: %w", err)
}
defer tx.Rollback()
- if err := updateCanonicalChainInTheDatabase(ctx, tx, headSlot, headRoot, cfg); err != nil {
+ canonicalUpdate, err := updateCanonicalChainInTheDatabase(ctx, tx, headSlot, headRoot, cfg)
+ if err != nil {
return fmt.Errorf("failed to update canonical chain in the database: %w", err)
}
- if err := postForkchoiceOperations(ctx, tx, logger, cfg, headSlot, headRoot); err != nil {
- return fmt.Errorf("failed to post forkchoice operations: %w", err)
+ headState, err := prepareForkchoiceCommit(tx, cfg, headRoot)
+ if err != nil {
+ return fmt.Errorf("failed to prepare forkchoice commit: %w", err)
}
var m runtime.MemStats
@@ -429,7 +444,39 @@ func doForkchoiceRoutine(ctx context.Context, logger log.Logger, cfg *Cfg, args
"alloc", common.ByteCount(m.Alloc),
"sys", common.ByteCount(m.Sys))
- return tx.Commit()
+ if err := commitCanonicalUpdate(tx, cfg.blobDownloader, headSlot, headRoot, canonicalUpdate, cfg.emitter); err != nil {
+ return err
+ }
+ if err := postForkchoiceOperations(ctx, logger, cfg, headSlot, headRoot, headState); err != nil {
+ return fmt.Errorf("failed to post forkchoice operations: %w", err)
+ }
+ return nil
+}
+
+type blobHeadTracker interface {
+ InvalidateCompletionAbove(uint64)
+ AbortHeadUpdate()
+ SetHead(uint64, common.Hash, uint64)
+}
+
+func commitCanonicalHead(tx kv.RwTx, downloader blobHeadTracker, headSlot uint64, headRoot common.Hash, commonAncestorSlot uint64) error {
+ downloader.InvalidateCompletionAbove(commonAncestorSlot)
+ if err := tx.Commit(); err != nil {
+ downloader.AbortHeadUpdate()
+ return err
+ }
+ downloader.SetHead(headSlot, headRoot, commonAncestorSlot)
+ return nil
+}
+
+func commitCanonicalUpdate(tx kv.RwTx, downloader blobHeadTracker, headSlot uint64, headRoot common.Hash, update canonicalUpdate, emitter *beaconevents.EventEmitter) error {
+ if err := commitCanonicalHead(tx, downloader, headSlot, headRoot, update.commonAncestorSlot); err != nil {
+ return err
+ }
+ if update.reorgEvent != nil {
+ emitter.State().SendChainReorg(update.reorgEvent)
+ }
+ return nil
}
// we need to generate only one goroutine for pre-caching shuffled set
diff --git a/cl/phase1/stages/forkchoice_test.go b/cl/phase1/stages/forkchoice_test.go
index 7376427b040..abdf1df11bd 100644
--- a/cl/phase1/stages/forkchoice_test.go
+++ b/cl/phase1/stages/forkchoice_test.go
@@ -2,6 +2,7 @@ package stages
import (
"context"
+ "errors"
"testing"
"github.com/stretchr/testify/require"
@@ -9,12 +10,121 @@ import (
"github.com/erigontech/erigon/cl/beacon/beaconevents"
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/persistence/beacon_indicies"
+ network2 "github.com/erigontech/erigon/cl/phase1/network"
"github.com/erigontech/erigon/common"
"github.com/erigontech/erigon/db/kv"
"github.com/erigontech/erigon/db/kv/dbcfg"
"github.com/erigontech/erigon/db/kv/memdb"
)
+type failingCommitTx struct{ kv.RwTx }
+
+func (failingCommitTx) Commit() error { return errors.New("commit failed") }
+
+type observingCommitTx struct {
+ kv.RwTx
+ onCommit func()
+ err error
+}
+
+func (tx observingCommitTx) Commit() error {
+ tx.onCommit()
+ return tx.err
+}
+
+type recordingBlobHeadTracker struct {
+ invalidated bool
+ published bool
+ aborted bool
+}
+
+func (d *recordingBlobHeadTracker) InvalidateCompletionAbove(uint64) { d.invalidated = true }
+
+func (d *recordingBlobHeadTracker) SetHead(uint64, common.Hash, uint64) { d.published = true }
+
+func (d *recordingBlobHeadTracker) AbortHeadUpdate() { d.aborted = true }
+
+func TestCommitCanonicalHeadInvalidatesBeforeCommitVisibility(t *testing.T) {
+ db := memdb.NewTestDB(t, dbcfg.ChainDB)
+ tx, err := db.BeginRw(t.Context())
+ require.NoError(t, err)
+ defer tx.Rollback()
+ tracker := &recordingBlobHeadTracker{}
+
+ err = commitCanonicalHead(observingCommitTx{
+ RwTx: tx,
+ onCommit: func() {
+ require.True(t, tracker.invalidated)
+ require.False(t, tracker.published)
+ },
+ err: errors.New("commit failed"),
+ }, tracker, 10, common.Hash{1}, 9)
+
+ require.Error(t, err)
+ require.True(t, tracker.invalidated)
+ require.False(t, tracker.published)
+ require.True(t, tracker.aborted)
+ tx.Rollback()
+}
+
+func TestCommitCanonicalHeadPublishesOnlyAfterSuccessfulCommit(t *testing.T) {
+ newDownloader := func() *network2.BlobHistoryDownloader {
+ return network2.NewBlobHistoryDownloader(t.Context(), &clparams.MainnetBeaconConfig, nil, nil, nil, nil, nil, nil, nil, false, false, nil)
+ }
+ db := memdb.NewTestDB(t, dbcfg.ChainDB)
+
+ failedTx, err := db.BeginRw(t.Context())
+ require.NoError(t, err)
+ defer failedTx.Rollback()
+ failedDownloader := newDownloader()
+ require.Error(t, commitCanonicalHead(failingCommitTx{failedTx}, failedDownloader, 10, common.Hash{1}, 9))
+ require.Zero(t, failedDownloader.HeadSlot())
+ failedTx.Rollback()
+
+ successfulTx, err := db.BeginRw(t.Context())
+ require.NoError(t, err)
+ defer successfulTx.Rollback()
+ successfulDownloader := newDownloader()
+ require.NoError(t, commitCanonicalHead(successfulTx, successfulDownloader, 10, common.Hash{1}, 9))
+ require.Equal(t, uint64(10), successfulDownloader.HeadSlot())
+}
+
+func TestCommitCanonicalUpdateEmitsReorgOnlyAfterCommitAndHeadPublication(t *testing.T) {
+ emitter := beaconevents.NewEventEmitter()
+ events := make(chan *beaconevents.EventStream, 1)
+ sub := emitter.State().Subscribe(events)
+ defer sub.Unsubscribe()
+ want := &beaconevents.ChainReorgData{Slot: 10}
+
+ db := memdb.NewTestDB(t, dbcfg.ChainDB)
+ failedTx, err := db.BeginRw(t.Context())
+ require.NoError(t, err)
+ defer failedTx.Rollback()
+ failedTracker := &recordingBlobHeadTracker{}
+ require.Error(t, commitCanonicalUpdate(failingCommitTx{failedTx}, failedTracker, 10, common.Hash{1}, canonicalUpdate{commonAncestorSlot: 9, reorgEvent: want}, emitter))
+ require.False(t, failedTracker.published)
+ select {
+ case <-events:
+ t.Fatal("reorg event emitted for failed commit")
+ default:
+ }
+ failedTx.Rollback()
+
+ successfulTx, err := db.BeginRw(t.Context())
+ require.NoError(t, err)
+ defer successfulTx.Rollback()
+ successfulTracker := &recordingBlobHeadTracker{}
+ require.NoError(t, commitCanonicalUpdate(successfulTx, successfulTracker, 10, common.Hash{1}, canonicalUpdate{commonAncestorSlot: 9, reorgEvent: want}, emitter))
+ require.True(t, successfulTracker.published)
+ select {
+ case event := <-events:
+ require.Equal(t, beaconevents.StateChainReorg, event.Event)
+ require.Same(t, want, event.Data)
+ default:
+ t.Fatal("reorg event not emitted after successful commit")
+ }
+}
+
func TestUpdateCanonicalChainReorgEvent(t *testing.T) {
db := memdb.NewTestDB(t, dbcfg.ChainDB)
defer db.Close()
@@ -53,7 +163,8 @@ func TestUpdateCanonicalChainReorgEvent(t *testing.T) {
writeBlock(root101b, root100, state101b, 101, false)
writeBlock(root102b, root101b, state102b, 102, false)
- reorg := drainReorgEvent(t, ctx, tx, 102, root102b)
+ reorg, commonAncestorSlot := drainReorgEvent(t, ctx, tx, 102, root102b)
+ require.Equal(t, uint64(100), commonAncestorSlot)
require.NotNil(t, reorg, "expected a chain_reorg event to be emitted")
require.Equal(t, uint64(102), reorg.Slot, "reorg Slot")
require.Equal(t, uint64(2), reorg.Depth, "reorg Depth should be oldHeadSlot - forkPointSlot")
@@ -101,7 +212,7 @@ func TestUpdateCanonicalChainReorgShorterFork(t *testing.T) {
writeBlock(root101b, root100, common.Hash{0xb1}, 101, false)
writeBlock(root102b, root101b, state102b, 102, false)
- reorg := drainReorgEvent(t, ctx, tx, 102, root102b)
+ reorg, _ := drainReorgEvent(t, ctx, tx, 102, root102b)
require.NotNil(t, reorg, "expected a chain_reorg event to be emitted")
require.Equal(t, uint64(102), reorg.Slot, "reorg Slot")
require.Equal(t, uint64(3), reorg.Depth, "reorg Depth: old tip 103 - fork point 100 = 3")
@@ -148,7 +259,8 @@ func TestUpdateCanonicalChainReorgLongerFork(t *testing.T) {
writeBlock(root102b, root101b, common.Hash{0xb2}, 102, false)
writeBlock(root103b, root102b, state103b, 103, false)
- reorg := drainReorgEvent(t, ctx, tx, 103, root103b)
+ reorg, commonAncestorSlot := drainReorgEvent(t, ctx, tx, 103, root103b)
+ require.Equal(t, uint64(100), commonAncestorSlot)
require.NotNil(t, reorg, "expected a chain_reorg event to be emitted")
require.Equal(t, uint64(103), reorg.Slot, "reorg Slot")
require.Equal(t, uint64(2), reorg.Depth, "reorg Depth: old tip 102 - fork point 100 = 2")
@@ -186,7 +298,8 @@ func TestUpdateCanonicalChainNoReorg(t *testing.T) {
writeBlock(root102, root101, common.Hash{0xa2}, 102, false)
- reorg := drainReorgEvent(t, ctx, tx, 102, root102)
+ reorg, commonAncestorSlot := drainReorgEvent(t, ctx, tx, 102, root102)
+ require.Equal(t, uint64(101), commonAncestorSlot)
require.Nil(t, reorg, "chain extension should NOT emit a chain_reorg event")
}
@@ -220,7 +333,7 @@ func TestUpdateCanonicalChainReorgOneSlot(t *testing.T) {
writeBlock(root101a, root100, state101a, 101, true)
writeBlock(root101b, root100, state101b, 101, false)
- reorg := drainReorgEvent(t, ctx, tx, 101, root101b)
+ reorg, _ := drainReorgEvent(t, ctx, tx, 101, root101b)
require.NotNil(t, reorg, "expected a chain_reorg event to be emitted")
require.Equal(t, uint64(101), reorg.Slot, "reorg Slot")
require.Equal(t, uint64(1), reorg.Depth, "reorg Depth: old tip 101 - fork point 100 = 1")
@@ -230,7 +343,7 @@ func TestUpdateCanonicalChainReorgOneSlot(t *testing.T) {
require.Equal(t, state101b, reorg.NewHeadState, "NewHeadState")
}
-func drainReorgEvent(t *testing.T, ctx context.Context, tx kv.RwTx, headSlot uint64, headRoot common.Hash) *beaconevents.ChainReorgData {
+func drainReorgEvent(t *testing.T, ctx context.Context, tx kv.RwTx, headSlot uint64, headRoot common.Hash) (*beaconevents.ChainReorgData, uint64) {
t.Helper()
emitter := beaconevents.NewEventEmitter()
ch := make(chan *beaconevents.EventStream, 16)
@@ -242,17 +355,21 @@ func drainReorgEvent(t *testing.T, ctx context.Context, tx kv.RwTx, headSlot uin
beaconCfg: &clparams.MainnetBeaconConfig,
}
- err := updateCanonicalChainInTheDatabase(ctx, tx, headSlot, headRoot, cfg)
+ update, err := updateCanonicalChainInTheDatabase(ctx, tx, headSlot, headRoot, cfg)
require.NoError(t, err)
+ require.NoError(t, tx.Commit())
+ if update.reorgEvent != nil {
+ cfg.emitter.State().SendChainReorg(update.reorgEvent)
+ }
for {
select {
case evt := <-ch:
if evt.Event == beaconevents.StateChainReorg {
- return evt.Data.(*beaconevents.ChainReorgData)
+ return evt.Data.(*beaconevents.ChainReorgData), update.commonAncestorSlot
}
default:
- return nil
+ return nil, update.commonAncestorSlot
}
}
}
diff --git a/cl/phase1/stages/stage_history_download.go b/cl/phase1/stages/stage_history_download.go
index 8d3aaa904e7..2b2961338b4 100644
--- a/cl/phase1/stages/stage_history_download.go
+++ b/cl/phase1/stages/stage_history_download.go
@@ -405,7 +405,7 @@ func SpawnStageHistoryDownload(cfg StageHistoryReconstructionCfg, ctx context.Co
}
if cfg.blobDownloader != nil {
- cfg.blobDownloader.SetHeadSlot(cfg.startingSlot + 1)
+ cfg.blobDownloader.SetHeadSlot(cfg.startingSlot)
cfg.blobDownloader.SetNotifyBlobBackfilled(cfg.antiquary.NotifyBlobBackfilled)
cfg.blobDownloader.Start()
}
diff --git a/cl/rpc/rpc.go b/cl/rpc/rpc.go
index 9dac6a11221..54ae4d17bc3 100644
--- a/cl/rpc/rpc.go
+++ b/cl/rpc/rpc.go
@@ -24,6 +24,7 @@ import (
"fmt"
"io"
"strings"
+ "sync"
"time"
"github.com/c2h5oh/datasize"
@@ -44,7 +45,11 @@ import (
"github.com/erigontech/erigon/node/gointerfaces/sentinelproto"
)
-const maxMessageLength = 18 * datasize.MB
+const (
+ maxMessageLength = 18 * datasize.MB
+ maxConcurrentBlobSidecarRequests = 2
+ maxConcurrentBlobSidecarBackfillRequests = 1
+)
var errBlockForkSchemaSlotMismatch = errors.New("block schema fork disagrees with the fork implied by its slot")
@@ -70,6 +75,10 @@ type BeaconRpcP2P struct {
ethClock eth_clock.EthereumClock
columnDataPeers *columnDataPeers
+
+ blobSidecarAdmissionOnce sync.Once
+ blobSidecarRequests chan struct{}
+ blobSidecarBackfill chan struct{}
}
// NewBeaconRpcP2P creates a new BeaconRpcP2P struct and returns a pointer to it.
@@ -240,8 +249,34 @@ func (b *BeaconRpcP2P) SendExecutionPayloadEnvelopesByRootReq(ctx context.Contex
return envelopes, pid, nil
}
-// SendBeaconBlocksByRangeReq retrieves blocks range from beacon chain.
func (b *BeaconRpcP2P) SendBlobsSidecarByIdentifierReq(ctx context.Context, req *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) {
+ return b.sendBlobsSidecarByIdentifierReq(ctx, req, false)
+}
+
+// SendBlobsSidecarByIdentifierReqForBackfill admits a background request without exhausting live request capacity.
+func (b *BeaconRpcP2P) SendBlobsSidecarByIdentifierReqForBackfill(ctx context.Context, req *solid.ListSSZ[*cltypes.BlobIdentifier]) ([]*cltypes.BlobSidecar, string, error) {
+ return b.sendBlobsSidecarByIdentifierReq(ctx, req, true)
+}
+
+func (b *BeaconRpcP2P) sendBlobsSidecarByIdentifierReq(ctx context.Context, req *solid.ListSSZ[*cltypes.BlobIdentifier], backfill bool) ([]*cltypes.BlobSidecar, string, error) {
+ b.blobSidecarAdmissionOnce.Do(func() {
+ b.blobSidecarRequests = make(chan struct{}, maxConcurrentBlobSidecarRequests)
+ b.blobSidecarBackfill = make(chan struct{}, maxConcurrentBlobSidecarBackfillRequests)
+ })
+ if backfill {
+ select {
+ case b.blobSidecarBackfill <- struct{}{}:
+ defer func() { <-b.blobSidecarBackfill }()
+ case <-ctx.Done():
+ return nil, "", ctx.Err()
+ }
+ }
+ select {
+ case b.blobSidecarRequests <- struct{}{}:
+ defer func() { <-b.blobSidecarRequests }()
+ case <-ctx.Done():
+ return nil, "", ctx.Err()
+ }
var buffer buffer.Buffer
if err := ssz_snappy.EncodeAndWrite(&buffer, req); err != nil {
return nil, "", err
diff --git a/cl/rpc/rpc_test.go b/cl/rpc/rpc_test.go
index 46ab29211ce..6707d94c590 100644
--- a/cl/rpc/rpc_test.go
+++ b/cl/rpc/rpc_test.go
@@ -3,13 +3,16 @@ package rpc
import (
"bytes"
"context"
+ "sync/atomic"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"github.com/erigontech/erigon/cl/clparams"
"github.com/erigontech/erigon/cl/cltypes"
+ "github.com/erigontech/erigon/cl/cltypes/solid"
"github.com/erigontech/erigon/cl/sentinel/communication/ssz_snappy"
"github.com/erigontech/erigon/cl/utils/eth_clock"
"github.com/erigontech/erigon/common"
@@ -22,6 +25,23 @@ type blockResponseSentinel struct {
bannedPeer string
}
+type blockingBlobSentinel struct {
+ sentinelproto.SentinelClient
+ active atomic.Int64
+ max atomic.Int64
+ enter chan struct{}
+}
+
+func (s *blockingBlobSentinel) SendRequest(ctx context.Context, _ *sentinelproto.RequestData, _ ...grpc.CallOption) (*sentinelproto.ResponseData, error) {
+ active := s.active.Add(1)
+ for current := s.max.Load(); active > current && !s.max.CompareAndSwap(current, active); current = s.max.Load() {
+ }
+ s.enter <- struct{}{}
+ <-ctx.Done()
+ s.active.Add(-1)
+ return nil, ctx.Err()
+}
+
func (s *blockResponseSentinel) SendRequest(context.Context, *sentinelproto.RequestData, ...grpc.CallOption) (*sentinelproto.ResponseData, error) {
return &sentinelproto.ResponseData{
Data: s.response,
@@ -61,6 +81,90 @@ func TestMaxRequestPayloadsFallback(t *testing.T) {
require.ErrorContains(t, err, "17")
}
+func TestBlobSidecarByRootRequestsShareConcurrencyLimit(t *testing.T) {
+ sentinel := &blockingBlobSentinel{enter: make(chan struct{}, 3)}
+ client := &BeaconRpcP2P{ctx: t.Context(), sentinel: sentinel, beaconConfig: &clparams.MainnetBeaconConfig}
+ req := solid.NewStaticListSSZ[*cltypes.BlobIdentifier](1, 40)
+ req.Append(&cltypes.BlobIdentifier{})
+ contexts := make([]context.Context, 3)
+ cancels := make([]context.CancelFunc, 3)
+ contexts[0], cancels[0] = context.WithCancel(t.Context())
+ contexts[1], cancels[1] = context.WithCancel(t.Context())
+ contexts[2], cancels[2] = context.WithCancel(t.Context())
+ defer cancels[0]()
+ defer cancels[1]()
+ defer cancels[2]()
+ done := make(chan struct{}, 3)
+ launch := func(index int) {
+ go func() {
+ _, _, _ = client.SendBlobsSidecarByIdentifierReq(contexts[index], req)
+ done <- struct{}{}
+ }()
+ }
+ launch(0)
+ <-sentinel.enter
+ launch(1)
+ <-sentinel.enter
+ launch(2)
+ select {
+ case <-sentinel.enter:
+ t.Fatal("third blob request crossed the shared concurrency boundary")
+ case <-time.After(100 * time.Millisecond):
+ }
+ cancels[0]()
+ select {
+ case <-sentinel.enter:
+ case <-time.After(time.Second):
+ t.Fatal("waiting blob request did not acquire a released permit")
+ }
+ require.Equal(t, int64(2), sentinel.max.Load())
+ cancels[1]()
+ cancels[2]()
+ for range 3 {
+ <-done
+ }
+}
+
+func TestBlobSidecarBackfillLeavesCapacityForLiveSync(t *testing.T) {
+ sentinel := &blockingBlobSentinel{enter: make(chan struct{}, 2)}
+ client := &BeaconRpcP2P{ctx: t.Context(), sentinel: sentinel, beaconConfig: &clparams.MainnetBeaconConfig}
+ req := solid.NewStaticListSSZ[*cltypes.BlobIdentifier](1, 40)
+ req.Append(&cltypes.BlobIdentifier{})
+ backfillCtx, cancelBackfill := context.WithCancel(t.Context())
+ defer cancelBackfill()
+ backfillDone := make(chan struct{}, 2)
+ for range 2 {
+ go func() {
+ _, _, _ = client.SendBlobsSidecarByIdentifierReqForBackfill(backfillCtx, req)
+ backfillDone <- struct{}{}
+ }()
+ }
+ select {
+ case <-sentinel.enter:
+ case <-time.After(time.Second):
+ t.Fatal("backfill request did not enter")
+ }
+
+ liveCtx, cancelLive := context.WithCancel(t.Context())
+ liveDone := make(chan struct{})
+ go func() {
+ _, _, _ = client.SendBlobsSidecarByIdentifierReq(liveCtx, req)
+ close(liveDone)
+ }()
+ select {
+ case <-sentinel.enter:
+ case <-time.After(time.Second):
+ t.Fatal("live request was starved by backfill")
+ }
+ require.Equal(t, int64(2), sentinel.max.Load())
+ cancelLive()
+ cancelBackfill()
+ <-liveDone
+ for range 2 {
+ <-backfillDone
+ }
+}
+
func TestSendBeaconBlocksByRangeReqRejectsForkSchemaSlotMismatch(t *testing.T) {
cfg := clparams.MainnetBeaconConfig
cfg.InitializeForkSchedule()
diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go
index a744357bc8e..75e33c04f74 100644
--- a/cmd/caplin/caplin1/run.go
+++ b/cmd/caplin/caplin1/run.go
@@ -581,6 +581,26 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
}
statesReader := historical_states_reader.NewHistoricalStatesReader(beaconConfig, rcsn, vTables, genesisState, stateSnapshots, syncedDataManager)
+ stageCfg := stages.ClStagesCfg(
+ ctx,
+ beaconRpc,
+ antiq,
+ ethClock,
+ beaconConfig,
+ state,
+ engine,
+ forkChoice,
+ indexDB,
+ csn,
+ rcsn,
+ dirs,
+ config,
+ syncedDataManager,
+ emitters,
+ blobStorage,
+ attestationProducer,
+ peerDas,
+ )
if config.BeaconAPIRouter.Active {
apiHandler := handler.NewApiHandler(
logger,
@@ -597,9 +617,12 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
version.NodeVersion(),
&config.BeaconAPIRouter,
emitters,
- blobStorage,
- columnStorage,
- csn,
+ handler.BlobDataDependencies{
+ Storage: blobStorage,
+ ColumnStorage: columnStorage,
+ Snapshots: csn,
+ BackfillStatus: stageCfg.BlobDownloader(),
+ },
validatorParameters,
attestationProducer,
engine,
@@ -629,27 +652,6 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
log.Info("Beacon API started", "addr", config.BeaconAPIRouter.Address)
}
- stageCfg := stages.ClStagesCfg(
- ctx,
- beaconRpc,
- antiq,
- ethClock,
- beaconConfig,
- state,
- engine,
- //gossipManager,
- forkChoice,
- indexDB,
- csn,
- rcsn,
- dirs,
- config,
- syncedDataManager,
- emitters,
- blobStorage,
- attestationProducer,
- peerDas,
- )
sync := stages.ConsensusClStages(ctx, stageCfg)
logger.Info("[Caplin] starting clstages loop")