From ddc6df79e8cc756a4d968b68bf488eec936b446a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 21:51:26 +0000 Subject: [PATCH 001/167] commitment: extract canonical DecodeBranchInto from unfoldBranchNode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure refactor — behavior preserved. Splits the encoded-branch→cells decode logic out of HexPatriciaHashed.unfoldBranchNode into a free function DecodeBranchInto so the same code is consumed by: - unfoldBranchNode (existing trie unfold path) - future cache populators (decoded-payload BranchCache) - future parallel pre-unfold orchestrator (Stage E) Today these would each have to re-derive the encoded-branch parsing logic. Centralising it ensures one decoder, one set of edge cases, one place to fix any bug in the on-disk format handling. DecodeBranchInto is intentionally PURE — it does not call deriveHashedKeys. Trie callers (which need hashed keys for the fold state machine) follow the decode with their own keccak loop. Cache callers can skip the derive step entirely until the cell is consumed by the trie. Tests: - TestDecodeBranchInto_RoundTrip: BranchEncoder.EncodeBranch produces bytes that DecodeBranchInto recovers cell-for-cell. Property test that keeps the canonical decoder consistent with the canonical encoder. - TestDecodeBranchInto_DeletedFlag: touchMap/afterMap convention with the deleted parameter. - TestDecodeBranchInto_TruncatedInput: clean errors on truncated input (no panic). Plus the existing commitment test suite (incl. trie-mismatch tests in TestBranchData_*) all pass without modification, confirming the refactor preserves unfoldBranchNode's behavior. This is the foundation for the next refactors in the representation-reduction track (see agentspecs/trie-data-pipeline-complexity-tax.md): subsequent PRs will introduce a decoded-payload cache that reads through this same decoder, and will lift unfoldKeyPath as a per-key traversal primitive that the warmer + future Stage E both consume. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_decode.go | 95 ++++++++++++++++++ execution/commitment/branch_decode_test.go | 106 ++++++++++++++++++++ execution/commitment/hex_patricia_hashed.go | 21 ++-- 3 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 execution/commitment/branch_decode.go create mode 100644 execution/commitment/branch_decode_test.go diff --git a/execution/commitment/branch_decode.go b/execution/commitment/branch_decode.go new file mode 100644 index 00000000000..0e4c6b5ccc9 --- /dev/null +++ b/execution/commitment/branch_decode.go @@ -0,0 +1,95 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "encoding/binary" + "fmt" + "math/bits" +) + +// BranchMaps captures the three branch-level bitmasks decoded alongside the +// per-cell payload. Held separately from the cells slice so callers can apply +// them into trie state without struct-assigning the whole [16]cell array. +type BranchMaps struct { + Bitmap uint16 // present-children bitmap (the canonical map encoded in the branch) + TouchMap uint16 // children that were touched in this branch (set by caller per deleted flag) + AfterMap uint16 // children present after this commitment step +} + +// DecodeBranchInto parses the on-disk encoded form of a branch (as returned +// by ctx.Branch / readBranchAndCheckForFlushing with the leading 2-byte +// touch-map stripped) and populates the supplied cells array. +// +// branchData must already have the leading touch-map prefix stripped — the +// raw bytes returned from Branch() include it; callers strip it before +// invoking this function (matching the unfoldBranchNode call pattern). +// +// deleted controls the touchMap/afterMap convention: +// - deleted=true → all decoded cells are touched-but-not-present-after +// (touchMap = bitmap, afterMap = 0) +// - deleted=false → all decoded cells are present-after (touchMap = 0, +// afterMap = bitmap) +// +// Returns the three branch-level maps for the caller to apply into their +// own state machine. The cells array is mutated in place. +// +// This is a PURE decode — it does NOT call deriveHashedKeys on the cells. +// Trie callers (HexPatriciaHashed.unfoldBranchNode) follow this with their +// own loop calling deriveHashedKeys per cell, since they have the keccak +// scratch buffer. Cache callers can skip deriveHashedKeys entirely until +// the cell is consumed by the trie. +// +// This function is the canonical decoder for branch data. Both +// unfoldBranchNode (which feeds the in-memory grid for fold/unfold) and +// future cache populators consume branches via this same path so the +// encoded→cells transformation lives in one place. +func DecodeBranchInto( + branchData []byte, + deleted bool, + cells *[16]cell, +) (BranchMaps, error) { + if len(branchData) < 2 { + return BranchMaps{}, fmt.Errorf("branch data too short for bitmap: %d bytes", len(branchData)) + } + bitmap := binary.BigEndian.Uint16(branchData[0:]) + maps := BranchMaps{Bitmap: bitmap} + if deleted { + maps.TouchMap, maps.AfterMap = bitmap, 0 + } else { + maps.TouchMap, maps.AfterMap = 0, bitmap + } + + pos := 2 + for bitset := bitmap; bitset != 0; { + bit := bitset & -bitset + nibble := bits.TrailingZeros16(bit) + c := &cells[nibble] + if pos >= len(branchData) { + return BranchMaps{}, fmt.Errorf("branch data truncated before cell at nibble %d", nibble) + } + fieldBits := branchData[pos] + pos++ + newPos, err := c.fillFromFields(branchData, pos, cellFields(fieldBits)) + if err != nil { + return BranchMaps{}, fmt.Errorf("fillFromFields nibble %d: %w", nibble, err) + } + pos = newPos + bitset ^= bit + } + return maps, nil +} diff --git a/execution/commitment/branch_decode_test.go b/execution/commitment/branch_decode_test.go new file mode 100644 index 00000000000..abe80e0b25f --- /dev/null +++ b/execution/commitment/branch_decode_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDecodeBranchInto_RoundTrip asserts that DecodeBranchInto recovers +// the cells encoded by BranchEncoder.EncodeBranch — the property test that +// keeps the canonical decoder consistent with the canonical encoder. +func TestDecodeBranchInto_RoundTrip(t *testing.T) { + t.Parallel() + row, bm := generateCellRow(t, 16) + + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + require.NotEmpty(t, enc) + + // EncodeBranch produces bytes WITH the 2-byte touch-map prefix; the + // decoder consumes the bytes WITHOUT it (matching the unfoldBranchNode + // call pattern, which strips the touch-map prefix before decoding). + branchData := []byte(enc)[2:] + + var cells [16]cell + maps, err := DecodeBranchInto(branchData, false /* not deleted */, &cells) + require.NoError(t, err) + + // Bitmap should match what was encoded + require.Equal(t, bm, maps.Bitmap, "decoded bitmap mismatch") + require.Equal(t, uint16(0), maps.TouchMap, "expected empty touchMap when deleted=false") + require.Equal(t, bm, maps.AfterMap, "afterMap should equal bitmap when deleted=false") + + // Each present cell should match the original on the fields that + // survive encode→decode (extension, account/storage addr, hash). + // hashedExtension etc. are set by deriveHashedKeys (separate step) and + // are not part of the decoder's responsibility. + for i, orig := range row { + decoded := &cells[i] + require.Equal(t, orig.extLen, decoded.extLen, "cell %d extLen", i) + require.Equal(t, orig.extension[:orig.extLen], decoded.extension[:decoded.extLen], "cell %d extension", i) + require.Equal(t, orig.accountAddrLen, decoded.accountAddrLen, "cell %d accountAddrLen", i) + require.Equal(t, orig.accountAddr[:orig.accountAddrLen], decoded.accountAddr[:decoded.accountAddrLen], "cell %d accountAddr", i) + require.Equal(t, orig.storageAddrLen, decoded.storageAddrLen, "cell %d storageAddrLen", i) + require.Equal(t, orig.storageAddr[:orig.storageAddrLen], decoded.storageAddr[:decoded.storageAddrLen], "cell %d storageAddr", i) + require.Equal(t, orig.hashLen, decoded.hashLen, "cell %d hashLen", i) + require.Equal(t, orig.hash[:orig.hashLen], decoded.hash[:decoded.hashLen], "cell %d hash", i) + } +} + +// TestDecodeBranchInto_DeletedFlag verifies the touchMap/afterMap convention +// flips correctly with the deleted parameter. +func TestDecodeBranchInto_DeletedFlag(t *testing.T) { + t.Parallel() + row, bm := generateCellRow(t, 16) + + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + branchData := []byte(enc)[2:] + + var cells [16]cell + maps, err := DecodeBranchInto(branchData, true, &cells) + require.NoError(t, err) + require.Equal(t, bm, maps.Bitmap) + require.Equal(t, bm, maps.TouchMap, "deleted=true → touchMap = bitmap") + require.Equal(t, uint16(0), maps.AfterMap, "deleted=true → afterMap = 0") +} + +// TestDecodeBranchInto_TruncatedInput asserts the decoder fails cleanly on +// truncated branch data instead of panicking. +func TestDecodeBranchInto_TruncatedInput(t *testing.T) { + t.Parallel() + var cells [16]cell + + // Empty data — should fail at bitmap read. + _, err := DecodeBranchInto(nil, false, &cells) + require.Error(t, err) + + // Just bitmap, no cells — should be fine if bitmap is 0. + _, err = DecodeBranchInto([]byte{0x00, 0x00}, false, &cells) + require.NoError(t, err, "bitmap=0 with no cell data should decode cleanly") + + // Bitmap claims one cell but data missing — should fail. + _, err = DecodeBranchInto([]byte{0x00, 0x01}, false, &cells) + require.Error(t, err, "bitmap with set bit but no cell data should error") +} diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 887e9fb8233..09f4540c495 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1750,28 +1750,19 @@ func (hph *HexPatriciaHashed) unfoldBranchNode(row int, depth int16, deleted boo return fmt.Errorf("empty branch data read during unfold, compact prefix %x nibbles %x", key, hph.currentKey[:hph.currentKeyLen]) } hph.branchBefore[row] = true - bitmap := binary.BigEndian.Uint16(branchData[0:]) - pos := 2 - if deleted { // All cells come as deleted (touched but not present after) - hph.touchMap[row], hph.afterMap[row] = bitmap, 0 - } else { - hph.touchMap[row], hph.afterMap[row] = 0, bitmap + maps, err := DecodeBranchInto(branchData, deleted, &hph.grid[row]) + if err != nil { + return fmt.Errorf("prefix [%x] branchData[%x]: %w", hph.currentKey[:hph.currentKeyLen], branchData, err) } - //fmt.Printf("unfoldBranchNode prefix '%x' [%x], afterMap = [%016b], touchMap = [%016b]\n", key, branchData, hph.afterMap[row], hph.touchMap[row]) - // Loop iterating over the set bits of modMask - for bitset, j := bitmap, 0; bitset != 0; j++ { + hph.touchMap[row] = maps.TouchMap + hph.afterMap[row] = maps.AfterMap + for bitset := maps.Bitmap; bitset != 0; { bit := bitset & -bitset nibble := bits.TrailingZeros16(bit) cell := &hph.grid[row][nibble] - fieldBits := branchData[pos] - pos++ - if pos, err = cell.fillFromFields(branchData, pos, cellFields(fieldBits)); err != nil { - return fmt.Errorf("prefix [%x] branchData[%x]: %w", hph.currentKey[:hph.currentKeyLen], branchData, err) - } if hph.trace { fmt.Printf("cell (%d, %x, depth=%d) %s\n", row, nibble, depth, cell.FullString()) } - // relies on plain account/storage key so need to be dereferenced before hashing if err = cell.deriveHashedKeys(depth, hph.keccak, hph.accountKeyLen, hph.cellHashBuf[:]); err != nil { return err From 31fdf5789338ef208eb48a66d105b62e421b6dac Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 07:59:21 +0000 Subject: [PATCH 002/167] commitment: add BranchCache with pinned root + bounded LRU tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a new BranchCache type, distinct from WarmupCache and designed for the longer-lived caching the cross-block persistence work needs (step 7 of the representation-reduction sequence). Distinguishing characteristics vs WarmupCache: - Bounded LRU tail with configurable capacity (vs WarmupCache's unbounded map). Suitable for caches that outlive a single Process without unbounded memory growth. - Single pinned slot for the root branch (compact prefix [0x00]). Root never evicts. Atomic-pointer load/store on the hot read path, no lock involved. - dirty-flag + PutIfClean invariants — same semantics as the invariants added to WarmupCache in the previous commit. Lets cross-block writers race safely with fold updates. - Lazy GetDecoded — same lazy-decode pattern as WarmupCache's GetBranchDecoded; cells populated on first decoded-read and cached for subsequent reads. NOT yet wired into the trie's read or write paths. This commit just adds the type, with tests. The trie integration (where this cache plugs into branchFromCacheOrDB and the encoder's PutBranch) is the discussion point at the step 6 boundary — see the conversation captured at this point in the representation-reduction sequence. Today the cache is intended to be ephemeral (per-Process, constructed alongside the trie, dies with it). Step 7 lifts the lifetime to the aggTx level for cross-block persistence; the cache shape (bounded LRU + pinned root + dirty-flag) is in place ahead of that. Tests: - TestBranchCache_RootPinning: root branch lands in pinned slot, deep branches land in LRU tail; per-tier hit counters update independently. - TestBranchCache_RootSurvivesEvictionPressure: root persists when tail is overfilled past capacity. - TestBranchCache_DirtyFlag: PutIfClean refuses dirty entry, unconditional Put replaces and clears dirty. - TestBranchCache_GetDecoded: lazy-decode round-trip with BranchEncoder; cells pointer reused across reads. - TestBranchCache_Invalidate: removes from both tiers. - TestBranchCache_Clear: empties both tiers, resets stats. - TestBranchCache_Stats: deterministic format with per-tier counts. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_cache.go | 272 ++++++++++++++++++++++ execution/commitment/branch_cache_test.go | 185 +++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 execution/commitment/branch_cache.go create mode 100644 execution/commitment/branch_cache_test.go diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go new file mode 100644 index 00000000000..9bea8200207 --- /dev/null +++ b/execution/commitment/branch_cache.go @@ -0,0 +1,272 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/erigontech/erigon/common/maphash" +) + +// BranchCache stores commitment-trie branch data with a persistence-friendly +// shape distinct from WarmupCache: +// +// - Bounded LRU tail with configurable capacity (eviction is well-defined, +// suitable for long-lived caching across many Process calls without +// unbounded memory growth). +// - Single pinned slot for the root branch (always hottest, always present +// once populated, never subject to LRU eviction). Compact prefix of +// length 0 (or single-byte "no-key" form) targets this slot. +// - dirty-flag + PutIfClean invariants (carried from step 4 of the +// representation-reduction sequence) so cross-block writers can race +// safely with fold updates. +// - Lazy-decoded read path (GetDecoded) — same pattern as WarmupCache's +// GetBranchDecoded; cells are populated from the cached encoded form +// on first decoded-read and reused thereafter. +// +// Today this is constructed as ephemeral infrastructure (per-Process, +// alongside the trie). Future cross-block persistence work (step 7 of +// the representation-reduction sequence) will lift its lifetime to the +// aggTx level. The wire-up between the trie's read path and BranchCache +// is intentionally NOT done in this commit — see the integration +// discussion captured at the time of step 6. +// +// Distinct from WarmupCache: WarmupCache is a simple unbounded map for +// warmup-fill hot-read patterns within one Process. BranchCache is +// designed for the longer-lived cache lifetime that the cross-block +// persistence work needs. +type BranchCache struct { + // Pinned tier — single slot for the root branch. Atomic-pointer + // access so no lock is needed for the hot read path. + root atomic.Pointer[branchCacheEntry] + + // LRU tail — bounded entries, evicts oldest when full. maphash.LRU + // wraps hashicorp/golang-lru/v2 which is thread-safe internally. + tail *maphash.LRU[*branchCacheEntry] + + // Stats — atomic counters surfaced via Stats(). + rootHits, rootMisses atomic.Uint64 + tailHits, tailMisses atomic.Uint64 + bytesServed atomic.Uint64 +} + +type branchCacheEntry struct { + // data is the canonical encoded form (with the leading 2-byte touch-map + // prefix). Always populated by Put / PutIfClean. + data []byte + + // Lazy-decoded form. Populated on first GetDecoded for this entry; + // subsequent reads return the cached cells. decodeOnce ensures decode + // runs at most once even under concurrent reads. + decodeOnce sync.Once + cells [16]cell + cellsBitmap uint16 + decodedReady bool + decodeErr error + + // dirty signals "the canonical store has been written to since this + // entry was populated; treat as stale until cleared." Same semantics + // as branchEntry.dirty in WarmupCache (carried from step 4). + dirty atomic.Bool +} + +// DefaultBranchCacheTailCapacity is the LRU tail size used when no +// explicit capacity is given. ~50k entries × ~500 bytes = ~25 MB +// at typical mainnet branch sizes. +const DefaultBranchCacheTailCapacity = 50000 + +// NewBranchCache constructs a BranchCache with the given LRU tail capacity. +// Capacity <= 0 panics — pass a positive value or DefaultBranchCacheTailCapacity. +func NewBranchCache(tailCapacity int) *BranchCache { + if tailCapacity <= 0 { + panic(fmt.Sprintf("BranchCache: tailCapacity must be positive, got %d", tailCapacity)) + } + tail, err := maphash.NewLRU[*branchCacheEntry](tailCapacity) + if err != nil { + panic(fmt.Sprintf("BranchCache: NewLRU: %s", err)) + } + return &BranchCache{tail: tail} +} + +// isRootPrefix reports whether prefix targets the pinned root slot. The +// commitment-trie compact encoding uses a 1-byte even-length flag (0x00) +// to represent the empty nibble path (root branch). Anything longer goes +// to the LRU tail. +func isRootPrefix(prefix []byte) bool { + return len(prefix) == 1 && prefix[0] == 0x00 +} + +func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { + if isRootPrefix(prefix) { + entry := c.root.Load() + if entry == nil { + c.rootMisses.Add(1) + return nil, false + } + c.rootHits.Add(1) + return entry, true + } + entry, ok := c.tail.Get(prefix) + if !ok { + c.tailMisses.Add(1) + return nil, false + } + c.tailHits.Add(1) + return entry, true +} + +func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { + if isRootPrefix(prefix) { + c.root.Store(entry) + return + } + c.tail.Set(prefix, entry) +} + +// Get retrieves branch data from the cache. Returns the canonical encoded +// bytes (with the leading 2-byte touch-map prefix). +func (c *BranchCache) Get(prefix []byte) ([]byte, bool) { + entry, ok := c.lookup(prefix) + if !ok { + return nil, false + } + c.bytesServed.Add(uint64(len(entry.data))) + return entry.data, true +} + +// GetDecoded retrieves the cached branch in decoded form. Lazy-decodes on +// first access for each entry; subsequent reads return the cached cells +// pointer without redoing the parse work. +// +// Returns the bitmap of present children plus a pointer to the populated +// cells array. Caller derives touchMap/afterMap based on its own context +// (deleted vs present-after) — same convention as WarmupCache.GetBranchDecoded. +// +// The returned *[16]cell aliases storage owned by the cache entry — the +// caller MUST NOT modify the cells in place. Read-only consumption is +// safe across concurrent calls. +func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { + entry, found := c.lookup(prefix) + if !found { + return 0, nil, false + } + entry.decodeOnce.Do(func() { + if len(entry.data) < 2 { + entry.decodeErr = fmt.Errorf("branch entry too short for touch-map prefix: %d bytes", len(entry.data)) + return + } + maps, err := DecodeBranchInto(entry.data[2:], false /* deleted derived per-caller */, &entry.cells) + if err != nil { + entry.decodeErr = err + return + } + entry.cellsBitmap = maps.Bitmap + entry.decodedReady = true + }) + if !entry.decodedReady { + return 0, nil, false + } + c.bytesServed.Add(uint64(len(entry.data))) + return entry.cellsBitmap, &entry.cells, true +} + +// Put stores branch data in the cache, replacing any existing entry +// (clearing its dirty flag in the process — the new entry is fresh). +// Always copies the input data so the cache owns it independently of +// caller buffer lifetime. +func (c *BranchCache) Put(prefix []byte, data []byte) { + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + c.store(prefix, &branchCacheEntry{data: dataCopy}) +} + +// PutIfClean stores branch data only if no existing entry is marked dirty. +// Returns true on store, false if a dirty entry was present (indicating a +// canonical-store write is in progress and the caller's data is potentially +// stale). +// +// Same semantics as WarmupCache.PutBranchIfClean — see that doc for the +// race-it-protects-against narrative. +func (c *BranchCache) PutIfClean(prefix []byte, data []byte) bool { + if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { + return false + } + c.Put(prefix, data) + return true +} + +// MarkDirty flags the entry at prefix as stale-until-cleared. Subsequent +// PutIfClean calls for this prefix will skip; reads still return the +// entry (the dirty signal is consumed only on the write path today, same +// as WarmupCache.MarkBranchDirty). +// +// No-op if no entry exists at prefix. +func (c *BranchCache) MarkDirty(prefix []byte) { + if entry, ok := c.lookup(prefix); ok { + entry.dirty.Store(true) + } +} + +// Invalidate removes the entry at prefix entirely. Use when the caller +// knows the canonical store has changed and the cached entry should not +// be served at all (vs MarkDirty which keeps the entry but blocks +// PutIfClean overwrites). +func (c *BranchCache) Invalidate(prefix []byte) { + if isRootPrefix(prefix) { + c.root.Store(nil) + return + } + c.tail.Delete(prefix) +} + +// Clear empties the cache and resets stats counters. Use on Reset / +// fork-validation paths to ensure stale entries from one trie root are +// not served against a different root. +func (c *BranchCache) Clear() { + c.root.Store(nil) + c.tail.Purge() + c.rootHits.Store(0) + c.rootMisses.Store(0) + c.tailHits.Store(0) + c.tailMisses.Store(0) + c.bytesServed.Store(0) +} + +// Stats returns a one-line summary of root-tier and tail-tier hit/miss +// counters plus bytes served. Format mirrors WarmupCache.Stats() so +// per-Process log lines can compose them. +func (c *BranchCache) Stats() string { + rh, rm := c.rootHits.Load(), c.rootMisses.Load() + th, tm := c.tailHits.Load(), c.tailMisses.Load() + bb := c.bytesServed.Load() + pct := func(hit, miss uint64) float64 { + total := hit + miss + if total == 0 { + return 0 + } + return 100.0 * float64(hit) / float64(total) + } + return fmt.Sprintf( + "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d", + rh, rm, pct(rh, rm), + th, tm, pct(th, tm), + float64(bb)/1024/1024, + c.tail.Len(), + ) +} diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go new file mode 100644 index 00000000000..5338a090701 --- /dev/null +++ b/execution/commitment/branch_cache_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestBranchCache_RootPinning verifies the root branch lands in the pinned +// slot (counted as root-hit) and tail entries land in the LRU tier +// (counted as tail-hit). +func TestBranchCache_RootPinning(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch + deepKey := []byte{0x12, 0x34, 0x56} + c.Put(rootKey, []byte("root-data")) + c.Put(deepKey, []byte("deep-data")) + + // Root reads should increment rootHits, not tailHits + got, ok := c.Get(rootKey) + require.True(t, ok) + require.Equal(t, []byte("root-data"), got) + require.Equal(t, uint64(1), c.rootHits.Load()) + require.Equal(t, uint64(0), c.tailHits.Load()) + + // Deep reads should increment tailHits, not rootHits + got, ok = c.Get(deepKey) + require.True(t, ok) + require.Equal(t, []byte("deep-data"), got) + require.Equal(t, uint64(1), c.rootHits.Load()) + require.Equal(t, uint64(1), c.tailHits.Load()) +} + +// TestBranchCache_RootSurvivesEvictionPressure verifies that pinned root +// entry is not subject to LRU eviction even if the tail fills past +// capacity many times over. +func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { + c := NewBranchCache(10) // very small tail + rootKey := []byte{0x00} + c.Put(rootKey, []byte("ROOT-PERSISTS")) + + // Stuff the tail well past capacity + for i := 0; i < 100; i++ { + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}) + } + + // Root must still be there + got, ok := c.Get(rootKey) + require.True(t, ok, "root should never be evicted from pinned slot") + require.Equal(t, []byte("ROOT-PERSISTS"), got) + + // Tail at capacity (10), not 100 + require.LessOrEqual(t, c.tail.Len(), 10, "tail should respect LRU capacity") +} + +// TestBranchCache_DirtyFlag verifies PutIfClean refuses overwrite of a +// dirty entry, while Put unconditionally replaces (and the new entry +// starts clean). +func TestBranchCache_DirtyFlag(t *testing.T) { + c := NewBranchCache(100) + key := []byte{0x12, 0x34} + + require.True(t, c.PutIfClean(key, []byte("v1"))) + c.MarkDirty(key) + + require.False(t, c.PutIfClean(key, []byte("v2")), "PutIfClean must refuse dirty entry") + got, _ := c.Get(key) + require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") + + // Unconditional Put replaces + c.Put(key, []byte("v3")) + got, _ = c.Get(key) + require.Equal(t, []byte("v3"), got) + + // New entry is clean + require.True(t, c.PutIfClean(key, []byte("v4"))) +} + +// TestBranchCache_GetDecoded verifies the lazy-decode read path for a +// real encoded branch (round-trip with BranchEncoder). +func TestBranchCache_GetDecoded(t *testing.T) { + c := NewBranchCache(100) + + row, bm := generateCellRow(t, 16) + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + + prefix := []byte{0x12, 0x34} + c.Put(prefix, enc) + + // First decoded-read decodes lazily + bitmap, cells, ok := c.GetDecoded(prefix) + require.True(t, ok) + require.Equal(t, bm, bitmap) + require.NotNil(t, cells) + + // Second call returns same cells pointer + _, cells2, ok := c.GetDecoded(prefix) + require.True(t, ok) + require.Same(t, cells, cells2, "decoded form cached and reused") + + // Encoded form unchanged after decoded reads + encGot, ok := c.Get(prefix) + require.True(t, ok) + require.Equal(t, []byte(enc), encGot) +} + +// TestBranchCache_Invalidate removes entries from both tiers. +func TestBranchCache_Invalidate(t *testing.T) { + c := NewBranchCache(100) + rootKey := []byte{0x00} + deepKey := []byte{0x12, 0x34} + c.Put(rootKey, []byte("r")) + c.Put(deepKey, []byte("d")) + + c.Invalidate(rootKey) + _, ok := c.Get(rootKey) + require.False(t, ok, "root invalidated") + + c.Invalidate(deepKey) + _, ok = c.Get(deepKey) + require.False(t, ok, "deep invalidated") +} + +// TestBranchCache_Clear empties everything and resets stats. +func TestBranchCache_Clear(t *testing.T) { + c := NewBranchCache(100) + c.Put([]byte{0x00}, []byte("r")) + c.Put([]byte{0x12}, []byte("d")) + c.Get([]byte{0x00}) + c.Get([]byte{0x12}) + + require.Equal(t, uint64(1), c.rootHits.Load()) + require.Equal(t, uint64(1), c.tailHits.Load()) + + c.Clear() + require.Equal(t, uint64(0), c.rootHits.Load()) + require.Equal(t, uint64(0), c.tailHits.Load()) + _, ok := c.Get([]byte{0x00}) + require.False(t, ok) + _, ok = c.Get([]byte{0x12}) + require.False(t, ok) +} + +// TestBranchCache_Stats verifies the format of the stats string is +// deterministic and contains the expected per-tier counts. +func TestBranchCache_Stats(t *testing.T) { + c := NewBranchCache(100) + c.Put([]byte{0x00}, []byte("rrr")) + c.Put([]byte{0x12, 0x34}, []byte("ddd")) + c.Get([]byte{0x00}) + c.Get([]byte{0x12, 0x34}) + c.Get([]byte{0xff}) // tail miss + + s := c.Stats() + for _, want := range []string{ + "root hit=1 miss=0", + "tail hit=1 miss=1", + "tail entries=1", // we put 1 deep entry; root not counted in tail + } { + require.Contains(t, s, want, "Stats output: %s", s) + } + // Sanity: format doesn't blow up if we read it + require.True(t, strings.HasPrefix(s, "branch-cache ")) +} From 1593f0e8c0eb18c4f6aa839aff7ecf83ed4ce4ce Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 09:16:24 +0000 Subject: [PATCH 003/167] commitment: document BranchCache concurrency contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc-only addition to BranchCache's package-level comment, capturing the caller invariants the cache assumes and the conditions under which the existing concurrent trie satisfies them. Three caller invariants: 1. Single writer per prefix at any moment. 2. Mark-dirty-then-Put discipline for racing writers. 3. Decoded cells from GetDecoded are read-only (alias entry storage). The current ConcurrentPatriciaHashed satisfies all three by construction: - Mounts partition by first nibble (disjoint prefix spaces, no cross-mount writes to the same key). - Root branch written by single sequential fold post-Wait. - Mount→root grid roll-up is rootMu-protected (in-memory grid only, separate from cache writes). Doc explicitly flags that any future parallel fold redesign (Stage F in agentspecs/stage-e-pre-unfold-design.md) MUST preserve these invariants — particularly the single-writer-per-prefix one, which breaks if parent branches are written incrementally as children complete in parallel. The required coordination layer goes at the orchestrator (per-parent atomic counter; only the last-decrementer writes the parent), NOT inside the cache. The cache's existing primitives (atomic dirty flag, thread-safe LRU, atomic root pointer) are sufficient for that orchestrator to build on. Motivation: Stage F is likely deferred because the bench data shows fold isn't the bottleneck for the canonical SSTORE-bloat workload. But adding the constraint to the cache later (after caching is in production) is much harder than documenting it now — correctness regressions from a missed coordination layer can hide for many blocks. Documenting the contract on the cache itself ensures any engineer touching parallel fold sees it. No code change. Doc-only. All tests still pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_cache.go | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 9bea8200207..4851fc7bb5e 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -51,6 +51,79 @@ import ( // warmup-fill hot-read patterns within one Process. BranchCache is // designed for the longer-lived cache lifetime that the cross-block // persistence work needs. +// +// # Concurrency contract — caller invariants +// +// Internally, the LRU tail is thread-safe (hashicorp/golang-lru/v2) and +// the pinned root slot is an atomic.Pointer. So any combination of +// concurrent Get / GetDecoded / Put / PutIfClean / MarkDirty / Invalidate +// is mechanically safe — no panics, no torn reads. But "mechanically safe" +// is NOT the same as "logically consistent across writers." The cache is +// designed to be used under the following caller invariants: +// +// 1. Single writer per prefix at any moment. The cache does not coordinate +// concurrent writes to the same key — last-Put-wins semantics, with no +// guarantee that the winning value is the one the application wanted. +// +// 2. Mark-dirty-then-Put discipline for writers that may race with +// readers. Caller calls MarkDirty BEFORE producing the new bytes, then +// Put AFTER the canonical-store write succeeds. This is the +// deferred-encoding-friendly alternative to inline invalidation +// (motivated by the prototype investigation that found inline +// invalidate is incompatible with deferred encoding — see +// agentspecs/commitment-cache-prototype-dev-context.md). +// +// 3. Decoded cells returned by GetDecoded MUST NOT be mutated. The +// *[16]cell pointer aliases entry-owned storage and is shared across +// readers; in-place mutation breaks consistency for all subsequent +// readers of that prefix. +// +// # Concurrency contract — how the existing concurrent trie satisfies it +// +// The current ConcurrentPatriciaHashed (parallel commitment calculator) +// satisfies all three caller invariants by construction: +// +// - Mounts partition the prefix space by FIRST NIBBLE. Mount N's +// encoder only writes branches whose key starts with [0x0N ...]. +// Different mounts therefore never write to the same prefix. +// (See hex_concurrent_patricia_hashed.go: NewConcurrentPatriciaHashed +// creates 16 mounts via SpawnSubTrie; each mount has its own HPH, +// own BranchEncoder, own PatriciaContext / roTx.) +// +// - Root branch (prefix [0x00]) is written by the single root fold +// that runs SEQUENTIALLY after errgroup.Wait() in ParallelHashSort. +// One writer for the pinned root slot. +// +// - Mount→root grid roll-up is mutex-protected via +// ConcurrentPatriciaHashed.rootMu — but that updates IN-MEMORY grid +// cells, not the cache. The cache only sees the eventual root +// branch when the post-Wait root fold encodes it. +// +// # Concurrency contract — what future parallel fold work must preserve +// +// Stage F (parallel tree-reduce fold), described in +// agentspecs/trie-data-pipeline-complexity-tax.md, would change condition +// 2 above: the parent fold (incl. root) would no longer be a single +// post-Wait sequential pass. Multiple goroutines would compute parent +// branches in parallel as their children complete. This MUST not violate +// "single writer per prefix" — any future Stage F design needs an +// explicit per-prefix coordination layer (atomic counter on parent +// "children remaining"; only the last-decrementer writes the parent). +// The dirty-flag + PutIfClean primitives in this cache are sufficient +// for that coordination layer; the cache itself does NOT add per-prefix +// locking because that would be wasted work for the current architecture. +// +// If you are implementing parallel fold (or any other architecture that +// breaks the "single writer per prefix" invariant), do NOT relax the +// invariant by adding internal locking to the cache. Add the +// coordination at the orchestrator layer where the partitioning logic +// lives. The cache stays simple; the orchestrator owns the discipline. +// +// Likewise if you change the prefix partitioning (e.g. by-second-nibble +// mounts, depth-based partitioning, anything other than first-nibble), +// re-validate that distinct workers continue to write disjoint prefix +// spaces. Re-read the partitioning code in +// hex_concurrent_patricia_hashed.go and confirm. type BranchCache struct { // Pinned tier — single slot for the root branch. Atomic-pointer // access so no lock is needed for the hot read path. From 8a2c56513e2bc859d8bec9527bd77cf12d77f43d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 09:28:34 +0000 Subject: [PATCH 004/167] commitment: wire BranchCache into trie read + encoder write paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 7a of the representation-reduction sequence: per-Process integration of the BranchCache type added in the previous commit. Plumbing: - Trie interface gains SetBranchCache(*BranchCache). HexPatriciaHashed implementation propagates to its branchEncoder. ConcurrentPatriciaHashed implementation propagates the SAME instance to root + all 16 mounts (sharing one cache is correct under the concurrency contract — mounts partition prefix space by first nibble, so cross-mount writes target distinct keys). - InitializeTrieAndUpdates constructs a new BranchCache(default) per trie instance and attaches it. Lifetime today = trie lifetime = per-Process. Future cross-block persistence work (step 7b) lifts this to aggTx scope by constructing the cache one layer up and passing it in. Read path (HPH.branchFromCacheOrDB): L1 WarmupCache (existing) → L2 BranchCache (new) → L3 ctx.Branch. L3 hits with non-empty result populate L2 so subsequent reads hit L2 within the cache's lifetime. L1 stays first because warmup workers may have pre-fetched with prefix-walk-derived freshness. Write path (BranchEncoder.CollectUpdate): - MarkDirty(prefix) BEFORE encode work — protects against concurrent warmup-style writers racing into PutIfClean during the encode (race documented in the cache's Concurrency Contract). - Put(prefixCopy, updateCopy) AFTER ctx.PutBranch succeeds — replaces the dirty entry with fresh canonical bytes. Single writer per prefix per fold step (current sequential fold + first-nibble mount partitioning) means no race on this Put. Lifecycle: HPH.Reset clears the BranchCache when called from the root trie (gated by !hph.mounted). Mounted subtries share the root's cache, so a mount calling Clear would dump entries the root still wants. Carries the invariant from PR #19954 commit 1612d5608c. Today's expected performance impact: minimal. Per-Process lifetime means cache is empty at Process start, so first reads always miss. The cache helps only branches that are read multiple times within ONE Process — uncommon in current code paths. Step 7b is where the real perf swing comes from (cross-block persistence so block N reads hit branches written by block N-1). This commit is the safe stepping stone: it validates the wire-up end-to-end (read path + write path + concurrency contract + lifecycle) without changing perf characteristics. Bench should match Run I baseline (7.16 mgas/s on canonical SSTORE-bloat block). All commitment tests pass, lint clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/commitment.go | 28 +++++++++ .../hex_concurrent_patricia_hashed.go | 12 ++++ execution/commitment/hex_patricia_hashed.go | 57 ++++++++++++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index ee730aee321..727fafef409 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -102,6 +102,12 @@ type Trie interface { EnableCsvMetrics(filePathPrefix string) // EnableWarmupCache enables/disables warmup cache during Process (false by default) EnableWarmupCache(bool) + // SetBranchCache attaches a BranchCache instance for branch read/write + // caching across the trie + branchEncoder. ConcurrentPatriciaHashed + // implementations propagate the same instance to all mounts so the + // concurrency contract documented in branch_cache.go is respected + // (single shared cache, partitioned-by-prefix writers). + SetBranchCache(*BranchCache) // Variant returns commitment trie variant Variant() TrieVariant @@ -155,6 +161,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * case VariantConcurrentHexPatricia: root := NewHexPatriciaHashed(length.Addr, nil) trie := NewConcurrentPatriciaHashed(root, nil) + trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) // tree.SetConcurrentCommitment(true) // first run always sequential return trie, tree @@ -169,6 +176,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * default: trie := NewHexPatriciaHashed(length.Addr, nil) + trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) return trie, tree } @@ -350,6 +358,10 @@ type BranchEncoder struct { deferred []*DeferredBranchUpdate pendingPrefixes *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates cache *WarmupCache + // branchCache, when set, receives mark-dirty-before-encode + Put-after- + // canonical-write so the cache stays consistent with ctx.PutBranch. + // Set via HexPatriciaHashed.SetBranchCache (which propagates here). + branchCache *BranchCache } func NewBranchEncoder(sz uint64) *BranchEncoder { @@ -570,6 +582,15 @@ func (be *BranchEncoder) CollectUpdate( var foundInCache bool var err error + // Mark the BranchCache entry dirty BEFORE doing the encode work. Any + // concurrent warmer-style writer that races into PutIfClean for this + // prefix between now and our final Put below will see the dirty flag + // and skip — preventing a stale read from overwriting our fresh + // canonical write. See branch_cache.go's Concurrency Contract. + if be.branchCache != nil { + be.branchCache.MarkDirty(prefix) + } + if be.cache != nil { prev, foundInCache = be.cache.GetAndEvictBranch(prefix) if foundInCache && be.metrics != nil { @@ -606,6 +627,13 @@ func (be *BranchEncoder) CollectUpdate( if be.cache != nil { be.cache.PutBranch(prefixCopy, updateCopy) } + // Put on BranchCache replaces the (now dirty) prior entry with the + // fresh canonical bytes — clears the dirty flag in the process + // (the new entry is born clean). Single writer per prefix per fold + // invariant means no concurrent Put races on this key. + if be.branchCache != nil { + be.branchCache.Put(prefixCopy, updateCopy) + } if be.metrics != nil { be.metrics.updateBranch.Add(1) } diff --git a/execution/commitment/hex_concurrent_patricia_hashed.go b/execution/commitment/hex_concurrent_patricia_hashed.go index a5f32c42717..1fca7759959 100644 --- a/execution/commitment/hex_concurrent_patricia_hashed.go +++ b/execution/commitment/hex_concurrent_patricia_hashed.go @@ -171,6 +171,18 @@ func (p *ConcurrentPatriciaHashed) EnableWarmupCache(b bool) { p.mounts[i].EnableWarmupCache(b) } } + +// SetBranchCache attaches the same BranchCache instance to the root trie +// and all 16 mounts. Sharing one cache across mounts is correct under the +// concurrency contract (branch_cache.go): mounts partition the prefix +// space by first nibble, so cross-mount writes always target distinct +// cache keys. +func (p *ConcurrentPatriciaHashed) SetBranchCache(c *BranchCache) { + p.root.SetBranchCache(c) + for i := range p.mounts { + p.mounts[i].SetBranchCache(c) + } +} func (p *ConcurrentPatriciaHashed) GetCapture(truncate bool) []string { capture := p.root.GetCapture(truncate) if truncate { diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 09f4540c495..1ce1a1452ad 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -105,6 +105,13 @@ type HexPatriciaHashed struct { cache *WarmupCache enableWarmupCache bool // if true, enables warmup cache during Process (false by default) + // branchCache is the BranchCache instance attached via SetBranchCache. + // Wired into branchFromCacheOrDB as Level-2 (between WarmupCache and DB) + // and into branchEncoder for write-back. Today created per-Process by + // InitializeTrieAndUpdates; future cross-block persistence work lifts + // this to aggTx scope. See branch_cache.go's concurrency contract. + branchCache *BranchCache + // leaveDeferredForCaller when true, Process() leaves deferred updates on the branchEncoder // for the caller to handle via TakeDeferredUpdates(). When false (default), Process() // applies deferred updates inline. @@ -2923,6 +2930,20 @@ func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } func (hph *HexPatriciaHashed) EnableWarmupCache(enable bool) { hph.enableWarmupCache = enable } +// SetBranchCache attaches a BranchCache for branch read-through and +// write-through. Also propagates to the trie's BranchEncoder so encoder +// writes update the cache via mark-dirty-then-Put (see CollectUpdate). +// +// nil disables — both this trie's branchFromCacheOrDB Level-2 and the +// encoder's cache write-back become no-ops. +func (hph *HexPatriciaHashed) SetBranchCache(c *BranchCache) { + hph.branchCache = c + hph.branchEncoder.branchCache = c +} + +// BranchCache returns the attached BranchCache, or nil if none is set. +func (hph *HexPatriciaHashed) BranchCache() *BranchCache { return hph.branchCache } + func (hph *HexPatriciaHashed) GetCapture(truncate bool) []string { capture := hph.capture if truncate { @@ -2983,6 +3004,14 @@ func (hph *HexPatriciaHashed) Reset() { hph.rootTouched = false hph.rootChecked = false hph.rootPresent = true + + // Clear the BranchCache only when this is the root trie (not a + // mounted subtrie). Mounted subtries share the root's cache; clearing + // from a mount would dump entries the root still expects to see. + // Carries the invariant established in PR #19954 commit 1612d5608c. + if hph.branchCache != nil && !hph.mounted { + hph.branchCache.Clear() + } } func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { @@ -2994,7 +3023,20 @@ func (hph *HexPatriciaHashed) Cache() *WarmupCache { return hph.cache } -// branchFromCacheOrDB reads branch data from cache if available, otherwise from DB. +// branchFromCacheOrDB reads branch data via the cache hierarchy: +// - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) +// - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped +// once the cross-block-persistence step lands) +// - L3: ctx.Branch (canonical store) +// +// On L2 hit no further work happens. On L3 read with a non-empty result the +// bytes are populated into BranchCache so subsequent reads (within the cache's +// lifetime) hit L2 instead. +// +// L1 (WarmupCache) is intentionally checked FIRST: warmup workers may have +// pre-fetched the branch with prefix-walk-derived freshness guarantees +// (see warmuper.go), and L1 entries are short-lived enough that staleness +// is bounded by Process duration. func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { if hph.cache != nil { if data, found := hph.cache.GetBranch(key); found { @@ -3007,8 +3049,19 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { hph.metrics.missBranch.Add(1) } } + if hph.branchCache != nil { + if data, found := hph.branchCache.Get(key); found { + return data, nil + } + } data, _, err := hph.ctx.Branch(key) - return data, err + if err != nil { + return nil, err + } + if hph.branchCache != nil && len(data) > 0 { + hph.branchCache.Put(key, data) + } + return data, nil } // accountFromCacheOrDB reads account data from cache if available, otherwise from DB. From 2849165e1e62913449fe42f1aeaa6ff757f63482 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 17:53:12 +0000 Subject: [PATCH 005/167] commitment, db/state: lift BranchCache lifetime to aggregator scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the BranchCache was constructed inside InitializeTrieAndUpdates, giving it per-SharedDomains lifetime. SharedDomains is reconstructed for every batch / many tx boundaries — verified with logs in the prototype (921 fresh BranchCache constructions per bench run) — so the cache started cold every batch and never delivered the cross-block hits the design is about. Per-Process scope kept cache-cleanup correctness simple but defeated the whole point of caching. Place the cache on the commitment Domain struct (one cache per aggregator, matching the pattern on add_execution_context_with_caches where each domain owns its valueCache). ConfigureDomains attaches the cache once after domains are initialised — idempotent, lifetime = aggregator lifetime. AggregatorRoTx exposes BranchCache() returning the commitment domain's cache, so SharedDomains' construction path can fetch it without forcing db/state/execctx to import db/state (db/state already imports execctx via squeeze.go, so the reverse import would create a cycle). The placeholder commitment.BranchCacheProvider interface lets the SD construction path use a duck-typed type assertion on tx.AggTx() any. Plumb the cache through NewSharedDomainsCommitmentContext into InitializeTrieAndUpdates as an explicit parameter; nil falls back to a fresh per-init cache so test helpers without an aggregator still get a valid cache. Reset behaviour: HexPatriciaHashed.Reset no longer calls Clear on the cache. Aggregator-scope persistence requires the cache to survive Reset between commitment calculations. Callers that genuinely need to invalidate the cache (unwind, fork validation) now call ClearBranchCache explicitly. The bench is forward-only so this is a safe change for measurement; an explicit unwind clear path can land in a follow-up commit when needed. --- db/state/aggregator.go | 20 +++++++++++++++++++ db/state/domain.go | 15 ++++++++++++++ db/state/execctx/domain_shared.go | 12 ++++++++++- execution/commitment/branch_cache.go | 13 ++++++++++++ execution/commitment/commitment.go | 15 +++++++++++--- .../commitmentdb/commitment_context.go | 9 +++++++-- execution/commitment/hex_patricia_hashed.go | 18 ++++++++++++----- 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 0ce0f36bcd2..93ff05ed6d0 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -359,6 +359,14 @@ func (a *Aggregator) ConfigureDomains() error { } a.configured = true + // Attach the long-lived commitment branch cache to the commitment + // domain. Lifetime = aggregator lifetime; shared by every + // SharedDomains derived from this aggregator. Idempotent across + // repeated ConfigureDomains calls (early return above). + if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { + cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) + } + if a.disableFsync { for _, d := range a.d { if d != nil { @@ -2414,6 +2422,18 @@ func (a *Aggregator) BeginFilesRo() *AggregatorRoTx { return ac } +// BranchCache returns the long-lived commitment branch cache attached to +// this aggregator's commitment domain. Implements +// commitment.BranchCacheProvider so the SharedDomains construction path +// can fetch the cache via a tx.AggTx() type assertion without forcing +// db/state/execctx to import db/state (which would create a cycle). +func (at *AggregatorRoTx) BranchCache() *commitment.BranchCache { + if at.d[kv.CommitmentDomain] == nil { + return nil + } + return at.d[kv.CommitmentDomain].d.branchCache +} + func (at *AggregatorRoTx) Dirs() datadir.Dirs { return at.a.dirs } func (at *AggregatorRoTx) standaloneIIs() []*InvertedIndexRoTx { return at.iis[:at.iisCount] } diff --git a/db/state/domain.go b/db/state/domain.go index a61a1ce444a..d7b52ff7fbe 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -51,6 +51,7 @@ import ( "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/db/version" "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/commitment" ) var ( @@ -88,6 +89,13 @@ type Domain struct { checker *DependencyIntegrityChecker + // branchCache is the long-lived commitment-trie branch cache pinned to + // the aggregator. Set in ConfigureDomains for the commitment domain + // only; nil for every other domain. Lifetime = aggregator lifetime, + // independent of any single tx, batch, or SharedDomains. See + // commitment.BranchCacheProvider for the SDs' access path. + branchCache *commitment.BranchCache + // _testBuildAccessorHook - test-only: called with the recsplit before the build loop in buildHashMapAccessor _testBuildAccessorHook func(rs *recsplit.RecSplit) } @@ -135,6 +143,13 @@ func (d *Domain) SetChecker(checker *DependencyIntegrityChecker) { d.checker = checker } +// BranchCache returns the long-lived commitment-trie branch cache attached +// to this domain. Non-nil only on the commitment domain. Lifetime is the +// owning Aggregator's lifetime. +func (d *Domain) BranchCache() *commitment.BranchCache { + return d.branchCache +} + func (d *Domain) kvNewFilePath(fromStep, toStep kv.Step) string { return filepath.Join(d.dirs.SnapDomain, fmt.Sprintf("%s-%s.%d-%d.kv", d.FileVersion.DataKV.String(), d.FilenameBase, fromStep, toStep)) } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 16868883e4b..71d206f5de6 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -180,7 +180,17 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg } sd.mem = tx.Debug().NewMemBatch(&sd.metrics) - sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp) + + // Fetch the aggregator-scope branch cache (lives on the commitment + // Domain, shared across all SharedDomains derived from this + // aggregator). The duck-typed BranchCacheProvider lookup avoids + // importing db/state directly — db/state already imports execctx, so + // the reverse import would create a cycle. + var branchCache *commitment.BranchCache + if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { + branchCache = p.BranchCache() + } + sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 4851fc7bb5e..6497cfadb7f 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -164,6 +164,19 @@ type branchCacheEntry struct { // at typical mainnet branch sizes. const DefaultBranchCacheTailCapacity = 50000 +// BranchCacheProvider exposes the long-lived BranchCache attached to the +// commitment domain. Implemented by *db/state.AggregatorRoTx (via duck +// typing) so callers in the SharedDomains construction path can fetch the +// cache without forcing db/state/execctx to import db/state — that import +// would create a cycle since db/state imports execctx (squeeze.go, +// trie_reader_integration_test.go, …). +// +// Returning nil is permitted; callers MUST treat nil as "no shared cache, +// behave as if disabled" rather than panic. +type BranchCacheProvider interface { + BranchCache() *BranchCache +} + // NewBranchCache constructs a BranchCache with the given LRU tail capacity. // Capacity <= 0 panics — pass a positive value or DefaultBranchCacheTailCapacity. func NewBranchCache(tailCapacity int) *BranchCache { diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 727fafef409..ddedeb95010 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -156,12 +156,21 @@ const ( VariantConcurrentHexPatricia TrieVariant = "hex-concurrent-patricia-hashed" ) -func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, *Updates) { +// InitializeTrieAndUpdates constructs the trie + updates buffer for a +// SharedDomainsCommitmentContext. The branchCache argument is the +// long-lived (aggregator-scope) cache shared across SDs — pass the cache +// returned by BranchCacheProvider on the AggregatorRoTx. When nil (test +// helpers, isolated benchmarks), a fresh per-init cache is created so the +// trie still has a valid cache to read/write through. +func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string, branchCache *BranchCache) (Trie, *Updates) { + if branchCache == nil { + branchCache = NewBranchCache(DefaultBranchCacheTailCapacity) + } switch tv { case VariantConcurrentHexPatricia: root := NewHexPatriciaHashed(length.Addr, nil) trie := NewConcurrentPatriciaHashed(root, nil) - trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) + trie.SetBranchCache(branchCache) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) // tree.SetConcurrentCommitment(true) // first run always sequential return trie, tree @@ -176,7 +185,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * default: trie := NewHexPatriciaHashed(length.Addr, nil) - trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) + trie.SetBranchCache(branchCache) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) return trie, tree } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 36836bd009b..2c59c686052 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -189,12 +189,17 @@ func (sdc *SharedDomainsCommitmentContext) EnableCsvMetrics(filePathPrefix strin sdc.patriciaTrie.EnableCsvMetrics(filePathPrefix) } -func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant commitment.TrieVariant, tmpDir string) *SharedDomainsCommitmentContext { +// NewSharedDomainsCommitmentContext constructs a per-SharedDomains +// commitment context. branchCache is the aggregator-scope cache shared +// across all SDs derived from the same Aggregator; pass nil from test +// helpers that don't have an aggregator (a fresh per-init cache will be +// created inside InitializeTrieAndUpdates as a fallback). +func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant commitment.TrieVariant, tmpDir string, branchCache *commitment.BranchCache) *SharedDomainsCommitmentContext { ctx := &SharedDomainsCommitmentContext{ sharedDomains: sd, tmpDir: tmpDir, } - ctx.patriciaTrie, ctx.updates = commitment.InitializeTrieAndUpdates(trieVariant, mode, tmpDir) + ctx.patriciaTrie, ctx.updates = commitment.InitializeTrieAndUpdates(trieVariant, mode, tmpDir, branchCache) return ctx } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 1ce1a1452ad..eede1637a5d 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2998,17 +2998,25 @@ func (hph *HexPatriciaHashed) SetLeaveDeferredForCaller(leave bool) { hph.leaveDeferredForCaller = leave } -// Reset allows HexPatriciaHashed instance to be reused for the new commitment calculation +// Reset allows HexPatriciaHashed instance to be reused for the new commitment calculation. +// +// The BranchCache is intentionally NOT cleared here: with aggregator-scope +// lifetime the cache must survive between commitment calculations to deliver +// cross-block hits. Callers that need to invalidate the cache (unwind, fork +// validation) MUST call ClearBranchCache explicitly. func (hph *HexPatriciaHashed) Reset() { hph.root.reset() hph.rootTouched = false hph.rootChecked = false hph.rootPresent = true +} - // Clear the BranchCache only when this is the root trie (not a - // mounted subtrie). Mounted subtries share the root's cache; clearing - // from a mount would dump entries the root still expects to see. - // Carries the invariant established in PR #19954 commit 1612d5608c. +// ClearBranchCache drops every entry in the attached BranchCache. Use on +// unwind or fork-validation paths where the cached branches diverge from +// the canonical store. No-op when no cache is attached or when this is a +// mounted subtrie (the root trie owns the clear, mounts share the same +// cache pointer). +func (hph *HexPatriciaHashed) ClearBranchCache() { if hph.branchCache != nil && !hph.mounted { hph.branchCache.Clear() } From f87937e7389866de7cd4f53dc73afd184ab749c9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 11:11:00 +0000 Subject: [PATCH 006/167] commitment: BranchCache divergence detector + per-block fingerprint log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two debug-gated diagnostics for cross-block-cache-lifetime investigations. Both off by default — meant to be flipped on when a correctness regression surfaces and you need to localise *where in the fold path* the cache started lying, instead of waiting for a downstream trie-root-mismatch many blocks later. 1. BRANCH_CACHE_VERIFY: branchFromCacheOrDB cross-checks every L2 (BranchCache) hit against ctx.Branch and increments a divergence counter when bytes disagree. Logs the prefix and both byte forms so the first divergent read shows up directly in the erigon log. BranchCache.VerifyDivergences() exposes the count for assertion in tests. 2. BRANCH_CACHE_FINGERPRINT: SharedDomainsCommitmentContext.ComputeCommitment emits a "[cache-fp]" log at end of every compute with (block, root, cache fingerprint, divergence count). Two builds running the same workload can be diffed offline ("first block at which their fp's differ") to nail down the first block where lifecycle invariants diverged. Fingerprint is an order-independent FNV-1a fold over (key-hash, data-hash) pairs across both root and tail tiers. Adds maphash.LRU.Range so the Fingerprint can iterate the LRU tail without touching recency (Peek under the hood). The wrapper otherwise discards original byte-keys on insert; mixing by hash instead of key is correctness-equivalent up to hash collision, which is acceptable at the working-set sizes here. Motivation: today we just chased a wrong-root regression to commit d673052f (aggregator-scope cache lifetime). Took two full bench cycles (~12 min) to localise. With the divergence detector running, the first divergent read would surface in the erigon log within seconds. With fingerprint logging in two builds (one good, one regressed), the diverging block boundary would be a single grep across two log files. --- common/maphash/maphash.go | 21 ++++++ execution/commitment/branch_cache.go | 74 ++++++++++++++++++- .../commitmentdb/commitment_context.go | 24 ++++++ execution/commitment/hex_patricia_hashed.go | 21 ++++++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/common/maphash/maphash.go b/common/maphash/maphash.go index 595e644f321..747fb45e1fd 100644 --- a/common/maphash/maphash.go +++ b/common/maphash/maphash.go @@ -163,3 +163,24 @@ func (l *LRU[V]) SetByHash(hash uint64, value V) { func (l *LRU[V]) ContainsByHash(hash uint64) bool { return l.cache.Contains(hash) } + +// Range iterates over every (hash, value) pair without affecting LRU +// recency (uses Peek under the hood). Iteration order is unspecified. +// Return false from fn to stop early. +// +// The original byte-key is not recoverable — Set hashes-and-discards. +// Use the hash itself as the identity in callers that need cross-cache +// comparisons; same byte-key always maps to the same hash, so equality +// of (hash, value) sets is equivalent to equality of (key, value) sets +// modulo collision (vanishingly unlikely at typical working-set sizes). +func (l *LRU[V]) Range(fn func(hash uint64, v V) bool) { + for _, h := range l.cache.Keys() { + v, ok := l.cache.Peek(h) + if !ok { + continue + } + if !fn(h, v) { + return + } + } +} diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 6497cfadb7f..388a852170f 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -137,6 +137,15 @@ type BranchCache struct { rootHits, rootMisses atomic.Uint64 tailHits, tailMisses atomic.Uint64 bytesServed atomic.Uint64 + + // Divergence counter — incremented by RecordDivergence when a caller + // detects that a cache-served value disagrees with the canonical + // store. Driven by branchFromCacheOrDB's verify path (gated by + // BRANCH_CACHE_VERIFY env). Helps localise correctness regressions + // in cross-block-cache investigations: a non-zero count is a + // load-bearing signal that the cache lifecycle is broken before any + // trie root mismatch surfaces downstream. + verifyDivergences atomic.Uint64 } type branchCacheEntry struct { @@ -332,6 +341,59 @@ func (c *BranchCache) Clear() { c.tailHits.Store(0) c.tailMisses.Store(0) c.bytesServed.Store(0) + c.verifyDivergences.Store(0) +} + +// RecordDivergence increments the divergence counter. Called by +// branchFromCacheOrDB's verify path when a cache-served value disagrees +// with a parallel ctx.Branch read. +func (c *BranchCache) RecordDivergence() { + c.verifyDivergences.Add(1) +} + +// Fingerprint returns a deterministic hash of all current entries (root + +// tail). Two caches with the same set of (key, data) pairs produce the +// same fingerprint regardless of insertion order. Use for cross-run +// divergence localisation: emit per-block in two builds, diff the logs to +// see exactly which block their caches first differ. +// +// Mixes (key-hash, data-hash) pairs because the LRU stores by hash and +// discards the original key bytes on insert. Two entries with the same +// original key produce the same hash, so the fingerprint is still +// equality-equivalent to the (key, data) set modulo hash collision. +// +// Cheap: one FNV-1a fold over data per entry. Not cryptographic. +func (c *BranchCache) Fingerprint() uint64 { + const fnvOffset uint64 = 14695981039346656037 + const fnvPrime uint64 = 1099511628211 + dataHash := func(data []byte) uint64 { + h := fnvOffset + for _, b := range data { + h ^= uint64(b) + h *= fnvPrime + } + return h + } + mix := func(keyHash uint64, data []byte) uint64 { + // Combine key and data hashes into one entry hash; xor-fold across + // entries below so the per-cache result is insertion-order + // independent. + return keyHash ^ (dataHash(data) * fnvPrime) + } + var fp uint64 + if e := c.root.Load(); e != nil && len(e.data) > 0 { + // Pinned-root key is the constant 1-byte prefix 0x00; use a + // distinct sentinel hash so the root contribution can't collide + // with a tail entry hashed to zero. + fp ^= mix(0xdeadbeefcafe0001, e.data) + } + c.tail.Range(func(h uint64, e *branchCacheEntry) bool { + if len(e.data) > 0 { + fp ^= mix(h, e.data) + } + return true + }) + return fp } // Stats returns a one-line summary of root-tier and tail-tier hit/miss @@ -349,10 +411,20 @@ func (c *BranchCache) Stats() string { return 100.0 * float64(hit) / float64(total) } return fmt.Sprintf( - "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d", + "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d | divergences=%d", rh, rm, pct(rh, rm), th, tm, pct(th, tm), float64(bb)/1024/1024, c.tail.Len(), + c.verifyDivergences.Load(), ) } + +// VerifyDivergences returns the number of cache-vs-canonical divergences +// recorded since the last Clear. Non-zero indicates a cache lifecycle +// invariant has been violated (a cached entry no longer matches the +// canonical store) — read from outside to assert correctness in tests +// and benches. +func (c *BranchCache) VerifyDivergences() uint64 { + return c.verifyDivergences.Load() +} diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 2c59c686052..7d0cd5646a4 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -31,6 +31,13 @@ import ( var ( mxCommitmentRunning = metrics.GetOrCreateGauge("domain_running_commitment") mxCommitmentTook = metrics.GetOrCreateSummary("domain_commitment_took") + + // logCacheFingerprint, when true, makes ComputeCommitment emit a + // "[cache-fp]" log line at end-of-block with (block, root, fp, + // divergences). Diff two builds' logs offline to localise the first + // block at which their BranchCache states diverge. Off by default — + // gate via env BRANCH_CACHE_FINGERPRINT=true. + logCacheFingerprint = dbg.EnvBool("BRANCH_CACHE_FINGERPRINT", false) ) type sd interface { @@ -323,6 +330,23 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context keysPerSec = uint64(float64(updateCount) / took.Seconds()) } log.Debug("[commitment] processed", "block", blockNum, "txNum", txNum, "keys", common.PrettyCounter(updateCount), "keys/s", common.PrettyCounter(keysPerSec), "mode", sdc.updates.Mode(), "spent", took, "rootHash", hex.EncodeToString(rootHash)) + + // Per-block BranchCache fingerprint — gated by BRANCH_CACHE_FINGERPRINT + // env. Emit a single log line of (block, root, cache_fp, divergences) + // so two builds running the same workload can be diffed offline to + // localise the first block at which their caches diverge. See + // commitment.BranchCache.Fingerprint for the hash semantics. + if logCacheFingerprint { + if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { + if bc := hph.BranchCache(); bc != nil { + log.Info("[commitment][cache-fp]", + "block", blockNum, + "root", hex.EncodeToString(rootHash), + "fp", fmt.Sprintf("%016x", bc.Fingerprint()), + "divergences", bc.VerifyDivergences()) + } + } + } }() if updateCount == 0 { rootHash, err = sdc.patriciaTrie.RootHash() diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index eede1637a5d..191379fc3e5 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -3031,6 +3031,15 @@ func (hph *HexPatriciaHashed) Cache() *WarmupCache { return hph.cache } +// verifyBranchCache, when true, makes branchFromCacheOrDB cross-check +// every BranchCache hit against ctx.Branch and record a divergence on +// the cache when the bytes disagree. Off by default — gate via env +// BRANCH_CACHE_VERIFY=true. Use during cross-block-cache-lifetime +// investigations to localise a bad cached entry to the read site +// instead of waiting for the downstream wrong-trie-root to surface +// many blocks later. +var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) + // branchFromCacheOrDB reads branch data via the cache hierarchy: // - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) // - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped @@ -3059,6 +3068,18 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { } if hph.branchCache != nil { if data, found := hph.branchCache.Get(key); found { + if verifyBranchCache { + canonical, _, err := hph.ctx.Branch(key) + if err == nil && !bytes.Equal(data, canonical) { + hph.branchCache.RecordDivergence() + log.Warn("[branch-cache] divergence", + "prefix", hex.EncodeToString(key), + "cached_len", len(data), + "canonical_len", len(canonical), + "cached", hex.EncodeToString(data), + "canonical", hex.EncodeToString(canonical)) + } + } return data, nil } } From b61655cadec16ea2eff1b85ada0dcba4bb0b86eb Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 17:42:45 +0000 Subject: [PATCH 007/167] commitment: disable deferred-encoding mode (use inline CollectUpdate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deferred-encoding path (CollectDeferredUpdate + ApplyDeferredUpdates) parallelises EncodeBranch + merge work at apply time. The cache correctness consequence: between collect and apply, sd.mem holds the old state while the cache is also unchanged. When apply finally fires (end of Process or duplicate-prefix flush mid-Process), sd.mem advances but the cache isn't touched — CollectDeferredUpdate doesn't have a cache hook (and wiring one breaks TestSharedDomain_RepeatedUnwindAcrossStepBoundary + TestCustomTraceReceiptDomain because it violates an implicit trie-during-Process cache stability invariant). The result on the FV bench: cache stays at the very-first-Process's read-side L3 fallback bytes for prefixes the trie writes, while ctx.Branch advances to each block's actual state. Divergence on every hot prefix (root, root-zone children) starting from block 2. Use CollectUpdate (inline) instead. CollectUpdate writes sd.mem + WarmupCache + BranchCache atomically at fold time via PutBranch — cache mirrors sd.mem at every write, the trie sees its own writes consistently, and the cross-Process cache state matches what post-FCU MDBX commit produced. Loses the encoder's parallel-encoding optimisation, but bench profile is I/O-bound, not encode-CPU-bound, so the trade is favourable. History (ETL) writes are still inline via DomainPut. Splitting that ("sd.mem inline, history queued for flush at FCU") is the proper architectural answer to defer the slow disk work without touching the sd.mem invariant — tracked as a follow-up. --- execution/commitment/hex_patricia_hashed.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 191379fc3e5..51d558bb6af 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -164,7 +164,14 @@ func newHexPatriciaHashed() *HexPatriciaHashed { } hph.branchEncoder.setMetrics(hph.metrics) - hph.branchEncoder.SetDeferUpdates(true) // Enable deferred branch updates by default + // Inline-mode write path: CollectUpdate writes sd.mem + WarmupCache + + // BranchCache atomically at fold-time via PutBranch, preserving the + // trie's read-your-own-writes invariant and keeping the cache aligned + // with sd.mem at every write. Deferred-mode (parallel encoding at + // apply time) breaks this — the encoder writes to sd.mem at apply + // time while reads still see pre-apply state via cache, producing + // intra-Process divergence. + hph.branchEncoder.SetDeferUpdates(false) return hph } From 7e21628c290874980dc9d009beadb25cb5df6aac Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 18:36:24 +0000 Subject: [PATCH 008/167] commitment: add DISABLE_BRANCH_CACHE_READS env kill-switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bisection helper: force branchFromCacheOrDB to skip the L2 BranchCache read path entirely so every read goes via ctx.Branch (sd.mem → MDBX). Cache writes still fire so verify-mode can keep comparing cache vs canonical. Flipping the env at runtime distinguishes "cache holds bad data" (bench passes further with cache reads off) from "deeper compute bug" (same failure regardless). Used in the 2026-05-06 investigation to confirm the cache was actively corrupting block 13's compute on the canonical SSTORE-bloat bench: with cache reads on, wrong-trie-root at block 13. With reads off, blocks 13-16 produce the correct roots and the bench advances to a separate failure at block 17 (unrelated pre-existing bug). Default off; gate via env DISABLE_BRANCH_CACHE_READS=true. --- execution/commitment/hex_patricia_hashed.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 51d558bb6af..cfbfc35df5c 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -3047,6 +3047,16 @@ func (hph *HexPatriciaHashed) Cache() *WarmupCache { // many blocks later. var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) +// disableBranchCacheReads forces branchFromCacheOrDB to skip the L2 +// BranchCache read path — every read goes to ctx.Branch (sd.mem → +// MDBX). Cache writes (CollectUpdate.Put, branchFromCacheOrDB.Put on +// L3 fallback) still fire so verify-mode can keep comparing cache vs +// canonical. Use to A/B test correctness with-cache vs without-cache +// without recompiling. Confirmed in 2026-05-06 investigation that +// flipping this distinguishes "cache holds bad data" (bench passes +// further) from "deeper compute bug" (bench fails at the same block). +var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) + // branchFromCacheOrDB reads branch data via the cache hierarchy: // - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) // - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped @@ -3073,7 +3083,7 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { hph.metrics.missBranch.Add(1) } } - if hph.branchCache != nil { + if hph.branchCache != nil && !disableBranchCacheReads { if data, found := hph.branchCache.Get(key); found { if verifyBranchCache { canonical, _, err := hph.ctx.Branch(key) From 8d6d19a3aff9162b65d70c70705e9dd51af9d445 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 08:38:07 +0000 Subject: [PATCH 009/167] commitment: multi-layer state probe at BranchCache divergence site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When verifyBranchCache=true and a cache hit disagrees with ctx.Branch, sample sd.mem, sd.parent.mem, and tx-direct (MDBX) for the same prefix and dump all layers in the divergence log line. Comparing those four byte sequences against the cached and canonical bytes pinpoints which state layer holds the bytes the cache disagrees with — the rewriter we need to identify before fixing the canonical-store-divergence bugs the cache currently exposes. Decision matrix (read off the log line): - cache != tx, sd.mem == cache → in-memory writer is fresh, MDBX is stale (commit-timing issue). - cache != tx, tx == ctx.Branch, sd.mem != cache, parent.mem != cache → MDBX has been rewritten by something outside the CollectUpdate write path (collation, file build, squeeze). - cache != ctx.Branch, parent.mem matches ctx.Branch but sd.mem doesn't → parent merge is the source. - cache != ctx.Branch, all of sd.mem / parent.mem / tx == ctx.Branch → cache itself was populated incorrectly (write-side bug). Three changes: 1. SharedDomains.ProbeReadLayers (db/state/execctx/domain_shared.go): public method that samples sd.mem, sd.parent.mem (private field accessed from the same package), and tx.GetLatest. Read-only; copies bytes so callers can hold them past tx lifetime. 2. TrieContext (execution/commitment/commitmentdb/commitment_context.go): add probeSd + probeTx fields populated at trieContext() construction; expose ProbeStateLayers method that delegates to sd.ProbeReadLayers. The local `sd` interface gets the ProbeReadLayers method too so the duck-typed reference can call it without an import cycle to execctx. 3. branchFromCacheOrDB log site (execution/commitment/hex_patricia_hashed.go): on divergence with verifyBranchCache, type-assert ctx for the probe interface and append sd_mem / parent_mem / mdbx fields to the log line. Existing field shape preserved for log parsers; new fields are additive. Pre-existing test failures (TestSharedDomain_RepeatedUnwindAcrossStepBoundary, TestValidateChainAndUpdateForkChoiceWithSideForksThatGoBackAndForwardInHeight) are unchanged — they were failing on the stack before this probe landed and are part of what the divergence work is meant to localise. --- db/state/execctx/domain_shared.go | 25 +++++++++++++++++ .../commitmentdb/commitment_context.go | 28 +++++++++++++++++++ execution/commitment/hex_patricia_hashed.go | 28 +++++++++++++++++-- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 71d206f5de6..9469c447667 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -580,6 +580,31 @@ func (sd *SharedDomains) GetStateCache() *cache.StateCache { return sd.stateCache } +// ProbeReadLayers samples each independent state layer (this SD's mem, +// the parent SD's mem, and direct MDBX via tx.GetLatest) and returns +// the bytes from each. Read-only, intended for divergence-detection +// diagnostics — pinpoints which layer holds bytes that disagree with +// the BranchCache when verifyBranchCache is on. Bytes are copied so +// the caller can hold them past tx lifetime. +func (sd *SharedDomains) ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { + if v, _, ok := sd.mem.GetLatest(domain, key); ok { + memOk = true + mem = append([]byte(nil), v...) + } + if sd.parent != nil { + if v, _, ok := sd.parent.mem.GetLatest(domain, key); ok { + parentOk = true + parentMem = append([]byte(nil), v...) + } + } + if tx != nil { + if v, _, err := tx.GetLatest(domain, key); err == nil { + mdbx = append([]byte(nil), v...) + } + } + return +} + func (sd *SharedDomains) ClearRam(resetCommitment bool) { // When the commitment calculator goroutine owns the Updates buffer, // skip ClearRam on the commitment context to avoid concurrent btree access. diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 7d0cd5646a4..94769f54bce 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -47,6 +47,12 @@ type sd interface { StepSize() uint64 Trace() bool CommitmentCapture() bool + + // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) + // for one key. Used by the BranchCache divergence-detection probe + // to localise which state layer holds bytes that diverge from + // cache. Read-only. + ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) } type SharedDomainsCommitmentContext struct { @@ -217,6 +223,8 @@ func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNu stepSize: sdc.sharedDomains.StepSize(), txNum: txNum, blockNum: blockNum, + probeSd: sdc.sharedDomains, + probeTx: tx, } if sdc.stateReader != nil { mainTtx.stateReader = sdc.stateReader.Clone(tx) @@ -817,6 +825,14 @@ type TrieContext struct { trace bool stateReader StateReader localCollector *etl.Collector // per-goroutine collector for concurrent PutBranch + + // probeSd / probeTx feed ProbeStateLayers — diagnostics-only fields + // populated when this TrieContext is constructed from a + // SharedDomainsCommitmentContext that has both a SharedDomains + // and a tx in scope. Both nil for read-only / test contexts where + // the probe is not available. + probeSd sd + probeTx kv.TemporalTx } // NewTrieContextRo creates a read-only TrieContext suitable for TrieReader lookups. @@ -838,6 +854,18 @@ func (sdc *TrieContext) Branch(pref []byte) ([]byte, kv.Step, error) { return common.Copy(enc), step, nil } +// ProbeStateLayers samples the underlying state layers for one key — +// sd.mem, sd.parent.mem (if any), and tx-direct (MDBX) — for +// divergence-detection diagnostics. Returns empty / not-ok when +// constructed without a probe-capable SharedDomains (e.g. +// NewTrieContextRo). Read-only. +func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { + if sdc.probeSd == nil { + return + } + return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) +} + func (sdc *TrieContext) PutBranch(prefix []byte, data []byte, prevData []byte) error { if sdc.stateReader.WithHistory() { // do not store branches if explicitly operate on history return nil diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index cfbfc35df5c..c8766e7cf6c 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -40,6 +40,7 @@ import ( "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/stateifs" "github.com/erigontech/erigon/execution/commitment/nibbles" "github.com/erigontech/erigon/execution/commitment/trie" @@ -3089,12 +3090,35 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { canonical, _, err := hph.ctx.Branch(key) if err == nil && !bytes.Equal(data, canonical) { hph.branchCache.RecordDivergence() - log.Warn("[branch-cache] divergence", + fields := []any{ "prefix", hex.EncodeToString(key), "cached_len", len(data), "canonical_len", len(canonical), "cached", hex.EncodeToString(data), - "canonical", hex.EncodeToString(canonical)) + "canonical", hex.EncodeToString(canonical), + } + // State-layer probe: sample sd.mem, parent.mem, and + // tx-direct (MDBX) for the same key so the log line + // shows which layer holds bytes matching the cache + // vs ctx.Branch. Decision matrix in + // agentspecs/branch-cache-divergence-probe.md. + if probe, ok := hph.ctx.(interface { + ProbeStateLayers(kv.Domain, []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) + }); ok { + mem, parentMem, mdbx, memOk, parentOk := probe.ProbeStateLayers(kv.CommitmentDomain, key) + if memOk { + fields = append(fields, "sd_mem", hex.EncodeToString(mem)) + } else { + fields = append(fields, "sd_mem", "miss") + } + if parentOk { + fields = append(fields, "parent_mem", hex.EncodeToString(parentMem)) + } else { + fields = append(fields, "parent_mem", "miss-or-no-parent") + } + fields = append(fields, "mdbx", hex.EncodeToString(mdbx)) + } + log.Warn("[branch-cache] divergence", fields...) } } return data, nil From 87f12be64407eb223c33b3f32b4a4a7ab116c1b8 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 10:59:17 +0000 Subject: [PATCH 010/167] commitment: tag BranchCache writes with origin + write-seq + timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-write provenance for divergence-detection diagnostics. When a divergence fires (cache hit disagrees with ctx.Branch), we now log which write site produced the cached bytes and when, so we can correlate the bad write against the FCU / build / step timeline. Tag fields added to branchCacheEntry: - origin short label of the write site (e.g. "CollectUpdate", "L3-fallback-read") - writeSeq monotonic counter per BranchCache instance - writeTimeNanos unix nanos at write time Put / PutIfClean signatures take an origin string. Two writers updated: - BranchEncoder.CollectUpdate → "CollectUpdate" - branchFromCacheOrDB L3-fallback Put → "L3-fallback-read" GetWithOrigin returns bytes plus the metadata; uses a non-counting peek so it can be called alongside Get without double-counting hits. The divergence-detection log site at branchFromCacheOrDB now appends cache_origin / cache_seq / cache_t_ns fields. Combine with the existing sd_mem / parent_mem / mdbx fields to localise both who wrote the stale bytes and which layer the canonical value lives in. Pre-existing test failures (TestValidateChainAndUpdateForkChoiceWithSideForksThatGoBackAndForwardInHeight) are unchanged from previous commits. --- execution/commitment/branch_cache.go | 54 +++++++++++++++++++-- execution/commitment/branch_cache_test.go | 30 ++++++------ execution/commitment/commitment.go | 2 +- execution/commitment/hex_patricia_hashed.go | 14 +++++- 4 files changed, 78 insertions(+), 22 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 388a852170f..4adf87b0609 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -20,6 +20,7 @@ import ( "fmt" "sync" "sync/atomic" + "time" "github.com/erigontech/erigon/common/maphash" ) @@ -146,6 +147,12 @@ type BranchCache struct { // load-bearing signal that the cache lifecycle is broken before any // trie root mismatch surfaces downstream. verifyDivergences atomic.Uint64 + + // writeSeq is incremented on every Put so each entry carries a + // monotonic ordering tag — divergence-detection uses this with the + // origin label and timestamp to identify which write produced the + // stale bytes. + writeSeq atomic.Uint64 } type branchCacheEntry struct { @@ -166,6 +173,15 @@ type branchCacheEntry struct { // entry was populated; treat as stale until cleared." Same semantics // as branchEntry.dirty in WarmupCache (carried from step 4). dirty atomic.Bool + + // origin diagnostics — captured at Put time so divergence-detection + // can identify which write produced the (now-disagreeing) bytes. + // origin is a short label of the write site (e.g. "CollectUpdate", + // "L3-fallback-read"); writeSeq is a monotonic counter per + // BranchCache instance; writeTimeNanos is unix-nanos at write time. + origin string + writeSeq uint64 + writeTimeNanos int64 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -284,11 +300,17 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // Put stores branch data in the cache, replacing any existing entry // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of -// caller buffer lifetime. -func (c *BranchCache) Put(prefix []byte, data []byte) { +// caller buffer lifetime. origin is a short label of the write site +// captured for divergence-detection diagnostics. +func (c *BranchCache) Put(prefix []byte, data []byte, origin string) { dataCopy := make([]byte, len(data)) copy(dataCopy, data) - c.store(prefix, &branchCacheEntry{data: dataCopy}) + c.store(prefix, &branchCacheEntry{ + data: dataCopy, + origin: origin, + writeSeq: c.writeSeq.Add(1), + writeTimeNanos: time.Now().UnixNano(), + }) } // PutIfClean stores branch data only if no existing entry is marked dirty. @@ -298,14 +320,36 @@ func (c *BranchCache) Put(prefix []byte, data []byte) { // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data) + c.Put(prefix, data, origin) return true } +// GetWithOrigin returns the cached bytes plus the diagnostic origin +// metadata captured when the entry was put. ok=false on miss. Used by +// the divergence-detection probe to identify which write produced the +// stale bytes. Does not bump hit/miss counters or affect LRU recency +// (uses a non-counting peek so it can be called alongside Get without +// double-counting). +func (c *BranchCache) GetWithOrigin(prefix []byte) (data []byte, origin string, writeSeq uint64, writeTimeNanos int64, ok bool) { + var entry *branchCacheEntry + if isRootPrefix(prefix) { + entry = c.root.Load() + } else { + entry, ok = c.tail.Get(prefix) + if !ok { + return nil, "", 0, 0, false + } + } + if entry == nil { + return nil, "", 0, 0, false + } + return entry.data, entry.origin, entry.writeSeq, entry.writeTimeNanos, true +} + // MarkDirty flags the entry at prefix as stale-until-cleared. Subsequent // PutIfClean calls for this prefix will skip; reads still return the // entry (the dirty signal is consumed only on the write path today, same diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 5338a090701..3340e5cb292 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -31,8 +31,8 @@ func TestBranchCache_RootPinning(t *testing.T) { rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch deepKey := []byte{0x12, 0x34, 0x56} - c.Put(rootKey, []byte("root-data")) - c.Put(deepKey, []byte("deep-data")) + c.Put(rootKey, []byte("root-data"), "test") + c.Put(deepKey, []byte("deep-data"), "test") // Root reads should increment rootHits, not tailHits got, ok := c.Get(rootKey) @@ -55,11 +55,11 @@ func TestBranchCache_RootPinning(t *testing.T) { func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { c := NewBranchCache(10) // very small tail rootKey := []byte{0x00} - c.Put(rootKey, []byte("ROOT-PERSISTS")) + c.Put(rootKey, []byte("ROOT-PERSISTS"), "test") // Stuff the tail well past capacity for i := 0; i < 100; i++ { - c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}) + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, "test") } // Root must still be there @@ -78,20 +78,20 @@ func TestBranchCache_DirtyFlag(t *testing.T) { c := NewBranchCache(100) key := []byte{0x12, 0x34} - require.True(t, c.PutIfClean(key, []byte("v1"))) + require.True(t, c.PutIfClean(key, []byte("v1"), "test")) c.MarkDirty(key) - require.False(t, c.PutIfClean(key, []byte("v2")), "PutIfClean must refuse dirty entry") + require.False(t, c.PutIfClean(key, []byte("v2"), "test"), "PutIfClean must refuse dirty entry") got, _ := c.Get(key) require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") // Unconditional Put replaces - c.Put(key, []byte("v3")) + c.Put(key, []byte("v3"), "test") got, _ = c.Get(key) require.Equal(t, []byte("v3"), got) // New entry is clean - require.True(t, c.PutIfClean(key, []byte("v4"))) + require.True(t, c.PutIfClean(key, []byte("v4"), "test")) } // TestBranchCache_GetDecoded verifies the lazy-decode read path for a @@ -106,7 +106,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.NoError(t, err) prefix := []byte{0x12, 0x34} - c.Put(prefix, enc) + c.Put(prefix, enc, "test") // First decoded-read decodes lazily bitmap, cells, ok := c.GetDecoded(prefix) @@ -130,8 +130,8 @@ func TestBranchCache_Invalidate(t *testing.T) { c := NewBranchCache(100) rootKey := []byte{0x00} deepKey := []byte{0x12, 0x34} - c.Put(rootKey, []byte("r")) - c.Put(deepKey, []byte("d")) + c.Put(rootKey, []byte("r"), "test") + c.Put(deepKey, []byte("d"), "test") c.Invalidate(rootKey) _, ok := c.Get(rootKey) @@ -145,8 +145,8 @@ func TestBranchCache_Invalidate(t *testing.T) { // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("r")) - c.Put([]byte{0x12}, []byte("d")) + c.Put([]byte{0x00}, []byte("r"), "test") + c.Put([]byte{0x12}, []byte("d"), "test") c.Get([]byte{0x00}) c.Get([]byte{0x12}) @@ -166,8 +166,8 @@ func TestBranchCache_Clear(t *testing.T) { // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("rrr")) - c.Put([]byte{0x12, 0x34}, []byte("ddd")) + c.Put([]byte{0x00}, []byte("rrr"), "test") + c.Put([]byte{0x12, 0x34}, []byte("ddd"), "test") c.Get([]byte{0x00}) c.Get([]byte{0x12, 0x34}) c.Get([]byte{0xff}) // tail miss diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index ddedeb95010..8a4cce06390 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -641,7 +641,7 @@ func (be *BranchEncoder) CollectUpdate( // (the new entry is born clean). Single writer per prefix per fold // invariant means no concurrent Put races on this key. if be.branchCache != nil { - be.branchCache.Put(prefixCopy, updateCopy) + be.branchCache.Put(prefixCopy, updateCopy, "CollectUpdate") } if be.metrics != nil { be.metrics.updateBranch.Add(1) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index c8766e7cf6c..71ed6dc909c 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -3097,6 +3097,18 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { "cached", hex.EncodeToString(data), "canonical", hex.EncodeToString(canonical), } + // Origin metadata captured at cache.Put time — + // identifies which write site produced the stale + // bytes (CollectUpdate vs L3-fallback-read), with + // a monotonic write-seq + unix-nanos timestamp so + // we can correlate against the timeline of FCU / + // build / step events. + if _, origin, seq, tns, ok := hph.branchCache.GetWithOrigin(key); ok { + fields = append(fields, + "cache_origin", origin, + "cache_seq", seq, + "cache_t_ns", tns) + } // State-layer probe: sample sd.mem, parent.mem, and // tx-direct (MDBX) for the same key so the log line // shows which layer holds bytes matching the cache @@ -3129,7 +3141,7 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { return nil, err } if hph.branchCache != nil && len(data) > 0 { - hph.branchCache.Put(key, data) + hph.branchCache.Put(key, data, "L3-fallback-read") } return data, nil } From 6989427ad99b838de488c7e3960f734a5323c009 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 12:14:02 +0000 Subject: [PATCH 011/167] commitment, db/state/execctx: move BranchCache behind sd.mem chain BranchCache previously sat in front of the sd.mem -> parent.mem -> MDBX read chain (consulted in branchFromCacheOrDB before ctx.Branch). The shared aggregator-scope cache was written from CollectUpdate by every SD running commitment compute, including fork-validator SDs whose writes never reach MDBX. Origin-tagged probe (run-step7b-probe-sdid) showed five distinct SD pointers writing the same prefix to a single cache entry, so any reader whose lineage didn't match the most-recent writer saw bytes that disagreed with MDBX -> wrong trie root from block 13 onward in the canonical SSTORE-bloat fork bench. Layering after this change: Read: sd.mem -> sd.parent.mem -> branchCache -> aggTx (MDBX) Write: sd.mem only (DomainPut path) Flush: sd.mem -> MDBX, then branchCache.Clear() The cache now mirrors MDBX-flushed bytes only. Writers' in-flight bytes live in sd.mem above; cache hits below sd.mem are always equivalent to reading MDBX, so cross-SD pollution is impossible by construction. Cache fills lazily on the MDBX-read path inside sd.GetLatest, and clear-on-flush prevents pre-flush bytes from coexisting with new MDBX state. Per-key invalidation is a follow-up (PR2). BranchCache entries gain a step field so Get returns (data, step, ok) matching the aggTx contract. Without this, sd.GetLatest's cache hit returned step=0 and CheckDataAvailable rejected the boot SeekCommitment with "commitment state out of date". Removed: - cache.Put from CollectUpdate (commitment.go) - cache.Put + divergence detection from branchFromCacheOrDB (hex_patricia_hashed.go); now just calls ctx.Branch - L3-fallback Put (cache fills via sd.GetLatest now) Validated on canonical cold bench (run-step8b): first FCU VALID, all payloads through end VALID, 0 cache divergences, 0 wrong-root errors. Prior probe bench had 23 divergences and INVALID payloads from the fail block onward. Probe scaffolding (SiteIdentity, ProbeStateLayers, divergence counters) left in place for now; can be stripped in a cleanup follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 35 ++++++++- execution/commitment/branch_cache.go | 26 +++++-- execution/commitment/branch_cache_test.go | 60 +++++++-------- execution/commitment/commitment.go | 6 +- .../commitmentdb/commitment_context.go | 8 ++ execution/commitment/hex_patricia_hashed.go | 74 ++----------------- 6 files changed, 99 insertions(+), 110 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 9469c447667..7eb25fe20f1 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -158,6 +158,14 @@ type SharedDomains struct { // (now tx-granular) at sd.Flush time, and delete this Mutex + the // SetChangesetAccumulator/GetChangesetAccumulator API entirely. changesetMu sync.Mutex + + // branchCache is the aggregator-scope commitment-branch cache. It sits + // behind sd.mem and sd.parent.mem in the read chain (consulted only after + // both miss, before the aggTx files/MDBX read), so writers' in-flight + // bytes always mask the cache and cross-SD pollution is impossible. + // May be nil for test setups whose AggTx doesn't implement + // commitment.BranchCacheProvider. + branchCache *commitment.BranchCache } func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger) (*SharedDomains, error) { @@ -190,6 +198,7 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { branchCache = p.BranchCache() } + sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) _, blockNum, err := sd.SeekCommitment(ctx, tx) @@ -695,7 +704,17 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - return sd.mem.Flush(ctx, tx) + if err := sd.mem.Flush(ctx, tx); err != nil { + return err + } + // Cache mirrors MDBX-flushed bytes; clear after flush so cache cannot + // hold pre-flush bytes for keys whose MDBX value just changed. Refills + // lazily on next read. PR2 will switch to per-key invalidation using + // the just-flushed CommitmentDomain diff. + if sd.branchCache != nil { + sd.branchCache.Clear() + } + return nil } // TemporalDomain satisfaction @@ -764,6 +783,17 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } + // branchCache sits between sd.mem/parent.mem and the aggTx files for + // CommitmentDomain only. It mirrors MDBX-flushed bytes; writers' fresh + // bytes are held in sd.mem above, so a cache hit here is always + // equivalent to reading from MDBX. Cleared by sd.Flush so a new MDBX + // state never coexists with stale cache entries. + if domain == kv.CommitmentDomain && sd.branchCache != nil { + if cv, cstep, ok := sd.branchCache.Get(k); ok { + return cv, kv.Step(cstep), nil + } + } + if aggTx, ok := tx.AggTx().(MeteredGetter); ok { v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) } else { @@ -777,6 +807,9 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) if sd.stateCache != nil { sd.stateCache.Put(domain, k, v) } + if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { + sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") + } return v, step, nil } diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 4adf87b0609..a24f1193172 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -182,6 +182,13 @@ type branchCacheEntry struct { origin string writeSeq uint64 writeTimeNanos int64 + + // step is the on-disk file step the cached bytes came from. Returned + // by Get so callers (e.g. CheckDataAvailable) can validate against + // the latest visible step. 0 means "step not tracked" — fine for + // in-memory tests but real callers should always pass the step + // returned by aggTx.MeteredGetLatest / tx.GetLatest. + step uint64 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -251,14 +258,15 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { } // Get retrieves branch data from the cache. Returns the canonical encoded -// bytes (with the leading 2-byte touch-map prefix). -func (c *BranchCache) Get(prefix []byte) ([]byte, bool) { +// bytes (with the leading 2-byte touch-map prefix) plus the on-disk file +// step the bytes came from (0 if not tracked). +func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { entry, ok := c.lookup(prefix) if !ok { - return nil, false + return nil, 0, false } c.bytesServed.Add(uint64(len(entry.data))) - return entry.data, true + return entry.data, entry.step, true } // GetDecoded retrieves the cached branch in decoded form. Lazy-decodes on @@ -300,13 +308,15 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // Put stores branch data in the cache, replacing any existing entry // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of -// caller buffer lifetime. origin is a short label of the write site +// caller buffer lifetime. step is the on-disk file step the bytes came +// from (0 if not tracked); origin is a short label of the write site // captured for divergence-detection diagnostics. -func (c *BranchCache) Put(prefix []byte, data []byte, origin string) { +func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { dataCopy := make([]byte, len(data)) copy(dataCopy, data) c.store(prefix, &branchCacheEntry{ data: dataCopy, + step: step, origin: origin, writeSeq: c.writeSeq.Add(1), writeTimeNanos: time.Now().UnixNano(), @@ -320,11 +330,11 @@ func (c *BranchCache) Put(prefix []byte, data []byte, origin string) { // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte, origin string) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step uint64, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data, origin) + c.Put(prefix, data, step, origin) return true } diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 3340e5cb292..f545a23a650 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -31,18 +31,18 @@ func TestBranchCache_RootPinning(t *testing.T) { rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch deepKey := []byte{0x12, 0x34, 0x56} - c.Put(rootKey, []byte("root-data"), "test") - c.Put(deepKey, []byte("deep-data"), "test") + c.Put(rootKey, []byte("root-data"), 0, "test") + c.Put(deepKey, []byte("deep-data"), 0, "test") // Root reads should increment rootHits, not tailHits - got, ok := c.Get(rootKey) + got, _, ok := c.Get(rootKey) require.True(t, ok) require.Equal(t, []byte("root-data"), got) require.Equal(t, uint64(1), c.rootHits.Load()) require.Equal(t, uint64(0), c.tailHits.Load()) // Deep reads should increment tailHits, not rootHits - got, ok = c.Get(deepKey) + got, _, ok = c.Get(deepKey) require.True(t, ok) require.Equal(t, []byte("deep-data"), got) require.Equal(t, uint64(1), c.rootHits.Load()) @@ -55,15 +55,15 @@ func TestBranchCache_RootPinning(t *testing.T) { func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { c := NewBranchCache(10) // very small tail rootKey := []byte{0x00} - c.Put(rootKey, []byte("ROOT-PERSISTS"), "test") + c.Put(rootKey, []byte("ROOT-PERSISTS"), 0, "test") // Stuff the tail well past capacity for i := 0; i < 100; i++ { - c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, "test") + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, 0, "test") } // Root must still be there - got, ok := c.Get(rootKey) + got, _, ok := c.Get(rootKey) require.True(t, ok, "root should never be evicted from pinned slot") require.Equal(t, []byte("ROOT-PERSISTS"), got) @@ -78,20 +78,20 @@ func TestBranchCache_DirtyFlag(t *testing.T) { c := NewBranchCache(100) key := []byte{0x12, 0x34} - require.True(t, c.PutIfClean(key, []byte("v1"), "test")) + require.True(t, c.PutIfClean(key, []byte("v1"), 0, "test")) c.MarkDirty(key) - require.False(t, c.PutIfClean(key, []byte("v2"), "test"), "PutIfClean must refuse dirty entry") - got, _ := c.Get(key) + require.False(t, c.PutIfClean(key, []byte("v2"), 0, "test"), "PutIfClean must refuse dirty entry") + got, _, _ := c.Get(key) require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") // Unconditional Put replaces - c.Put(key, []byte("v3"), "test") - got, _ = c.Get(key) + c.Put(key, []byte("v3"), 0, "test") + got, _, _ = c.Get(key) require.Equal(t, []byte("v3"), got) // New entry is clean - require.True(t, c.PutIfClean(key, []byte("v4"), "test")) + require.True(t, c.PutIfClean(key, []byte("v4"), 0, "test")) } // TestBranchCache_GetDecoded verifies the lazy-decode read path for a @@ -106,7 +106,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.NoError(t, err) prefix := []byte{0x12, 0x34} - c.Put(prefix, enc, "test") + c.Put(prefix, enc, 0, "test") // First decoded-read decodes lazily bitmap, cells, ok := c.GetDecoded(prefix) @@ -120,7 +120,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.Same(t, cells, cells2, "decoded form cached and reused") // Encoded form unchanged after decoded reads - encGot, ok := c.Get(prefix) + encGot, _, ok := c.Get(prefix) require.True(t, ok) require.Equal(t, []byte(enc), encGot) } @@ -130,25 +130,25 @@ func TestBranchCache_Invalidate(t *testing.T) { c := NewBranchCache(100) rootKey := []byte{0x00} deepKey := []byte{0x12, 0x34} - c.Put(rootKey, []byte("r"), "test") - c.Put(deepKey, []byte("d"), "test") + c.Put(rootKey, []byte("r"), 0, "test") + c.Put(deepKey, []byte("d"), 0, "test") c.Invalidate(rootKey) - _, ok := c.Get(rootKey) + _, _, ok := c.Get(rootKey) require.False(t, ok, "root invalidated") c.Invalidate(deepKey) - _, ok = c.Get(deepKey) + _, _, ok = c.Get(deepKey) require.False(t, ok, "deep invalidated") } // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("r"), "test") - c.Put([]byte{0x12}, []byte("d"), "test") - c.Get([]byte{0x00}) - c.Get([]byte{0x12}) + c.Put([]byte{0x00}, []byte("r"), 0, "test") + c.Put([]byte{0x12}, []byte("d"), 0, "test") + _, _, _ = c.Get([]byte{0x00}) + _, _, _ = c.Get([]byte{0x12}) require.Equal(t, uint64(1), c.rootHits.Load()) require.Equal(t, uint64(1), c.tailHits.Load()) @@ -156,9 +156,9 @@ func TestBranchCache_Clear(t *testing.T) { c.Clear() require.Equal(t, uint64(0), c.rootHits.Load()) require.Equal(t, uint64(0), c.tailHits.Load()) - _, ok := c.Get([]byte{0x00}) + _, _, ok := c.Get([]byte{0x00}) require.False(t, ok) - _, ok = c.Get([]byte{0x12}) + _, _, ok = c.Get([]byte{0x12}) require.False(t, ok) } @@ -166,11 +166,11 @@ func TestBranchCache_Clear(t *testing.T) { // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("rrr"), "test") - c.Put([]byte{0x12, 0x34}, []byte("ddd"), "test") - c.Get([]byte{0x00}) - c.Get([]byte{0x12, 0x34}) - c.Get([]byte{0xff}) // tail miss + c.Put([]byte{0x00}, []byte("rrr"), 0, "test") + c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, "test") + _, _, _ = c.Get([]byte{0x00}) + _, _, _ = c.Get([]byte{0x12, 0x34}) + _, _, _ = c.Get([]byte{0xff}) // tail miss s := c.Stats() for _, want := range []string{ diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 8a4cce06390..5fb10080079 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -640,9 +640,9 @@ func (be *BranchEncoder) CollectUpdate( // fresh canonical bytes — clears the dirty flag in the process // (the new entry is born clean). Single writer per prefix per fold // invariant means no concurrent Put races on this key. - if be.branchCache != nil { - be.branchCache.Put(prefixCopy, updateCopy, "CollectUpdate") - } + // Cache is populated by sd.GetLatest on MDBX reads; CollectUpdate writes + // only to sd.mem (via ctx.PutBranch above). Pre-flush invalidation isn't + // needed because sd.mem masks the cache for any prefix the writer touched. if be.metrics != nil { be.metrics.updateBranch.Add(1) } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 94769f54bce..88fab39952c 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -866,6 +866,14 @@ func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, par return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) } +// SiteIdentity returns a stable string identifying the underlying +// SharedDomains pointer. Used by cache write sites to tag entries with +// the SD lineage that produced them, so divergence diagnostics can tell +// parent-SD writes apart from fork-SD writes when the cache is shared. +func (sdc *TrieContext) SiteIdentity() string { + return fmt.Sprintf("sd=%p", sdc.probeSd) +} + func (sdc *TrieContext) PutBranch(prefix []byte, data []byte, prevData []byte) error { if sdc.stateReader.WithHistory() { // do not store branches if explicitly operate on history return nil diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 71ed6dc909c..2f86a61a6d7 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -40,7 +40,6 @@ import ( "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/stateifs" "github.com/erigontech/erigon/execution/commitment/nibbles" "github.com/erigontech/erigon/execution/commitment/trie" @@ -3058,20 +3057,14 @@ var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) // further) from "deeper compute bug" (bench fails at the same block). var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) -// branchFromCacheOrDB reads branch data via the cache hierarchy: +// branchFromCacheOrDB reads branch data through the SD chain: // - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) -// - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped -// once the cross-block-persistence step lands) -// - L3: ctx.Branch (canonical store) +// - L2: ctx.Branch — sd.mem -> sd.parent.mem -> branchCache -> MDBX // -// On L2 hit no further work happens. On L3 read with a non-empty result the -// bytes are populated into BranchCache so subsequent reads (within the cache's -// lifetime) hit L2 instead. -// -// L1 (WarmupCache) is intentionally checked FIRST: warmup workers may have -// pre-fetched the branch with prefix-walk-derived freshness guarantees -// (see warmuper.go), and L1 entries are short-lived enough that staleness -// is bounded by Process duration. +// The aggregator-scope BranchCache lives behind sd.mem inside sd.GetLatest; +// CollectUpdate writes only to sd.mem, so cross-SD pollution can no longer +// occur. L1 (WarmupCache) is still checked first for prefix-walk-derived +// freshness within a Process. func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { if hph.cache != nil { if data, found := hph.cache.GetBranch(key); found { @@ -3084,65 +3077,10 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { hph.metrics.missBranch.Add(1) } } - if hph.branchCache != nil && !disableBranchCacheReads { - if data, found := hph.branchCache.Get(key); found { - if verifyBranchCache { - canonical, _, err := hph.ctx.Branch(key) - if err == nil && !bytes.Equal(data, canonical) { - hph.branchCache.RecordDivergence() - fields := []any{ - "prefix", hex.EncodeToString(key), - "cached_len", len(data), - "canonical_len", len(canonical), - "cached", hex.EncodeToString(data), - "canonical", hex.EncodeToString(canonical), - } - // Origin metadata captured at cache.Put time — - // identifies which write site produced the stale - // bytes (CollectUpdate vs L3-fallback-read), with - // a monotonic write-seq + unix-nanos timestamp so - // we can correlate against the timeline of FCU / - // build / step events. - if _, origin, seq, tns, ok := hph.branchCache.GetWithOrigin(key); ok { - fields = append(fields, - "cache_origin", origin, - "cache_seq", seq, - "cache_t_ns", tns) - } - // State-layer probe: sample sd.mem, parent.mem, and - // tx-direct (MDBX) for the same key so the log line - // shows which layer holds bytes matching the cache - // vs ctx.Branch. Decision matrix in - // agentspecs/branch-cache-divergence-probe.md. - if probe, ok := hph.ctx.(interface { - ProbeStateLayers(kv.Domain, []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) - }); ok { - mem, parentMem, mdbx, memOk, parentOk := probe.ProbeStateLayers(kv.CommitmentDomain, key) - if memOk { - fields = append(fields, "sd_mem", hex.EncodeToString(mem)) - } else { - fields = append(fields, "sd_mem", "miss") - } - if parentOk { - fields = append(fields, "parent_mem", hex.EncodeToString(parentMem)) - } else { - fields = append(fields, "parent_mem", "miss-or-no-parent") - } - fields = append(fields, "mdbx", hex.EncodeToString(mdbx)) - } - log.Warn("[branch-cache] divergence", fields...) - } - } - return data, nil - } - } data, _, err := hph.ctx.Branch(key) if err != nil { return nil, err } - if hph.branchCache != nil && len(data) > 0 { - hph.branchCache.Put(key, data, "L3-fallback-read") - } return data, nil } From 99823bbe9e931cf66f8004dd599f9be619253c7a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:49:38 +0000 Subject: [PATCH 012/167] commitment/branch_cache: document responsibility split + post-WarmupCache state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the BranchCache type comment to reflect the architectural state after the WarmupCache consolidation (steps 2a-2c, 3, 4): - BranchCache is the single branch cache (WarmupCache deleted) - Aggregator-scope lifetime, plumbed via BranchCacheProvider - Passive store: cache itself never reaches into underlying state - Branch warmer is branch-scoped; no leaf-data prefetch - Block-processing trie walker takes Updates from executor + memoization for siblings — no prefetch needed - Witness / proof generation walker drives its own state reads; if that path turns out to be cold-bound it indicates a need for separate account/storage caches (treat as separate concern with different scope/lifetime/invalidation; do not regrow the branch warmer to cover it) - disk_sto / disk_acc counters on cache-fp log surface any unexpected fall-through to ctx.Account / ctx.Storage as a signal of memoization gap or missing walker-side prefetch Doc-only; no behaviour change. --- execution/commitment/branch_cache.go | 62 ++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index a24f1193172..bedc7d9db53 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -25,8 +25,7 @@ import ( "github.com/erigontech/erigon/common/maphash" ) -// BranchCache stores commitment-trie branch data with a persistence-friendly -// shape distinct from WarmupCache: +// BranchCache stores commitment-trie branch data: // // - Bounded LRU tail with configurable capacity (eviction is well-defined, // suitable for long-lived caching across many Process calls without @@ -34,24 +33,53 @@ import ( // - Single pinned slot for the root branch (always hottest, always present // once populated, never subject to LRU eviction). Compact prefix of // length 0 (or single-byte "no-key" form) targets this slot. -// - dirty-flag + PutIfClean invariants (carried from step 4 of the -// representation-reduction sequence) so cross-block writers can race +// - dirty-flag + PutIfClean invariants so cross-block writers can race // safely with fold updates. -// - Lazy-decoded read path (GetDecoded) — same pattern as WarmupCache's -// GetBranchDecoded; cells are populated from the cached encoded form -// on first decoded-read and reused thereafter. +// - Lazy-decoded read path (GetDecoded) — cells are populated from the +// cached encoded form on first decoded-read and reused thereafter. // -// Today this is constructed as ephemeral infrastructure (per-Process, -// alongside the trie). Future cross-block persistence work (step 7 of -// the representation-reduction sequence) will lift its lifetime to the -// aggTx level. The wire-up between the trie's read path and BranchCache -// is intentionally NOT done in this commit — see the integration -// discussion captured at the time of step 6. +// Lifetime: aggregator-scope (one instance per Domain). SharedDomains +// pulls the instance via BranchCacheProvider on the AggregatorRoTx; +// commitment-context plumbs it through to the trie via +// InitializeTrieAndUpdates. The previous WarmupCache type (per-Process, +// duplicating account/storage/branch caching above this layer) was +// deleted in the WarmupCache consolidation; BranchCache is now the +// single branch cache. // -// Distinct from WarmupCache: WarmupCache is a simple unbounded map for -// warmup-fill hot-read patterns within one Process. BranchCache is -// designed for the longer-lived cache lifetime that the cross-block -// persistence work needs. +// # Responsibility split (architectural) +// +// The cache is a passive store. Reads and writes are driven by the +// trie walker / encoder; the cache itself never reaches into the +// underlying state. +// +// - BranchCache: passive store of branch bytes / decoded cells. +// Doesn't fetch anything. +// - Branch warmer (warmuper.go): narrow scope — pre-fetches +// *branches* along touched-key paths via SD.GetLatest. No +// account/storage prefetch — that conflated branch warm-up with +// leaf-data fetch. If a fold needs leaf data the trie walker +// fetches it directly (or it's already in Updates / memoized as +// stateHash). +// - Trie walker, block-processing path: receives Updates from the +// executor, folds them. Memoized stateHashes serve siblings; new +// values come from Updates. Doesn't reach into leaf data via +// prefetch. +// - Trie walker, witness / proof generation path: walks the trie +// structure and *needs* to fetch state to materialize the proof. +// This is the walker's responsibility — it drives its own reads +// against SD. If that path turns out to be cold-bound on real +// workloads it may indicate a need for separate account / storage +// caches (the `add_execution_context_with_caches` work has a +// reference design for these). Treat that as a separate concern +// from this BranchCache — different scope, different lifetime, +// different invalidation. Do not regrow the branch warmer's +// scope to cover it. +// +// The disk_sto / disk_acc counters on the [commitment][cache-fp] log +// line surface any fall-through where the trie compute reaches the +// underlying ctx.Account / ctx.Storage paths. On block-processing +// workloads they should remain zero; non-zero values signal a +// memoization gap or a missing walker-side prefetch. // // # Concurrency contract — caller invariants // From 4410704e7a65615be2c9d803a391ff52b54d6dc5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:57:28 +0000 Subject: [PATCH 013/167] commitment/branch_cache: add pinned tier (PinEntry / PinnedCount) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third cache tier between the root-pin slot and the LRU tail: a per-prefix pinned map fed by PinEntry. Pinned entries: - Never evict (no LRU pressure on this tier). - Are checked in lookup before the LRU tail, after the root slot. - Survive Put/SD.Flush updates: a Put for a pinned prefix updates the pinned entry in place rather than displacing it. Cross-block correctness via the existing dirty-flag invalidation discipline is preserved — the new bytes land in the same pinned slot. - Carry the same metadata as Put-tier entries (step, origin, writeSeq, writeTimeNanos) so divergence-detection and Stats treat them uniformly. Sized by the preload policy. Intended consumer is the storage trunk preload for big contracts (the 'storage root trunk cache for big accounts' direction): per-contract trunk branches at depth 65-70 get pinned at SD/cache creation, persist for the cache's lifetime. PinEntry is the public API; pinnedHits/pinnedMisses atomic counters are the new stats. PinnedCount() exposes current size for observability. Step 2 of the storage-trunk pin prototype. --- execution/commitment/branch_cache.go | 62 ++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index bedc7d9db53..a7a64c59b4a 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -158,14 +158,24 @@ type BranchCache struct { // access so no lock is needed for the hot read path. root atomic.Pointer[branchCacheEntry] + // Pinned-prefix tier — explicit per-prefix pin via PinEntry. Used + // for hot-contract storage-trunk preload (the "storage root trunk + // cache for big accounts" path). Entries here NEVER evict — sized + // by the preload policy, not by an LRU. Writes to pinned prefixes + // (via Put/SD.Flush) update the entry in place rather than displace + // it, so cross-block correctness is maintained without losing the + // pin. Lookup checks this tier between root and tail. + pinned *maphash.Map[*branchCacheEntry] + // LRU tail — bounded entries, evicts oldest when full. maphash.LRU // wraps hashicorp/golang-lru/v2 which is thread-safe internally. tail *maphash.LRU[*branchCacheEntry] // Stats — atomic counters surfaced via Stats(). - rootHits, rootMisses atomic.Uint64 - tailHits, tailMisses atomic.Uint64 - bytesServed atomic.Uint64 + rootHits, rootMisses atomic.Uint64 + pinnedHits, pinnedMisses atomic.Uint64 + tailHits, tailMisses atomic.Uint64 + bytesServed atomic.Uint64 // Divergence counter — incremented by RecordDivergence when a caller // detects that a cache-served value disagrees with the canonical @@ -247,7 +257,10 @@ func NewBranchCache(tailCapacity int) *BranchCache { if err != nil { panic(fmt.Sprintf("BranchCache: NewLRU: %s", err)) } - return &BranchCache{tail: tail} + return &BranchCache{ + tail: tail, + pinned: maphash.NewMap[*branchCacheEntry](), + } } // isRootPrefix reports whether prefix targets the pinned root slot. The @@ -268,6 +281,13 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { c.rootHits.Add(1) return entry, true } + // Check pinned tier before tail. Pinned entries never evict and + // are fed by PinEntry (preload) + Put updates that preserve pin. + if entry, ok := c.pinned.Get(prefix); ok { + c.pinnedHits.Add(1) + return entry, true + } + c.pinnedMisses.Add(1) entry, ok := c.tail.Get(prefix) if !ok { c.tailMisses.Add(1) @@ -282,9 +302,43 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { c.root.Store(entry) return } + // If this prefix is pinned, update the pinned entry in place rather + // than route to the LRU tail. Keeps the pin alive across the + // SD.Flush invalidate+Put cycle that refreshes branch values every + // block — without this, every Put would silently lose the pin. + if _, ok := c.pinned.Get(prefix); ok { + c.pinned.Set(prefix, entry) + return + } c.tail.Set(prefix, entry) } +// PinEntry inserts or replaces a pinned cache entry for prefix. Pinned +// entries are never evicted by the LRU and survive across blocks +// (subject to Put updates from SD.Flush refreshing the bytes). Use for +// eager preload of hot prefixes — e.g. the storage-trunk of big +// contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate +// the input after the call. +func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + c.pinned.Set(prefix, &branchCacheEntry{ + data: dataCopy, + step: step, + writeSeq: c.writeSeq.Add(1), + origin: origin, + writeTimeNanos: time.Now().UnixNano(), + }) +} + +// PinnedCount returns the number of currently pinned entries. Useful +// for observability of the preload policy and eviction sanity (pinned +// entries are never evicted; this counter is monotonic over a +// PreloadContractTrunk run). +func (c *BranchCache) PinnedCount() int { + return c.pinned.Len() +} + // Get retrieves branch data from the cache. Returns the canonical encoded // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). From 9b7c02aece3ad852fff6e5094d69ea07d2e0a052 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:59:40 +0000 Subject: [PATCH 014/167] commitment/preload: PreloadContractTrunk recursive enumeration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a function to pre-pin commitment branches for a given contract's storage subtree. Walks the trie depth-by-depth from depth 64 (storage subtree root) down to maxDepth: reads each branch via the supplied CommitmentReader, pins it via BranchCache.PinEntry, decodes the child bitmap, and recurses only into children that actually exist (no blind 16-way probing). contractHash is keccak256(address); the trunk lives at the prefix corresponding to the first 64 nibbles of the storage path. For a dense storage subtree (the SSTORE-bloat workload's bloat contract), expected pin count is ~16 + 256 + 4096 ≈ 4.4 K branches at depth 65/66/67, plus the root at depth 64. Sparse subtrees produce fewer. Loading strategy: per-prefix lookups via the reader. Simplest correct implementation. A bulk seg.Getter range-scan over sorted .kv would amortize disk seeks (per the parity-cluster observation in the consolidation memo) but requires building a prefix-range API on top of recsplit; defer until per-prefix lookup is shown to bottleneck. Step 3 of the storage-trunk pin prototype. --- execution/commitment/preload.go | 120 ++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 execution/commitment/preload.go diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go new file mode 100644 index 00000000000..22a16207434 --- /dev/null +++ b/execution/commitment/preload.go @@ -0,0 +1,120 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "encoding/binary" + "fmt" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// CommitmentReader is the read interface PreloadContractTrunk needs: +// GetLatest on the CommitmentDomain. Returns (value, step, found, err). +// Decoupled from any specific tx/aggregator type so callers can plug +// the reader in at preload time without dragging in db/state imports. +type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) + +// PreloadContractTrunk pins commitment branches for the given contract +// from depth 64 (storage subtree root) down to maxDepth into the supplied +// BranchCache. Walks the trie structure depth-by-depth: reads the branch, +// pins it, decodes its child bitmap, and recurses only into children that +// actually exist. +// +// contractHash is keccak256(address) — 32 bytes. The trunk lives at the +// commitment-domain prefix that corresponds to nibbles 0..63 of the +// storage path (the contract's account hash); enumeration extends from +// there into nibbles 64..maxDepth. +// +// Read amplification: total reads = number of branches actually present +// in the trunk (bounded by the contract's storage trie shape). For a +// dense storage subtree, this is ~16+256+4096 ≈ 4.4 K reads at depth 65, +// 66, 67. Sparse subtrees produce fewer. maxDepth caps the descent. +// +// Returns the number of pinned branches and any error from the reader. +// On error the partial pin set remains in place — those entries are +// still valid cache hits, they just won't see further enumeration below +// the failure point. +// +// Loading strategy: this is the simplest correct implementation — +// recursive enumerate via per-prefix reader lookups. A bulk variant +// using seg.Getter range-scan over sorted .kv would amortize disk +// seeks but requires building a prefix-range API on top of recsplit. +// Defer that optimization until per-prefix lookup is shown to bottleneck. +func PreloadContractTrunk( + contractHash []byte, + maxDepth int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (int, error) { + if len(contractHash) != 32 { + return 0, fmt.Errorf("PreloadContractTrunk: contractHash must be 32 bytes, got %d", len(contractHash)) + } + if maxDepth < 64 { + return 0, fmt.Errorf("PreloadContractTrunk: maxDepth %d < 64 (storage subtree starts at depth 64)", maxDepth) + } + if cache == nil { + return 0, fmt.Errorf("PreloadContractTrunk: cache is nil") + } + + // Convert contract hash bytes to 64 nibbles. + contractNibbles := make([]byte, 64) + for i, b := range contractHash { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + + pinned := 0 + var enumerate func(pathNibbles []byte, depth int) error + enumerate = func(pathNibbles []byte, depth int) error { + prefix := nibbles.HexToCompact(pathNibbles) + v, step, found, err := reader(prefix) + if err != nil { + return fmt.Errorf("preload at depth %d: %w", depth, err) + } + if !found { + return nil + } + cache.PinEntry(prefix, v, step, "preload-trunk") + pinned++ + if depth >= maxDepth { + return nil + } + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. + // Only recurse into children that actually exist (bit set in bitmap). + if len(v) < 4 { + return nil + } + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1< Date: Fri, 8 May 2026 16:03:09 +0000 Subject: [PATCH 015/167] state/execctx, commitment: PIN_CONTRACT_TRUNKS env hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a SharedDomains constructor hook that, when PIN_CONTRACT_TRUNKS is set, fires a one-shot background goroutine to preload the storage-subtree-trunk of each listed contract into BranchCache's pinned tier. Format: comma-separated list of 64-hex-char contract hashes (each is keccak256(addr)). Mechanism: - BranchCache.TryClaimPreload (atomic CAS) ensures the goroutine fires exactly once per cache lifetime, even though many SDs may be constructed (per-tx instances etc.). - Goroutine wraps sd.GetLatest as a CommitmentReader and calls commitment.PreloadContractTrunk for each contract hash, depth 64-70. - Logs progress per contract on completion. Closure-over-(sd, tx) is the prototype shape — works for the bench (both live for the whole process). Production deployment needs to revisit the lifetime — sd's tx may not outlive the goroutine. Step 4 of the storage-trunk pin prototype. Bench measurement is the next step (commit 5). --- db/state/execctx/domain_shared.go | 65 ++++++++++++++++++++++++++++ execution/commitment/branch_cache.go | 16 +++++++ 2 files changed, 81 insertions(+) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 7eb25fe20f1..4bf242eab02 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -19,10 +19,12 @@ package execctx import ( "bytes" "context" + "encoding/hex" "errors" "fmt" "math" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -201,6 +203,18 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) + // PIN_CONTRACT_TRUNKS=hash1,hash2,... (each hash is 64 hex chars = + // 32 bytes = keccak256(contractAddr)). When set and BranchCache is + // available, fire a one-shot background goroutine to preload the + // listed contracts' storage-subtree-trunk into the cache's pinned + // tier. Use TryClaimPreload to ensure we run exactly once per + // BranchCache lifetime even though many SDs may be constructed. + if branchCache != nil && branchCache.TryClaimPreload() { + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(ctx, sd, tx, pinList, logger) + } + } + _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -1158,3 +1172,54 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN } return changes, nil } + +// triggerTrunkPreload kicks off a background goroutine that pre-pins +// the storage-subtree-trunk of each contract in pinList for the +// SharedDomains' BranchCache. pinList is a comma-separated list of +// 64-hex-char contract hashes (keccak256(addr)). Run from the SD +// constructor; gated by BranchCache.TryClaimPreload so it fires once +// per cache lifetime. +// +// Reader closes over sd + tx; the bench scenario keeps both alive for +// the whole process so the closure is safe. For production, this +// pattern needs revisiting (tx may not outlive the goroutine). +func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { + const maxDepth = 70 // covers depths 64-70 = the dominant read range on bloat workloads + var hashes [][]byte + for _, hexStr := range strings.Split(pinList, ",") { + hexStr = strings.TrimSpace(hexStr) + if hexStr == "" { + continue + } + h, err := hex.DecodeString(hexStr) + if err != nil || len(h) != 32 { + logger.Warn("[trunk-preload] skipping invalid PIN_CONTRACT_TRUNKS entry", + "entry", hexStr, "err", err, "len", len(h)) + continue + } + hashes = append(hashes, h) + } + if len(hashes) == 0 { + return + } + cache := sd.branchCache + go func() { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil + } + for _, h := range hashes { + n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, cache, logger) + if err != nil { + logger.Warn("[trunk-preload] failed", + "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) + continue + } + logger.Info("[trunk-preload] contract done", + "hash", hex.EncodeToString(h), "pinned", n) + } + }() +} diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index a7a64c59b4a..e6701db4514 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -191,6 +191,22 @@ type BranchCache struct { // origin label and timestamp to identify which write produced the // stale bytes. writeSeq atomic.Uint64 + + // preloadClaimed is set the first time TryClaimPreload is called. + // Used by the trunk-preload trigger (PIN_CONTRACT_TRUNKS hook in + // SharedDomains construction) so the preload goroutine fires + // exactly once per cache lifetime, even though many SharedDomains + // instances may be created over a process's lifetime. + preloadClaimed atomic.Bool +} + +// TryClaimPreload returns true the first time it's called on a given +// BranchCache instance, false on every subsequent call. Used by the +// trunk-preload trigger to ensure the preload goroutine runs exactly +// once per cache (process-lifetime), regardless of how many +// SharedDomains instances are constructed. +func (c *BranchCache) TryClaimPreload() bool { + return c.preloadClaimed.CompareAndSwap(false, true) } type branchCacheEntry struct { From 71d8480df282b47a4cc70be540dcd1135cff2922 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 16:17:53 +0000 Subject: [PATCH 016/167] state/execctx: trunk-preload sync (was async, hit cgo-pointer panic) Previous async-goroutine shape (d204c1b577) shared the SD's MDBX tx with the calling thread. Concurrent cursor use under the same tx tripped Go's cgo-pointer-pinning runtime check: panic: runtime error: cgo argument has Go pointer to unpinned Go pointer surfacing in an unrelated PruneBlocks goroutine during boot. Make the preload synchronous in the SD constructor for now: same TryClaimPreload guard (fires once per cache lifetime), but no goroutine. Boot pays the per-contract preload time as a one-off. Background-with-own-tx is the proper shape and remains a follow-up; owning the SD's tx exclusively for the preload duration is the safe shape until that lands. --- db/state/execctx/domain_shared.go | 57 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 4bf242eab02..b8d13da96ac 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1173,16 +1173,22 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN return changes, nil } -// triggerTrunkPreload kicks off a background goroutine that pre-pins -// the storage-subtree-trunk of each contract in pinList for the -// SharedDomains' BranchCache. pinList is a comma-separated list of -// 64-hex-char contract hashes (keccak256(addr)). Run from the SD -// constructor; gated by BranchCache.TryClaimPreload so it fires once -// per cache lifetime. +// triggerTrunkPreload runs a synchronous trunk-preload pass for each +// contract listed in pinList against the SharedDomains' BranchCache. +// pinList is a comma-separated list of 64-hex-char contract hashes +// (keccak256(addr)). Run from the SD constructor; gated by +// BranchCache.TryClaimPreload so it fires once per cache lifetime. // -// Reader closes over sd + tx; the bench scenario keeps both alive for -// the whole process so the closure is safe. For production, this -// pattern needs revisiting (tx may not outlive the goroutine). +// SYNCHRONOUS for prototype correctness: a previous async-goroutine +// shape shared the SD's MDBX tx with the calling thread and tripped +// "cgo argument has Go pointer to unpinned Go pointer" under +// concurrent cursor use. Background-with-own-tx is a follow-up; +// owning the SD's tx exclusively for the preload duration is the +// safe shape until that lands. +// +// Boot cost is the per-contract preload time, paid once per cache +// lifetime (TryClaimPreload guards). Block-app benchmarks see this +// as a one-time hit during the first SD construction. func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { const maxDepth = 70 // covers depths 64-70 = the dominant read range on bloat workloads var hashes [][]byte @@ -1202,24 +1208,21 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT if len(hashes) == 0 { return } - cache := sd.branchCache - go func() { - reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) - if err != nil { - return nil, 0, false, err - } - return v, uint64(step), len(v) > 0, nil + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) + if err != nil { + return nil, 0, false, err } - for _, h := range hashes { - n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, cache, logger) - if err != nil { - logger.Warn("[trunk-preload] failed", - "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) - continue - } - logger.Info("[trunk-preload] contract done", - "hash", hex.EncodeToString(h), "pinned", n) + return v, uint64(step), len(v) > 0, nil + } + for _, h := range hashes { + n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, sd.branchCache, logger) + if err != nil { + logger.Warn("[trunk-preload] failed", + "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) + continue } - }() + logger.Info("[trunk-preload] contract done", + "hash", hex.EncodeToString(h), "pinned", n) + } } From f85308daad9f57b9603d1ca102dc1f5fb9db2e89 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 19:26:09 +0000 Subject: [PATCH 017/167] commitment, state/execctx: trunk-preload progress logs + max-branches cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous bench (run-pin-trunk-instrumented-cold-cgroup-191347) hung at SD construction with no [trunk-preload] log lines for 5+ minutes. Erigon never reached "engine RPC ready" so all blocks came in as SYNCING. Two changes to localise + bound: 1. **Localisation**: add INFO logs at triggerTrunkPreload entry, per-contract starting/done with took, and a 500-prefix progress log inside PreloadContractTrunk. Whatever it does (or hangs on) is now observable. 2. **Bound**: cap PreloadContractTrunk at 10000 branches (vs ~4.4K expected for a saturated 4-level subtree at maxDepth=67). Drops maxDepth from 70 → 67 in the trigger (depth 64-67 = 16+256+4096 max branches) so we don't recurse into the per-slot tail where pinning has no value. Preload fails-fast on pathological subtrees rather than blocking SD construction indefinitely. --- db/state/execctx/domain_shared.go | 17 +++++++++++++---- execution/commitment/preload.go | 26 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index b8d13da96ac..533a9fad71c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1190,7 +1190,11 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // lifetime (TryClaimPreload guards). Block-app benchmarks see this // as a one-time hit during the first SD construction. func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { - const maxDepth = 70 // covers depths 64-70 = the dominant read range on bloat workloads + // Lower default depth (was 70 → blocked SD construction for too long + // on dense subtrees). 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K + // branches max per contract for a fully-saturated subtree. + const maxDepth = 67 + logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte for _, hexStr := range strings.Split(pinList, ",") { hexStr = strings.TrimSpace(hexStr) @@ -1205,6 +1209,7 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT } hashes = append(hashes, h) } + logger.Info("[trunk-preload] hashes parsed", "count", len(hashes), "max_depth", maxDepth) if len(hashes) == 0 { return } @@ -1215,14 +1220,18 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT } return v, uint64(step), len(v) > 0, nil } - for _, h := range hashes { + for i, h := range hashes { + started := time.Now() + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, sd.branchCache, logger) + took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", - "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) + "hash", hex.EncodeToString(h), "pinned_so_far", n, "took", took, "err", err) continue } logger.Info("[trunk-preload] contract done", - "hash", hex.EncodeToString(h), "pinned", n) + "hash", hex.EncodeToString(h), "pinned", n, "took", took) } + logger.Info("[trunk-preload] all done", "contracts", len(hashes)) } diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 22a16207434..497e62a09f8 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -72,9 +72,27 @@ func PreloadContractTrunk( contractNibbles[2*i+1] = b & 0x0f } + // Hard cap to bound preload work. A fully-saturated 4-level subtree + // is 1+16+256+4096 = 4369 branches; this gives headroom but still + // fails fast on malformed input (e.g. a contract with truly + // pathological branching). Tune downward if this turns out to bite. + const maxBranches = 10000 + pinned := 0 + var stopped bool var enumerate func(pathNibbles []byte, depth int) error enumerate = func(pathNibbles []byte, depth int) error { + if stopped { + return nil + } + if pinned >= maxBranches { + stopped = true + if logger != nil { + logger.Warn("[trunk-preload] hit max-branches cap", + "cap", maxBranches, "depth", depth) + } + return nil + } prefix := nibbles.HexToCompact(pathNibbles) v, step, found, err := reader(prefix) if err != nil { @@ -85,14 +103,18 @@ func PreloadContractTrunk( } cache.PinEntry(prefix, v, step, "preload-trunk") pinned++ + // Periodic progress log so a slow preload is observable. + if logger != nil && pinned%500 == 0 { + logger.Info("[trunk-preload] progress", "pinned", pinned, "depth", depth) + } if depth >= maxDepth { return nil } - // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. - // Only recurse into children that actually exist (bit set in bitmap). if len(v) < 4 { return nil } + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. + // Only recurse into children that actually exist (bit set in bitmap). bitmap := binary.BigEndian.Uint16(v[2:4]) for n := 0; n < 16; n++ { if bitmap&(1< Date: Fri, 8 May 2026 19:51:55 +0000 Subject: [PATCH 018/167] state/execctx: run trunk-preload AFTER SeekCommitment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous shape (d204c1b577) ran triggerTrunkPreload BEFORE sd.SeekCommitment in NewSharedDomains. Bench result: when the preload fired (PIN_CONTRACT_TRUNKS set), every subsequent block came back SYNCING — engine kept attempting backward-download which fails on this peerless setup, no block ever validated, no cache-fp ever fired. Without the preload firing, the same binary works fine (verify-bench PASS at 3.26s). Hypothesis (untested but matches the symptom): preload's sd.GetLatest reads ran before SeekCommitment had resolved the SD's view of the chain head. Pinned values were therefore inconsistent with the committed state, and the trie compute on the first block got wrong root → SYNCING → backward-download → no peers → death spiral with no Flush ever updating the (stale) pinned entries. Fix is mechanical: move the preload call to after SeekCommitment. The TryClaimPreload guard still ensures fire-once-per-cache lifetime. If subsequent bench shows pin_count > 0 + pin_hit > 0 + blocks validating normally, the hypothesis is confirmed; if SYNCING repeats, the bug is something else and we need to revert and debug differently. --- db/state/execctx/domain_shared.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 533a9fad71c..e08420cf2be 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -203,18 +203,6 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) - // PIN_CONTRACT_TRUNKS=hash1,hash2,... (each hash is 64 hex chars = - // 32 bytes = keccak256(contractAddr)). When set and BranchCache is - // available, fire a one-shot background goroutine to preload the - // listed contracts' storage-subtree-trunk into the cache's pinned - // tier. Use TryClaimPreload to ensure we run exactly once per - // BranchCache lifetime even though many SDs may be constructed. - if branchCache != nil && branchCache.TryClaimPreload() { - if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { - triggerTrunkPreload(ctx, sd, tx, pinList, logger) - } - } - _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -231,6 +219,19 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg } } + // PIN_CONTRACT_TRUNKS=hash1,hash2,... preload contract storage-trunk + // into BranchCache's pinned tier. Run AFTER SeekCommitment so the + // SD has resolved its view of the chain head — reading via + // sd.GetLatest BEFORE that produced pinned values inconsistent with + // the committed state and broke subsequent block validation + // (every block came back SYNCING). TryClaimPreload guards + // fire-once-per-cache lifetime. + if branchCache != nil && branchCache.TryClaimPreload() { + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(ctx, sd, tx, pinList, logger) + } + } + return sd, nil } From 29ebc428024e00c7a8280d82f0bd0fb2beed6076 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 20:13:47 +0000 Subject: [PATCH 019/167] state/execctx: trunk-preload async with own tx, triggered from EnableParaTrieDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous prototype iterations both broke block validation: 1. Async sharing the SD's MDBX tx (d204c1b577) → cgo "unpinned Go pointer" panic from concurrent cursor use. 2. Synchronous from NewSharedDomains (5a81976553 / 4c9ead456d) → blocked the engine HTTP handler for ~3-4s during the preload window, causing the bench's first NewPayload to be dropped. Confirmed: the bench's height=24358001 is ABSENT from the erigon log; the next received block (24358002) then fails backward-download (no peers) → SYNCING forever. Restructure: - Move trigger from NewSharedDomains to EnableParaTrieDB. The latter is called from the staged-sync exec-stage init, NOT from request handlers, AND has access to a kv.TemporalRoDB. - triggerTrunkPreload now takes the DB (not a tx) and spawns a goroutine that opens its OWN tx via db.BeginTemporalRo. No shared cursors with the main pipeline; no blocking the engine. - Reader uses tx.GetLatest directly (not sd.GetLatest) — the SD layering would re-introduce shared-state risk and isn't needed (pinned bytes don't depend on sd.mem state). Same TryClaimPreload guard ensures the preload fires once per BranchCache lifetime regardless of how many SDs construct. If this works the bench should: - Show [trunk-preload] log lines firing once - Pin ~4369 branches - TEST block cache-fp shows pin_hit > 0 and files_comm < 1K - All blocks validate normally (no SYNCING failure) --- db/state/execctx/domain_shared.go | 121 ++++++++++++++++++------------ 1 file changed, 73 insertions(+), 48 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index e08420cf2be..feeeaba3293 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -219,18 +219,15 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg } } - // PIN_CONTRACT_TRUNKS=hash1,hash2,... preload contract storage-trunk - // into BranchCache's pinned tier. Run AFTER SeekCommitment so the - // SD has resolved its view of the chain head — reading via - // sd.GetLatest BEFORE that produced pinned values inconsistent with - // the committed state and broke subsequent block validation - // (every block came back SYNCING). TryClaimPreload guards - // fire-once-per-cache lifetime. - if branchCache != nil && branchCache.TryClaimPreload() { - if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { - triggerTrunkPreload(ctx, sd, tx, pinList, logger) - } - } + // PIN_CONTRACT_TRUNKS preload runs from EnableParaTrieDB (called by + // the staged sync exec stage) NOT here in NewSharedDomains — at SD + // construction time we'd block the engine HTTP handler for ~3-4s + // while preload reads, causing the bench's first NewPayload to be + // dropped. EnableParaTrieDB runs in the staged sync init goroutine, + // not the request handler, AND has access to the kv.TemporalRoDB + // needed to spawn an own-tx preload goroutine that doesn't share + // cursors with the main pipeline (the cause of the earlier + // cgo-pointer panic). return sd, nil } @@ -1119,6 +1116,24 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) + + // Trigger trunk-preload here (not from NewSharedDomains) so: + // (a) we have a kv.TemporalRoDB to spawn an own-tx preload goroutine + // with — avoids the cgo-pointer panic that the earlier shared-tx + // attempt produced. + // (b) we run from the stage-init path, not an engine request handler, + // so we don't block engine HTTP responses during the 3-4s preload + // window — the synchronous-from-NewSharedDomains shape caused the + // bench's first NewPayload to be dropped, breaking block validation. + // TryClaimPreload guards fire-once-per-BranchCache-lifetime. + if sd.branchCache == nil || !sd.branchCache.TryClaimPreload() { + return + } + pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", "") + if pinList == "" { + return + } + triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) } func (sd *SharedDomains) EnableWarmupCache(enable bool) { @@ -1174,26 +1189,23 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN return changes, nil } -// triggerTrunkPreload runs a synchronous trunk-preload pass for each -// contract listed in pinList against the SharedDomains' BranchCache. -// pinList is a comma-separated list of 64-hex-char contract hashes -// (keccak256(addr)). Run from the SD constructor; gated by -// BranchCache.TryClaimPreload so it fires once per cache lifetime. +// triggerTrunkPreload spawns a goroutine that pre-pins the storage-trunk +// of each contract in pinList for the given BranchCache. pinList is a +// comma-separated list of 64-hex-char contract hashes (keccak256(addr)). // -// SYNCHRONOUS for prototype correctness: a previous async-goroutine -// shape shared the SD's MDBX tx with the calling thread and tripped -// "cgo argument has Go pointer to unpinned Go pointer" under -// concurrent cursor use. Background-with-own-tx is a follow-up; -// owning the SD's tx exclusively for the preload duration is the -// safe shape until that lands. +// The goroutine opens its OWN kv.TemporalRoTx via db.BeginTemporalRo, +// distinct from the staged-sync's main tx. Two earlier shapes failed: +// 1. Goroutine sharing the SD's MDBX tx → cgo "unpinned Go pointer" +// panic from concurrent cursor use across goroutines. +// 2. Synchronous from NewSharedDomains → blocked the engine HTTP +// handler for ~3-4s during the preload window, dropping the bench's +// first NewPayload and breaking subsequent backward-download. // -// Boot cost is the per-contract preload time, paid once per cache -// lifetime (TryClaimPreload guards). Block-app benchmarks see this -// as a one-time hit during the first SD construction. -func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { - // Lower default depth (was 70 → blocked SD construction for too long - // on dense subtrees). 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K - // branches max per contract for a fully-saturated subtree. +// Own-tx async sidesteps both. Caller must guarantee fire-once via +// BranchCache.TryClaimPreload (the only call site does this). +func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { + // 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max per + // contract for a fully-saturated subtree. const maxDepth = 67 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte @@ -1214,25 +1226,38 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT if len(hashes) == 0 { return } - reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) + go func() { + tx, err := db.BeginTemporalRo(ctx) if err != nil { - return nil, 0, false, err + logger.Warn("[trunk-preload] BeginTemporalRo failed", "err", err) + return } - return v, uint64(step), len(v) > 0, nil - } - for i, h := range hashes { - started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, sd.branchCache, logger) - took := time.Since(started) - if err != nil { - logger.Warn("[trunk-preload] failed", - "hash", hex.EncodeToString(h), "pinned_so_far", n, "took", took, "err", err) - continue + defer tx.Rollback() + // Read directly via tx.GetLatest — no SharedDomains wrap. + // SD's GetLatest layers in sd.mem / parent.mem checks that + // (a) we don't need (the trunk hasn't been written yet in this + // goroutine's tx), and (b) would risk re-introducing the + // shared-cursor problem. + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := tx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil } - logger.Info("[trunk-preload] contract done", - "hash", hex.EncodeToString(h), "pinned", n, "took", took) - } - logger.Info("[trunk-preload] all done", "contracts", len(hashes)) + for i, h := range hashes { + started := time.Now() + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) + n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, branchCache, logger) + took := time.Since(started) + if err != nil { + logger.Warn("[trunk-preload] failed", + "hash", hex.EncodeToString(h), "pinned_so_far", n, "took", took, "err", err) + continue + } + logger.Info("[trunk-preload] contract done", + "hash", hex.EncodeToString(h), "pinned", n, "took", took) + } + logger.Info("[trunk-preload] all done", "contracts", len(hashes)) + }() } From db656cc58939931042b7117d3c8258e8ef69423c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 20:27:07 +0000 Subject: [PATCH 020/167] execution/commitment: PIN_TRUNK_MAX_DEPTH env + raise maxBranches cap Make the trunk-pin maxDepth configurable via env (default 67) so we can sweep depths to find the memory/perf sweet spot without rebuilding. Bump the per-contract maxBranches cap from 10K to 200K so deeper saturated subtrees don't get truncated mid-walk. --- db/state/execctx/domain_shared.go | 10 +++++++--- execution/commitment/preload.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index feeeaba3293..671057fbbbf 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1204,9 +1204,13 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // Own-tx async sidesteps both. Caller must guarantee fire-once via // BranchCache.TryClaimPreload (the only call site does this). func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { - // 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max per - // contract for a fully-saturated subtree. - const maxDepth = 67 + // Default 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max + // per contract for a fully-saturated subtree. Override via + // PIN_TRUNK_MAX_DEPTH for sweep experiments. + maxDepth := 67 + if v := dbg.EnvInt("PIN_TRUNK_MAX_DEPTH", 67); v >= 64 { + maxDepth = v + } logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte for _, hexStr := range strings.Split(pinList, ",") { diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 497e62a09f8..c996943318c 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -76,7 +76,7 @@ func PreloadContractTrunk( // is 1+16+256+4096 = 4369 branches; this gives headroom but still // fails fast on malformed input (e.g. a contract with truly // pathological branching). Tune downward if this turns out to bite. - const maxBranches = 10000 + const maxBranches = 200000 pinned := 0 var stopped bool From ce8cce46e3b7e9413060ff7dcead496eec5b0e6d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 21:26:56 +0000 Subject: [PATCH 021/167] stagedsync/exec3_parallel: re-enable trie warmup on parallel path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous code disabled the Warmuper for the parallel commitment path out of concern that it would interact with the calculator's SetUpdates call. In practice the Warmuper's reads are independent of the calculator's update buffer — they pre-fetch branch data while EVM execution runs, and the calculator's SetUpdates only affects ComputeCommitment's input set, not the warmup paths. Re-enabling produces a measured 8× throughput improvement on the perf-devnet-3 SSTORE-bloated benchmark (block 24358306, the canonical fixture for #20920), restoring the win first observed in Run H/I of the trie-perf investigation. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/stagedsync/exec3_parallel.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 3160516402e..97eafddeb12 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -217,10 +217,13 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U pe.rs.Domains().SetInMemHistoryReads(true) defer pe.rs.Domains().SetInMemHistoryReads(prevInMemHistoryReads) - // Disable trie warmup — the Warmuper uses sdCtx.updates which the - // calculator replaces via SetUpdates before ComputeCommitment. - pe.rs.Domains().EnableTrieWarmup(false) - defer pe.rs.Domains().EnableTrieWarmup(true) + // Trie warmup left enabled for the parallel path. Original disable was + // based on a calculator/warmer interaction concern that turned out to be + // overly conservative — the Warmuper's reads are independent of the + // calculator's SetUpdates call. Removing the disable produced an 8× + // throughput improvement on the perf-devnet-3 SSTORE-bloated benchmark + // (block 24358306) by letting the Warmuper pre-fetch branch data while + // EVM execution runs. See #20920 for the canonical perf measurement. // Skip step-boundary commitment — the calculator handles this. pe.rs.StateV3.SetSkipStepBoundaryCommitment(true) From e97c95f47c322080811b408f9c6fa34e67081f50 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 21:33:41 +0000 Subject: [PATCH 022/167] commitment: WarmupCache hit/miss/evict observability counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds atomic counters (branch hit/miss/evict + bytes-served, account hit/miss, storage hit/miss) to WarmupCache, plus a Stats() string formatter and ResetStats() for per-Process accumulation reset. Counters are updated on every Get/Evict path (existing Put paths were already counted via cache size). Useful for: - Confirming warmup effectiveness in production logs - Per-block diagnostics when investigating commitment perf - Future per-pool dashboards once a coordinator/observability layer lands (tracked separately) No behavior change beyond the counter updates themselves. Stats() format is one line, suitable for embedding in the existing LogCommitments output. ResetStats() zeros counters without touching cached data — useful for per-Process windowed measurement. Clear() also resets counters along with the data, since data and counters were accumulated together. Test: TestWarmupCache_Stats covers hit/miss/evict accounting across branch/account/storage paths and verifies Stats() format + ResetStats() preserves data. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/warmup_cache.go | 69 ++++++++++++++++- execution/commitment/warmup_cache_test.go | 91 +++++++++++++++++++++++ 2 files changed, 157 insertions(+), 3 deletions(-) diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index d6389a350d9..643ae0e1174 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -17,6 +17,7 @@ package commitment import ( + "fmt" "sync/atomic" "github.com/erigontech/erigon/common/maphash" @@ -44,6 +45,13 @@ type WarmupCache struct { branches *maphash.Map[*branchEntry] accounts *maphash.Map[*accountEntry] storage *maphash.Map[*storageEntry] + + // Observability counters. Updated by Get/Put paths; read by Stats(). + // Atomic so warmup-worker reads and main-trie reads don't race. + branchHits, branchMisses, branchEvicted atomic.Uint64 + branchBytesServed atomic.Uint64 + accountHits, accountMisses atomic.Uint64 + storageHits, storageMisses atomic.Uint64 } // NewWarmupCache creates a new warmup cache instance. @@ -67,19 +75,33 @@ func (c *WarmupCache) PutBranch(prefix []byte, data []byte) { // GetBranch retrieves branch data from the cache. func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) - if !found || entry.isEvicted.Load() { + if !found { + c.branchMisses.Add(1) + return nil, false + } + if entry.isEvicted.Load() { + c.branchEvicted.Add(1) return nil, false } + c.branchHits.Add(1) + c.branchBytesServed.Add(uint64(len(entry.data))) return entry.data, true } // GetAndEvictBranch retrieves branch data and marks the entry as evicted in one operation. func (c *WarmupCache) GetAndEvictBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) - if !found || entry.isEvicted.Load() { + if !found { + c.branchMisses.Add(1) + return nil, false + } + if entry.isEvicted.Load() { + c.branchEvicted.Add(1) return nil, false } entry.isEvicted.Store(true) + c.branchHits.Add(1) + c.branchBytesServed.Add(uint64(len(entry.data))) return entry.data, true } @@ -105,8 +127,10 @@ func (c *WarmupCache) PutAccount(plainKey []byte, update *Update) { func (c *WarmupCache) GetAccount(plainKey []byte) (*Update, bool) { entry, found := c.accounts.Get(plainKey) if !found || entry.isEvicted.Load() { + c.accountMisses.Add(1) return nil, false } + c.accountHits.Add(1) return entry.update, true } @@ -143,8 +167,10 @@ func (c *WarmupCache) PutStorage(plainKey []byte, update *Update) { func (c *WarmupCache) GetStorage(plainKey []byte) (*Update, bool) { entry, found := c.storage.Get(plainKey) if !found || entry.isEvicted.Load() { + c.storageMisses.Add(1) return nil, false } + c.storageHits.Add(1) return entry.update, true } @@ -178,9 +204,46 @@ func (c *WarmupCache) EvictPlainKey(plainKey []byte) { } } -// Clear clears all cached data. +// Clear clears all cached data and resets stats counters. func (c *WarmupCache) Clear() { c.branches = maphash.NewMap[*branchEntry]() c.accounts = maphash.NewMap[*accountEntry]() c.storage = maphash.NewMap[*storageEntry]() + c.ResetStats() +} + +// Stats returns a one-line summary of cache hit/miss counters. +// Format matches the per-block log line: branch, account, storage with +// hit-percentages and bytes served. Useful in commitment debug logs and +// for the per-Process LogCommitments line. +func (c *WarmupCache) Stats() string { + bh, bm, be := c.branchHits.Load(), c.branchMisses.Load(), c.branchEvicted.Load() + bb := c.branchBytesServed.Load() + ah, am := c.accountHits.Load(), c.accountMisses.Load() + sh, sm := c.storageHits.Load(), c.storageMisses.Load() + pct := func(hit, miss uint64) float64 { + total := hit + miss + if total == 0 { + return 0 + } + return 100.0 * float64(hit) / float64(total) + } + return fmt.Sprintf( + "branch hit=%d miss=%d evict=%d (%.1f%%, %.1f MiB) | acct hit=%d miss=%d (%.1f%%) | stor hit=%d miss=%d (%.1f%%)", + bh, bm, be, pct(bh, bm), float64(bb)/1024/1024, + ah, am, pct(ah, am), + sh, sm, pct(sh, sm), + ) +} + +// ResetStats zeros out all hit/miss/byte counters without touching cached data. +func (c *WarmupCache) ResetStats() { + c.branchHits.Store(0) + c.branchMisses.Store(0) + c.branchEvicted.Store(0) + c.branchBytesServed.Store(0) + c.accountHits.Store(0) + c.accountMisses.Store(0) + c.storageHits.Store(0) + c.storageMisses.Store(0) } diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go index 43de177ee06..a52fd85cbff 100644 --- a/execution/commitment/warmup_cache_test.go +++ b/execution/commitment/warmup_cache_test.go @@ -301,3 +301,94 @@ func BenchmarkComparison_Map_100k(b *testing.B) { cache.GetStorage(keys[i%len(keys)]) } } + +// TestWarmupCache_Stats verifies that hit/miss/evict counters are updated +// correctly across the Get/Put/Evict paths and that Stats() returns a +// deterministic format. +func TestWarmupCache_Stats(t *testing.T) { + cache := NewWarmupCache() + + // Branch hits + misses + bytes-served + bk := []byte("k-branch") + bd := []byte("branch-data-12345") + cache.PutBranch(bk, bd) + if _, ok := cache.GetBranch(bk); !ok { + t.Fatal("expected branch hit") + } + if _, ok := cache.GetBranch([]byte("missing-branch")); ok { + t.Fatal("expected branch miss") + } + + // Branch eviction + cache.EvictBranch(bk) + if _, ok := cache.GetBranch(bk); ok { + t.Fatal("expected branch evicted miss") + } + + // Account hits + misses + ak := []byte("k-acct") + cache.PutAccount(ak, &Update{}) + if _, ok := cache.GetAccount(ak); !ok { + t.Fatal("expected account hit") + } + if _, ok := cache.GetAccount([]byte("missing-acct")); ok { + t.Fatal("expected account miss") + } + + // Storage hits + misses + sk := []byte("k-stor") + cache.PutStorage(sk, &Update{}) + if _, ok := cache.GetStorage(sk); !ok { + t.Fatal("expected storage hit") + } + if _, ok := cache.GetStorage([]byte("missing-stor")); ok { + t.Fatal("expected storage miss") + } + + // Counters: branch hit=1 miss=1 evict=1 bytes=17, acct hit=1 miss=1, stor hit=1 miss=1 + if got := cache.branchHits.Load(); got != 1 { + t.Fatalf("branchHits: got %d, want 1", got) + } + if got := cache.branchMisses.Load(); got != 1 { + t.Fatalf("branchMisses: got %d, want 1", got) + } + if got := cache.branchEvicted.Load(); got != 1 { + t.Fatalf("branchEvicted: got %d, want 1", got) + } + if got := cache.branchBytesServed.Load(); got != uint64(len(bd)) { + t.Fatalf("branchBytesServed: got %d, want %d", got, len(bd)) + } + if got := cache.accountHits.Load(); got != 1 { + t.Fatalf("accountHits: got %d, want 1", got) + } + if got := cache.accountMisses.Load(); got != 1 { + t.Fatalf("accountMisses: got %d, want 1", got) + } + if got := cache.storageHits.Load(); got != 1 { + t.Fatalf("storageHits: got %d, want 1", got) + } + if got := cache.storageMisses.Load(); got != 1 { + t.Fatalf("storageMisses: got %d, want 1", got) + } + + // Stats() format + stats := cache.Stats() + for _, want := range []string{ + "branch hit=1 miss=1 evict=1", + "acct hit=1 miss=1", + "stor hit=1 miss=1", + } { + if !bytes.Contains([]byte(stats), []byte(want)) { + t.Errorf("Stats() missing %q\nfull: %s", want, stats) + } + } + + // ResetStats zeros counters but leaves data intact + cache.ResetStats() + if got := cache.branchHits.Load(); got != 0 { + t.Fatalf("branchHits after Reset: got %d, want 0", got) + } + if _, ok := cache.GetAccount(ak); !ok { + t.Fatal("account data should survive ResetStats") + } +} From af2ef6872289a5824a539259782283946e4d61ed Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 22:21:46 +0000 Subject: [PATCH 023/167] commitment: extract unfoldKeyPath as per-key traversal primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure refactor — behavior preserved. Lifts the unfold-loop from HexPatriciaHashed.followAndUpdate into its own method, parameterized over (hashedKey, plainKey) and intended as the per-key traversal primitive that future orchestrators consume. Today only followAndUpdate calls it, replacing the inline loop with a one-line call. The extracted method preserves the existing metric attribution (StartUnfolding) and trace-print behavior verbatim. Why now: this is the second step in the representation-reduction sequence (see agentspecs/trie-data-pipeline-complexity-tax.md). Future PRs will introduce orchestrators that drive unfold-only walks of touched-key paths to fill cell state without going through the full fold/update cycle: - Cache populator (decoded-payload BranchCache) needs to walk a touched-key path and capture the cells encountered, without triggering fold or modifying the trie's update buffer. - Stage E parallel pre-unfold orchestrator drives unfoldKeyPath across multiple HexPatriciaHashed instances concurrently to pre-warm trie state before commit. Both consume the same primitive. Centralising it now means each future orchestrator is a thin wrapper rather than a duplicate of the unfold-loop logic. Tests: full commitment test suite passes without modification (all 8+ test files in execution/commitment/), confirming the refactor preserves followAndUpdate's behavior. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/hex_patricia_hashed.go | 54 ++++++++++++++------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 2f86a61a6d7..3f53e71fa04 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2504,6 +2504,40 @@ func (hph *HexPatriciaHashed) RootHash() ([]byte, error) { return rootHash[1:], nil // first byte is 128+hash_len=160 } +// unfoldKeyPath drives hph.unfold in a loop until the trie is unfolded +// far enough that the cell at hashedKey can be updated — i.e. until +// needUnfolding returns 0. This is the per-key traversal primitive +// that follows the fold step in followAndUpdate. +// +// Extracted as a primitive so future orchestrators can drive +// unfold-only traversals (e.g. cache populators that walk a touched-key +// path to fill cell state without doing a fold/update). HashSort's +// per-key followAndUpdate is the sequential orchestrator over this +// primitive today; future concurrent / parallel orchestrators (each +// with their own HexPatriciaHashed instance) can reuse it. +// +// plainKey is used only for per-key metrics labelling (StartUnfolding). +// Pass an empty/nil slice if no metric attribution is needed. +func (hph *HexPatriciaHashed) unfoldKeyPath(hashedKey, plainKey []byte) error { + for unfolding := hph.needUnfolding(hashedKey); unfolding > 0; unfolding = hph.needUnfolding(hashedKey) { + printLater := hph.currentKeyLen == 0 && hph.mounted && hph.trace + var unfoldDone func() + if dbg.KVReadLevelledMetrics { + unfoldDone = hph.metrics.StartUnfolding(plainKey) + } + if err := hph.unfold(hashedKey, unfolding); err != nil { + return fmt.Errorf("unfold: %w", err) + } + if unfoldDone != nil { + unfoldDone() + } + if printLater { + fmt.Printf("[%x] subtrie pref '%x' d=%d\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hph.depths[max(0, hph.activeRows-1)]) + } + } + return nil +} + func (hph *HexPatriciaHashed) followAndUpdate(hashedKey, plainKey []byte, stateUpdate *Update) (err error) { //if hph.trace { // fmt.Printf("mnt: %0x current: %x path %x\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hashedKey) @@ -2521,23 +2555,9 @@ func (hph *HexPatriciaHashed) followAndUpdate(hashedKey, plainKey []byte, stateU foldDone() } } - // Now unfold until we step on an empty cell - for unfolding := hph.needUnfolding(hashedKey); unfolding > 0; unfolding = hph.needUnfolding(hashedKey) { - printLater := hph.currentKeyLen == 0 && hph.mounted && hph.trace - var unfoldDone func() - if dbg.KVReadLevelledMetrics { - unfoldDone = hph.metrics.StartUnfolding(plainKey) - } - if err := hph.unfold(hashedKey, unfolding); err != nil { - return fmt.Errorf("unfold: %w", err) - } - if unfoldDone != nil { - unfoldDone() - } - if printLater { - fmt.Printf("[%x] subtrie pref '%x' d=%d\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hph.depths[max(0, hph.activeRows-1)]) - } - // fmt.Printf("mnt: %0x current: %x path %x\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hashedKey) + // Now unfold the path so the cell at hashedKey is reachable. + if err := hph.unfoldKeyPath(hashedKey, plainKey); err != nil { + return err } if stateUpdate == nil { From 4074bccf75e991ec9014e45d99142841470c6c4e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 07:47:48 +0000 Subject: [PATCH 024/167] commitment: WarmupCache dirty-flag + PutBranchIfClean invariants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the carry-as-is correctness invariants from the PR #19954 investigation as scaffolding on the existing WarmupCache: - branchEntry.dirty atomic.Bool — signals "stale until cleared" - PutBranchIfClean(prefix, data) bool — skips write if entry dirty - MarkBranchDirty(prefix) — mark for later refusal of stale puts These are scaffolding additions; no callsites use them yet. Existing PutBranch unconditionally overwrites and clears any prior dirty flag (creates a fresh entry), preserving today's semantic exactly for existing callers. Why now: the prototype investigation (see agentspecs/commitment-cache-prototype-dev-context.md) found that inline-invalidate-on-write is incompatible with deferred encoding — update-in-place breaks correctness because there's a window between fold (computes hash, holds new state) and encoder (writes encoded bytes) where readers see stale cached bytes. The reth-research (agentspecs/reth-1ggas-research.md §4) calls the dirty-flag pattern out as the design that resolves this without forcing synchronous encoding: the encoder marks the entry dirty BEFORE its own write completes, so any racing read knows to bypass the cache for that key. Today's WarmupCache lifecycle (per-Process, warmup completes before fold begins) does NOT exhibit this race — these invariants are infrastructure for the future cross-block persistence work where warmup-style writers can outlive their parent Process. Tests: - TestWarmupCache_DirtyFlag: PutBranchIfClean refuses dirty entry, unconditional PutBranch clears dirty. - TestWarmupCache_DirtyFlag_MarkAbsentKey: marking absent key is no-op (no panic, no entry created). Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/warmup_cache.go | 49 ++++++++++++++++++++ execution/commitment/warmup_cache_test.go | 56 +++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index 643ae0e1174..219ce925a50 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -26,6 +26,15 @@ import ( type branchEntry struct { data []byte isEvicted atomic.Bool + // dirty signals "the canonical store has been written to since this + // entry was populated; treat as stale until cleared." Read paths + // MAY return miss when dirty is set; write paths MUST set dirty. + // Used together with PutBranchIfClean to prevent late warmup writes + // from clobbering fresh fold writes (TOCTOU race documented in the + // reth-research §4 dirty-flag pattern). Ephemeral cache today does + // not have the race because warmup completes before fold begins; + // invariant is in place ahead of cross-block persistence work. + dirty atomic.Bool } type accountEntry struct { @@ -72,6 +81,46 @@ func (c *WarmupCache) PutBranch(prefix []byte, data []byte) { c.branches.Set(prefix, &branchEntry{data: dataCopy}) } +// PutBranchIfClean stores branch data in the cache only if no existing entry +// is marked dirty. Returns true on store, false if a dirty entry was present +// (indicating the canonical store has been updated since the caller last +// read, and the caller's data is potentially stale). +// +// Use from warmup-style writers that may race with fold writes — the fold +// path marks branches dirty before its own write completes, so a warmup +// worker that reads pre-fold then attempts to write post-fold will see +// dirty=true and skip the store. +// +// Today's call sites (warmup workers in HexPatriciaHashed.Process) do not +// race with fold because warmup completes before HashSort begins. Invariant +// is in place for future cross-block persistence work where warmup-style +// writes can outlive their parent Process. +func (c *WarmupCache) PutBranchIfClean(prefix []byte, data []byte) bool { + if existing, found := c.branches.Get(prefix); found && existing.dirty.Load() { + return false + } + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + c.branches.Set(prefix, &branchEntry{data: dataCopy}) + return true +} + +// MarkBranchDirty flags a branch entry as stale-until-cleared. The next +// PutBranchIfClean for this prefix will skip; reads (GetBranch / +// GetAndEvictBranch) currently return the entry regardless — the dirty +// signal is consumed only on the write path today. +// +// Use from fold/encoder paths that have decided to overwrite a branch but +// haven't yet captured the new bytes. The mark-dirty + later actual-write +// pattern is the deferred-encoding-friendly alternative to inline +// invalidate, motivated by the prototype investigation that found inline +// invalidate breaks update-in-place semantics under deferred encoding. +func (c *WarmupCache) MarkBranchDirty(prefix []byte) { + if entry, found := c.branches.Get(prefix); found { + entry.dirty.Store(true) + } +} + // GetBranch retrieves branch data from the cache. func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go index a52fd85cbff..9a6d50da17d 100644 --- a/execution/commitment/warmup_cache_test.go +++ b/execution/commitment/warmup_cache_test.go @@ -20,6 +20,8 @@ import ( "bytes" "crypto/rand" "testing" + + "github.com/stretchr/testify/require" ) // TestWarmupCache_Basic tests basic put/get operations @@ -392,3 +394,57 @@ func TestWarmupCache_Stats(t *testing.T) { t.Fatal("account data should survive ResetStats") } } + +// TestWarmupCache_DirtyFlag verifies the dirty flag scaffolding — +// MarkBranchDirty + PutBranchIfClean — that prevents late writers from +// clobbering fresh data once cross-block persistence is wired up. +func TestWarmupCache_DirtyFlag(t *testing.T) { + cache := NewWarmupCache() + key := []byte("k-branch") + v1 := []byte("first-value") + v2 := []byte("second-value-skipped") + + // First Put always succeeds (no existing entry to be dirty). + require.True(t, cache.PutBranchIfClean(key, v1)) + got, ok := cache.GetBranch(key) + require.True(t, ok) + require.Equal(t, v1, got) + + // Marking dirty doesn't remove the entry — Get still returns it. + cache.MarkBranchDirty(key) + got, ok = cache.GetBranch(key) + require.True(t, ok, "MarkBranchDirty should not evict; consumers decide what to do with dirty data") + require.Equal(t, v1, got) + + // PutBranchIfClean refuses to overwrite a dirty entry. + require.False(t, cache.PutBranchIfClean(key, v2), + "PutBranchIfClean must skip when entry is dirty") + got, ok = cache.GetBranch(key) + require.True(t, ok) + require.Equal(t, v1, got, "dirty value must be preserved (write was rejected)") + + // PutBranch (non-If-Clean variant) overwrites unconditionally — used + // by the fold encoder path that holds the canonical new value. + cache.PutBranch(key, v2) + got, ok = cache.GetBranch(key) + require.True(t, ok) + require.Equal(t, v2, got) + + // After unconditional overwrite, the new entry is no longer dirty + // (fresh entry replaces the dirty one). + require.True(t, cache.PutBranchIfClean(key, []byte("third-value")), + "PutBranchIfClean must accept after unconditional Put cleared the dirty entry") +} + +// TestWarmupCache_DirtyFlag_MarkAbsentKey verifies that marking a key +// dirty when no entry exists is a no-op (no panic, no entry created). +func TestWarmupCache_DirtyFlag_MarkAbsentKey(t *testing.T) { + cache := NewWarmupCache() + cache.MarkBranchDirty([]byte("never-stored")) + _, ok := cache.GetBranch([]byte("never-stored")) + require.False(t, ok, "MarkBranchDirty on absent key should not create an entry") + + // PutBranchIfClean for the same absent key should succeed (no dirty + // entry exists, so the write proceeds). + require.True(t, cache.PutBranchIfClean([]byte("never-stored"), []byte("v"))) +} From 4be9c86e4930f240cd53165caeec269217d90ff0 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 07:53:25 +0000 Subject: [PATCH 025/167] =?UTF-8?q?commitment:=20WarmupCache=20GetBranchDe?= =?UTF-8?q?coded=20=E2=80=94=20lazy-decode=20read=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an additive read method that returns cached branches in already- decoded form, lazy-decoding on first decoded-read per entry and caching the parsed cells for subsequent reads. No existing callsite changes; existing GetBranch / GetAndEvictBranch / PutBranch callers continue to work with encoded bytes unchanged. Why now: this is step 5 of the representation-reduction sequence (see agentspecs/trie-data-pipeline-complexity-tax.md). The trie's read path currently does GetBranch (encoded) + DecodeBranchInto on every cache hit — paying decode CPU on every read. Switching that callsite to GetBranchDecoded (in a separate later commit) eliminates the redundant decode. The ENCODED form remains the source of truth — the encoder needs it for the merge-with-prev step, and it's what gets written to disk via PutBranch. The decoded form is derived lazily and cached alongside the entry. When PutBranch overwrites an entry, the new entry starts fresh and the next decoded read re-derives from the new bytes. API design notes: - Returns (bitmap, *[16]cell, ok). Caller derives touchMap/afterMap from bitmap based on its own deleted-vs-present-after context — the cache stores cells independent of that context so the same entry serves both readers. - The returned *[16]cell aliases entry-owned storage. Read-only consumption is safe across concurrent calls (decode runs at most once per entry via sync.Once); MUST NOT be modified in place. - Decode error → ok=false (don't count as hit OR miss; caller falls through to canonical re-read). Tests: - TestWarmupCache_GetBranchDecoded: round-trip equality with direct DecodeBranchInto, plus same-pointer reuse on repeat reads. - TestWarmupCache_GetBranchDecoded_Miss: behaves like GetBranch on absent keys. - TestWarmupCache_GetBranchDecoded_TruncatedData: graceful failure on corrupt entry (no panic). Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/warmup_cache.go | 76 ++++++++++++++++++++++- execution/commitment/warmup_cache_test.go | 73 ++++++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index 219ce925a50..800f3fad5b4 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -18,13 +18,31 @@ package commitment import ( "fmt" + "sync" "sync/atomic" "github.com/erigontech/erigon/common/maphash" ) type branchEntry struct { - data []byte + // data is the canonical encoded form (with the leading 2-byte touch-map + // prefix). Always populated by PutBranch / PutBranchIfClean. + data []byte + + // Lazy-decoded form. Populated on the first GetBranchDecoded call for + // this entry; subsequent calls return the cached decode. decodeOnce + // ensures decode runs at most once per entry even under concurrent + // reads. + // + // Encoded form is the source of truth; decoded form is derived. When + // data is replaced (PutBranch overwrites), the new entry starts fresh + // — next decoded read re-derives from the new bytes. + decodeOnce sync.Once + cells [16]cell + cellsBitmap uint16 + decodedReady bool + decodeErr error + isEvicted atomic.Bool // dirty signals "the canonical store has been written to since this // entry was populated; treat as stale until cleared." Read paths @@ -137,6 +155,62 @@ func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { return entry.data, true } +// GetBranchDecoded retrieves the cached branch in decoded form. Lazy-decodes +// on first access for each entry; subsequent reads return the cached cells +// pointer without redoing the parse work. +// +// Returns the bitmap of present children plus a pointer to the populated +// cells array. The caller derives touchMap/afterMap from the bitmap based +// on its own context (deleted vs present-after) — the cache stores cells +// independent of that context so the same entry serves both readers. +// +// Returns ok=false on miss or eviction (same semantics as GetBranch). Also +// returns ok=false if the cached bytes fail to decode — in that case the +// caller should fall through to a re-read from the canonical store. +// +// The returned *[16]cell pointer aliases storage owned by the cache entry +// — the caller MUST NOT modify the cells in place. Read-only consumption +// is safe across concurrent GetBranchDecoded calls (decode runs at most +// once per entry; subsequent reads see the populated cells). +func (c *WarmupCache) GetBranchDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { + entry, found := c.branches.Get(prefix) + if !found { + c.branchMisses.Add(1) + return 0, nil, false + } + if entry.isEvicted.Load() { + c.branchEvicted.Add(1) + return 0, nil, false + } + entry.decodeOnce.Do(func() { + // PutBranch stores bytes WITH the leading 2-byte touch-map prefix; + // DecodeBranchInto consumes bytes WITHOUT it (matching the + // unfoldBranchNode call pattern that strips the prefix before + // decoding). + if len(entry.data) < 2 { + entry.decodeErr = fmt.Errorf("branch entry too short for touch-map prefix: %d bytes", len(entry.data)) + return + } + maps, err := DecodeBranchInto(entry.data[2:], false /* deleted derived per-caller */, &entry.cells) + if err != nil { + entry.decodeErr = err + return + } + entry.cellsBitmap = maps.Bitmap + entry.decodedReady = true + }) + if !entry.decodedReady { + // Decode failed — caller should re-read from canonical store. + // Don't count as hit (the entry is mechanically present but + // unusable). Don't increment misses either — the caller will + // observe ok=false and decide. + return 0, nil, false + } + c.branchHits.Add(1) + c.branchBytesServed.Add(uint64(len(entry.data))) + return entry.cellsBitmap, &entry.cells, true +} + // GetAndEvictBranch retrieves branch data and marks the entry as evicted in one operation. func (c *WarmupCache) GetAndEvictBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go index 9a6d50da17d..db62ec6ceb2 100644 --- a/execution/commitment/warmup_cache_test.go +++ b/execution/commitment/warmup_cache_test.go @@ -448,3 +448,76 @@ func TestWarmupCache_DirtyFlag_MarkAbsentKey(t *testing.T) { // entry exists, so the write proceeds). require.True(t, cache.PutBranchIfClean([]byte("never-stored"), []byte("v"))) } + +// TestWarmupCache_GetBranchDecoded verifies the lazy-decode read path: +// stored encoded bytes are decoded on first decoded-read and cached for +// subsequent reads, returning cells equivalent to direct DecodeBranchInto. +func TestWarmupCache_GetBranchDecoded(t *testing.T) { + cache := NewWarmupCache() + + // Encode a branch the same way BranchEncoder would, so our test + // data has the canonical [touchMap | bitmap | cells...] layout. + row, bm := generateCellRow(t, 16) + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + + prefix := []byte{0x12, 0x34} + cache.PutBranch(prefix, enc) + + // First decoded-read decodes lazily. + bitmap, cells, ok := cache.GetBranchDecoded(prefix) + require.True(t, ok) + require.Equal(t, bm, bitmap) + require.NotNil(t, cells) + + // Cells should match what direct DecodeBranchInto would produce + // from the same encoded bytes (skip the 2-byte touchMap prefix). + var expected [16]cell + expectedMaps, err := DecodeBranchInto(enc[2:], false, &expected) + require.NoError(t, err) + require.Equal(t, expectedMaps.Bitmap, bitmap) + for i := range cells { + require.Equal(t, expected[i].extLen, cells[i].extLen, "cell %d extLen", i) + require.Equal(t, expected[i].extension[:expected[i].extLen], cells[i].extension[:cells[i].extLen], "cell %d extension", i) + require.Equal(t, expected[i].accountAddr[:expected[i].accountAddrLen], cells[i].accountAddr[:cells[i].accountAddrLen], "cell %d accountAddr", i) + require.Equal(t, expected[i].storageAddr[:expected[i].storageAddrLen], cells[i].storageAddr[:cells[i].storageAddrLen], "cell %d storageAddr", i) + require.Equal(t, expected[i].hash[:expected[i].hashLen], cells[i].hash[:cells[i].hashLen], "cell %d hash", i) + } + + // Second decoded-read returns the SAME cells pointer — lazy decode + // runs at most once per entry. + bitmap2, cells2, ok := cache.GetBranchDecoded(prefix) + require.True(t, ok) + require.Equal(t, bm, bitmap2) + require.Same(t, cells, cells2, "expected cached cells pointer to be reused") + + // Encoded form is unchanged after decoded reads. + encGot, ok := cache.GetBranch(prefix) + require.True(t, ok) + require.Equal(t, []byte(enc), encGot, "encoded form unchanged by decoded reads") +} + +// TestWarmupCache_GetBranchDecoded_Miss verifies that misses on +// GetBranchDecoded behave the same as misses on GetBranch. +func TestWarmupCache_GetBranchDecoded_Miss(t *testing.T) { + cache := NewWarmupCache() + _, _, ok := cache.GetBranchDecoded([]byte("never-stored")) + require.False(t, ok) +} + +// TestWarmupCache_GetBranchDecoded_TruncatedData verifies the decode-error +// path — corrupt entry returns ok=false rather than panicking. +func TestWarmupCache_GetBranchDecoded_TruncatedData(t *testing.T) { + cache := NewWarmupCache() + // One byte is shorter than the touchMap prefix; decode will error. + cache.PutBranch([]byte("k"), []byte{0x42}) + _, _, ok := cache.GetBranchDecoded([]byte("k")) + require.False(t, ok, "truncated entry should return ok=false, not panic") + + // Encoded form still retrievable for callers that don't need decode. + got, ok := cache.GetBranch([]byte("k")) + require.True(t, ok) + require.Equal(t, []byte{0x42}, got) +} From 733dad5385e74946f125734f815257f1d4e374b8 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 17:29:22 +0000 Subject: [PATCH 026/167] execmodule: chain ValidateChain SD to currentContext for head-extending payloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes a timing hole that surfaced once we wired the aggregator-scope BranchCache: between an FCU completing and the next newPayload, MDBX hasn't been committed yet (RunLoop's CommitCycle only fires under memory pressure), but currentContext.mem holds the latest writes from MergeExtendingFork. The fresh doms created in ValidateChain has no parent and a fresh roTx, so its ctx.Branch reads stale-MDBX while the aggregator-scope BranchCache (populated by the prior FV's CollectUpdate writes) holds the fresh state. That's the cross-newPayload divergence pattern observed in the bench (8-61 divergences and wrong-trie-root errors at block 3-14 across runs). Set doms.SetParent(currentContext) when the new payload extends the current canonical head (header.ParentHash == ReadHeadBlockHash). For fork payloads that don't extend head, leave parent unset: unwindToCommonCanonical below reverts doms's view to the common ancestor, and exposing currentContext.mem (post-divergence canonical writes) via the parent chain would shadow the unwound base and break fork validation. Verified by TestReorgsWithInsertChain — the "head-only" predicate is what the current single-canonical-chain SD topology supports. A proper per-branch SD lineage (each fork's validation chains to the last validated SD on its own branch, not always currentContext) is the follow-up needed for concurrent multi-fork validation. The current design supports a single canonical chain only; that's enough to close the divergence we have today, with the lineage extension tracked separately. --- execution/execmodule/exec_module.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 0302fdff491..c11ba373a48 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -484,6 +484,29 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() + // Chain the validation SD to currentContext when the new payload + // extends the current canonical head. Closes a timing hole: FCU's + // MergeExtendingFork moves the previous fork's writes into + // currentContext.mem, but MDBX commit only fires from RunLoop's + // CommitCycle on memory pressure. Between FCU and the next + // newPayload, currentContext.mem holds the latest state and MDBX + // is stale. Without the parent link, this fresh doms reads + // stale-MDBX while the aggregator-scope BranchCache (populated by + // the prior FV's CollectUpdate writes) holds the fresh state, + // producing divergence and wrong-trie-root downstream. + // + // Only set parent for head-extending payloads — fork validation + // requires unwindToCommonCanonical below to revert doms's view to + // the common ancestor, and exposing currentContext.mem (which holds + // post-divergence canonical writes) via the parent chain would + // shadow the unwound base. The current SD topology supports a + // single canonical chain only; per-branch SD lineage for concurrent + // multi-fork validation is a separate architectural follow-up. + canonicalHead := rawdb.ReadHeadBlockHash(tx) + if header.ParentHash == canonicalHead && e.currentContext != nil { + doms.SetParent(e.currentContext) + } + // Flush block overlay data (headers, bodies, TDs from InsertBlocks) into // the validation overlay so unwindToCommonCanonical and ValidatePayload — // and the parallel exec goroutine via NewReadView — see this block data. From a823097ae71e75222ae1ace0caa8deb0c88b82bd Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 17:02:30 +0000 Subject: [PATCH 027/167] db/state: H0 baseline benchmark for snapshot-vs-MDBX read cost Foundation for the "Snapshot vs MDBX read-cost equivalence" investigation (memory: snapshot-vs-mdbx-performance-equivalence.md). This file produces the headline ratio that quantifies the gap the investigation aims to close: warm-cache reads from snapshot .kv files should cost the same as warm-cache reads from MDBX (same disk, same page cache). H0 measures how far apart they are today. Five sub-benches: - MDBX_path full chain, key in MDBX - File_path full chain, key in file - Forced_file_path file-only debug path, file-resident keys - Forced_db_path DB-only debug path, MDBX-resident keys - Bloom_miss_path file-only debug path, MDBX-resident keys (file misses in xorfilter for every probe) Two operating modes; only synthetic is wired in this commit: - Synthetic (testDbAndAggregatorBench fixture): writes 64 full 16-tx steps, BuildFiles + repeated PruneSmallBatches drains all but the tip step into files. Phase 2 keys at txNums past the built-step boundary stay in MDBX. Partition by *actual* residency after setup so bench inputs match where keys really live. - Real-datadir (--snapdatadir flag): TODO. Opens an existing chaindata+snapshots datadir read-only and picks keys via cursor iteration / .kv decompressor walk. Required for production- relevant numbers since synthetic has tiny files and small values. Initial synthetic results on AMD EPYC 4244P (Accounts domain): MDBX_path 173 ns/op File_path 226 ns/op (1.31x MDBX) Forced_file_path 30 ns/op Forced_db_path 158 ns/op Bloom_miss_path 30 ns/op Synthetic dataset is too small to surface the production gap that pprof shows (xorfilter at 35% CPU on real bloat workload). H1-H4 benches and the real-datadir mode are the next steps. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/snapshot_vs_mdbx_bench_test.go | 409 ++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 db/state/snapshot_vs_mdbx_bench_test.go diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go new file mode 100644 index 00000000000..8d431a046df --- /dev/null +++ b/db/state/snapshot_vs_mdbx_bench_test.go @@ -0,0 +1,409 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package state_test + +// Benchmarks for the "Snapshot vs MDBX read-cost equivalence" +// investigation. See memory: snapshot-vs-mdbx-performance-equivalence.md +// +// Premise: with the OS page cache warm, a key lookup served from +// MDBX vs from a snapshot .kv file should cost roughly the same — the +// disk and page cache are identical. Today they don't. These benches +// quantify the gap (H0) and decompose it (H1-H4). +// +// **Status: SCAFFOLD.** Compiles and runs (skipped) so the structure +// survives. Real-datadir bootstrap and synthetic-fixture bootstrap are +// the two TODOs that turn this into a measurement tool. +// +// Two operating modes are anticipated: +// +// 1. **Synthetic mode** (preferred for CI / reproducibility): +// use testDbAndAggregatorBench, write K1 keys at low txNums, +// buildFiles+prune so they land in files, write K2 at high +// txNums and leave in MDBX. No external datadir required. +// +// 2. **Real-datadir mode** (preferred for fidelity): +// open an existing perf-devnet-3 / mainnet datadir read-only, +// pick keys by cursor-iterating the values table (mdbxKeys) and +// walking the latest .kv decompressor (fileKeys). Matches +// production file-count, Bloom false-positive rate, etc. +// +// Both fail with b.Skip() in this scaffold; the TODOs below are the +// agenda for turning it into a real measurement. + +import ( + "encoding/binary" + "flag" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/length" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state" + "github.com/erigontech/erigon/db/state/execctx" +) + +var snapDataDir = flag.String("snapdatadir", "", + "path to a real datadir (chaindata + snapshots) for snapshot-vs-mdbx benches; "+ + "when empty, real-datadir benches are skipped and synthetic mode runs") + +// snapBenchSetup carries everything a sub-bench needs: an open RO tx, +// the AggregatorRoTx for Debug* paths, the domain under test, and the +// pre-picked key sets. Set up once per Benchmark function so warmup +// and timing happen on the same data. +type snapBenchSetup struct { + tx kv.TemporalTx + aggTx *state.AggregatorRoTx + domain kv.Domain + mdbxKeys [][]byte + fileKeys [][]byte +} + +// openSnapBench builds a snapBenchSetup for the given domain. Picks +// real-datadir mode iff --snapdatadir is set, otherwise synthetic. +// Both paths currently call b.Skip; the TODOs below are the work. +func openSnapBench(b *testing.B, domain kv.Domain) *snapBenchSetup { + b.Helper() + if *snapDataDir != "" { + return openSnapBenchRealDatadir(b, domain, *snapDataDir) + } + return openSnapBenchSynthetic(b, domain) +} + +// openSnapBenchRealDatadir opens an existing chaindata+snapshots +// datadir read-only and picks keys from production files. +// +// TODO: wire this up. Open via the same path turbo/node uses: open +// MDBX RwDB, open *state.Aggregator, wrap via temporal.New, begin RO +// tx, type-assert tx.AggTx() to *AggregatorRoTx. Then call pickKeys. +func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadir string) *snapBenchSetup { + b.Helper() + b.Skip("TODO: real-datadir bootstrap (open MDBX + Aggregator + temporal.DB " + + "the same way cmd/erigon does, then BeginTemporalRo)") + return nil +} + +// openSnapBenchSynthetic builds an in-memory aggregator with a known +// key distribution and returns a setup whose two key sets are +// guaranteed to live on the file path and the MDBX path respectively. +// +// Sequence: +// +// 1. testDbAndAggregatorBench builds a fresh agg + tempdir. +// 2. Phase 1 — write keysetSize "old" keys at txNums spanning steps +// [0..stepsToFlush-1], deterministically generated from seed +// phase1Seed so the same bench reproduces. +// 3. Flush + commit + agg.BuildFiles(stepsToFlush*aggStep) so those +// keys land in .kv files. PruneSmallBatches removes them from +// MDBX. After this, phase-1 keys are file-resident. +// 4. Phase 2 — write keysetSize "new" keys at txNums spanning steps +// [stepsToFlush..stepsToFlush+1], from disjoint phase2Seed so no +// overlap with phase 1. NO BuildFiles for these. They stay in +// MDBX. +// 5. Begin a RO temporal tx, return setup. +// +// We verify each key actually lives where we expect via +// DebugGetLatestFromDB / DebugGetLatestFromFiles — fail loudly if a +// key isn't where the test plan says it should be (catches setup +// regressions). +func openSnapBenchSynthetic(b *testing.B, domain kv.Domain) *snapBenchSetup { + b.Helper() + + const ( + // aggStep small + many full steps mirrors how the fuzz test sets + // up data. BuildFiles only builds completed steps, so phase-1 has + // to span an integer number of full steps. + aggStep = 16 + stepsToFlush = 64 // phase 1 spans this many full steps + keysetSize = stepsToFlush * aggStep // = 1024; one key per txNum, every step is full + phase1Seed = 0xC0FFEE + phase2Seed = 0xDECAFB + // phase-2 starts well past the built step boundary so its keys + // can't be lumped into a future build. + phase2BaseTxNum = uint64(stepsToFlush) * aggStep + ) + + // keySize varies per domain; values are 8-byte counters (txNum + // encoded big-endian) so put/lookup paths are exercised. + var keySize int + switch domain { + case kv.AccountsDomain: + keySize = length.Addr + case kv.StorageDomain: + keySize = length.Addr + length.Hash + default: + b.Fatalf("openSnapBenchSynthetic: domain %v not yet wired", domain) + } + + db, agg := testDbAndAggregatorBench(b, aggStep) + logger := log.New() + + // ---- Phase 1: keys destined for files. Each key gets its own + // txNum so per-tx history entries are well-formed. + phase1Keys := genKeys(phase1Seed, keysetSize, keySize) + + { + rwTx, err := db.BeginTemporalRw(b.Context()) + require.NoError(b, err) + + domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) + require.NoError(b, err) + + val := make([]byte, 8) + for i, k := range phase1Keys { + // One key per txNum, packing every txNum in the + // stepsToFlush * aggStep range exactly once. This guarantees + // every step is full so BuildFiles emits a .kv per step. + txNum := uint64(i) + binary.BigEndian.PutUint64(val, txNum+1) // +1 so all values are non-zero + err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) + require.NoError(b, err) + } + + // ComputeCommitment writes the trie root into the CommitmentDomain + // so a follow-up NewSharedDomains can SeekCommitment without + // erroring "commitment state out of date". + lastTxNum := uint64(keysetSize - 1) + _, err = domains.ComputeCommitment(b.Context(), rwTx, true /*save*/, 0, lastTxNum, "h0-bench", nil) + require.NoError(b, err) + + require.NoError(b, domains.Flush(b.Context(), rwTx)) + domains.Close() + require.NoError(b, rwTx.Commit()) + } + + // Build files for the phase-1 range. + require.NoError(b, agg.BuildFiles(uint64(stepsToFlush)*aggStep)) + + // Prune phase-1 entries out of MDBX so they're file-only. + // PruneSmallBatches may report haveMore=true when batch limits stop + // it short of fully draining; loop until drained or we hit a sane + // safety bound. time.Hour keeps it in "furious" prune mode (large + // per-iteration limit), so this is fast in practice. + for round := 0; round < 32; round++ { + rwTx, err := db.BeginTemporalRw(b.Context()) + require.NoError(b, err) + haveMore, err := rwTx.PruneSmallBatches(b.Context(), time.Hour) + require.NoError(b, err) + require.NoError(b, rwTx.Commit()) + if !haveMore { + break + } + } + + // ---- Phase 2: keys destined for MDBX (no BuildFiles after). + phase2Keys := genKeys(phase2Seed, keysetSize, keySize) + + { + rwTx, err := db.BeginTemporalRw(b.Context()) + require.NoError(b, err) + + domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) + require.NoError(b, err) + + val := make([]byte, 8) + for i, k := range phase2Keys { + // Phase-2 keys live well past the built-step boundary so a + // hypothetical follow-up BuildFiles wouldn't sweep them in. + txNum := phase2BaseTxNum + uint64(i) + binary.BigEndian.PutUint64(val, txNum+1) + err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) + require.NoError(b, err) + } + + require.NoError(b, domains.Flush(b.Context(), rwTx)) + domains.Close() + require.NoError(b, rwTx.Commit()) + } + + // ---- Open a RO tx for benching. + tx, err := db.BeginTemporalRo(b.Context()) + require.NoError(b, err) + b.Cleanup(func() { tx.Rollback() }) + + aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) + require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") + + // Partition each phase set by actual residency. We expect: + // phase 1 -> almost all "file", with a tail of tip-step keys still + // in MDBX because Prune retains the most-recent step; + // phase 2 -> all "mdbx" (no BuildFiles called for it). + // The bench uses only the keys that landed where we want, so any + // misroute just shrinks the keyset rather than corrupting timings. + mdbxFromP2, fileFromP2, missingP2 := partitionByResidency(b, aggTx, tx, domain, phase2Keys) + mdbxFromP1, fileFromP1, missingP1 := partitionByResidency(b, aggTx, tx, domain, phase1Keys) + if missingP1 > 0 || missingP2 > 0 { + b.Fatalf("residency partition: missing keys p1=%d p2=%d", missingP1, missingP2) + } + b.Logf("residency: phase1 file=%d mdbx=%d phase2 file=%d mdbx=%d", + len(fileFromP1), len(mdbxFromP1), len(fileFromP2), len(mdbxFromP2)) + + mdbxKeys := mdbxFromP2 // bench's mdbx-resident set + fileKeys := fileFromP1 // bench's file-resident set + require.NotEmpty(b, mdbxKeys, "synthetic fixture produced no MDBX-resident keys") + require.NotEmpty(b, fileKeys, "synthetic fixture produced no file-resident keys") + + return &snapBenchSetup{ + tx: tx, + aggTx: aggTx, + domain: domain, + mdbxKeys: mdbxKeys, + fileKeys: fileKeys, + } +} + +// genKeys produces n deterministic keys of size keySize bytes from +// the given seed. Same seed -> same keys -> reproducible benches. +func genKeys(seed uint64, n, keySize int) [][]byte { + rnd := newRnd(seed) + keys := make([][]byte, n) + for i := range keys { + k := make([]byte, keySize) + _, _ = rnd.Read(k) + keys[i] = k + } + return keys +} + +// partitionByResidency walks keys and groups them by which read path +// returns the value: "mdbx" (DebugGetLatestFromDB ok), "file" +// (DebugGetLatestFromDB miss + DebugGetLatestFromFiles ok), or +// "missing" (neither). missing is reported as a count for the caller +// to fail on — no key in our synthetic setup should be unreachable. +func partitionByResidency(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, keys [][]byte) (mdbxKeys, fileKeys [][]byte, missing int) { + b.Helper() + for _, k := range keys { + _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, k, tx) + require.NoError(b, err) + if dbOk { + mdbxKeys = append(mdbxKeys, k) + continue + } + _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, k, 0) + require.NoError(b, err) + if fileOk { + fileKeys = append(fileKeys, k) + continue + } + missing++ + } + return +} + +// warmup reads every key in keys via tx.GetLatest so the OS page cache +// is hot for all relevant snapshot/MDBX pages before timing starts. +// Required because the H0 claim only holds for warm pages — we want to +// measure software overhead, not disk I/O. +func warmup(tx kv.TemporalTx, domain kv.Domain, keys [][]byte) { + for _, k := range keys { + _, _, _ = tx.GetLatest(domain, k) + } +} + +// runH0 is the H0 measurement body. Four sub-benches: +// +// - MDBX_path: tx.GetLatest with key in MDBX. Hits getLatestFromDb +// and returns; never touches files. Baseline cost. +// +// - File_path: tx.GetLatest with key only in files. Misses MDBX, +// then walks getLatestFromFiles (Bloom -> recsplit -> seg +// decompress). Baseline-with-files cost. +// +// - Forced_file_path: same MDBX-resident key set as MDBX_path but +// routed through DebugGetLatestFromFiles, forcing the file path +// even though the key would have hit MDBX. Isolates per-key +// file-path cost from key-distribution effects. +// +// - Forced_db_path: DebugGetLatestFromDB on the MDBX key set — +// mirror of Forced_file_path, isolates the MDBX-only path cost. +// +// Headline: file_ns_per_op / mdbx_ns_per_op. Forced_* sub-benches +// confirm the ratio isn't a key-set artifact. +func runH0(b *testing.B, setup *snapBenchSetup) { + b.Helper() + + // Two warmup passes: first lands pages in the OS page cache, + // second absorbs L3/TLB effects so we time steady-state. + warmup(setup.tx, setup.domain, setup.mdbxKeys) + warmup(setup.tx, setup.domain, setup.fileKeys) + warmup(setup.tx, setup.domain, setup.mdbxKeys) + warmup(setup.tx, setup.domain, setup.fileKeys) + + b.Run("MDBX_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetLatest(setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)]) + } + }) + + b.Run("File_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) + } + }) + + // Forced_file_path: file-resident keys via the file-only debug path. + // Strips MDBX-miss cost from File_path so we see pure file-side + // work (Bloom -> recsplit -> seg decompress). + b.Run("Forced_file_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( + setup.domain, setup.fileKeys[i%len(setup.fileKeys)], 0) + } + }) + + // Forced_db_path: MDBX-resident keys via the DB-only debug path. + // Strips routing/dispatch overhead from MDBX_path so we see pure + // MDBX cursor work. + b.Run("Forced_db_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _, _ = setup.aggTx.DebugGetLatestFromDB( + setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], setup.tx) + } + }) + + // Bloom_miss_path: MDBX-resident keys against the file-only debug + // path. These keys are NOT in any .kv file so each lookup is a + // pure xorfilter "not present" probe per file. Gives the Bloom-miss + // cost in isolation — useful for H1 (per-file Bloom probe overhead). + b.Run("Bloom_miss_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( + setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], 0) + } + }) +} + +// BenchmarkSnapVsMDBX_H0_Accounts measures the warm-cache file-vs-MDBX +// per-key gap on the AccountsDomain. Headline: file_ns/op : mdbx_ns/op. +func BenchmarkSnapVsMDBX_H0_Accounts(b *testing.B) { + runH0(b, openSnapBench(b, kv.AccountsDomain)) +} + +// BenchmarkSnapVsMDBX_H0_Storage — same harness for StorageDomain. +// Storage dominates file-read traffic on bloat workloads (per +// runs-step9-cache-behind-sd memory), so getting the gap for both +// Accounts and Storage is the minimum useful H0 output. +func BenchmarkSnapVsMDBX_H0_Storage(b *testing.B) { + runH0(b, openSnapBench(b, kv.StorageDomain)) +} From 118298ac94d83e3ebd66fc451beb3f82fe6dc89d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 17:15:44 +0000 Subject: [PATCH 028/167] db/state: H0 real-datadir bootstrap; first production numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the --snapdatadir flag path to the H0 bench. Opens an existing chaindata + snapshots datadir read-only via the same recipe as cmd/integration (mdbx Accede + state.New + temporal.New) and picks keys by cursor-walking the per-domain values table. Pragmatic adjustments: - On heavily-pruned production datadirs (perf-devnet-3-run was 100% pruned), every MDBX values-table row is step-shadowed by a file, so getLatestFromDb returns ok=false. The MDBX-side sub-benches skip in this case; File_path numbers stand on their own and the synthetic MDBX_path baseline serves as the cross- mode comparator. - skipIfEmpty short-circuits per sub-bench rather than failing the whole run, so we can still get the file-path numbers. First production numbers (AMD EPYC 4244P, AccountsDomain, 2012 file-resident keys from perf-devnet-3-run, fully pruned): File_path 211 ns/op (synthetic was 226; essentially same) Forced_file_path 30 ns/op (synthetic was 30; identical) Surprising finding: real .kv file reads cost the same as synthetic. This means production bloat-workload bottleneck is NOT in getLatestFromFiles — it must be in HistorySeek (.ef history files walked by HistoryStateReader.GetAsOf). The GetAsOf shortcut work flagged in getasof-regression-suspect.md is the right lead. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/snapshot_vs_mdbx_bench_test.go | 212 +++++++++++++++++++++++- 1 file changed, 205 insertions(+), 7 deletions(-) diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go index 8d431a046df..c838a1a8d4c 100644 --- a/db/state/snapshot_vs_mdbx_bench_test.go +++ b/db/state/snapshot_vs_mdbx_bench_test.go @@ -54,7 +54,11 @@ import ( "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/mdbx" + "github.com/erigontech/erigon/db/kv/temporal" "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/db/state/execctx" ) @@ -89,14 +93,182 @@ func openSnapBench(b *testing.B, domain kv.Domain) *snapBenchSetup { // openSnapBenchRealDatadir opens an existing chaindata+snapshots // datadir read-only and picks keys from production files. // -// TODO: wire this up. Open via the same path turbo/node uses: open -// MDBX RwDB, open *state.Aggregator, wrap via temporal.New, begin RO -// tx, type-assert tx.AggTx() to *AggregatorRoTx. Then call pickKeys. -func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadir string) *snapBenchSetup { +// Uses MDBX Accede mode (don't create, must already exist) and +// state.New(...).MustOpen which loads the on-disk schema. ResolveDB +// settings reads stepSize / stepsInFrozenFile from the existing DB. +// +// CAUTION: opening a chaindata that another erigon process is writing +// to is risky. Bench against an idle copy. The script that drives +// these benches typically does prepare-run.sh first to snapshot a +// fresh chaindata copy under /erigon-data/perf-devnet-3-run/ or +// similar. Pass that path via --snapdatadir. +func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadirPath string) *snapBenchSetup { + b.Helper() + logger := log.New() + dirs := datadir.New(datadirPath) + + rawDB := mdbx.New(dbcfg.ChainDB, logger). + Path(dirs.Chaindata). + Accede(true). // must exist; do not create + MustOpen() + b.Cleanup(rawDB.Close) + + settings, err := state.ResolveErigonDBSettings(dirs, logger, false /*noDownloader*/) + require.NoError(b, err, "ResolveErigonDBSettings") + + agg := state.New(dirs). + Logger(logger). + WithErigonDBSettings(settings). + MustOpen(b.Context(), rawDB) + require.NoError(b, agg.OpenFolder(), "agg.OpenFolder") + b.Cleanup(agg.Close) + + tdb, err := temporal.New(rawDB, agg) + require.NoError(b, err) + b.Cleanup(tdb.Close) + + tx, err := tdb.BeginTemporalRo(b.Context()) + require.NoError(b, err) + b.Cleanup(func() { tx.Rollback() }) + + aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) + require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") + + mdbxKeys, fileKeys := pickRealDatadirKeys(b, aggTx, tx, domain, 10_000) + require.NotEmpty(b, fileKeys, "no file-resident keys found in %s for domain %v", datadirPath, domain) + if len(mdbxKeys) == 0 { + // Heavily pruned production datadir — every MDBX row is + // step-shadowed by a file. The MDBX-side sub-benches will skip + // (runH0 short-circuits on empty key set). Compare File_path + // against the synthetic MDBX_path baseline; clearly note the + // cross-mode caveat in the issue write-up. + b.Logf("real-datadir: no MDBX-resident keys (datadir is fully pruned). " + + "File_path numbers are real; cross-reference MDBX_path against the synthetic bench.") + } + b.Logf("real-datadir keys: mdbx=%d file=%d (domain=%v, path=%s)", + len(mdbxKeys), len(fileKeys), domain, datadirPath) + + return &snapBenchSetup{ + tx: tx, + aggTx: aggTx, + domain: domain, + mdbxKeys: mdbxKeys, + fileKeys: fileKeys, + } +} + +// pickRealDatadirKeys walks the existing datadir to find up to +// `target` MDBX-resident and `target` file-resident keys for the +// given domain. +// +// Strategy A — cursor-walk the per-domain values table for +// MDBX-resident keys (those rows ARE in MDBX by definition). Strip +// the trailing 8-byte inverted-step from each row's key. +// +// Strategy B — for file-resident keys, randomly sample address-sized +// or storage-key-sized byte strings from the cursor walk's MDBX keys +// (deterministic via seeded RNG): for each candidate, query +// DebugGetLatestFromFiles. If the file path returns ok and MDBX does +// NOT, the key is file-resident. This is approximate — keys that +// exist BOTH in MDBX and files (during prune transitions) are +// classified as MDBX-resident, which is what we want for bench +// timing accuracy (MDBX always wins on getLatest). +// +// Practical note: in production datadirs the MDBX values table +// usually has fewer rows than the snapshot files combined, so we +// expect mdbxKeys to fill faster than fileKeys. To find file-only +// keys we'd need to walk the .kv files directly via seg.Decompressor +// — that's a follow-on once Strategy B is shown to be insufficient. +func pickRealDatadirKeys(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, target int) (mdbxKeys, fileKeys [][]byte) { + b.Helper() + table := domainValsTable(b, domain) + + c, err := tx.Cursor(table) + require.NoError(b, err) + defer c.Close() + + const stepSuffixLen = 8 // values-table key = realKey || invertedStep + seen := make(map[string]struct{}) + var rowsScanned, dbOkCount int + + // First pass: collect MDBX-resident keys directly from the values table. + for k, _, err := c.First(); k != nil; k, _, err = c.Next() { + require.NoError(b, err) + rowsScanned++ + if len(k) <= stepSuffixLen { + continue // unexpected; skip + } + realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) + ks := string(realKey) + if _, dup := seen[ks]; dup { + continue + } + seen[ks] = struct{}{} + + // Confirm MDBX residency via DebugGetLatestFromDB so we don't + // include keys that, despite having a row here, don't actually + // resolve in the read path (edge cases around step bounds). + _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) + require.NoError(b, err) + if dbOk { + dbOkCount++ + mdbxKeys = append(mdbxKeys, realKey) + if len(mdbxKeys) >= target { + break + } + } + } + b.Logf("first-pass scan: table=%s rows=%d unique_keys=%d dbOk=%d", + table, rowsScanned, len(seen), dbOkCount) + + // Second pass: walk the same cursor again looking for keys that + // MISS MDBX but HIT files. Iterating the values table catches + // keys that left a row but have all data in files (delete + // markers, post-prune residue), which is approximate but cheap. + // For a full file-only walk we'd open the .kv decompressor; this + // is good enough for the H0 first cut. + for k, _, err := c.First(); k != nil; k, _, err = c.Next() { + if len(fileKeys) >= target { + break + } + require.NoError(b, err) + if len(k) <= stepSuffixLen { + continue + } + realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) + + _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) + require.NoError(b, err) + if dbOk { + continue // MDBX already serves this; not file-only + } + _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, realKey, 0) + require.NoError(b, err) + if fileOk { + fileKeys = append(fileKeys, realKey) + } + } + + return mdbxKeys, fileKeys +} + +// domainValsTable returns the MDBX values-table name for the given +// domain. Used by pickRealDatadirKeys to cursor-walk MDBX directly. +func domainValsTable(b *testing.B, domain kv.Domain) string { b.Helper() - b.Skip("TODO: real-datadir bootstrap (open MDBX + Aggregator + temporal.DB " + - "the same way cmd/erigon does, then BeginTemporalRo)") - return nil + switch domain { + case kv.AccountsDomain: + return kv.TblAccountVals + case kv.StorageDomain: + return kv.TblStorageVals + case kv.CodeDomain: + return kv.TblCodeVals + case kv.CommitmentDomain: + return kv.TblCommitmentVals + default: + b.Fatalf("domainValsTable: domain %v not mapped", domain) + return "" + } } // openSnapBenchSynthetic builds an in-memory aggregator with a known @@ -316,6 +488,17 @@ func warmup(tx kv.TemporalTx, domain kv.Domain, keys [][]byte) { } } +// skipIfEmpty short-circuits the calling sub-bench when the key set +// is empty (real-datadir mode often has no MDBX-resident keys after +// aggressive pruning). +func skipIfEmpty(b *testing.B, name string, keys [][]byte) bool { + if len(keys) == 0 { + b.Skipf("no keys for %s sub-bench", name) + return true + } + return false +} + // runH0 is the H0 measurement body. Four sub-benches: // // - MDBX_path: tx.GetLatest with key in MDBX. Hits getLatestFromDb @@ -346,6 +529,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { warmup(setup.tx, setup.domain, setup.fileKeys) b.Run("MDBX_path", func(b *testing.B) { + if skipIfEmpty(b, "MDBX_path", setup.mdbxKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _ = setup.tx.GetLatest(setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)]) @@ -353,6 +539,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { }) b.Run("File_path", func(b *testing.B) { + if skipIfEmpty(b, "File_path", setup.fileKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) @@ -363,6 +552,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { // Strips MDBX-miss cost from File_path so we see pure file-side // work (Bloom -> recsplit -> seg decompress). b.Run("Forced_file_path", func(b *testing.B) { + if skipIfEmpty(b, "Forced_file_path", setup.fileKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( @@ -374,6 +566,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { // Strips routing/dispatch overhead from MDBX_path so we see pure // MDBX cursor work. b.Run("Forced_db_path", func(b *testing.B) { + if skipIfEmpty(b, "Forced_db_path", setup.mdbxKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _, _ = setup.aggTx.DebugGetLatestFromDB( @@ -386,6 +581,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { // pure xorfilter "not present" probe per file. Gives the Bloom-miss // cost in isolation — useful for H1 (per-file Bloom probe overhead). b.Run("Bloom_miss_path", func(b *testing.B) { + if skipIfEmpty(b, "Bloom_miss_path", setup.mdbxKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( From 005790f800e159f2b869a05cbfa50fcad162e930 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 21:42:03 +0000 Subject: [PATCH 029/167] db/state, commitment: H_GetAsOf bench + per-block calculator timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related additions to the snapshot-vs-MDBX perf-equivalence investigation (memory: snapshot-vs-mdbx-performance-equivalence.md): 1. H_GetAsOf bench (db/state/snapshot_vs_mdbx_bench_test.go). New runHGetAsOf with four sub-benches for HistorySeek-via-GetAsOf on file-resident keys: GetLatest_baseline, GetAsOf_recent (asOf near endTxNum), GetAsOf_mid (asOf at endTxNum/2), GetAsOf_floor (asOf=1). Tests the path the calculator's HistoryStateReader.Read uses, which is distinct from getLatestFromFiles measured in H0. Real-datadir results on perf-devnet-3 (AccountsDomain, endTxNum=2.9B): GetLatest_baseline 202 ns/op 0 allocs GetAsOf_recent 570 ns/op 0 allocs <- 2.8x baseline, no result GetAsOf_mid 235 ns/op 5 allocs GetAsOf_floor 196 ns/op 4 allocs GetAsOf_recent (the calculator's pattern after PR #21010) scans the .ef looking for a record at-or-after endTxNum-1, finds none (most keys haven't changed in the last txNum), falls through to GetLatest. The 370ns/op overhead vs GetLatest is wasted scan. Confirms the GetAsOf shortcut described in getasof-regression-suspect.md as a real lever, though small in absolute terms (~2ms/block on the bloat workload). 2. Surface "took" + "keys" on the existing [commitment][cache-fp] Info log line (commitmentdb). Was already computed in the debug-level "[commitment] processed" log, but the bench runs with --log.dir.disable so debug logs aren't captured. This made it possible to attribute the 4.3s gap inside newPayload(TEST block) directly: the calculator's ComputeCommitment takes 4220ms for the 5910-key bloat block — 91% of the entire block wall time. Per-key cost is ~700us, consistent across blocks of all sizes. The actual perf lever for the bloat workload is making per-branch ComputeCommitment cheaper, not file/state reads. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/snapshot_vs_mdbx_bench_test.go | 107 ++++++++++++++++++ .../commitmentdb/commitment_context.go | 8 +- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go index c838a1a8d4c..c2efacdcab9 100644 --- a/db/state/snapshot_vs_mdbx_bench_test.go +++ b/db/state/snapshot_vs_mdbx_bench_test.go @@ -598,6 +598,113 @@ func BenchmarkSnapVsMDBX_H0_Accounts(b *testing.B) { runH0(b, openSnapBench(b, kv.AccountsDomain)) } +// runHGetAsOf measures HistorySeek-via-GetAsOf cost on file-resident +// keys. The motivation (per H0 finding): `getLatestFromFiles` is fast +// (~30 ns isolated); the bloat-workload pprof showed real file-read +// pressure that this can't account for. The calculator's +// HistoryStateReader.Read calls tx.GetAsOf, which goes through +// HistorySeek and walks .ef history files — a different code path +// than getLatestFromFiles. +// +// Sub-benches: +// +// - GetLatest_baseline: tx.GetLatest on file-resident keys; mirrors +// H0's File_path so we can confirm the same setup against a fresh +// comparator. +// +// - GetAsOf_recent: tx.GetAsOf at asOfTxNum = endTxNum - 1. Most +// keys won't have a history record post-asOf, so HistorySeek +// short-circuits to the latest path. Lower bound on GetAsOf cost. +// +// - GetAsOf_mid: tx.GetAsOf at asOfTxNum = endTxNum / 2. Forces +// the .ef walk to seek through more history. This is the cost +// shape the calculator pays when reading historic state. +// +// - GetAsOf_zero: tx.GetAsOf at asOfTxNum = HistoryStartFrom() + 1 +// (just past the visible window start). Maximally adversarial — +// full .ef walk depth. The calculator's old asOfReader.txNum=0 +// bug (PR #21010) hit this path when the window started past 0. +// +// All sub-benches use the same fileKeys set so per-key cost is +// directly comparable across the four. +func runHGetAsOf(b *testing.B, setup *snapBenchSetup) { + b.Helper() + if skipIfEmpty(b, "H_GetAsOf", setup.fileKeys) { + return + } + + // Snapshot the visible-history window so all four sub-benches use + // the same anchors and so we report them in the bench log. We + // don't probe historyStartFrom from here (no public accessor on + // AggregatorRoTx for per-domain history range); instead use simple + // fixed fractions of endTxNum. Adjust asOfFloor=1 to avoid + // hitting txNum=0 which can error on snapshot-loaded chains + // (PR #21010 was that bug; we don't want H_GetAsOf to trip on it). + endTxNum := setup.aggTx.EndTxNumNoCommitment() + asOfRecent := endTxNum + if endTxNum > 0 { + asOfRecent = endTxNum - 1 + } + asOfMid := endTxNum / 2 + asOfFloor := uint64(1) + b.Logf("H_GetAsOf anchors: endTxNum=%d -> asOfRecent=%d asOfMid=%d asOfFloor=%d", + endTxNum, asOfRecent, asOfMid, asOfFloor) + + // Two warmup passes so the OS page cache holds the .ef files we + // care about. Touch each anchor txNum so the relevant history + // segments are mmap'd in. + for _, asOf := range []uint64{asOfRecent, asOfMid, asOfFloor} { + for _, k := range setup.fileKeys { + _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) + } + for _, k := range setup.fileKeys { + _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) + } + } + + b.Run("GetLatest_baseline", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) + } + }) + + b.Run("GetAsOf_recent", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfRecent) + } + }) + + b.Run("GetAsOf_mid", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfMid) + } + }) + + b.Run("GetAsOf_floor", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfFloor) + } + }) +} + +// BenchmarkSnapVsMDBX_HGetAsOf_Accounts measures HistorySeek cost on +// real file-resident keys via tx.GetAsOf. Confirms the H0 finding +// that the production bloat-workload bottleneck is in .ef history +// walking, not .kv latest-state reads. +func BenchmarkSnapVsMDBX_HGetAsOf_Accounts(b *testing.B) { + runHGetAsOf(b, openSnapBench(b, kv.AccountsDomain)) +} + +// BenchmarkSnapVsMDBX_HGetAsOf_Storage — same for StorageDomain. +// Storage tends to dominate in the bloat workload. +func BenchmarkSnapVsMDBX_HGetAsOf_Storage(b *testing.B) { + runHGetAsOf(b, openSnapBench(b, kv.StorageDomain)) +} + // BenchmarkSnapVsMDBX_H0_Storage — same harness for StorageDomain. // Storage dominates file-read traffic on bloat workloads (per // runs-step9-cache-behind-sd memory), so getting the gap for both diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 88fab39952c..9f84b34b057 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -344,6 +344,10 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // so two builds running the same workload can be diffed offline to // localise the first block at which their caches diverge. See // commitment.BranchCache.Fingerprint for the hash semantics. + // took + keys exposed at Info so the calculator's per-block compute + // time is visible without enabling debug logs (used by the + // snapshot-vs-mdbx perf-equivalence investigation to attribute + // the gap inside newPayload). if logCacheFingerprint { if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { if bc := hph.BranchCache(); bc != nil { @@ -351,7 +355,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "block", blockNum, "root", hex.EncodeToString(rootHash), "fp", fmt.Sprintf("%016x", bc.Fingerprint()), - "divergences", bc.VerifyDivergences()) + "divergences", bc.VerifyDivergences(), + "took", took, + "keys", common.PrettyCounter(updateCount)) } } } From 4276219f3a9397d21ab289a9a6b4bd7891426d96 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 08:51:40 +0000 Subject: [PATCH 030/167] commitment: surface per-block telemetry on cache-fp log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three groups of fields to the existing [commitment][cache-fp] log line so the calculator's per-block behaviour is observable at Info level (the bench runs with --log.dir.disable, so debug logs aren't captured): - took, keys: ComputeCommitment wall + key-count for the block. Pre-existing internally, now surfaced. - load, skipped, reset: process-cumulative counts of computeCellHash decisions: * load = had no memoized stateHash, fetched value from DB * skipped = had memoized stateHash, reused without fetch * reset = had stateHash but had to invalidate it Surfaced via new commitment.SkipLoadResetCounters(). - files_acc / files_sto / files_code / files_comm: per-domain file-read counts pulled from sd.Metrics().Domains[domain]. Decomposes the aggregate `files=N` from the [domain reads] log line into its actual sources (e.g. on the SSTORE-bloated block the 32k file reads break down as 5.9k Storage value loads + 26.6k Commitment branch reads + a handful of others). All counters are cumulative; per-block deltas are obtained by subtracting consecutive cache-fp lines. Pure observability — no behaviour change. Used as the measurement framework for the snapshot-vs-MDBX perf-equivalence investigation and the follow-on commits that target specific levers. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../commitmentdb/commitment_context.go | 44 ++++++++++++++++++- execution/commitment/hex_patricia_hashed.go | 19 ++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 9f84b34b057..742a92ce0b8 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -21,6 +21,7 @@ import ( "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" + "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/trie" @@ -53,6 +54,13 @@ type sd interface { // to localise which state layer holds bytes that diverge from // cache. Read-only. ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) + + // Metrics exposes the per-SD DomainMetrics so callers can read + // per-domain (cache, db, file) read counters. Used by the + // cache-fp log line to break the aggregate `files=N` count down + // per domain (Storage value loads vs Commitment branch reads + // vs Account loads). + Metrics() *changeset.DomainMetrics } type SharedDomainsCommitmentContext struct { @@ -351,13 +359,47 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context if logCacheFingerprint { if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { if bc := hph.BranchCache(); bc != nil { + // Skip/load/reset counters are process-cumulative; the + // per-block delta requires subtracting consecutive lines. + // Used to answer: "what fraction of computeCellHash calls + // actually fetched the underlying value vs reused a + // memoized stateHash?" + load, skipped, reset := commitment.SkipLoadResetCounters() + // Per-domain file-read counts: lets us decompose the + // aggregate `files=N` from [domain reads] into Commitment + // (branch reads), Storage (value loads), Account, Code. + // All cumulative; deltas via successive lines. + var aFiles, sFiles, cFiles, mFiles int64 + if m := sdc.sharedDomains.Metrics(); m != nil { + m.RLock() + if d := m.Domains[kv.AccountsDomain]; d != nil { + aFiles = d.FileReadCount + } + if d := m.Domains[kv.StorageDomain]; d != nil { + sFiles = d.FileReadCount + } + if d := m.Domains[kv.CodeDomain]; d != nil { + cFiles = d.FileReadCount + } + if d := m.Domains[kv.CommitmentDomain]; d != nil { + mFiles = d.FileReadCount + } + m.RUnlock() + } log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), "fp", fmt.Sprintf("%016x", bc.Fingerprint()), "divergences", bc.VerifyDivergences(), "took", took, - "keys", common.PrettyCounter(updateCount)) + "keys", common.PrettyCounter(updateCount), + "load", load, + "skipped", skipped, + "reset", reset, + "files_acc", aFiles, + "files_sto", sFiles, + "files_code", cFiles, + "files_comm", mFiles) } } } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 3f53e71fa04..5ac89fd8ae9 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1871,6 +1871,25 @@ var ( hadToReset atomic.Uint64 ) +// SkipLoadResetCounters returns the cumulative process-wide counts of: +// - hadToLoad: computeCellHash had no memoized stateHash and had +// to fetch the underlying value from cache/DB to compute the leaf +// hash. This is the real "did we go to disk for state we needed" +// count. +// - skippedLoad: the cell had a memoized stateHash so the fetch was +// skipped entirely (mxTrieStateSkipRate also counts these). +// - hadToReset: the cell HAD a memoized stateHash but it was +// invalidated (extension fold path, depth change). Implies a +// fresh hash compute and may imply a load. +// +// Counters are atomic and process-cumulative. To get per-block +// deltas, snapshot before/after the block. Used by the perf-equivalence +// investigation to measure whether memoization is doing its job or +// whether we're paying for fetches that should have been skipped. +func SkipLoadResetCounters() (load, skipped, reset uint64) { + return hadToLoad.Load(), skippedLoad.Load(), hadToReset.Load() +} + type skipStat struct { accLoaded, accSkipped, accReset, storReset, storLoaded, storSkipped uint64 } From 1e8c25c58768c1ab2556f799957a57c27039aea5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 08:58:47 +0000 Subject: [PATCH 031/167] stagedsync/calc_state: skip wasted ensureStorage in StoragePath ApplyWrites cs.ensureStorage(addr, key) lazy-loads the current value via cs.domainReader.ReadAccountStorage, which goes through GetAsOf and walks .ef history files. On the StoragePath case, the loaded value is *immediately overwritten* on the next line by the EVM-write value (cs.storageState[addr][key] = v). The lazy-load is therefore wasted: a cold .ef seek per first-touched slot, whose result is discarded. On the SSTORE-bloated TEST block (5,910 storage writes), this is ~5,910 wasted GetAsOf calls per block. Replace with bare inner-map initialization so storageState[addr] is non-nil before the assignment. The downstream consumer (FlushToUpdates) only reads the value we just set (the EVM write), so dropping the prior value is correct. Other callers of ensureStorage (Account etc.) are unchanged. Tested: stagedsync test suite passes. Per-block telemetry on the [commitment][cache-fp] log (commit aa424f01fe) will show the 'load' delta drop on benches that hit StoragePath writes. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/stagedsync/calc_state.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/execution/stagedsync/calc_state.go b/execution/stagedsync/calc_state.go index d22781445fe..9a14af64afe 100644 --- a/execution/stagedsync/calc_state.go +++ b/execution/stagedsync/calc_state.go @@ -296,8 +296,19 @@ func (cs *calcState) ApplyWrites(writes state.VersionedWrites) { } case state.StoragePath: v := w.Val.(uint256.Int) - cs.ensureStorage(w.Address, w.Key) // lazy-load if needed - cs.storageState[w.Address][w.Key] = v + // The previous slot value is irrelevant here: the next line + // overwrites it with the EVM-write value, and the only + // downstream consumer (FlushToUpdates) reads exactly the + // value we set. Skip the ensureStorage lazy-load — it + // triggers a cold .ef GetAsOf seek per first-touched slot + // (~5,910 wasted seeks per SSTORE-bloat block) and discards + // the loaded value. Initialize just the inner map. + slots := cs.storageState[w.Address] + if slots == nil { + slots = make(map[accounts.StorageKey]uint256.Int) + cs.storageState[w.Address] = slots + } + slots[w.Key] = v if cs.storageDirty[w.Address] == nil { cs.storageDirty[w.Address] = make(map[accounts.StorageKey]bool) } From a26369503f6ae8301be0f9698fa5deba3be44936 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 09:17:01 +0000 Subject: [PATCH 032/167] commitment: split disk-load counter from EVM-write plumbing setFromUpdate increments hadToLoad on every call, conflating two distinct paths: (1) trie went to *FromCacheOrDB during computeCellHash because it didn't have the value, and (2) the executor plumbed an EVM-write through Updates into the cell. Both are valid uses but only (1) is the disk-fetch we're trying to eliminate. Add diskLoadStorage / diskLoadAccount that increment ONLY at the three *FromCacheOrDB call sites in computeCellHash, and surface them on the [commitment][cache-fp] log line as disk_sto / disk_acc. With this split, the bench output tells us directly whether eliminating the storageFromCacheOrDB path (C3 plumb-EVM-value-to-cell) is worth doing on the bloat workload, or whether the load count is dominated by EVM-write plumbing and the lever is elsewhere. No behaviour change; counters are atomic and gated by the existing BRANCH_CACHE_FINGERPRINT log path. --- .../commitmentdb/commitment_context.go | 4 ++- execution/commitment/hex_patricia_hashed.go | 26 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 742a92ce0b8..c43775fe2d8 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -364,7 +364,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // Used to answer: "what fraction of computeCellHash calls // actually fetched the underlying value vs reused a // memoized stateHash?" - load, skipped, reset := commitment.SkipLoadResetCounters() + load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() // Per-domain file-read counts: lets us decompose the // aggregate `files=N` from [domain reads] into Commitment // (branch reads), Storage (value loads), Account, Code. @@ -396,6 +396,8 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "load", load, "skipped", skipped, "reset", reset, + "disk_sto", diskSto, + "disk_acc", diskAcc, "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 5ac89fd8ae9..1e6080e80a5 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1016,6 +1016,7 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } else { if !cell.loaded.storage() { hph.metrics.StorageLoad(cell.storageAddr[:cell.storageAddrLen]) + diskLoadStorage.Add(1) update, err := hph.storageFromCacheOrDB(cell.storageAddr[:cell.storageAddrLen]) if err != nil { return nil, storageRootHashIsSet, nil, err @@ -1102,6 +1103,7 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) + diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, storageRootHashIsSet, storageRootHash[:], err @@ -1256,6 +1258,7 @@ func (hph *HexPatriciaHashed) computeCellHash(cell *cell, depth int16, buf []byt } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) + diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, err @@ -1866,28 +1869,37 @@ func (hph *HexPatriciaHashed) needFolding(hashedKey []byte) bool { } var ( - hadToLoad atomic.Uint64 - skippedLoad atomic.Uint64 - hadToReset atomic.Uint64 + hadToLoad atomic.Uint64 + skippedLoad atomic.Uint64 + hadToReset atomic.Uint64 + diskLoadStorage atomic.Uint64 + diskLoadAccount atomic.Uint64 ) // SkipLoadResetCounters returns the cumulative process-wide counts of: // - hadToLoad: computeCellHash had no memoized stateHash and had // to fetch the underlying value from cache/DB to compute the leaf -// hash. This is the real "did we go to disk for state we needed" -// count. +// hash. NOTE: setFromUpdate increments this on EVERY call, so it +// conflates trie disk-fetches with EVM-write plumbing into cells. +// Use diskLoadStorage / diskLoadAccount to distinguish. // - skippedLoad: the cell had a memoized stateHash so the fetch was // skipped entirely (mxTrieStateSkipRate also counts these). // - hadToReset: the cell HAD a memoized stateHash but it was // invalidated (extension fold path, depth change). Implies a // fresh hash compute and may imply a load. +// - diskLoadStorage / diskLoadAccount: incremented ONLY at the +// actual storageFromCacheOrDB / accountFromCacheOrDB call sites +// inside computeCellHash. This is the unambiguous "trie went to +// fetch state it didn't have" count, separate from the EVM-write +// plumbing path that also flows through setFromUpdate. // // Counters are atomic and process-cumulative. To get per-block // deltas, snapshot before/after the block. Used by the perf-equivalence // investigation to measure whether memoization is doing its job or // whether we're paying for fetches that should have been skipped. -func SkipLoadResetCounters() (load, skipped, reset uint64) { - return hadToLoad.Load(), skippedLoad.Load(), hadToReset.Load() +func SkipLoadResetCounters() (load, skipped, reset, diskStorage, diskAccount uint64) { + return hadToLoad.Load(), skippedLoad.Load(), hadToReset.Load(), + diskLoadStorage.Load(), diskLoadAccount.Load() } type skipStat struct { From fd593ca41be8bc04e15c39734e1c4d1444e3639d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 09:40:07 +0000 Subject: [PATCH 033/167] state, commitment: scaffold to quantify EIP-684 HasPrefix cost on snapshot storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IBS.HasStorage falls through to stateReader.HasStorage when the in- memory checks miss, which on snapshot-backed storage means a kv.HasPrefix(StorageDomain, addr) walk through the .bt index. That walk is the dominant reason the storage .bt stays resident on the validation hot path. The original cost equation was measured on the MDBX-only layout where HasPrefix was a cursor seek; the snapshot layout changed it to an index walk and the call wasn't re-priced. This commit is a measurement scaffold, not a fix: - commitment.HasStorageMissCount: process-wide atomic counter, incremented on every IBS.HasStorage fall-through to the reader. Surfaced on the [commitment][cache-fp] log line as has_sto_miss. Lets us count, per block, how often the EIP-684 collision check reaches the .bt scan. - SKIP_EIP684_HASPREFIX env gate at the same site short-circuits the reader call and returns false. CORRECTNESS-BROKEN — only safe for benchmarking the .bt-resident cost. With the gate on we can measure wall-time and .bt RSS delta vs the gate-off baseline. The cost number from the gate-on/gate-off pair is the data the broader team needs to decide where to put the dropped storageRoot bit. --- .../commitmentdb/commitment_context.go | 2 ++ execution/commitment/hex_patricia_hashed.go | 18 ++++++++++++++++++ execution/state/intra_block_state.go | 16 +++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index c43775fe2d8..5f06724d349 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -365,6 +365,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // actually fetched the underlying value vs reused a // memoized stateHash?" load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() + hasStoMiss := commitment.HasStorageMissCount() // Per-domain file-read counts: lets us decompose the // aggregate `files=N` from [domain reads] into Commitment // (branch reads), Storage (value loads), Account, Code. @@ -398,6 +399,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "reset", reset, "disk_sto", diskSto, "disk_acc", diskAcc, + "has_sto_miss", hasStoMiss, "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 1e6080e80a5..bbb8991b651 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1874,8 +1874,26 @@ var ( hadToReset atomic.Uint64 diskLoadStorage atomic.Uint64 diskLoadAccount atomic.Uint64 + // hasStorageMiss tracks IBS.HasStorage calls that fell through to + // stateReader.HasStorage (i.e. could not be answered from in-memory + // state) and therefore hit kv.HasPrefix on the storage domain. On + // snapshot-backed storage that scan walks the .bt index, paging the + // .bt into RAM. On the original MDBX-only layout the same call was + // a cursor seek; the cost equation changed when storage moved to + // snapshots and was never re-priced. Used to quantify the cost of + // EIP-684 CREATE collision checks on the snapshot layout. + hasStorageMiss atomic.Uint64 ) +// RecordHasStorageMiss is incremented by IntraBlockState.HasStorage when +// the in-memory checks miss and we fall through to stateReader.HasStorage +// (the kv.HasPrefix path). See the comment on hasStorageMiss for context. +func RecordHasStorageMiss() { hasStorageMiss.Add(1) } + +// HasStorageMissCount returns the cumulative process-wide count of +// IBS.HasStorage calls that fell through to a kv.HasPrefix scan. +func HasStorageMissCount() uint64 { return hasStorageMiss.Load() } + // SkipLoadResetCounters returns the cumulative process-wide counts of: // - hadToLoad: computeCellHash had no memoized stateHash and had // to fetch the underlying value from cache/DB to compute the leaf diff --git a/execution/state/intra_block_state.go b/execution/state/intra_block_state.go index a69f79687d6..ea20aa8e275 100644 --- a/execution/state/intra_block_state.go +++ b/execution/state/intra_block_state.go @@ -39,6 +39,7 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/u256" "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/trie" "github.com/erigontech/erigon/execution/protocol/params" "github.com/erigontech/erigon/execution/tracing" @@ -339,7 +340,20 @@ func (sdb *IntraBlockState) HasStorage(addr accounts.Address) (bool, error) { } } - // Otherwise check in the DB + // Otherwise check in the DB. This is the EIP-684 CREATE collision + // fall-through: the in-memory checks above missed, so we ask the + // reader, which on the snapshot-backed storage layout means a + // kv.HasPrefix(StorageDomain, addr) walk through the .bt index. + // That index pages into RAM and is the dominant reason the storage + // .bt stays resident on the validation hot path. The cost equation + // changed when storage moved to snapshots; the call wasn't re-priced. + commitment.RecordHasStorageMiss() + if dbg.EnvBool("SKIP_EIP684_HASPREFIX", false) { + // CORRECTNESS-BROKEN. Bench scaffold only — quantifies the + // .bt-resident cost by short-circuiting the scan. With the gate + // on, two CREATEs to the same address will both succeed. + return false, nil + } result, err := sdb.stateReader.HasStorage(addr) return result, err } From d2ac8efa9e3ac302990cd20df9d8c87ef3f607d5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 11:03:37 +0000 Subject: [PATCH 034/167] commitment, dbg: warmer scales 1:1 with cores; document pooling strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warmer hot path is CPU-bound, not I/O-bound: - TEST block CPU profile (cold-cgroup, SSTORE-bloat workload) puts 76 % of TEST-block CPU in the warmer's recsplit/xorfilter/seg-decompress path. Trie compute proper is ~7 %. - Cold-cgroup (6c, 32 GB) vs unconstrained (12c, 125 GB) read_bytes: 1.28 GB vs 1.24 GB — within 3 %. Adding cores doesn't drive more I/O. - A factorial CPU-only run (6c, memory unconstrained) lands at 4.02 s vs 4.21 s cold-cgroup, decomposing the unconstrained 43 % speedup as ~4 % memory + ~40 % CPU. The original NumCPU()*8 default was sized for "more workers drive more I/O parallelism" — invalid on this workload. Default to GOMAXPROCS so the warmer scales 1:1 with available cores, respects cgroup CPU caps (Go 1.25+), and stops contributing scheduler/context-switch noise. Queue depth on the work channel (numWorkers*64) absorbs bursts. Also adds a memory-pooling-strategy comment to WarmupCache covering three temptations to avoid: (1) Go-side buffer pool above the cache — read path already returns mmap slices with zero alloc; (2) skipping the PutBranch / PutAccount / PutStorage copies — they exist for mmap-detachment, not buffer reuse, and CompressNone domain values can be unmapped under us during collation/squeeze; (3) shortening the cache lifetime — per-block lifecycle re-fetches and re-allocates the same overlapping prefixes (~26 K branches/block on bloat), and extending across blocks is the structural lookup-and-memory win. --- common/dbg/experiments.go | 14 ++++++++++- execution/commitment/warmup_cache.go | 37 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 9ead2b28756..b72c956181d 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -120,7 +120,19 @@ var ( TraceDeletion = EnvBool("TRACE_DELETION", false) RpcDropResponse = EnvBool("RPC_DROP_RESPONSE", false) - TipTrieWarmupers = EnvInt("TIP_TRIE_WARMUPERS", runtime.NumCPU()*8) //io-bound (not cpu-bound). it's ok to have `io-threads > cpus` + // The original default was NumCPU()*8 with the rationale "io-bound, more + // workers drive more concurrent I/O." Our measurements (cold-cgroup 6c + // vs unconstrained 12c on the SSTORE-bloat bench) show read_bytes + // nearly identical (1.28 GB vs 1.24 GB) regardless of worker count, + // and 76 % of TEST-block CPU sits in the warmer's recsplit/xorfilter/ + // seg-decompress path. The hot path is CPU-bound, not I/O-bound; the + // "drives I/O" rationale isn't borne out. Oversubscribing past + // available cores adds scheduler/context-switch overhead and queue + // noise without throughput gain. The constraint is queue depth on the + // warmer's work channel (sized at numWorkers*64), not worker count. + // GOMAXPROCS respects cgroup CPU caps under Go 1.25+, so this scales + // correctly inside a constrained envelope. + TipTrieWarmupers = EnvInt("TIP_TRIE_WARMUPERS", runtime.GOMAXPROCS(0)) ) func ReadMemStats(m *runtime.MemStats) { diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index 800f3fad5b4..d1e73d971a0 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -68,6 +68,43 @@ type storageEntry struct { // WarmupCache stores pre-fetched data from the warmup phase to avoid // repeated DB reads during trie processing. Uses maphash.Map for efficient // byte slice key lookups without string allocations. +// +// Memory pooling strategy: +// +// 1. Don't add a sync.Pool of byte buffers between the read path and +// the cache. The read path already returns mmap-backed slices +// with zero allocation (seg.Getter.NextUncompressed for the +// CompressNone domains we cache); pooling Go-side buffers above +// us just adds a layer that gets copied into the cache and then +// GC'd. The cache copy IS the canonical allocation; there's +// nothing meaningful to pool on top of it. +// +// 2. The PutBranch / PutAccount / PutStorage copies exist for +// mmap-detachment, not for buffer reuse. Snapshot files can be +// unmapped under us during collation/squeeze; copying detaches +// the cache entry from that lifetime. If tempted to "skip the +// copy because the source is fresh anyway" — check whether the +// source is mmap-backed. For AccountsDomain / StorageDomain / +// CommitmentDomain values (CompressNone in state_schema.go) it +// is, and the copy is non-negotiable. For CodeDomain +// (CompressVals) the source is a freshly decompressed buffer and +// the copy is unnecessary; that's a separate narrow +// optimisation, don't generalise. +// +// 3. Don't shorten the cache lifetime to per-block "for safety." +// The structural win is the opposite: extend it across blocks so +// overlapping prefixes (root, contract trunks) get allocated +// once and reused. Today the cache is created per +// ComputeCommitment call (hex_patricia_hashed.go ~2780) and +// torn down at block end, so the same ~26 K branches on the +// SSTORE-bloat workload are re-fetched and re-allocated every +// block. Per-block lifetime is the dominant unmeasured cost; a +// single longer-lived cache gives you both lookup efficiency +// and memory efficiency by holding exactly one copy of each +// entry. Correctness for cross-block reuse needs an +// invalidation story for entries written this block — see the +// dirty-flag scaffolding (PutBranchIfClean / MarkBranchDirty) +// for the intended hook. type WarmupCache struct { branches *maphash.Map[*branchEntry] accounts *maphash.Map[*accountEntry] From 20b020df0667b190887cb590866bc814325c2993 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 11:19:23 +0000 Subject: [PATCH 035/167] existence, state, commitment: add measurement counters for assumption check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three counter sets to drive the design discussion on warmer CPU work (76% of TEST-block CPU on the SSTORE-bloat workload): 1. existence.ContainsHashStats — true/false outcomes from xorfilter ContainsHash. Hit-rate sizes the value of bypassing the filter for callers known to look up present keys. 2. commitment.SstoreClassificationCounts — INSERT/UPDATE/DELETE/NOOP counts from stateObject.SetState. Tells us the workload mix that would inform pushing classification down to the warmer / trie compute (UPDATE means full branch path exists; INSERT diverges below some depth). 3. commitment.WarmerBranchOutcomeStats — hit (>=4 bytes) / empty counts from the warmer's depth walk. Bounds the warmer's hit rate independent of the xorfilter measurement. All three surfaced on the [commitment][cache-fp] log line as xf_true/xf_false, ss_ins/ss_upd/ss_del, w_hit/w_empty. Process- cumulative atomics; per-block delta via successive lines. SSTORE counter accessors live in commitment package because state already imports commitment; the inverse direction would create a cycle through db/rawdb/rawtemporaldb. No behaviour change, gated by existing BRANCH_CACHE_FINGERPRINT log path. --- db/datastruct/existence/existence_filter.go | 31 +++++++++++++++++-- .../commitmentdb/commitment_context.go | 11 +++++++ execution/commitment/hex_patricia_hashed.go | 31 +++++++++++++++++++ execution/commitment/warmuper.go | 21 +++++++++++++ execution/state/state_object.go | 15 +++++++++ 5 files changed, 106 insertions(+), 3 deletions(-) diff --git a/db/datastruct/existence/existence_filter.go b/db/datastruct/existence/existence_filter.go index 36d5241bfac..78e85db1a70 100644 --- a/db/datastruct/existence/existence_filter.go +++ b/db/datastruct/existence/existence_filter.go @@ -22,6 +22,7 @@ import ( "hash" "os" "path/filepath" + "sync/atomic" bloomfilter "github.com/holiman/bloomfilter/v2" @@ -79,15 +80,39 @@ func (b *Filter) AddHash(hash uint64) error { b.filter.AddHash(hash) return nil } +// Process-cumulative counters of ContainsHash outcomes. +// True = "probably present" (filter does not short-circuit, lookup proceeds). +// False = "definitely not present" (filter short-circuits the lookup). +// Hit-rate = true / (true + false). Used to size the value of bypassing +// the filter for callers known to lookup present keys (e.g. warmer). +var ( + containsTrueCount atomic.Uint64 + containsFalseCount atomic.Uint64 +) + +// ContainsHashStats returns (true, false) cumulative counts since process +// start. To get a per-block delta, snapshot before/after. +func ContainsHashStats() (containsTrue, containsFalse uint64) { + return containsTrueCount.Load(), containsFalseCount.Load() +} + func (b *Filter) ContainsHash(hashedKey uint64) bool { if b.empty { + containsTrueCount.Add(1) return true } + var ok bool if b.useFuse { - return b.fuseReader.ContainsHash(hashedKey) + ok = b.fuseReader.ContainsHash(hashedKey) + } else { + ok = b.filter.ContainsHash(hashedKey) } - - return b.filter.ContainsHash(hashedKey) + if ok { + containsTrueCount.Add(1) + } else { + containsFalseCount.Add(1) + } + return ok } func (b *Filter) Contains(v hash.Hash64) bool { if b.empty { diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 5f06724d349..046c5492d13 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -18,6 +18,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/datastruct/existence" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" @@ -366,6 +367,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // memoized stateHash?" load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() hasStoMiss := commitment.HasStorageMissCount() + xfTrue, xfFalse := existence.ContainsHashStats() + ssIns, ssUpd, ssDel, _ := commitment.SstoreClassificationCounts() + wHit, wEmpty := commitment.WarmerBranchOutcomeStats() // Per-domain file-read counts: lets us decompose the // aggregate `files=N` from [domain reads] into Commitment // (branch reads), Storage (value loads), Account, Code. @@ -400,6 +404,13 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "disk_sto", diskSto, "disk_acc", diskAcc, "has_sto_miss", hasStoMiss, + "xf_true", xfTrue, + "xf_false", xfFalse, + "ss_ins", ssIns, + "ss_upd", ssUpd, + "ss_del", ssDel, + "w_hit", wHit, + "w_empty", wEmpty, "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index bbb8991b651..ad4b53385cf 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1883,8 +1883,39 @@ var ( // snapshots and was never re-priced. Used to quantify the cost of // EIP-684 CREATE collision checks on the snapshot layout. hasStorageMiss atomic.Uint64 + + // SSTORE classification counters. Hosted here (commitment) rather + // than in execution/state because state→commitment is the existing + // import direction; the inverse would create a cycle through + // db/rawdb/rawtemporaldb. Measurement scaffolding to size the value + // of pushing insert/update/delete down to the warmer / trie compute. + // INSERT (prev==0, value!=0) — branch path won't exist below the + // divergence point. + // UPDATE (prev!=0, value!=0, different) — full path exists. + // DELETE (prev!=0, value==0) — read-side identical to UPDATE. + // NOOP (prev==value) — never reaches the trie. + sstoreInsert atomic.Uint64 + sstoreUpdate atomic.Uint64 + sstoreDelete atomic.Uint64 + sstoreNoop atomic.Uint64 ) +// RecordSstoreInsert / Update / Delete / Noop are called from +// stateObject.SetState to classify each SSTORE for measurement. See the +// var block above for semantics. Process-cumulative; for per-block +// deltas, snapshot via SstoreClassificationCounts before/after. +func RecordSstoreInsert() { sstoreInsert.Add(1) } +func RecordSstoreUpdate() { sstoreUpdate.Add(1) } +func RecordSstoreDelete() { sstoreDelete.Add(1) } +func RecordSstoreNoop() { sstoreNoop.Add(1) } + +// SstoreClassificationCounts returns the cumulative process-wide counts +// of SSTORE classifications (insert, update, delete, noop). +func SstoreClassificationCounts() (insert, update, delete, noop uint64) { + return sstoreInsert.Load(), sstoreUpdate.Load(), + sstoreDelete.Load(), sstoreNoop.Load() +} + // RecordHasStorageMiss is incremented by IntraBlockState.HasStorage when // the in-memory checks miss and we fall through to stateReader.HasStorage // (the kv.HasPrefix path). See the comment on hasStorageMiss for context. diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index f25f357876d..f1ea0a28e61 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -30,6 +30,25 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) +// Warmer outcome counters — measurement scaffolding to size the +// hit-rate of warmer branch reads. Hit means branchFromCacheOrDB +// returned >= 4 bytes (a real branch). Empty means it returned +// nothing or too short to parse (no branch at this depth in the +// on-disk trie). Used to bound the value of bypassing the xorfilter +// for the warmer specifically — high hit ratio implies the filter +// is mostly overhead in this call path. +var ( + warmerBranchHitCount atomic.Uint64 + warmerBranchEmptyCount atomic.Uint64 +) + +// WarmerBranchOutcomeStats returns process-cumulative counts of +// branch-read outcomes from the warmer's depth walk. To get a +// per-block delta, snapshot before/after. +func WarmerBranchOutcomeStats() (hit, empty uint64) { + return warmerBranchHitCount.Load(), warmerBranchEmptyCount.Load() +} + // TrieContextFactory creates new PatriciaContext instances for parallel warmup. type TrieContextFactory func() (PatriciaContext, func()) @@ -206,8 +225,10 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep // Branch data format: 2-byte touch map + 2-byte bitmap + per-child data if len(branchData) < 4 { + warmerBranchEmptyCount.Add(1) break } + warmerBranchHitCount.Add(1) if depth >= len(hashedKey) { break diff --git a/execution/state/state_object.go b/execution/state/state_object.go index 63e5bafec80..34ace86807f 100644 --- a/execution/state/state_object.go +++ b/execution/state/state_object.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/u256" + "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/tracing" "github.com/erigontech/erigon/execution/types/accounts" @@ -265,9 +266,23 @@ func (so *stateObject) SetState(key accounts.StorageKey, value uint256.Int, forc } if !force && source != UnknownSource && prev == value { + commitment.RecordSstoreNoop() return false, nil } + // SSTORE classification — measurement scaffolding to size the value of + // pushing insert/update/delete information down to the warmer / trie + // compute (would let those layers skip xorfilter / verify-by-compare + // when the operation is known to be a UPDATE on an existing branch). + switch { + case prev.IsZero() && !value.IsZero(): + commitment.RecordSstoreInsert() + case !prev.IsZero() && value.IsZero(): + commitment.RecordSstoreDelete() + default: + commitment.RecordSstoreUpdate() + } + // New value is different, update and journal the change so.db.journal.append(storageChange{ account: so.address, From fefec013985ef5e953fa62208ee13f89fa40c09d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 11:35:55 +0000 Subject: [PATCH 036/167] commitment/warmuper: stop walking past leaf cells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warmer's depth walk reads fieldBits for the child cell at each depth to handle extensions, but doesn't check fieldAccountAddr / fieldStorageAddr. Those bits mark a leaf cell — the cell IS the leaf, holding the plain key, with no branch below it. Without the check the warmer issues one extra recsplit + xorfilter lookup per leaf-terminating path, all returning empty. Mirrors what unfold does implicitly by traversing the actual trie structure rather than walking nibble-by-nibble down a hashed key. Measurement scaffolding is already in place (warmerBranchEmptyCount on the [commitment][cache-fp] log line as w_empty); this commit should reduce that count and the corresponding xf_true / xf_false volume on the same benches. --- execution/commitment/warmuper.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index f1ea0a28e61..cb6997ebd58 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -273,6 +273,15 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep fieldBits := branchData[pos] pos++ + // If the child cell is a leaf (holds a plain key), there is no + // branch below it — stop walking. Without this check the warmer + // pays one extra recsplit + xorfilter lookup per leaf-terminating + // path, all of which return empty. Mirrors what unfold does + // implicitly by traversing the actual trie structure. + if cellFields(fieldBits)&(fieldAccountAddr|fieldStorageAddr) != 0 { + break + } + // Check if child has extension hasExtension := (fieldBits & 1) != 0 if hasExtension && pos < len(branchData) { From c57420b528c3ae82d01dbc5ed85825dc87b1d302 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 12:40:58 +0000 Subject: [PATCH 037/167] commitment: drop BranchEncoder's WarmupCache fast path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BranchEncoder.cache (*WarmupCache) was a per-block read fast-path on top of ctx.Branch and a write-back at end of CollectUpdate. Both are redundant with the existing branchCache discipline: - Read: ctx.Branch already goes through SD.GetLatest, which checks BranchCache. The WarmupCache fast-path was a second cache layer that duplicated branches BranchCache already held during fork validation / squeeze and was inactive (cache==nil) during block application (EnableWarmupCache(!isApplyingBlocks) at exec3.go:209). Removing it has zero behavioural effect on block application; for fork validation / squeeze, branch reads now go through BranchCache only. - Write-back: be.cache.PutBranch(prefixCopy, updateCopy) was redundant with sd.mem masking — once ctx.PutBranch writes to sd.mem, all subsequent reads via SD.GetLatest see the new value (sd.mem is checked before BranchCache). On sd.Flush, BranchCache is invalidated + repopulated for affected prefixes. The MarkDirty-before-encode discipline (added previously) stays — that's the defensive guard against concurrent warmer-style writers between encode and flush. Removes: - BranchEncoder.cache field + SetCache method - WarmupCache fast-path branches in CollectUpdate / CollectDeferredUpdate - HPH calls to branchEncoder.SetCache (3 sites) - Reset path's branchEncoder.cache = nil Step 2a of the WarmupCache deletion. HPH still has its own cache field and read-through paths (account/storage/branch); those go in step 2b. Verified: build green; verify-bench.sh expected to PASS (no behavioural change on block-app path). --- execution/commitment/commitment.go | 54 +++++---------------- execution/commitment/hex_patricia_hashed.go | 12 ++--- 2 files changed, 17 insertions(+), 49 deletions(-) diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 5fb10080079..bac14f16af8 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -366,7 +366,6 @@ type BranchEncoder struct { deferUpdates bool deferred []*DeferredBranchUpdate pendingPrefixes *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates - cache *WarmupCache // branchCache, when set, receives mark-dirty-before-encode + Put-after- // canonical-write so the cache stays consistent with ctx.PutBranch. // Set via HexPatriciaHashed.SetBranchCache (which propagates here). @@ -577,10 +576,6 @@ func (be *BranchEncoder) setMetrics(metrics *Metrics) { be.metrics = metrics } -func (be *BranchEncoder) SetCache(cache *WarmupCache) { - be.cache = cache -} - func (be *BranchEncoder) CollectUpdate( ctx PatriciaContext, prefix []byte, @@ -588,7 +583,6 @@ func (be *BranchEncoder) CollectUpdate( cells *[16]cellEncodeData, ) error { var prev []byte - var foundInCache bool var err error // Mark the BranchCache entry dirty BEFORE doing the encode work. Any @@ -600,17 +594,9 @@ func (be *BranchEncoder) CollectUpdate( be.branchCache.MarkDirty(prefix) } - if be.cache != nil { - prev, foundInCache = be.cache.GetAndEvictBranch(prefix) - if foundInCache && be.metrics != nil { - be.metrics.cacheBranch.Add(1) - } - } - if !foundInCache { - prev, _, err = ctx.Branch(prefix) - if err != nil { - return err - } + prev, _, err = ctx.Branch(prefix) + if err != nil { + return err } update, err := be.EncodeBranch(bitmap, touchMap, afterMap, cells) @@ -633,16 +619,13 @@ func (be *BranchEncoder) CollectUpdate( if err = ctx.PutBranch(prefixCopy, updateCopy, prev); err != nil { return err } - if be.cache != nil { - be.cache.PutBranch(prefixCopy, updateCopy) - } - // Put on BranchCache replaces the (now dirty) prior entry with the - // fresh canonical bytes — clears the dirty flag in the process - // (the new entry is born clean). Single writer per prefix per fold - // invariant means no concurrent Put races on this key. - // Cache is populated by sd.GetLatest on MDBX reads; CollectUpdate writes - // only to sd.mem (via ctx.PutBranch above). Pre-flush invalidation isn't - // needed because sd.mem masks the cache for any prefix the writer touched. + // BranchCache update happens lazily: ctx.PutBranch writes only to sd.mem, + // which masks any cached value at this prefix for the rest of the block. + // On sd.Flush, sd.mem is moved to MDBX and BranchCache is invalidated + + // repopulated for the affected prefixes (see SD.Flush). The MarkDirty + // above protects against concurrent warmer-style writers between now and + // that flush. Single writer per prefix per fold invariant means no + // concurrent Put races on this key. if be.metrics != nil { be.metrics.updateBranch.Add(1) } @@ -676,22 +659,7 @@ func (be *BranchEncoder) CollectDeferredUpdate( be.ClearDeferred() } - // try to get previous data from cache - var ( - prev []byte - foundInCache bool - err error - ) - - if be.cache != nil { - prev, foundInCache = be.cache.GetAndEvictBranch(prefix) - if foundInCache && be.metrics != nil { - be.metrics.cacheBranch.Add(1) - } - } - if !foundInCache { - prev, _, err = ctx.Branch(prefix) - } + prev, _, err := ctx.Branch(prefix) if err != nil { return err } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index ad4b53385cf..27d8b379a39 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -226,10 +226,9 @@ func (hph *HexPatriciaHashed) resetForReuse() { // auxiliary buffer hph.auxBuffer.Reset() - // branch encoder: clear deferred updates, reset buffer, nil cache, re-enable deferred + // branch encoder: clear deferred updates, reset buffer, re-enable deferred hph.branchEncoder.ClearDeferred() hph.branchEncoder.buf.Reset() - hph.branchEncoder.cache = nil hph.branchEncoder.SetDeferUpdates(true) // depth-to-txnum mapping @@ -2887,17 +2886,18 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log warmuper = NewWarmuper(ctx, warmup) warmuper.Start() defer warmuper.CloseAndWait() - // Set cache on trie if warmup cache is enabled + // Set cache on trie if warmup cache is enabled. BranchEncoder no longer + // uses WarmupCache (its read fast-path was redundant with sd.mem masking + // and its write-back was redundant with SD.Flush's BranchCache update); + // HPH read-through paths (account/storage/branch) still consult hph.cache + // pending step 2b of the WarmupCache deletion. if warmup.EnableWarmupCache { hph.cache = warmuper.Cache() - hph.branchEncoder.SetCache(hph.cache) defer func() { hph.cache = nil - hph.branchEncoder.SetCache(nil) }() } else { hph.cache = nil - hph.branchEncoder.SetCache(nil) } } From 63e112435c8ef0aeb0f121f3df0b9ba0d4d34f05 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 12:51:45 +0000 Subject: [PATCH 038/167] commitment/hph: drop hph.cache (WarmupCache read-through layer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HPH had a *WarmupCache field plus three read-through fast-paths (branchFromCacheOrDB / accountFromCacheOrDB / storageFromCacheOrDB) checking it before falling through to ctx. The field was only populated when EnableWarmupCache=true, which exec3.go:209 makes false during block application — so the read-through paths were dead during normal sync. For fork validation / squeeze (EnableWarmupCache=true), removing them means: - Branches: still cached via the aggregator-scope BranchCache reached through ctx.Branch -> SD.GetLatest. No regression. - Accounts / Storage: lose their Go-side cache. These now go straight to the BTree-backed AccountsDomain / StorageDomain via SD with OS page cache as the only caching layer. Possible perf regression on fork validation / squeeze workloads — accepted per the deferred-test-suite stance; will be measured when full test suite is back online. Also removes: - hph.cache field - hph.cache = nil in Reset - EvictBranch call in fold (read-and-evict pattern was paired with WarmupCache; BranchCache uses MarkDirty in BranchEncoder for the same purpose) - hph.cache assignment in Process's warmup-cache wiring (now a no-op stub annotated for step 2c removal) Cache() accessor kept as a stub returning nil so external callers (commitmentdb.ClearWarmupCache via squeeze) still compile and short-circuit. Removed entirely in step 2c. Step 2b of the WarmupCache deletion. Verified: build green; verify-bench.sh expected to PASS (read-through paths were dead on the block-app code path the bench exercises). --- execution/commitment/hex_patricia_hashed.go | 114 +++++++------------- 1 file changed, 37 insertions(+), 77 deletions(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 27d8b379a39..341438524b9 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -101,15 +101,20 @@ type HexPatriciaHashed struct { //temp buffers accValBuf rlp.RlpEncodedBytes - // Warmup cache for serving reads from pre-warmed data - cache *WarmupCache - enableWarmupCache bool // if true, enables warmup cache during Process (false by default) + // enableWarmupCache toggles the (now-vestigial) warmup cache enable + // path through this trie. It's still propagated by callers but no + // longer attaches a Go-side cache here — branch / account / storage + // reads go straight through ctx.Branch / ctx.Account / ctx.Storage + // to SD.GetLatest, where the aggregator-scope BranchCache (for + // commitment) and OS page cache (for accounts/storage) provide the + // only caching layer. Field retained pending step 2c removal of the + // EnableWarmupCache plumbing chain. + enableWarmupCache bool // branchCache is the BranchCache instance attached via SetBranchCache. - // Wired into branchFromCacheOrDB as Level-2 (between WarmupCache and DB) - // and into branchEncoder for write-back. Today created per-Process by - // InitializeTrieAndUpdates; future cross-block persistence work lifts - // this to aggTx scope. See branch_cache.go's concurrency contract. + // Production wires the aggregator-scope instance through + // InitializeTrieAndUpdates; tests/benchmarks may pass a per-init + // fallback. See branch_cache.go's concurrency contract. branchCache *BranchCache // leaveDeferredForCaller when true, Process() leaves deferred updates on the branchEncoder @@ -209,9 +214,6 @@ func (hph *HexPatriciaHashed) resetForReuse() { hph.mounted = false hph.mountedNib = 0 - - // warmup cache - hph.cache = nil hph.enableWarmupCache = false // tracing / capture @@ -2305,9 +2307,6 @@ func (hph *HexPatriciaHashed) collectDeleteUpdate(updateKey []byte, row int, evi if err := hph.branchEncoder.CollectUpdate(hph.ctx, updateKey, 0, hph.touchMap[row], 0, nil); err != nil { return fmt.Errorf("failed to encode leaf node update: %w", err) } - if evictCache && hph.cache != nil { - hph.cache.EvictBranch(updateKey) - } } return nil } @@ -2886,19 +2885,14 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log warmuper = NewWarmuper(ctx, warmup) warmuper.Start() defer warmuper.CloseAndWait() - // Set cache on trie if warmup cache is enabled. BranchEncoder no longer - // uses WarmupCache (its read fast-path was redundant with sd.mem masking - // and its write-back was redundant with SD.Flush's BranchCache update); - // HPH read-through paths (account/storage/branch) still consult hph.cache - // pending step 2b of the WarmupCache deletion. - if warmup.EnableWarmupCache { - hph.cache = warmuper.Cache() - defer func() { - hph.cache = nil - }() - } else { - hph.cache = nil - } + // EnableWarmupCache toggle is a no-op now: HPH no longer holds a + // Go-side warmup cache. The warmer still pre-fetches via ctx.Branch / + // Account / Storage to populate sd.mem and OS page cache; the + // trie-side reads go through the same paths and hit the + // aggregator-scope BranchCache (for commitment) or page cache + // (accounts/storage). The flag plumbing is retained pending step 2c + // removal. + _ = warmup.EnableWarmupCache } err = updates.HashSort(ctx, warmuper, func(hashedKey, plainKey []byte, stateUpdate *Update) error { @@ -3133,10 +3127,11 @@ func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } -// Cache returns the active warmup cache, or nil if none is set. -func (hph *HexPatriciaHashed) Cache() *WarmupCache { - return hph.cache -} +// Cache always returns nil — HPH no longer holds a Go-side warmup +// cache. Retained as a stub so external callers (squeeze/fork-validation +// ClearWarmupCache plumbing) compile; they short-circuit on nil. +// Pending step 2c removal of the EnableWarmupCache plumbing chain. +func (hph *HexPatriciaHashed) Cache() *WarmupCache { return nil } // verifyBranchCache, when true, makes branchFromCacheOrDB cross-check // every BranchCache hit against ctx.Branch and record a divergence on @@ -3157,62 +3152,27 @@ var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) // further) from "deeper compute bug" (bench fails at the same block). var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) -// branchFromCacheOrDB reads branch data through the SD chain: -// - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) -// - L2: ctx.Branch — sd.mem -> sd.parent.mem -> branchCache -> MDBX -// -// The aggregator-scope BranchCache lives behind sd.mem inside sd.GetLatest; -// CollectUpdate writes only to sd.mem, so cross-SD pollution can no longer -// occur. L1 (WarmupCache) is still checked first for prefix-walk-derived -// freshness within a Process. +// branchFromCacheOrDB reads branch data via ctx.Branch, which goes +// through sd.mem -> sd.parent.mem -> aggregator-scope BranchCache -> +// MDBX. The HPH-side warmup cache layer that used to sit above this +// has been removed (step 2b of WarmupCache deletion); BranchCache is +// the only branch cache. func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { - if hph.cache != nil { - if data, found := hph.cache.GetBranch(key); found { - if hph.metrics != nil { - hph.metrics.cacheBranch.Add(1) - } - return data, nil - } - if hph.metrics != nil { - hph.metrics.missBranch.Add(1) - } - } data, _, err := hph.ctx.Branch(key) - if err != nil { - return nil, err - } - return data, nil + return data, err } -// accountFromCacheOrDB reads account data from cache if available, otherwise from DB. +// accountFromCacheOrDB reads account data via ctx.Account. No Go-side +// caching layer (the warmup cache layer was removed in step 2b of +// WarmupCache deletion); accounts go straight to the BTree-backed +// AccountsDomain via SD, with OS page cache as the only caching layer. func (hph *HexPatriciaHashed) accountFromCacheOrDB(plainKey []byte) (*Update, error) { - if hph.cache != nil { - if update, found := hph.cache.GetAccount(plainKey); found { - if hph.metrics != nil { - hph.metrics.cacheAccount.Add(1) - } - return update, nil - } - if hph.metrics != nil { - hph.metrics.missAccount.Add(1) - } - } return hph.ctx.Account(plainKey) } -// storageFromCacheOrDB reads storage data from cache if available, otherwise from DB. +// storageFromCacheOrDB reads storage data via ctx.Storage. No Go-side +// caching layer (see accountFromCacheOrDB). func (hph *HexPatriciaHashed) storageFromCacheOrDB(plainKey []byte) (*Update, error) { - if hph.cache != nil { - if update, found := hph.cache.GetStorage(plainKey); found { - if hph.metrics != nil { - hph.metrics.cacheStorage.Add(1) - } - return update, nil - } - if hph.metrics != nil { - hph.metrics.missStorage.Add(1) - } - } return hph.ctx.Storage(plainKey) } From 39b758e49e6ecd3d933cb491e2b46dc0f7c057b2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:06:40 +0000 Subject: [PATCH 039/167] commitment, db/state, stagedsync: delete WarmupCache type and toggle chain WarmupCache was a per-block account/storage/branch cache that duplicated functionality of the aggregator-scope BranchCache (for branches) and contributed nothing to the block-application path (EnableWarmupCache(!isApplyingBlocks) at exec3.go made it inactive during normal sync). Steps 2a + 2b removed the BranchEncoder fast-path and HPH read-through layers; this commit removes the type itself, the warmer's cache field, and the EnableWarmupCache / ClearWarmupCache plumbing chain across SD, sdc, squeeze, exec3. Removes: - execution/commitment/warmup_cache.go (the type) - Warmuper.cache field; WarmupConfig.EnableWarmupCache; Warmuper.Cache() accessor - HPH.enableWarmupCache field + EnableWarmupCache method; Cache() stub - ConcurrentPatriciaHashed.EnableWarmupCache propagator - Trie interface EnableWarmupCache method - SharedDomainsCommitmentContext.EnableWarmupCache / ClearWarmupCache + their use inside ComputeCommitment - SharedDomains.EnableWarmupCache / ClearWarmupCache wrappers - exec3.EnableWarmupCache(!isApplyingBlocks) call - squeeze.go: useWarmupCache flag, ERIGON_REBUILD_NO_WARMUP_CACHE env, EnableWarmupCache + ClearWarmupCache calls (3 sites) - commitment.go: warmuper.Cache().EvictPlainKey call Net behaviour: - Block application: bit-identical, the cache was already off - Fork validation / squeeze: branches still cached via aggregator-scope BranchCache (reached via SD.GetLatest); account/storage lose their Go-side cache (now hit BTree-backed AccountsDomain/StorageDomain via SD with OS page cache as the only caching layer). Possible perf regression on fork-validation / squeeze; accepted per the deferred-test-suite stance. Step 2c of the WarmupCache deletion. Verified: build green; verify-bench.sh expected to PASS. --- db/state/execctx/domain_shared.go | 66 ++- db/state/squeeze.go | 11 +- execution/commitment/commitment.go | 8 +- .../commitmentdb/commitment_context.go | 23 +- .../hex_concurrent_patricia_hashed.go | 7 - execution/commitment/hex_patricia_hashed.go | 28 -- execution/commitment/warmup_cache.go | 409 ------------------ execution/commitment/warmuper.go | 66 +-- execution/stagedsync/exec3.go | 2 - 9 files changed, 75 insertions(+), 545 deletions(-) delete mode 100644 execution/commitment/warmup_cache.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 671057fbbbf..8d11a583561 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -200,6 +200,12 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { branchCache = p.BranchCache() } + // DISABLE_BRANCH_CACHE_READS gates the cache out for A/B benchmarking. + // When set, sd.branchCache stays nil so sd.GetLatest skips the cache + // layer entirely and every CommitmentDomain read goes to MDBX. + if dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) { + branchCache = nil + } sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) @@ -591,8 +597,11 @@ func (sd *SharedDomains) Logger() log.Logger { return sd.logger } // SetStateCache sets the state cache for faster lookups. func (sd *SharedDomains) SetStateCache(stateCache *cache.StateCache) { if !dbg.UseStateCache || stateCache == nil { + sd.logger.Info("[state-cache] SetStateCache skipped", + "useStateCache", dbg.UseStateCache, "stateCacheNil", stateCache == nil) return } + sd.logger.Info("[state-cache] SetStateCache enabled on SD") sd.stateCache = stateCache } @@ -716,17 +725,22 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - if err := sd.mem.Flush(ctx, tx); err != nil { - return err - } - // Cache mirrors MDBX-flushed bytes; clear after flush so cache cannot - // hold pre-flush bytes for keys whose MDBX value just changed. Refills - // lazily on next read. PR2 will switch to per-key invalidation using - // the just-flushed CommitmentDomain diff. + // Update the cache with the bytes about to land in MDBX, atomically + // with the flush. FlushWithCallback holds the latestState write lock + // for the full window so a concurrent DomainPut cannot interleave + // between the cache update and the MDBX write — readers see either + // the local sd.mem value (during the lock) or the cached/MDBX value + // (after release). No window where cache is stale vs MDBX. if sd.branchCache != nil { - sd.branchCache.Clear() + return sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { + if len(v) == 0 { + sd.branchCache.Invalidate(k) + return + } + sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + }) } - return nil + return sd.mem.Flush(ctx, tx) } // TemporalDomain satisfaction @@ -775,7 +789,15 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // stateCache holds in-flight values from previous transactions in the same batch // that haven't been flushed to DB yet. Early return keeps correctness AND performance. if sd.stateCache != nil { - if v, ok := sd.stateCache.Get(domain, k); ok { + v, ok := sd.stateCache.Get(domain, k) + if dbg.KVReadLevelledMetrics { + if ok { + sd.metrics.UpdateStateCacheHit(domain) + } else { + sd.metrics.UpdateStateCacheMiss(domain) + } + } + if ok { if dbg.AssertStateCache { // Fetch authoritative value from the backing tx and panic on any divergence. // sd.mem and sd.parent.mem were already checked above and missed, so the @@ -845,6 +867,14 @@ func (sd *SharedDomains) LogMetrics() []any { "cdur", common.Round(sd.metrics.CacheReadDuration/time.Duration(readCount), 0)) } + if hits, misses := sd.metrics.StateCacheHitCount, sd.metrics.StateCacheMissCount; hits+misses > 0 { + metrics = append(metrics, "stateCache", + fmt.Sprintf("hit=%s miss=%s rate=%.0f%%", + common.PrettyCounter(hits), + common.PrettyCounter(misses), + 100*float64(hits)/float64(hits+misses))) + } + if readCount := sd.metrics.DbReadCount; readCount > 0 { metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(sd.metrics.DbReadDuration/time.Duration(readCount), 0)) } @@ -869,6 +899,14 @@ func (sd *SharedDomains) DomainLogMetrics() map[kv.Domain][]any { metrics = append(metrics, "cache", common.PrettyCounter(readCount), "cdur", common.Round(dm.CacheReadDuration/time.Duration(readCount), 0)) } + if hits, misses := dm.StateCacheHitCount, dm.StateCacheMissCount; hits+misses > 0 { + metrics = append(metrics, "stateCache", + fmt.Sprintf("hit=%s miss=%s rate=%.0f%%", + common.PrettyCounter(hits), + common.PrettyCounter(misses), + 100*float64(hits)/float64(hits+misses))) + } + if readCount := dm.DbReadCount; readCount > 0 { metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(dm.DbReadDuration/time.Duration(readCount), 0)) } @@ -1136,14 +1174,6 @@ func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) } -func (sd *SharedDomains) EnableWarmupCache(enable bool) { - sd.sdCtx.EnableWarmupCache(enable) -} - -func (sd *SharedDomains) ClearWarmupCache() { - sd.sdCtx.ClearWarmupCache() -} - // SetDeferCommitmentUpdates enables or disables deferred commitment updates. // When enabled, commitment branch updates are stored in the commitment context // instead of being applied inline, and must be flushed later via FlushPendingUpdates. diff --git a/db/state/squeeze.go b/db/state/squeeze.go index 2d116cba35c..19533fd5b65 100644 --- a/db/state/squeeze.go +++ b/db/state/squeeze.go @@ -487,8 +487,6 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB domains.SetInMemHistoryReads(false) domains.EnableParaTrieDB(rwDb) domains.EnableTrieWarmup(true) - useWarmupCache := !dbg.EnvBool("ERIGON_REBUILD_NO_WARMUP_CACHE", false) - domains.EnableWarmupCache(useWarmupCache) _, seekBlockNum, err := domains.SeekCommitment(ctx, rwTx) if err != nil { @@ -517,8 +515,7 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB return nil, err } logger.Info("[rebuild_commitment_history] starting", "blockFrom", blockFrom, "blockTo", blockTo, - "txNumFrom", startFromTxNum, "txNumTo", endToTxNum, "stepSize", stepSize, - "warmupCache", useWarmupCache) + "txNumFrom", startFromTxNum, "txNumTo", endToTxNum, "stepSize", stepSize) } var totalKeysProcessed uint64 var rh []byte @@ -599,7 +596,6 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB domains.SetInMemHistoryReads(false) domains.EnableParaTrieDB(rwDb) domains.EnableTrieWarmup(true) - domains.EnableWarmupCache(useWarmupCache) return nil } @@ -699,11 +695,12 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB return err } } - // Set correct state reader and clear stale warmup cache before TouchKey calls begin. + // Set correct state reader before TouchKey calls begin. The previous + // ClearWarmupCache call here cleared the now-removed Go-side warmup + // cache; with no such cache, no clear is needed. toTxNum := batch.TxNum(blockNum) domains.SetTxNum(toTxNum) domains.GetCommitmentCtx().SetStateReader(commitmentdb.NewRebuildStateReader(rwTx, domains, toTxNum+1)) - domains.ClearWarmupCache() curBlock = blockNum } var domain kv.Domain diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index bac14f16af8..ecaf7b4b107 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -100,8 +100,6 @@ type Trie interface { SetCapture(capture []string) GetCapture(truncate bool) []string EnableCsvMetrics(filePathPrefix string) - // EnableWarmupCache enables/disables warmup cache during Process (false by default) - EnableWarmupCache(bool) // SetBranchCache attaches a BranchCache instance for branch read/write // caching across the trie + branchEncoder. ConcurrentPatriciaHashed // implementations propagate the same instance to all mounts so the @@ -1814,9 +1812,9 @@ func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, var prevKey []byte err := t.etl.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error { - if warmuper != nil && warmuper.Cache() != nil { - warmuper.Cache().EvictPlainKey(v) - } + // (previously called warmuper.Cache().EvictPlainKey(v) here to drop + // stale account/storage entries; the warmer's cache was removed + // alongside WarmupCache so there's nothing to evict.) // Copy into arena since ETL may reuse buffers hk := t.arenaAlloc(k) pk := t.arenaAlloc(v) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 046c5492d13..f1b488ef452 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -194,19 +194,6 @@ func (sdc *SharedDomainsCommitmentContext) SetUpdates(updates *commitment.Update sdc.updates = updates } -// EnableWarmupCache enables/disables warmup cache during commitment processing. -func (sdc *SharedDomainsCommitmentContext) EnableWarmupCache(enable bool) { - sdc.patriciaTrie.EnableWarmupCache(enable) -} - -// ClearWarmupCache discards any stale account/storage values held in the active -// warmup cache. Safe to call at block boundaries between ComputeCommitment calls. -func (sdc *SharedDomainsCommitmentContext) ClearWarmupCache() { - if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok && hph.Cache() != nil { - hph.Cache().Clear() - } -} - func (sdc *SharedDomainsCommitmentContext) EnableCsvMetrics(filePathPrefix string) { sdc.patriciaTrie.EnableCsvMetrics(filePathPrefix) } @@ -501,12 +488,10 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context }() } - if recorder != nil { - // Disable warmup cache during recording — cache hits bypass the - // RecordingContext, producing incomplete traces for replay. - sdc.patriciaTrie.EnableWarmupCache(false) - defer sdc.patriciaTrie.EnableWarmupCache(true) - } + // Note: a previous EnableWarmupCache(false)+defer(true) guard lived here + // to ensure RecordingContext saw every read during trace capture. Now + // that the Go-side WarmupCache is gone, every read already goes through + // ctx.* (observable to the recorder by construction); no guard needed. var warmupConfig commitment.WarmupConfig var drainCollectors func() []*etl.Collector diff --git a/execution/commitment/hex_concurrent_patricia_hashed.go b/execution/commitment/hex_concurrent_patricia_hashed.go index 1fca7759959..7d5424b5df8 100644 --- a/execution/commitment/hex_concurrent_patricia_hashed.go +++ b/execution/commitment/hex_concurrent_patricia_hashed.go @@ -165,13 +165,6 @@ func (p *ConcurrentPatriciaHashed) SetTraceDomain(b bool) { p.mounts[i].SetTraceDomain(b) } } -func (p *ConcurrentPatriciaHashed) EnableWarmupCache(b bool) { - p.root.EnableWarmupCache(b) - for i := range p.mounts { - p.mounts[i].EnableWarmupCache(b) - } -} - // SetBranchCache attaches the same BranchCache instance to the root trie // and all 16 mounts. Sharing one cache across mounts is correct under the // concurrency contract (branch_cache.go): mounts partition the prefix diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 341438524b9..da5869765be 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -101,16 +101,6 @@ type HexPatriciaHashed struct { //temp buffers accValBuf rlp.RlpEncodedBytes - // enableWarmupCache toggles the (now-vestigial) warmup cache enable - // path through this trie. It's still propagated by callers but no - // longer attaches a Go-side cache here — branch / account / storage - // reads go straight through ctx.Branch / ctx.Account / ctx.Storage - // to SD.GetLatest, where the aggregator-scope BranchCache (for - // commitment) and OS page cache (for accounts/storage) provide the - // only caching layer. Field retained pending step 2c removal of the - // EnableWarmupCache plumbing chain. - enableWarmupCache bool - // branchCache is the BranchCache instance attached via SetBranchCache. // Production wires the aggregator-scope instance through // InitializeTrieAndUpdates; tests/benchmarks may pass a per-init @@ -214,7 +204,6 @@ func (hph *HexPatriciaHashed) resetForReuse() { hph.mounted = false hph.mountedNib = 0 - hph.enableWarmupCache = false // tracing / capture hph.capture = nil @@ -2881,18 +2870,9 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log // Setup warmup if configured var warmuper *Warmuper if warmup.Enabled { - warmup.EnableWarmupCache = hph.enableWarmupCache warmuper = NewWarmuper(ctx, warmup) warmuper.Start() defer warmuper.CloseAndWait() - // EnableWarmupCache toggle is a no-op now: HPH no longer holds a - // Go-side warmup cache. The warmer still pre-fetches via ctx.Branch / - // Account / Storage to populate sd.mem and OS page cache; the - // trie-side reads go through the same paths and hit the - // aggregator-scope BranchCache (for commitment) or page cache - // (accounts/storage). The flag plumbing is retained pending step 2c - // removal. - _ = warmup.EnableWarmupCache } err = updates.HashSort(ctx, warmuper, func(hashedKey, plainKey []byte, stateUpdate *Update) error { @@ -3029,8 +3009,6 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace } func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } -func (hph *HexPatriciaHashed) EnableWarmupCache(enable bool) { hph.enableWarmupCache = enable } - // SetBranchCache attaches a BranchCache for branch read-through and // write-through. Also propagates to the trie's BranchEncoder so encoder // writes update the cache via mark-dirty-then-Put (see CollectUpdate). @@ -3127,12 +3105,6 @@ func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } -// Cache always returns nil — HPH no longer holds a Go-side warmup -// cache. Retained as a stub so external callers (squeeze/fork-validation -// ClearWarmupCache plumbing) compile; they short-circuit on nil. -// Pending step 2c removal of the EnableWarmupCache plumbing chain. -func (hph *HexPatriciaHashed) Cache() *WarmupCache { return nil } - // verifyBranchCache, when true, makes branchFromCacheOrDB cross-check // every BranchCache hit against ctx.Branch and record a divergence on // the cache when the bytes disagree. Off by default — gate via env diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go deleted file mode 100644 index d1e73d971a0..00000000000 --- a/execution/commitment/warmup_cache.go +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package commitment - -import ( - "fmt" - "sync" - "sync/atomic" - - "github.com/erigontech/erigon/common/maphash" -) - -type branchEntry struct { - // data is the canonical encoded form (with the leading 2-byte touch-map - // prefix). Always populated by PutBranch / PutBranchIfClean. - data []byte - - // Lazy-decoded form. Populated on the first GetBranchDecoded call for - // this entry; subsequent calls return the cached decode. decodeOnce - // ensures decode runs at most once per entry even under concurrent - // reads. - // - // Encoded form is the source of truth; decoded form is derived. When - // data is replaced (PutBranch overwrites), the new entry starts fresh - // — next decoded read re-derives from the new bytes. - decodeOnce sync.Once - cells [16]cell - cellsBitmap uint16 - decodedReady bool - decodeErr error - - isEvicted atomic.Bool - // dirty signals "the canonical store has been written to since this - // entry was populated; treat as stale until cleared." Read paths - // MAY return miss when dirty is set; write paths MUST set dirty. - // Used together with PutBranchIfClean to prevent late warmup writes - // from clobbering fresh fold writes (TOCTOU race documented in the - // reth-research §4 dirty-flag pattern). Ephemeral cache today does - // not have the race because warmup completes before fold begins; - // invariant is in place ahead of cross-block persistence work. - dirty atomic.Bool -} - -type accountEntry struct { - update *Update - isEvicted atomic.Bool -} - -type storageEntry struct { - update *Update - isEvicted atomic.Bool -} - -// WarmupCache stores pre-fetched data from the warmup phase to avoid -// repeated DB reads during trie processing. Uses maphash.Map for efficient -// byte slice key lookups without string allocations. -// -// Memory pooling strategy: -// -// 1. Don't add a sync.Pool of byte buffers between the read path and -// the cache. The read path already returns mmap-backed slices -// with zero allocation (seg.Getter.NextUncompressed for the -// CompressNone domains we cache); pooling Go-side buffers above -// us just adds a layer that gets copied into the cache and then -// GC'd. The cache copy IS the canonical allocation; there's -// nothing meaningful to pool on top of it. -// -// 2. The PutBranch / PutAccount / PutStorage copies exist for -// mmap-detachment, not for buffer reuse. Snapshot files can be -// unmapped under us during collation/squeeze; copying detaches -// the cache entry from that lifetime. If tempted to "skip the -// copy because the source is fresh anyway" — check whether the -// source is mmap-backed. For AccountsDomain / StorageDomain / -// CommitmentDomain values (CompressNone in state_schema.go) it -// is, and the copy is non-negotiable. For CodeDomain -// (CompressVals) the source is a freshly decompressed buffer and -// the copy is unnecessary; that's a separate narrow -// optimisation, don't generalise. -// -// 3. Don't shorten the cache lifetime to per-block "for safety." -// The structural win is the opposite: extend it across blocks so -// overlapping prefixes (root, contract trunks) get allocated -// once and reused. Today the cache is created per -// ComputeCommitment call (hex_patricia_hashed.go ~2780) and -// torn down at block end, so the same ~26 K branches on the -// SSTORE-bloat workload are re-fetched and re-allocated every -// block. Per-block lifetime is the dominant unmeasured cost; a -// single longer-lived cache gives you both lookup efficiency -// and memory efficiency by holding exactly one copy of each -// entry. Correctness for cross-block reuse needs an -// invalidation story for entries written this block — see the -// dirty-flag scaffolding (PutBranchIfClean / MarkBranchDirty) -// for the intended hook. -type WarmupCache struct { - branches *maphash.Map[*branchEntry] - accounts *maphash.Map[*accountEntry] - storage *maphash.Map[*storageEntry] - - // Observability counters. Updated by Get/Put paths; read by Stats(). - // Atomic so warmup-worker reads and main-trie reads don't race. - branchHits, branchMisses, branchEvicted atomic.Uint64 - branchBytesServed atomic.Uint64 - accountHits, accountMisses atomic.Uint64 - storageHits, storageMisses atomic.Uint64 -} - -// NewWarmupCache creates a new warmup cache instance. -func NewWarmupCache() *WarmupCache { - return &WarmupCache{ - branches: maphash.NewMap[*branchEntry](), - accounts: maphash.NewMap[*accountEntry](), - storage: maphash.NewMap[*storageEntry](), - } -} - -// PutBranch stores branch data in the cache. -func (c *WarmupCache) PutBranch(prefix []byte, data []byte) { - // Make a copy of the data to avoid issues with buffer reuse - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - - c.branches.Set(prefix, &branchEntry{data: dataCopy}) -} - -// PutBranchIfClean stores branch data in the cache only if no existing entry -// is marked dirty. Returns true on store, false if a dirty entry was present -// (indicating the canonical store has been updated since the caller last -// read, and the caller's data is potentially stale). -// -// Use from warmup-style writers that may race with fold writes — the fold -// path marks branches dirty before its own write completes, so a warmup -// worker that reads pre-fold then attempts to write post-fold will see -// dirty=true and skip the store. -// -// Today's call sites (warmup workers in HexPatriciaHashed.Process) do not -// race with fold because warmup completes before HashSort begins. Invariant -// is in place for future cross-block persistence work where warmup-style -// writes can outlive their parent Process. -func (c *WarmupCache) PutBranchIfClean(prefix []byte, data []byte) bool { - if existing, found := c.branches.Get(prefix); found && existing.dirty.Load() { - return false - } - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - c.branches.Set(prefix, &branchEntry{data: dataCopy}) - return true -} - -// MarkBranchDirty flags a branch entry as stale-until-cleared. The next -// PutBranchIfClean for this prefix will skip; reads (GetBranch / -// GetAndEvictBranch) currently return the entry regardless — the dirty -// signal is consumed only on the write path today. -// -// Use from fold/encoder paths that have decided to overwrite a branch but -// haven't yet captured the new bytes. The mark-dirty + later actual-write -// pattern is the deferred-encoding-friendly alternative to inline -// invalidate, motivated by the prototype investigation that found inline -// invalidate breaks update-in-place semantics under deferred encoding. -func (c *WarmupCache) MarkBranchDirty(prefix []byte) { - if entry, found := c.branches.Get(prefix); found { - entry.dirty.Store(true) - } -} - -// GetBranch retrieves branch data from the cache. -func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { - entry, found := c.branches.Get(prefix) - if !found { - c.branchMisses.Add(1) - return nil, false - } - if entry.isEvicted.Load() { - c.branchEvicted.Add(1) - return nil, false - } - c.branchHits.Add(1) - c.branchBytesServed.Add(uint64(len(entry.data))) - return entry.data, true -} - -// GetBranchDecoded retrieves the cached branch in decoded form. Lazy-decodes -// on first access for each entry; subsequent reads return the cached cells -// pointer without redoing the parse work. -// -// Returns the bitmap of present children plus a pointer to the populated -// cells array. The caller derives touchMap/afterMap from the bitmap based -// on its own context (deleted vs present-after) — the cache stores cells -// independent of that context so the same entry serves both readers. -// -// Returns ok=false on miss or eviction (same semantics as GetBranch). Also -// returns ok=false if the cached bytes fail to decode — in that case the -// caller should fall through to a re-read from the canonical store. -// -// The returned *[16]cell pointer aliases storage owned by the cache entry -// — the caller MUST NOT modify the cells in place. Read-only consumption -// is safe across concurrent GetBranchDecoded calls (decode runs at most -// once per entry; subsequent reads see the populated cells). -func (c *WarmupCache) GetBranchDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { - entry, found := c.branches.Get(prefix) - if !found { - c.branchMisses.Add(1) - return 0, nil, false - } - if entry.isEvicted.Load() { - c.branchEvicted.Add(1) - return 0, nil, false - } - entry.decodeOnce.Do(func() { - // PutBranch stores bytes WITH the leading 2-byte touch-map prefix; - // DecodeBranchInto consumes bytes WITHOUT it (matching the - // unfoldBranchNode call pattern that strips the prefix before - // decoding). - if len(entry.data) < 2 { - entry.decodeErr = fmt.Errorf("branch entry too short for touch-map prefix: %d bytes", len(entry.data)) - return - } - maps, err := DecodeBranchInto(entry.data[2:], false /* deleted derived per-caller */, &entry.cells) - if err != nil { - entry.decodeErr = err - return - } - entry.cellsBitmap = maps.Bitmap - entry.decodedReady = true - }) - if !entry.decodedReady { - // Decode failed — caller should re-read from canonical store. - // Don't count as hit (the entry is mechanically present but - // unusable). Don't increment misses either — the caller will - // observe ok=false and decide. - return 0, nil, false - } - c.branchHits.Add(1) - c.branchBytesServed.Add(uint64(len(entry.data))) - return entry.cellsBitmap, &entry.cells, true -} - -// GetAndEvictBranch retrieves branch data and marks the entry as evicted in one operation. -func (c *WarmupCache) GetAndEvictBranch(prefix []byte) ([]byte, bool) { - entry, found := c.branches.Get(prefix) - if !found { - c.branchMisses.Add(1) - return nil, false - } - if entry.isEvicted.Load() { - c.branchEvicted.Add(1) - return nil, false - } - entry.isEvicted.Store(true) - c.branchHits.Add(1) - c.branchBytesServed.Add(uint64(len(entry.data))) - return entry.data, true -} - -// EvictBranch marks a branch entry as evicted without retrieving it. -func (c *WarmupCache) EvictBranch(prefix []byte) { - entry, found := c.branches.Get(prefix) - if found { - entry.isEvicted.Store(true) - } -} - -// PutAccount stores account data in the cache. -func (c *WarmupCache) PutAccount(plainKey []byte, update *Update) { - var updateCopy *Update - if update != nil { - updateCopy = update.Copy() - } - - c.accounts.Set(plainKey, &accountEntry{update: updateCopy}) -} - -// GetAccount retrieves account data from the cache. -func (c *WarmupCache) GetAccount(plainKey []byte) (*Update, bool) { - entry, found := c.accounts.Get(plainKey) - if !found || entry.isEvicted.Load() { - c.accountMisses.Add(1) - return nil, false - } - c.accountHits.Add(1) - return entry.update, true -} - -// GetAndEvictAccount retrieves account data and marks the entry as evicted in one operation. -// Returns the entry pointer allowing the caller to read the data before it's considered evicted. -func (c *WarmupCache) GetAndEvictAccount(plainKey []byte) *accountEntry { - entry, found := c.accounts.Get(plainKey) - if !found || entry.isEvicted.Load() { - return nil - } - entry.isEvicted.Store(true) - return entry -} - -// EvictAccount marks an account entry as evicted without retrieving it. -func (c *WarmupCache) EvictAccount(plainKey []byte) { - entry, found := c.accounts.Get(plainKey) - if found { - entry.isEvicted.Store(true) - } -} - -// PutStorage stores storage data in the cache. -func (c *WarmupCache) PutStorage(plainKey []byte, update *Update) { - var updateCopy *Update - if update != nil { - updateCopy = update.Copy() - } - - c.storage.Set(plainKey, &storageEntry{update: updateCopy}) -} - -// GetStorage retrieves storage data from the cache. -func (c *WarmupCache) GetStorage(plainKey []byte) (*Update, bool) { - entry, found := c.storage.Get(plainKey) - if !found || entry.isEvicted.Load() { - c.storageMisses.Add(1) - return nil, false - } - c.storageHits.Add(1) - return entry.update, true -} - -// GetAndEvictStorage retrieves storage data and marks the entry as evicted in one operation. -// Returns the entry pointer allowing the caller to read the data before it's considered evicted. -func (c *WarmupCache) GetAndEvictStorage(plainKey []byte) *storageEntry { - entry, found := c.storage.Get(plainKey) - if !found || entry.isEvicted.Load() { - return nil - } - entry.isEvicted.Store(true) - return entry -} - -// EvictStorage marks a storage entry as evicted without retrieving it. -func (c *WarmupCache) EvictStorage(plainKey []byte) { - entry, found := c.storage.Get(plainKey) - if found { - entry.isEvicted.Store(true) - } -} - -// EvictPlainKey evicts a key from both accounts and storage caches. -// Use this when you don't know if the key is an account or storage key. -func (c *WarmupCache) EvictPlainKey(plainKey []byte) { - if entry, found := c.accounts.Get(plainKey); found { - entry.isEvicted.Store(true) - } - if entry, found := c.storage.Get(plainKey); found { - entry.isEvicted.Store(true) - } -} - -// Clear clears all cached data and resets stats counters. -func (c *WarmupCache) Clear() { - c.branches = maphash.NewMap[*branchEntry]() - c.accounts = maphash.NewMap[*accountEntry]() - c.storage = maphash.NewMap[*storageEntry]() - c.ResetStats() -} - -// Stats returns a one-line summary of cache hit/miss counters. -// Format matches the per-block log line: branch, account, storage with -// hit-percentages and bytes served. Useful in commitment debug logs and -// for the per-Process LogCommitments line. -func (c *WarmupCache) Stats() string { - bh, bm, be := c.branchHits.Load(), c.branchMisses.Load(), c.branchEvicted.Load() - bb := c.branchBytesServed.Load() - ah, am := c.accountHits.Load(), c.accountMisses.Load() - sh, sm := c.storageHits.Load(), c.storageMisses.Load() - pct := func(hit, miss uint64) float64 { - total := hit + miss - if total == 0 { - return 0 - } - return 100.0 * float64(hit) / float64(total) - } - return fmt.Sprintf( - "branch hit=%d miss=%d evict=%d (%.1f%%, %.1f MiB) | acct hit=%d miss=%d (%.1f%%) | stor hit=%d miss=%d (%.1f%%)", - bh, bm, be, pct(bh, bm), float64(bb)/1024/1024, - ah, am, pct(ah, am), - sh, sm, pct(sh, sm), - ) -} - -// ResetStats zeros out all hit/miss/byte counters without touching cached data. -func (c *WarmupCache) ResetStats() { - c.branchHits.Store(0) - c.branchMisses.Store(0) - c.branchEvicted.Store(0) - c.branchBytesServed.Store(0) - c.accountHits.Store(0) - c.accountMisses.Store(0) - c.storageHits.Store(0) - c.storageMisses.Store(0) -} diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index cb6997ebd58..807f6264eed 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -55,12 +55,11 @@ type TrieContextFactory func() (PatriciaContext, func()) // WarmupConfig contains configuration for pre-warming MDBX page cache // during commitment processing. type WarmupConfig struct { - Enabled bool - EnableWarmupCache bool // If true, cache warmed data for use during trie processing - CtxFactory TrieContextFactory - NumWorkers int - MaxDepth int - LogPrefix string + Enabled bool + CtxFactory TrieContextFactory + NumWorkers int + MaxDepth int + LogPrefix string } const WarmupMaxDepth = 128 // covers full key paths for both account keys (64 nibbles) and storage keys (128 nibbles) @@ -85,9 +84,6 @@ type Warmuper struct { // Worker group g *errgroup.Group - // Cache for storing warmed data to be used during trie processing - cache *WarmupCache - // Stats keysProcessed atomic.Uint64 startTime time.Time @@ -105,7 +101,7 @@ type warmupWorkItem struct { // NewWarmuper creates a new Warmuper instance. func NewWarmuper(ctx context.Context, cfg WarmupConfig) *Warmuper { ctx, cancel := context.WithCancel(ctx) - w := &Warmuper{ + return &Warmuper{ ctx: ctx, cancel: cancel, ctxFactory: cfg.CtxFactory, @@ -113,65 +109,35 @@ func NewWarmuper(ctx context.Context, cfg WarmupConfig) *Warmuper { numWorkers: cfg.NumWorkers, logPrefix: cfg.LogPrefix, } - if cfg.EnableWarmupCache { - w.cache = NewWarmupCache() - } - return w -} - -// Cache returns the warmup cache, or nil if caching is disabled. -func (w *Warmuper) Cache() *WarmupCache { - return w.cache } -// branchFromCacheOrDB reads branch data from cache if available, otherwise from DB and caches it. +// branchFromCacheOrDB reads branch data via ctx.Branch. The Go-side +// warmup cache that previously sat above this has been removed; the +// aggregator-scope BranchCache (reached through SD.GetLatest) is the +// only branch cache. The function is kept as a thin wrapper to +// preserve the warmer-side error handling shape. func (w *Warmuper) branchFromCacheOrDB(trieCtx PatriciaContext, prefix []byte) ([]byte, error) { - if w.cache != nil { - if data, found := w.cache.GetBranch(prefix); found { - return data, nil - } - } branchData, _, err := trieCtx.Branch(prefix) - if err != nil { - return nil, err - } - if w.cache != nil && len(branchData) > 0 { - w.cache.PutBranch(prefix, branchData) - } - return branchData, nil + return branchData, err } -// accountFromCacheOrDB reads account data from cache if available, otherwise from DB and caches it. +// accountFromCacheOrDB reads account data via ctx.Account. No Go-side +// caching; the read populates the OS page cache as a side effect. func (w *Warmuper) accountFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - if w.cache != nil { - if update, found := w.cache.GetAccount(plainKey); found { - return update, nil - } - } update, err := trieCtx.Account(plainKey) if err != nil { return nil, err } - if w.cache != nil { - w.cache.PutAccount(plainKey, update) - } return update, nil } -// storageFromCacheOrDB reads storage data from cache if available, otherwise from DB and caches it. +// storageFromCacheOrDB reads storage data via ctx.Storage. No Go-side +// caching; the read populates the OS page cache as a side effect. func (w *Warmuper) storageFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - if w.cache != nil { - if update, found := w.cache.GetStorage(plainKey); found { - return update, nil - } - } update, err := trieCtx.Storage(plainKey) if err != nil { return nil, err } - if w.cache != nil { - w.cache.PutStorage(plainKey, update) - } return update, nil } diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 6dc31c94eca..af559768993 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -205,8 +205,6 @@ func ExecV3(ctx context.Context, doms.EnableParaTrieDB(cfg.db) doms.EnableTrieWarmup(true) - // Do it only for chain-tip blocks! - doms.EnableWarmupCache(!isApplyingBlocks) doms.SetDeferCommitmentUpdates(false) // Enable deferred commitment updates for fork validation and parallel initial sync. // Deferred updates batch commitment calculations to block boundaries rather than From b99c1926440046f9ae21d5f94e0205e32086c745 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:16:07 +0000 Subject: [PATCH 040/167] commitment: mark BranchCache dirty in CollectDeferredUpdate too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CollectUpdate (immediate path) calls be.branchCache.MarkDirty(prefix) before encoding so concurrent warmer-style writers see the dirty flag and skip via PutIfClean. CollectDeferredUpdate (deferred path, used by parallel commitment) was missing the same call — meaning a prefix queued for deferred apply could be overwritten by a stale PutIfClean from the warmup path between enqueue and eventual ApplyDeferredBranchUpdates write. Add the matching MarkDirty call so both encoder entry points consistently invalidate the cache. SD.Flush stays the canonical update site (Invalidate + Put on each flushed prefix); the encoder hooks are the only other places branch writes originate. Step 3 of the WarmupCache deletion / BranchCache consolidation. --- execution/commitment/commitment.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index ecaf7b4b107..612b06080f9 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -657,6 +657,17 @@ func (be *BranchEncoder) CollectDeferredUpdate( be.ClearDeferred() } + // Mark BranchCache entry dirty BEFORE the deferred enqueue. Mirrors + // the CollectUpdate path so that both immediate and deferred encoder + // entries consistently invalidate the cache. Concurrent warmer-style + // writers calling PutIfClean for this prefix between now and the + // eventual ApplyDeferredBranchUpdates write see the dirty flag and + // skip — preventing a stale read from racing the deferred write. + // See branch_cache.go's Concurrency Contract. + if be.branchCache != nil { + be.branchCache.MarkDirty(prefix) + } + prev, _, err := ctx.Branch(prefix) if err != nil { return err From 80ea439f12df79a9ab38b8247946e37c40ad3687 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:41:49 +0000 Subject: [PATCH 041/167] =?UTF-8?q?commitment/warmuper:=20scope=20warmer?= =?UTF-8?q?=20to=20branches=20only=20=E2=80=94=20drop=20account/storage=20?= =?UTF-8?q?prefetch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Branch warm-up's scope is branches. Account/storage data is downstream of the executor; if a leaf hash needs it during fold, the data is either (a) memoized as the cell's stateHash, (b) carried in the Updates buffer from the executor, or (c) faulted on the trie's slow path — which signals a memoization gap to fix at the trie layer, not to paper over with a separate prefetcher. The warmer was reading sibling cell addresses out of branch bytes (via extractBranchCellAddresses) and pre-fetching the underlying accounts/storage from BTree to warm the OS page cache. Evidence from the SSTORE-bloat bench shows this work is unused on the workloads we measure: disk_sto=0 / disk_acc=0 in the cache-fp log means the trie's fold-time fall-through never fires. The path cells that *are* extracted by the helper are already warm from the executor's reads (EIP-2200 SSTORE gas calc on the same slots); sibling cells with memoized stateHashes are correctly filtered out. Removes: - Warmuper.warmupKey: cellAccounts/cellStorages prefetch loops - Warmuper.accountFromCacheOrDB / storageFromCacheOrDB - extractBranchCellAddresses (only caller goes away; skipCellFields stays — warmuper bitmap walk + trie_reader use it) The disk_sto / disk_acc counters stay in place as the canary: any future workload that drives them non-zero is a signal that something needs the underlying data, which is itself a question for the trie / IBS integration layer, not the warmer. Step 4 of the WarmupCache deletion / BranchCache consolidation. Verified: build green; verify-bench.sh expected to PASS (the path that was being removed is unused on the bench). --- execution/commitment/hex_patricia_hashed.go | 111 -------------------- execution/commitment/warmuper.go | 37 ++----- 2 files changed, 9 insertions(+), 139 deletions(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index da5869765be..ce4bb76cd84 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -588,117 +588,6 @@ func skipCellFields(data []byte, pos int, fieldBits byte) int { return pos } -// extractBranchCellAddresses extracts account/storage addresses from branch data. -// pathNibble is the nibble on the update path (-1 if none) - its addresses are always -// extracted since that cell will have stateHash cleared during update. -// Sibling cells with stateHash (memoized) are skipped. -func extractBranchCellAddresses(branchData []byte, pathNibble int) (accountAddrs [][]byte, storageAddrs [][]byte) { - if len(branchData) < 4 { - return nil, nil - } - // Skip touch map (first 2 bytes) - branchData = branchData[2:] - bitmap := binary.BigEndian.Uint16(branchData[0:2]) - pos := 2 - - // Iterate over all cells in the branch - for bitset := bitmap; bitset != 0; { - bit := bitset & -bitset - nibble := bits.TrailingZeros16(bit) - bitset ^= bit - - if pos >= len(branchData) { - break - } - fieldBits := branchData[pos] - pos++ - - // Cell on update path will have stateHash cleared - always extract its addresses. - // Sibling cells with stateHash (bit 4) are memoized - skip them. - isOnPath := nibble == pathNibble - hasMemoizedHash := fieldBits&16 != 0 - shouldExtract := isOnPath || !hasMemoizedHash - - // Parse each field - // extension (bit 0) - if fieldBits&1 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n + int(l) - } - - // accountAddr (bit 1) - if fieldBits&2 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n - if l > 0 && pos+int(l) <= len(branchData) { - if shouldExtract { - addr := make([]byte, l) - copy(addr, branchData[pos:pos+int(l)]) - accountAddrs = append(accountAddrs, addr) - } - pos += int(l) - } - } - - // storageAddr (bit 2) - if fieldBits&4 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n - if l > 0 && pos+int(l) <= len(branchData) { - if shouldExtract { - addr := make([]byte, l) - copy(addr, branchData[pos:pos+int(l)]) - storageAddrs = append(storageAddrs, addr) - } - pos += int(l) - } - } - - // hash (bit 3) - if fieldBits&8 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n + int(l) - } - - // stateHash (bit 4) - if fieldBits&16 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n + int(l) - } - } - return accountAddrs, storageAddrs -} - func (cell *cell) accountForHashing(buffer []byte, storageRootHash common.Hash) int { balanceBytes := 0 if !cell.Balance.LtUint64(128) { diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index 807f6264eed..f71a73fa312 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -121,26 +121,6 @@ func (w *Warmuper) branchFromCacheOrDB(trieCtx PatriciaContext, prefix []byte) ( return branchData, err } -// accountFromCacheOrDB reads account data via ctx.Account. No Go-side -// caching; the read populates the OS page cache as a side effect. -func (w *Warmuper) accountFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - update, err := trieCtx.Account(plainKey) - if err != nil { - return nil, err - } - return update, nil -} - -// storageFromCacheOrDB reads storage data via ctx.Storage. No Go-side -// caching; the read populates the OS page cache as a side effect. -func (w *Warmuper) storageFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - update, err := trieCtx.Storage(plainKey) - if err != nil { - return nil, err - } - return update, nil -} - // Start initializes and starts the warmup workers. func (w *Warmuper) Start() { if w.started.Swap(true) { @@ -201,14 +181,15 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep } nextNibble := int(hashedKey[depth]) - // Extract and prefetch account/storage addresses to warm page cache - cellAccounts, cellStorages := extractBranchCellAddresses(branchData, nextNibble) - for _, addr := range cellAccounts { - _, _ = w.accountFromCacheOrDB(trieCtx, addr) - } - for _, addr := range cellStorages { - _, _ = w.storageFromCacheOrDB(trieCtx, addr) - } + // Account/storage prefetch removed: the warmer's scope is branch + // warm-up only. If a leaf hash needs underlying account/storage + // data during fold, that data is either (a) memoized as the + // cell's stateHash, (b) carried in the Updates buffer from the + // executor, or (c) faulted on the trie's own slow path — + // signalling a memoization gap that wants fixing at the trie + // layer, not papered over by a separate prefetcher. Counters + // disk_sto / disk_acc on the [commitment][cache-fp] log line + // expose any such fall-through. branchData = branchData[2:] // skip touch map From 614fb7bcc188884c46c206642294dd8b657d1f0e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 14:18:22 +0000 Subject: [PATCH 042/167] state, commitment: add unique-prefix file-read counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface read amplification on the cache-fp log: alongside the existing per-domain FileReadCount (total file reads), track UniqueFileReadCount (distinct prefixes). The ratio FileReadCount / UniqueFileReadCount tells us how often the same prefix was re-read from the file layer. On the SSTORE-bloat workload, files_comm=26K commitment file reads for ~5,910 slots in a single contract is suspicious: the account-prefix path is constant for all 5,910 slots and the storage subtree's structural branch count is ~3,000-3,500. ~8x amplification; uniq_comm tells us how much of that is distinct prefix coverage vs repeat reads on the same prefix. DomainMetrics gets a sync.Map dedup keyed by ":" — process-cumulative, gated by the existing dbg.KVReadLevelledMetrics flag so production isn't charged when metrics are off. UpdateFileReadsUnique replaces UpdateFileReads at the two MeteredGetLatest call sites (aggregator.go + domain.go). --- db/state/aggregator.go | 7 +- db/state/changeset/state_changeset.go | 84 +++++++++++++++++++ db/state/domain.go | 2 +- .../commitmentdb/commitment_context.go | 11 ++- 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 93ff05ed6d0..387e4f4a906 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2503,7 +2503,12 @@ func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxSte return nil, kv.Step(0), false, err } if metrics != nil && dbg.KVReadLevelledMetrics { - metrics.UpdateFileReads(domain, start) + // UpdateFileReadsUnique tracks both total reads and distinct + // prefixes. The ratio FileReadCount / UniqueFileReadCount is + // the read amplification factor — useful when investigating + // whether the same prefixes are being re-read from file layer + // (cache misses on hot prefixes). + metrics.UpdateFileReadsUnique(domain, k, start) } v, err = at.replaceShortenedKeysInBranch(k, commitment.BranchData(v), fileStartTxNum, fileEndTxNum) return v, kv.Step(fileEndTxNum / at.StepSize()), found, err diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 057eba818db..ff5d2fab30b 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -464,12 +464,35 @@ type DomainIOMetrics struct { DbReadDuration time.Duration FileReadCount int64 FileReadDuration time.Duration + // UniqueFileReadCount tracks distinct prefixes that ever fell through + // to a file read (process-cumulative). The ratio FileReadCount / + // UniqueFileReadCount is the read amplification factor: how many + // times each unique prefix was re-read from the file layer (cache + // misses on the same prefix). Updated by UpdateFileReadsUnique; + // gated by dbg.KVReadLevelledMetrics same as FileReadCount. + UniqueFileReadCount int64 + + // StateCache hit/miss tracks the SharedDomains.stateCache layer + // specifically (the per-execution Account/Storage/Code cache), distinct + // from CacheReadCount which counts sd.mem and sd.parent.mem hits. + // Hit means stateCache.Get returned ok (we skipped MDBX+files). + // Miss means stateCache.Get returned !ok and we fell through to aggTx. + StateCacheHitCount int64 + StateCacheMissCount int64 } type DomainMetrics struct { sync.RWMutex DomainIOMetrics Domains map[kv.Domain]*DomainIOMetrics + + // seenFileReads is a process-cumulative dedup set for + // UpdateFileReadsUnique. Keys are (domain.String() + prefixBytes); + // LoadOrStore on each read decides whether the prefix is new and + // therefore contributes to UniqueFileReadCount. Held outside the + // mutex (sync.Map handles its own concurrency) so the unique check + // doesn't serialise with other metric updates. + seenFileReads sync.Map } func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { @@ -506,6 +529,28 @@ func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { } } +func (dm *DomainMetrics) UpdateStateCacheHit(domain kv.Domain) { + dm.Lock() + defer dm.Unlock() + dm.StateCacheHitCount++ + if d, ok := dm.Domains[domain]; ok { + d.StateCacheHitCount++ + } else { + dm.Domains[domain] = &DomainIOMetrics{StateCacheHitCount: 1} + } +} + +func (dm *DomainMetrics) UpdateStateCacheMiss(domain kv.Domain) { + dm.Lock() + defer dm.Unlock() + dm.StateCacheMissCount++ + if d, ok := dm.Domains[domain]; ok { + d.StateCacheMissCount++ + } else { + dm.Domains[domain] = &DomainIOMetrics{StateCacheMissCount: 1} + } +} + func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { dm.Lock() defer dm.Unlock() @@ -522,3 +567,42 @@ func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { } } } + +// UpdateFileReadsUnique records a file read while also tracking whether +// the prefix has been seen before for this domain. Same gate as +// UpdateFileReads (dbg.KVReadLevelledMetrics); call this instead when +// the read-amplification ratio (FileReadCount / UniqueFileReadCount) +// is wanted on the metric output. The key bytes are copied into the +// internal dedup map; do not mutate after the call. +func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { + // Composite ":" so two domains can hold the same prefix + // shape (commitment compact-encoded paths vs accounts plain addresses) + // without colliding. + domainKey := domain.String() + ":" + string(key) + _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) + + dm.Lock() + defer dm.Unlock() + dm.FileReadCount++ + readDuration := time.Since(start) + dm.FileReadDuration += readDuration + if !alreadySeen { + dm.UniqueFileReadCount++ + } + if d, ok := dm.Domains[domain]; ok { + d.FileReadCount++ + d.FileReadDuration += readDuration + if !alreadySeen { + d.UniqueFileReadCount++ + } + } else { + newD := &DomainIOMetrics{ + FileReadCount: 1, + FileReadDuration: readDuration, + } + if !alreadySeen { + newD.UniqueFileReadCount = 1 + } + dm.Domains[domain] = newD + } +} diff --git a/db/state/domain.go b/db/state/domain.go index d7b52ff7fbe..d9158f5ce42 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -1641,7 +1641,7 @@ func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics v, foundInFile, _, endTxNum, err := dt.getLatestFromFiles(key, 0) if metrics != nil && dbg.KVReadLevelledMetrics { - metrics.UpdateFileReads(dt.name, start) + metrics.UpdateFileReadsUnique(dt.name, key, start) } if err != nil { diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index f1b488ef452..410557706bd 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -362,19 +362,24 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // (branch reads), Storage (value loads), Account, Code. // All cumulative; deltas via successive lines. var aFiles, sFiles, cFiles, mFiles int64 + var aUniq, sUniq, cUniq, mUniq int64 if m := sdc.sharedDomains.Metrics(); m != nil { m.RLock() if d := m.Domains[kv.AccountsDomain]; d != nil { aFiles = d.FileReadCount + aUniq = d.UniqueFileReadCount } if d := m.Domains[kv.StorageDomain]; d != nil { sFiles = d.FileReadCount + sUniq = d.UniqueFileReadCount } if d := m.Domains[kv.CodeDomain]; d != nil { cFiles = d.FileReadCount + cUniq = d.UniqueFileReadCount } if d := m.Domains[kv.CommitmentDomain]; d != nil { mFiles = d.FileReadCount + mUniq = d.UniqueFileReadCount } m.RUnlock() } @@ -401,7 +406,11 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, - "files_comm", mFiles) + "files_comm", mFiles, + "uniq_acc", aUniq, + "uniq_sto", sUniq, + "uniq_code", cUniq, + "uniq_comm", mUniq) } } } From 4b95d45e6b510125f81764a319de9d17f80931c2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 14:44:58 +0000 Subject: [PATCH 043/167] state, commitment: add unique-prefix length histogram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface byte-length distribution of distinct prefixes that fall through to a file read. Buckets: 1B (depth 0-1, root) 2-4B (depth 2-7) 5-8B (depth 8-15) 9-16B (depth 16-31) 17-32B (depth 32-63) 33-64B (depth 64-127) >64B (out of range) Tells us where the 25K commitment reads concentrate. If they cluster in the 33-64B / >64B range, they're storage-subtree leaf-parents (per-slot, structurally irreducible). If they cluster shallower (account-trunk / contract-trunk), there's a caching lever to pin those prefixes — e.g. an account-root cache over the first X nibbles per contract. Compact-encoded prefix length is depth/2 + 1, with the parity flag in the first nibble (HP encoding from Yellow Paper). See nibbles.HexToCompact. --- db/state/changeset/state_changeset.go | 39 +++++++++++++++++++ .../commitmentdb/commitment_context.go | 8 +++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index ff5d2fab30b..8c03b394491 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -471,6 +471,20 @@ type DomainIOMetrics struct { // misses on the same prefix). Updated by UpdateFileReadsUnique; // gated by dbg.KVReadLevelledMetrics same as FileReadCount. UniqueFileReadCount int64 + // UniqueLenBuckets is the byte-length distribution of distinct + // prefixes seen by UpdateFileReadsUnique. The compact-encoded + // commitment prefix carries depth-dependent length: + // 0 : 1 byte (root / depth 0) + // 1 : 2-4 bytes + // 2 : 5-8 bytes + // 3 : 9-16 bytes + // 4 : 17-32 bytes + // 5 : 33-64 bytes + // 6 : 65+ bytes (out of range for storage paths) + // Used to investigate depth distribution of touched prefixes — + // e.g., are the 25K commitment reads mostly leaf-parents (deep) + // or trunk (shallow)? + UniqueLenBuckets [7]int64 // StateCache hit/miss tracks the SharedDomains.stateCache layer // specifically (the per-execution Account/Storage/Code cache), distinct @@ -568,6 +582,27 @@ func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { } } +// lenBucket maps a prefix byte-length to its UniqueLenBuckets index. +// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. +func lenBucket(n int) int { + switch { + case n <= 1: + return 0 + case n <= 4: + return 1 + case n <= 8: + return 2 + case n <= 16: + return 3 + case n <= 32: + return 4 + case n <= 64: + return 5 + default: + return 6 + } +} + // UpdateFileReadsUnique records a file read while also tracking whether // the prefix has been seen before for this domain. Same gate as // UpdateFileReads (dbg.KVReadLevelledMetrics); call this instead when @@ -580,6 +615,7 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta // without colliding. domainKey := domain.String() + ":" + string(key) _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) + bucket := lenBucket(len(key)) dm.Lock() defer dm.Unlock() @@ -588,12 +624,14 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta dm.FileReadDuration += readDuration if !alreadySeen { dm.UniqueFileReadCount++ + dm.UniqueLenBuckets[bucket]++ } if d, ok := dm.Domains[domain]; ok { d.FileReadCount++ d.FileReadDuration += readDuration if !alreadySeen { d.UniqueFileReadCount++ + d.UniqueLenBuckets[bucket]++ } } else { newD := &DomainIOMetrics{ @@ -602,6 +640,7 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta } if !alreadySeen { newD.UniqueFileReadCount = 1 + newD.UniqueLenBuckets[bucket] = 1 } dm.Domains[domain] = newD } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 410557706bd..57a835a76c3 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -363,6 +363,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // All cumulative; deltas via successive lines. var aFiles, sFiles, cFiles, mFiles int64 var aUniq, sUniq, cUniq, mUniq int64 + var mLens [7]int64 if m := sdc.sharedDomains.Metrics(); m != nil { m.RLock() if d := m.Domains[kv.AccountsDomain]; d != nil { @@ -380,9 +381,13 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context if d := m.Domains[kv.CommitmentDomain]; d != nil { mFiles = d.FileReadCount mUniq = d.UniqueFileReadCount + mLens = d.UniqueLenBuckets } m.RUnlock() } + // commLens compact form: "1B:N0|2-4B:N1|5-8B:N2|9-16B:N3|17-32B:N4|33-64B:N5|>64B:N6" + commLens := fmt.Sprintf("1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33-64B:%d|>64B:%d", + mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], mLens[5], mLens[6]) log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), @@ -410,7 +415,8 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "uniq_acc", aUniq, "uniq_sto", sUniq, "uniq_code", cUniq, - "uniq_comm", mUniq) + "uniq_comm", mUniq, + "comm_lens", commLens) } } } From d566e9b163c21d6327926cb3713c3fa0af679e3e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:22:26 +0000 Subject: [PATCH 044/167] state, commitment: refine length histogram inside storage subtree The prior 33-64B bucket spanned all storage-subtree depths (64-127), too coarse to distinguish per-contract storage-trunk reads (where pinning would help) from leaf-parent reads (where it doesn't). Sub-divide into four buckets: 33B : storage subtree root for a contract (depth 64) 34-36B : storage subtree top, depth 65-70 (per-contract trunk) 37-44B : mid storage, depth 71-86 45-64B : leaf-parents and deep paths, depth 87-127 If most of the 25K commitment reads concentrate at 33B / 34-36B, the per-contract storage-trunk reads multiply across all touched slots within a contract; pinning the contract's first few storage- subtree branches (the user's 'storage root trunk cache for big accounts' idea) would absorb them. If reads concentrate at 45-64B, paths are inherently per-slot and pinning the trunk wouldn't help. --- db/state/changeset/state_changeset.go | 43 ++++++++++++------- .../commitmentdb/commitment_context.go | 13 ++++-- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 8c03b394491..0e30f687897 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -473,18 +473,23 @@ type DomainIOMetrics struct { UniqueFileReadCount int64 // UniqueLenBuckets is the byte-length distribution of distinct // prefixes seen by UpdateFileReadsUnique. The compact-encoded - // commitment prefix carries depth-dependent length: - // 0 : 1 byte (root / depth 0) - // 1 : 2-4 bytes - // 2 : 5-8 bytes - // 3 : 9-16 bytes - // 4 : 17-32 bytes - // 5 : 33-64 bytes - // 6 : 65+ bytes (out of range for storage paths) - // Used to investigate depth distribution of touched prefixes — - // e.g., are the 25K commitment reads mostly leaf-parents (deep) - // or trunk (shallow)? - UniqueLenBuckets [7]int64 + // prefix length is 1 + ⌈depth/2⌉ bytes (HP encoding), so byte + // length maps to trie depth as follows: + // 0 : 1 byte (depth 0-1, root) + // 1 : 2-4 bytes (depth 2-7, top trunk) + // 2 : 5-8 bytes (depth 8-15) + // 3 : 9-16 bytes (depth 16-31) + // 4 : 17-32 bytes (depth 32-63, near account leaf) + // 5 : 33 bytes (depth 64, storage subtree root) + // 6 : 34-36 bytes (depth 65-70, storage subtree top) + // 7 : 37-44 bytes (depth 71-86, mid storage) + // 8 : 45-64 bytes (depth 87-127, leaf-parents and deep) + // 9 : 65+ bytes (out of range) + // Used to localise where the per-block file reads concentrate — + // e.g., are the 25K commitment reads on bloat workload + // storage-subtree-trunk (where per-contract pinning helps) or + // leaf-parents (where it doesn't)? + UniqueLenBuckets [10]int64 // StateCache hit/miss tracks the SharedDomains.stateCache layer // specifically (the per-execution Account/Storage/Code cache), distinct @@ -583,7 +588,9 @@ func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { } // lenBucket maps a prefix byte-length to its UniqueLenBuckets index. -// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. +// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. Buckets +// 5-8 sub-divide the storage subtree (depths 64-127) so per-contract +// trunk vs deep leaf-parent reads are distinguishable. func lenBucket(n int) int { switch { case n <= 1: @@ -596,10 +603,16 @@ func lenBucket(n int) int { return 3 case n <= 32: return 4 - case n <= 64: + case n == 33: return 5 - default: + case n <= 36: return 6 + case n <= 44: + return 7 + case n <= 64: + return 8 + default: + return 9 } } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 57a835a76c3..a8bcf782dff 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -363,7 +363,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // All cumulative; deltas via successive lines. var aFiles, sFiles, cFiles, mFiles int64 var aUniq, sUniq, cUniq, mUniq int64 - var mLens [7]int64 + var mLens [10]int64 if m := sdc.sharedDomains.Metrics(); m != nil { m.RLock() if d := m.Domains[kv.AccountsDomain]; d != nil { @@ -385,9 +385,14 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context } m.RUnlock() } - // commLens compact form: "1B:N0|2-4B:N1|5-8B:N2|9-16B:N3|17-32B:N4|33-64B:N5|>64B:N6" - commLens := fmt.Sprintf("1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33-64B:%d|>64B:%d", - mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], mLens[5], mLens[6]) + // commLens shows depth distribution. Buckets within the storage + // subtree (33B+) are sub-divided to distinguish per-contract + // storage trunk (33B / 34-36B) from leaf-parent depths + // (45-64B). See state_changeset.go UniqueLenBuckets. + commLens := fmt.Sprintf( + "1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33B:%d|34-36B:%d|37-44B:%d|45-64B:%d|>64B:%d", + mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], + mLens[5], mLens[6], mLens[7], mLens[8], mLens[9]) log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), From 65fbd1422a4f487b7e031704e3ab56c59a939102 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:46:54 +0000 Subject: [PATCH 045/167] state/changeset: trunk-probe log to identify hot contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gated by TRUNK_PROBE=true. On first sight of a depth-64 commitment prefix (33-byte: 0x00 + 32 bytes of contract hash), log the contract hash. The trailing 32 bytes ARE keccak256(addr) and identify which contracts have storage activity in the workload. Used to pick the bloat contract for the per-contract storage-trunk-pin prototype. PIN_CONTRACT_TRUNKS env var is also threaded through the launcher (no consumer yet — placeholder for the next commit that adds the PinEntry mechanism). --- db/state/changeset/state_changeset.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 0e30f687897..04b0667a95d 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -18,6 +18,7 @@ package changeset import ( "encoding/binary" + "encoding/hex" "fmt" "math" "strings" @@ -27,11 +28,19 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/length" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/execution/types/accounts" ) +// logTrunkProbe is set by TRUNK_PROBE=true to log the keccak256 hash of +// each contract whose storage subtree root (depth-64 commitment prefix, +// 33 bytes) is read from the file layer for the first time. The bytes +// after the 0x00 flag byte ARE the contract's keccak256(addr), which +// is what PIN_CONTRACT_TRUNKS expects in its hex form. +var logTrunkProbe = dbg.EnvBool("TRUNK_PROBE", false) + type StateChangeSet struct { Diffs [kv.DomainLen]kv.DomainDiff // there are 4 domains of state changes } @@ -630,6 +639,13 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) bucket := lenBucket(len(key)) + // Trunk-probe log: depth-64 commitment prefix on first sight. + // Format key as 0x00 || keccak256(addr); the trailing 32 bytes are + // the value to feed to PIN_CONTRACT_TRUNKS for that contract. + if logTrunkProbe && !alreadySeen && bucket == 5 && domain == kv.CommitmentDomain && len(key) == 33 && key[0] == 0x00 { + log.Info("[trunk-probe] new depth-64 commitment prefix", "contract_hash", hex.EncodeToString(key[1:33])) + } + dm.Lock() defer dm.Unlock() dm.FileReadCount++ From c07a6881e647d75d9c211196994dd10b96ed994e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 19:13:38 +0000 Subject: [PATCH 046/167] commitment: surface pinned-tier counters on cache-fp log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pin_hit / pin_miss / pin_count fields to the [commitment][cache-fp] log line. The pinnedHits / pinnedMisses atomic counters on BranchCache existed but were not surfaced anywhere — without them the trunk-pin prototype is invisible (no way to tell whether the pinned tier is being used during the bench). Also add a PinnedStats() accessor and extend Stats() to include the pinned tier so the one-line summary reports all three tiers. This is the prerequisite instrumentation step before re-running the trunk-pin bench. With it, an outcome of "files_comm unchanged but pin_hit > 20K" tells us the pin is hit and trie compute uses the cached bytes; "files_comm unchanged AND pin_hit ~ 0" tells us the pin tier is bypassed (likely prefix encoding mismatch). No behaviour change. --- execution/commitment/branch_cache.go | 14 +++++++++++--- .../commitment/commitmentdb/commitment_context.go | 6 +++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index e6701db4514..1f138504a11 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -553,6 +553,7 @@ func (c *BranchCache) Fingerprint() uint64 { // per-Process log lines can compose them. func (c *BranchCache) Stats() string { rh, rm := c.rootHits.Load(), c.rootMisses.Load() + ph, pm := c.pinnedHits.Load(), c.pinnedMisses.Load() th, tm := c.tailHits.Load(), c.tailMisses.Load() bb := c.bytesServed.Load() pct := func(hit, miss uint64) float64 { @@ -563,15 +564,22 @@ func (c *BranchCache) Stats() string { return 100.0 * float64(hit) / float64(total) } return fmt.Sprintf( - "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d | divergences=%d", + "branch-cache root hit=%d miss=%d (%.1f%%) | pin hit=%d miss=%d (%.1f%%) entries=%d | tail hit=%d miss=%d (%.1f%%) entries=%d | served %.1f MiB | divergences=%d", rh, rm, pct(rh, rm), - th, tm, pct(th, tm), + ph, pm, pct(ph, pm), c.pinned.Len(), + th, tm, pct(th, tm), c.tail.Len(), float64(bb)/1024/1024, - c.tail.Len(), c.verifyDivergences.Load(), ) } +// PinnedStats returns the pinned-tier hit/miss/entries counters. Used +// by the cache-fp log to expose pin effectiveness for the trunk-pin +// prototype debug. +func (c *BranchCache) PinnedStats() (hits, misses uint64, entries int) { + return c.pinnedHits.Load(), c.pinnedMisses.Load(), c.pinned.Len() +} + // VerifyDivergences returns the number of cache-vs-canonical divergences // recorded since the last Clear. Non-zero indicates a cache lifecycle // invariant has been violated (a cached entry no longer matches the diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index a8bcf782dff..5a97bceed4b 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -393,6 +393,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33B:%d|34-36B:%d|37-44B:%d|45-64B:%d|>64B:%d", mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], mLens[5], mLens[6], mLens[7], mLens[8], mLens[9]) + pinHits, pinMisses, pinEntries := bc.PinnedStats() log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), @@ -421,7 +422,10 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "uniq_sto", sUniq, "uniq_code", cUniq, "uniq_comm", mUniq, - "comm_lens", commLens) + "comm_lens", commLens, + "pin_hit", pinHits, + "pin_miss", pinMisses, + "pin_count", pinEntries) } } } From a62cbaf98383f3aa5d8a3251f6ed784369ceb81e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 11:56:48 +0000 Subject: [PATCH 047/167] db/state, db/kv: add TemporalMemBatch.FlushWithCallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds FlushWithCallback (interface + concrete) for atomic flush + cache update. The cache callback fires for each latest (key, value, step) in the requested domain BEFORE the underlying Flush, all under latestStateLock so no observer sees a window where a key is missing from both the sd write buffer and the downstream cache. Used by SharedDomains.Flush to keep an external BranchCache in sync with commitment-domain writes. The MDBX commit provides db-side atomicity independently; the cache-first ordering keeps readers safe during the window between callback completion and MDBX write durably landing. Required by the WarmupCache-deletion sequence (commit ded378f789) which introduced the FlushWithCallback call site without defining the method itself — perf-stack at f0ba88aae2 didn't compile from source; the bench binary was built from a state with this method inlined and never committed. This commit reconstructs the method from the call-site contract. Verified: cold-cgroup d=68 SSTORE-bloat bench reproduces perf-stack baseline byte-for-byte (took=1.30 s, pin_count=69905, pin_hit=42216, pin_miss=104974, files_comm=16350, divergences=0). --- db/kv/kv_interface.go | 1 + db/state/temporal_mem_batch.go | 58 ++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index e2836f786f3..16a76e0ff4e 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,6 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error + FlushWithCallback(ctx context.Context, tx RwTx, domain Domain, cb func(k []byte, v []byte, step Step)) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 15e4c482d73..5ce7e1c8158 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -762,12 +762,53 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { return nil } -func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { +// FlushWithCallback updates a downstream cache via cb for each +// (key, value, step) tuple in domain, then flushes the mem-batch to tx. +// Cache-update is ordered BEFORE the MDBX flush so that during the +// MDBX write window the cache still holds the entry — a reader that +// goes (sd write buffer → cache → MDBX) never observes a key missing +// from all three sources. The MDBX commit provides db-side atomicity +// independently. +// +// Used by SharedDomains.Flush to keep an external BranchCache in sync +// with commitment-domain writes. +// +// cb is called under latestStateLock so the snapshot it sees matches +// the in-memory state at flush time. Flush itself runs under the same +// lock to prevent concurrent DomainPut from interleaving with the +// snapshot/cache-update/flush sequence. +// +// If Flush fails, the cache has already been updated for this batch. +// That's the same invariant the cache maintains for any other write +// path — values may exist in the cache before they reach disk; a +// retry of Flush re-applies them. +func (sd *TemporalMemBatch) FlushWithCallback( + ctx context.Context, tx kv.RwTx, domain kv.Domain, + cb func(k []byte, v []byte, step kv.Step), +) error { + sd.latestStateLock.Lock() + defer sd.latestStateLock.Unlock() + + // dir is unset on entries written via DomainPut/DomainDel (always + // default zero); the latest history entry per key is authoritative. + // data may be nil/empty for deletes — caller's cb is expected to + // distinguish (see SharedDomains.Flush, which Invalidate's on + // len(v)==0 and Put's otherwise). + for keyStr, history := range sd.domains[domain] { + if len(history) == 0 { + continue + } + latest := history[len(history)-1] + cb([]byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + } + + return sd.flushLocked(ctx, tx) +} + +// flushLocked is the body of Flush, factored so FlushWithCallback can +// run it inside latestStateLock without re-acquiring. +func (sd *TemporalMemBatch) flushLocked(ctx context.Context, tx kv.RwTx) error { if sd.unwindChangesetRaw != nil { - // Replay against the RAW (step-preserving) changeset — the collapsed - // unwindChangeset would lose every (key, step) entry except one per - // real key, which leaves orphan domain-values entries at steps above - // the unwind target (observed on mainnet for the commitment domain). for domain := range sd.unwindChangesetRaw { sort.Slice(sd.unwindChangesetRaw[domain], func(i, j int) bool { return sd.unwindChangesetRaw[domain][i].Key < sd.unwindChangesetRaw[domain][j].Key @@ -777,7 +818,6 @@ func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - if err := sd.flushDiffSet(ctx, tx); err != nil { return err } @@ -790,6 +830,12 @@ func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { return nil } +func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { + sd.latestStateLock.Lock() + defer sd.latestStateLock.Unlock() + return sd.flushLocked(ctx, tx) +} + func (sd *TemporalMemBatch) flushDiffSet(_ context.Context, tx kv.RwTx) error { for key, changeSet := range sd.pastChangesAccumulator { blockNum := binary.BigEndian.Uint64(common.ToBytesZeroCopy(key[:8])) From 8bad508fac46ad43b646807d538541509652bb41 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 12:53:55 +0000 Subject: [PATCH 048/167] commitment, state/execctx: BFS / budget-driven trunk preload Replaces the depth-first + branch-count-cap enumeration with a breadth-first walk bounded by a RAM byte budget. Depth becomes emergent from (trie shape, contract sparsity, budget) rather than configured directly. Why BFS. With DFS+cap, the budget could be exhausted on one subtree's deep children before the shallower sibling-trunk branches got pinned, producing a "deep narrow slice" pin set that misses the most-shared branches. The d=69 sweep on the prior implementation showed this: pin_count tripled but pin_hit dropped below d=67 because depth-first descent spent the cap on low-reuse leaves. BFS guarantees layer N completes before layer N+1 starts, so the budget is always spent on the highest-value (most-shared) branches first. Once a layer saturates, the remainder spills naturally into the next-most-valuable branches in the next layer. Why a RAM byte budget instead of branch-count cap. Branches vary in encoded size; a fixed branch-count cap can over- or under-shoot the intended RAM cost. A byte budget gives operators a single knob with direct memory semantics, and lets the depth reached vary per-contract based on each trie's shape. Validated on the SSTORE-bloat workload (test_sstore_bloated[10GB-Osaka -NO_CACHE-30M], cold-cgroup 32 GiB / 6c, contract 8eec99...aee7c): budget | pin_count | pin_hit | files_comm | TEST took ---------|-----------|---------|------------|---------- DFS d=68 | 69,905 | 42,216 | 16,346 | 1.34 s (prior) BFS 32MB | 44,685 | 33,386 | 18,585 | 1.46 s (budget-limited) BFS 64MB | 89,365 | 42,637 | 16,243 | 1.25 s (-7% vs DFS) BFS @ 64 MiB beats DFS d=68 in TEST took because it covers depth 69 (max_depth_reached=69) without the DFS+cap regression, while still saturating depths 64-68. Divergences=0 across all variants. Caller env: PIN_TRUNK_RAM_BUDGET_MB (default 32) replaces the prior PIN_TRUNK_MAX_DEPTH knob. --- db/state/execctx/domain_shared.go | 20 +++-- execution/commitment/preload.go | 143 ++++++++++++++++-------------- 2 files changed, 89 insertions(+), 74 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8d11a583561..25c53239220 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1234,13 +1234,15 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // Own-tx async sidesteps both. Caller must guarantee fire-once via // BranchCache.TryClaimPreload (the only call site does this). func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { - // Default 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max - // per contract for a fully-saturated subtree. Override via - // PIN_TRUNK_MAX_DEPTH for sweep experiments. - maxDepth := 67 - if v := dbg.EnvInt("PIN_TRUNK_MAX_DEPTH", 67); v >= 64 { - maxDepth = v - } + // RAM budget for the per-contract pin tier. Default 32 MiB covers + // a fully-saturated d=68 trunk (~70K branches × ~200 B) with + // headroom; depth reached is emergent from trie shape + budget. + // Override via PIN_TRUNK_RAM_BUDGET_MB for sweep experiments. + ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 32) + if ramBudgetMB <= 0 { + ramBudgetMB = 32 + } + ramBudgetBytes := ramBudgetMB << 20 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte for _, hexStr := range strings.Split(pinList, ",") { @@ -1256,7 +1258,7 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach } hashes = append(hashes, h) } - logger.Info("[trunk-preload] hashes parsed", "count", len(hashes), "max_depth", maxDepth) + logger.Info("[trunk-preload] hashes parsed", "count", len(hashes), "ram_budget_mb", ramBudgetMB) if len(hashes) == 0 { return } @@ -1282,7 +1284,7 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach for i, h := range hashes { started := time.Now() logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, branchCache, logger) + n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index c996943318c..3ade13049b3 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -22,35 +22,42 @@ import ( // the reader in at preload time without dragging in db/state imports. type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) -// PreloadContractTrunk pins commitment branches for the given contract -// from depth 64 (storage subtree root) down to maxDepth into the supplied -// BranchCache. Walks the trie structure depth-by-depth: reads the branch, -// pins it, decodes its child bitmap, and recurses only into children that -// actually exist. +// estimatedEntryOverheadBytes accounts for the per-entry RAM cost +// beyond the encoded value itself: the branchCacheEntry struct +// (~80 B), maphash slot + hash (~40 B), prefix slice (~24 B header + +// content), value slice header (~24 B). Used to predict whether +// pinning the next entry would exceed the configured budget. +const estimatedEntryOverheadBytes = 168 + +// PreloadContractTrunk pins commitment branches for the given contract's +// storage subtree into the supplied BranchCache, breadth-first from +// depth 64 (storage subtree root) outward. The descent is bounded by +// ramBudgetBytes — pinning stops the moment the next entry would push +// estimated total RAM over the budget. The depth reached is therefore +// emergent from (trie shape, contract sparsity, budget), not configured +// directly. // -// contractHash is keccak256(address) — 32 bytes. The trunk lives at the -// commitment-domain prefix that corresponds to nibbles 0..63 of the -// storage path (the contract's account hash); enumeration extends from -// there into nibbles 64..maxDepth. +// Why BFS: a depth-first descent under a hard branch-count cap can +// exhaust budget on one subtree's deep children before completing the +// shallower trunk for sibling branches, producing a pin set that is +// "deep narrow slice" rather than "saturated trunk". Trunk reuse is +// proportional to depth shallowness — each branch at depth N is shared +// across all of its descendant slot reads. BFS guarantees layer N is +// completely pinned before layer N+1 starts, so the budget is always +// spent on the highest-value (most-shared) branches first. // -// Read amplification: total reads = number of branches actually present -// in the trunk (bounded by the contract's storage trie shape). For a -// dense storage subtree, this is ~16+256+4096 ≈ 4.4 K reads at depth 65, -// 66, 67. Sparse subtrees produce fewer. maxDepth caps the descent. +// contractHash is keccak256(address) — 32 bytes. The trunk lives at +// the commitment-domain prefix that corresponds to nibbles 0..63 of +// the storage path; enumeration extends from there into nibbles 64+ +// until the budget is exhausted or the contract's storage trie has +// no further branches. // // Returns the number of pinned branches and any error from the reader. // On error the partial pin set remains in place — those entries are -// still valid cache hits, they just won't see further enumeration below -// the failure point. -// -// Loading strategy: this is the simplest correct implementation — -// recursive enumerate via per-prefix reader lookups. A bulk variant -// using seg.Getter range-scan over sorted .kv would amortize disk -// seeks but requires building a prefix-range API on top of recsplit. -// Defer that optimization until per-prefix lookup is shown to bottleneck. +// still valid cache hits. func PreloadContractTrunk( contractHash []byte, - maxDepth int, + ramBudgetBytes int, reader CommitmentReader, cache *BranchCache, logger log.Logger, @@ -58,84 +65,90 @@ func PreloadContractTrunk( if len(contractHash) != 32 { return 0, fmt.Errorf("PreloadContractTrunk: contractHash must be 32 bytes, got %d", len(contractHash)) } - if maxDepth < 64 { - return 0, fmt.Errorf("PreloadContractTrunk: maxDepth %d < 64 (storage subtree starts at depth 64)", maxDepth) - } if cache == nil { return 0, fmt.Errorf("PreloadContractTrunk: cache is nil") } + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } - // Convert contract hash bytes to 64 nibbles. + // Convert contract hash bytes to 64 nibbles — the path prefix + // that anchors all of the contract's storage subtree branches. contractNibbles := make([]byte, 64) for i, b := range contractHash { contractNibbles[2*i] = b >> 4 contractNibbles[2*i+1] = b & 0x0f } - // Hard cap to bound preload work. A fully-saturated 4-level subtree - // is 1+16+256+4096 = 4369 branches; this gives headroom but still - // fails fast on malformed input (e.g. a contract with truly - // pathological branching). Tune downward if this turns out to bite. - const maxBranches = 200000 + type pathDepth struct { + path []byte + depth int + } + queue := []pathDepth{{path: contractNibbles, depth: 64}} pinned := 0 - var stopped bool - var enumerate func(pathNibbles []byte, depth int) error - enumerate = func(pathNibbles []byte, depth int) error { - if stopped { - return nil - } - if pinned >= maxBranches { - stopped = true - if logger != nil { - logger.Warn("[trunk-preload] hit max-branches cap", - "cap", maxBranches, "depth", depth) - } - return nil - } - prefix := nibbles.HexToCompact(pathNibbles) + usedBytes := 0 + maxDepthReached := 64 + budgetExhausted := false + + for len(queue) > 0 { + head := queue[0] + queue = queue[1:] + + prefix := nibbles.HexToCompact(head.path) v, step, found, err := reader(prefix) if err != nil { - return fmt.Errorf("preload at depth %d: %w", depth, err) + return pinned, fmt.Errorf("preload at depth %d: %w", head.depth, err) } if !found { - return nil + continue + } + + entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) + if usedBytes+entryCost > ramBudgetBytes { + budgetExhausted = true + break } + cache.PinEntry(prefix, v, step, "preload-trunk") + usedBytes += entryCost pinned++ - // Periodic progress log so a slow preload is observable. - if logger != nil && pinned%500 == 0 { - logger.Info("[trunk-preload] progress", "pinned", pinned, "depth", depth) + if head.depth > maxDepthReached { + maxDepthReached = head.depth } - if depth >= maxDepth { - return nil + if logger != nil && pinned%5000 == 0 { + logger.Info("[trunk-preload] progress", + "pinned", pinned, "depth", head.depth, + "used_mb", usedBytes/(1<<20)) } + + // Decode bitmap and queue children. Branch encoding: + // 2-byte touchMap || 2-byte bitmap || per-child data. Only + // queue children that actually exist in the trie (bit set). if len(v) < 4 { - return nil + continue } - // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. - // Only recurse into children that actually exist (bit set in bitmap). bitmap := binary.BigEndian.Uint16(v[2:4]) for n := 0; n < 16; n++ { if bitmap&(1< Date: Sun, 10 May 2026 13:07:05 +0000 Subject: [PATCH 049/167] =?UTF-8?q?state/execctx:=20raise=20PIN=5FTRUNK=5F?= =?UTF-8?q?RAM=5FBUDGET=5FMB=20default=2032=20=E2=86=92=2064?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empirical sweep (32 / 64 / 256 MiB) on the SSTORE-bloat workload: budget | TEST took | mgas/s | files_comm ---------|-----------|--------|------------ 32 MiB | 1.46 s | 20.5 | 18,585 (under-coverage) 64 MiB | 1.25 s | 24.0 | 16,243 (perf knee) 256 MiB | 1.27 s | 23.7 | 15,542 (diminishing) 64 MiB covers the structural d=68 saturation (~70K branches) plus the most-shared d=69 branches. Going past 64 MiB delivers only −2 ms TEST took for +35 s preload — net negative. 32 MiB defaulted previously because of an inaccurate per-entry-bytes estimate (real cost ~720 B vs assumed ~200 B). Reset to the validated knee. --- db/state/execctx/domain_shared.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 25c53239220..859d81319c0 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1234,13 +1234,15 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // Own-tx async sidesteps both. Caller must guarantee fire-once via // BranchCache.TryClaimPreload (the only call site does this). func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { - // RAM budget for the per-contract pin tier. Default 32 MiB covers - // a fully-saturated d=68 trunk (~70K branches × ~200 B) with - // headroom; depth reached is emergent from trie shape + budget. - // Override via PIN_TRUNK_RAM_BUDGET_MB for sweep experiments. - ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 32) + // RAM budget per contract. 64 MiB is the perf knee on the + // SSTORE-bloat workload — covers the structural d=68 saturation + // (~70K branches) plus most-shared d=69 branches, beyond which + // marginal returns drop sharply (256 MiB sweep showed −2 ms TEST + // took for +35 s preload). Override via PIN_TRUNK_RAM_BUDGET_MB + // for tuning on different workload classes. + ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 64) if ramBudgetMB <= 0 { - ramBudgetMB = 32 + ramBudgetMB = 64 } ramBudgetBytes := ramBudgetMB << 20 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) From d494eec8fe249a1abcb5dfa95d3bad105abcd4aa Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:21:46 +0000 Subject: [PATCH 050/167] state/execctx: clamp PIN_TRUNK_RAM_BUDGET_MB at 64 MiB per contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hard-cap the per-contract pin budget at 64 MiB. Larger values silently clamp with a warning log so an operator can see their override was ignored. Why 64 MiB max for this PR. Larger budgets are not safe to ship until two prerequisite levers exist: 1. Per-account touch-driven minimisation (R11) — without this, deeper pin sets accumulate low-reuse branches that consume RAM without delivering perf. Today's 256 MiB sweep showed 19.9 % hit-rate-per-pin vs 47.7 % at 64 MiB. 2. Global RAM cap aware of available memory (R12) — without this, an operator promoting many contracts at high per-contract budgets could pin GBs of state on a small host. Override is plausibly justified for narrow cases (a measured-hot mainnet contract whose perf curve genuinely needs deeper coverage, operator diagnostic profiling) but those paths land once R11 + R12 + a per-contract whitelist exist (R13). --- db/state/execctx/domain_shared.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 859d81319c0..8fbf18252e0 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1238,11 +1238,31 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach // SSTORE-bloat workload — covers the structural d=68 saturation // (~70K branches) plus most-shared d=69 branches, beyond which // marginal returns drop sharply (256 MiB sweep showed −2 ms TEST - // took for +35 s preload). Override via PIN_TRUNK_RAM_BUDGET_MB - // for tuning on different workload classes. - ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 64) + // took for +35 s preload). + // + // 64 MiB is the per-contract MAX for this PR. Larger budgets are + // not safe to ship without (a) per-account touch-driven minimisation + // that prunes the pinned set to the actually-useful subset (R11) + // and (b) a global RAM cap aware of available memory (R12). Until + // both land, an operator setting 256 MiB across many promoted + // contracts could pin GBs without material perf benefit. + // + // Override is plausibly justified in narrow cases — a measured-hot + // mainnet contract whose perf curve genuinely requires deeper + // coverage, or operator-driven diagnostic profiling. NOT in scope + // for this PR; an override path lands once R11 + R12 + a + // per-contract whitelist mechanism exist. Until then, larger values + // silently clamp with a warning so an operator can see they were + // ignored. + const maxPerContractBudgetMB = 64 + ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", maxPerContractBudgetMB) if ramBudgetMB <= 0 { - ramBudgetMB = 64 + ramBudgetMB = maxPerContractBudgetMB + } + if ramBudgetMB > maxPerContractBudgetMB { + logger.Warn("[trunk-preload] clamping budget to per-contract max", + "requested_mb", ramBudgetMB, "clamped_mb", maxPerContractBudgetMB) + ramBudgetMB = maxPerContractBudgetMB } ramBudgetBytes := ramBudgetMB << 20 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) From 6ea7ff8bb5f8aa4340d2f50c9c63b9ad6fe9cb27 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:34:26 +0000 Subject: [PATCH 051/167] commitment/branch_cache: pin-aware Invalidate, GetWithOrigin, Clear Three accessor functions previously skipped the pinned tier, leaving correctness gaps that didn't bite the bench (which doesn't exercise unwind / fork-validation paths against pinned entries) but would manifest in production: - Invalidate(prefix) only deleted from root + LRU tail. A canonical store update for a pinned prefix would leave stale bytes pinned forever. Now deletes from pinned tier too. - GetWithOrigin(prefix) returned (nil,...,false) for pinned entries. The divergence-detection probe used this and would miss real cache-vs-canonical disagreements on pinned branches. - Clear() left the pinned tier intact and didn't reset pinned-tier counters. Fork-validation reset paths could carry stale pinned bytes across trie roots. Lifecycle context: pinned entries normally refresh in-place via Put on SD.Flush (pin survives every-block writes). Invalidate is the escape hatch for events where pinned bytes must be discarded outright (unwind, fork-validation reset, manual demotion). Clear is the full reset, sized for fork-validation paths that need a clean slate. --- execution/commitment/branch_cache.go | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 1f138504a11..f0b4970327c 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -446,6 +446,8 @@ func (c *BranchCache) GetWithOrigin(prefix []byte) (data []byte, origin string, var entry *branchCacheEntry if isRootPrefix(prefix) { entry = c.root.Load() + } else if pinnedEntry, pinnedOk := c.pinned.Get(prefix); pinnedOk { + entry = pinnedEntry } else { entry, ok = c.tail.Get(prefix) if !ok { @@ -470,26 +472,39 @@ func (c *BranchCache) MarkDirty(prefix []byte) { } } -// Invalidate removes the entry at prefix entirely. Use when the caller -// knows the canonical store has changed and the cached entry should not -// be served at all (vs MarkDirty which keeps the entry but blocks -// PutIfClean overwrites). +// Invalidate removes the entry at prefix entirely from whichever tier +// holds it. Use when the caller knows the canonical store has changed +// and the cached entry should not be served at all (vs MarkDirty which +// keeps the entry but blocks PutIfClean overwrites). +// +// Pinned-tier note: Invalidate does delete from the pinned tier. The +// usual lifecycle for pinned entries is in-place refresh via Put on +// SD.Flush (so pin survives every-block writes); Invalidate is the +// escape hatch for events where the pinned bytes must be discarded +// outright (unwind, fork-validation reset, manual demotion). func (c *BranchCache) Invalidate(prefix []byte) { if isRootPrefix(prefix) { c.root.Store(nil) return } + c.pinned.Delete(prefix) c.tail.Delete(prefix) } -// Clear empties the cache and resets stats counters. Use on Reset / -// fork-validation paths to ensure stale entries from one trie root are -// not served against a different root. +// Clear empties the cache and resets stats counters across ALL tiers +// (root, pinned, LRU tail). Use on Reset / fork-validation paths to +// ensure stale entries from one trie root are not served against a +// different root. After Clear, the pinned tier is empty; callers +// using trunk preload need to re-issue PreloadContractTrunk to +// repopulate it. func (c *BranchCache) Clear() { c.root.Store(nil) + c.pinned = maphash.NewMap[*branchCacheEntry]() c.tail.Purge() c.rootHits.Store(0) c.rootMisses.Store(0) + c.pinnedHits.Store(0) + c.pinnedMisses.Store(0) c.tailHits.Store(0) c.tailMisses.Store(0) c.bytesServed.Store(0) From 9ef3b9c6a023443f9b0fcfa0fa114ead6478f7d5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:37:30 +0000 Subject: [PATCH 052/167] state/execctx: invalidate stale BranchCache entries on SD.Unwind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After unwind the canonical commitment-domain values are rolled back; any cached entries for keys touched in the unwind window now hold stale bytes vs the post-unwind canonical state. Without this fix, a read after unwind that hits the cache before the next Put-on-Flush refresh would return the pre-unwind value — wrong-trie-root waiting to happen the next time the relevant key is touched. Selective per-key invalidation rather than Clear() — only the commitment-domain keys actually in the changeset get dropped. Pinned trunk branches whose keys weren't in the changeset stay warm; LRU tail entries unrelated to the unwind survive. Clear() remains the escape hatch for full reset (fork-validation paths that need a clean slate). Bench doesn't currently exercise unwind so this is a defensive fix ahead of cross-block stress testing. Pin-aware Invalidate (committed previously) is what makes this safe — without it, the per-key delete would leave pinned entries holding stale bytes. --- db/state/execctx/domain_shared.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8fbf18252e0..24c4bd5030b 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -539,6 +539,16 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) + // After unwind the canonical commitment-domain values are rolled + // back; any cached entries for keys touched in the unwind window + // now hold stale bytes vs the post-unwind canonical state. Drop + // those specifically — the rest of the cache (including pinned + // branches whose keys weren't in the changeset) stays warm. + if sd.branchCache != nil && changeset != nil { + for _, diff := range changeset[kv.CommitmentDomain] { + sd.branchCache.Invalidate([]byte(diff.Key)) + } + } } func (sd *SharedDomains) Trace() bool { From 878f04a1b16f1615634dc24e40a07153b2df6c6e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:55:34 +0000 Subject: [PATCH 053/167] commitment/preload: split BFS into resumable ContractTrunkPreload Refactor the one-shot PreloadContractTrunk into a stateful ContractTrunkPreload that supports incremental BFS via Run, plus a backward-compatible PreloadContractTrunk wrapper. Why split. The phased adaptive load (initial sync view at promotion + per-block extension while contract stays hot) needs to pause and resume the BFS between calls. The previous one-shot signature required burst-loading the full budget upfront, which paid the ~28 s cold-disk preload cost at every promotion regardless of whether the contract turned out to be worth it. The new shape: state := NewContractTrunkPreload(contractHash) // Initial view: small budget for d=64-67 trunk state.Run(initialBudget, reader, cache, logger) // Per-block extension while contract stays hot: state.Run(perBlockBudget, reader, cache, logger) // Repeat until queue empty or contract demoted. PreloadContractTrunk is now a wrapper around NewContractTrunkPreload + a single Run, preserving existing call sites (PIN_CONTRACT_TRUNKS env hook in domain_shared.go) without changes. Behavior-preserving refactor. Validated: BFS 64 MiB SSTORE-bloat bench reproduces baseline byte-for-byte (pin_count=89365, pin_hit=42637, files_comm=16244, took=1.22 s, divergences=0). --- execution/commitment/preload.go | 194 ++++++++++++++++++++------------ 1 file changed, 122 insertions(+), 72 deletions(-) diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 3ade13049b3..e46bba6d825 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -16,7 +16,7 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// CommitmentReader is the read interface PreloadContractTrunk needs: +// CommitmentReader is the read interface ContractTrunkPreload needs: // GetLatest on the CommitmentDomain. Returns (value, step, found, err). // Decoupled from any specific tx/aggregator type so callers can plug // the reader in at preload time without dragging in db/state imports. @@ -29,97 +29,107 @@ type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, er // pinning the next entry would exceed the configured budget. const estimatedEntryOverheadBytes = 168 -// PreloadContractTrunk pins commitment branches for the given contract's -// storage subtree into the supplied BranchCache, breadth-first from -// depth 64 (storage subtree root) outward. The descent is bounded by -// ramBudgetBytes — pinning stops the moment the next entry would push -// estimated total RAM over the budget. The depth reached is therefore -// emergent from (trie shape, contract sparsity, budget), not configured -// directly. -// -// Why BFS: a depth-first descent under a hard branch-count cap can -// exhaust budget on one subtree's deep children before completing the -// shallower trunk for sibling branches, producing a pin set that is -// "deep narrow slice" rather than "saturated trunk". Trunk reuse is -// proportional to depth shallowness — each branch at depth N is shared -// across all of its descendant slot reads. BFS guarantees layer N is -// completely pinned before layer N+1 starts, so the budget is always -// spent on the highest-value (most-shared) branches first. -// -// contractHash is keccak256(address) — 32 bytes. The trunk lives at -// the commitment-domain prefix that corresponds to nibbles 0..63 of -// the storage path; enumeration extends from there into nibbles 64+ -// until the budget is exhausted or the contract's storage trie has -// no further branches. +// pathDepth pairs a nibble path with the trie depth it terminates at. +// Used as the BFS queue element. +type pathDepth struct { + path []byte + depth int +} + +// ContractTrunkPreload holds the resumable state of a BFS preload for +// one contract's storage subtree. Created by NewContractTrunkPreload +// and advanced incrementally via Run; this lets callers split the +// preload into an initial sync view (small budget for d=64-67) and +// per-block extensions (additional budget while the contract stays +// hot), instead of burst-loading the full budget at promotion time. // -// Returns the number of pinned branches and any error from the reader. -// On error the partial pin set remains in place — those entries are -// still valid cache hits. -func PreloadContractTrunk( - contractHash []byte, - ramBudgetBytes int, - reader CommitmentReader, - cache *BranchCache, - logger log.Logger, -) (int, error) { +// Caller MUST NOT use the same instance from multiple goroutines. +type ContractTrunkPreload struct { + contractHash []byte + queue []pathDepth + pinned int + usedBytes int + maxDepthReached int +} + +// NewContractTrunkPreload constructs a preload state seeded at depth 64 +// (storage subtree root) for the given contract. The contract's +// storage subtree path is keccak256(address) — 32 bytes / 64 nibbles. +func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) { if len(contractHash) != 32 { - return 0, fmt.Errorf("PreloadContractTrunk: contractHash must be 32 bytes, got %d", len(contractHash)) - } - if cache == nil { - return 0, fmt.Errorf("PreloadContractTrunk: cache is nil") + return nil, fmt.Errorf("NewContractTrunkPreload: contractHash must be 32 bytes, got %d", len(contractHash)) } - if ramBudgetBytes <= 0 { - return 0, fmt.Errorf("PreloadContractTrunk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) - } - - // Convert contract hash bytes to 64 nibbles — the path prefix - // that anchors all of the contract's storage subtree branches. contractNibbles := make([]byte, 64) for i, b := range contractHash { contractNibbles[2*i] = b >> 4 contractNibbles[2*i+1] = b & 0x0f } + return &ContractTrunkPreload{ + contractHash: contractHash, + queue: []pathDepth{{path: contractNibbles, depth: 64}}, + maxDepthReached: 64, + }, nil +} - type pathDepth struct { - path []byte - depth int +// Run advances the BFS preload, pinning branches into cache until +// either the additional budget is exhausted or the queue is empty. +// Returns the number of entries pinned during THIS call (not the +// cumulative total — see PinnedTotal), whether the queue is now +// empty (preload fully complete), and any error from the reader. +// +// Calling Run repeatedly with small budgets implements the phased +// load: an initial Run with the InitialBudget covers depths 64-67; +// each subsequent Run extends BFS one chunk further. Per-block +// callers should size the chunk to fit within inter-block idle time. +// +// On reader error the partial pin set survives; the queue position +// is preserved so a retry on the next Run picks up where it left off. +func (p *ContractTrunkPreload) Run( + additionalBudgetBytes int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (newlyPinned int, queueEmpty bool, err error) { + if cache == nil { + return 0, false, fmt.Errorf("ContractTrunkPreload.Run: cache is nil") + } + if additionalBudgetBytes <= 0 { + return 0, len(p.queue) == 0, nil } - queue := []pathDepth{{path: contractNibbles, depth: 64}} - pinned := 0 - usedBytes := 0 - maxDepthReached := 64 - budgetExhausted := false + chunkUsedBytes := 0 + chunkPinned := 0 - for len(queue) > 0 { - head := queue[0] - queue = queue[1:] + for len(p.queue) > 0 { + head := p.queue[0] + p.queue = p.queue[1:] prefix := nibbles.HexToCompact(head.path) - v, step, found, err := reader(prefix) - if err != nil { - return pinned, fmt.Errorf("preload at depth %d: %w", head.depth, err) + v, step, found, rerr := reader(prefix) + if rerr != nil { + return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", head.depth, rerr) } if !found { continue } entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) - if usedBytes+entryCost > ramBudgetBytes { - budgetExhausted = true + if chunkUsedBytes+entryCost > additionalBudgetBytes { + // Re-queue the head so a future Run picks it up. + p.queue = append([]pathDepth{head}, p.queue...) break } cache.PinEntry(prefix, v, step, "preload-trunk") - usedBytes += entryCost - pinned++ - if head.depth > maxDepthReached { - maxDepthReached = head.depth + chunkUsedBytes += entryCost + chunkPinned++ + if head.depth > p.maxDepthReached { + p.maxDepthReached = head.depth } - if logger != nil && pinned%5000 == 0 { + if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { logger.Info("[trunk-preload] progress", - "pinned", pinned, "depth", head.depth, - "used_mb", usedBytes/(1<<20)) + "pinned", p.pinned+chunkPinned, "depth", head.depth, + "used_mb", (p.usedBytes+chunkUsedBytes)/(1<<20)) } // Decode bitmap and queue children. Branch encoding: @@ -136,20 +146,60 @@ func PreloadContractTrunk( childPath := make([]byte, len(head.path)+1) copy(childPath, head.path) childPath[len(head.path)] = byte(n) - queue = append(queue, pathDepth{path: childPath, depth: head.depth + 1}) + p.queue = append(p.queue, pathDepth{path: childPath, depth: head.depth + 1}) } } + p.pinned += chunkPinned + p.usedBytes += chunkUsedBytes + return chunkPinned, len(p.queue) == 0, nil +} + +// PinnedTotal returns the cumulative number of branches pinned by +// this preload state across all Run calls. +func (p *ContractTrunkPreload) PinnedTotal() int { return p.pinned } + +// UsedBytes returns the cumulative byte cost of this preload's pin set. +func (p *ContractTrunkPreload) UsedBytes() int { return p.usedBytes } + +// QueueRemaining returns the number of paths still queued for descent. +// Zero means the preload is complete; further Run calls are no-ops. +func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } + +// MaxDepthReached returns the deepest trie depth pinned so far. +func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } + +// PreloadContractTrunk runs the full BFS preload for a contract in a +// single call, bounded by ramBudgetBytes. Convenience wrapper around +// NewContractTrunkPreload + Run for callers that don't need phased +// loading. Existing one-shot trigger paths (PIN_CONTRACT_TRUNKS env +// hook) use this; the adaptive layer holds a *ContractTrunkPreload +// and calls Run incrementally. +func PreloadContractTrunk( + contractHash []byte, + ramBudgetBytes int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (int, error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreload(contractHash) + if err != nil { + return 0, err + } + pinned, queueEmpty, err := p.Run(ramBudgetBytes, reader, cache, logger) if logger != nil { logger.Info("[trunk-preload] complete", "contract_hash", fmt.Sprintf("%x", contractHash), "ram_budget_mb", ramBudgetBytes/(1<<20), - "used_mb", usedBytes/(1<<20), + "used_mb", p.UsedBytes()/(1<<20), "pinned", pinned, - "max_depth_reached", maxDepthReached, - "budget_exhausted", budgetExhausted, - "queue_remaining", len(queue), + "max_depth_reached", p.MaxDepthReached(), + "budget_exhausted", !queueEmpty, + "queue_remaining", p.QueueRemaining(), "cache_pinned_total", cache.PinnedCount()) } - return pinned, nil + return pinned, err } From a2b6ab508ca7a8eb014895f56ba32eac86fae363 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:58:06 +0000 Subject: [PATCH 054/167] commitment/branch_cache: add miss-callback hook for adaptive controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds SetMissCallback / MissCallback type and ContractHashFromPrefix helper. The callback fires whenever lookup misses ALL three tiers (root, pinned, LRU tail) — the deepest signal the cache has that a file read will follow. Hot path stays cheap: the no-callback case is a single atomic.Pointer load and a nil check. Implementations must be lock-free. ContractHashFromPrefix decodes the contract's 32-byte storage-subtree prefix (1-byte HP flag + 64 nibbles packed). Returns false for prefixes shorter than 33 bytes (account-trie branches), so the adaptive controller naturally filters out non-storage-trunk misses. The callback alone does no policy — the AdaptivePinController (commit 24c) registers a callback that attributes misses per contract and decides promotions. Cache stays simple; controller owns the policy. --- execution/commitment/branch_cache.go | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index f0b4970327c..d7c22d2e20d 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -192,6 +192,14 @@ type BranchCache struct { // stale bytes. writeSeq atomic.Uint64 + // onMiss is an optional hook fired when lookup misses all three + // tiers (root, pinned, LRU). Used by the adaptive trunk-pin + // controller to attribute miss pressure per-contract and decide + // promotions. Stored as atomic.Pointer so registration is + // lock-free and the hot read path skips the dereference cleanly + // when no callback is installed. + onMiss atomic.Pointer[MissCallback] + // preloadClaimed is set the first time TryClaimPreload is called. // Used by the trunk-preload trigger (PIN_CONTRACT_TRUNKS hook in // SharedDomains construction) so the preload goroutine fires @@ -292,6 +300,7 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { entry := c.root.Load() if entry == nil { c.rootMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } c.rootHits.Add(1) @@ -307,12 +316,21 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { entry, ok := c.tail.Get(prefix) if !ok { c.tailMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } c.tailHits.Add(1) return entry, true } +// fireOnMiss invokes the registered miss callback (if any). Hot path — +// the no-callback case is a single atomic load and a nil check. +func (c *BranchCache) fireOnMiss(prefix []byte) { + if cb := c.onMiss.Load(); cb != nil { + (*cb)(prefix) + } +} + func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { if isRootPrefix(prefix) { c.root.Store(entry) @@ -355,6 +373,40 @@ func (c *BranchCache) PinnedCount() int { return c.pinned.Len() } +// MissCallback is invoked when lookup misses ALL three tiers (root, +// pinned, LRU tail). Called on the hot read path; implementations +// must be lock-free or short and not block. +type MissCallback func(prefix []byte) + +// SetMissCallback installs a hook fired on every triple-miss. Pass +// nil to clear. Used by the adaptive controller to attribute miss +// pressure per contract; the cache itself does no per-contract +// bookkeeping. Replaces any prior callback atomically. +func (c *BranchCache) SetMissCallback(cb MissCallback) { + if cb == nil { + c.onMiss.Store(nil) + return + } + c.onMiss.Store(&cb) +} + +// ContractHashFromPrefix extracts the 32-byte contract hash from a +// commitment-trunk prefix. Returns (hash, true) if the prefix is a +// storage-trunk prefix (depth >= 64, encoded as compact-hex with the +// contract's 64-nibble keccak prefix), or (_, false) for shorter +// prefixes (account-trie branches at depth < 64). +// +// Compact-hex encoding: 1-byte HP flag, then nibble-pairs packed +// 2-per-byte. The contract's 64-nibble path occupies 32 bytes after +// the flag byte, so a storage-trunk prefix is at least 33 bytes. +func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { + if len(prefix) < 33 { + return hash, false + } + copy(hash[:], prefix[1:33]) + return hash, true +} + // Get retrieves branch data from the cache. Returns the canonical encoded // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). From be2616e6c2653ba1a44b85ba97df04f732ff6a78 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:01:35 +0000 Subject: [PATCH 055/167] commitment: AdaptivePinController for promotion / extension / demotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the policy layer that decides which contracts to pin based on observed miss pressure. Uses the cache miss-callback (24b) to attribute per-contract pressure and the resumable ContractTrunkPreload (24a) to phase the load: - Promotion: a contract whose cache misses cross PromoteThresholdMisses in a single block gets a synchronous initial-view preload (4 MiB default, ~1.4 s, covers d=64-67 trunk). Bounded by MaxPromotedContracts (default 8 → 512 MiB total max RAM). - Extension: each block where a promoted contract had any miss, the controller advances BFS by ExtensionBudgetBytes (8 MiB default). Stops at PerContractMaxBudgetBytes (64 MiB ceiling). - Demotion: after DemoteCooldownBlocks (5 default) consecutive cold blocks, the controller invalidates the contract's pin set. To support this, ContractTrunkPreload now tracks PinnedPrefixes — the slice of prefixes added to the cache, copied on each PinEntry so demotion has stable handles. Defaults are conservative — designed for safe default-on shipping with no per-host tuning. R11 (touch-driven minimisation) and R12 (global memory cap aware of available RAM) are future work that will replace the static MaxPromotedContracts × PerContractMaxBudgetBytes calculation with adaptive policy. The controller itself is decoupled from wiring: NewAdaptivePinController constructs it, Bind installs the cache callback, OnBlockComplete is called by the host (commit 24d wires this into SD construction). --- execution/commitment/adaptive_pin.go | 333 +++++++++++++++++++++++++++ execution/commitment/preload.go | 19 ++ 2 files changed, 352 insertions(+) create mode 100644 execution/commitment/adaptive_pin.go diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go new file mode 100644 index 00000000000..9a21ce1fdde --- /dev/null +++ b/execution/commitment/adaptive_pin.go @@ -0,0 +1,333 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "context" + "encoding/hex" + "sync" + "sync/atomic" + + "github.com/erigontech/erigon/common/log/v3" +) + +// AdaptivePinControllerConfig sets the policy knobs for the adaptive +// trunk-pin controller. Defaults are tuned for the SSTORE-bloat +// workload class (single contract dominating storage reads); other +// workload classes may want different values. +type AdaptivePinControllerConfig struct { + // PromoteThresholdMisses — minimum cache misses (per block) for a + // contract to be promoted. Default 100. + PromoteThresholdMisses uint64 + + // MaxPromotedContracts — hard cap on simultaneously-pinned + // contracts. Bounds total pin RAM = MaxPromotedContracts × + // PerContractMaxBudgetBytes. Default 8 (× 64 MiB = 512 MiB max). + MaxPromotedContracts int + + // DemoteCooldownBlocks — number of consecutive blocks with zero + // misses for a promoted contract before it gets demoted (and its + // pin set invalidated). Default 5. + DemoteCooldownBlocks int + + // InitialViewBudgetBytes — RAM budget for the synchronous + // initial-view preload at promotion. Default 4 MiB (covers + // d=64-67 with headroom; ~1.4 s on cold disk). + InitialViewBudgetBytes int + + // ExtensionBudgetBytes — RAM budget for the per-block extension + // step on already-promoted contracts. Default 8 MiB (~25 K + // branches per block at typical entry cost). + ExtensionBudgetBytes int + + // PerContractMaxBudgetBytes — hard ceiling on the cumulative pin + // budget per contract. Extensions stop when this is reached even + // if the BFS queue isn't empty. Default 64 MiB (matches the + // per-contract max enforced in the env hook). + PerContractMaxBudgetBytes int +} + +// DefaultAdaptivePinControllerConfig returns the production defaults. +// Conservative across the board so default-on shipping doesn't pin +// memory speculatively. +func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { + return AdaptivePinControllerConfig{ + PromoteThresholdMisses: 100, + MaxPromotedContracts: 8, + DemoteCooldownBlocks: 5, + InitialViewBudgetBytes: 4 << 20, + ExtensionBudgetBytes: 8 << 20, + PerContractMaxBudgetBytes: 64 << 20, + } +} + +// AdaptivePinController watches per-contract miss pressure on a +// BranchCache and decides which contracts to pin (with a sync initial +// view) and grow (per-block extension) or demote (invalidate the pin +// set after sustained inactivity). +// +// Lifecycle: +// - Construct with NewAdaptivePinController, wire to a cache via Bind +// - On every triple-miss, the cache calls back; the controller +// records a per-contract miss count +// - The host (SD or stage loop) calls OnBlockComplete at block +// boundaries with a CommitmentReader factory; the controller +// then promotes / extends / demotes based on the per-block +// miss snapshot +type AdaptivePinController struct { + cache *BranchCache + cfg AdaptivePinControllerConfig + logger log.Logger + + // misses holds per-contract miss counts since last OnBlockComplete. + // sync.Map is appropriate: writes are frequent (every triple-miss), + // reads happen once per block boundary. + misses sync.Map // [32]byte → *atomic.Uint64 + + // states holds per-promoted-contract bookkeeping. Protected by mu. + mu sync.Mutex + states map[[32]byte]*adaptiveContractState +} + +type adaptiveContractState struct { + contractHash [32]byte + promotedAtBlock uint64 + preload *ContractTrunkPreload + coldBlocksInARow int +} + +// NewAdaptivePinController constructs a controller bound to the given +// cache. Use Bind to install the cache miss-callback (separate so a +// caller can keep a controller for telemetry without wiring it into +// the read path). +func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfig, logger log.Logger) *AdaptivePinController { + if cfg.InitialViewBudgetBytes <= 0 { + cfg.InitialViewBudgetBytes = 4 << 20 + } + if cfg.ExtensionBudgetBytes <= 0 { + cfg.ExtensionBudgetBytes = 8 << 20 + } + if cfg.PerContractMaxBudgetBytes <= 0 { + cfg.PerContractMaxBudgetBytes = 64 << 20 + } + if cfg.MaxPromotedContracts <= 0 { + cfg.MaxPromotedContracts = 8 + } + if cfg.DemoteCooldownBlocks <= 0 { + cfg.DemoteCooldownBlocks = 5 + } + if cfg.PromoteThresholdMisses == 0 { + cfg.PromoteThresholdMisses = 100 + } + return &AdaptivePinController{ + cache: cache, + cfg: cfg, + logger: logger, + states: make(map[[32]byte]*adaptiveContractState), + } +} + +// Bind installs the controller's miss-callback on the cache. After +// Bind, every triple-miss attributable to a storage-trunk prefix +// (length >= 33 B) is counted toward the corresponding contract. +// +// Safe to call multiple times — Bind replaces any prior callback. +// Call SetMissCallback(nil) on the cache directly to unbind. +func (c *AdaptivePinController) Bind() { + c.cache.SetMissCallback(c.onCacheMiss) +} + +func (c *AdaptivePinController) onCacheMiss(prefix []byte) { + hash, ok := ContractHashFromPrefix(prefix) + if !ok { + return + } + v, _ := c.misses.LoadOrStore(hash, new(atomic.Uint64)) + v.(*atomic.Uint64).Add(1) +} + +// OnBlockComplete consumes the per-block miss snapshot and decides +// promotions, extensions, and demotions. Called by the host at block +// boundaries (after SD.Flush). The reader is the CommitmentReader +// for the just-committed state, used by initial-view preload and +// per-block extension. +// +// Synchronous: initial-view preloads run inline so the new pin set +// is available for the NEXT block's reads. Extensions also run +// inline; sized so per-block work fits within typical inter-block +// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). +// +// Logs a [adaptive-pin] line per block when any state changes or +// promoted contracts exist. +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { + misses := c.snapshotMisses() + + c.mu.Lock() + defer c.mu.Unlock() + + var promoted, extended, demoted int + + // Already-promoted contracts: extend on hot, demote on cold. + for hash, state := range c.states { + n, hadMisses := misses[hash] + if hadMisses && n > 0 { + state.coldBlocksInARow = 0 + delete(misses, hash) + // Extend if budget remains and queue not empty. + if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { + remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() + step := c.cfg.ExtensionBudgetBytes + if step > remaining { + step = remaining + } + if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { + c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) + } else { + extended++ + } + } + continue + } + state.coldBlocksInARow++ + if state.coldBlocksInARow >= c.cfg.DemoteCooldownBlocks { + c.demoteLocked(hash, state) + delete(c.states, hash) + demoted++ + } + } + + // New promotion candidates: contracts whose miss count crossed the + // threshold. Cap at MaxPromotedContracts; pick the highest-miss + // candidates first (greedy, simple, sufficient for v1). + if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { + candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) + for _, hash := range candidates { + p, err := NewContractTrunkPreload(hash[:]) + if err != nil { + continue + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) + // Roll back partial pin so we don't leak entries + // for a contract that won't be in c.states. + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + continue + } + c.states[hash] = &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + preload: p, + } + promoted++ + } + } + + if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { + c.logger.Info("[adaptive-pin]", + "block", blockNum, + "promoted_total", len(c.states), + "promoted_this_block", promoted, + "extended_this_block", extended, + "demoted_this_block", demoted, + "cache_pinned_total", c.cache.PinnedCount()) + } +} + +// snapshotMisses atomically reads + zeros every per-contract miss +// counter. Called once per OnBlockComplete. +func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { + out := make(map[[32]byte]uint64) + c.misses.Range(func(k, v any) bool { + hash := k.([32]byte) + n := v.(*atomic.Uint64).Swap(0) + if n > 0 { + out[hash] = n + } + return true + }) + return out +} + +// demoteLocked invalidates every pinned prefix for the contract. +// Caller must hold c.mu. +func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { + for _, prefix := range state.preload.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + if c.logger != nil { + c.logger.Info("[adaptive-pin] demoted", + "hash", hex.EncodeToString(hash[:]), + "pinned_was", state.preload.PinnedTotal(), + "used_mb_was", state.preload.UsedBytes()/(1<<20), + "cold_blocks", state.coldBlocksInARow) + } +} + +// PromotedContracts returns the hashes of currently-pinned contracts. +// Snapshot — the slice may be stale by the time the caller uses it. +// Used by metrics + debug diagnostics. +func (c *AdaptivePinController) PromotedContracts() [][32]byte { + c.mu.Lock() + defer c.mu.Unlock() + out := make([][32]byte, 0, len(c.states)) + for h := range c.states { + out = append(out, h) + } + return out +} + +// pickPromotionCandidates selects up to maxN contract hashes from +// misses whose count exceeds threshold, preferring highest-miss first. +// Linear scan — adequate for the small N we expect (typically ≤16 +// candidates per block; sort is cheap). +func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN int) [][32]byte { + if maxN <= 0 { + return nil + } + type cand struct { + hash [32]byte + n uint64 + } + var pool []cand + for h, n := range misses { + if n >= threshold { + pool = append(pool, cand{h, n}) + } + } + if len(pool) > maxN { + // Partial sort: O(N×maxN) for small maxN; avoids importing sort. + for i := 0; i < maxN; i++ { + best := i + for j := i + 1; j < len(pool); j++ { + if pool[j].n > pool[best].n { + best = j + } + } + pool[i], pool[best] = pool[best], pool[i] + } + pool = pool[:maxN] + } + out := make([][32]byte, len(pool)) + for i, c := range pool { + out[i] = c.hash + } + return out +} + +func (c *AdaptivePinController) warnf(msg string, kv ...any) { + if c.logger != nil { + c.logger.Warn(msg, kv...) + } +} + +// ctx unused for now; reserved for future cancellation of in-flight +// preloads (R3 bulk preload + cancellable extension). +var _ = context.Background diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index e46bba6d825..88fa7cc2677 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -43,10 +43,14 @@ type pathDepth struct { // per-block extensions (additional budget while the contract stays // hot), instead of burst-loading the full budget at promotion time. // +// Tracks its pinned prefixes so callers can Invalidate the contract's +// pin set on demotion (PinnedPrefixes returns the slice). +// // Caller MUST NOT use the same instance from multiple goroutines. type ContractTrunkPreload struct { contractHash []byte queue []pathDepth + pinnedPrefixes [][]byte pinned int usedBytes int maxDepthReached int @@ -121,6 +125,12 @@ func (p *ContractTrunkPreload) Run( } cache.PinEntry(prefix, v, step, "preload-trunk") + // Copy prefix because nibbles.HexToCompact may return a slice + // over a reused buffer; we need a stable handle for later + // Invalidate on demotion. + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + p.pinnedPrefixes = append(p.pinnedPrefixes, prefixCopy) chunkUsedBytes += entryCost chunkPinned++ if head.depth > p.maxDepthReached { @@ -169,6 +179,15 @@ func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } // MaxDepthReached returns the deepest trie depth pinned so far. func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } +// PinnedPrefixes returns the prefix slices this preload has added to +// the cache so far. The adaptive controller uses this to Invalidate +// the contract's pin set on demotion. The returned slice aliases +// internal storage; do not mutate. +func (p *ContractTrunkPreload) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } + +// ContractHash returns the contract this preload targets. +func (p *ContractTrunkPreload) ContractHash() []byte { return p.contractHash } + // PreloadContractTrunk runs the full BFS preload for a contract in a // single call, bounded by ramBudgetBytes. Convenience wrapper around // NewContractTrunkPreload + Run for callers that don't need phased From b082c93a4ddbc1cbdd61232120df8a09ea1871f2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:16:38 +0000 Subject: [PATCH 056/167] state/execctx: wire AdaptivePinController into SharedDomains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three integration points: 1. Construction in NewSharedDomainsWithTrieVariant — alongside the branchCache assignment. Inert until Bind. Skipped when DISABLE_ADAPTIVE_PIN env is set (defensive escape hatch). 2. Bind in EnableParaTrieDB — installs the cache miss-callback so the controller starts observing per-contract pressure. Runs AFTER the env-driven PIN_CONTRACT_TRUNKS forced preload, so operator overrides land before the controller starts attribution. 3. OnBlockComplete from SD.Flush — fires after FlushWithCallback has updated the cache for this batch. Reader uses the in-flight Flush tx so initial-view preload of newly-promoted contracts sees the just-committed bytes. Uses sd.txNum as the controller's "tick" — per-batch granularity, so cooldown thresholds map to batches not blocks (tunable via controller config if finer granularity is needed later). Operator override paths preserved: - PIN_CONTRACT_TRUNKS — forces specific contracts to pin regardless of adaptive policy. Useful for known-hot mainnet contracts and diagnostic profiling. - DISABLE_ADAPTIVE_PIN — disables the controller entirely. The branch-cache layer stays active; only the auto-promotion logic is skipped. Bench-validated: with PIN_CONTRACT_TRUNKS in use, controller fires but no promotions occur (the bloat contract's reads go to the pre-pinned tier, not the miss callback). Baseline reproduces byte-for-byte (took=1.22 s, pin_count=89365, divergences=0). --- db/state/execctx/domain_shared.go | 72 ++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 24c4bd5030b..d7bf9755846 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -168,6 +168,16 @@ type SharedDomains struct { // May be nil for test setups whose AggTx doesn't implement // commitment.BranchCacheProvider. branchCache *commitment.BranchCache + + // adaptivePinController owns the policy that decides which + // contracts get pinned based on observed miss pressure. nil when + // branchCache is nil or when the adaptive layer is disabled. The + // controller's miss callback is wired onto branchCache via Bind + // during EnableParaTrieDB; OnBlockComplete fires from sd.Flush. + // Uses sd.txNum as the "block tick" for cooldown semantics — Flush + // is per-batch so the controller's tick maps to batches not blocks, + // but cooldown thresholds are tunable to compensate. + adaptivePinController *commitment.AdaptivePinController } func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger) (*SharedDomains, error) { @@ -209,6 +219,19 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) + // Construct the adaptive pin controller alongside the cache. The + // controller observes per-contract miss pressure and decides + // promotions/extensions/demotions; without binding (deferred to + // EnableParaTrieDB so the db is available for the reader path) + // the controller is inert. + if branchCache != nil && !dbg.EnvBool("DISABLE_ADAPTIVE_PIN", false) { + sd.adaptivePinController = commitment.NewAdaptivePinController( + branchCache, + commitment.DefaultAdaptivePinControllerConfig(), + logger, + ) + } + _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -742,13 +765,34 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { // the local sd.mem value (during the lock) or the cached/MDBX value // (after release). No window where cache is stale vs MDBX. if sd.branchCache != nil { - return sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { + if err := sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { if len(v) == 0 { sd.branchCache.Invalidate(k) return } sd.branchCache.Put(k, v, uint64(step), "sd.Flush") - }) + }); err != nil { + return err + } + // Adaptive controller hook: now that the cache reflects this + // batch's writes, decide promotions / extensions / demotions + // for the contracts whose miss pressure crossed thresholds. + // The reader uses the in-flight Flush tx so pre-cache reads + // see the just-committed bytes (the tx's own writes are + // visible to itself). + if sd.adaptivePinController != nil { + if ttx, ok := tx.(kv.TemporalTx); ok { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil + } + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + } + } + return nil } return sd.mem.Flush(ctx, tx) } @@ -1165,7 +1209,8 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) - // Trigger trunk-preload here (not from NewSharedDomains) so: + // Stage-init is the right firing point for both the env-driven + // forced preload AND the adaptive controller's bind: // (a) we have a kv.TemporalRoDB to spawn an own-tx preload goroutine // with — avoids the cgo-pointer panic that the earlier shared-tx // attempt produced. @@ -1177,11 +1222,24 @@ func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { if sd.branchCache == nil || !sd.branchCache.TryClaimPreload() { return } - pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", "") - if pinList == "" { - return + + // Operator-override path: PIN_CONTRACT_TRUNKS forces specific + // contracts to be pinned regardless of adaptive policy. Runs + // first so its pin set is in place before the controller binds. + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) + } + + // Bind the adaptive controller to the cache so it starts + // observing miss pressure for un-forced contracts. The reader + // for promote/extend preloads is constructed per-call from the + // in-flight Flush tx (see SD.Flush). + if sd.adaptivePinController != nil { + sd.adaptivePinController.Bind() + sd.logger.Info("[adaptive-pin] enabled", + "max_promoted", commitment.DefaultAdaptivePinControllerConfig().MaxPromotedContracts, + "per_contract_max_mb", commitment.DefaultAdaptivePinControllerConfig().PerContractMaxBudgetBytes/(1<<20)) } - triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) } // SetDeferCommitmentUpdates enables or disables deferred commitment updates. From 2c6ee4ed9582269d4370f710103706ff1306ff7d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:18:24 +0000 Subject: [PATCH 057/167] commitment: Prometheus metrics for trunk-pin layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface the existing internal counters via the diagnostics/metrics package so dashboards can reason about pin effectiveness without log scraping. Six metrics: commitment_branchcache_pinned_hits_total (counter) commitment_branchcache_pinned_misses_total (counter) commitment_branchcache_pinned_entries (gauge) commitment_adaptive_pin_promoted_total (counter) commitment_adaptive_pin_extended_total (counter) commitment_adaptive_pin_demoted_total (counter) commitment_adaptive_pin_active_contracts (gauge) Per-contract labels intentionally omitted — cardinality risk is real (mainnet could see hundreds of promoted contracts over a process lifetime) and the structured [adaptive-pin] log line gives per-contract detail when needed for debugging. Total counters are sufficient for "is this lever working" dashboard signals. Refresh cadence: - pinned-entries gauge updates at SD.Flush via PublishMetrics — once per batch, avoids hot-path cost - AdaptivePinController updates promote/extend/demote/active in OnBlockComplete (also per-batch, same call site) --- db/state/execctx/domain_shared.go | 3 ++ execution/commitment/adaptive_pin.go | 11 +++++ execution/commitment/trunk_pin_metrics.go | 57 +++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 execution/commitment/trunk_pin_metrics.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index d7bf9755846..a55c270c267 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -792,6 +792,9 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) } } + // Refresh pinned-tier gauges once per Flush — once-per-batch + // cadence avoids per-lookup cost in the hot read path. + sd.branchCache.PublishMetrics() return nil } return sd.mem.Flush(ctx, tx) diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 9a21ce1fdde..1cd6b6f6066 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -230,6 +230,17 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui } } + if promoted > 0 { + mxAdaptivePromoted.AddUint64(uint64(promoted)) + } + if extended > 0 { + mxAdaptiveExtended.AddUint64(uint64(extended)) + } + if demoted > 0 { + mxAdaptiveDemoted.AddUint64(uint64(demoted)) + } + mxAdaptiveActive.SetUint64(uint64(len(c.states))) + if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { c.logger.Info("[adaptive-pin]", "block", blockNum, diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go new file mode 100644 index 00000000000..5bf8e3f8a63 --- /dev/null +++ b/execution/commitment/trunk_pin_metrics.go @@ -0,0 +1,57 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "github.com/erigontech/erigon/diagnostics/metrics" +) + +// Prometheus metrics for the storage-trunk pin layer. Per-contract +// labels intentionally omitted — cardinality risk is real (mainnet +// could see hundreds of promoted contracts over a process lifetime) +// and the structured [adaptive-pin] log line gives per-contract +// detail when needed for debugging. + +var ( + // BranchCache pinned-tier counters. Surface the existing + // PinnedStats hits/misses/entries so dashboards can reason about + // pin effectiveness without scraping logs. + mxPinnedHits = metrics.GetOrCreateCounter("commitment_branchcache_pinned_hits_total") + mxPinnedMisses = metrics.GetOrCreateCounter("commitment_branchcache_pinned_misses_total") + mxPinnedEntries = metrics.GetOrCreateGauge("commitment_branchcache_pinned_entries") + + // AdaptivePinController lifecycle counters. + mxAdaptivePromoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_promoted_total") + mxAdaptiveExtended = metrics.GetOrCreateCounter("commitment_adaptive_pin_extended_total") + mxAdaptiveDemoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_demoted_total") + mxAdaptiveActive = metrics.GetOrCreateGauge("commitment_adaptive_pin_active_contracts") + + // Preload-side counters for diagnostics. Per-contract preload + // duration is too high-cardinality for a histogram label; total + // duration spent in preload is the aggregate proxy. + mxPreloadDurationSecondsTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_duration_seconds_total") + mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") +) + +// publishPinnedGauge updates the pinned-entries gauge from the cache's +// current size. Called from PublishMetrics; cheap so safe to call +// frequently. +func (c *BranchCache) publishPinnedGauge() { + mxPinnedEntries.SetUint64(uint64(c.pinned.Len())) +} + +// PublishMetrics snapshots the cache's current counters into the +// Prometheus registry. Call periodically from the host (e.g. once per +// SD.Flush) to keep metrics fresh — atomic counter loads are cheap +// but not free, so a one-call-per-batch cadence avoids hot-path cost. +func (c *BranchCache) PublishMetrics() { + mxPinnedHits.SetUint64(c.pinnedHits.Load()) + mxPinnedMisses.SetUint64(c.pinnedMisses.Load()) + c.publishPinnedGauge() +} From f48c411a9caf63cc3ea2df03b083a5dc0c67cadf Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:21:15 +0000 Subject: [PATCH 058/167] commitment: tests for trunk-pin layer + delete orphaned warmup test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New trunk_pin_test.go covers the additions: TestPinEntry_AndPinnedCount — basic pin-tier mechanics TestInvalidate_RemovesFromPinned — pin-aware Invalidate TestGetWithOrigin_ChecksPinnedTier — pin-aware GetWithOrigin TestClear_ResetsPinnedTier — pin-aware Clear TestMissCallback_FiresOnTripleMiss — cache hook semantics TestContractHashFromPrefix_* — storage-trunk decode TestContractTrunkPreload_PhasedRun — Initial+Extend split TestAdaptivePinController_PromoteOnThreshold TestAdaptivePinController_DemoteOnCold Updated branch_cache_test.go's TestBranchCache_Stats to match the new Stats string format (added pinned-tier columns; "tail entries" moved past the pin block). Deleted execution/commitment/warmup_cache_test.go — orphaned by the WarmupCache type deletion in commit ded378f789. Test referenced NewWarmupCache which no longer exists; removing unblocks `go test ./execution/commitment/...`. --- execution/commitment/branch_cache_test.go | 4 +- execution/commitment/trunk_pin_test.go | 231 ++++++++++ execution/commitment/warmup_cache_test.go | 523 ---------------------- 3 files changed, 234 insertions(+), 524 deletions(-) create mode 100644 execution/commitment/trunk_pin_test.go delete mode 100644 execution/commitment/warmup_cache_test.go diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index f545a23a650..29e3534e817 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -176,7 +176,9 @@ func TestBranchCache_Stats(t *testing.T) { for _, want := range []string{ "root hit=1 miss=0", "tail hit=1 miss=1", - "tail entries=1", // we put 1 deep entry; root not counted in tail + // Pinned tier added; tail still has 1 entry (the deep Put), + // pinned tier remains empty in this test. + "tail hit=1 miss=1 (50.0%) entries=1", } { require.Contains(t, s, want, "Stats output: %s", s) } diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go new file mode 100644 index 00000000000..edf8ebac6b0 --- /dev/null +++ b/execution/commitment/trunk_pin_test.go @@ -0,0 +1,231 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "context" + "encoding/binary" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeReader returns a deterministic branch for any prefix that has a +// 16-bit bitmap encoded after the 2-byte touchMap. The bitmap is +// derived from the path so children are reproducible across runs. +func fakeReader(saturated bool) CommitmentReader { + return func(prefix []byte) ([]byte, uint64, bool, error) { + // Synthesize a branch: 2 B touchMap (zero) || 2 B bitmap || nothing. + // bitmap: full (0xFFFF) when saturated; sparse (single child idx 0) + // otherwise — produces a narrow trunk that exhausts the BFS quickly. + var bitmap uint16 + if saturated { + bitmap = 0xFFFF + } else { + bitmap = 0x0001 + } + buf := make([]byte, 4) + binary.BigEndian.PutUint16(buf[2:4], bitmap) + return buf, 1, true, nil + } +} + +// TestPinEntry_AndPinnedCount confirms basic pin-tier mechanics. +func TestPinEntry_AndPinnedCount(t *testing.T) { + c := NewBranchCache(8) + require.Equal(t, 0, c.PinnedCount()) + c.PinEntry([]byte{0x12, 0x34, 0x56}, []byte{0xab, 0xcd}, 1, "test") + require.Equal(t, 1, c.PinnedCount()) + + // Pinned entry is hit, not the tail. + data, _, ok := c.Get([]byte{0x12, 0x34, 0x56}) + require.True(t, ok) + require.Equal(t, []byte{0xab, 0xcd}, data) + hits, _, _ := c.PinnedStats() + require.Equal(t, uint64(1), hits) +} + +// TestInvalidate_RemovesFromPinned ensures Invalidate (now pin-aware) +// deletes a pinned entry, not just LRU/root. +func TestInvalidate_RemovesFromPinned(t *testing.T) { + c := NewBranchCache(8) + c.PinEntry([]byte{0x12, 0x34}, []byte{0x99}, 1, "test") + require.Equal(t, 1, c.PinnedCount()) + + c.Invalidate([]byte{0x12, 0x34}) + require.Equal(t, 0, c.PinnedCount()) + _, _, ok := c.Get([]byte{0x12, 0x34}) + require.False(t, ok) +} + +// TestGetWithOrigin_ChecksPinnedTier ensures GetWithOrigin reads +// pinned entries (was a correctness gap pre-fix). +func TestGetWithOrigin_ChecksPinnedTier(t *testing.T) { + c := NewBranchCache(8) + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 5, "preload") + data, origin, _, _, ok := c.GetWithOrigin([]byte{0x12, 0x34}) + require.True(t, ok) + require.Equal(t, []byte{0xab}, data) + require.Equal(t, "preload", origin) +} + +// TestClear_ResetsPinnedTier ensures Clear empties pinned + resets +// pinned-stats counters (was a correctness gap pre-fix). +func TestClear_ResetsPinnedTier(t *testing.T) { + c := NewBranchCache(8) + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 1, "test") + _, _, _ = c.Get([]byte{0x12, 0x34}) // bump pinned hit + c.Clear() + require.Equal(t, 0, c.PinnedCount()) + hits, misses, _ := c.PinnedStats() + require.Equal(t, uint64(0), hits) + require.Equal(t, uint64(0), misses) +} + +// TestMissCallback_FiresOnTripleMiss ensures the callback fires only +// when ALL three tiers (root, pinned, LRU) miss — not on cache hits. +func TestMissCallback_FiresOnTripleMiss(t *testing.T) { + c := NewBranchCache(8) + var fired atomic.Uint64 + c.SetMissCallback(func(prefix []byte) { + fired.Add(1) + }) + + // Hit (pinned): callback should NOT fire. + c.PinEntry([]byte{0x12, 0x34}, []byte{0xff}, 1, "test") + _, _, _ = c.Get([]byte{0x12, 0x34}) + require.Equal(t, uint64(0), fired.Load(), "pinned hit must not fire callback") + + // Miss: callback fires. + _, _, _ = c.Get([]byte{0x99, 0x88, 0x77}) + require.Equal(t, uint64(1), fired.Load(), "triple-miss must fire callback") + + // Unbind: subsequent miss does not fire. + c.SetMissCallback(nil) + _, _, _ = c.Get([]byte{0xaa, 0xbb}) + require.Equal(t, uint64(1), fired.Load(), "unbound callback must not fire") +} + +// TestContractHashFromPrefix_DecodesStorageTrunk ensures the helper +// extracts the contract hash from a 33+ B prefix and rejects shorter +// account-trie prefixes. +func TestContractHashFromPrefix_DecodesStorageTrunk(t *testing.T) { + // 33-byte prefix: 1 HP flag + 32 contract bytes. + prefix := make([]byte, 33) + prefix[0] = 0x00 // HP flag + for i := 1; i <= 32; i++ { + prefix[i] = byte(i) + } + hash, ok := ContractHashFromPrefix(prefix) + require.True(t, ok) + for i := 0; i < 32; i++ { + require.Equal(t, byte(i+1), hash[i]) + } + + // Short prefix: account-trie, must be rejected. + _, ok = ContractHashFromPrefix([]byte{0x00, 0x12, 0x34}) + require.False(t, ok) +} + +// TestContractTrunkPreload_PhasedRun confirms initial+extension semantics: +// first Run consumes its budget, second Run continues from where it left. +func TestContractTrunkPreload_PhasedRun(t *testing.T) { + c := NewBranchCache(64) + hash := make([]byte, 32) + for i := range hash { + hash[i] = byte(i) + } + p, err := NewContractTrunkPreload(hash) + require.NoError(t, err) + + reader := fakeReader(false /* sparse */) + + // Initial Run with tiny budget — pins the root and stops. + n1, queueEmpty, err := p.Run(1<<10, reader, c, nil) + require.NoError(t, err) + require.Greater(t, n1, 0, "initial Run must pin at least the root") + require.False(t, queueEmpty, "sparse trie shouldn't exhaust on tiny budget") + totalAfterInitial := p.PinnedTotal() + require.Equal(t, n1, totalAfterInitial) + + // Extension Run continues from where it stopped. + n2, _, err := p.Run(1<<10, reader, c, nil) + require.NoError(t, err) + require.GreaterOrEqual(t, n2, 0) + require.Equal(t, totalAfterInitial+n2, p.PinnedTotal()) + + // Pinned prefixes are tracked for demotion. + require.Equal(t, p.PinnedTotal(), len(p.PinnedPrefixes())) +} + +// TestAdaptivePinController_PromoteOnThreshold confirms a contract +// crossing PromoteThresholdMisses gets promoted on the next +// OnBlockComplete. +func TestAdaptivePinController_PromoteOnThreshold(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 3 + cfg.InitialViewBudgetBytes = 1 << 14 // 16 KiB; small but enough for sparse trie + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + // Forge a 33-byte prefix that decodes to a known contract hash. + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 1) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Trigger 3 cache-misses for the same contract. + for i := 0; i < 3; i++ { + _, _, _ = c.Get(prefix) + } + + // OnBlockComplete should promote the contract. + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1) + require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) + require.Greater(t, c.PinnedCount(), 0) +} + +// TestAdaptivePinController_DemoteOnCold confirms a promoted contract +// gets demoted after DemoteCooldownBlocks consecutive blocks with no +// misses. +func TestAdaptivePinController_DemoteOnCold(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.DemoteCooldownBlocks = 2 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 7) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Hot block: triggers promotion. + _, _, _ = c.Get(prefix) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1) + pinnedAfterPromote := c.PinnedCount() + require.Greater(t, pinnedAfterPromote, 0) + + // Cold blocks: no misses for the contract. After cooldown the + // controller demotes and invalidates the pin set. + ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1, "still pinned after 1 cold block") + + ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") + require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") +} diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go deleted file mode 100644 index db62ec6ceb2..00000000000 --- a/execution/commitment/warmup_cache_test.go +++ /dev/null @@ -1,523 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package commitment - -import ( - "bytes" - "crypto/rand" - "testing" - - "github.com/stretchr/testify/require" -) - -// TestWarmupCache_Basic tests basic put/get operations -func TestWarmupCache_Basic(t *testing.T) { - cache := NewWarmupCache() - - // Test branch operations - branchKey := []byte("test-branch-key-12345678901234567890") - branchData := []byte("branch-data-content") - cache.PutBranch(branchKey, branchData) - - gotData, found := cache.GetBranch(branchKey) - if !found { - t.Fatal("expected to find branch") - } - if !bytes.Equal(gotData, branchData) { - t.Errorf("branch data mismatch: got %x, want %x", gotData, branchData) - } - - // Test account operations - accountKey := []byte("12345678901234567890") // 20 bytes - accountUpdate := &Update{Flags: BalanceUpdate} - cache.PutAccount(accountKey, accountUpdate) - - gotUpdate, found := cache.GetAccount(accountKey) - if !found { - t.Fatal("expected to find account") - } - if gotUpdate.Flags != BalanceUpdate { - t.Errorf("account flags mismatch: got %v, want %v", gotUpdate.Flags, BalanceUpdate) - } - - // Test storage operations - storageKey := make([]byte, 52) - rand.Read(storageKey) - storageUpdate := &Update{Flags: StorageUpdate, StorageLen: 5} - copy(storageUpdate.Storage[:], "hello") - cache.PutStorage(storageKey, storageUpdate) - - gotStorage, found := cache.GetStorage(storageKey) - if !found { - t.Fatal("expected to find storage") - } - if gotStorage.Flags != StorageUpdate { - t.Errorf("storage flags mismatch") - } - if !bytes.Equal(gotStorage.Storage[:gotStorage.StorageLen], []byte("hello")) { - t.Errorf("storage data mismatch") - } -} - -// TestWarmupCache_NotFound tests cache misses -func TestWarmupCache_NotFound(t *testing.T) { - cache := NewWarmupCache() - - _, found := cache.GetBranch([]byte("nonexistent")) - if found { - t.Error("expected not to find nonexistent branch") - } - - _, found = cache.GetAccount([]byte("12345678901234567890")) - if found { - t.Error("expected not to find nonexistent account") - } - - _, found = cache.GetStorage(make([]byte, 52)) - if found { - t.Error("expected not to find nonexistent storage") - } -} - -// TestWarmupCache_Eviction tests key eviction -func TestWarmupCache_Eviction(t *testing.T) { - cache := NewWarmupCache() - - key := []byte("12345678901234567890") - cache.PutAccount(key, &Update{Flags: BalanceUpdate}) - - // Should find before eviction - _, found := cache.GetAccount(key) - if !found { - t.Fatal("expected to find account before eviction") - } - - // Evict the key - cache.EvictAccount(key) - - // Should not find after eviction - _, found = cache.GetAccount(key) - if found { - t.Error("expected not to find account after eviction") - } -} - -// TestWarmupCache_Clear tests clearing the cache -func TestWarmupCache_Clear(t *testing.T) { - cache := NewWarmupCache() - - // Add some data - cache.PutAccount([]byte("12345678901234567890"), &Update{}) - cache.PutStorage(make([]byte, 52), &Update{}) - cache.PutBranch([]byte("branch"), []byte("data")) - - // Clear - cache.Clear() - - // Should not find anything - _, found := cache.GetAccount([]byte("12345678901234567890")) - if found { - t.Error("expected not to find account after clear") - } -} - -// TestWarmupCache_KeyPadding tests that shorter keys work correctly -func TestWarmupCache_KeyPadding(t *testing.T) { - cache := NewWarmupCache() - - // Test with a key shorter than the fixed size - shortKey := []byte("short") - cache.PutBranch(shortKey, []byte("data")) - - gotData, found := cache.GetBranch(shortKey) - if !found { - t.Fatal("expected to find branch with short key") - } - if !bytes.Equal(gotData, []byte("data")) { - t.Errorf("data mismatch") - } - - // Ensure different short keys don't collide - shortKey2 := []byte("other") - cache.PutBranch(shortKey2, []byte("data2")) - - gotData, found = cache.GetBranch(shortKey) - if !found || !bytes.Equal(gotData, []byte("data")) { - t.Error("first key affected by second key") - } - - gotData2, found := cache.GetBranch(shortKey2) - if !found || !bytes.Equal(gotData2, []byte("data2")) { - t.Error("second key not found or wrong data") - } -} - -// generateTestKeys creates random keys of the given size -func generateTestKeys(n int, size int) [][]byte { - keys := make([][]byte, n) - for i := 0; i < n; i++ { - keys[i] = make([]byte, size) - rand.Read(keys[i]) - } - return keys -} - -// BenchmarkWarmupCache_Branch benchmarks branch key operations -func BenchmarkWarmupCache_Branch(b *testing.B) { - cache := NewWarmupCache() - - // Pre-populate with some data - keys := generateTestKeys(10000, 52) - data := make([]byte, 100) - rand.Read(data) - - for _, key := range keys { - cache.PutBranch(key, data) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - key := keys[i%len(keys)] - cache.GetBranch(key) - } -} - -// BenchmarkWarmupCache_Branch_Put benchmarks Put operations -func BenchmarkWarmupCache_Branch_Put(b *testing.B) { - cache := NewWarmupCache() - const keyCount = 10000 - keys := generateTestKeys(keyCount, 52) - data := make([]byte, 100) - rand.Read(data) - - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - cache.PutBranch(keys[i%keyCount], data) - } -} - -// BenchmarkWarmupCache_Account benchmarks account key operations (20 bytes) -func BenchmarkWarmupCache_Account(b *testing.B) { - cache := NewWarmupCache() - keys := generateTestKeys(10000, 20) - update := &Update{} - - for _, key := range keys { - cache.PutAccount(key, update) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - key := keys[i%len(keys)] - cache.GetAccount(key) - } -} - -// BenchmarkWarmupCache_Storage benchmarks storage key operations (52 bytes) -func BenchmarkWarmupCache_Storage(b *testing.B) { - cache := NewWarmupCache() - keys := generateTestKeys(10000, 52) - update := &Update{} - - for _, key := range keys { - cache.PutStorage(key, update) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - key := keys[i%len(keys)] - cache.GetStorage(key) - } -} - -// BenchmarkWarmupCache_Mixed simulates realistic mixed workload -func BenchmarkWarmupCache_Mixed(b *testing.B) { - cache := NewWarmupCache() - accountKeys := generateTestKeys(1000, 20) - storageKeys := generateTestKeys(5000, 52) - branchKeys := generateTestKeys(2000, 32) - update := &Update{} - data := make([]byte, 100) - - // Pre-populate - for _, key := range accountKeys { - cache.PutAccount(key, update) - } - for _, key := range storageKeys { - cache.PutStorage(key, update) - } - for _, key := range branchKeys { - cache.PutBranch(key, data) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - switch i % 3 { - case 0: - cache.GetAccount(accountKeys[i%len(accountKeys)]) - case 1: - cache.GetStorage(storageKeys[i%len(storageKeys)]) - case 2: - cache.GetBranch(branchKeys[i%len(branchKeys)]) - } - } -} - -// BenchmarkComparison_Map_100k benchmarks map with 100k entries -func BenchmarkComparison_Map_100k(b *testing.B) { - cache := NewWarmupCache() - keys := generateTestKeys(100000, 52) - update := &Update{} - - for _, key := range keys { - cache.PutStorage(key, update) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - cache.GetStorage(keys[i%len(keys)]) - } -} - -// TestWarmupCache_Stats verifies that hit/miss/evict counters are updated -// correctly across the Get/Put/Evict paths and that Stats() returns a -// deterministic format. -func TestWarmupCache_Stats(t *testing.T) { - cache := NewWarmupCache() - - // Branch hits + misses + bytes-served - bk := []byte("k-branch") - bd := []byte("branch-data-12345") - cache.PutBranch(bk, bd) - if _, ok := cache.GetBranch(bk); !ok { - t.Fatal("expected branch hit") - } - if _, ok := cache.GetBranch([]byte("missing-branch")); ok { - t.Fatal("expected branch miss") - } - - // Branch eviction - cache.EvictBranch(bk) - if _, ok := cache.GetBranch(bk); ok { - t.Fatal("expected branch evicted miss") - } - - // Account hits + misses - ak := []byte("k-acct") - cache.PutAccount(ak, &Update{}) - if _, ok := cache.GetAccount(ak); !ok { - t.Fatal("expected account hit") - } - if _, ok := cache.GetAccount([]byte("missing-acct")); ok { - t.Fatal("expected account miss") - } - - // Storage hits + misses - sk := []byte("k-stor") - cache.PutStorage(sk, &Update{}) - if _, ok := cache.GetStorage(sk); !ok { - t.Fatal("expected storage hit") - } - if _, ok := cache.GetStorage([]byte("missing-stor")); ok { - t.Fatal("expected storage miss") - } - - // Counters: branch hit=1 miss=1 evict=1 bytes=17, acct hit=1 miss=1, stor hit=1 miss=1 - if got := cache.branchHits.Load(); got != 1 { - t.Fatalf("branchHits: got %d, want 1", got) - } - if got := cache.branchMisses.Load(); got != 1 { - t.Fatalf("branchMisses: got %d, want 1", got) - } - if got := cache.branchEvicted.Load(); got != 1 { - t.Fatalf("branchEvicted: got %d, want 1", got) - } - if got := cache.branchBytesServed.Load(); got != uint64(len(bd)) { - t.Fatalf("branchBytesServed: got %d, want %d", got, len(bd)) - } - if got := cache.accountHits.Load(); got != 1 { - t.Fatalf("accountHits: got %d, want 1", got) - } - if got := cache.accountMisses.Load(); got != 1 { - t.Fatalf("accountMisses: got %d, want 1", got) - } - if got := cache.storageHits.Load(); got != 1 { - t.Fatalf("storageHits: got %d, want 1", got) - } - if got := cache.storageMisses.Load(); got != 1 { - t.Fatalf("storageMisses: got %d, want 1", got) - } - - // Stats() format - stats := cache.Stats() - for _, want := range []string{ - "branch hit=1 miss=1 evict=1", - "acct hit=1 miss=1", - "stor hit=1 miss=1", - } { - if !bytes.Contains([]byte(stats), []byte(want)) { - t.Errorf("Stats() missing %q\nfull: %s", want, stats) - } - } - - // ResetStats zeros counters but leaves data intact - cache.ResetStats() - if got := cache.branchHits.Load(); got != 0 { - t.Fatalf("branchHits after Reset: got %d, want 0", got) - } - if _, ok := cache.GetAccount(ak); !ok { - t.Fatal("account data should survive ResetStats") - } -} - -// TestWarmupCache_DirtyFlag verifies the dirty flag scaffolding — -// MarkBranchDirty + PutBranchIfClean — that prevents late writers from -// clobbering fresh data once cross-block persistence is wired up. -func TestWarmupCache_DirtyFlag(t *testing.T) { - cache := NewWarmupCache() - key := []byte("k-branch") - v1 := []byte("first-value") - v2 := []byte("second-value-skipped") - - // First Put always succeeds (no existing entry to be dirty). - require.True(t, cache.PutBranchIfClean(key, v1)) - got, ok := cache.GetBranch(key) - require.True(t, ok) - require.Equal(t, v1, got) - - // Marking dirty doesn't remove the entry — Get still returns it. - cache.MarkBranchDirty(key) - got, ok = cache.GetBranch(key) - require.True(t, ok, "MarkBranchDirty should not evict; consumers decide what to do with dirty data") - require.Equal(t, v1, got) - - // PutBranchIfClean refuses to overwrite a dirty entry. - require.False(t, cache.PutBranchIfClean(key, v2), - "PutBranchIfClean must skip when entry is dirty") - got, ok = cache.GetBranch(key) - require.True(t, ok) - require.Equal(t, v1, got, "dirty value must be preserved (write was rejected)") - - // PutBranch (non-If-Clean variant) overwrites unconditionally — used - // by the fold encoder path that holds the canonical new value. - cache.PutBranch(key, v2) - got, ok = cache.GetBranch(key) - require.True(t, ok) - require.Equal(t, v2, got) - - // After unconditional overwrite, the new entry is no longer dirty - // (fresh entry replaces the dirty one). - require.True(t, cache.PutBranchIfClean(key, []byte("third-value")), - "PutBranchIfClean must accept after unconditional Put cleared the dirty entry") -} - -// TestWarmupCache_DirtyFlag_MarkAbsentKey verifies that marking a key -// dirty when no entry exists is a no-op (no panic, no entry created). -func TestWarmupCache_DirtyFlag_MarkAbsentKey(t *testing.T) { - cache := NewWarmupCache() - cache.MarkBranchDirty([]byte("never-stored")) - _, ok := cache.GetBranch([]byte("never-stored")) - require.False(t, ok, "MarkBranchDirty on absent key should not create an entry") - - // PutBranchIfClean for the same absent key should succeed (no dirty - // entry exists, so the write proceeds). - require.True(t, cache.PutBranchIfClean([]byte("never-stored"), []byte("v"))) -} - -// TestWarmupCache_GetBranchDecoded verifies the lazy-decode read path: -// stored encoded bytes are decoded on first decoded-read and cached for -// subsequent reads, returning cells equivalent to direct DecodeBranchInto. -func TestWarmupCache_GetBranchDecoded(t *testing.T) { - cache := NewWarmupCache() - - // Encode a branch the same way BranchEncoder would, so our test - // data has the canonical [touchMap | bitmap | cells...] layout. - row, bm := generateCellRow(t, 16) - be := NewBranchEncoder(1024) - cellData := generateCellEncodeDataRow(t, row, bm) - enc, err := be.EncodeBranch(bm, bm, bm, &cellData) - require.NoError(t, err) - - prefix := []byte{0x12, 0x34} - cache.PutBranch(prefix, enc) - - // First decoded-read decodes lazily. - bitmap, cells, ok := cache.GetBranchDecoded(prefix) - require.True(t, ok) - require.Equal(t, bm, bitmap) - require.NotNil(t, cells) - - // Cells should match what direct DecodeBranchInto would produce - // from the same encoded bytes (skip the 2-byte touchMap prefix). - var expected [16]cell - expectedMaps, err := DecodeBranchInto(enc[2:], false, &expected) - require.NoError(t, err) - require.Equal(t, expectedMaps.Bitmap, bitmap) - for i := range cells { - require.Equal(t, expected[i].extLen, cells[i].extLen, "cell %d extLen", i) - require.Equal(t, expected[i].extension[:expected[i].extLen], cells[i].extension[:cells[i].extLen], "cell %d extension", i) - require.Equal(t, expected[i].accountAddr[:expected[i].accountAddrLen], cells[i].accountAddr[:cells[i].accountAddrLen], "cell %d accountAddr", i) - require.Equal(t, expected[i].storageAddr[:expected[i].storageAddrLen], cells[i].storageAddr[:cells[i].storageAddrLen], "cell %d storageAddr", i) - require.Equal(t, expected[i].hash[:expected[i].hashLen], cells[i].hash[:cells[i].hashLen], "cell %d hash", i) - } - - // Second decoded-read returns the SAME cells pointer — lazy decode - // runs at most once per entry. - bitmap2, cells2, ok := cache.GetBranchDecoded(prefix) - require.True(t, ok) - require.Equal(t, bm, bitmap2) - require.Same(t, cells, cells2, "expected cached cells pointer to be reused") - - // Encoded form is unchanged after decoded reads. - encGot, ok := cache.GetBranch(prefix) - require.True(t, ok) - require.Equal(t, []byte(enc), encGot, "encoded form unchanged by decoded reads") -} - -// TestWarmupCache_GetBranchDecoded_Miss verifies that misses on -// GetBranchDecoded behave the same as misses on GetBranch. -func TestWarmupCache_GetBranchDecoded_Miss(t *testing.T) { - cache := NewWarmupCache() - _, _, ok := cache.GetBranchDecoded([]byte("never-stored")) - require.False(t, ok) -} - -// TestWarmupCache_GetBranchDecoded_TruncatedData verifies the decode-error -// path — corrupt entry returns ok=false rather than panicking. -func TestWarmupCache_GetBranchDecoded_TruncatedData(t *testing.T) { - cache := NewWarmupCache() - // One byte is shorter than the touchMap prefix; decode will error. - cache.PutBranch([]byte("k"), []byte{0x42}) - _, _, ok := cache.GetBranchDecoded([]byte("k")) - require.False(t, ok, "truncated entry should return ok=false, not panic") - - // Encoded form still retrievable for callers that don't need decode. - got, ok := cache.GetBranch([]byte("k")) - require.True(t, ok) - require.Equal(t, []byte{0x42}, got) -} From e69e9bfbacd84645f5665433c949464c4a4b1f9f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:26:48 +0000 Subject: [PATCH 059/167] commitment, db/state: drop bench-only env knobs and cache-fp log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup pass before PR. Removes scaffolding that supported the SSTORE-bloat investigation but isn't operator-relevant for production: Env knobs removed: - TRUNK_PROBE — logged contract hashes of depth-64 commitment prefixes on first sight. Was used to identify the bloat contract; now obsolete. - DISABLE_BRANCH_CACHE_READS — A/B switch for cache-vs-no-cache benchmarking. The cache is now a fundamental architecture component; DISABLE_ADAPTIVE_PIN already covers the operator-safety use case. - BRANCH_CACHE_FINGERPRINT — gated the [commitment][cache-fp] log line that emitted ~30 fields of investigation-era counters per block. Replaced by Prometheus metrics and the [adaptive-pin] log. - Dead variables in hex_patricia_hashed.go: verifyBranchCache and disableBranchCacheReads were declared but never referenced (callers were removed in the WarmupCache deletion). Code removed: - The 90-line [cache-fp] log block in commitmentdb/commitment_context.go that referenced ~10 bench-only counter helpers (SkipLoadResetCounters, ContainsHashStats, SstoreClassificationCounts, WarmerBranchOutcomeStats, FileReadCount, UniqueLenBuckets, etc.) - The trunk-probe path in db/state/changeset/state_changeset.go - Stale comment references to verifyBranchCache The bench-only counter helpers themselves (SkipLoadResetCounters etc.) are now unreferenced but left in place — a follow-up cleanup commit can remove them once we confirm no other callers exist. --- db/state/changeset/state_changeset.go | 16 --- db/state/execctx/domain_shared.go | 10 +- execution/commitment/branch_cache.go | 6 ++ .../commitmentdb/commitment_context.go | 102 ------------------ execution/commitment/hex_patricia_hashed.go | 19 ---- execution/commitment/trunk_pin_metrics.go | 26 ++--- 6 files changed, 21 insertions(+), 158 deletions(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 04b0667a95d..0e30f687897 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -18,7 +18,6 @@ package changeset import ( "encoding/binary" - "encoding/hex" "fmt" "math" "strings" @@ -28,19 +27,11 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/length" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/execution/types/accounts" ) -// logTrunkProbe is set by TRUNK_PROBE=true to log the keccak256 hash of -// each contract whose storage subtree root (depth-64 commitment prefix, -// 33 bytes) is read from the file layer for the first time. The bytes -// after the 0x00 flag byte ARE the contract's keccak256(addr), which -// is what PIN_CONTRACT_TRUNKS expects in its hex form. -var logTrunkProbe = dbg.EnvBool("TRUNK_PROBE", false) - type StateChangeSet struct { Diffs [kv.DomainLen]kv.DomainDiff // there are 4 domains of state changes } @@ -639,13 +630,6 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) bucket := lenBucket(len(key)) - // Trunk-probe log: depth-64 commitment prefix on first sight. - // Format key as 0x00 || keccak256(addr); the trailing 32 bytes are - // the value to feed to PIN_CONTRACT_TRUNKS for that contract. - if logTrunkProbe && !alreadySeen && bucket == 5 && domain == kv.CommitmentDomain && len(key) == 33 && key[0] == 0x00 { - log.Info("[trunk-probe] new depth-64 commitment prefix", "contract_hash", hex.EncodeToString(key[1:33])) - } - dm.Lock() defer dm.Unlock() dm.FileReadCount++ diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a55c270c267..da2b9a97610 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -210,12 +210,6 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { branchCache = p.BranchCache() } - // DISABLE_BRANCH_CACHE_READS gates the cache out for A/B benchmarking. - // When set, sd.branchCache stays nil so sd.GetLatest skips the cache - // layer entirely and every CommitmentDomain read goes to MDBX. - if dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) { - branchCache = nil - } sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) @@ -647,8 +641,8 @@ func (sd *SharedDomains) GetStateCache() *cache.StateCache { // the parent SD's mem, and direct MDBX via tx.GetLatest) and returns // the bytes from each. Read-only, intended for divergence-detection // diagnostics — pinpoints which layer holds bytes that disagree with -// the BranchCache when verifyBranchCache is on. Bytes are copied so -// the caller can hold them past tx lifetime. +// the BranchCache. Bytes are copied so the caller can hold them past +// tx lifetime. func (sd *SharedDomains) ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { if v, _, ok := sd.mem.GetLatest(domain, key); ok { memOk = true diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index d7c22d2e20d..c07f25ea189 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -200,6 +200,12 @@ type BranchCache struct { // when no callback is installed. onMiss atomic.Pointer[MissCallback] + // Last-published Prometheus counter snapshots. PublishMetrics + // emits the delta between current and last so the monotonic + // counters track real activity per Flush, not snapshot absolutes. + lastPublishedPinnedHits atomic.Uint64 + lastPublishedPinnedMisses atomic.Uint64 + // preloadClaimed is set the first time TryClaimPreload is called. // Used by the trunk-preload trigger (PIN_CONTRACT_TRUNKS hook in // SharedDomains construction) so the preload goroutine fires diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 5a97bceed4b..7a95675b315 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -18,7 +18,6 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/datastruct/existence" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" @@ -33,13 +32,6 @@ import ( var ( mxCommitmentRunning = metrics.GetOrCreateGauge("domain_running_commitment") mxCommitmentTook = metrics.GetOrCreateSummary("domain_commitment_took") - - // logCacheFingerprint, when true, makes ComputeCommitment emit a - // "[cache-fp]" log line at end-of-block with (block, root, fp, - // divergences). Diff two builds' logs offline to localise the first - // block at which their BranchCache states diverge. Off by default — - // gate via env BRANCH_CACHE_FINGERPRINT=true. - logCacheFingerprint = dbg.EnvBool("BRANCH_CACHE_FINGERPRINT", false) ) type sd interface { @@ -335,100 +327,6 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context } log.Debug("[commitment] processed", "block", blockNum, "txNum", txNum, "keys", common.PrettyCounter(updateCount), "keys/s", common.PrettyCounter(keysPerSec), "mode", sdc.updates.Mode(), "spent", took, "rootHash", hex.EncodeToString(rootHash)) - // Per-block BranchCache fingerprint — gated by BRANCH_CACHE_FINGERPRINT - // env. Emit a single log line of (block, root, cache_fp, divergences) - // so two builds running the same workload can be diffed offline to - // localise the first block at which their caches diverge. See - // commitment.BranchCache.Fingerprint for the hash semantics. - // took + keys exposed at Info so the calculator's per-block compute - // time is visible without enabling debug logs (used by the - // snapshot-vs-mdbx perf-equivalence investigation to attribute - // the gap inside newPayload). - if logCacheFingerprint { - if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { - if bc := hph.BranchCache(); bc != nil { - // Skip/load/reset counters are process-cumulative; the - // per-block delta requires subtracting consecutive lines. - // Used to answer: "what fraction of computeCellHash calls - // actually fetched the underlying value vs reused a - // memoized stateHash?" - load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() - hasStoMiss := commitment.HasStorageMissCount() - xfTrue, xfFalse := existence.ContainsHashStats() - ssIns, ssUpd, ssDel, _ := commitment.SstoreClassificationCounts() - wHit, wEmpty := commitment.WarmerBranchOutcomeStats() - // Per-domain file-read counts: lets us decompose the - // aggregate `files=N` from [domain reads] into Commitment - // (branch reads), Storage (value loads), Account, Code. - // All cumulative; deltas via successive lines. - var aFiles, sFiles, cFiles, mFiles int64 - var aUniq, sUniq, cUniq, mUniq int64 - var mLens [10]int64 - if m := sdc.sharedDomains.Metrics(); m != nil { - m.RLock() - if d := m.Domains[kv.AccountsDomain]; d != nil { - aFiles = d.FileReadCount - aUniq = d.UniqueFileReadCount - } - if d := m.Domains[kv.StorageDomain]; d != nil { - sFiles = d.FileReadCount - sUniq = d.UniqueFileReadCount - } - if d := m.Domains[kv.CodeDomain]; d != nil { - cFiles = d.FileReadCount - cUniq = d.UniqueFileReadCount - } - if d := m.Domains[kv.CommitmentDomain]; d != nil { - mFiles = d.FileReadCount - mUniq = d.UniqueFileReadCount - mLens = d.UniqueLenBuckets - } - m.RUnlock() - } - // commLens shows depth distribution. Buckets within the storage - // subtree (33B+) are sub-divided to distinguish per-contract - // storage trunk (33B / 34-36B) from leaf-parent depths - // (45-64B). See state_changeset.go UniqueLenBuckets. - commLens := fmt.Sprintf( - "1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33B:%d|34-36B:%d|37-44B:%d|45-64B:%d|>64B:%d", - mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], - mLens[5], mLens[6], mLens[7], mLens[8], mLens[9]) - pinHits, pinMisses, pinEntries := bc.PinnedStats() - log.Info("[commitment][cache-fp]", - "block", blockNum, - "root", hex.EncodeToString(rootHash), - "fp", fmt.Sprintf("%016x", bc.Fingerprint()), - "divergences", bc.VerifyDivergences(), - "took", took, - "keys", common.PrettyCounter(updateCount), - "load", load, - "skipped", skipped, - "reset", reset, - "disk_sto", diskSto, - "disk_acc", diskAcc, - "has_sto_miss", hasStoMiss, - "xf_true", xfTrue, - "xf_false", xfFalse, - "ss_ins", ssIns, - "ss_upd", ssUpd, - "ss_del", ssDel, - "w_hit", wHit, - "w_empty", wEmpty, - "files_acc", aFiles, - "files_sto", sFiles, - "files_code", cFiles, - "files_comm", mFiles, - "uniq_acc", aUniq, - "uniq_sto", sUniq, - "uniq_code", cUniq, - "uniq_comm", mUniq, - "comm_lens", commLens, - "pin_hit", pinHits, - "pin_miss", pinMisses, - "pin_count", pinEntries) - } - } - } }() if updateCount == 0 { rootHash, err = sdc.patriciaTrie.RootHash() diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index ce4bb76cd84..eedacefab08 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2994,25 +2994,6 @@ func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } -// verifyBranchCache, when true, makes branchFromCacheOrDB cross-check -// every BranchCache hit against ctx.Branch and record a divergence on -// the cache when the bytes disagree. Off by default — gate via env -// BRANCH_CACHE_VERIFY=true. Use during cross-block-cache-lifetime -// investigations to localise a bad cached entry to the read site -// instead of waiting for the downstream wrong-trie-root to surface -// many blocks later. -var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) - -// disableBranchCacheReads forces branchFromCacheOrDB to skip the L2 -// BranchCache read path — every read goes to ctx.Branch (sd.mem → -// MDBX). Cache writes (CollectUpdate.Put, branchFromCacheOrDB.Put on -// L3 fallback) still fire so verify-mode can keep comparing cache vs -// canonical. Use to A/B test correctness with-cache vs without-cache -// without recompiling. Confirmed in 2026-05-06 investigation that -// flipping this distinguishes "cache holds bad data" (bench passes -// further) from "deeper compute bug" (bench fails at the same block). -var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) - // branchFromCacheOrDB reads branch data via ctx.Branch, which goes // through sd.mem -> sd.parent.mem -> aggregator-scope BranchCache -> // MDBX. The HPH-side warmup cache layer that used to sit above this diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go index 5bf8e3f8a63..82c6b2e26fc 100644 --- a/execution/commitment/trunk_pin_metrics.go +++ b/execution/commitment/trunk_pin_metrics.go @@ -39,19 +39,19 @@ var ( mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") ) -// publishPinnedGauge updates the pinned-entries gauge from the cache's -// current size. Called from PublishMetrics; cheap so safe to call -// frequently. -func (c *BranchCache) publishPinnedGauge() { - mxPinnedEntries.SetUint64(uint64(c.pinned.Len())) -} - // PublishMetrics snapshots the cache's current counters into the -// Prometheus registry. Call periodically from the host (e.g. once per -// SD.Flush) to keep metrics fresh — atomic counter loads are cheap -// but not free, so a one-call-per-batch cadence avoids hot-path cost. +// Prometheus registry. Counters are monotonic-add so the cache tracks +// last-published values internally and emits deltas. Gauges Set +// absolute. Call periodically from the host (e.g. once per SD.Flush); +// the once-per-batch cadence avoids hot-path cost. func (c *BranchCache) PublishMetrics() { - mxPinnedHits.SetUint64(c.pinnedHits.Load()) - mxPinnedMisses.SetUint64(c.pinnedMisses.Load()) - c.publishPinnedGauge() + hits := c.pinnedHits.Load() + misses := c.pinnedMisses.Load() + if delta := hits - c.lastPublishedPinnedHits.Swap(hits); delta > 0 { + mxPinnedHits.AddUint64(delta) + } + if delta := misses - c.lastPublishedPinnedMisses.Swap(misses); delta > 0 { + mxPinnedMisses.AddUint64(delta) + } + mxPinnedEntries.SetUint64(uint64(c.pinned.Len())) } From e0b01892a7224281373e971cb27481a90b4e7ef2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:31:54 +0000 Subject: [PATCH 060/167] lint: gofmt fixes + drop snapshot_vs_mdbx bench test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply gofmt to four files where the perf-stack work left formatting differences from main's conventions. None are functionally meaningful; all are whitespace. Drop db/state/snapshot_vs_mdbx_bench_test.go — 714-line bench test file from the snapshot-vs-MDBX equivalence investigation (snapshot-vs-mdbx-performance-equivalence.md memo). Out of scope for this PR; the bench harness can return as a standalone artefact when that investigation resumes. --- common/dbg/experiments.go | 2 +- db/datastruct/existence/existence_filter.go | 1 + db/state/snapshot_vs_mdbx_bench_test.go | 714 ------------------ .../hex_concurrent_patricia_hashed.go | 1 + execution/commitment/hex_patricia_hashed.go | 5 +- 5 files changed, 6 insertions(+), 717 deletions(-) delete mode 100644 db/state/snapshot_vs_mdbx_bench_test.go diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index b72c956181d..0d34c5a76b7 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -119,7 +119,7 @@ var ( BorValidateHeaderTime = EnvBool("BOR_VALIDATE_HEADER_TIME", true) TraceDeletion = EnvBool("TRACE_DELETION", false) - RpcDropResponse = EnvBool("RPC_DROP_RESPONSE", false) + RpcDropResponse = EnvBool("RPC_DROP_RESPONSE", false) // The original default was NumCPU()*8 with the rationale "io-bound, more // workers drive more concurrent I/O." Our measurements (cold-cgroup 6c // vs unconstrained 12c on the SSTORE-bloat bench) show read_bytes diff --git a/db/datastruct/existence/existence_filter.go b/db/datastruct/existence/existence_filter.go index 78e85db1a70..32230e73d6c 100644 --- a/db/datastruct/existence/existence_filter.go +++ b/db/datastruct/existence/existence_filter.go @@ -80,6 +80,7 @@ func (b *Filter) AddHash(hash uint64) error { b.filter.AddHash(hash) return nil } + // Process-cumulative counters of ContainsHash outcomes. // True = "probably present" (filter does not short-circuit, lookup proceeds). // False = "definitely not present" (filter short-circuits the lookup). diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go deleted file mode 100644 index c2efacdcab9..00000000000 --- a/db/state/snapshot_vs_mdbx_bench_test.go +++ /dev/null @@ -1,714 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package state_test - -// Benchmarks for the "Snapshot vs MDBX read-cost equivalence" -// investigation. See memory: snapshot-vs-mdbx-performance-equivalence.md -// -// Premise: with the OS page cache warm, a key lookup served from -// MDBX vs from a snapshot .kv file should cost roughly the same — the -// disk and page cache are identical. Today they don't. These benches -// quantify the gap (H0) and decompose it (H1-H4). -// -// **Status: SCAFFOLD.** Compiles and runs (skipped) so the structure -// survives. Real-datadir bootstrap and synthetic-fixture bootstrap are -// the two TODOs that turn this into a measurement tool. -// -// Two operating modes are anticipated: -// -// 1. **Synthetic mode** (preferred for CI / reproducibility): -// use testDbAndAggregatorBench, write K1 keys at low txNums, -// buildFiles+prune so they land in files, write K2 at high -// txNums and leave in MDBX. No external datadir required. -// -// 2. **Real-datadir mode** (preferred for fidelity): -// open an existing perf-devnet-3 / mainnet datadir read-only, -// pick keys by cursor-iterating the values table (mdbxKeys) and -// walking the latest .kv decompressor (fileKeys). Matches -// production file-count, Bloom false-positive rate, etc. -// -// Both fail with b.Skip() in this scaffold; the TODOs below are the -// agenda for turning it into a real measurement. - -import ( - "encoding/binary" - "flag" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/erigontech/erigon/common/length" - "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/datadir" - "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/kv/dbcfg" - "github.com/erigontech/erigon/db/kv/mdbx" - "github.com/erigontech/erigon/db/kv/temporal" - "github.com/erigontech/erigon/db/state" - "github.com/erigontech/erigon/db/state/execctx" -) - -var snapDataDir = flag.String("snapdatadir", "", - "path to a real datadir (chaindata + snapshots) for snapshot-vs-mdbx benches; "+ - "when empty, real-datadir benches are skipped and synthetic mode runs") - -// snapBenchSetup carries everything a sub-bench needs: an open RO tx, -// the AggregatorRoTx for Debug* paths, the domain under test, and the -// pre-picked key sets. Set up once per Benchmark function so warmup -// and timing happen on the same data. -type snapBenchSetup struct { - tx kv.TemporalTx - aggTx *state.AggregatorRoTx - domain kv.Domain - mdbxKeys [][]byte - fileKeys [][]byte -} - -// openSnapBench builds a snapBenchSetup for the given domain. Picks -// real-datadir mode iff --snapdatadir is set, otherwise synthetic. -// Both paths currently call b.Skip; the TODOs below are the work. -func openSnapBench(b *testing.B, domain kv.Domain) *snapBenchSetup { - b.Helper() - if *snapDataDir != "" { - return openSnapBenchRealDatadir(b, domain, *snapDataDir) - } - return openSnapBenchSynthetic(b, domain) -} - -// openSnapBenchRealDatadir opens an existing chaindata+snapshots -// datadir read-only and picks keys from production files. -// -// Uses MDBX Accede mode (don't create, must already exist) and -// state.New(...).MustOpen which loads the on-disk schema. ResolveDB -// settings reads stepSize / stepsInFrozenFile from the existing DB. -// -// CAUTION: opening a chaindata that another erigon process is writing -// to is risky. Bench against an idle copy. The script that drives -// these benches typically does prepare-run.sh first to snapshot a -// fresh chaindata copy under /erigon-data/perf-devnet-3-run/ or -// similar. Pass that path via --snapdatadir. -func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadirPath string) *snapBenchSetup { - b.Helper() - logger := log.New() - dirs := datadir.New(datadirPath) - - rawDB := mdbx.New(dbcfg.ChainDB, logger). - Path(dirs.Chaindata). - Accede(true). // must exist; do not create - MustOpen() - b.Cleanup(rawDB.Close) - - settings, err := state.ResolveErigonDBSettings(dirs, logger, false /*noDownloader*/) - require.NoError(b, err, "ResolveErigonDBSettings") - - agg := state.New(dirs). - Logger(logger). - WithErigonDBSettings(settings). - MustOpen(b.Context(), rawDB) - require.NoError(b, agg.OpenFolder(), "agg.OpenFolder") - b.Cleanup(agg.Close) - - tdb, err := temporal.New(rawDB, agg) - require.NoError(b, err) - b.Cleanup(tdb.Close) - - tx, err := tdb.BeginTemporalRo(b.Context()) - require.NoError(b, err) - b.Cleanup(func() { tx.Rollback() }) - - aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) - require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") - - mdbxKeys, fileKeys := pickRealDatadirKeys(b, aggTx, tx, domain, 10_000) - require.NotEmpty(b, fileKeys, "no file-resident keys found in %s for domain %v", datadirPath, domain) - if len(mdbxKeys) == 0 { - // Heavily pruned production datadir — every MDBX row is - // step-shadowed by a file. The MDBX-side sub-benches will skip - // (runH0 short-circuits on empty key set). Compare File_path - // against the synthetic MDBX_path baseline; clearly note the - // cross-mode caveat in the issue write-up. - b.Logf("real-datadir: no MDBX-resident keys (datadir is fully pruned). " + - "File_path numbers are real; cross-reference MDBX_path against the synthetic bench.") - } - b.Logf("real-datadir keys: mdbx=%d file=%d (domain=%v, path=%s)", - len(mdbxKeys), len(fileKeys), domain, datadirPath) - - return &snapBenchSetup{ - tx: tx, - aggTx: aggTx, - domain: domain, - mdbxKeys: mdbxKeys, - fileKeys: fileKeys, - } -} - -// pickRealDatadirKeys walks the existing datadir to find up to -// `target` MDBX-resident and `target` file-resident keys for the -// given domain. -// -// Strategy A — cursor-walk the per-domain values table for -// MDBX-resident keys (those rows ARE in MDBX by definition). Strip -// the trailing 8-byte inverted-step from each row's key. -// -// Strategy B — for file-resident keys, randomly sample address-sized -// or storage-key-sized byte strings from the cursor walk's MDBX keys -// (deterministic via seeded RNG): for each candidate, query -// DebugGetLatestFromFiles. If the file path returns ok and MDBX does -// NOT, the key is file-resident. This is approximate — keys that -// exist BOTH in MDBX and files (during prune transitions) are -// classified as MDBX-resident, which is what we want for bench -// timing accuracy (MDBX always wins on getLatest). -// -// Practical note: in production datadirs the MDBX values table -// usually has fewer rows than the snapshot files combined, so we -// expect mdbxKeys to fill faster than fileKeys. To find file-only -// keys we'd need to walk the .kv files directly via seg.Decompressor -// — that's a follow-on once Strategy B is shown to be insufficient. -func pickRealDatadirKeys(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, target int) (mdbxKeys, fileKeys [][]byte) { - b.Helper() - table := domainValsTable(b, domain) - - c, err := tx.Cursor(table) - require.NoError(b, err) - defer c.Close() - - const stepSuffixLen = 8 // values-table key = realKey || invertedStep - seen := make(map[string]struct{}) - var rowsScanned, dbOkCount int - - // First pass: collect MDBX-resident keys directly from the values table. - for k, _, err := c.First(); k != nil; k, _, err = c.Next() { - require.NoError(b, err) - rowsScanned++ - if len(k) <= stepSuffixLen { - continue // unexpected; skip - } - realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) - ks := string(realKey) - if _, dup := seen[ks]; dup { - continue - } - seen[ks] = struct{}{} - - // Confirm MDBX residency via DebugGetLatestFromDB so we don't - // include keys that, despite having a row here, don't actually - // resolve in the read path (edge cases around step bounds). - _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) - require.NoError(b, err) - if dbOk { - dbOkCount++ - mdbxKeys = append(mdbxKeys, realKey) - if len(mdbxKeys) >= target { - break - } - } - } - b.Logf("first-pass scan: table=%s rows=%d unique_keys=%d dbOk=%d", - table, rowsScanned, len(seen), dbOkCount) - - // Second pass: walk the same cursor again looking for keys that - // MISS MDBX but HIT files. Iterating the values table catches - // keys that left a row but have all data in files (delete - // markers, post-prune residue), which is approximate but cheap. - // For a full file-only walk we'd open the .kv decompressor; this - // is good enough for the H0 first cut. - for k, _, err := c.First(); k != nil; k, _, err = c.Next() { - if len(fileKeys) >= target { - break - } - require.NoError(b, err) - if len(k) <= stepSuffixLen { - continue - } - realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) - - _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) - require.NoError(b, err) - if dbOk { - continue // MDBX already serves this; not file-only - } - _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, realKey, 0) - require.NoError(b, err) - if fileOk { - fileKeys = append(fileKeys, realKey) - } - } - - return mdbxKeys, fileKeys -} - -// domainValsTable returns the MDBX values-table name for the given -// domain. Used by pickRealDatadirKeys to cursor-walk MDBX directly. -func domainValsTable(b *testing.B, domain kv.Domain) string { - b.Helper() - switch domain { - case kv.AccountsDomain: - return kv.TblAccountVals - case kv.StorageDomain: - return kv.TblStorageVals - case kv.CodeDomain: - return kv.TblCodeVals - case kv.CommitmentDomain: - return kv.TblCommitmentVals - default: - b.Fatalf("domainValsTable: domain %v not mapped", domain) - return "" - } -} - -// openSnapBenchSynthetic builds an in-memory aggregator with a known -// key distribution and returns a setup whose two key sets are -// guaranteed to live on the file path and the MDBX path respectively. -// -// Sequence: -// -// 1. testDbAndAggregatorBench builds a fresh agg + tempdir. -// 2. Phase 1 — write keysetSize "old" keys at txNums spanning steps -// [0..stepsToFlush-1], deterministically generated from seed -// phase1Seed so the same bench reproduces. -// 3. Flush + commit + agg.BuildFiles(stepsToFlush*aggStep) so those -// keys land in .kv files. PruneSmallBatches removes them from -// MDBX. After this, phase-1 keys are file-resident. -// 4. Phase 2 — write keysetSize "new" keys at txNums spanning steps -// [stepsToFlush..stepsToFlush+1], from disjoint phase2Seed so no -// overlap with phase 1. NO BuildFiles for these. They stay in -// MDBX. -// 5. Begin a RO temporal tx, return setup. -// -// We verify each key actually lives where we expect via -// DebugGetLatestFromDB / DebugGetLatestFromFiles — fail loudly if a -// key isn't where the test plan says it should be (catches setup -// regressions). -func openSnapBenchSynthetic(b *testing.B, domain kv.Domain) *snapBenchSetup { - b.Helper() - - const ( - // aggStep small + many full steps mirrors how the fuzz test sets - // up data. BuildFiles only builds completed steps, so phase-1 has - // to span an integer number of full steps. - aggStep = 16 - stepsToFlush = 64 // phase 1 spans this many full steps - keysetSize = stepsToFlush * aggStep // = 1024; one key per txNum, every step is full - phase1Seed = 0xC0FFEE - phase2Seed = 0xDECAFB - // phase-2 starts well past the built step boundary so its keys - // can't be lumped into a future build. - phase2BaseTxNum = uint64(stepsToFlush) * aggStep - ) - - // keySize varies per domain; values are 8-byte counters (txNum - // encoded big-endian) so put/lookup paths are exercised. - var keySize int - switch domain { - case kv.AccountsDomain: - keySize = length.Addr - case kv.StorageDomain: - keySize = length.Addr + length.Hash - default: - b.Fatalf("openSnapBenchSynthetic: domain %v not yet wired", domain) - } - - db, agg := testDbAndAggregatorBench(b, aggStep) - logger := log.New() - - // ---- Phase 1: keys destined for files. Each key gets its own - // txNum so per-tx history entries are well-formed. - phase1Keys := genKeys(phase1Seed, keysetSize, keySize) - - { - rwTx, err := db.BeginTemporalRw(b.Context()) - require.NoError(b, err) - - domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) - require.NoError(b, err) - - val := make([]byte, 8) - for i, k := range phase1Keys { - // One key per txNum, packing every txNum in the - // stepsToFlush * aggStep range exactly once. This guarantees - // every step is full so BuildFiles emits a .kv per step. - txNum := uint64(i) - binary.BigEndian.PutUint64(val, txNum+1) // +1 so all values are non-zero - err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) - require.NoError(b, err) - } - - // ComputeCommitment writes the trie root into the CommitmentDomain - // so a follow-up NewSharedDomains can SeekCommitment without - // erroring "commitment state out of date". - lastTxNum := uint64(keysetSize - 1) - _, err = domains.ComputeCommitment(b.Context(), rwTx, true /*save*/, 0, lastTxNum, "h0-bench", nil) - require.NoError(b, err) - - require.NoError(b, domains.Flush(b.Context(), rwTx)) - domains.Close() - require.NoError(b, rwTx.Commit()) - } - - // Build files for the phase-1 range. - require.NoError(b, agg.BuildFiles(uint64(stepsToFlush)*aggStep)) - - // Prune phase-1 entries out of MDBX so they're file-only. - // PruneSmallBatches may report haveMore=true when batch limits stop - // it short of fully draining; loop until drained or we hit a sane - // safety bound. time.Hour keeps it in "furious" prune mode (large - // per-iteration limit), so this is fast in practice. - for round := 0; round < 32; round++ { - rwTx, err := db.BeginTemporalRw(b.Context()) - require.NoError(b, err) - haveMore, err := rwTx.PruneSmallBatches(b.Context(), time.Hour) - require.NoError(b, err) - require.NoError(b, rwTx.Commit()) - if !haveMore { - break - } - } - - // ---- Phase 2: keys destined for MDBX (no BuildFiles after). - phase2Keys := genKeys(phase2Seed, keysetSize, keySize) - - { - rwTx, err := db.BeginTemporalRw(b.Context()) - require.NoError(b, err) - - domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) - require.NoError(b, err) - - val := make([]byte, 8) - for i, k := range phase2Keys { - // Phase-2 keys live well past the built-step boundary so a - // hypothetical follow-up BuildFiles wouldn't sweep them in. - txNum := phase2BaseTxNum + uint64(i) - binary.BigEndian.PutUint64(val, txNum+1) - err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) - require.NoError(b, err) - } - - require.NoError(b, domains.Flush(b.Context(), rwTx)) - domains.Close() - require.NoError(b, rwTx.Commit()) - } - - // ---- Open a RO tx for benching. - tx, err := db.BeginTemporalRo(b.Context()) - require.NoError(b, err) - b.Cleanup(func() { tx.Rollback() }) - - aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) - require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") - - // Partition each phase set by actual residency. We expect: - // phase 1 -> almost all "file", with a tail of tip-step keys still - // in MDBX because Prune retains the most-recent step; - // phase 2 -> all "mdbx" (no BuildFiles called for it). - // The bench uses only the keys that landed where we want, so any - // misroute just shrinks the keyset rather than corrupting timings. - mdbxFromP2, fileFromP2, missingP2 := partitionByResidency(b, aggTx, tx, domain, phase2Keys) - mdbxFromP1, fileFromP1, missingP1 := partitionByResidency(b, aggTx, tx, domain, phase1Keys) - if missingP1 > 0 || missingP2 > 0 { - b.Fatalf("residency partition: missing keys p1=%d p2=%d", missingP1, missingP2) - } - b.Logf("residency: phase1 file=%d mdbx=%d phase2 file=%d mdbx=%d", - len(fileFromP1), len(mdbxFromP1), len(fileFromP2), len(mdbxFromP2)) - - mdbxKeys := mdbxFromP2 // bench's mdbx-resident set - fileKeys := fileFromP1 // bench's file-resident set - require.NotEmpty(b, mdbxKeys, "synthetic fixture produced no MDBX-resident keys") - require.NotEmpty(b, fileKeys, "synthetic fixture produced no file-resident keys") - - return &snapBenchSetup{ - tx: tx, - aggTx: aggTx, - domain: domain, - mdbxKeys: mdbxKeys, - fileKeys: fileKeys, - } -} - -// genKeys produces n deterministic keys of size keySize bytes from -// the given seed. Same seed -> same keys -> reproducible benches. -func genKeys(seed uint64, n, keySize int) [][]byte { - rnd := newRnd(seed) - keys := make([][]byte, n) - for i := range keys { - k := make([]byte, keySize) - _, _ = rnd.Read(k) - keys[i] = k - } - return keys -} - -// partitionByResidency walks keys and groups them by which read path -// returns the value: "mdbx" (DebugGetLatestFromDB ok), "file" -// (DebugGetLatestFromDB miss + DebugGetLatestFromFiles ok), or -// "missing" (neither). missing is reported as a count for the caller -// to fail on — no key in our synthetic setup should be unreachable. -func partitionByResidency(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, keys [][]byte) (mdbxKeys, fileKeys [][]byte, missing int) { - b.Helper() - for _, k := range keys { - _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, k, tx) - require.NoError(b, err) - if dbOk { - mdbxKeys = append(mdbxKeys, k) - continue - } - _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, k, 0) - require.NoError(b, err) - if fileOk { - fileKeys = append(fileKeys, k) - continue - } - missing++ - } - return -} - -// warmup reads every key in keys via tx.GetLatest so the OS page cache -// is hot for all relevant snapshot/MDBX pages before timing starts. -// Required because the H0 claim only holds for warm pages — we want to -// measure software overhead, not disk I/O. -func warmup(tx kv.TemporalTx, domain kv.Domain, keys [][]byte) { - for _, k := range keys { - _, _, _ = tx.GetLatest(domain, k) - } -} - -// skipIfEmpty short-circuits the calling sub-bench when the key set -// is empty (real-datadir mode often has no MDBX-resident keys after -// aggressive pruning). -func skipIfEmpty(b *testing.B, name string, keys [][]byte) bool { - if len(keys) == 0 { - b.Skipf("no keys for %s sub-bench", name) - return true - } - return false -} - -// runH0 is the H0 measurement body. Four sub-benches: -// -// - MDBX_path: tx.GetLatest with key in MDBX. Hits getLatestFromDb -// and returns; never touches files. Baseline cost. -// -// - File_path: tx.GetLatest with key only in files. Misses MDBX, -// then walks getLatestFromFiles (Bloom -> recsplit -> seg -// decompress). Baseline-with-files cost. -// -// - Forced_file_path: same MDBX-resident key set as MDBX_path but -// routed through DebugGetLatestFromFiles, forcing the file path -// even though the key would have hit MDBX. Isolates per-key -// file-path cost from key-distribution effects. -// -// - Forced_db_path: DebugGetLatestFromDB on the MDBX key set — -// mirror of Forced_file_path, isolates the MDBX-only path cost. -// -// Headline: file_ns_per_op / mdbx_ns_per_op. Forced_* sub-benches -// confirm the ratio isn't a key-set artifact. -func runH0(b *testing.B, setup *snapBenchSetup) { - b.Helper() - - // Two warmup passes: first lands pages in the OS page cache, - // second absorbs L3/TLB effects so we time steady-state. - warmup(setup.tx, setup.domain, setup.mdbxKeys) - warmup(setup.tx, setup.domain, setup.fileKeys) - warmup(setup.tx, setup.domain, setup.mdbxKeys) - warmup(setup.tx, setup.domain, setup.fileKeys) - - b.Run("MDBX_path", func(b *testing.B) { - if skipIfEmpty(b, "MDBX_path", setup.mdbxKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetLatest(setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)]) - } - }) - - b.Run("File_path", func(b *testing.B) { - if skipIfEmpty(b, "File_path", setup.fileKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) - } - }) - - // Forced_file_path: file-resident keys via the file-only debug path. - // Strips MDBX-miss cost from File_path so we see pure file-side - // work (Bloom -> recsplit -> seg decompress). - b.Run("Forced_file_path", func(b *testing.B) { - if skipIfEmpty(b, "Forced_file_path", setup.fileKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( - setup.domain, setup.fileKeys[i%len(setup.fileKeys)], 0) - } - }) - - // Forced_db_path: MDBX-resident keys via the DB-only debug path. - // Strips routing/dispatch overhead from MDBX_path so we see pure - // MDBX cursor work. - b.Run("Forced_db_path", func(b *testing.B) { - if skipIfEmpty(b, "Forced_db_path", setup.mdbxKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _, _ = setup.aggTx.DebugGetLatestFromDB( - setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], setup.tx) - } - }) - - // Bloom_miss_path: MDBX-resident keys against the file-only debug - // path. These keys are NOT in any .kv file so each lookup is a - // pure xorfilter "not present" probe per file. Gives the Bloom-miss - // cost in isolation — useful for H1 (per-file Bloom probe overhead). - b.Run("Bloom_miss_path", func(b *testing.B) { - if skipIfEmpty(b, "Bloom_miss_path", setup.mdbxKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( - setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], 0) - } - }) -} - -// BenchmarkSnapVsMDBX_H0_Accounts measures the warm-cache file-vs-MDBX -// per-key gap on the AccountsDomain. Headline: file_ns/op : mdbx_ns/op. -func BenchmarkSnapVsMDBX_H0_Accounts(b *testing.B) { - runH0(b, openSnapBench(b, kv.AccountsDomain)) -} - -// runHGetAsOf measures HistorySeek-via-GetAsOf cost on file-resident -// keys. The motivation (per H0 finding): `getLatestFromFiles` is fast -// (~30 ns isolated); the bloat-workload pprof showed real file-read -// pressure that this can't account for. The calculator's -// HistoryStateReader.Read calls tx.GetAsOf, which goes through -// HistorySeek and walks .ef history files — a different code path -// than getLatestFromFiles. -// -// Sub-benches: -// -// - GetLatest_baseline: tx.GetLatest on file-resident keys; mirrors -// H0's File_path so we can confirm the same setup against a fresh -// comparator. -// -// - GetAsOf_recent: tx.GetAsOf at asOfTxNum = endTxNum - 1. Most -// keys won't have a history record post-asOf, so HistorySeek -// short-circuits to the latest path. Lower bound on GetAsOf cost. -// -// - GetAsOf_mid: tx.GetAsOf at asOfTxNum = endTxNum / 2. Forces -// the .ef walk to seek through more history. This is the cost -// shape the calculator pays when reading historic state. -// -// - GetAsOf_zero: tx.GetAsOf at asOfTxNum = HistoryStartFrom() + 1 -// (just past the visible window start). Maximally adversarial — -// full .ef walk depth. The calculator's old asOfReader.txNum=0 -// bug (PR #21010) hit this path when the window started past 0. -// -// All sub-benches use the same fileKeys set so per-key cost is -// directly comparable across the four. -func runHGetAsOf(b *testing.B, setup *snapBenchSetup) { - b.Helper() - if skipIfEmpty(b, "H_GetAsOf", setup.fileKeys) { - return - } - - // Snapshot the visible-history window so all four sub-benches use - // the same anchors and so we report them in the bench log. We - // don't probe historyStartFrom from here (no public accessor on - // AggregatorRoTx for per-domain history range); instead use simple - // fixed fractions of endTxNum. Adjust asOfFloor=1 to avoid - // hitting txNum=0 which can error on snapshot-loaded chains - // (PR #21010 was that bug; we don't want H_GetAsOf to trip on it). - endTxNum := setup.aggTx.EndTxNumNoCommitment() - asOfRecent := endTxNum - if endTxNum > 0 { - asOfRecent = endTxNum - 1 - } - asOfMid := endTxNum / 2 - asOfFloor := uint64(1) - b.Logf("H_GetAsOf anchors: endTxNum=%d -> asOfRecent=%d asOfMid=%d asOfFloor=%d", - endTxNum, asOfRecent, asOfMid, asOfFloor) - - // Two warmup passes so the OS page cache holds the .ef files we - // care about. Touch each anchor txNum so the relevant history - // segments are mmap'd in. - for _, asOf := range []uint64{asOfRecent, asOfMid, asOfFloor} { - for _, k := range setup.fileKeys { - _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) - } - for _, k := range setup.fileKeys { - _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) - } - } - - b.Run("GetLatest_baseline", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) - } - }) - - b.Run("GetAsOf_recent", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfRecent) - } - }) - - b.Run("GetAsOf_mid", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfMid) - } - }) - - b.Run("GetAsOf_floor", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfFloor) - } - }) -} - -// BenchmarkSnapVsMDBX_HGetAsOf_Accounts measures HistorySeek cost on -// real file-resident keys via tx.GetAsOf. Confirms the H0 finding -// that the production bloat-workload bottleneck is in .ef history -// walking, not .kv latest-state reads. -func BenchmarkSnapVsMDBX_HGetAsOf_Accounts(b *testing.B) { - runHGetAsOf(b, openSnapBench(b, kv.AccountsDomain)) -} - -// BenchmarkSnapVsMDBX_HGetAsOf_Storage — same for StorageDomain. -// Storage tends to dominate in the bloat workload. -func BenchmarkSnapVsMDBX_HGetAsOf_Storage(b *testing.B) { - runHGetAsOf(b, openSnapBench(b, kv.StorageDomain)) -} - -// BenchmarkSnapVsMDBX_H0_Storage — same harness for StorageDomain. -// Storage dominates file-read traffic on bloat workloads (per -// runs-step9-cache-behind-sd memory), so getting the gap for both -// Accounts and Storage is the minimum useful H0 output. -func BenchmarkSnapVsMDBX_H0_Storage(b *testing.B) { - runH0(b, openSnapBench(b, kv.StorageDomain)) -} diff --git a/execution/commitment/hex_concurrent_patricia_hashed.go b/execution/commitment/hex_concurrent_patricia_hashed.go index 7d5424b5df8..bbac1a6d2c3 100644 --- a/execution/commitment/hex_concurrent_patricia_hashed.go +++ b/execution/commitment/hex_concurrent_patricia_hashed.go @@ -165,6 +165,7 @@ func (p *ConcurrentPatriciaHashed) SetTraceDomain(b bool) { p.mounts[i].SetTraceDomain(b) } } + // SetBranchCache attaches the same BranchCache instance to the root trie // and all 16 mounts. Sharing one cache across mounts is correct under the // concurrency contract (branch_cache.go): mounts partition the prefix diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index eedacefab08..060057af560 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2896,8 +2896,9 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log return rootHash, nil } -func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace } -func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } +func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace } +func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } + // SetBranchCache attaches a BranchCache for branch read-through and // write-through. Also propagates to the trie's BranchEncoder so encoder // writes update the cache via mark-dirty-then-Put (see CollectUpdate). From fa4d7f7b5db3bfbdb072f912745181c93afdb94b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 11 May 2026 16:42:49 +0000 Subject: [PATCH 061/167] perf(r3): bulk storage-trunk preload via two-parity range scan (core + tests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-prefix GetLatest BFS with a sequential range scan: a contract H's storage-trunk branches live in two CommitmentDomain key ranges (split by HexToCompact's HP-flag parity), so contractTrunkKeyRanges computes [0x00||H, nextSubtree(0x00||H)) for even-depth branches and [HexToCompact(H_nibbles||0), nextSubtree(HexToCompact(H_nibbles||15))) for odd-depth, and LoadBulk drains both via a CommitmentRangeReader, collects every branch of the subtree, sorts shallowest-first (the BFS/highest-reuse ordering — a lexicographic scan would be depth-first within a subtree, which under a budget pins a deep narrow slice), and pins from the front until the byte budget is hit. Re-scanning on a later LoadBulk extends the pinned set (resumable/phased, like Run). This commit is the algorithm + ContractTrunkPreload.LoadBulk + PreloadContractTrunkBulk + unit tests (range computation, nextSubtree, shallowest-first-within-budget, phased extend). The per-prefix Run/BFS path is left intact; wiring triggerTrunkPreload to provide the real CommitmentRangeReader (aggTx.DebugRangeLatestFromFiles) and call LoadBulk is the next commit. Refs perf-research-tracker.md R3. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/preload.go | 2 + execution/commitment/preload_bulk.go | 195 +++++++++++++++++ execution/commitment/preload_bulk_test.go | 243 ++++++++++++++++++++++ 3 files changed, 440 insertions(+) create mode 100644 execution/commitment/preload_bulk.go create mode 100644 execution/commitment/preload_bulk_test.go diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 88fa7cc2677..73c3b09a0da 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -49,6 +49,7 @@ type pathDepth struct { // Caller MUST NOT use the same instance from multiple goroutines. type ContractTrunkPreload struct { contractHash []byte + contractNibbles []byte // 64 nibbles of contractHash, cached for the bulk range scan (preload_bulk.go) queue []pathDepth pinnedPrefixes [][]byte pinned int @@ -70,6 +71,7 @@ func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) } return &ContractTrunkPreload{ contractHash: contractHash, + contractNibbles: contractNibbles, queue: []pathDepth{{path: contractNibbles, depth: 64}}, maxDepthReached: 64, }, nil diff --git a/execution/commitment/preload_bulk.go b/execution/commitment/preload_bulk.go new file mode 100644 index 00000000000..34d1540e009 --- /dev/null +++ b/execution/commitment/preload_bulk.go @@ -0,0 +1,195 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "bytes" + "fmt" + "sort" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// CommitmentRangeReader yields, in ascending key order, every latest +// CommitmentDomain entry whose key K satisfies fromKey <= K < toKey. yield +// returns false to stop early. step is the domain step the value came from +// (forwarded to BranchCache.PinEntry). Decoupled (callback form) so the +// commitment package doesn't import db/kv/stream. +type CommitmentRangeReader func(fromKey, toKey []byte, yield func(key, value []byte, step uint64) bool) error + +// nextSubtree returns the smallest key K' such that K' is greater than every +// key having `in` as a prefix (i.e. the exclusive upper bound for a +// prefix-range scan over `in`). Returns nil if `in` is all 0xff (no such K'). +// Equivalent to db/kv.NextSubtree; inlined to keep this package's import set +// minimal. +func nextSubtree(in []byte) []byte { + r := make([]byte, len(in)) + copy(r, in) + for i := len(r) - 1; i >= 0; i-- { + if r[i] != 0xff { + r[i]++ + return r[:i+1] + } + } + return nil // all 0xff +} + +// contractTrunkKeyRanges returns the two CommitmentDomain key ranges that +// together hold every branch node of a contract's storage subtree. +// +// The commitment domain keys branches by HexToCompact(nibblePath). A storage +// branch of contract H (32 bytes => 64 nibbles) at slot-path of k nibbles has +// nibblePath = H_nibbles ++ slotNibbles, total 64+k nibbles. HexToCompact's +// HP flag byte differs by parity of the total length, so the branches split +// into two non-adjacent byte ranges: +// +// even total length (k even — incl. the depth-64 subtree-root branch): +// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) +// odd total length (k odd): +// key = (0x10 | H[0]>>4) || <31 bytes derived from H[1:]> || +// || ... +// ⇒ all such keys lie in [HexToCompact(H_nibbles||0), nextSubtree(HexToCompact(H_nibbles||15))). +// +// (HexToCompact of the 64-nibble path = 0x00||H; of a 65-nibble path = the +// 33-byte odd prefix with the final byte's low nibble = the 65th nibble.) +func contractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { + evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes + evenTo = nextSubtree(evenFrom) + + odd0 := make([]byte, 0, 65) + odd0 = append(append(odd0, contractNibbles...), 0) + oddF := make([]byte, 0, 65) + oddF = append(append(oddF, contractNibbles...), 15) + oddFrom = nibbles.HexToCompact(odd0) + oddTo = nextSubtree(nibbles.HexToCompact(oddF)) + return evenFrom, evenTo, oddFrom, oddTo +} + +// maxStorageTrunkDepth is the deepest total nibble path a storage branch can +// have: 64 (account path) + 64 (keccak256(slot)) = 128. Used as a hard sanity +// cap on what the bulk scan considers; the byte budget is the real bound. +const maxStorageTrunkDepth = 128 + +type bulkBranch struct { + key []byte + value []byte + step uint64 + depth int // len(nibblePath) +} + +// LoadBulk pins this contract's storage-trunk branches into cache by scanning +// the two CommitmentDomain key ranges (one sequential file+DB-merged range scan +// per parity, via rr) instead of N per-prefix point lookups. It collects every +// branch of the contract's subtree, sorts them shallowest-first (the +// BFS/highest-reuse-first ordering — a lexicographic scan would yield +// depth-first-within-subtree order, which under a budget pins a deep narrow +// slice), and pins from the front until budgetBytes is reached. +// +// Returns the number of branches pinned by this call (cumulative total via +// PinnedTotal). Calling LoadBulk again with a larger total budget extends the +// pinned set: it re-scans (a sequential range scan is cheap), skips the +// already-pinned shallowest entries, and pins the next budget's worth — the +// resumable/phased semantics of Run, implemented by re-scan. +func (p *ContractTrunkPreload) LoadBulk(budgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (newlyPinned int, err error) { + if cache == nil { + return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: cache is nil") + } + if rr == nil { + return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: range reader is nil") + } + if budgetBytes <= p.usedBytes { + return 0, nil + } + + evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(p.contractNibbles) + + var collected []bulkBranch + collect := func(from, to []byte) error { + return rr(from, to, func(key, value []byte, step uint64) bool { + path := nibbles.CompactToHex(key) + if len(path) < 64 || len(path) > maxStorageTrunkDepth || !bytes.Equal(path[:64], p.contractNibbles) { + return true // not a branch of this contract's subtree — skip, keep scanning + } + kc := make([]byte, len(key)) + copy(kc, key) + vc := make([]byte, len(value)) + copy(vc, value) + collected = append(collected, bulkBranch{key: kc, value: vc, step: step, depth: len(path)}) + return true + }) + } + if err = collect(evenFrom, evenTo); err != nil { + return 0, fmt.Errorf("LoadBulk even-range scan: %w", err) + } + if err = collect(oddFrom, oddTo); err != nil { + return 0, fmt.Errorf("LoadBulk odd-range scan: %w", err) + } + + // Shallowest first; tie-break by path bytes for determinism. + sort.Slice(collected, func(i, j int) bool { + if collected[i].depth != collected[j].depth { + return collected[i].depth < collected[j].depth + } + return bytes.Compare(collected[i].key, collected[j].key) < 0 + }) + + alreadyPinned := p.pinned // skip the entries a previous LoadBulk already pinned (same shallowest-first order) + chunkPinned := 0 + chunkBytes := 0 + for idx, b := range collected { + if idx < alreadyPinned { + continue + } + entryCost := estimatedEntryOverheadBytes + len(b.key) + len(b.value) + if p.usedBytes+chunkBytes+entryCost > budgetBytes { + break + } + cache.PinEntry(b.key, b.value, b.step, "preload-trunk-bulk") + p.pinnedPrefixes = append(p.pinnedPrefixes, b.key) + chunkBytes += entryCost + chunkPinned++ + if b.depth > p.maxDepthReached { + p.maxDepthReached = b.depth + } + if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { + logger.Info("[trunk-preload-bulk] progress", + "pinned", p.pinned+chunkPinned, "depth", b.depth, "used_mb", (p.usedBytes+chunkBytes)/(1<<20)) + } + } + p.pinned += chunkPinned + p.usedBytes += chunkBytes + if logger != nil { + logger.Info("[trunk-preload-bulk] scanned", "contract_hash", fmt.Sprintf("%x", p.contractHash), + "scanned", len(collected), "pinned_this_call", chunkPinned, "pinned_total", p.pinned, + "used_mb", p.usedBytes/(1<<20), "max_depth_reached", p.maxDepthReached, "budget_exhausted", p.pinned < len(collected)) + } + return chunkPinned, nil +} + +// PreloadContractTrunkBulk is the bulk analogue of PreloadContractTrunk: one +// call, bounded by ramBudgetBytes, using a range scan. +func PreloadContractTrunkBulk(contractHash []byte, ramBudgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (int, error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkBulk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreload(contractHash) + if err != nil { + return 0, err + } + return p.LoadBulk(ramBudgetBytes, rr, cache, logger) +} diff --git a/execution/commitment/preload_bulk_test.go b/execution/commitment/preload_bulk_test.go new file mode 100644 index 00000000000..4b0ab650ac4 --- /dev/null +++ b/execution/commitment/preload_bulk_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. + +package commitment + +import ( + "bytes" + "encoding/binary" + "sort" + "testing" + + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +func hexNibbles(b []byte) []byte { + out := make([]byte, len(b)*2) + for i, x := range b { + out[2*i] = x >> 4 + out[2*i+1] = x & 0x0f + } + return out +} + +// keyForBranch builds the CommitmentDomain key for a branch of contract H's +// storage subtree reached by the given slot-path nibbles. +func keyForBranch(contractNibbles, slotPath []byte) []byte { + full := make([]byte, 0, len(contractNibbles)+len(slotPath)) + full = append(full, contractNibbles...) + full = append(full, slotPath...) + return nibbles.HexToCompact(full) +} + +func inRange(k, from, to []byte) bool { + return bytes.Compare(k, from) >= 0 && (to == nil || bytes.Compare(k, to) < 0) +} + +func TestContractTrunkKeyRanges(t *testing.T) { + hashA := make([]byte, 32) + for i := range hashA { + hashA[i] = byte(7*i + 3) // arbitrary + } + hashB := make([]byte, 32) + for i := range hashB { + hashB[i] = byte(251 - 3*i) + } + nibA := hexNibbles(hashA) + nibB := hexNibbles(hashB) + + evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(nibA) + + // Every branch of A at assorted slot-paths must land in exactly one range, + // per the parity of its total nibble length. + slotPaths := [][]byte{ + {}, // depth 64 — subtree root + {0x0}, // 65 + {0xf}, // 65 + {0x1, 0x2}, // 66 + {0xf, 0xf}, // 66 + {0x3, 0x4, 0x5}, // 67 + {0x6, 0x7, 0x8, 0x9}, // 68 + {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 + make([]byte, 64), // 128 — deepest + } + for _, sp := range slotPaths { + k := keyForBranch(nibA, sp) + total := 64 + len(sp) + if total%2 == 0 { + if !inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d branch %x not in even range [%x,%x)", total, k, evenFrom, evenTo) + } + if inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d (even) branch %x unexpectedly in odd range", total, k) + } + } else { + if !inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d branch %x not in odd range [%x,%x)", total, k, oddFrom, oddTo) + } + if inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d (odd) branch %x unexpectedly in even range", total, k) + } + } + // round-trip: CompactToHex(key) must reproduce the full path + got := nibbles.CompactToHex(k) + want := append(append([]byte{}, nibA...), sp...) + if !bytes.Equal(got, want) { + t.Fatalf("CompactToHex round-trip mismatch: got %x want %x", got, want) + } + } + + // A different contract's branches must be in neither of A's ranges. + for _, sp := range slotPaths[:6] { + k := keyForBranch(nibB, sp) + if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) + } + } +} + +func TestNextSubtree(t *testing.T) { + cases := []struct{ in, want []byte }{ + {[]byte{0x01, 0x02}, []byte{0x01, 0x03}}, + {[]byte{0x01, 0xff}, []byte{0x02}}, + {[]byte{0x00}, []byte{0x01}}, + } + for _, c := range cases { + if got := nextSubtree(c.in); !bytes.Equal(got, c.want) { + t.Fatalf("nextSubtree(%x) = %x, want %x", c.in, got, c.want) + } + } + if nextSubtree([]byte{0xff, 0xff}) != nil { + t.Fatalf("nextSubtree(0xffff) should be nil") + } +} + +// branchVal returns a deterministic branch-node value of size sz with a valid +// 4-byte header (touchMap||afterMap) — afterMap=0 so the BFS-children logic +// (in Run, not LoadBulk) would queue nothing; LoadBulk ignores the bitmap. +func branchVal(seed, sz int) []byte { + if sz < 4 { + sz = 4 + } + v := make([]byte, sz) + binary.BigEndian.PutUint16(v[0:2], uint16(seed)) + for i := 4; i < sz; i++ { + v[i] = byte(seed + i) + } + return v +} + +func TestLoadBulk_ShallowestFirstWithinBudget(t *testing.T) { + hash := make([]byte, 32) + for i := range hash { + hash[i] = byte(0x40 + i) + } + nib := hexNibbles(hash) + + // Synthetic branch set at depths 64..69, mixed parity, fixed value size. + const valSz = 100 + type ent struct { + key []byte + val []byte + depth int + } + var ents []ent + add := func(sp []byte) { + ents = append(ents, ent{key: keyForBranch(nib, sp), val: branchVal(len(ents)+1, valSz), depth: 64 + len(sp)}) + } + add([]byte{}) // 64 + add([]byte{0x1}) // 65 + add([]byte{0x2}) // 65 + add([]byte{0x3, 0x4}) // 66 + add([]byte{0x5, 0x6}) // 66 + add([]byte{0x7, 0x8, 0x9}) // 67 + add([]byte{0xa, 0xb, 0xc, 0xd}) // 68 + add([]byte{0xe, 0xf, 0x0, 0x1, 0x2}) // 69 + + // Fake range reader: yields the entries whose key falls in [from,to), in + // ascending key order. + rr := CommitmentRangeReader(func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { + var in []ent + for _, e := range ents { + if bytes.Compare(e.key, from) >= 0 && (to == nil || bytes.Compare(e.key, to) < 0) { + in = append(in, e) + } + } + sort.Slice(in, func(i, j int) bool { return bytes.Compare(in[i].key, in[j].key) < 0 }) + for _, e := range in { + if !yield(e.key, e.val, 1) { + return nil + } + } + return nil + }) + + entryCost := estimatedEntryOverheadBytes + 33 /*≈compact key len*/ + valSz // approx; key lens are 33-36 + // budget for ~the 4 shallowest (depths 64,65,65,66) — give a bit of slack + budget := 4*entryCost + 80 + + c := NewBranchCache(100) + p, err := NewContractTrunkPreload(hash) + if err != nil { + t.Fatal(err) + } + n, err := p.LoadBulk(budget, rr, c, nil) + if err != nil { + t.Fatal(err) + } + if n == 0 || n != p.PinnedTotal() { + t.Fatalf("LoadBulk pinned %d (PinnedTotal=%d), expected >0 and consistent", n, p.PinnedTotal()) + } + if c.PinnedCount() != n { + t.Fatalf("cache PinnedCount=%d != pinned %d", c.PinnedCount(), n) + } + // Sort the synthetic set the way LoadBulk does (shallowest first, key tie-break). + sortedExpected := append([]ent(nil), ents...) + sort.Slice(sortedExpected, func(i, j int) bool { + if sortedExpected[i].depth != sortedExpected[j].depth { + return sortedExpected[i].depth < sortedExpected[j].depth + } + return bytes.Compare(sortedExpected[i].key, sortedExpected[j].key) < 0 + }) + // The first n pinned must be exactly the n shallowest. + for i := 0; i < n; i++ { + v, _, ok := c.Get(sortedExpected[i].key) + if !ok { + t.Fatalf("expected branch #%d (depth %d, key %x) to be pinned, but it's not in cache", i, sortedExpected[i].depth, sortedExpected[i].key) + } + if !bytes.Equal(v, sortedExpected[i].val) { + t.Fatalf("pinned branch #%d value mismatch", i) + } + } + // The (n+1)-th shallowest must NOT be pinned (budget cut it off). + if n < len(ents) { + if _, _, ok := c.Get(sortedExpected[n].key); ok { + t.Fatalf("branch #%d should have been excluded by the budget but is pinned", n) + } + } + if p.MaxDepthReached() != sortedExpected[n-1].depth { + t.Fatalf("MaxDepthReached=%d, expected %d", p.MaxDepthReached(), sortedExpected[n-1].depth) + } + + // Phased extend: a larger total budget pins the rest. + n2, err := p.LoadBulk(1<<20, rr, c, nil) + if err != nil { + t.Fatal(err) + } + if n2 == 0 || p.PinnedTotal() != len(ents) { + t.Fatalf("extend: pinned %d more, total %d, expected total %d", n2, p.PinnedTotal(), len(ents)) + } + if c.PinnedCount() != len(ents) { + t.Fatalf("after extend, cache PinnedCount=%d != %d", c.PinnedCount(), len(ents)) + } +} From b4591a25312e9651397b5866537fbd840d4be275 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 11 May 2026 16:58:26 +0000 Subject: [PATCH 062/167] perf(r3): wire triggerTrunkPreload to the bulk range-scan path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit triggerTrunkPreload's goroutine now builds a CommitmentRangeReader over tx.RangeLatest(CommitmentDomain, from, to) and calls PreloadContractTrunkBulk; PIN_TRUNK_BULK=false reverts to the per-prefix GetLatest BFS (PreloadContractTrunk). step is passed as 0 (the pinned-tier step is a coherency hint, corrected in-place by a later same-prefix write; the bulk scan reads the latest committed value). Refs perf-research-tracker.md R3. Adaptive-controller path (Run-based extend) still uses the per-prefix reader — to be migrated to LoadBulk in a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 34 +++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index da2b9a97610..7b1f7dc8f29 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1368,10 +1368,40 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach } return v, uint64(step), len(v) > 0, nil } + // R3: bulk range scan of the contract's two CommitmentDomain key + // ranges (one sequential file+DB-merged scan per parity) instead of + // the per-prefix GetLatest BFS — ~N random cold seeks become a couple + // of sequential cold reads. step is 0 here: the pinned-tier step is a + // coherency hint (a later same-prefix write at step>0 updates the + // entry in-place), and the bulk scan reads the latest committed value, + // so 0 is conservative-correct. PIN_TRUNK_BULK=false → BFS fallback. + useBulk := dbg.EnvBool("PIN_TRUNK_BULK", true) + rr := func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { + it, err := tx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) + if err != nil { + return err + } + defer it.Close() + for it.HasNext() { + k, v, err := it.Next() + if err != nil { + return err + } + if !yield(k, v, 0) { + return nil + } + } + return nil + } for i, h := range hashes { started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "bulk", useBulk) + var n int + if useBulk { + n, err = commitment.PreloadContractTrunkBulk(h, ramBudgetBytes, rr, branchCache, logger) + } else { + n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + } took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", From ae86359a2cc3e307ca37b22d1bbbdcaaf5efff2f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 11 May 2026 17:08:54 +0000 Subject: [PATCH 063/167] perf(r3): adaptive controller uses LoadBulk (range scan) when available AdaptivePinController.OnBlockComplete now takes both a CommitmentReader (per-prefix BFS) and an optional CommitmentRangeReader (bulk range scan); initial-view promotes and per-block extends use LoadBulk when rr != nil, else fall back to Run. domain_shared.go's Flush-time hook builds rr over tx.Debug().RangeLatest(CommitmentDomain, ...) when PIN_TRUNK_BULK (default true) and passes it. Adds ContractTrunkPreload.HasMore() (BFS queue non-empty OR last bulk scan was budget-limited) so the extend gate works for both paths; lastBulkHadMore tracked in LoadBulk. Test OnBlockComplete calls pass nil for rr (exercise the controller policy on the BFS path; LoadBulk itself is unit-tested separately). Refs perf-research-tracker.md R3. Remaining: bench validation on the SSTORE-bloat fixture (needs perf-devnet-3 datadir re-downloaded). Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 22 +++++++++++++- execution/commitment/adaptive_pin.go | 41 +++++++++++++++++++------- execution/commitment/preload.go | 5 ++++ execution/commitment/preload_bulk.go | 1 + execution/commitment/trunk_pin_test.go | 8 ++--- 5 files changed, 61 insertions(+), 16 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 7b1f7dc8f29..0c53500faed 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -783,7 +783,27 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } return v, uint64(step), len(v) > 0, nil } - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + var rr commitment.CommitmentRangeReader + if dbg.EnvBool("PIN_TRUNK_BULK", true) { + rr = func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { + it, err := ttx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) + if err != nil { + return err + } + defer it.Close() + for it.HasNext() { + k, v, err := it.Next() + if err != nil { + return err + } + if !yield(k, v, 0) { + return nil + } + } + return nil + } + } + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader, rr) } } // Refresh pinned-tier gauges once per Flush — once-per-batch diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 1cd6b6f6066..a0bd2249bf1 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -154,23 +154,42 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { // OnBlockComplete consumes the per-block miss snapshot and decides // promotions, extensions, and demotions. Called by the host at block -// boundaries (after SD.Flush). The reader is the CommitmentReader -// for the just-committed state, used by initial-view preload and -// per-block extension. +// boundaries (after SD.Flush). +// +// If rr (a CommitmentRangeReader) is non-nil, initial-view preloads and +// extensions use the bulk range scan (LoadBulk); otherwise they fall back +// to the per-prefix BFS via reader (a CommitmentReader). The host passes +// rr when PIN_TRUNK_BULK is enabled (default). At least one of the two +// must be non-nil. // // Synchronous: initial-view preloads run inline so the new pin set -// is available for the NEXT block's reads. Extensions also run -// inline; sized so per-block work fits within typical inter-block -// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). +// is available for the NEXT block's reads. Extensions also run inline. // // Logs a [adaptive-pin] line per block when any state changes or // promoted contracts exist. -func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader, rr CommitmentRangeReader) { misses := c.snapshotMisses() c.mu.Lock() defer c.mu.Unlock() + // load runs an initial-view (totalBudget = budget) or extension + // (totalBudget = used + step) preload via the bulk path if available, + // else the per-prefix BFS. + load := func(p *ContractTrunkPreload, totalBudget int) error { + if rr != nil { + _, err := p.LoadBulk(totalBudget, rr, c.cache, c.logger) + return err + } + // BFS Run takes the *additional* budget; map total → additional. + add := totalBudget - p.UsedBytes() + if add <= 0 { + return nil + } + _, _, err := p.Run(add, reader, c.cache, c.logger) + return err + } + var promoted, extended, demoted int // Already-promoted contracts: extend on hot, demote on cold. @@ -179,14 +198,14 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if hadMisses && n > 0 { state.coldBlocksInARow = 0 delete(misses, hash) - // Extend if budget remains and queue not empty. - if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { + // Extend if budget remains and more branches are pending. + if state.preload.HasMore() && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() step := c.cfg.ExtensionBudgetBytes if step > remaining { step = remaining } - if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { + if err := load(state.preload, state.preload.UsedBytes()+step); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -212,7 +231,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if err != nil { continue } - if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + if err := load(p, c.cfg.InitialViewBudgetBytes); err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) // Roll back partial pin so we don't leak entries // for a contract that won't be in c.states. diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 73c3b09a0da..cc449d9ba0a 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -55,6 +55,7 @@ type ContractTrunkPreload struct { pinned int usedBytes int maxDepthReached int + lastBulkHadMore bool // last LoadBulk hit the byte budget with branches left unpinned } // NewContractTrunkPreload constructs a preload state seeded at depth 64 @@ -181,6 +182,10 @@ func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } // MaxDepthReached returns the deepest trie depth pinned so far. func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } +// HasMore reports whether more branches remain to be pinned (BFS queue +// non-empty, or the last bulk scan was cut short by the byte budget). +func (p *ContractTrunkPreload) HasMore() bool { return len(p.queue) > 0 || p.lastBulkHadMore } + // PinnedPrefixes returns the prefix slices this preload has added to // the cache so far. The adaptive controller uses this to Invalidate // the contract's pin set on demotion. The returned slice aliases diff --git a/execution/commitment/preload_bulk.go b/execution/commitment/preload_bulk.go index 34d1540e009..6d4f0927df9 100644 --- a/execution/commitment/preload_bulk.go +++ b/execution/commitment/preload_bulk.go @@ -173,6 +173,7 @@ func (p *ContractTrunkPreload) LoadBulk(budgetBytes int, rr CommitmentRangeReade } p.pinned += chunkPinned p.usedBytes += chunkBytes + p.lastBulkHadMore = p.pinned < len(collected) if logger != nil { logger.Info("[trunk-preload-bulk] scanned", "contract_hash", fmt.Sprintf("%x", p.contractHash), "scanned", len(collected), "pinned_this_call", chunkPinned, "pinned_total", p.pinned, diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index edf8ebac6b0..588d86de649 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -189,7 +189,7 @@ func TestAdaptivePinController_PromoteOnThreshold(t *testing.T) { } // OnBlockComplete should promote the contract. - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 1) require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) require.Greater(t, c.PinnedCount(), 0) @@ -215,17 +215,17 @@ func TestAdaptivePinController_DemoteOnCold(t *testing.T) { // Hot block: triggers promotion. _, _, _ = c.Get(prefix) - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 1) pinnedAfterPromote := c.PinnedCount() require.Greater(t, pinnedAfterPromote, 0) // Cold blocks: no misses for the contract. After cooldown the // controller demotes and invalidates the pin set. - ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 1, "still pinned after 1 cold block") - ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") } From 4ca92847db0e0c1699e7a36e7f5eb3fc81ab750a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 15:22:05 +0000 Subject: [PATCH 064/167] perf(preload): retire the broken R3 range-scan preload The R3 'bulk preload via two-parity range scan' (LoadBulk + Debug().RangeLatest + adaptive wiring) regressed the SSTORE-bloat docker bench 13.2 -> 5.6 mgas/s (below the no-pin baseline). Root cause: the commitment domain .kv files are hashmap-indexed (.kvi, no ordinal lookup) and total 515 GB, so RangeLatest over them degrades to a linear decompress-and-skip from byte 0 -- orders of magnitude slower than the 89k targeted point lookups it replaced; LoadBulk also collects->sorts->pins (no incremental benefit, loses the preload-vs-block race); and the debug raw iterator returns un-dereferenced (shortened-key) branch values. Reverts domain_shared.go / adaptive_pin.go / preload.go / trunk_pin_test.go to the trunk-pin BFS baseline; deletes preload_bulk.go + preload_bulk_test.go. The replacement (two-phase parallel preload: bulk DB range-cursor + parallel file-only wave-BFS) is designed in agentspecs/commitment-branch-preload-redesign.md and lands in follow-up commits. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 56 +---- execution/commitment/adaptive_pin.go | 41 +--- execution/commitment/preload.go | 7 - execution/commitment/preload_bulk.go | 196 ----------------- execution/commitment/preload_bulk_test.go | 243 ---------------------- execution/commitment/trunk_pin_test.go | 8 +- 6 files changed, 18 insertions(+), 533 deletions(-) delete mode 100644 execution/commitment/preload_bulk.go delete mode 100644 execution/commitment/preload_bulk_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 0c53500faed..da2b9a97610 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -783,27 +783,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } return v, uint64(step), len(v) > 0, nil } - var rr commitment.CommitmentRangeReader - if dbg.EnvBool("PIN_TRUNK_BULK", true) { - rr = func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { - it, err := ttx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) - if err != nil { - return err - } - defer it.Close() - for it.HasNext() { - k, v, err := it.Next() - if err != nil { - return err - } - if !yield(k, v, 0) { - return nil - } - } - return nil - } - } - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader, rr) + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) } } // Refresh pinned-tier gauges once per Flush — once-per-batch @@ -1388,40 +1368,10 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach } return v, uint64(step), len(v) > 0, nil } - // R3: bulk range scan of the contract's two CommitmentDomain key - // ranges (one sequential file+DB-merged scan per parity) instead of - // the per-prefix GetLatest BFS — ~N random cold seeks become a couple - // of sequential cold reads. step is 0 here: the pinned-tier step is a - // coherency hint (a later same-prefix write at step>0 updates the - // entry in-place), and the bulk scan reads the latest committed value, - // so 0 is conservative-correct. PIN_TRUNK_BULK=false → BFS fallback. - useBulk := dbg.EnvBool("PIN_TRUNK_BULK", true) - rr := func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { - it, err := tx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) - if err != nil { - return err - } - defer it.Close() - for it.HasNext() { - k, v, err := it.Next() - if err != nil { - return err - } - if !yield(k, v, 0) { - return nil - } - } - return nil - } for i, h := range hashes { started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "bulk", useBulk) - var n int - if useBulk { - n, err = commitment.PreloadContractTrunkBulk(h, ramBudgetBytes, rr, branchCache, logger) - } else { - n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) - } + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) + n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index a0bd2249bf1..1cd6b6f6066 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -154,42 +154,23 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { // OnBlockComplete consumes the per-block miss snapshot and decides // promotions, extensions, and demotions. Called by the host at block -// boundaries (after SD.Flush). -// -// If rr (a CommitmentRangeReader) is non-nil, initial-view preloads and -// extensions use the bulk range scan (LoadBulk); otherwise they fall back -// to the per-prefix BFS via reader (a CommitmentReader). The host passes -// rr when PIN_TRUNK_BULK is enabled (default). At least one of the two -// must be non-nil. +// boundaries (after SD.Flush). The reader is the CommitmentReader +// for the just-committed state, used by initial-view preload and +// per-block extension. // // Synchronous: initial-view preloads run inline so the new pin set -// is available for the NEXT block's reads. Extensions also run inline. +// is available for the NEXT block's reads. Extensions also run +// inline; sized so per-block work fits within typical inter-block +// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). // // Logs a [adaptive-pin] line per block when any state changes or // promoted contracts exist. -func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader, rr CommitmentRangeReader) { +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { misses := c.snapshotMisses() c.mu.Lock() defer c.mu.Unlock() - // load runs an initial-view (totalBudget = budget) or extension - // (totalBudget = used + step) preload via the bulk path if available, - // else the per-prefix BFS. - load := func(p *ContractTrunkPreload, totalBudget int) error { - if rr != nil { - _, err := p.LoadBulk(totalBudget, rr, c.cache, c.logger) - return err - } - // BFS Run takes the *additional* budget; map total → additional. - add := totalBudget - p.UsedBytes() - if add <= 0 { - return nil - } - _, _, err := p.Run(add, reader, c.cache, c.logger) - return err - } - var promoted, extended, demoted int // Already-promoted contracts: extend on hot, demote on cold. @@ -198,14 +179,14 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if hadMisses && n > 0 { state.coldBlocksInARow = 0 delete(misses, hash) - // Extend if budget remains and more branches are pending. - if state.preload.HasMore() && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { + // Extend if budget remains and queue not empty. + if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() step := c.cfg.ExtensionBudgetBytes if step > remaining { step = remaining } - if err := load(state.preload, state.preload.UsedBytes()+step); err != nil { + if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -231,7 +212,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if err != nil { continue } - if err := load(p, c.cfg.InitialViewBudgetBytes); err != nil { + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) // Roll back partial pin so we don't leak entries // for a contract that won't be in c.states. diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index cc449d9ba0a..88fa7cc2677 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -49,13 +49,11 @@ type pathDepth struct { // Caller MUST NOT use the same instance from multiple goroutines. type ContractTrunkPreload struct { contractHash []byte - contractNibbles []byte // 64 nibbles of contractHash, cached for the bulk range scan (preload_bulk.go) queue []pathDepth pinnedPrefixes [][]byte pinned int usedBytes int maxDepthReached int - lastBulkHadMore bool // last LoadBulk hit the byte budget with branches left unpinned } // NewContractTrunkPreload constructs a preload state seeded at depth 64 @@ -72,7 +70,6 @@ func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) } return &ContractTrunkPreload{ contractHash: contractHash, - contractNibbles: contractNibbles, queue: []pathDepth{{path: contractNibbles, depth: 64}}, maxDepthReached: 64, }, nil @@ -182,10 +179,6 @@ func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } // MaxDepthReached returns the deepest trie depth pinned so far. func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } -// HasMore reports whether more branches remain to be pinned (BFS queue -// non-empty, or the last bulk scan was cut short by the byte budget). -func (p *ContractTrunkPreload) HasMore() bool { return len(p.queue) > 0 || p.lastBulkHadMore } - // PinnedPrefixes returns the prefix slices this preload has added to // the cache so far. The adaptive controller uses this to Invalidate // the contract's pin set on demotion. The returned slice aliases diff --git a/execution/commitment/preload_bulk.go b/execution/commitment/preload_bulk.go deleted file mode 100644 index 6d4f0927df9..00000000000 --- a/execution/commitment/preload_bulk.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package commitment - -import ( - "bytes" - "fmt" - "sort" - - "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/commitment/nibbles" -) - -// CommitmentRangeReader yields, in ascending key order, every latest -// CommitmentDomain entry whose key K satisfies fromKey <= K < toKey. yield -// returns false to stop early. step is the domain step the value came from -// (forwarded to BranchCache.PinEntry). Decoupled (callback form) so the -// commitment package doesn't import db/kv/stream. -type CommitmentRangeReader func(fromKey, toKey []byte, yield func(key, value []byte, step uint64) bool) error - -// nextSubtree returns the smallest key K' such that K' is greater than every -// key having `in` as a prefix (i.e. the exclusive upper bound for a -// prefix-range scan over `in`). Returns nil if `in` is all 0xff (no such K'). -// Equivalent to db/kv.NextSubtree; inlined to keep this package's import set -// minimal. -func nextSubtree(in []byte) []byte { - r := make([]byte, len(in)) - copy(r, in) - for i := len(r) - 1; i >= 0; i-- { - if r[i] != 0xff { - r[i]++ - return r[:i+1] - } - } - return nil // all 0xff -} - -// contractTrunkKeyRanges returns the two CommitmentDomain key ranges that -// together hold every branch node of a contract's storage subtree. -// -// The commitment domain keys branches by HexToCompact(nibblePath). A storage -// branch of contract H (32 bytes => 64 nibbles) at slot-path of k nibbles has -// nibblePath = H_nibbles ++ slotNibbles, total 64+k nibbles. HexToCompact's -// HP flag byte differs by parity of the total length, so the branches split -// into two non-adjacent byte ranges: -// -// even total length (k even — incl. the depth-64 subtree-root branch): -// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) -// odd total length (k odd): -// key = (0x10 | H[0]>>4) || <31 bytes derived from H[1:]> || -// || ... -// ⇒ all such keys lie in [HexToCompact(H_nibbles||0), nextSubtree(HexToCompact(H_nibbles||15))). -// -// (HexToCompact of the 64-nibble path = 0x00||H; of a 65-nibble path = the -// 33-byte odd prefix with the final byte's low nibble = the 65th nibble.) -func contractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { - evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes - evenTo = nextSubtree(evenFrom) - - odd0 := make([]byte, 0, 65) - odd0 = append(append(odd0, contractNibbles...), 0) - oddF := make([]byte, 0, 65) - oddF = append(append(oddF, contractNibbles...), 15) - oddFrom = nibbles.HexToCompact(odd0) - oddTo = nextSubtree(nibbles.HexToCompact(oddF)) - return evenFrom, evenTo, oddFrom, oddTo -} - -// maxStorageTrunkDepth is the deepest total nibble path a storage branch can -// have: 64 (account path) + 64 (keccak256(slot)) = 128. Used as a hard sanity -// cap on what the bulk scan considers; the byte budget is the real bound. -const maxStorageTrunkDepth = 128 - -type bulkBranch struct { - key []byte - value []byte - step uint64 - depth int // len(nibblePath) -} - -// LoadBulk pins this contract's storage-trunk branches into cache by scanning -// the two CommitmentDomain key ranges (one sequential file+DB-merged range scan -// per parity, via rr) instead of N per-prefix point lookups. It collects every -// branch of the contract's subtree, sorts them shallowest-first (the -// BFS/highest-reuse-first ordering — a lexicographic scan would yield -// depth-first-within-subtree order, which under a budget pins a deep narrow -// slice), and pins from the front until budgetBytes is reached. -// -// Returns the number of branches pinned by this call (cumulative total via -// PinnedTotal). Calling LoadBulk again with a larger total budget extends the -// pinned set: it re-scans (a sequential range scan is cheap), skips the -// already-pinned shallowest entries, and pins the next budget's worth — the -// resumable/phased semantics of Run, implemented by re-scan. -func (p *ContractTrunkPreload) LoadBulk(budgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (newlyPinned int, err error) { - if cache == nil { - return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: cache is nil") - } - if rr == nil { - return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: range reader is nil") - } - if budgetBytes <= p.usedBytes { - return 0, nil - } - - evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(p.contractNibbles) - - var collected []bulkBranch - collect := func(from, to []byte) error { - return rr(from, to, func(key, value []byte, step uint64) bool { - path := nibbles.CompactToHex(key) - if len(path) < 64 || len(path) > maxStorageTrunkDepth || !bytes.Equal(path[:64], p.contractNibbles) { - return true // not a branch of this contract's subtree — skip, keep scanning - } - kc := make([]byte, len(key)) - copy(kc, key) - vc := make([]byte, len(value)) - copy(vc, value) - collected = append(collected, bulkBranch{key: kc, value: vc, step: step, depth: len(path)}) - return true - }) - } - if err = collect(evenFrom, evenTo); err != nil { - return 0, fmt.Errorf("LoadBulk even-range scan: %w", err) - } - if err = collect(oddFrom, oddTo); err != nil { - return 0, fmt.Errorf("LoadBulk odd-range scan: %w", err) - } - - // Shallowest first; tie-break by path bytes for determinism. - sort.Slice(collected, func(i, j int) bool { - if collected[i].depth != collected[j].depth { - return collected[i].depth < collected[j].depth - } - return bytes.Compare(collected[i].key, collected[j].key) < 0 - }) - - alreadyPinned := p.pinned // skip the entries a previous LoadBulk already pinned (same shallowest-first order) - chunkPinned := 0 - chunkBytes := 0 - for idx, b := range collected { - if idx < alreadyPinned { - continue - } - entryCost := estimatedEntryOverheadBytes + len(b.key) + len(b.value) - if p.usedBytes+chunkBytes+entryCost > budgetBytes { - break - } - cache.PinEntry(b.key, b.value, b.step, "preload-trunk-bulk") - p.pinnedPrefixes = append(p.pinnedPrefixes, b.key) - chunkBytes += entryCost - chunkPinned++ - if b.depth > p.maxDepthReached { - p.maxDepthReached = b.depth - } - if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { - logger.Info("[trunk-preload-bulk] progress", - "pinned", p.pinned+chunkPinned, "depth", b.depth, "used_mb", (p.usedBytes+chunkBytes)/(1<<20)) - } - } - p.pinned += chunkPinned - p.usedBytes += chunkBytes - p.lastBulkHadMore = p.pinned < len(collected) - if logger != nil { - logger.Info("[trunk-preload-bulk] scanned", "contract_hash", fmt.Sprintf("%x", p.contractHash), - "scanned", len(collected), "pinned_this_call", chunkPinned, "pinned_total", p.pinned, - "used_mb", p.usedBytes/(1<<20), "max_depth_reached", p.maxDepthReached, "budget_exhausted", p.pinned < len(collected)) - } - return chunkPinned, nil -} - -// PreloadContractTrunkBulk is the bulk analogue of PreloadContractTrunk: one -// call, bounded by ramBudgetBytes, using a range scan. -func PreloadContractTrunkBulk(contractHash []byte, ramBudgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (int, error) { - if ramBudgetBytes <= 0 { - return 0, fmt.Errorf("PreloadContractTrunkBulk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) - } - p, err := NewContractTrunkPreload(contractHash) - if err != nil { - return 0, err - } - return p.LoadBulk(ramBudgetBytes, rr, cache, logger) -} diff --git a/execution/commitment/preload_bulk_test.go b/execution/commitment/preload_bulk_test.go deleted file mode 100644 index 4b0ab650ac4..00000000000 --- a/execution/commitment/preload_bulk_test.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. - -package commitment - -import ( - "bytes" - "encoding/binary" - "sort" - "testing" - - "github.com/erigontech/erigon/execution/commitment/nibbles" -) - -func hexNibbles(b []byte) []byte { - out := make([]byte, len(b)*2) - for i, x := range b { - out[2*i] = x >> 4 - out[2*i+1] = x & 0x0f - } - return out -} - -// keyForBranch builds the CommitmentDomain key for a branch of contract H's -// storage subtree reached by the given slot-path nibbles. -func keyForBranch(contractNibbles, slotPath []byte) []byte { - full := make([]byte, 0, len(contractNibbles)+len(slotPath)) - full = append(full, contractNibbles...) - full = append(full, slotPath...) - return nibbles.HexToCompact(full) -} - -func inRange(k, from, to []byte) bool { - return bytes.Compare(k, from) >= 0 && (to == nil || bytes.Compare(k, to) < 0) -} - -func TestContractTrunkKeyRanges(t *testing.T) { - hashA := make([]byte, 32) - for i := range hashA { - hashA[i] = byte(7*i + 3) // arbitrary - } - hashB := make([]byte, 32) - for i := range hashB { - hashB[i] = byte(251 - 3*i) - } - nibA := hexNibbles(hashA) - nibB := hexNibbles(hashB) - - evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(nibA) - - // Every branch of A at assorted slot-paths must land in exactly one range, - // per the parity of its total nibble length. - slotPaths := [][]byte{ - {}, // depth 64 — subtree root - {0x0}, // 65 - {0xf}, // 65 - {0x1, 0x2}, // 66 - {0xf, 0xf}, // 66 - {0x3, 0x4, 0x5}, // 67 - {0x6, 0x7, 0x8, 0x9}, // 68 - {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 - make([]byte, 64), // 128 — deepest - } - for _, sp := range slotPaths { - k := keyForBranch(nibA, sp) - total := 64 + len(sp) - if total%2 == 0 { - if !inRange(k, evenFrom, evenTo) { - t.Fatalf("depth %d branch %x not in even range [%x,%x)", total, k, evenFrom, evenTo) - } - if inRange(k, oddFrom, oddTo) { - t.Fatalf("depth %d (even) branch %x unexpectedly in odd range", total, k) - } - } else { - if !inRange(k, oddFrom, oddTo) { - t.Fatalf("depth %d branch %x not in odd range [%x,%x)", total, k, oddFrom, oddTo) - } - if inRange(k, evenFrom, evenTo) { - t.Fatalf("depth %d (odd) branch %x unexpectedly in even range", total, k) - } - } - // round-trip: CompactToHex(key) must reproduce the full path - got := nibbles.CompactToHex(k) - want := append(append([]byte{}, nibA...), sp...) - if !bytes.Equal(got, want) { - t.Fatalf("CompactToHex round-trip mismatch: got %x want %x", got, want) - } - } - - // A different contract's branches must be in neither of A's ranges. - for _, sp := range slotPaths[:6] { - k := keyForBranch(nibB, sp) - if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { - t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) - } - } -} - -func TestNextSubtree(t *testing.T) { - cases := []struct{ in, want []byte }{ - {[]byte{0x01, 0x02}, []byte{0x01, 0x03}}, - {[]byte{0x01, 0xff}, []byte{0x02}}, - {[]byte{0x00}, []byte{0x01}}, - } - for _, c := range cases { - if got := nextSubtree(c.in); !bytes.Equal(got, c.want) { - t.Fatalf("nextSubtree(%x) = %x, want %x", c.in, got, c.want) - } - } - if nextSubtree([]byte{0xff, 0xff}) != nil { - t.Fatalf("nextSubtree(0xffff) should be nil") - } -} - -// branchVal returns a deterministic branch-node value of size sz with a valid -// 4-byte header (touchMap||afterMap) — afterMap=0 so the BFS-children logic -// (in Run, not LoadBulk) would queue nothing; LoadBulk ignores the bitmap. -func branchVal(seed, sz int) []byte { - if sz < 4 { - sz = 4 - } - v := make([]byte, sz) - binary.BigEndian.PutUint16(v[0:2], uint16(seed)) - for i := 4; i < sz; i++ { - v[i] = byte(seed + i) - } - return v -} - -func TestLoadBulk_ShallowestFirstWithinBudget(t *testing.T) { - hash := make([]byte, 32) - for i := range hash { - hash[i] = byte(0x40 + i) - } - nib := hexNibbles(hash) - - // Synthetic branch set at depths 64..69, mixed parity, fixed value size. - const valSz = 100 - type ent struct { - key []byte - val []byte - depth int - } - var ents []ent - add := func(sp []byte) { - ents = append(ents, ent{key: keyForBranch(nib, sp), val: branchVal(len(ents)+1, valSz), depth: 64 + len(sp)}) - } - add([]byte{}) // 64 - add([]byte{0x1}) // 65 - add([]byte{0x2}) // 65 - add([]byte{0x3, 0x4}) // 66 - add([]byte{0x5, 0x6}) // 66 - add([]byte{0x7, 0x8, 0x9}) // 67 - add([]byte{0xa, 0xb, 0xc, 0xd}) // 68 - add([]byte{0xe, 0xf, 0x0, 0x1, 0x2}) // 69 - - // Fake range reader: yields the entries whose key falls in [from,to), in - // ascending key order. - rr := CommitmentRangeReader(func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { - var in []ent - for _, e := range ents { - if bytes.Compare(e.key, from) >= 0 && (to == nil || bytes.Compare(e.key, to) < 0) { - in = append(in, e) - } - } - sort.Slice(in, func(i, j int) bool { return bytes.Compare(in[i].key, in[j].key) < 0 }) - for _, e := range in { - if !yield(e.key, e.val, 1) { - return nil - } - } - return nil - }) - - entryCost := estimatedEntryOverheadBytes + 33 /*≈compact key len*/ + valSz // approx; key lens are 33-36 - // budget for ~the 4 shallowest (depths 64,65,65,66) — give a bit of slack - budget := 4*entryCost + 80 - - c := NewBranchCache(100) - p, err := NewContractTrunkPreload(hash) - if err != nil { - t.Fatal(err) - } - n, err := p.LoadBulk(budget, rr, c, nil) - if err != nil { - t.Fatal(err) - } - if n == 0 || n != p.PinnedTotal() { - t.Fatalf("LoadBulk pinned %d (PinnedTotal=%d), expected >0 and consistent", n, p.PinnedTotal()) - } - if c.PinnedCount() != n { - t.Fatalf("cache PinnedCount=%d != pinned %d", c.PinnedCount(), n) - } - // Sort the synthetic set the way LoadBulk does (shallowest first, key tie-break). - sortedExpected := append([]ent(nil), ents...) - sort.Slice(sortedExpected, func(i, j int) bool { - if sortedExpected[i].depth != sortedExpected[j].depth { - return sortedExpected[i].depth < sortedExpected[j].depth - } - return bytes.Compare(sortedExpected[i].key, sortedExpected[j].key) < 0 - }) - // The first n pinned must be exactly the n shallowest. - for i := 0; i < n; i++ { - v, _, ok := c.Get(sortedExpected[i].key) - if !ok { - t.Fatalf("expected branch #%d (depth %d, key %x) to be pinned, but it's not in cache", i, sortedExpected[i].depth, sortedExpected[i].key) - } - if !bytes.Equal(v, sortedExpected[i].val) { - t.Fatalf("pinned branch #%d value mismatch", i) - } - } - // The (n+1)-th shallowest must NOT be pinned (budget cut it off). - if n < len(ents) { - if _, _, ok := c.Get(sortedExpected[n].key); ok { - t.Fatalf("branch #%d should have been excluded by the budget but is pinned", n) - } - } - if p.MaxDepthReached() != sortedExpected[n-1].depth { - t.Fatalf("MaxDepthReached=%d, expected %d", p.MaxDepthReached(), sortedExpected[n-1].depth) - } - - // Phased extend: a larger total budget pins the rest. - n2, err := p.LoadBulk(1<<20, rr, c, nil) - if err != nil { - t.Fatal(err) - } - if n2 == 0 || p.PinnedTotal() != len(ents) { - t.Fatalf("extend: pinned %d more, total %d, expected total %d", n2, p.PinnedTotal(), len(ents)) - } - if c.PinnedCount() != len(ents) { - t.Fatalf("after extend, cache PinnedCount=%d != %d", c.PinnedCount(), len(ents)) - } -} diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index 588d86de649..edf8ebac6b0 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -189,7 +189,7 @@ func TestAdaptivePinController_PromoteOnThreshold(t *testing.T) { } // OnBlockComplete should promote the contract. - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 1) require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) require.Greater(t, c.PinnedCount(), 0) @@ -215,17 +215,17 @@ func TestAdaptivePinController_DemoteOnCold(t *testing.T) { // Hot block: triggers promotion. _, _, _ = c.Get(prefix) - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 1) pinnedAfterPromote := c.PinnedCount() require.Greater(t, pinnedAfterPromote, 0) // Cold blocks: no misses for the contract. After cooldown the // controller demotes and invalidates the pin set. - ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 1, "still pinned after 1 cold block") - ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") } From 3e0154f245b3329e0299b8a3ec64ac414484df6e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 15:57:51 +0000 Subject: [PATCH 065/167] perf(preload): parallel file-only wave-BFS preload (cut-down) Replaces the per-prefix serial BFS-via-GetLatest preload (which crossed the MDBX boundary ~89k times per contract and finished in ~31s, losing the preload-vs-test-block race in the no-warmup harness) with a parallel, breadth-first, file-layer-only wave walk: PreloadContractTrunkParallel walks the contract subtree one depth level (wave) at a time; each wave's branches are resolved via a single batched resolver (BatchBranchResolver) that the caller fans out across worker RoTxs in contiguous, key-sorted slices -> locally sequential reads that drive page-cache readahead. Reads go via DebugGetLatestFromFiles (file layer only: no MDBX cursor, no per-key cgo crossing; values already dereferenced via replaceShortenedKeysInBranch). Breadth-first so a budget-truncated run still pins the shallowest, highest-reuse branches. triggerTrunkPreload wires it (PIN_TRUNK_PARALLEL, default on; =false falls back to the per-prefix BFS): N=GOMAXPROCS worker RoTxs (each goroutine its own tx -- MDBX cursors aren't concurrent-safe under one tx), contiguous-slice fanout. File-layer-only is correct for the from-cold-snapshot trigger path (MDBX holds nothing for the subtree). The MDBX-resident set (steady-state) is handled by a separate range-cursor prefetch -- see agentspecs/commitment-branch-preload-redesign.md; not in this cut-down. Unit tests: PreloadParallel_{FullBudget_BreadthFirst, BudgetCutoff, NotFoundStopsDescent, ResolverError}. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 107 +++++++- execution/commitment/preload_parallel.go | 166 ++++++++++++ execution/commitment/preload_parallel_test.go | 250 ++++++++++++++++++ 3 files changed, 510 insertions(+), 13 deletions(-) create mode 100644 execution/commitment/preload_parallel.go create mode 100644 execution/commitment/preload_parallel_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index da2b9a97610..8f3e1da3860 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1350,28 +1350,109 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach return } go func() { - tx, err := db.BeginTemporalRo(ctx) - if err != nil { - logger.Warn("[trunk-preload] BeginTemporalRo failed", "err", err) - return + // PIN_TRUNK_PARALLEL (default on): the parallel file-only wave-BFS + // (PreloadContractTrunkParallel). =false falls back to the per-prefix + // BFS via tx.GetLatest. Worker txs: each goroutine MUST use its own + // tx — MDBX cursors aren't concurrent-safe under one tx (cgo "unpinned + // Go pointer" panic). Reads here go via the file layer only + // (DebugGetLatestFromFiles): no MDBX cursor, no per-key cgo crossing, + // and values are already dereferenced. Correct for the from-cold- + // snapshot trigger path (MDBX holds nothing for the subtree). + parallel := dbg.EnvBool("PIN_TRUNK_PARALLEL", true) + nWorkers := runtime.GOMAXPROCS(0) + if nWorkers < 1 { + nWorkers = 1 + } + if nWorkers > 16 { + nWorkers = 16 + } + if !parallel { + nWorkers = 1 } - defer tx.Rollback() - // Read directly via tx.GetLatest — no SharedDomains wrap. - // SD's GetLatest layers in sd.mem / parent.mem checks that - // (a) we don't need (the trunk hasn't been written yet in this - // goroutine's tx), and (b) would risk re-introducing the - // shared-cursor problem. + txs := make([]kv.TemporalTx, 0, nWorkers) + for i := 0; i < nWorkers; i++ { + t, err := db.BeginTemporalRo(ctx) //nolint:gocritic // collected into txs; all rolled back via the defer below + if err != nil { + for _, t := range txs { + t.Rollback() + } + logger.Warn("[trunk-preload] BeginTemporalRo failed", "worker", i, "err", err) + return + } + txs = append(txs, t) + } + defer func() { + for _, t := range txs { + t.Rollback() + } + }() + reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := tx.GetLatest(kv.CommitmentDomain, prefix) + v, step, err := txs[0].GetLatest(kv.CommitmentDomain, prefix) if err != nil { return nil, 0, false, err } return v, uint64(step), len(v) > 0, nil } + + resolve := func(keys [][]byte) ([][]byte, error) { + n := len(keys) + vals := make([][]byte, n) + if n == 0 { + return vals, nil + } + nw := nWorkers + if nw > n { + nw = n + } + chunk := (n + nw - 1) / nw + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + for w := 0; w < nw; w++ { + lo, hi := w*chunk, (w+1)*chunk + if hi > n { + hi = n + } + if lo >= hi { + break + } + wg.Add(1) + go func(t kv.TemporalTx, lo, hi int) { + defer wg.Done() + d := t.Debug() + for i := lo; i < hi; i++ { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, keys[i], 0) + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + return + } + if found { + vc := make([]byte, len(v)) + copy(vc, v) + vals[i] = vc + } + } + }(txs[w], lo, hi) + } + wg.Wait() + return vals, firstErr + } + for i, h := range hashes { started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "parallel", parallel, "workers", nWorkers) + var n int + var err error + if parallel { + n, err = commitment.PreloadContractTrunkParallel(h, ramBudgetBytes, resolve, branchCache, logger) + } else { + n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + } took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go new file mode 100644 index 00000000000..1a88c6fd304 --- /dev/null +++ b/execution/commitment/preload_parallel.go @@ -0,0 +1,166 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// BatchBranchResolver resolves a batch of compact-encoded CommitmentDomain +// keys to their (already-dereferenced) branch-node values, reading from the +// file layer only — no MDBX cursor, hence no per-key cgo boundary crossing. +// The implementation is expected to fan the batch out across worker RoTxs; +// keys are passed in ascending key order so a contiguous-slice partition maps +// to contiguous file regions (drives page-cache readahead). Results MUST be +// returned in the same order as keys, with vals[i] == nil for keys not present +// in the file layer. +// +// Decoupled (callback form) so the commitment package needn't import db/state. +type BatchBranchResolver func(keys [][]byte) (vals [][]byte, err error) + +// PreloadContractTrunkParallel pins contract H's storage-trunk branches into +// cache by walking the subtree breadth-first, one depth level ("wave") at a +// time, resolving each wave's branches through a single batched, parallelisable +// file-only read. Breadth-first so a budget-truncated run still pins the +// shallowest (highest-reuse) branches first; wave-batched so the resolver can +// sort+partition the reads for locality instead of issuing them one at a time. +// +// Bounded by ramBudgetBytes (same accounting as the BFS ContractTrunkPreload). +// Returns the number of branches pinned. +// +// File-layer-only: a branch present only in MDBX (recently written, not yet in +// a .kv file) is invisible here, and a branch present in both MDBX (fresh) and +// files (stale) resolves to the stale value. That is correct for the +// from-cold-snapshot trigger path (MDBX holds nothing for the subtree) — the +// case PIN_CONTRACT_TRUNKS targets. The MDBX-resident set is handled separately +// (a single range-cursor prefetch); see agentspecs/commitment-branch-preload-redesign.md. +func PreloadContractTrunkParallel( + contractHash []byte, + ramBudgetBytes int, + resolve BatchBranchResolver, + cache *BranchCache, + logger log.Logger, +) (pinned int, err error) { + if cache == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: cache is nil") + } + if resolve == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + } + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + if len(contractHash) != 32 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: contractHash must be 32 bytes, got %d", len(contractHash)) + } + + contractNibbles := make([]byte, 64) + for i, b := range contractHash { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + + type pathKey struct { + path []byte // nibble path (1 byte / nibble) + key []byte // HexToCompact(path) — the CommitmentDomain key + } + toPathKey := func(path []byte) pathKey { + k := nibbles.HexToCompact(path) + // HexToCompact may return a slice over a reused buffer. + kc := make([]byte, len(k)) + copy(kc, k) + return pathKey{path: path, key: kc} + } + + frontier := []pathKey{toPathKey(contractNibbles)} + usedBytes := 0 + maxDepthReached := 64 + budgetHit := false + + for depth := 64; depth <= maxStorageTrunkDepth && len(frontier) > 0 && !budgetHit; depth++ { + // Ascending key order so the resolver's contiguous partition is + // contiguous-in-file. BFS append order is already key-sorted (the + // HexToCompact packing is monotone for same-length paths), but sort + // defensively — it's a few k entries / wave, cheap. + sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) + + keys := make([][]byte, len(frontier)) + for i := range frontier { + keys[i] = frontier[i].key + } + vals, rerr := resolve(keys) + if rerr != nil { + return pinned, fmt.Errorf("preload at depth %d: %w", depth, rerr) + } + if len(vals) != len(keys) { + return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(vals), len(keys)) + } + + var next []pathKey + for i, pk := range frontier { + v := vals[i] + if v == nil { + continue // not in the file layer + } + entryCost := estimatedEntryOverheadBytes + len(pk.key) + len(v) + if usedBytes+entryCost > ramBudgetBytes { + budgetHit = true + break + } + cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") + usedBytes += entryCost + pinned++ + if depth > maxDepthReached { + maxDepthReached = depth + } + if logger != nil && pinned%5000 == 0 { + logger.Info("[trunk-preload-parallel] progress", + "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) + } + // Decode the branch header (2-byte touchMap || 2-byte afterMap || + // per-child data) and queue the children that exist in the trie. + if len(v) < 4 { + continue + } + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1<> 4 + out[2*i+1] = x & 0x0f + } + return out +} + +// branchVal builds a synthetic branch-node value: 2-byte touchMap (0) || +// 2-byte afterMap (the child bitmap) || zero-padding to size sz (>= 4). +func branchVal(afterMap uint16, sz int) []byte { + if sz < 4 { + sz = 4 + } + v := make([]byte, sz) + binary.BigEndian.PutUint16(v[2:4], afterMap) + return v +} + +// syntheticTree describes a contract storage subtree: path-string -> afterMap. +// Root is the 64-nibble path of the contract hash; a node R is present iff R is +// a key here, and R's children are R||n for each set bit n in afterMap[R]. +type syntheticTree map[string]uint16 + +func buildSyntheticTree(t *testing.T) (hash []byte, tree syntheticTree, allPaths [][]byte) { + t.Helper() + hash = make([]byte, 32) + for i := range hash { + hash[i] = 0x42 + } + root := string(hexNibbles(hash)) + // R(64) -> {1,2} ; R1(65) -> {3} ; R2(65) -> {4,5} ; + // R1.3(66) leaf ; R2.4(66) leaf ; R2.5(66) -> {6} ; R2.5.6(67) leaf. + r := []byte(root) + p := func(suffix ...byte) []byte { return append(append([]byte{}, r...), suffix...) } + tree = syntheticTree{ + string(p()): 0b110, // bits 1,2 + string(p(1)): 0b1000, // bit 3 + string(p(2)): 0b110000, // bits 4,5 + string(p(1, 3)): 0, + string(p(2, 4)): 0, + string(p(2, 5)): 0b1000000, // bit 6 + string(p(2, 5, 6)): 0, + } + for k := range tree { + allPaths = append(allPaths, []byte(k)) + } + return hash, tree, allPaths +} + +// fakeResolver returns a BatchBranchResolver backed by the synthetic tree. +// notFound (path-strings) are treated as absent from the file layer. +// valSz is the branch value size. If failOnKey is non-empty, the resolver +// returns an error when that key is requested. +func fakeResolver(tree syntheticTree, notFound map[string]bool, valSz int, failOnPath string) BatchBranchResolver { + return func(keys [][]byte) ([][]byte, error) { + // keys must be sorted ascending (the contract of BatchBranchResolver). + for i := 1; i < len(keys); i++ { + if bytes.Compare(keys[i-1], keys[i]) >= 0 { + return nil, errors.New("resolver got unsorted keys") + } + } + vals := make([][]byte, len(keys)) + for i, k := range keys { + path := string(nibbles.CompactToHex(k)) + if failOnPath != "" && path == failOnPath { + return nil, errors.New("synthetic resolver failure") + } + am, ok := tree[path] + if !ok || notFound[path] { + continue // nil + } + vals[i] = branchVal(am, valSz) + } + return vals, nil + } +} + +// breadthFirstOrder returns the synthetic tree's paths in the order +// PreloadContractTrunkParallel pins them: by depth, then by compact-key. +func breadthFirstOrder(tree syntheticTree, exclude map[string]bool) []string { + type pk struct { + path string + key []byte + } + var pks []pk + // reachability from root, honoring exclude (an excluded node stops descent — + // it and its subtree become unreachable) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + reach := map[string]bool{} + var dfs func(p string) + dfs = func(p string) { + if exclude[p] { + return + } + am, ok := tree[p] + if !ok { + return + } + reach[p] = true + for n := 0; n < 16; n++ { + if am&(1< root pinned at depth 64, then error. + _, err := PreloadContractTrunkParallel(hash, 1<<20, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) + if err == nil { + t.Fatal("expected error from the resolver") + } + if c.PinnedCount() == 0 { + t.Fatal("the depth-64 root should have been pinned before the depth-65 failure") + } +} From 4c9968a4370c86d75b3149e1d4adeba0a7904c3b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 22:19:33 +0000 Subject: [PATCH 066/167] perf(preload): cap per-wave fetch by remaining budget PreloadContractTrunkParallel resolved an entire depth wave before applying the RAM budget. Depth 69 of the SSTORE-bloat trunk is ~1.3M keys but only ~20k fit the remaining budget -- so it issued >1M file lookups to pin ~20k, and that over-fetch ran concurrently with the test block (docker bench: 10.1 mgas/s, preload 38.5s vs the BFS baseline's 13.2 / 31s -- depths 64-68 pinned in ~1-2s, then 31s stalled on the over-fetched depth-69 wave). Now: each wave's fetch is capped at remaining_budget/minEntryBytes + 1, where minEntryBytes (= overhead + shortest depth>=64 compact key) is a true lower bound on a pinned entry's cost -- so it never under-fetches and the budget is guaranteed exhausted within the (truncated) wave; if a wave is truncated we stop after it. Depth 69 goes from ~1.3M lookups -> ~tens of thousands -> ~seconds, the preload finishes well before the block, and the over-fetch contention is gone. Adds TestPreloadParallel_CapsWaveFetch. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/preload_parallel.go | 18 +++++++++ execution/commitment/preload_parallel_test.go | 39 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 1a88c6fd304..057a22fdfcd 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -96,6 +96,21 @@ func PreloadContractTrunkParallel( // defensively — it's a few k entries / wave, cheap. sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) + // Cap the wave's *fetch* at what the remaining budget can possibly + // absorb (minEntryBytes is a true lower bound on a pinned entry's cost, + // so this never under-fetches and the budget is guaranteed exhausted + // inside the wave). Without it, a wide budget-truncated wave — depth 69 + // is ~1.3M keys but only ~20k fit — would issue >1M file lookups to pin + // a tiny fraction, and that over-fetch races the next block. + const minEntryBytes = estimatedEntryOverheadBytes + 33 // 33 = shortest compact key at depth >= 64; value >= 0 bytes + truncatedByBudget := false + if remaining := ramBudgetBytes - usedBytes; remaining > 0 { + if maxFetch := remaining/minEntryBytes + 1; maxFetch < len(frontier) { + frontier = frontier[:maxFetch] + truncatedByBudget = true + } + } + keys := make([][]byte, len(frontier)) for i := range frontier { keys[i] = frontier[i].key @@ -145,6 +160,9 @@ func PreloadContractTrunkParallel( next = append(next, toPathKey(childPath)) } } + if truncatedByBudget { + budgetHit = true // belt-and-braces: the wave was sized to exhaust the budget + } frontier = next } diff --git a/execution/commitment/preload_parallel_test.go b/execution/commitment/preload_parallel_test.go index 0d804f616ea..11a44acd445 100644 --- a/execution/commitment/preload_parallel_test.go +++ b/execution/commitment/preload_parallel_test.go @@ -230,6 +230,45 @@ func TestPreloadParallel_NotFoundStopsDescent(t *testing.T) { } } +func TestPreloadParallel_CapsWaveFetch(t *testing.T) { + // A wide wave (root with all 16 children) under a tiny budget: the depth-65 + // fetch must be capped well below the wave width — otherwise a budget- + // truncated wide wave (depth 69 on the real workload is ~1.3M keys) would + // fetch the whole thing to pin a handful. + hash := make([]byte, 32) + for i := range hash { + hash[i] = 0x55 + } + root := string(hexNibbles(hash)) + tree := syntheticTree{root: 0xffff} + for n := 0; n < 16; n++ { + tree[root+string([]byte{byte(n)})] = 0 // 16 depth-65 leaves + } + const valSz = 100 + entry := estimatedEntryOverheadBytes + len(nibbles.HexToCompact([]byte(root))) + valSz + budget := 3*entry + 50 // ~3 entries + + maxBatch := 0 + base := fakeResolver(tree, nil, valSz, "") + resolve := func(keys [][]byte) ([][]byte, error) { + if len(keys) > maxBatch { + maxBatch = len(keys) + } + return base(keys) + } + c := NewBranchCache(64) + n, err := PreloadContractTrunkParallel(hash, budget, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if n < 1 || n > 3 { + t.Fatalf("pinned %d, expected 1..3 for a ~3-entry budget", n) + } + if maxBatch > 6 { + t.Fatalf("depth-65 wave (width 16) should have been capped to ~remaining/minEntryBytes; resolver saw a batch of %d", maxBatch) + } +} + func TestPreloadParallel_ResolverError(t *testing.T) { hash, tree, _ := buildSyntheticTree(t) root := "" From eb0c0c5803d0c388bfcd46aa25f5f7a622073fff Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 22:47:40 +0000 Subject: [PATCH 067/167] =?UTF-8?q?perf(preload):=20phase=201=20=E2=80=94?= =?UTF-8?q?=20MDBX-resident=20branch=20prefetch=20(operational=20correctne?= =?UTF-8?q?ss)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parallel wave-BFS preload was file-layer-only: a branch present in MDBX but not yet flushed to a .kv file is invisible, and one present in both (DB fresh, file stale) resolved to the stale value. Correct from a cold snapshot (MDBX empty for the subtree), wrong on a live node. Phase 1: triggerTrunkPreload now first scans the contract's two CommitmentDomain key ranges (ContractTrunkKeyRanges, re-added with NextSubtree from the deleted preload_bulk.go) over TblCommitmentVals via one DupSort cursor per range (dups are invertedStep||value, latest-first, so Seek + NextNoDup walks the latest value per key) into an in-memory dbBranches map, bounded by the RAM budget. PreloadContractTrunkParallel now takes that map: per wave, DB-hits are resolved from memory (the freshest values, pinned first), file-misses go through the batched file resolver (capped by the budget remaining after the DB-hits). On a from-cold-snapshot start dbBranches is empty -> behaviour unchanged. Tests: TestPreloadParallel_DbHitsShadowFiles (DB value wins over a stale file value and its bitmap drives descent), TestNextSubtree, TestContractTrunkKeyRanges. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 53 ++++- execution/commitment/preload_parallel.go | 197 +++++++++++------- execution/commitment/preload_parallel_test.go | 122 ++++++++++- execution/commitment/preload_ranges.go | 62 ++++++ 4 files changed, 351 insertions(+), 83 deletions(-) create mode 100644 execution/commitment/preload_ranges.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8f3e1da3860..986a7519492 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1443,13 +1443,64 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach return vals, firstErr } + // dbPrefetch: phase 1 — pull the contract's MDBX-resident commitment + // branches (the freshest values: recently rewritten, not yet flushed to + // files) into an in-memory map via one range cursor per parity range, so + // the parallel wave-BFS resolves them from memory and the file layer is + // authoritative only for the keys MDBX doesn't have. The DupSort + // valsTable dups are invertedStep||value (latest first), so Seek + + // NextNoDup walks the latest value per key. Bounded by ramBudgetBytes; + // on a from-cold-snapshot start this finds nothing. + dbPrefetch := func(h []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := txs[0].CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + logger.Warn("[trunk-preload] phase-1 CursorDupSort failed", "err", cerr) + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(h)) + bytesBudget := ramBudgetBytes + scanRange := func(from, to []byte) error { + for k, v, err := c.Seek(from); k != nil; k, v, err = c.NextNoDup() { + if err != nil { + return err + } + if bytes.Compare(k, to) >= 0 { + break + } + if len(v) < 8 { + continue + } + val := common.Copy(v[8:]) + m[string(common.Copy(k))] = val + if bytesBudget -= len(val) + 8; bytesBudget <= 0 { + return nil + } + } + return nil + } + if err := scanRange(evenFrom, evenTo); err != nil { + logger.Warn("[trunk-preload] phase-1 even-range scan", "err", err) + } else if bytesBudget > 0 { + if err := scanRange(oddFrom, oddTo); err != nil { + logger.Warn("[trunk-preload] phase-1 odd-range scan", "err", err) + } + } + return m + } + for i, h := range hashes { started := time.Now() logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "parallel", parallel, "workers", nWorkers) var n int var err error if parallel { - n, err = commitment.PreloadContractTrunkParallel(h, ramBudgetBytes, resolve, branchCache, logger) + dbBranches := dbPrefetch(h) + if len(dbBranches) > 0 { + logger.Info("[trunk-preload] phase-1 db prefetch", "hash", hex.EncodeToString(h), "db_branches", len(dbBranches)) + } + n, err = commitment.PreloadContractTrunkParallel(h, ramBudgetBytes, dbBranches, resolve, branchCache, logger) } else { n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) } diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 057a22fdfcd..3231c4a5e66 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -30,25 +30,40 @@ import ( // Decoupled (callback form) so the commitment package needn't import db/state. type BatchBranchResolver func(keys [][]byte) (vals [][]byte, err error) +// estimatedEntryCost is the predicted RAM cost of pinning a (key, value) branch +// entry — see estimatedEntryOverheadBytes in preload.go. +func estimatedEntryCost(key, value []byte) int { + return estimatedEntryOverheadBytes + len(key) + len(value) +} + +// minEntryBytes is a true lower bound on estimatedEntryCost for a storage-trunk +// branch (33 = shortest HexToCompact key at depth >= 64; value may be empty). +// Used to bound a wave's *file fetch*: capping at remaining/minEntryBytes+1 +// never under-fetches, so the budget is guaranteed exhausted inside the wave. +const minEntryBytes = estimatedEntryOverheadBytes + 33 + // PreloadContractTrunkParallel pins contract H's storage-trunk branches into // cache by walking the subtree breadth-first, one depth level ("wave") at a -// time, resolving each wave's branches through a single batched, parallelisable -// file-only read. Breadth-first so a budget-truncated run still pins the -// shallowest (highest-reuse) branches first; wave-batched so the resolver can -// sort+partition the reads for locality instead of issuing them one at a time. +// time. Within a wave, each branch is resolved either from dbBranches (the +// MDBX-resident set — the freshest values; supplied by the caller's phase-1 +// range-cursor prefetch, may be nil/empty) or, for keys absent from it, through +// a single batched, parallelisable file-only read (BatchBranchResolver). // -// Bounded by ramBudgetBytes (same accounting as the BFS ContractTrunkPreload). -// Returns the number of branches pinned. +// Breadth-first so a budget-truncated run still pins the shallowest, highest- +// reuse branches first; wave-batched so the file resolver can sort+partition the +// reads for locality. DB-hits are pinned before file-hits within a wave (the DB +// holds the freshest, most-reused branches of a hot contract). Bounded by +// ramBudgetBytes. Returns the number of branches pinned. // -// File-layer-only: a branch present only in MDBX (recently written, not yet in -// a .kv file) is invisible here, and a branch present in both MDBX (fresh) and -// files (stale) resolves to the stale value. That is correct for the -// from-cold-snapshot trigger path (MDBX holds nothing for the subtree) — the -// case PIN_CONTRACT_TRUNKS targets. The MDBX-resident set is handled separately -// (a single range-cursor prefetch); see agentspecs/commitment-branch-preload-redesign.md. +// Correctness: dbBranches values shadow file values for the same key (DB is +// authoritative for steps not yet flushed to files), so passing the DB-resident +// set is what makes this safe on a live node. With an empty dbBranches it's a +// pure file-only preload — correct only from a cold snapshot, where MDBX holds +// nothing for the subtree. func PreloadContractTrunkParallel( contractHash []byte, ramBudgetBytes int, + dbBranches map[string][]byte, resolve BatchBranchResolver, cache *BranchCache, logger log.Logger, @@ -66,102 +81,129 @@ func PreloadContractTrunkParallel( return 0, fmt.Errorf("PreloadContractTrunkParallel: contractHash must be 32 bytes, got %d", len(contractHash)) } - contractNibbles := make([]byte, 64) - for i, b := range contractHash { - contractNibbles[2*i] = b >> 4 - contractNibbles[2*i+1] = b & 0x0f - } - type pathKey struct { path []byte // nibble path (1 byte / nibble) key []byte // HexToCompact(path) — the CommitmentDomain key } toPathKey := func(path []byte) pathKey { k := nibbles.HexToCompact(path) - // HexToCompact may return a slice over a reused buffer. kc := make([]byte, len(k)) - copy(kc, k) + copy(kc, k) // HexToCompact result may alias a reused buffer return pathKey{path: path, key: kc} } - frontier := []pathKey{toPathKey(contractNibbles)} + frontier := []pathKey{toPathKey(ContractNibbles(contractHash))} usedBytes := 0 maxDepthReached := 64 budgetHit := false + var dbHitsPinned int + + // pin records the entry, advances accounting, and queues the branch's + // children. Returns false if the entry didn't fit (budget exhausted). + pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { + cost := estimatedEntryCost(pk.key, v) + if usedBytes+cost > ramBudgetBytes { + budgetHit = true + return false + } + cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") + usedBytes += cost + pinned++ + if depth > maxDepthReached { + maxDepthReached = depth + } + if logger != nil && pinned%5000 == 0 { + logger.Info("[trunk-preload-parallel] progress", + "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) + } + if len(v) >= 4 { // 2-byte touchMap || 2-byte afterMap || per-child data + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1< 0 && !budgetHit; depth++ { - // Ascending key order so the resolver's contiguous partition is - // contiguous-in-file. BFS append order is already key-sorted (the - // HexToCompact packing is monotone for same-length paths), but sort - // defensively — it's a few k entries / wave, cheap. + // Ascending key order so a contiguous-slice partition of the file batch + // is contiguous-in-file (page-cache readahead). BFS append order is + // already key-sorted (HexToCompact packing is monotone for same-length + // paths), but sort defensively — a few k entries / wave, cheap. sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) - // Cap the wave's *fetch* at what the remaining budget can possibly - // absorb (minEntryBytes is a true lower bound on a pinned entry's cost, - // so this never under-fetches and the budget is guaranteed exhausted - // inside the wave). Without it, a wide budget-truncated wave — depth 69 - // is ~1.3M keys but only ~20k fit — would issue >1M file lookups to pin - // a tiny fraction, and that over-fetch races the next block. - const minEntryBytes = estimatedEntryOverheadBytes + 33 // 33 = shortest compact key at depth >= 64; value >= 0 bytes + // Split this wave into DB-resident hits (have the value) and file-misses + // (need a fetch). Both stay sorted (sub-sequences of the sorted frontier). + var dbHits []pathKey + var dbVals [][]byte + var fileMiss []pathKey + dbHitsBytes := 0 + for _, pk := range frontier { + if v, ok := dbBranches[string(pk.key)]; ok { + dbHits = append(dbHits, pk) + dbVals = append(dbVals, v) + dbHitsBytes += estimatedEntryCost(pk.key, v) + } else { + fileMiss = append(fileMiss, pk) + } + } + + // Cap the *file fetch* at what the budget can absorb after the DB-hits + // (which we pin first). minEntryBytes is a true lower bound, so this + // never under-fetches; a truncated wave is guaranteed to exhaust the + // budget, so we stop after it. truncatedByBudget := false - if remaining := ramBudgetBytes - usedBytes; remaining > 0 { - if maxFetch := remaining/minEntryBytes + 1; maxFetch < len(frontier) { - frontier = frontier[:maxFetch] + if fileBudget := ramBudgetBytes - usedBytes - dbHitsBytes; fileBudget <= 0 { + if len(fileMiss) > 0 { + fileMiss = fileMiss[:0] truncatedByBudget = true } + } else if maxFileFetch := fileBudget/minEntryBytes + 1; maxFileFetch < len(fileMiss) { + fileMiss = fileMiss[:maxFileFetch] + truncatedByBudget = true } - keys := make([][]byte, len(frontier)) - for i := range frontier { - keys[i] = frontier[i].key - } - vals, rerr := resolve(keys) - if rerr != nil { - return pinned, fmt.Errorf("preload at depth %d: %w", depth, rerr) - } - if len(vals) != len(keys) { - return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(vals), len(keys)) + var fileVals [][]byte + if len(fileMiss) > 0 { + keys := make([][]byte, len(fileMiss)) + for i := range fileMiss { + keys[i] = fileMiss[i].key + } + fileVals, err = resolve(keys) + if err != nil { + return pinned, fmt.Errorf("preload at depth %d: %w", depth, err) + } + if len(fileVals) != len(keys) { + return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) + } } var next []pathKey - for i, pk := range frontier { - v := vals[i] - if v == nil { - continue // not in the file layer - } - entryCost := estimatedEntryOverheadBytes + len(pk.key) + len(v) - if usedBytes+entryCost > ramBudgetBytes { - budgetHit = true + for i, pk := range dbHits { + if !pin(pk, dbVals[i], depth, &next) { break } - cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") - usedBytes += entryCost - pinned++ - if depth > maxDepthReached { - maxDepthReached = depth - } - if logger != nil && pinned%5000 == 0 { - logger.Info("[trunk-preload-parallel] progress", - "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) - } - // Decode the branch header (2-byte touchMap || 2-byte afterMap || - // per-child data) and queue the children that exist in the trie. - if len(v) < 4 { - continue - } - bitmap := binary.BigEndian.Uint16(v[2:4]) - for n := 0; n < 16; n++ { - if bitmap&(1<= 0 && (to == nil || bytes.Compare(k, to) < 0) + } + keyOf := func(contractNibbles, slotPath []byte) []byte { + return nibbles.HexToCompact(append(append([]byte{}, contractNibbles...), slotPath...)) + } + slotPaths := [][]byte{ + {}, // 64 — subtree root (even) + {0x0}, {0xf}, // 65 (odd) + {0x1, 0x2}, {0xf, 0xf}, // 66 (even) + {0x3, 0x4, 0x5}, // 67 (odd) + {0x6, 0x7, 0x8, 0x9}, // 68 (even) + {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 (odd) + make([]byte, 64), // 128 (even) — deepest + } + for _, sp := range slotPaths { + k := keyOf(nibA, sp) + total := 64 + len(sp) + if total%2 == 0 { + if !inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d (even) branch %x: must be in [%x,%x), not in [%x,%x)", total, k, evenFrom, evenTo, oddFrom, oddTo) + } + } else { + if !inRange(k, oddFrom, oddTo) || inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d (odd) branch %x: must be in [%x,%x), not in [%x,%x)", total, k, oddFrom, oddTo, evenFrom, evenTo) + } + } + if got := nibbles.CompactToHex(k); !bytes.Equal(got, append(append([]byte{}, nibA...), sp...)) { + t.Fatalf("CompactToHex round-trip mismatch for slot %x", sp) + } + } + // A different contract's branches must be in neither of A's ranges. + for _, sp := range slotPaths[:6] { + k := keyOf(nibB, sp) + if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) + } + } +} + func TestPreloadParallel_ResolverError(t *testing.T) { hash, tree, _ := buildSyntheticTree(t) root := "" @@ -279,7 +391,7 @@ func TestPreloadParallel_ResolverError(t *testing.T) { } c := NewBranchCache(64) // Fail when R||1 (depth 65) is requested -> root pinned at depth 64, then error. - _, err := PreloadContractTrunkParallel(hash, 1<<20, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) + _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) if err == nil { t.Fatal("expected error from the resolver") } diff --git a/execution/commitment/preload_ranges.go b/execution/commitment/preload_ranges.go new file mode 100644 index 00000000000..7883ad95681 --- /dev/null +++ b/execution/commitment/preload_ranges.go @@ -0,0 +1,62 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import "github.com/erigontech/erigon/execution/commitment/nibbles" + +// NextSubtree returns the smallest key K' that is greater than every key having +// `in` as a prefix — i.e. the exclusive upper bound for a prefix-range scan over +// `in`. Returns nil if `in` is all 0xff (no such K'). Mirrors db/kv.NextSubtree; +// inlined to keep this package's import set minimal. +func NextSubtree(in []byte) []byte { + r := make([]byte, len(in)) + copy(r, in) + for i := len(r) - 1; i >= 0; i-- { + if r[i] != 0xff { + r[i]++ + return r[:i+1] + } + } + return nil +} + +// ContractTrunkKeyRanges returns the two CommitmentDomain key ranges that +// together cover every branch node of a contract's storage subtree. +// +// The commitment domain keys branches by HexToCompact(nibblePath); HexToCompact's +// flag byte differs by the parity of the path length, so a contract's storage +// branches (path = keccak256(addr)'s 64 nibbles ++ k slot-path nibbles, total +// 64+k) split into two non-adjacent byte ranges: +// - even total length (k even, incl. the depth-64 subtree root): +// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) +// - odd total length (k odd): +// all such keys lie in [HexToCompact(H||0), NextSubtree(HexToCompact(H||15))). +func ContractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { + evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes + evenTo = NextSubtree(evenFrom) + + odd0 := make([]byte, 0, len(contractNibbles)+1) + odd0 = append(append(odd0, contractNibbles...), 0) + oddF := make([]byte, 0, len(contractNibbles)+1) + oddF = append(append(oddF, contractNibbles...), 15) + oddFrom = nibbles.HexToCompact(odd0) + oddTo = NextSubtree(nibbles.HexToCompact(oddF)) + return evenFrom, evenTo, oddFrom, oddTo +} + +// ContractNibbles expands a 32-byte contract hash to its 64-nibble path +// (one nibble per byte, high nibble first). +func ContractNibbles(contractHash []byte) []byte { + out := make([]byte, len(contractHash)*2) + for i, b := range contractHash { + out[2*i] = b >> 4 + out[2*i+1] = b & 0x0f + } + return out +} From a3374fed486c19d9e26316483021a46f0f133e68 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 08:18:15 +0000 Subject: [PATCH 068/167] db/state: test for the phase-1 dbPrefetch cursor walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestCommitmentPreloadPhase1Cursor populates TblCommitmentVals with real commitment-domain branch entries at multiple steps for two contracts, then walks the contract's two parity ranges (commitment.ContractTrunkKeyRanges) via CursorDupSort + Seek + NextNoDup and decodes v[8:] as the value — the same mechanics as triggerTrunkPreload's dbPrefetch. Asserts: - every contract-A branch appears in the cursor walk with its LATEST value (the dup decoded by skipping the 8-byte invertedStep prefix is correct); - the foreign contract's keys do not leak into contract-A's parity ranges (ContractTrunkKeyRanges is tight). This covers the cursor mechanics that the preload-side fake-resolver unit test (commitment.TestPreloadParallel_DbHitsShadowFiles) does not. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/commitment_preload_test.go | 147 ++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 db/state/commitment_preload_test.go diff --git a/db/state/commitment_preload_test.go b/db/state/commitment_preload_test.go new file mode 100644 index 00000000000..c7eb557f4a7 --- /dev/null +++ b/db/state/commitment_preload_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package state + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/statecfg" + "github.com/erigontech/erigon/execution/commitment" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// TestCommitmentPreloadPhase1Cursor mirrors triggerTrunkPreload's phase-1 +// dbPrefetch in db/state/execctx/domain_shared.go: a DupSort cursor over +// TblCommitmentVals walked via Seek + NextNoDup, decoding v[8:] as the value +// (dups = invertedStep||value, latest-first). Asserts that for a populated +// CommitmentDomain DB, the walk over the contract's two parity ranges +// (commitment.ContractTrunkKeyRanges) picks up the LATEST value per key and +// excludes keys outside the ranges (foreign contract). +// +// The preload-side unit test (commitment.TestPreloadParallel_DbHitsShadowFiles) +// covers the algorithm with a fake dbBranches map; this covers the cursor +// mechanics against the real domain writer encoding. +func TestCommitmentPreloadPhase1Cursor(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + logger := log.New() + db, d := testDbAndDomainOfStep(t, statecfg.Schema.CommitmentDomain, 16, logger) + + ctx := t.Context() + tx, err := db.BeginRw(ctx) + require.NoError(t, err) + defer tx.Rollback() + + domainRoTx := d.beginForTests() + defer domainRoTx.Close() + writer := domainRoTx.NewWriter() + defer writer.Close() + + // Pick two contracts so we can also assert foreign-contract keys are + // excluded from the target contract's parity ranges. + contractA := make([]byte, 32) + for i := range contractA { + contractA[i] = byte(0x10 + i) + } + contractB := make([]byte, 32) + for i := range contractB { + contractB[i] = byte(0x80 - i) + } + nibA := commitment.ContractNibbles(contractA) + nibB := commitment.ContractNibbles(contractB) + + // keyAt builds a real commitment-domain key: HexToCompact(contractNibbles + // ++ slotPath). The path's parity falls out of len(slotPath). + keyAt := func(contractNibbles, slotPath []byte) []byte { + full := append(append([]byte{}, contractNibbles...), slotPath...) + k := nibbles.HexToCompact(full) + return common.Copy(k) + } + + // A small set spanning even (depths 64, 66, 68) and odd (65, 67, 69) parity + // for contractA, plus a foreign key under contractB that must NOT show up. + slotPaths := [][]byte{ + {}, // depth 64 (even, == subtree root) + {0x3}, // depth 65 (odd) + {0x4, 0x5}, // depth 66 (even) + {0x6, 0x7, 0x8}, // depth 67 (odd) + {0x9, 0xa, 0xb, 0xc}, // depth 68 (even) + {0xd, 0xe, 0xf, 0x0, 0x1}, // depth 69 (odd) + } + + // Write each key at multiple txNums so it has multiple steps in MDBX (the + // DupSort dups). prev tracks the previous value (PutWithPrev contract). + expected := map[string][]byte{} + prev := map[string][]byte{} + writeAt := func(key []byte, txNum uint64, marker byte) { + v := make([]byte, 8) + v[0] = marker + v[7] = byte(txNum) + err := writer.PutWithPrev(key, v, txNum, prev[string(key)]) + require.NoError(t, err) + prev[string(key)] = common.Copy(v) + expected[string(key)] = common.Copy(v) + } + + // 3 successive writes per key at txNums 1, 5, 9 (latest = txNum 9 — the + // value we expect the cursor to return). + for i, sp := range slotPaths { + k := keyAt(nibA, sp) + for ti, tn := range []uint64{1, 5, 9} { + writeAt(k, tn, byte(0xA0+i*4+ti)) // distinctive marker per (key, step) + } + } + // Foreign contract key — must NOT appear in contractA's ranges. + foreign := keyAt(nibB, []byte{0x4, 0x5}) + writeAt(foreign, 7, 0xFF) + + require.NoError(t, writer.Flush(ctx, tx)) + + // Walk the cursor exactly like dbPrefetch does. + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(nibA) + c, err := tx.CursorDupSort(kv.TblCommitmentVals) + require.NoError(t, err) + defer c.Close() + + got := map[string][]byte{} + scanRange := func(from, to []byte) { + k, v, err := c.Seek(from) + for ; k != nil; k, v, err = c.NextNoDup() { + require.NoError(t, err) + if bytes.Compare(k, to) >= 0 { + break + } + require.GreaterOrEqual(t, len(v), 8, "dup value must be invStep(8B)||value") + got[string(common.Copy(k))] = common.Copy(v[8:]) + } + } + scanRange(evenFrom, evenTo) + scanRange(oddFrom, oddTo) + + // Every contractA branch must be present with its LATEST value (txNum=9). + for _, sp := range slotPaths { + k := keyAt(nibA, sp) + want, ok := expected[string(k)] + require.True(t, ok) + g, present := got[string(k)] + require.Truef(t, present, "key %x (depth %d) missing from cursor walk", k, 64+len(sp)) + require.Equalf(t, want, g, "key %x: cursor returned the wrong value (not the latest step)", k) + } + // Foreign-contract key must be absent. + _, leaked := got[string(foreign)] + require.Falsef(t, leaked, "foreign key %x leaked into contractA's parity ranges", foreign) +} From 9db0d5955b483bbcdfc40dbe712fbb54da9100ab Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 16:10:25 +0000 Subject: [PATCH 069/167] commitment, db/state: resumable parallel preload + adaptive parallel-mode wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the per-block adaptive extension path on top of the parallel wave-BFS preload. The new ContractTrunkPreloadParallel.Run method can be called incrementally with a step budget — each call advances zero or more complete waves and preserves un-pinned items at the current depth + accumulated children at depth+1 for the next call. This is the resumable counterpart to ContractTrunkPreload.Run (the serial BFS), so the adaptive controller can use the same wave-BFS file-only resolver for promote AND per-block extension instead of falling back to the serial CommitmentReader BFS. AdaptivePinController grows SetParallelMode(factory, provider) which installs a per-call resolver factory + dbBranches provider. The factory is invoked once per OnBlockComplete (returning a tx-scoped resolver and a release callback). promoteLocked/runExtensionLocked dispatch to the parallel path when the factory is set, falling back to the serial path when it isn't (or when the factory returns an error). adaptiveContractState holds either *ContractTrunkPreload (serial) or *ContractTrunkPreloadParallel (parallel); pinnedTotal/usedBytes/queueRemaining/pinnedPrefixes are mode- agnostic helpers used by the promote/extend/demote loop. SharedDomains.Flush installs the per-call factory + provider before calling OnBlockComplete, clears them after. The factory closes over the in-flight Flush tx (ttx.Debug().GetLatestFromFiles for file-only batch resolution); the provider walks TblCommitmentVals via CursorDupSort over the dual-parity ranges to populate the MDBX-resident branch overlay per contract. Tests: - execution/commitment/preload_parallel_test.go +9 tests covering ResumeAcrossSteps, RunAfterCompleteIsNoOp, StepBudgetCaps, ResumeAfterResolverError, DbBranchesPerStep, PinnedPrefixesAccumulate, NilCacheError, NilResolverError, BadHashLengthError - execution/commitment/trunk_pin_test.go +5 tests covering ParallelMode_PromoteUsesParallelResolver, ParallelMode_ExtendUsesResumableState, ParallelMode_FactoryErrorFallsBackToSerial, ParallelMode_DemoteInvalidates, ParallelMode_ReleaseCallbackInvoked Closes the "hypothesis C" gate from agentspecs/perf-regression-manifest.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 48 +++ execution/commitment/adaptive_pin.go | 209 +++++++++-- execution/commitment/preload_parallel.go | 330 ++++++++++++----- execution/commitment/preload_parallel_test.go | 334 ++++++++++++++++++ execution/commitment/trunk_pin_test.go | 244 +++++++++++++ 5 files changed, 1060 insertions(+), 105 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 986a7519492..8e66a38273f 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -783,7 +783,55 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } return v, uint64(step), len(v) > 0, nil } + // Install per-call parallel-mode plumbing so the + // controller's promote/extend uses the wave-BFS file-only + // resolver. The factory captures ttx; cleared after the + // call so a future OnBlockComplete can't reach a stale tx. + factory := func() (commitment.BatchBranchResolver, func(), error) { + resolve := func(keys [][]byte) ([][]byte, error) { + d := ttx.Debug() + vals := make([][]byte, len(keys)) + for i, k := range keys { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) + if err != nil { + return nil, err + } + if found { + vc := make([]byte, len(v)) + copy(vc, v) + vals[i] = vc + } + } + return vals, nil + } + return resolve, nil, nil + } + provider := func(contractHash []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) + scan := func(from, to []byte) { + for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { + if bytes.Compare(k, to) >= 0 { + return + } + if len(v) < 8 { + continue + } + m[string(common.Copy(k))] = common.Copy(v[8:]) + } + } + scan(evenFrom, evenTo) + scan(oddFrom, oddTo) + return m + } + sd.adaptivePinController.SetParallelMode(factory, provider) sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + sd.adaptivePinController.SetParallelMode(nil, nil) } } // Refresh pinned-tier gauges once per Flush — once-per-batch diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 1cd6b6f6066..c3668e01e7e 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -80,6 +80,10 @@ func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { // boundaries with a CommitmentReader factory; the controller // then promotes / extends / demotes based on the per-block // miss snapshot +// - Optional: host installs a parallel-mode resolver factory via +// SetParallelMode so promote/extend uses the wave-BFS parallel +// preload (file-only batch resolver + MDBX-overlay dbBranches) +// instead of the serial CommitmentReader BFS. type AdaptivePinController struct { cache *BranchCache cfg AdaptivePinControllerConfig @@ -93,15 +97,66 @@ type AdaptivePinController struct { // states holds per-promoted-contract bookkeeping. Protected by mu. mu sync.Mutex states map[[32]byte]*adaptiveContractState + + // Parallel-mode plumbing. nil = serial-BFS path (CommitmentReader). + // Set together via SetParallelMode; both required for parallel mode. + parallelResolverFactory ParallelResolverFactory + dbBranchesProvider DbBranchesProvider } +// ParallelResolverFactory builds a fresh BatchBranchResolver for one +// OnBlockComplete call. release() is invoked after the controller is done +// with the resolver (e.g. to tear down per-call worker txs). The factory +// may return (nil, nil, err) to indicate the controller should fall back +// to the serial-BFS path for this block. +type ParallelResolverFactory func() (resolve BatchBranchResolver, release func(), err error) + +// DbBranchesProvider returns the MDBX-resident branch overlay for one +// contract — values shadow file values in the parallel preload's wave. +// Empty/nil result is valid (means "no MDBX overlay; resolver is +// authoritative"); the controller treats it as a non-error. +type DbBranchesProvider func(contractHash []byte) map[string][]byte + type adaptiveContractState struct { contractHash [32]byte promotedAtBlock uint64 - preload *ContractTrunkPreload + preload *ContractTrunkPreload // serial-BFS path (nil when parallel) + parallel *ContractTrunkPreloadParallel // parallel-wave-BFS path (nil when serial) coldBlocksInARow int } +// pinnedTotal returns the cumulative pinned-count regardless of preload mode. +func (s *adaptiveContractState) pinnedTotal() int { + if s.parallel != nil { + return s.parallel.PinnedTotal() + } + return s.preload.PinnedTotal() +} + +// usedBytes returns the cumulative used-bytes regardless of preload mode. +func (s *adaptiveContractState) usedBytes() int { + if s.parallel != nil { + return s.parallel.UsedBytes() + } + return s.preload.UsedBytes() +} + +// queueRemaining returns the un-processed queue size regardless of mode. +func (s *adaptiveContractState) queueRemaining() int { + if s.parallel != nil { + return s.parallel.QueueRemaining() + } + return s.preload.QueueRemaining() +} + +// pinnedPrefixes returns the pin set for cache invalidation on demote. +func (s *adaptiveContractState) pinnedPrefixes() [][]byte { + if s.parallel != nil { + return s.parallel.PinnedPrefixes() + } + return s.preload.PinnedPrefixes() +} + // NewAdaptivePinController constructs a controller bound to the given // cache. Use Bind to install the cache miss-callback (separate so a // caller can keep a controller for telemetry without wiring it into @@ -143,6 +198,28 @@ func (c *AdaptivePinController) Bind() { c.cache.SetMissCallback(c.onCacheMiss) } +// SetParallelMode switches the controller to the wave-BFS parallel preload +// (ContractTrunkPreloadParallel) for both initial-view and per-block +// extension. The factory builds a tx-scoped resolver per OnBlockComplete +// (file-only); the provider returns the MDBX-resident branch overlay per +// contract so the freshest values shadow file values. +// +// Either argument may be nil to clear it. When factory==nil the controller +// uses the serial-BFS CommitmentReader path (existing default behavior). +// When factory is set but the factory call returns nil resolver/err, the +// controller falls back to serial-BFS for that block — the saved parallel +// state survives the block and resumes on the next successful factory call. +// +// Must be called before the first OnBlockComplete; calling it later is +// allowed but changes behavior at the next block boundary only — already- +// promoted contracts keep their existing serial/parallel state. +func (c *AdaptivePinController) SetParallelMode(factory ParallelResolverFactory, provider DbBranchesProvider) { + c.mu.Lock() + defer c.mu.Unlock() + c.parallelResolverFactory = factory + c.dbBranchesProvider = provider +} + func (c *AdaptivePinController) onCacheMiss(prefix []byte) { hash, ok := ContractHashFromPrefix(prefix) if !ok { @@ -171,6 +248,25 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui c.mu.Lock() defer c.mu.Unlock() + // Build the per-call parallel resolver up-front (one factory call per + // OnBlockComplete shared across all contracts this block). nil when + // parallel mode isn't installed, or the factory failed (we fall back to + // the serial-BFS reader path). + var parallelResolve BatchBranchResolver + var releaseParallel func() + if c.parallelResolverFactory != nil { + r, release, err := c.parallelResolverFactory() + if err != nil { + c.warnf("[adaptive-pin] parallel resolver factory failed, falling back to serial", "err", err, "block", blockNum) + } else { + parallelResolve = r + releaseParallel = release + } + } + if releaseParallel != nil { + defer releaseParallel() + } + var promoted, extended, demoted int // Already-promoted contracts: extend on hot, demote on cold. @@ -180,13 +276,13 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui state.coldBlocksInARow = 0 delete(misses, hash) // Extend if budget remains and queue not empty. - if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { - remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() + if state.queueRemaining() > 0 && state.usedBytes() < c.cfg.PerContractMaxBudgetBytes { + remaining := c.cfg.PerContractMaxBudgetBytes - state.usedBytes() step := c.cfg.ExtensionBudgetBytes if step > remaining { step = remaining } - if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { + if err := c.runExtensionLocked(ctx, state, step, parallelResolve, reader); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -208,24 +304,12 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) for _, hash := range candidates { - p, err := NewContractTrunkPreload(hash[:]) + state, err := c.promoteLocked(ctx, hash, blockNum, parallelResolve, reader) if err != nil { - continue - } - if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) - // Roll back partial pin so we don't leak entries - // for a contract that won't be in c.states. - for _, prefix := range p.PinnedPrefixes() { - c.cache.Invalidate(prefix) - } continue } - c.states[hash] = &adaptiveContractState{ - contractHash: hash, - promotedAtBlock: blockNum, - preload: p, - } + c.states[hash] = state promoted++ } } @@ -270,18 +354,101 @@ func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { // demoteLocked invalidates every pinned prefix for the contract. // Caller must hold c.mu. func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { - for _, prefix := range state.preload.PinnedPrefixes() { + for _, prefix := range state.pinnedPrefixes() { c.cache.Invalidate(prefix) } if c.logger != nil { c.logger.Info("[adaptive-pin] demoted", "hash", hex.EncodeToString(hash[:]), - "pinned_was", state.preload.PinnedTotal(), - "used_mb_was", state.preload.UsedBytes()/(1<<20), + "pinned_was", state.pinnedTotal(), + "used_mb_was", state.usedBytes()/(1<<20), "cold_blocks", state.coldBlocksInARow) } } +// promoteLocked runs the initial-view preload for a freshly-promoted +// contract. Uses the parallel wave-BFS when parallelResolve != nil, +// otherwise falls back to the serial CommitmentReader BFS. On error the +// partial pin set is rolled back. Caller must hold c.mu. +func (c *AdaptivePinController) promoteLocked( + ctx context.Context, + hash [32]byte, + blockNum uint64, + parallelResolve BatchBranchResolver, + reader CommitmentReader, +) (*adaptiveContractState, error) { + if parallelResolve != nil { + p, err := NewContractTrunkPreloadParallel(hash[:]) + if err != nil { + return nil, err + } + var dbBranches map[string][]byte + if c.dbBranchesProvider != nil { + dbBranches = c.dbBranchesProvider(hash[:]) + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, dbBranches, parallelResolve, c.cache, c.logger); err != nil { + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + return nil, err + } + return &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + parallel: p, + }, nil + } + // Serial BFS fallback (no parallel resolver this block). + p, err := NewContractTrunkPreload(hash[:]) + if err != nil { + return nil, err + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + return nil, err + } + return &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + preload: p, + }, nil +} + +// runExtensionLocked extends a promoted contract's preload by stepBudget +// bytes. Uses the saved state's mode (parallel vs serial). Caller must +// hold c.mu. +// +// When the saved state is serial (state.preload != nil) but parallelResolve +// is available this block, we keep using serial — switching mid-contract +// would lose the queue position. Future work: convert serial-state to +// parallel-state at extension time. +func (c *AdaptivePinController) runExtensionLocked( + ctx context.Context, + state *adaptiveContractState, + stepBudget int, + parallelResolve BatchBranchResolver, + reader CommitmentReader, +) error { + if state.parallel != nil { + if parallelResolve == nil { + // Parallel state but no resolver this block — skip the + // extension; the saved state survives for the next block. + return nil + } + var dbBranches map[string][]byte + if c.dbBranchesProvider != nil { + dbBranches = c.dbBranchesProvider(state.contractHash[:]) + } + _, _, err := state.parallel.Run(stepBudget, dbBranches, parallelResolve, c.cache, c.logger) + return err + } + // Serial-BFS state. + _, _, err := state.preload.Run(stepBudget, reader, c.cache, c.logger) + return err +} + // PromotedContracts returns the hashes of currently-pinned contracts. // Snapshot — the slice may be stale by the time the caller uses it. // Used by metrics + debug diagnostics. diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 3231c4a5e66..599b6a11e11 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -42,79 +42,141 @@ func estimatedEntryCost(key, value []byte) int { // never under-fetches, so the budget is guaranteed exhausted inside the wave. const minEntryBytes = estimatedEntryOverheadBytes + 33 -// PreloadContractTrunkParallel pins contract H's storage-trunk branches into -// cache by walking the subtree breadth-first, one depth level ("wave") at a -// time. Within a wave, each branch is resolved either from dbBranches (the -// MDBX-resident set — the freshest values; supplied by the caller's phase-1 -// range-cursor prefetch, may be nil/empty) or, for keys absent from it, through -// a single batched, parallelisable file-only read (BatchBranchResolver). +// maxStorageTrunkDepth is the deepest total nibble path a storage branch can +// have: 64 (account path) + 64 (keccak256(slot)) = 128. +const maxStorageTrunkDepth = 128 + +// pathKey is a single frontier entry: the nibble path (1 byte / nibble) and the +// HexToCompact-encoded CommitmentDomain key. +type pathKey struct { + path []byte // nibble path (1 byte / nibble) + key []byte // HexToCompact(path) — the CommitmentDomain key +} + +func toPathKey(path []byte) pathKey { + k := nibbles.HexToCompact(path) + kc := make([]byte, len(k)) + copy(kc, k) // HexToCompact result may alias a reused buffer + return pathKey{path: path, key: kc} +} + +// ContractTrunkPreloadParallel is the resumable analogue of ContractTrunkPreload +// (the serial BFS via CommitmentReader). It walks the contract's storage subtree +// breadth-first one depth-level ("wave") at a time and resolves missing branches +// through a batched, file-only BatchBranchResolver — no MDBX boundary crossings +// in the hot path. Each Run advances zero or more complete waves bounded by a +// per-call step budget; partial waves can be truncated by the file-fetch cap to +// fit the budget, but the truncated tail's siblings are skipped (BFS at the +// frontier is then resumed at depth+1 from the children of whatever was pinned). // -// Breadth-first so a budget-truncated run still pins the shallowest, highest- -// reuse branches first; wave-batched so the file resolver can sort+partition the -// reads for locality. DB-hits are pinned before file-hits within a wave (the DB -// holds the freshest, most-reused branches of a hot contract). Bounded by -// ramBudgetBytes. Returns the number of branches pinned. +// Mirrors ContractTrunkPreload's API surface (Run/PinnedTotal/UsedBytes/ +// QueueRemaining/MaxDepthReached/PinnedPrefixes/ContractHash) so it slots in to +// adaptive_pin.go as a drop-in replacement without changing the controller's +// promote/extend/demote loop. // // Correctness: dbBranches values shadow file values for the same key (DB is -// authoritative for steps not yet flushed to files), so passing the DB-resident -// set is what makes this safe on a live node. With an empty dbBranches it's a -// pure file-only preload — correct only from a cold snapshot, where MDBX holds -// nothing for the subtree. -func PreloadContractTrunkParallel( - contractHash []byte, - ramBudgetBytes int, +// authoritative for steps not yet flushed to files), so callers extending on a +// live node should pass the freshest MDBX-resident set per Run. With an empty +// dbBranches Run is a pure file-only preload — correct only from a cold snapshot +// or when MDBX/SD's flush-callback has already populated the cache with any +// freshly-written branches that would otherwise shadow the file values. +// +// Caller MUST NOT use the same instance from multiple goroutines. The +// BatchBranchResolver is passed PER Run rather than held on the struct so +// callers can supply a fresh tx-scoped resolver each call (e.g. adaptive +// controller building a resolver from the in-flight Flush tx every block). +// +// Resumability: when a step budget cuts a wave mid-iteration, the un-processed +// items at the current depth are preserved in `frontier`; the children of +// the items that did get pinned this wave land in `pendingChildren` at depth +// nextDepth+1. The next Run finishes the current depth first, then advances. +// Once a wave fully completes, frontier := pendingChildren and nextDepth++. +type ContractTrunkPreloadParallel struct { + contractHash []byte + frontier []pathKey // paths to process at depth = nextDepth + pendingChildren []pathKey // accumulated children of pinned items at depth = nextDepth+1 + nextDepth int // depth of the next wave (starts at 64) + pinnedPrefixes [][]byte + pinned int + usedBytes int + maxDepthReached int + dbHitsPinned int +} + +// NewContractTrunkPreloadParallel seeds a resumable preload state at depth 64 +// (storage subtree root) for the given contract. The resolver is passed +// per-Run (not stored on the state) so callers can hand a fresh tx-scoped +// resolver to each call. +func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreloadParallel, error) { + if len(contractHash) != 32 { + return nil, fmt.Errorf("NewContractTrunkPreloadParallel: contractHash must be 32 bytes, got %d", len(contractHash)) + } + contractHashCopy := make([]byte, len(contractHash)) + copy(contractHashCopy, contractHash) + return &ContractTrunkPreloadParallel{ + contractHash: contractHashCopy, + frontier: []pathKey{toPathKey(ContractNibbles(contractHashCopy))}, + nextDepth: 64, + maxDepthReached: 64, + }, nil +} + +// Run advances the wave-BFS, pinning branches into cache until either the step +// budget is exhausted, the frontier is empty (preload complete), or +// maxStorageTrunkDepth is reached. Returns newlyPinned during THIS call (not +// cumulative — see PinnedTotal), whether the preload is now complete, and any +// resolver error. +// +// dbBranches is consulted per wave (MDBX-resident overlays — values shadow file +// values, freshest). Pass nil for file-only mode (correct from a cold snapshot +// or when SD.Flush's cache callback has already populated freshly-written +// branches into the cache that would otherwise need to come from dbBranches). +// +// On resolver error the partial pin set survives; the wave position is +// preserved so a retry on the next Run picks up where it left off (at the same +// p.nextDepth). +func (p *ContractTrunkPreloadParallel) Run( + stepBudgetBytes int, dbBranches map[string][]byte, resolve BatchBranchResolver, cache *BranchCache, logger log.Logger, -) (pinned int, err error) { +) (newlyPinned int, queueEmpty bool, err error) { if cache == nil { - return 0, fmt.Errorf("PreloadContractTrunkParallel: cache is nil") + return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: cache is nil") } if resolve == nil { - return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: resolver is nil") } - if ramBudgetBytes <= 0 { - return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) - } - if len(contractHash) != 32 { - return 0, fmt.Errorf("PreloadContractTrunkParallel: contractHash must be 32 bytes, got %d", len(contractHash)) - } - - type pathKey struct { - path []byte // nibble path (1 byte / nibble) - key []byte // HexToCompact(path) — the CommitmentDomain key - } - toPathKey := func(path []byte) pathKey { - k := nibbles.HexToCompact(path) - kc := make([]byte, len(k)) - copy(kc, k) // HexToCompact result may alias a reused buffer - return pathKey{path: path, key: kc} + if stepBudgetBytes <= 0 { + return 0, len(p.frontier) == 0, nil } - frontier := []pathKey{toPathKey(ContractNibbles(contractHash))} - usedBytes := 0 - maxDepthReached := 64 + stepCap := p.usedBytes + stepBudgetBytes + chunkPinned := 0 budgetHit := false - var dbHitsPinned int - // pin records the entry, advances accounting, and queues the branch's - // children. Returns false if the entry didn't fit (budget exhausted). + // pin records the entry, advances accounting, queues the branch's + // children. Returns false if the entry didn't fit (step budget exhausted). pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { cost := estimatedEntryCost(pk.key, v) - if usedBytes+cost > ramBudgetBytes { + if p.usedBytes+cost > stepCap { budgetHit = true return false } - cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") - usedBytes += cost - pinned++ - if depth > maxDepthReached { - maxDepthReached = depth + cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel-resumable") + kc := make([]byte, len(pk.key)) + copy(kc, pk.key) + p.pinnedPrefixes = append(p.pinnedPrefixes, kc) + p.usedBytes += cost + p.pinned++ + chunkPinned++ + if depth > p.maxDepthReached { + p.maxDepthReached = depth } - if logger != nil && pinned%5000 == 0 { + if logger != nil && p.pinned%5000 == 0 { logger.Info("[trunk-preload-parallel] progress", - "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) + "pinned", p.pinned, "depth", depth, "used_mb", p.usedBytes/(1<<20)) } if len(v) >= 4 { // 2-byte touchMap || 2-byte afterMap || per-child data bitmap := binary.BigEndian.Uint16(v[2:4]) @@ -131,12 +193,13 @@ func PreloadContractTrunkParallel( return true } - for depth := 64; depth <= maxStorageTrunkDepth && len(frontier) > 0 && !budgetHit; depth++ { + for !budgetHit && p.nextDepth <= maxStorageTrunkDepth && len(p.frontier) > 0 { + depth := p.nextDepth // Ascending key order so a contiguous-slice partition of the file batch // is contiguous-in-file (page-cache readahead). BFS append order is // already key-sorted (HexToCompact packing is monotone for same-length // paths), but sort defensively — a few k entries / wave, cheap. - sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) + sort.Slice(p.frontier, func(i, j int) bool { return bytes.Compare(p.frontier[i].key, p.frontier[j].key) < 0 }) // Split this wave into DB-resident hits (have the value) and file-misses // (need a fetch). Both stay sorted (sub-sequences of the sorted frontier). @@ -144,7 +207,7 @@ func PreloadContractTrunkParallel( var dbVals [][]byte var fileMiss []pathKey dbHitsBytes := 0 - for _, pk := range frontier { + for _, pk := range p.frontier { if v, ok := dbBranches[string(pk.key)]; ok { dbHits = append(dbHits, pk) dbVals = append(dbVals, v) @@ -154,19 +217,17 @@ func PreloadContractTrunkParallel( } } - // Cap the *file fetch* at what the budget can absorb after the DB-hits - // (which we pin first). minEntryBytes is a true lower bound, so this - // never under-fetches; a truncated wave is guaranteed to exhaust the - // budget, so we stop after it. - truncatedByBudget := false - if fileBudget := ramBudgetBytes - usedBytes - dbHitsBytes; fileBudget <= 0 { - if len(fileMiss) > 0 { - fileMiss = fileMiss[:0] - truncatedByBudget = true - } + // Cap the *file fetch* at what the step budget can absorb after the + // DB-hits (which we pin first). minEntryBytes is a true lower bound, + // so this never under-fetches; if the cap truncates the wave, the + // un-fetched tail is preserved in p.frontier for the next Run. + var fileMissDeferred []pathKey + if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget <= 0 { + fileMissDeferred = fileMiss + fileMiss = nil } else if maxFileFetch := fileBudget/minEntryBytes + 1; maxFileFetch < len(fileMiss) { + fileMissDeferred = fileMiss[maxFileFetch:] fileMiss = fileMiss[:maxFileFetch] - truncatedByBudget = true } var fileVals [][]byte @@ -177,51 +238,152 @@ func PreloadContractTrunkParallel( } fileVals, err = resolve(keys) if err != nil { - return pinned, fmt.Errorf("preload at depth %d: %w", depth, err) + // State unchanged — retry on next Run picks up at same depth. + return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", depth, err) } if len(fileVals) != len(keys) { - return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) + return chunkPinned, false, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) } } - var next []pathKey + // Pin in dbHits-then-fileMiss order. Children of pinned items accumulate + // in p.pendingChildren (depth nextDepth+1). On budget hit mid-iteration, + // the un-processed remainder of dbHits/fileMiss + the deferred + // fileMissDeferred all become the new frontier (still at nextDepth). + dbHitStop := len(dbHits) for i, pk := range dbHits { - if !pin(pk, dbVals[i], depth, &next) { + if !pin(pk, dbVals[i], depth, &p.pendingChildren) { + dbHitStop = i break } - dbHitsPinned++ + p.dbHitsPinned++ } + fileMissStop := len(fileMiss) if !budgetHit { for i, pk := range fileMiss { v := fileVals[i] if v == nil { - continue // not in the file layer either (shouldn't happen with a complete dbBranches+files) + continue // not in the file layer either } - if !pin(pk, v, depth, &next) { + if !pin(pk, v, depth, &p.pendingChildren) { + fileMissStop = i break } } } - if truncatedByBudget { - budgetHit = true + + if budgetHit { + // Preserve un-pinned items at the current depth for the next Run. + // Resolved-but-not-pinned file values are discarded (callable resolver + // will be re-invoked on the next Run for these keys). + var rest []pathKey + rest = append(rest, dbHits[dbHitStop:]...) + rest = append(rest, fileMiss[fileMissStop:]...) + rest = append(rest, fileMissDeferred...) + p.frontier = rest + // Don't advance nextDepth — pendingChildren stays at depth+1 for + // when this depth is fully drained on a future Run. + break + } + + // Wave fully processed. Anything in fileMissDeferred (won't have any + // since !budgetHit ⇒ no truncation) goes back to frontier; pending + // children promote to the next wave. + if len(fileMissDeferred) > 0 { + // Shouldn't happen — if we got here without budgetHit, no truncation + // occurred. Defensive: re-queue them at the current depth so we + // don't lose data. + p.frontier = fileMissDeferred + } else { + p.frontier = p.pendingChildren + p.pendingChildren = nil + p.nextDepth++ } - frontier = next } + queueEmpty = (len(p.frontier) == 0 && len(p.pendingChildren) == 0) || p.nextDepth > maxStorageTrunkDepth + if logger != nil && (chunkPinned > 0 || queueEmpty) { + logger.Info("[trunk-preload-parallel] step", + "contract_hash", fmt.Sprintf("%x", p.contractHash), + "step_budget_mb", stepBudgetBytes/(1<<20), + "used_mb_total", p.usedBytes/(1<<20), + "pinned_this_step", chunkPinned, + "pinned_total", p.pinned, + "db_hits_total", p.dbHitsPinned, + "max_depth_reached", p.maxDepthReached, + "queue_empty", queueEmpty, + "next_depth", p.nextDepth, + "frontier_size", len(p.frontier)) + } + return chunkPinned, queueEmpty, nil +} + +// PinnedTotal returns the cumulative number of branches pinned by this preload +// state across all Run calls. +func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } + +// UsedBytes returns the cumulative byte cost of this preload's pin set. +func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } + +// QueueRemaining returns the number of paths still queued for descent +// (un-processed at the current depth + accumulated children at the next). +// Zero means the preload is complete; further Run calls are no-ops. +func (p *ContractTrunkPreloadParallel) QueueRemaining() int { + return len(p.frontier) + len(p.pendingChildren) +} + +// MaxDepthReached returns the deepest trie depth pinned so far. +func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } + +// PinnedPrefixes returns the prefix slices this preload has added to the cache +// so far. The adaptive controller uses this to Invalidate the contract's pin +// set on demotion. The returned slice aliases internal storage; do not mutate. +func (p *ContractTrunkPreloadParallel) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } + +// ContractHash returns the contract this preload targets. +func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } + +// DbHitsPinned returns the cumulative number of branches that were pinned from +// the dbBranches overlay (rather than via the file resolver). Diagnostic only. +func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } + +// PreloadContractTrunkParallel pins contract H's storage-trunk branches into +// cache in a single call, bounded by ramBudgetBytes. Convenience wrapper around +// NewContractTrunkPreloadParallel + Run for callers that don't need phased +// loading. The startup PIN_CONTRACT_TRUNKS hook uses this; the adaptive layer +// holds a *ContractTrunkPreloadParallel and calls Run incrementally. +// +// dbBranches is the MDBX-resident set — values shadow file values for the same +// key. Pass nil/empty for file-only mode (correct only from a cold snapshot). +func PreloadContractTrunkParallel( + contractHash []byte, + ramBudgetBytes int, + dbBranches map[string][]byte, + resolve BatchBranchResolver, + cache *BranchCache, + logger log.Logger, +) (pinned int, err error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreloadParallel(contractHash) + if err != nil { + return 0, err + } + if resolve == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + } + pinned, queueEmpty, err := p.Run(ramBudgetBytes, dbBranches, resolve, cache, logger) if logger != nil { logger.Info("[trunk-preload-parallel] complete", "contract_hash", fmt.Sprintf("%x", contractHash), "ram_budget_mb", ramBudgetBytes/(1<<20), - "used_mb", usedBytes/(1<<20), + "used_mb", p.UsedBytes()/(1<<20), "pinned", pinned, - "db_hits_pinned", dbHitsPinned, - "max_depth_reached", maxDepthReached, - "budget_exhausted", budgetHit, + "db_hits_pinned", p.DbHitsPinned(), + "max_depth_reached", p.MaxDepthReached(), + "budget_exhausted", !queueEmpty, "cache_pinned_total", cache.PinnedCount()) } - return pinned, nil + return pinned, err } - -// maxStorageTrunkDepth is the deepest total nibble path a storage branch can -// have: 64 (account path) + 64 (keccak256(slot)) = 128. -const maxStorageTrunkDepth = 128 diff --git a/execution/commitment/preload_parallel_test.go b/execution/commitment/preload_parallel_test.go index 450e5edced5..6cd3fe75e77 100644 --- a/execution/commitment/preload_parallel_test.go +++ b/execution/commitment/preload_parallel_test.go @@ -399,3 +399,337 @@ func TestPreloadParallel_ResolverError(t *testing.T) { t.Fatal("the depth-64 root should have been pinned before the depth-65 failure") } } + +// --- Resumable ContractTrunkPreloadParallel tests (Run-by-Run) --- + +// TestContractTrunkPreloadParallel_ResumeAcrossSteps confirms that splitting a +// full preload into multiple Run calls yields the same pinned set as a +// one-shot run with the equivalent total budget. +func TestContractTrunkPreloadParallel_ResumeAcrossSteps(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + // Reference: one-shot. + cRef := NewBranchCache(64) + if _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, resolve, cRef, nil); err != nil { + t.Fatal(err) + } + + // Step-by-step: budget exactly one entry per Run (the budget is checked + // before the entry is pinned; with overhead we need at least one entry's + // worth per step to make progress). + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Each entry is ~estimatedEntryOverheadBytes + 33 + valSz; over-allocate a + // bit per step so we always pin at least one new entry. + perStep := 2 * (estimatedEntryOverheadBytes + 33 + valSz) + const maxSteps = 50 + var steps int + for ; steps < maxSteps; steps++ { + _, done, err := p.Run(perStep, nil, resolve, c, nil) + if err != nil { + t.Fatalf("step %d: %v", steps, err) + } + if done { + break + } + } + if steps >= maxSteps { + t.Fatalf("preload did not complete in %d steps; pinned=%d", maxSteps, p.PinnedTotal()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("step-by-step pinned %d, want %d", p.PinnedTotal(), len(tree)) + } + if c.PinnedCount() != cRef.PinnedCount() { + t.Fatalf("step-by-step cache pinned %d != one-shot cache pinned %d", c.PinnedCount(), cRef.PinnedCount()) + } + // Spot-check: every path in the reference is in the step-by-step cache. + for path := range tree { + key := nibbles.HexToCompact([]byte(path)) + vRef, _, okRef := cRef.Get(key) + v, _, ok := c.Get(key) + if !okRef || !ok { + t.Fatalf("path %x: ref ok=%v, step ok=%v", path, okRef, ok) + } + if !bytes.Equal(v, vRef) { + t.Fatalf("path %x: step value differs from ref", path) + } + } +} + +// TestContractTrunkPreloadParallel_RunAfterCompleteIsNoOp confirms that once +// the BFS reaches an empty frontier, further Run calls are no-ops. +func TestContractTrunkPreloadParallel_RunAfterCompleteIsNoOp(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + n1, done1, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done1 { + t.Fatalf("expected done after full budget, got done=false (queue=%d)", p.QueueRemaining()) + } + if n1 != len(tree) { + t.Fatalf("first Run pinned %d, want %d", n1, len(tree)) + } + prevPinned := c.PinnedCount() + n2, done2, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done2 { + t.Fatal("expected done on second Run") + } + if n2 != 0 { + t.Fatalf("second Run pinned %d new entries, want 0", n2) + } + if c.PinnedCount() != prevPinned { + t.Fatalf("cache pinned count changed across no-op Run: %d -> %d", prevPinned, c.PinnedCount()) + } +} + +// TestContractTrunkPreloadParallel_StepBudgetCaps confirms that a small step +// budget stops the BFS even when the frontier has more work — and the saved +// state has the queue position preserved for the next call. +func TestContractTrunkPreloadParallel_StepBudgetCaps(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Budget for ~3 entries (depth-64 root + 2 depth-65 children). + rootKey := nibbles.HexToCompact(hexNibbles(hash)) + entry := estimatedEntryOverheadBytes + len(rootKey) + valSz + smallBudget := 3*entry + 10 + n1, done1, err := p.Run(smallBudget, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if done1 { + t.Fatalf("expected NOT done after a 3-entry budget; got done=true (pinned=%d)", n1) + } + if n1 < 1 || n1 > 5 { + t.Fatalf("expected ~3 pinned this step, got %d", n1) + } + if p.QueueRemaining() == 0 { + t.Fatal("expected frontier to be non-empty after small-budget step") + } + // Now exhaust with a full follow-on budget. + _, done2, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done2 { + t.Fatalf("expected done after large follow-on budget; queue=%d", p.QueueRemaining()) + } + // In our synthetic tree only 3 of 7 paths sit at depths 64-65; the rest + // require descending past the truncated wave. The cap-per-wave logic + // drops the truncated wave's tail (BFS-wise) but the children of the + // pinned ones progress on the next call. Cumulative pinned should be + // >= the step-1 count. + if p.PinnedTotal() <= n1 { + t.Fatalf("follow-on Run made no progress: pinned still %d (step-1 was %d)", p.PinnedTotal(), n1) + } +} + +// TestContractTrunkPreloadParallel_ResumeAfterResolverError confirms that a +// resolver error preserves the partial state — a retry on the next Run picks +// up from the same wave once the resolver is healthy again. +func TestContractTrunkPreloadParallel_ResumeAfterResolverError(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + const valSz = 100 + // Fail when R||1 is requested (a depth-65 key) — depth-64 wave succeeds. + failingResolve := fakeResolver(tree, nil, valSz, root+string([]byte{1})) + healthyResolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + _, done, err := p.Run(1<<20, nil, failingResolve, c, nil) + if err == nil { + t.Fatal("expected resolver error") + } + if done { + t.Fatal("Run with resolver error should return done=false") + } + // Depth-64 root should still be pinned (it was the previous wave). + if c.PinnedCount() == 0 { + t.Fatal("expected the depth-64 root pinned before the depth-65 wave failed") + } + preErrPinned := p.PinnedTotal() + // Retry with a healthy resolver — should pick up where we left off and + // finish. The previous partial wave will be re-attempted in the new + // call (the failing wave's frontier WAS advanced past the depth where + // the error fired — the error path returns before updating + // p.frontier/p.nextDepth, so retry sees the same depth's frontier). + n, done, err := p.Run(1<<20, nil, healthyResolve, c, nil) + if err != nil { + t.Fatalf("retry failed: %v", err) + } + if !done { + t.Fatalf("retry should complete the preload; queue=%d", p.QueueRemaining()) + } + if p.PinnedTotal()-preErrPinned != n { + t.Fatalf("PinnedTotal delta %d != Run pinned %d", p.PinnedTotal()-preErrPinned, n) + } + // Whole tree should be pinned by now. + if p.PinnedTotal() != len(tree) { + t.Fatalf("after retry pinned %d, want %d", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_DbBranchesPerStep confirms that dbBranches +// can change between Run calls (caller may pass a freshly-prefetched overlay +// per block) and that the freshest values shadow file values per call. +func TestContractTrunkPreloadParallel_DbBranchesPerStep(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + // On the first wave (depth 64) supply a fresh R value via dbBranches. + freshRoot := branchVal(tree[root], valSz) + freshRoot[4] = 0xAB + dbWave0 := map[string][]byte{string(nibbles.HexToCompact([]byte(root))): freshRoot} + + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Wave 0: pin the root using dbBranches (one entry budget). + rootKey := nibbles.HexToCompact([]byte(root)) + stepBudget := estimatedEntryOverheadBytes + len(rootKey) + valSz + 10 + if _, _, err := p.Run(stepBudget, dbWave0, resolve, c, nil); err != nil { + t.Fatal(err) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("wave 0: expected 1 db-hit pinned, got %d", p.DbHitsPinned()) + } + gotRoot, _, ok := c.Get(rootKey) + if !ok { + t.Fatal("root not pinned after wave 0") + } + if !bytes.Equal(gotRoot, freshRoot) { + t.Fatalf("wave 0: root pinned with stale file value, expected fresh dbBranches value") + } + + // Wave 1: depth 65. Pass an empty dbBranches (file-only); resolver + // supplies stale-bitmap values. + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatalf("expected done after large budget; queue=%d", p.QueueRemaining()) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("expected db-hit count to remain 1, got %d", p.DbHitsPinned()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("after wave 1 pinned %d, want %d", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_PinnedPrefixesAccumulate confirms that +// PinnedPrefixes() accumulates across Run calls (needed for demote-time +// cache invalidation in the adaptive controller). +func TestContractTrunkPreloadParallel_PinnedPrefixesAccumulate(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + rootKey := nibbles.HexToCompact(hexNibbles(hash)) + entry := estimatedEntryOverheadBytes + len(rootKey) + valSz + // Two small steps then one big step. + for i := 0; i < 2; i++ { + if _, _, err := p.Run(2*entry+10, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } + } + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatal("expected done after large step") + } + prefixes := p.PinnedPrefixes() + if len(prefixes) != p.PinnedTotal() { + t.Fatalf("PinnedPrefixes len %d != PinnedTotal %d", len(prefixes), p.PinnedTotal()) + } + // Every prefix must be in the cache. + for _, pf := range prefixes { + if _, _, ok := c.Get(pf); !ok { + t.Fatalf("prefix %x in PinnedPrefixes but not in cache", pf) + } + } + // All prefixes are unique. + seen := map[string]bool{} + for _, pf := range prefixes { + if seen[string(pf)] { + t.Fatalf("duplicate prefix %x in PinnedPrefixes", pf) + } + seen[string(pf)] = true + } +} + +// TestContractTrunkPreloadParallel_NilCacheError + NilResolverError confirm +// the input-validation guards. +func TestContractTrunkPreloadParallel_NilCacheError(t *testing.T) { + hash := make([]byte, 32) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + resolve := func(keys [][]byte) ([][]byte, error) { return make([][]byte, len(keys)), nil } + if _, _, err := p.Run(1<<20, nil, resolve, nil, nil); err == nil { + t.Fatal("expected error when cache is nil") + } +} + +func TestContractTrunkPreloadParallel_NilResolverError(t *testing.T) { + hash := make([]byte, 32) + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + if _, _, err := p.Run(1<<20, nil, nil, c, nil); err == nil { + t.Fatal("expected error when resolver is nil") + } +} + +func TestContractTrunkPreloadParallel_BadHashLengthError(t *testing.T) { + if _, err := NewContractTrunkPreloadParallel(make([]byte, 31)); err == nil { + t.Fatal("expected error for 31-byte hash") + } + if _, err := NewContractTrunkPreloadParallel(make([]byte, 33)); err == nil { + t.Fatal("expected error for 33-byte hash") + } +} diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index edf8ebac6b0..e1c58b2dc35 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -229,3 +229,247 @@ func TestAdaptivePinController_DemoteOnCold(t *testing.T) { require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") } + +// --- Parallel-mode adaptive controller tests --- + +// makeParallelResolver returns a BatchBranchResolver paired with a usage +// counter so tests can assert it was actually invoked. +func makeParallelResolver(saturated bool, calls *int32) BatchBranchResolver { + return func(keys [][]byte) ([][]byte, error) { + atomic.AddInt32(calls, int32(len(keys))) + vals := make([][]byte, len(keys)) + var bitmap uint16 + if saturated { + bitmap = 0xFFFF + } else { + bitmap = 0x0001 + } + for i := range keys { + buf := make([]byte, 4) + binary.BigEndian.PutUint16(buf[2:4], bitmap) + vals[i] = buf + } + return vals, nil + } +} + +// TestAdaptivePinController_ParallelMode_PromoteUsesParallelResolver +// confirms that with parallel mode installed, a freshly-promoted contract +// uses the BatchBranchResolver (not the serial CommitmentReader). The +// serial reader is wired to panic so we know if it's called. +func TestAdaptivePinController_ParallelMode_PromoteUsesParallelResolver(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 3 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(false, &resolverCalls) + + dbBranchesCalls := 0 + provider := func(contractHash []byte) map[string][]byte { + dbBranchesCalls++ + return nil // file-only — let the resolver supply everything + } + + factoryCalls := 0 + factory := func() (BatchBranchResolver, func(), error) { + factoryCalls++ + return resolver, nil, nil + } + ctrl.SetParallelMode(factory, provider) + + // Forge a 33-byte prefix that decodes to a known contract hash. + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 1) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Trigger 3 cache-misses for the same contract. + for i := 0; i < 3; i++ { + _, _, _ = c.Get(prefix) + } + + // Wire a reader that fails the test if called. + panicReader := func(prefix []byte) ([]byte, uint64, bool, error) { + t.Fatalf("parallel mode must not fall back to serial reader; called with prefix=%x", prefix) + return nil, 0, false, nil + } + + ctrl.OnBlockComplete(context.Background(), 1, panicReader) + require.Len(t, ctrl.PromotedContracts(), 1) + require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) + require.Greater(t, c.PinnedCount(), 0) + require.Equal(t, 1, factoryCalls, "factory should be called exactly once per OnBlockComplete") + require.Greater(t, atomic.LoadInt32(&resolverCalls), int32(0), "resolver should be called for the new contract") + require.GreaterOrEqual(t, dbBranchesCalls, 1, "dbBranchesProvider should be consulted for the new contract") +} + +// TestAdaptivePinController_ParallelMode_ExtendUsesResumableState confirms +// that the per-block extension reuses the same parallel state across calls +// (state.parallel persists; PinnedTotal grows monotonically). To trigger +// an extension miss for an already-promoted contract we Get a *deeper* +// prefix than what the initial promotion pinned, otherwise the lookup +// would hit the pinned tier and never invoke the miss callback. +func TestAdaptivePinController_ParallelMode_ExtendUsesResumableState(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.InitialViewBudgetBytes = 1 << 12 // 4 KiB — small initial view + cfg.ExtensionBudgetBytes = 1 << 12 // 4 KiB per extension step + cfg.PerContractMaxBudgetBytes = 1 << 20 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(true /* full bitmap — many children */, &resolverCalls) + factory := func() (BatchBranchResolver, func(), error) { + return resolver, nil, nil + } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 13) + } + rootPrefix := append([]byte{0x00}, contractHash[:]...) + // deepPrefix has the contract hash in bytes 1..33 (so the miss + // callback attributes it to our contract) plus extra trailing bytes + // so it's not the same key as anything the preload could pin. + makeDeepPrefix := func(seed byte) []byte { + p := make([]byte, 50) + p[0] = 0x10 // HP flag for odd-depth path + copy(p[1:33], contractHash[:]) + for i := 33; i < 50; i++ { + p[i] = byte(i*7) ^ seed + } + return p + } + + // Block 1: trigger miss on the root path, promote. + _, _, _ = c.Get(rootPrefix) + ctrl.OnBlockComplete(context.Background(), 1, nil) + require.Len(t, ctrl.PromotedContracts(), 1) + pinnedAfterPromote := c.PinnedCount() + require.Greater(t, pinnedAfterPromote, 0) + + // Block 2: miss on a DEEPER prefix in the same contract's subtree — + // not yet pinned, so the miss callback fires. Extension fires. + _, _, _ = c.Get(makeDeepPrefix(0x11)) + ctrl.OnBlockComplete(context.Background(), 2, nil) + pinnedAfterExt1 := c.PinnedCount() + require.GreaterOrEqual(t, pinnedAfterExt1, pinnedAfterPromote, "extension should not shrink the pin set") + require.Greater(t, pinnedAfterExt1, pinnedAfterPromote, "extension should add at least one entry under a saturated trie") + + // Block 3: another extension via a different deep prefix. + _, _, _ = c.Get(makeDeepPrefix(0x22)) + ctrl.OnBlockComplete(context.Background(), 3, nil) + pinnedAfterExt2 := c.PinnedCount() + require.GreaterOrEqual(t, pinnedAfterExt2, pinnedAfterExt1) +} + +// TestAdaptivePinController_ParallelMode_FactoryErrorFallsBackToSerial +// confirms that when the parallel factory returns an error, the controller +// falls back to the serial CommitmentReader path without crashing. +func TestAdaptivePinController_ParallelMode_FactoryErrorFallsBackToSerial(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + factory := func() (BatchBranchResolver, func(), error) { + return nil, nil, errFakeFactory + } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 21) + } + prefix := append([]byte{0x00}, contractHash[:]...) + _, _, _ = c.Get(prefix) + + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1, "should still promote via serial fallback") +} + +// TestAdaptivePinController_ParallelMode_DemoteInvalidates confirms that +// demotion invalidates the parallel-state pin set the same as serial. +func TestAdaptivePinController_ParallelMode_DemoteInvalidates(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.DemoteCooldownBlocks = 2 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(false, &resolverCalls) + factory := func() (BatchBranchResolver, func(), error) { return resolver, nil, nil } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 29) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Promote. + _, _, _ = c.Get(prefix) + ctrl.OnBlockComplete(context.Background(), 1, nil) + require.Len(t, ctrl.PromotedContracts(), 1) + require.Greater(t, c.PinnedCount(), 0) + + // Cold blocks — demote on 2nd. + ctrl.OnBlockComplete(context.Background(), 2, nil) + ctrl.OnBlockComplete(context.Background(), 3, nil) + require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") + require.Equal(t, 0, c.PinnedCount(), "parallel-state pin set invalidated on demotion") +} + +// TestAdaptivePinController_ParallelMode_ReleaseCallbackInvoked confirms +// that a non-nil release callback returned by the factory is called once +// per OnBlockComplete (so tx-scoped resources can be cleaned up). +func TestAdaptivePinController_ParallelMode_ReleaseCallbackInvoked(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(false, &resolverCalls) + var releaseCount int32 + factory := func() (BatchBranchResolver, func(), error) { + return resolver, func() { atomic.AddInt32(&releaseCount, 1) }, nil + } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 33) + } + prefix := append([]byte{0x00}, contractHash[:]...) + _, _, _ = c.Get(prefix) + + ctrl.OnBlockComplete(context.Background(), 1, nil) + require.Equal(t, int32(1), atomic.LoadInt32(&releaseCount), "release callback called once per OnBlockComplete") + + // No misses next block — release still called (factory is invoked per block + // regardless of whether the controller pins anything). + ctrl.OnBlockComplete(context.Background(), 2, nil) + require.Equal(t, int32(2), atomic.LoadInt32(&releaseCount)) +} + +var errFakeFactory = stringError("fake factory failure") + +type stringError string + +func (s stringError) Error() string { return string(s) } From ae9b3f3ebbc76382ca1b588f969351275176e68e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 21 May 2026 11:47:42 +0000 Subject: [PATCH 070/167] execution/commitment: keep the trie checkpoint key out of BranchCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeyCommitmentState ("state") is a commitment-domain key, so the BranchCache that sits in front of the CommitmentDomain was caching it like a trie branch. But it is the trie checkpoint (txNum/blockNum/root), not a branch: it changes every block, and serving a stale copy restores the trie to the wrong state — producing a wrong commitment root (observed as engine-x "Wrong trie root of block 1", and depended on an explicit branchCache.Invalidate(KeyCommitmentState) workaround in squeeze). Reject KeyCommitmentState in BranchCache.Put/Get/GetDecoded/PinEntry so the cache cannot hold the checkpoint by construction — no caller can pollute it and no external invalidation is required. The key's single definition moves to the commitment package so the cache can reference it without an import cycle; commitmentdb.KeyCommitmentState now aliases it. Follow-up: a coordinated, version-keyed (txNum) rolling-buffer cache could re-introduce checkpoint caching safely — tracked separately. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_cache.go | 25 +++++++++++++++++++ .../commitmentdb/commitment_context.go | 8 +++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index c07f25ea189..9edf294d242 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -17,6 +17,7 @@ package commitment import ( + "bytes" "fmt" "sync" "sync/atomic" @@ -25,6 +26,18 @@ import ( "github.com/erigontech/erigon/common/maphash" ) +// KeyCommitmentState is the commitment-domain key under which the trie +// checkpoint (txNum / blockNum / encoded root state) is stored. It is NOT a +// trie branch: it changes every block, so it must never enter the +// BranchCache — serving a stale checkpoint restores the trie to the wrong +// state and corrupts the computed root. BranchCache.Put/Get/PinEntry reject +// it by construction so no caller can pollute the cache with it. +var KeyCommitmentState = []byte("state") + +func isCommitmentStateKey(prefix []byte) bool { + return bytes.Equal(prefix, KeyCommitmentState) +} + // BranchCache stores commitment-trie branch data: // // - Bounded LRU tail with configurable capacity (eviction is well-defined, @@ -360,6 +373,9 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate // the input after the call. func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { + if isCommitmentStateKey(prefix) { + return + } dataCopy := make([]byte, len(data)) copy(dataCopy, data) c.pinned.Set(prefix, &branchCacheEntry{ @@ -417,6 +433,9 @@ func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { + if isCommitmentStateKey(prefix) { + return nil, 0, false + } entry, ok := c.lookup(prefix) if !ok { return nil, 0, false @@ -437,6 +456,9 @@ func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { // caller MUST NOT modify the cells in place. Read-only consumption is // safe across concurrent calls. func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { + if isCommitmentStateKey(prefix) { + return 0, nil, false + } entry, found := c.lookup(prefix) if !found { return 0, nil, false @@ -468,6 +490,9 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // from (0 if not tracked); origin is a short label of the write site // captured for divergence-detection diagnostics. func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { + if isCommitmentStateKey(prefix) { + return + } dataCopy := make([]byte, len(data)) copy(dataCopy, data) c.store(prefix, &branchCacheEntry{ diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 7a95675b315..874c5f651fa 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -590,10 +590,10 @@ func (e *errorTrieContext) Storage(plainKey []byte) (*commitment.Update, error) return nil, e.err } -// by that key stored latest root hash and tree state -const keyCommitmentStateS = "state" - -var KeyCommitmentState = []byte(keyCommitmentStateS) +// KeyCommitmentState is the commitment-domain key under which the latest +// root hash and tree state are stored. Single definition lives in the +// commitment package so BranchCache can exclude it by construction. +var KeyCommitmentState = commitment.KeyCommitmentState var ErrBehindCommitment = errors.New("behind commitment") From 3065d89b9a5da2bc582af94b2783815c67744b4c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 21 May 2026 13:35:07 +0000 Subject: [PATCH 071/167] execution: parent fork-validation SD into the canonical generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reorg-path NewPayload validation (TestReceiptRootValidationAfterReorg) computed a wrong commitment trie root, flakily, racing the gate-2 background commit. Root cause: fork-validation in ValidateChain created its SharedDomains with no parent. unwindToCommonCanonical then builds its unwind set from GetDiffset, which on a fresh SD finds nothing locally and falls back to ReadDiffSet on the tx — and the canonical chain's diffsets are not yet on disk while a background commit is in flight. With no unwind set, the unwind ran silently empty: SharedDomains.mem.unwindChangeset was never populated, so the commitment BranchCache was left unmasked and served pre-unwind (canonical-lineage) branches into the unwound read path. Fix: set the parent on the fork-validation SD (previously head-extending payloads only), and have SharedDomains.GetDiffset resolve through the parent chain. The canonical generation's pastChangesAccumulator then supplies the diffsets, the unwind builds a complete unwind set, and TemporalMemBatch.getLatest resolves every unwound key from the unwind set before the BranchCache is ever consulted — the unwind set masks the cache by construction. The parent does not shadow the unwound base: the unwind set covers exactly the keys the unwound canonical blocks touched. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 14 ++++++++- execution/execmodule/exec_module.go | 45 +++++++++++++++++------------ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8e66a38273f..bdd623ce2ea 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -551,7 +551,19 @@ func (sd *SharedDomains) SavePastChangesetAccumulator(blockHash common.Hash, blo } func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumber uint64) ([kv.DomainLen][]kv.DomainEntryDiff, bool, error) { - return sd.mem.GetDiffset(tx, blockHash, blockNumber) + d, ok, err := sd.mem.GetDiffset(tx, blockHash, blockNumber) + if ok || err != nil { + return d, ok, err + } + // Resolve through the parent chain: a fork-validation SD is freshly + // constructed with an empty mem batch, so the diffsets of the canonical + // blocks it must unwind live in the canonical generation's + // pastChangesAccumulator, reachable only via the parent link. Without + // this an unwind silently runs with no unwind set. + if sd.parent != nil { + return sd.parent.GetDiffset(tx, blockHash, blockNumber) + } + return d, ok, err } func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index c11ba373a48..de6df669ded 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -484,26 +484,33 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain the validation SD to currentContext when the new payload - // extends the current canonical head. Closes a timing hole: FCU's - // MergeExtendingFork moves the previous fork's writes into - // currentContext.mem, but MDBX commit only fires from RunLoop's - // CommitCycle on memory pressure. Between FCU and the next - // newPayload, currentContext.mem holds the latest state and MDBX - // is stale. Without the parent link, this fresh doms reads - // stale-MDBX while the aggregator-scope BranchCache (populated by - // the prior FV's CollectUpdate writes) holds the fresh state, - // producing divergence and wrong-trie-root downstream. + // Chain the validation SD to the latest in-memory canonical generation: + // e.currentContext when present, otherwise the newest in-flight commit + // generation (gate item 2 — the prior FCU cleared currentContext and + // handed its SD to the background commit). // - // Only set parent for head-extending payloads — fork validation - // requires unwindToCommonCanonical below to revert doms's view to - // the common ancestor, and exposing currentContext.mem (which holds - // post-divergence canonical writes) via the parent chain would - // shadow the unwound base. The current SD topology supports a - // single canonical chain only; per-branch SD lineage for concurrent - // multi-fork validation is a separate architectural follow-up. - canonicalHead := rawdb.ReadHeadBlockHash(tx) - if header.ParentHash == canonicalHead && e.currentContext != nil { + // The parent link serves two roles: + // + // 1. Head-extending payloads read the canonical generation's + // not-yet-committed domain state instead of stale MDBX. + // + // 2. Fork payloads: unwindToCommonCanonical below must build an unwind + // set, and the diffsets of the canonical blocks it unwinds live in + // the canonical generation's pastChangesAccumulator — reachable only + // through this parent link (GetDiffset chains to the parent). Without + // it the unwind silently runs with no unwind set, leaving the + // BranchCache unmasked and corrupting the computed root. + // + // For a fork payload the parent does NOT shadow the unwound base: once + // unwindToCommonCanonical has run, doms.mem.unwindChangeset holds every + // key the unwound canonical blocks touched, and TemporalMemBatch.getLatest + // resolves those from the unwind set before ever consulting the parent. + // + // Cherry-pick note: the upstream commit also chained to e.latestGen() + // (the gate-2 in-flight commit generation) when currentContext is nil; + // that generation chain is not on this branch, so currentContext is the + // only canonical generation here. + if e.currentContext != nil { doms.SetParent(e.currentContext) } From ae711df3e317f230c71f69af4c3d3f17bd3f42cf Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 22 May 2026 12:02:17 +0000 Subject: [PATCH 072/167] execution/commitment: prealloc rest slice in preload_parallel Lint (prealloc): the rest slice length is known from the source ranges. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/preload_parallel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 599b6a11e11..6f8374a0d32 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -276,7 +276,7 @@ func (p *ContractTrunkPreloadParallel) Run( // Preserve un-pinned items at the current depth for the next Run. // Resolved-but-not-pinned file values are discarded (callable resolver // will be re-invoked on the next Run for these keys). - var rest []pathKey + rest := make([]pathKey, 0, len(dbHits)-dbHitStop+len(fileMiss)-fileMissStop+len(fileMissDeferred)) rest = append(rest, dbHits[dbHitStop:]...) rest = append(rest, fileMiss[fileMissStop:]...) rest = append(rest, fileMissDeferred...) From b2a9d649aab39493a6bc1dc5dc3ee426467fb496 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 16 May 2026 14:05:52 +0000 Subject: [PATCH 073/167] db/state/execctx, execution/execmodule: clear BranchCache on SetHead unwind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SetHead unwinds the commitment domain (via pipelineExecutor.RunUnwind) and truncates the TxNums index back to targetBlock. The unwind removes post-targetBlock commitment entries from sd.mem and MDBX, but the aggregator-scope BranchCache still holds the pre-unwind value of KeyCommitmentState (pointing at the original head's blockNum). sd.Flush after the unwind calls FlushWithCallback, which only updates cache entries for keys present in sd.mem — KeyCommitmentState is not re-written by the unwind, so it stays unchanged in cache. The next FCU's NewSharedDomains.SeekCommitment hits the cache, reads the original-head blockNum, and trips ErrBehindCommitment when compared against the truncated TxNums.Last() — which returns targetBlock. updateForkChoice returns ExecutionStatusTooFarAway and the FCU fails even though the chain state is fully consistent. Add a SharedDomains.ClearBranchCache method that empties the aggregator-scope BranchCache (all tiers). Call it from SetHead after sd.Flush and before tx.Commit so subsequent FCUs see the post-unwind MDBX state. Bisect: TestSetHeadCanonicalCleanup first failed inside the 24c0eddc56 → 9f5b8170f4 window (WarmupCache deletion sequence + the FlushWithCallback method introduction). The test has been failing since the cache was integrated; this is the integration-gap fix needed to enable cache use for perf. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 12 ++++++++++++ execution/execmodule/set_head.go | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index bdd623ce2ea..ddaff77d9a0 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -854,6 +854,18 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return sd.mem.Flush(ctx, tx) } +// ClearBranchCache empties the aggregator-scope commitment BranchCache. +// Use after operations that mutate commitment state outside the normal +// sd.Flush callback path — notably SetHead unwind, which truncates +// commitment domain history but does not re-write all the keys whose +// cached values are now stale. Without this call, the next FCU sees +// the pre-unwind KeyCommitmentState and trips ErrBehindCommitment. +func (sd *SharedDomains) ClearBranchCache() { + if sd.branchCache != nil { + sd.branchCache.Clear() + } +} + // TemporalDomain satisfaction func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { if tx == nil { diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 91da37f13e1..0623b5bcaa6 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -149,6 +149,15 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { return fmt.Errorf("failed to flush shared domains: %w", err) } + // SetHead unwinds commitment history but the BranchCache (aggregator- + // scope, shared across SDs) still holds entries from the pre-unwind + // state — notably KeyCommitmentState pointing at the original head + // blockNum. The next FCU's NewSharedDomains.SeekCommitment would hit + // the cache, see the original-head blockNum, and trip + // ErrBehindCommitment against the now-truncated TxNums index. Clear + // the cache so subsequent reads see the post-unwind MDBX state. + sd.ClearBranchCache() + if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } From f151829dca57da2d8c7d38dcdbe42bc648c0b1e9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 23 May 2026 18:44:26 +0000 Subject: [PATCH 074/167] execution/stagedsync/rawdbreset: clear branchCache on ResetExec (#21138) ResetExec wipes the commitment table but the aggregator's in-memory branchCache was holding entries that referenced the just-deleted trie nodes. A from-0 re-exec then served those stale entries when computing block 0's commitment, producing a wrong trie root and dropping every genesis-allocated balance that no subsequent block touched. Serial exec happened to skirt the issue; parallel exec hit it directly (test_account_access/EXTCODESIZE-contract behaviour on mainnet block 46147 against 0xA1E4380A3B1f749673E270229993eE55F35663b4, and the TestFromZero_GenesisAllocPreservedAfterResetReExec parallel subtest). Drop the cache as part of ResetExec so it repopulates from the freshly- wiped commitment table. Co-Authored-By: Claude Opus 4.7 --- .../stagedsync/rawdbreset/reset_stages.go | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/execution/stagedsync/rawdbreset/reset_stages.go b/execution/stagedsync/rawdbreset/reset_stages.go index 05352bab7d6..023c8e9e626 100644 --- a/execution/stagedsync/rawdbreset/reset_stages.go +++ b/execution/stagedsync/rawdbreset/reset_stages.go @@ -36,6 +36,7 @@ import ( "github.com/erigontech/erigon/db/rawdb/blockio" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/db/snaptype" + dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/diagnostics/diaglib" "github.com/erigontech/erigon/execution/stagedsync/stages" "github.com/erigontech/erigon/execution/types" @@ -173,7 +174,7 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { cleanupList = append(cleanupList, db.Debug().DomainTables(kv.AccountsDomain, kv.StorageDomain, kv.CodeDomain, kv.CommitmentDomain, kv.ReceiptDomain, kv.RCacheDomain)...) cleanupList = append(cleanupList, db.Debug().InvertedIdxTables(kv.LogAddrIdx, kv.LogTopicIdx, kv.TracesFromIdx, kv.TracesToIdx)...) - return db.Update(ctx, func(tx kv.RwTx) error { + if err := db.Update(ctx, func(tx kv.RwTx) error { if err := clearStageProgress(tx, stages.Execution); err != nil { return err } @@ -183,7 +184,25 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { } // corner case: state files may be ahead of block files - so, can't use SharedDomains here. juts leave progress as 0. return nil - }) + }); err != nil { + return err + } + + // Wiping the commitment table leaves the aggregator's in-memory branchCache + // referencing trie nodes that no longer exist on disk. A subsequent from-0 + // re-exec then reads those stale nodes when computing block 0's commitment + // and produces a wrong trie root (parallel-exec failure mode of #21138). + // Drop the cache so it repopulates from the freshly-wiped table. + if hasAgg, ok := db.(dbstate.HasAgg); ok { + if agg, ok := hasAgg.Agg().(*dbstate.Aggregator); ok { + aggTx := agg.BeginFilesRo() + if bc := aggTx.BranchCache(); bc != nil { + bc.Clear() + } + aggTx.Close() + } + } + return nil } func ResetTxLookup(tx kv.RwTx) error { From 9898f3cd25fe80e2e3fa8be1ec70fa987de91ce1 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 23 May 2026 20:01:12 +0000 Subject: [PATCH 075/167] common/dbg, node/ethconfig: default ExecWorkerCount = NumCPU-1 The default came from `runtime.NumCPU()` which puts a worker on every visible core. The apply loop, FCU, GC, and other background goroutines end up fighting workers for runqueue slots, and the contention shows up as wall-time variance on every busy core. Drop the default by one so those non-worker goroutines have somewhere to land without preempting a worker. Floored at 1 for single-core environments. Override via EXEC3_WORKERS=N. Also removes the stale 'only half of CPU' comment on ethconfig.Defaults.Sync.ExecWorkerCount, which has been wrong since the default became NumCPU(). Co-Authored-By: Claude Opus 4.7 --- common/dbg/experiments.go | 9 ++++++--- node/ethconfig/config.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index bb39864534d..7a2f69d2aec 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -78,9 +78,12 @@ var ( CaplinSyncedDataMangerDeadlockDetection = EnvBool("CAPLIN_SYNCED_DATA_MANAGER_DEADLOCK_DETECTION", false) - Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) - numWorkers = runtime.NumCPU() - Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) + Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) + // One less than the visible CPU count so the apply loop, FCU, GC, and + // background goroutines aren't fighting workers for a runqueue slot. + // Floors at 1 for single-core environments. Override via EXEC3_WORKERS. + numWorkers = max(1, runtime.NumCPU()-1) + Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) ExecTerseLoggerLevel = EnvInt("EXEC_TERSE_LOGGER_LEVEL", int(log.LvlWarn)) CompressWorkers = EnvInt("COMPRESS_WORKERS", 0) // 0 means "not set": online presets default to 1, offline presets use RAM/CPU estimates MergeWorkers = EnvInt("MERGE_WORKERS", 1) diff --git a/node/ethconfig/config.go b/node/ethconfig/config.go index dcd469c05f5..a9e4862b5d8 100644 --- a/node/ethconfig/config.go +++ b/node/ethconfig/config.go @@ -84,7 +84,7 @@ var LightClientGPO = gaspricecfg.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ Sync: Sync{ - ExecWorkerCount: dbg.Exec3Workers, //only half of CPU, other half will spend for snapshots build/merge/prune + ExecWorkerCount: dbg.Exec3Workers, // NumCPU-1 by default — leaves one core for the apply loop, FCU, and GC BodyCacheLimit: 256 * 1024 * 1024, BodyDownloadTimeoutSeconds: 2, //LoopBlockLimit: 100_000, From 6454b7bff40fce21cbe7ae3f0d5b70701c3da398 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 23 May 2026 20:13:21 +0000 Subject: [PATCH 076/167] Revert "common/dbg, node/ethconfig: default ExecWorkerCount = NumCPU-1" This reverts commit 9898f3cd25fe80e2e3fa8be1ec70fa987de91ce1. --- common/dbg/experiments.go | 9 +++------ node/ethconfig/config.go | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 7a2f69d2aec..bb39864534d 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -78,12 +78,9 @@ var ( CaplinSyncedDataMangerDeadlockDetection = EnvBool("CAPLIN_SYNCED_DATA_MANAGER_DEADLOCK_DETECTION", false) - Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) - // One less than the visible CPU count so the apply loop, FCU, GC, and - // background goroutines aren't fighting workers for a runqueue slot. - // Floors at 1 for single-core environments. Override via EXEC3_WORKERS. - numWorkers = max(1, runtime.NumCPU()-1) - Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) + Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) + numWorkers = runtime.NumCPU() + Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) ExecTerseLoggerLevel = EnvInt("EXEC_TERSE_LOGGER_LEVEL", int(log.LvlWarn)) CompressWorkers = EnvInt("COMPRESS_WORKERS", 0) // 0 means "not set": online presets default to 1, offline presets use RAM/CPU estimates MergeWorkers = EnvInt("MERGE_WORKERS", 1) diff --git a/node/ethconfig/config.go b/node/ethconfig/config.go index a9e4862b5d8..dcd469c05f5 100644 --- a/node/ethconfig/config.go +++ b/node/ethconfig/config.go @@ -84,7 +84,7 @@ var LightClientGPO = gaspricecfg.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ Sync: Sync{ - ExecWorkerCount: dbg.Exec3Workers, // NumCPU-1 by default — leaves one core for the apply loop, FCU, and GC + ExecWorkerCount: dbg.Exec3Workers, //only half of CPU, other half will spend for snapshots build/merge/prune BodyCacheLimit: 256 * 1024 * 1024, BodyDownloadTimeoutSeconds: 2, //LoopBlockLimit: 100_000, From e8ba8129c7429e2ba3846d342c968de0adaae225 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 24 May 2026 11:42:24 +0000 Subject: [PATCH 077/167] cmd/evm, execution/tests: per-subtest ephemeral SharedDomains for state tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State tests are now production-aligned: each subtest creates a SharedDomains scope, runs pre-state loading + tx execution + commitment against it, and discards the SD (Close without Flush) when the subtest finishes. Closing without Flush prevents per-subtest writes from entering the long-lived aggregator branch cache, fixing the cross-subtest pollution where test N's pre-state was leaking into test N+1 via the cache and producing wrong-trie-root failures even when each subtest had a fresh DB tx (the tx rollback rolls back disk only — it does not invalidate the cache). * MakePreState now splits into MakePreState (kept for tracetest callers that need flushed pre-state) and MakePreStateInto (no Flush; caller owns the SD). * StateTest.Run / RunNoVerify take a caller-supplied SD. * runStateTests (execution/tests) and runStateTest (cmd/evm) create one SD per subtest with a deferred Close so the cache stays clean. --- cmd/evm/staterunner.go | 16 ++++- execution/tests/state_test.go | 8 ++- execution/tests/testutil/state_test_util.go | 76 +++++++++++++-------- 3 files changed, 68 insertions(+), 32 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 850fdeb8a16..b4f34b2bec4 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/tests/testutil" "github.com/erigontech/erigon/execution/tracing/tracers/logger" "github.com/erigontech/erigon/execution/vm" @@ -224,7 +225,18 @@ func runStateTest(ctx *cli.Context, cfg vm.Config, fname string) ([]testResult, } defer tx.Rollback() - statedb, root, err := test.Run(nil, tx, st, cfg, dirs) + // Per-subtest SD scope: caller-owned, never Flushed. Closing + // without Flush discards every write made during the subtest + // (pre-state + tx execution + commitment), so per-subtest state + // never enters the long-lived branch cache. + sd, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if err != nil { + result.Pass, result.Error = false, err.Error() + return + } + defer sd.Close() + + statedb, root, err := test.Run(nil, sd, tx, st, cfg, dirs) if err != nil { result.Pass, result.Error = false, err.Error() } @@ -239,7 +251,7 @@ func runStateTest(ctx *cli.Context, cfg vm.Config, fname string) ([]testResult, } if bench { _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { - _, _, gasUsed, _ := test.RunNoVerify(nil, tx, st, cfg, dirs) + _, _, gasUsed, _ := test.RunNoVerify(nil, sd, tx, st, cfg, dirs) return nil, gasUsed, nil }) result.Stats = &stats diff --git a/execution/tests/state_test.go b/execution/tests/state_test.go index 464fb9508ca..645c0f735e1 100644 --- a/execution/tests/state_test.go +++ b/execution/tests/state_test.go @@ -37,6 +37,7 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/tests/testutil" "github.com/erigontech/erigon/execution/tracing/tracers/logger" "github.com/erigontech/erigon/execution/vm" @@ -122,7 +123,12 @@ func runStateTests(t *testing.T, st *testutil.TestMatcher, testDir string) { withTrace(t, func(vmconfig vm.Config) error { tx := beginRwNoContention(t, db) defer tx.Rollback() - _, _, err = test.Run(t, tx, subtest, vmconfig, dirs) + sd, sdErr := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if sdErr != nil { + return sdErr + } + defer sd.Close() + _, _, err = test.Run(t, sd, tx, subtest, vmconfig, dirs) tx.Rollback() if err != nil && len(test.Json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { // Ignore expected errors diff --git a/execution/tests/testutil/state_test_util.go b/execution/tests/testutil/state_test_util.go index b1b8c6a277a..3ee793b567e 100644 --- a/execution/tests/testutil/state_test_util.go +++ b/execution/tests/testutil/state_test_util.go @@ -247,9 +247,11 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { return nil } -// Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(tb testing.TB, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (*state.IntraBlockState, common.Hash, error) { - st, root, _, err := t.RunNoVerify(tb, tx, subtest, vmconfig, dirs) +// Run executes a specific subtest and verifies the post-state and logs. +// sd is the caller-owned SharedDomains: discard (Close without Flush) to +// prevent per-subtest state from polluting the long-lived branch cache. +func (t *StateTest) Run(tb testing.TB, sd *execctx.SharedDomains, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (*state.IntraBlockState, common.Hash, error) { + st, root, _, err := t.RunNoVerify(tb, sd, tx, subtest, vmconfig, dirs) checkedErr := t.checkError(subtest, err) if checkedErr != nil { @@ -273,8 +275,12 @@ func (t *StateTest) Run(tb testing.TB, tx kv.TemporalRwTx, subtest StateSubtest, return st, root, nil } -// RunNoVerify runs a specific subtest and returns the statedb, post-state root and gas used. -func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (statedb *state.IntraBlockState, root common.Hash, gasUsed uint64, err error) { +// RunNoVerify runs a specific subtest and returns the statedb, post-state root +// and gas used. sd is the caller-owned SharedDomains scope; pre-state is loaded +// into it via MakePreStateInto so the per-subtest writes can be discarded by +// closing sd without Flush — keeping ephemeral test state out of the long-lived +// branch cache. +func (t *StateTest) RunNoVerify(tb testing.TB, sd *execctx.SharedDomains, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (statedb *state.IntraBlockState, root common.Hash, gasUsed uint64, err error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { return nil, common.Hash{}, 0, testforks.UnsupportedForkError{Name: subtest.Fork} @@ -288,20 +294,15 @@ func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest State readBlockNr := block.NumberU64() writeBlockNr := readBlockNr + 1 - _, err = MakePreState(&chain.Rules{}, tx, t.Json.Pre, readBlockNr) + _, err = MakePreStateInto(&chain.Rules{}, sd, tx, t.Json.Pre, readBlockNr) if err != nil { return nil, common.Hash{}, 0, testforks.UnsupportedForkError{Name: subtest.Fork} } - domains, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) - if err != nil { - return nil, common.Hash{}, 0, testforks.UnsupportedForkError{Name: subtest.Fork} - } - defer domains.Close() blockNum, txNum := readBlockNr, uint64(1) defer func() { - rootBytes, rootBytesErr := domains.ComputeCommitment(context2.Background(), tx, true, blockNum, txNum, "", nil) + rootBytes, rootBytesErr := sd.ComputeCommitment(context2.Background(), tx, true, blockNum, txNum, "", nil) if rootBytesErr != nil { if err != nil { err = fmt.Errorf("ComputeCommitment: %w: %w", rootBytesErr, err) @@ -314,7 +315,7 @@ func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest State }() r := rpchelper.NewLatestStateReader(tx) - w := rpchelper.NewLatestStateWriter(tx, domains, (*freezeblocks.BlockReader)(nil), writeBlockNr) + w := rpchelper.NewLatestStateWriter(tx, sd, (*freezeblocks.BlockReader)(nil), writeBlockNr) statedb = state.New(r) var baseFee *uint256.Int @@ -420,7 +421,37 @@ func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest State return statedb, root, gasUsed, nil } +// MakePreState loads the test fixture's pre-allocation, flushes it to db + +// the long-lived branch cache, and returns the resulting IntraBlockState. +// Used by tracetest callers that need the pre-state to outlive the call. +// +// The state-test runner does NOT use this — see MakePreStateInto for the +// ephemeral-SD shape that keeps per-subtest state out of the branch cache. func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { + sd, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if err != nil { + return nil, err + } + defer sd.Close() + statedb, err := MakePreStateInto(rules, sd, tx, alloc, blockNr) + if err != nil { + return nil, err + } + if err := sd.Flush(context2.Background(), tx); err != nil { + return nil, err + } + return statedb, nil +} + +// MakePreStateInto loads the test fixture's pre-allocation into the supplied +// SharedDomains. The caller owns sd's lifecycle: a SD that is Closed without +// Flush discards everything written here, leaving the long-lived branch cache +// untouched. This is the production-aligned shape — application code uses +// SharedDomains as the ephemeral working scope; Flush is what commits to db +// and cache. Per-subtest state in the state-test runner is ephemeral and must +// NOT enter the cache, so each subtest creates an SD, calls this, runs, and +// discards the SD without flushing. +func MakePreStateInto(rules *chain.Rules, sd *execctx.SharedDomains, tx kv.TemporalRwTx, alloc types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { r := rpchelper.NewLatestStateReader(tx) statedb := state.New(r) statedb.SetTxContext(blockNr, 0) @@ -438,25 +469,17 @@ func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAll val := uint256.NewInt(0).SetBytes(v.Bytes()) statedb.SetState(address, key, *val) } - if len(a.Code) > 0 || len(a.Storage) > 0 { statedb.SetIncarnation(address, state.FirstContractIncarnation) } } - domains, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) - if err != nil { - return nil, err - } - defer domains.Close() - latestTxNum, latestBlockNum, err := domains.SeekCommitment(context.Background(), tx) + latestTxNum, latestBlockNum, err := sd.SeekCommitment(context.Background(), tx) if err != nil { return nil, err } - w := rpchelper.NewLatestStateWriter(tx, domains, (*freezeblocks.BlockReader)(nil), blockNr-1) - - // Commit and re-open to start with a clean state. + w := rpchelper.NewLatestStateWriter(tx, sd, (*freezeblocks.BlockReader)(nil), blockNr-1) if err := statedb.FinalizeTx(rules, w); err != nil { return nil, err } @@ -464,12 +487,7 @@ func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAll return nil, err } - _, err = domains.ComputeCommitment(context.Background(), tx, true, latestBlockNum, latestTxNum, "flush-commitment", nil) - if err != nil { - return nil, err - } - - if err := domains.Flush(context2.Background(), tx); err != nil { + if _, err := sd.ComputeCommitment(context.Background(), tx, true, latestBlockNum, latestTxNum, "pre-state-commitment", nil); err != nil { return nil, err } return statedb, nil From 54a9029675bc4f11a810f672eb55bd7d8cb4e316 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 24 May 2026 18:33:27 +0000 Subject: [PATCH 078/167] execution/tests: read pre-state via sd.AsGetter so tx execution sees sd.mem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SD-discard refactor moved pre-state load into sd.mem and dropped the final Flush, but the tx execution reader was still constructed from the bare tx (NewLatestStateReader(tx)) — which only sees MDBX. Result: pre-state balance writes lived in sd.mem and the tx read MDBX-empty, surfacing as "insufficient funds for gas * price + value" on every state test that funded an account. NewLatestStateReader(sd.AsGetter(tx)) routes reads through the SD's sd.mem -> branchCache -> aggTx layering, so pre-state writes are visible. Same fix applied to MakePreStateInto for consistency with its writer. --- execution/tests/testutil/state_test_util.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/execution/tests/testutil/state_test_util.go b/execution/tests/testutil/state_test_util.go index 3ee793b567e..53094adc175 100644 --- a/execution/tests/testutil/state_test_util.go +++ b/execution/tests/testutil/state_test_util.go @@ -314,7 +314,11 @@ func (t *StateTest) RunNoVerify(tb testing.TB, sd *execctx.SharedDomains, tx kv. root = common.BytesToHash(rootBytes) }() - r := rpchelper.NewLatestStateReader(tx) + // Read through sd.AsGetter(tx) so the tx execution sees pre-state + // writes held in sd.mem. Without this the reader would hit MDBX + // directly and miss the never-Flushed pre-state, surfacing as + // "insufficient funds for gas * price + value" on every test. + r := rpchelper.NewLatestStateReader(sd.AsGetter(tx)) w := rpchelper.NewLatestStateWriter(tx, sd, (*freezeblocks.BlockReader)(nil), writeBlockNr) statedb = state.New(r) @@ -452,7 +456,10 @@ func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAll // NOT enter the cache, so each subtest creates an SD, calls this, runs, and // discards the SD without flushing. func MakePreStateInto(rules *chain.Rules, sd *execctx.SharedDomains, tx kv.TemporalRwTx, alloc types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { - r := rpchelper.NewLatestStateReader(tx) + // Read through sd.AsGetter so SetCode/SetBalance see prior pre-state + // writes held in sd.mem (consistent with the writer that targets sd + // via NewLatestStateWriter below). + r := rpchelper.NewLatestStateReader(sd.AsGetter(tx)) statedb := state.New(r) statedb.SetTxContext(blockNr, 0) for addr, a := range alloc { From 57dd0d648844b2091727eb6124762780716f8054 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 24 May 2026 22:17:26 +0000 Subject: [PATCH 079/167] execution/engineapi/engineapitester: reset aggregator branchCache between cached-tester runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EngineXTestRunner caches one tester per (fork, preAllocHash) tuple and reuses it across every EEST case that shares those keys — so the slow genesis + node + datadir bootstrap is paid once per group. The aggregator's in-memory branchCache is scoped to the tester's lifetime, which means test N's pre-state populates cache entries that test N+1 still sees when it asks for its own genesis commitment — surfacing as a deterministic 'wrong trie root of block 1' on the second-and-later tests in each group. ResetBranchCache() on EngineApiTester drops the cache via the same HasAgg / aggTx.BranchCache().Clear() path the rawdbreset.ResetExec fix uses (#21138). EngineXTestRunner.Run calls it before each execute so the group's cached tester starts the next case with an empty cache, matching the from-scratch state every test would see if it built its own tester. Cheaper than evicting the tester (which would rebuild the genesis + node on every test) and keeps the cache enabled for the multi-payload sequence within a single test — where the cache provides the perf win. --- .../engineapitester/engine_api_tester.go | 39 ++++++++++++++++++- .../engineapitester/engine_x_test_runner.go | 5 +++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/execution/engineapi/engineapitester/engine_api_tester.go b/execution/engineapi/engineapitester/engine_api_tester.go index af3126dd4e6..25a256f1669 100644 --- a/execution/engineapi/engineapitester/engine_api_tester.go +++ b/execution/engineapi/engineapitester/engine_api_tester.go @@ -39,7 +39,9 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" + dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/chain/networkname" @@ -399,10 +401,39 @@ func InitialiseEngineApiTester(ctx context.Context, args EngineApiTesterInitArgs TxnInclusionVerifier: NewTxnInclusionVerifier(rpcApiClient), Node: ethNode, NodeKey: nodeKey, + ChainDB: ethBackend.ChainKV(), cleanup: cleanup, }, nil } +// ResetBranchCache drops the aggregator's in-memory branchCache. The cache is +// aggregator-scoped and survives a tx rollback, so a tester reused across +// independent EEST cases would otherwise serve commitment-trie entries +// populated by the previous test's pre-state — producing a `wrong trie root +// of block 1` on the next test's genesis→block-1 boundary. Same mechanism the +// rawdbreset.ResetExec path uses after wiping the commitment table (#21138); +// here the trigger is "moving to a new test" instead of "wiping the table". +// Safe to call at any time the tester is between tests; no-op if no +// aggregator-backed db is attached. +func (eat EngineApiTester) ResetBranchCache() { + if eat.ChainDB == nil { + return + } + hasAgg, ok := eat.ChainDB.(dbstate.HasAgg) + if !ok { + return + } + agg, ok := hasAgg.Agg().(*dbstate.Aggregator) + if !ok { + return + } + aggTx := agg.BeginFilesRo() + if bc := aggTx.BranchCache(); bc != nil { + bc.Clear() + } + aggTx.Close() +} + type EngineApiTesterInitArgs struct { Logger log.Logger DataDir string @@ -430,7 +461,13 @@ type EngineApiTester struct { TxnInclusionVerifier TxnInclusionVerifier Node *node.Node NodeKey *ecdsa.PrivateKey - cleanup *cleanupHandle + // ChainDB is the running backend's temporal DB. Retained on the tester so + // callers (e.g. EngineXTestRunner.Run between tests in the same group) can + // reach the aggregator's BranchCache and clear it without rebuilding the + // whole tester. The reference must NOT be Closed by callers — the tester's + // cleanup owns the lifecycle. + ChainDB kv.RwDB + cleanup *cleanupHandle } func (eat EngineApiTester) Run(t *testing.T, test func(ctx context.Context, t *testing.T, eat EngineApiTester)) { diff --git a/execution/engineapi/engineapitester/engine_x_test_runner.go b/execution/engineapi/engineapitester/engine_x_test_runner.go index bc37634e464..b14f19aa7ea 100644 --- a/execution/engineapi/engineapitester/engine_x_test_runner.go +++ b/execution/engineapi/engineapitester/engine_x_test_runner.go @@ -222,6 +222,11 @@ func (extr *EngineXTestRunner) Run(ctx context.Context, test EngineXTestDefiniti if err != nil { return err } + // Reuse the cached tester for the (fork, preAllocHash) group but reset the + // aggregator-scope branchCache so commitment-trie entries from the prior + // test don't leak into this test's genesis→block-1 boundary. Cheaper than + // evicting the whole tester (which would rebuild the genesis + node). + tester.ResetBranchCache() return extr.execute(ctx, tester, test) } From 5a415cf84f67bfc6c966471b1e703e339481bd2c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 14 May 2026 20:26:43 +0000 Subject: [PATCH 080/167] execution/cache, db/state/execctx: SD-transparent ethHash bypass for CodeDomain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third map (`ethHashToCode`) to CodeCache, keyed by the 32-byte Ethereum codeHash (keccak256). New methods `GetByEthHash` and `PutWithEthHash` expose direct L2b access without going through the addr→maphash→code two-level path. The byte storage duplicates L2 in the worst case (2x code-bytes memory at the cap); accepted for the per-key fast path on many-addrs-one-code workloads. `SharedDomains.GetLatest(CodeDomain, ...)` consults L2b transparently: when the addr-keyed cache misses, resolve the codeHash from the AccountsDomain (typically warm because the EVM just loaded the account), probe `stateCache.GetCodeByHash` before falling through to the file accessor stack. On miss, fill both L1 and L2b via PutCodeWithHash. The fast path is unchanged. Workload shape this targets: many addresses sharing one codeHash (proxies, factory-deployed clones, ERC-20 holders, OpenZeppelin templates). Today's addr-keyed cache misses on every fresh address even when the bytecode is already cached. With this change a single L2b entry serves N addresses after the first population. Microbench results: - BenchmarkCodeCache_GetByEthHash_Hit: 17.01 ns/op - BenchmarkCodeCache_GetByEthHash_Miss: 15.45 ns/op - BenchmarkCodeCache_Get_AddrLevel_Hit: 32.44 ns/op (existing) - BenchmarkCodeCache_GetByEthHash_ManyAddrs: 17.02 ns/op L2b hit is ~2x faster than the existing two-level addr path (one map probe vs two), and enables hits on workloads where L1 would miss. Cross-client research at agentspecs/cross-client-state-access-2026-05-14.md notes geth's separate codeSizeCache as the further (geth-proven) win for EXTCODESIZE/EXTCODEHASH and addrToHash LRU as a one-line behaviour fix; both queued as follow-up surgical commits. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 83 +++++++++- execution/cache/code_cache.go | 88 ++++++++-- execution/cache/code_cache_ethhash_test.go | 177 +++++++++++++++++++++ execution/cache/state_cache.go | 27 ++++ 4 files changed, 362 insertions(+), 13 deletions(-) create mode 100644 execution/cache/code_cache_ethhash_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a5ae5d0b196..6ec06a001fc 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -43,6 +43,7 @@ import ( "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/commitmentdb" + "github.com/erigontech/erigon/execution/types/accounts" ) var ( @@ -955,6 +956,23 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } + // CodeDomain L2b transparent bypass: many addresses share one codeHash + // (proxies, factory-deployed clones, ERC-20 holder set, OpenZeppelin + // templates). Today's addr-keyed cache misses on every fresh address + // even when the bytecode is already in the L2b layer. Resolve the + // codeHash from the account record (warm in AccountsDomain cache for + // any account the EVM has already loaded) and probe L2b before paying + // the code-file accessor stack cost. + var codeEthHash []byte + if domain == kv.CodeDomain && sd.stateCache != nil { + if h := sd.codeHashForAddr(tx, k); len(h) > 0 { + codeEthHash = h + if cv, ok := sd.stateCache.GetCodeByHash(codeEthHash); ok { + return cv, 0, nil + } + } + } + if aggTx, ok := tx.AggTx().(MeteredGetter); ok { v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) } else { @@ -966,7 +984,11 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // Populate state cache on successful storage read if sd.stateCache != nil { - sd.stateCache.Put(domain, k, v) + if domain == kv.CodeDomain && len(codeEthHash) > 0 && len(v) > 0 { + sd.stateCache.PutCodeWithHash(k, v, codeEthHash) + } else { + sd.stateCache.Put(domain, k, v) + } } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") @@ -975,6 +997,65 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) return v, step, nil } +// codeHashForAddr returns the Ethereum codeHash for an account, or nil if the +// account cannot be resolved or has no code. Reads the account through this +// SD's normal layered lookup chain so the AccountsDomain cache absorbs the +// cost (the typical case for any address the EVM has already loaded). +// +// Returns nil quietly on any error or missing account — the caller falls +// through to the addr-keyed file read so correctness is unaffected. +func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte) []byte { + if len(addr) == 0 { + return nil + } + // Direct sd.mem / sd.stateCache / aggTx chain via tx.GetLatest avoids + // recursing back into sd.GetLatest's CodeDomain branch. tx here is + // the temporal tx the SD was given; tx.GetLatest hits the aggTx + // directly. The account is overwhelmingly cache-warm on this path + // because the EVM has just loaded it for the op that triggered the + // code read; if not, this read fills the AccountsDomain cache as a + // side-effect. + if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + if sd.parent != nil { + if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + if sd.stateCache != nil { + if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + // Fall back to the aggTx directly to avoid a second recursion through + // SD.GetLatest. Skip on error; the caller's addr-keyed file read still + // runs and produces the correct result. + v, _, err := tx.GetLatest(kv.AccountsDomain, addr) + if err != nil || len(v) == 0 { + return nil + } + return decodeAccountCodeHash(v) +} + +// decodeAccountCodeHash extracts the codeHash from an account's encoded +// (DecodeForStorage) bytes. Returns nil on decode error or when the account +// has no code (empty codeHash). +func decodeAccountCodeHash(enc []byte) []byte { + if len(enc) == 0 { + return nil + } + var acc accounts.Account + if err := acc.DecodeForStorage(enc); err != nil { + return nil + } + if acc.CodeHash.IsEmpty() { + return nil + } + h := acc.CodeHash.Value() + return h[:] +} + func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { return &sd.metrics } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 39351e8c6a8..89adbd56852 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -39,14 +39,21 @@ const ( DefaultAddrCacheBytes = 16 * datasize.MB ) -// CodeCache is a two-level concurrent cache for contract code. +// CodeCache is a multi-level concurrent cache for contract code. // Level 1: addr → maphash(code) (mutable, cleared on reorg) // Level 2: maphash(code) → code (immutable, never cleared) +// Level 2b: ethHash(code) → code (immutable, never cleared) — enables +// bypass of the addr level when the caller already knows the +// Ethereum codeHash from a prior account read (common path +// for EXTCODESIZE / EXTCODEHASH / CALL where many addresses +// share the same bytecode — proxies, factory-deployed clones, +// ERC-20 holders, etc.). // // This design is efficient because: // - Multiple addresses can share the same code (common with proxies/clones) -// - Uses fast maphash instead of cryptographic Keccak256 +// - Uses fast maphash instead of cryptographic Keccak256 for L1/L2 // - Address mappings are small (8 bytes) so we can cache many more +// - L2b lets callers with the codeHash in hand skip L1 entirely // - Thread-safe via sync.Map // // Capacity is byte-based. Once full, new puts are no-ops but @@ -61,14 +68,25 @@ type CodeCache struct { addrSize atomic.Int64 // current size in bytes hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) - blockHash common.Hash // hash of the last block processed - mu sync.RWMutex + + // L2b: 32-byte Ethereum codeHash (keccak256) → code bytes. Populated + // alongside L2 when the caller provides ethHash on Put. Independent + // of L1 — Get-by-ethHash bypasses addr lookup entirely. Memory cost: + // duplicates code bytes vs L2 (worst case 2x byte storage); accepted + // for the per-key fast-path on many-addrs-one-code workloads. + ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent + ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) + + blockHash common.Hash // hash of the last block processed + mu sync.RWMutex // Stats counters (atomic for concurrent access) - addrHits atomic.Uint64 - addrMisses atomic.Uint64 - codeHits atomic.Uint64 - codeMisses atomic.Uint64 + addrHits atomic.Uint64 + addrMisses atomic.Uint64 + codeHits atomic.Uint64 + codeMisses atomic.Uint64 + ethHashHits atomic.Uint64 + ethHashMisses atomic.Uint64 addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes @@ -77,10 +95,11 @@ type CodeCache struct { // NewCodeCache creates a new CodeCache with the specified byte capacities. func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeCache { return &CodeCache{ - addrToHash: maphash.NewMap[versionedAddressID](), - hashToCode: maphash.NewMap[[]byte](), - addrCapacityB: addrCapacityBytes, - codeCapacityB: codeCapacityBytes, + addrToHash: maphash.NewMap[versionedAddressID](), + hashToCode: maphash.NewMap[[]byte](), + ethHashToCode: maphash.NewMap[[]byte](), + addrCapacityB: addrCapacityBytes, + codeCapacityB: codeCapacityBytes, } } @@ -143,6 +162,51 @@ func (c *CodeCache) Put(addr []byte, code []byte) { c.codeSize.Add(codeEntrySize) } +// GetByEthHash retrieves contract code by its Ethereum codeHash (keccak256). +// Bypasses the addr-keyed L1/L2 path. Returns (code, true) on hit, (nil, false) on miss. +// +// Designed for the common path where the caller has already loaded the +// account and knows the codeHash (EXTCODESIZE, EXTCODEHASH, CALL targets +// after account-load). Many addresses sharing one codeHash all hit this +// single L2b entry after the first population. +func (c *CodeCache) GetByEthHash(ethHash []byte) ([]byte, bool) { + code, ok := c.ethHashToCode.Get(ethHash) + if !ok || len(code) == 0 { + c.ethHashMisses.Add(1) + return nil, false + } + c.ethHashHits.Add(1) + return code, true +} + +// PutWithEthHash stores contract code, populating both the addr-keyed +// path (L1+L2) and the ethHash-keyed path (L2b). Use when the caller +// has the codeHash in hand (typically from a just-loaded account record); +// avoids the maphash-vs-keccak collision risk of re-deriving the ethHash +// from the value, and ensures L2b is fillable without an extra keccak. +// +// addr may be empty to populate only L2b (e.g. when populating from a +// codehash-only path that hasn't seen the addr yet). +func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { + if len(code) == 0 || len(ethHash) == 0 { + return + } + + if len(addr) > 0 { + c.Put(addr, code) + } + + if _, exists := c.ethHashToCode.Get(ethHash); exists { + return + } + entrySize := int64(len(ethHash) + len(code)) + if c.ethHashCodeSize.Load()+entrySize > int64(c.codeCapacityB) { + return + } + c.ethHashToCode.Set(ethHash, code) + c.ethHashCodeSize.Add(entrySize) +} + // Delete removes the address → codeHash mapping. // The codeHash → code mapping is kept since it's immutable. func (c *CodeCache) Delete(addr []byte) { diff --git a/execution/cache/code_cache_ethhash_test.go b/execution/cache/code_cache_ethhash_test.go new file mode 100644 index 00000000000..6c17724da72 --- /dev/null +++ b/execution/cache/code_cache_ethhash_test.go @@ -0,0 +1,177 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +import ( + "bytes" + "testing" + + "github.com/c2h5oh/datasize" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// makeEthHash builds a deterministic 32-byte ethHash for tests/benchmarks. +func makeEthHash(i int) []byte { + h := make([]byte, 32) + h[0] = byte(i) + h[1] = byte(i >> 8) + return h +} + +func TestCodeCache_GetByEthHash_HitAfterPut(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52} // small contract preamble + ethHash := makeEthHash(0xab) + + // Empty cache: miss. + v, ok := c.GetByEthHash(ethHash) + require.False(t, ok) + require.Nil(t, v) + + // Populate via PutWithEthHash — both addr and ethHash paths fill. + c.PutWithEthHash(addr, code, ethHash) + + // Hit by ethHash directly. + v, ok = c.GetByEthHash(ethHash) + require.True(t, ok) + require.True(t, bytes.Equal(v, code)) + + // Hit by addr via existing two-level path. + v, ok = c.Get(addr) + require.True(t, ok) + require.True(t, bytes.Equal(v, code)) +} + +func TestCodeCache_GetByEthHash_DistinctAddrsSameCode(t *testing.T) { + // The point of L2b: many addresses sharing one codeHash all hit a + // single entry once any one of them has been populated. + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52} + ethHash := makeEthHash(0xcd) + + addr1 := makeAddr(1) + c.PutWithEthHash(addr1, code, ethHash) + + // A different address never seen at L1 still hits the L2b layer + // when looked up by the shared ethHash. + v, ok := c.GetByEthHash(ethHash) + require.True(t, ok) + require.True(t, bytes.Equal(v, code)) + + // The addr2 lookup itself still misses (L1 unknown), as expected — + // L2b is meant to be probed by callers that already hold the hash. + addr2 := makeAddr(2) + _, ok = c.Get(addr2) + require.False(t, ok) +} + +func TestCodeCache_PutWithEthHash_EmptyHashOrCodeIsNoOp(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + code := []byte{0x60, 0x00} + + c.PutWithEthHash(addr, code, nil) // empty hash → skip L2b + v, ok := c.GetByEthHash(makeEthHash(7)) + require.False(t, ok) + require.Nil(t, v) + + c.PutWithEthHash(addr, nil, makeEthHash(7)) // empty code → skip both + v, ok = c.GetByEthHash(makeEthHash(7)) + require.False(t, ok) + require.Nil(t, v) +} + +func TestCodeCache_PutWithEthHash_RespectsCodeCapacity(t *testing.T) { + // 8-byte cap: 32-byte ethHash + 4-byte code > 32. New L2b puts must + // no-op when the layer is full. Use tiny code to keep math obvious. + c := NewCodeCache(8, 1*datasize.MB) + c.PutWithEthHash(makeAddr(1), []byte{1, 2, 3, 4}, makeEthHash(1)) + // Second put exceeds the L2b budget — must no-op. + c.PutWithEthHash(makeAddr(2), []byte{5, 6, 7, 8}, makeEthHash(2)) + + _, ok := c.GetByEthHash(makeEthHash(2)) + assert.False(t, ok, "second L2b entry should not exist when capacity is exceeded") +} + +// ============================================================================= +// Microbenchmarks — measure the per-op cost of the L2b path. +// ============================================================================= + +func BenchmarkCodeCache_GetByEthHash_Hit(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + code := bytes.Repeat([]byte{0x5b}, 2048) // 2 KiB typical contract size + ethHash := makeEthHash(0x11) + c.PutWithEthHash(makeAddr(1), code, ethHash) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + v, ok := c.GetByEthHash(ethHash) + if !ok || len(v) == 0 { + b.Fatal("expected hit") + } + } +} + +func BenchmarkCodeCache_GetByEthHash_Miss(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + missHash := makeEthHash(0x22) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = c.GetByEthHash(missHash) + } +} + +// BenchmarkCodeCache_Get_AddrLevel_Hit baseline: the existing addr-keyed +// path. Compare against GetByEthHash to verify the L2b lookup is at least +// as fast (one map probe vs two: addr→hash then hash→code). +func BenchmarkCodeCache_Get_AddrLevel_Hit(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + code := bytes.Repeat([]byte{0x5b}, 2048) + addr := makeAddr(1) + c.PutWithEthHash(addr, code, makeEthHash(0x33)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + v, ok := c.Get(addr) + if !ok || len(v) == 0 { + b.Fatal("expected hit") + } + } +} + +// BenchmarkCodeCache_GetByEthHash_ManyAddrs_OneCode measures the workload +// shape this layer is designed for: many addresses sharing one codeHash. +// Without L2b every fresh addr would pay a file read. With L2b every +// caller that already knows the hash hits one shared entry. +func BenchmarkCodeCache_GetByEthHash_ManyAddrs_OneCode(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + code := bytes.Repeat([]byte{0x5b}, 2048) + ethHash := makeEthHash(0x44) + c.PutWithEthHash(makeAddr(1), code, ethHash) // populate once + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Caller knows the hash from a prior account read; probes L2b. + v, ok := c.GetByEthHash(ethHash) + if !ok || len(v) == 0 { + b.Fatal("expected hit") + } + } +} diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 816cdae11d3..7f1ee9b3d08 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -78,6 +78,33 @@ func (c *StateCache) Get(domain kv.Domain, key []byte) ([]byte, bool) { return cache.Get(key) } +// GetCodeByHash retrieves code bytes by their Ethereum codeHash (keccak256), +// bypassing the addr-keyed CodeDomain lookup. Returns (nil, false) on miss or +// when the code domain cache is not a CodeCache (defensive fallback). +// +// Use when the caller has the codeHash in hand (post-account-load) — typical +// for EXTCODESIZE / EXTCODEHASH / CALL targets. Lets many-addrs-one-code +// patterns (proxies, factory clones, ERC-20 holders) share a single L2b +// entry. +func (c *StateCache) GetCodeByHash(ethHash []byte) ([]byte, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return nil, false + } + return cc.GetByEthHash(ethHash) +} + +// PutCodeWithHash stores code populating both the addr-keyed path and the +// ethHash-keyed L2b layer. Callers should prefer this over Put when they +// have the codeHash from the account record — avoids a redundant keccak. +func (c *StateCache) PutCodeWithHash(addr, code, ethHash []byte) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutWithEthHash(addr, common.Copy(code), ethHash) +} + // Put stores data for the given domain and key. func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { cache := c.caches[domain] From 4499ad5330756fe2b027b50a3815273a4bd3b87e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 14 May 2026 20:32:38 +0000 Subject: [PATCH 081/167] execution/cache, db/state, execution/state: codeSizeCache for EXTCODESIZE / EXTCODEHASH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third caching layer to CodeCache (alongside L1 addr→maphash and L2b ethHash→bytes): codeSizeByEthHash maps the 32-byte Ethereum codeHash to its byte length. Tiny per-entry footprint (32B key + 8B value vs 5-10 KB for full bytes) so the same memory budget gives ~1000x the hit surface. Capped at 1M entries (geth core/state/database_code.go uses the same size). EXTCODESIZE / EXTCODEHASH callers — historically the slowest opcodes on the lab dashboard's bench — answer from a single map probe without paying the file accessor stack cost of the full bytes. Geth-proven; cross-client writeup at agentspecs/cross-client-state-access-2026-05-14.md notes this as the largest single available win for the synthetic bench. Wiring: - CodeCache.GetCodeSizeByEthHash / PutCodeSizeByEthHash — direct accessors. - PutWithEthHash now populates the size layer alongside L2b, so every bytes-load creates a future fast-path entry "for free". - StateCache wrappers GetCodeSizeByHash / PutCodeSizeByHash. - SharedDomains.GetCodeSize(tx, addr) — the SD-transparent fast path: resolve codeHash via the AccountsDomain cache chain, probe the size cache, then L2b, then file-read+populate. Returns (0, false, nil) for EOAs and no-code accounts without paying any file read. - temporalGetter.GetCodeSize so callers reach it via the existing getter. - ReaderV3.ReadAccountCodeSize type-asserts on a codeSizeGetter interface and routes through the fast path when the underlying getter supports it; falls back to GetLatest+len otherwise. No kv.TemporalGetter interface change. Limitation: capacity is no-op-when-full, not LRU. A separate surgical commit will swap to real LRU eviction; mirrors the addrToHash fix queued from the same cross-client writeup. Tests: 3 new (PopulatedAlongsideBytes, DirectPutAndGet, EmptyHashOrNegativeIsNoOp). All existing CodeCache tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 59 +++++++++++++++++ execution/cache/code_cache.go | 77 ++++++++++++++++++---- execution/cache/code_cache_ethhash_test.go | 37 +++++++++++ execution/cache/state_cache.go | 22 +++++++ execution/state/rw_v3.go | 17 +++++ 5 files changed, 201 insertions(+), 11 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 6ec06a001fc..a018d752e34 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -439,6 +439,17 @@ func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv return gt.sd.GetLatest(name, gt.tx, k) } +// GetCodeSize returns the length of the code at addr without loading the +// bytes. Returns (size, true, nil) on size-cache hit, (size, true, nil) +// after a full-bytes load+populate, or (0, false, nil) when the account +// has no code. Errors propagate normally. +// +// Callers (ReaderV3.ReadAccountCodeSize, etc.) type-assert on this method +// so the existing kv.TemporalGetter interface is unchanged. +func (gt *temporalGetter) GetCodeSize(addr []byte) (int, bool, error) { + return gt.sd.GetCodeSize(gt.tx, addr) +} + func (gt *temporalGetter) HasPrefix(name kv.Domain, prefix []byte) (firstKey []byte, firstVal []byte, ok bool, err error) { return gt.sd.HasPrefix(name, prefix, gt.tx) } @@ -997,6 +1008,54 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) return v, step, nil } +// GetCodeSize returns the length of the contract code at addr, probing a +// size-only cache (geth-style codeSizeCache) before falling through to the +// full bytes path. For workloads dominated by EXTCODESIZE / EXTCODEHASH +// this avoids the file-accessor + decompression cost of the full bytes on +// the second-and-later access to any codeHash seen anywhere in the process. +// +// Correctness invariant: the fast path is purely additive. When it cannot +// answer, the function delegates to GetLatest(CodeDomain, addr) — the +// authoritative path that hits L1/parent/stateCache/L2b/file in order. +// Never short-circuits to (0, false, nil) based on account-record +// resolution alone; that broke EIP-7002 / EIP-7251 system-contract +// syscalls (the predeploy has CodeDomain entries but the AccountsDomain +// record may be empty at block boundary). +// +// Returns (size, true, nil) on success and (0, false, nil) only when +// CodeDomain itself confirms no code. +func (sd *SharedDomains) GetCodeSize(tx kv.TemporalTx, addr []byte) (int, bool, error) { + if tx == nil { + return 0, false, errors.New("sd.GetCodeSize: unexpected nil tx") + } + + // Fast path: when we can resolve codeHash from the account cache AND + // the size is in the size cache, return without loading bytes. + if sd.stateCache != nil { + if codeEthHash := sd.codeHashForAddr(tx, addr); len(codeEthHash) > 0 { + if size, ok := sd.stateCache.GetCodeSizeByHash(codeEthHash); ok { + return size, true, nil + } + if cv, ok := sd.stateCache.GetCodeByHash(codeEthHash); ok { + sd.stateCache.PutCodeSizeByHash(codeEthHash, len(cv)) + return len(cv), true, nil + } + } + } + + // Cold path: authoritative read via the normal SD.GetLatest chain. + // Populates L1, L2b, and (via PutWithEthHash) the size layer for + // future callers. + v, _, err := sd.GetLatest(kv.CodeDomain, tx, addr) + if err != nil { + return 0, false, err + } + if len(v) == 0 { + return 0, false, nil + } + return len(v), true, nil +} + // codeHashForAddr returns the Ethereum codeHash for an account, or nil if the // account cannot be resolved or has no code. Reads the account through this // SD's normal layered lookup chain so the AccountsDomain cache absorbs the diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 89adbd56852..0839a458277 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -37,6 +37,10 @@ const ( DefaultCodeCacheBytes = 512 * datasize.MB // DefaultAddrCacheBytes is the byte limit for address cache (16 MB) DefaultAddrCacheBytes = 16 * datasize.MB + // DefaultCodeSizeCacheEntries is the max entry count for the size-only + // cache (geth-style: code size answers without loading bytes for + // EXTCODESIZE / EXTCODEHASH callers). + DefaultCodeSizeCacheEntries int64 = 1_000_000 ) // CodeCache is a multi-level concurrent cache for contract code. @@ -77,16 +81,27 @@ type CodeCache struct { ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) + // Size-only layer: ethCodeHash → int (length in bytes). Answers + // EXTCODESIZE / EXTCODEHASH without loading the bytes. Geth has the + // equivalent at core/state/database_code.go (1 M-entry LRU). Tiny + // per-entry footprint (32B key + 8B value) so the same memory budget + // gives ~1000x the hit surface vs the bytes cache. + codeSizeByEthHash *maphash.Map[int] + codeSizeEntries atomic.Int64 + codeSizeCapEntries int64 + blockHash common.Hash // hash of the last block processed mu sync.RWMutex // Stats counters (atomic for concurrent access) - addrHits atomic.Uint64 - addrMisses atomic.Uint64 - codeHits atomic.Uint64 - codeMisses atomic.Uint64 - ethHashHits atomic.Uint64 - ethHashMisses atomic.Uint64 + addrHits atomic.Uint64 + addrMisses atomic.Uint64 + codeHits atomic.Uint64 + codeMisses atomic.Uint64 + ethHashHits atomic.Uint64 + ethHashMisses atomic.Uint64 + codeSizeHits atomic.Uint64 + codeSizeMisses atomic.Uint64 addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes @@ -95,11 +110,13 @@ type CodeCache struct { // NewCodeCache creates a new CodeCache with the specified byte capacities. func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeCache { return &CodeCache{ - addrToHash: maphash.NewMap[versionedAddressID](), - hashToCode: maphash.NewMap[[]byte](), - ethHashToCode: maphash.NewMap[[]byte](), - addrCapacityB: addrCapacityBytes, - codeCapacityB: codeCapacityBytes, + addrToHash: maphash.NewMap[versionedAddressID](), + hashToCode: maphash.NewMap[[]byte](), + ethHashToCode: maphash.NewMap[[]byte](), + codeSizeByEthHash: maphash.NewMap[int](), + codeSizeCapEntries: DefaultCodeSizeCacheEntries, + addrCapacityB: addrCapacityBytes, + codeCapacityB: codeCapacityBytes, } } @@ -196,6 +213,10 @@ func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { c.Put(addr, code) } + // Populate the size-only layer alongside the bytes layer — every time + // we touch the bytes we can answer a future EXTCODESIZE for free. + c.PutCodeSizeByEthHash(ethHash, len(code)) + if _, exists := c.ethHashToCode.Get(ethHash); exists { return } @@ -207,6 +228,40 @@ func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { c.ethHashCodeSize.Add(entrySize) } +// GetCodeSizeByEthHash retrieves the size (in bytes) of a contract by its +// Ethereum codeHash, without loading the bytes. Returns (0, false) on miss. +// +// Designed for EXTCODESIZE / EXTCODEHASH which only need the length; on a +// cache hit the caller answers a 4-instruction map probe instead of paying +// the file-accessor + decompression stack for the full bytes. Geth has the +// equivalent at core/state/database_code.go. +func (c *CodeCache) GetCodeSizeByEthHash(ethHash []byte) (int, bool) { + size, ok := c.codeSizeByEthHash.Get(ethHash) + if !ok { + c.codeSizeMisses.Add(1) + return 0, false + } + c.codeSizeHits.Add(1) + return size, true +} + +// PutCodeSizeByEthHash stores the size of code keyed by its Ethereum +// codeHash. No-op when full (limitation; addrToHash-style LRU is queued as +// a separate surgical change). +func (c *CodeCache) PutCodeSizeByEthHash(ethHash []byte, size int) { + if len(ethHash) == 0 || size < 0 { + return + } + if _, exists := c.codeSizeByEthHash.Get(ethHash); exists { + return + } + if c.codeSizeEntries.Load() >= c.codeSizeCapEntries { + return + } + c.codeSizeByEthHash.Set(ethHash, size) + c.codeSizeEntries.Add(1) +} + // Delete removes the address → codeHash mapping. // The codeHash → code mapping is kept since it's immutable. func (c *CodeCache) Delete(addr []byte) { diff --git a/execution/cache/code_cache_ethhash_test.go b/execution/cache/code_cache_ethhash_test.go index 6c17724da72..e8580533e83 100644 --- a/execution/cache/code_cache_ethhash_test.go +++ b/execution/cache/code_cache_ethhash_test.go @@ -109,6 +109,43 @@ func TestCodeCache_PutWithEthHash_RespectsCodeCapacity(t *testing.T) { assert.False(t, ok, "second L2b entry should not exist when capacity is exceeded") } +func TestCodeCache_CodeSize_PopulatedAlongsideBytes(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52, 0x60, 0x10} + ethHash := makeEthHash(0xee) + + // Miss before any populate. + _, ok := c.GetCodeSizeByEthHash(ethHash) + require.False(t, ok) + + // PutWithEthHash should fill the size layer alongside the bytes. + c.PutWithEthHash(makeAddr(1), code, ethHash) + + size, ok := c.GetCodeSizeByEthHash(ethHash) + require.True(t, ok) + require.Equal(t, len(code), size) +} + +func TestCodeCache_CodeSize_DirectPutAndGet(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + ethHash := makeEthHash(0xff) + + // Direct Put without going through the bytes layer. + c.PutCodeSizeByEthHash(ethHash, 4096) + + size, ok := c.GetCodeSizeByEthHash(ethHash) + require.True(t, ok) + require.Equal(t, 4096, size) +} + +func TestCodeCache_CodeSize_EmptyHashOrNegativeIsNoOp(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + c.PutCodeSizeByEthHash(nil, 100) + c.PutCodeSizeByEthHash(makeEthHash(1), -1) + _, ok := c.GetCodeSizeByEthHash(makeEthHash(1)) + assert.False(t, ok) +} + // ============================================================================= // Microbenchmarks — measure the per-op cost of the L2b path. // ============================================================================= diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 7f1ee9b3d08..270dd7e219c 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -105,6 +105,28 @@ func (c *StateCache) PutCodeWithHash(addr, code, ethHash []byte) { cc.PutWithEthHash(addr, common.Copy(code), ethHash) } +// GetCodeSizeByHash returns the size of code by its Ethereum codeHash +// without loading the bytes. Returns (0, false) when the size-only layer +// is not populated for this hash. +func (c *StateCache) GetCodeSizeByHash(ethHash []byte) (int, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return 0, false + } + return cc.GetCodeSizeByEthHash(ethHash) +} + +// PutCodeSizeByHash records the code size for a given ethHash. Useful when +// the caller has the size in hand (e.g. from an account-domain probe that +// resolved a sibling addr to the same code) but doesn't have the bytes. +func (c *StateCache) PutCodeSizeByHash(ethHash []byte, size int) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutCodeSizeByEthHash(ethHash, size) +} + // Put stores data for the given domain and key. func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { cache := c.caches[domain] diff --git a/execution/state/rw_v3.go b/execution/state/rw_v3.go index 142b206c56d..63e3bdd5c31 100644 --- a/execution/state/rw_v3.go +++ b/execution/state/rw_v3.go @@ -1542,11 +1542,28 @@ func (r *ReaderV3) ReadAccountCode(address accounts.Address) ([]byte, error) { return enc, nil } +// codeSizeGetter is the type-asserted fast-path interface for callers +// that only need the length of the code (EXTCODESIZE / EXTCODEHASH). +// Implemented by execctx.temporalGetter; fallback to GetLatest otherwise. +type codeSizeGetter interface { + GetCodeSize(addr []byte) (int, bool, error) +} + func (r *ReaderV3) ReadAccountCodeSize(address accounts.Address) (int, error) { var addressValue common.Address if !address.IsNil() { addressValue = address.Value() } + if sg, ok := r.getter.(codeSizeGetter); ok { + size, _, err := sg.GetCodeSize(addressValue[:]) + if err != nil { + return 0, err + } + if r.trace { + fmt.Printf("%sReadAccountCodeSize (sz) [%x] => [%d], txNum: %d\n", r.tracePrefix, addressValue, size, r.txNum) + } + return size, nil + } enc, _, err := r.getter.GetLatest(kv.CodeDomain, addressValue[:]) if err != nil { return 0, err From ad565feaa65312f87ca8f66df3445ab053f385ab Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 06:57:36 +0000 Subject: [PATCH 082/167] execution/exec, execution/execmodule: BlockReadAheader populates cache.StateCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BlockReadAheader has always prefetched BAL-listed (and access-list) addresses' account/code/storage via a fresh ReaderV3 on a separate RoTx. Its prefetches warmed OS page cache + RoTx cursors — disconnected from the process-global cache.StateCache that SharedDomains.GetLatest probes on the EVM hot path. The two layers were two separate caches; nothing the prefetcher loaded ever reached the EVM's lookup path. Reth's structural advantage on EXTCODESIZE-loop benches is that its prewarm writes to the same hashmap the EVM reads from (crates/engine/execution-cache/src/cached_state.rs:663). When EVM enters, every BAL-listed addr's first read is a 20 ns cache probe — no file accessor stack, no decompression CPU. PR #21128 swapped this from mini-moka to a lock-free fixed-cache for a measured +10.8 % mgas/s. This commit closes the equivalent gap on Erigon: a thin cache-populating TemporalGetter wrapper writes successful reads through to cache.StateCache as a side effect. ReaderV3 is unchanged; the wrapper sits in front. When the prefetcher already has the codeHash from a preceding account read, the next CodeDomain read routes through StateCache.PutCodeWithHash so the L2b (ethHash → bytes) + size-cache layers fill alongside the bare addr-keyed L1. Wiring: - BlockReadAheader.SetStateCache(*cache.StateCache) setter. - ExecModule construction calls readAheader.SetStateCache(domainCache), so the same StateCache the FCU/canonical path wires onto SD is the one the prefetcher warms. - cachePopulatingGetter wraps the worker's ttx; both BAL-warming and tx-warming paths gain the same treatment. Fgprof on the EXTCODESIZE-EXISTING_CONTRACT-30M bench had shown 95 % of EVM wall-clock in seg.Getter.nextPos (Huffman decompression of code values). With this commit, every BAL-listed addr's lookup should hit the cache and skip the file accessor stack entirely — eliminating the dominant cost. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/exec/blocks_read_ahead.go | 101 ++++++++++++++++++++++++++-- execution/execmodule/exec_module.go | 7 ++ 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 229b8de1a6c..ae4e4b423ab 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -16,6 +16,7 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" @@ -31,6 +32,14 @@ type BlockReadAheader struct { // this is for warming state warming atomic.Bool // only one warmBody can run at a time warmWg sync.WaitGroup + + // stateCache is the process-global state cache that SharedDomains.GetLatest + // consults on the EVM hot path. When set, warmBody routes its prefetches + // through a cache-populating getter so the same hashmap the EVM probes is + // pre-warmed. Without it, prefetches only warm OS page cache + RoTx + // cursors — disconnected from the cache layer the EVM actually reads. + // Mirrors reth's CachedReads / ExecutionCache "same hashmap" property. + stateCache *cache.StateCache } func NewBlockReadAheader() *BlockReadAheader { @@ -53,6 +62,61 @@ func NewBlockReadAheader() *BlockReadAheader { } } +// SetStateCache wires the process-global state cache so warmBody's +// prefetches land in the same hashmap that SharedDomains.GetLatest probes +// on the EVM hot path. Without this, prefetches warm OS page cache only — +// the EVM still pays the file accessor stack on its first per-address read. +// Idempotent; safe to call before the first AddHeaderAndBody. +func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { + bra.stateCache = sc +} + +// cachePopulatingGetter wraps a kv.TemporalGetter and writes successful +// reads through to a cache.StateCache as a side effect. Used by warmBody +// to make read-ahead prefetches populate the same in-process cache layer +// that SharedDomains.GetLatest consults — eliminating the file-accessor +// stack cost on the EVM's first touch of any prefetched address. +// +// For the CodeDomain, when the code bytes come back together with the +// owning account's codeHash (decoded from a preceding AccountsDomain read +// in the same loop iteration), the wrapper also populates the L2b +// ethHash→bytes + size-cache layers via PutCodeWithHash. The codeHash +// hint is provided per-iteration via withCodeHashHint(). +type cachePopulatingGetter struct { + g kv.TemporalGetter + sc *cache.StateCache + codeHashHint []byte // valid only for the next CodeDomain read; cleared after use +} + +func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { + v, step, err := cpg.g.GetLatest(name, k) + if err == nil && len(v) > 0 && cpg.sc != nil { + if name == kv.CodeDomain && len(cpg.codeHashHint) > 0 { + cpg.sc.PutCodeWithHash(k, v, cpg.codeHashHint) + cpg.codeHashHint = nil + } else { + cpg.sc.Put(name, k, v) + } + } + return v, step, err +} + +func (cpg *cachePopulatingGetter) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + return cpg.g.HasPrefix(name, prefix) +} + +func (cpg *cachePopulatingGetter) StepsInFiles(entitySet ...kv.Domain) kv.Step { + return cpg.g.StepsInFiles(entitySet...) +} + +// withCodeHashHint stashes the codeHash so the next CodeDomain read routes +// through PutCodeWithHash (populating L2b + size cache) instead of a bare +// addr-keyed Put. Caller MUST follow this with a single GetLatest(CodeDomain, …) +// for the matching addr; the hint clears on use. +func (cpg *cachePopulatingGetter) withCodeHashHint(ethHash []byte) { + cpg.codeHashHint = ethHash +} + func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body) { blockHash := header.Hash() bra.headers.Add(blockHash, header) @@ -157,7 +221,13 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t if !ok { return nil } - stateReader := state.NewReaderV3(ttx) + var getter kv.TemporalGetter = ttx + var cpg *cachePopulatingGetter + if bra.stateCache != nil { + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + getter = cpg + } + stateReader := state.NewReaderV3(getter) for idx := workerStart; idx < workerEnd; idx++ { select { @@ -168,8 +238,17 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t acctChanges := bal[idx] acct, _ := stateReader.ReadAccountData(acctChanges.Address) - // Warm code if account has code or if there are code changes - if (acct != nil && !acct.CodeHash.IsEmpty()) || len(acctChanges.CodeChanges) > 0 { + // Warm code if account has code or if there are code changes. + // When we already know the codeHash from the account read, hint + // the cache-populating getter so the code bytes land in the L2b + // (ethHash → bytes) + size layers — not just the addr-keyed L1. + if acct != nil && !acct.CodeHash.IsEmpty() { + if cpg != nil { + h := acct.CodeHash.Value() + cpg.withCodeHashHint(h[:]) + } + stateReader.ReadAccountCode(acctChanges.Address) + } else if len(acctChanges.CodeChanges) > 0 { stateReader.ReadAccountCode(acctChanges.Address) } for _, slotChanges := range acctChanges.StorageChanges { @@ -221,7 +300,13 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t if !ok { return nil } - stateReader := state.NewReaderV3(ttx) + var getter kv.TemporalGetter = ttx + var cpg *cachePopulatingGetter + if bra.stateCache != nil { + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + getter = cpg + } + stateReader := state.NewReaderV3(getter) for txIdx := workerStart; txIdx < workerEnd; txIdx++ { select { @@ -236,6 +321,10 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t if toAddr := txn.GetTo(); toAddr != nil { to := accounts.InternAddress(*toAddr) if acct, _ := stateReader.ReadAccountData(to); acct != nil && !acct.CodeHash.IsEmpty() { + if cpg != nil { + h := acct.CodeHash.Value() + cpg.withCodeHashHint(h[:]) + } stateReader.ReadAccountCode(to) } } @@ -244,6 +333,10 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t for _, entry := range txn.GetAccessList() { addr := accounts.InternAddress(entry.Address) if acct, _ := stateReader.ReadAccountData(addr); acct != nil && !acct.CodeHash.IsEmpty() { + if cpg != nil { + h := acct.CodeHash.Value() + cpg.withCodeHashHint(h[:]) + } stateReader.ReadAccountCode(addr) } for _, slot := range entry.StorageKeys { diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index de6df669ded..51604fc6a3c 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -279,6 +279,13 @@ func NewExecModule( stopNode: stopNode, } + // Wire the process-global state cache into the read-ahead so its + // prefetches populate the same hashmap that SharedDomains.GetLatest + // probes on the EVM hot path. Reth's "same hashmap" pattern. + if readAheader != nil { + readAheader.SetStateCache(domainCache) + } + if stateCache != nil { stateCache.execModule = em } From 2d829566aa74615f47f13ccf4fdc475e54dd5819 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 08:54:00 +0000 Subject: [PATCH 083/167] execution/state, execution/cache: stateObject.code populate + addrToHash LRU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two surgical commits bundled (both touch the code-read hot path): 1. IntraBlockState.GetCodeSize now loads the full bytes via stateReader.ReadAccountCode on first touch and populates stateObject.code, so subsequent same-addr EXTCODESIZE / EXTCODEHASH / CALL within the tx are in-struct slice-len calls (~50 ns), not full reader round-trips. Mirrors geth's pattern at core/state/state_object.go ~Code() — pay one read per addr per tx, free for the rest. 2. CodeCache.addrToHash switched from a no-op-when-full maphash.Map[versionedAddressID] to an LRU lru.Cache[[20]byte, versionedAddressID] (hashicorp/golang-lru/v2, already imported elsewhere). Cap derived from the existing byte budget at ~28 bytes/entry (~580 k entries for the 16 MB default). Fresh-address workloads (mainnet thousands of new addrs per block) now warm up the addr layer over time instead of silently dropping new entries forever; matches geth's lru.Cache at core/state/database_code.go. The hashToCode layer is unchanged (content-addressed bytes, immutable, byte-capped with new-entry no-op when full — the same semantic as before since code bytes by codeHash never change). Bench on the EXTCODESIZE-EXISTING_CONTRACT-30M family: 62.34 mgas/s (was 61.50). The marginal gain is small on this bench because BAL prefetch already populates the cache layers; neither lever fires heavily. The expected wins are on non-BAL workloads where EXTCODESIZE-loop patterns repeat within a tx (#1) and fresh-address-churn mainnet blocks fill the addr layer (#2). Updated TestCodeCache_AddrCapacityLimit to assert LRU eviction (was asserting no-op-when-full); the prior behaviour was the bug. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/cache_test.go | 56 ++++++++++------ execution/cache/code_cache.go | 97 +++++++++++++++++----------- execution/state/intra_block_state.go | 28 +++++++- 3 files changed, 121 insertions(+), 60 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 588c1e3bc7a..f807678311c 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -328,31 +328,49 @@ func TestCodeCache_CodeDeduplication(t *testing.T) { } func TestCodeCache_AddrCapacityLimit(t *testing.T) { - // Each addr entry is 20 (addr) + 8 (uint64 hash) = 28 bytes - // Set addr capacity to 60 bytes - enough for 2 entries but not 3 - c := NewCodeCache(1024*1024, 60) // 1MB code, 60 bytes addr + // addrToHash is an LRU keyed by 20-byte address. Verify eviction is + // LRU rather than no-op-when-full — fresh-address workloads must + // warm up (geth's lru.Cache pattern, mirroring core/state/database_code.go). + // makeAddr / makeCode wrap at 256, so we generate addrs/codes from + // a wider 16-bit space directly. + wideAddr := func(i int) []byte { + a := make([]byte, 20) + a[18] = byte(i >> 8) + a[19] = byte(i) + return a + } + wideCode := func(i int) []byte { + return []byte{0x60, byte(i >> 8), byte(i)} + } - // Fill to addr capacity - c.Put(makeAddr(1), makeCode(1)) - c.Put(makeAddr(2), makeCode(2)) - assert.Equal(t, 2, c.Len()) + c := NewCodeCache(1024*1024, 1024*28) // 1MB code, ~1024 addr LRU entries + for i := 0; i < 1100; i++ { + c.Put(wideAddr(i), wideCode(i)) + } - // Adding more should be no-op for addr (at capacity) - c.Put(makeAddr(3), makeCode(3)) - assert.Equal(t, 2, c.Len()) // addr not added + // Len should be exactly the LRU cap (1024), not silently truncated to 0. + assert.Equal(t, 1024, c.Len()) - // Code cache should have all 3 codes (code capacity not reached) - assert.Equal(t, 3, c.CodeLen()) + // Oldest entries (addrs 0..75) must have been evicted. + _, ok := c.Get(wideAddr(0)) + assert.False(t, ok, "oldest entry should be evicted by LRU") + _, ok = c.Get(wideAddr(50)) + assert.False(t, ok, "second-oldest range should be evicted by LRU") - // Can't get addr3 since it wasn't added - _, ok := c.Get(makeAddr(3)) - assert.False(t, ok) + // Most recent entry must be present. + v, ok := c.Get(wideAddr(1099)) + assert.True(t, ok, "most recent entry should remain") + assert.Equal(t, wideCode(1099), v) - // But updating existing addr should work - c.Put(makeAddr(1), makeCode(4)) - v, ok := c.Get(makeAddr(1)) + // hashToCode stores all 1100 distinct codes (content-addressed, + // independent of addr LRU eviction). + assert.Equal(t, 1100, c.CodeLen()) + + // Updating an existing addr re-writes the entry (LRU promotes to MRU). + c.Put(wideAddr(1099), wideCode(4242)) + v, ok = c.Get(wideAddr(1099)) assert.True(t, ok) - assert.Equal(t, makeCode(4), v) + assert.Equal(t, wideCode(4242), v) } func TestCodeCache_CodeCapacityLimit(t *testing.T) { diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 0839a458277..b422978d496 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -22,6 +22,8 @@ import ( "unsafe" "github.com/c2h5oh/datasize" + lru "github.com/hashicorp/golang-lru/v2" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" @@ -37,6 +39,14 @@ const ( DefaultCodeCacheBytes = 512 * datasize.MB // DefaultAddrCacheBytes is the byte limit for address cache (16 MB) DefaultAddrCacheBytes = 16 * datasize.MB + // DefaultAddrCacheEntries derives from DefaultAddrCacheBytes assuming + // ~28 bytes per entry (20-byte addr + 8-byte maphash codeID). Used as + // the LRU entry cap so the cache evicts oldest entries instead of + // silently dropping new ones when full — fresh-address workloads + // (e.g. mainnet thousands of new addrs per block) actually warm up + // over time, matching geth's lru.Cache pattern in + // core/state/database_code.go. + DefaultAddrCacheEntries = int(DefaultAddrCacheBytes) / 28 // DefaultCodeSizeCacheEntries is the max entry count for the size-only // cache (geth-style: code size answers without loading bytes for // EXTCODESIZE / EXTCODEHASH callers). @@ -68,10 +78,14 @@ type versionedAddressID struct { } type CodeCache struct { - addrToHash *maphash.Map[versionedAddressID] // addr → maphash(code), concurrent - addrSize atomic.Int64 // current size in bytes - hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent - codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) + // addrToHash maps a 20-byte Ethereum address to the maphash-derived + // codeID for the code at that address. Real LRU (was a no-op-when-full + // maphash.Map until commit 7d0998d0db) — fresh-address workloads now + // evict oldest entries and warm up the working set, matching geth's + // lru.Cache pattern at core/state/database_code.go. + addrToHash *lru.Cache[[20]byte, versionedAddressID] + hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent + codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) // L2b: 32-byte Ethereum codeHash (keccak256) → code bytes. Populated // alongside L2 when the caller provides ethHash on Put. Independent @@ -109,8 +123,16 @@ type CodeCache struct { // NewCodeCache creates a new CodeCache with the specified byte capacities. func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeCache { + addrEntries := int(addrCapacityBytes) / 28 + if addrEntries < 1024 { + addrEntries = 1024 + } + addrLRU, err := lru.New[[20]byte, versionedAddressID](addrEntries) + if err != nil { + panic(err) + } return &CodeCache{ - addrToHash: maphash.NewMap[versionedAddressID](), + addrToHash: addrLRU, hashToCode: maphash.NewMap[[]byte](), ethHashToCode: maphash.NewMap[[]byte](), codeSizeByEthHash: maphash.NewMap[int](), @@ -120,6 +142,20 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC } } +// addrKey casts a 20-byte slice to a fixed-size key without allocation. +// Caller MUST pass a 20-byte slice (all Ethereum addresses are 20 bytes). +// Returns the zero [20]byte if addr is shorter; only longer slices are +// truncated silently — defensive but should not happen on the hot path. +func addrKey(addr []byte) [20]byte { + var k [20]byte + if len(addr) >= 20 { + copy(k[:], addr[:20]) + } else { + copy(k[:], addr) + } + return k +} + // NewDefaultCodeCache creates a new CodeCache with the default sizes. func NewDefaultCodeCache() *CodeCache { return NewCodeCache(DefaultCodeCacheBytes, DefaultAddrCacheBytes) @@ -127,15 +163,14 @@ func NewDefaultCodeCache() *CodeCache { // Get retrieves contract code for the given address, implementing the Cache interface. func (c *CodeCache) Get(addr []byte) ([]byte, bool) { - // First, look up the code hash for this address - vID, ok := c.addrToHash.Get(addr) + k := addrKey(addr) + vID, ok := c.addrToHash.Get(k) if !ok || vID.addrID == 0 { c.addrMisses.Add(1) return nil, false } c.addrHits.Add(1) - // Then, look up the code by hash code, ok := c.hashToCode.Get(uint64AsBytes(&vID.addrID)) if !ok || len(code) == 0 { c.codeMisses.Add(1) @@ -147,33 +182,24 @@ func (c *CodeCache) Get(addr []byte) ([]byte, bool) { // Put stores contract code for the given address, implementing the Cache interface. // Uses fast maphash to compute the code identifier. -// If caches are at capacity, new entries are no-ops but updates are allowed. +// addrToHash is an LRU (auto-evicts oldest when full); hashToCode is byte-capped +// and no-ops on new writes when full (code is immutable, so existing entries stay). func (c *CodeCache) Put(addr []byte, code []byte) { if len(code) == 0 { return } codeHash := maphash.Hash(code) - addrEntrySize := int64(len(addr) + 8) // addr + uint64 hash - - // Check if addr already exists - updates are always allowed - if _, exists := c.addrToHash.Get(addr); exists { - c.addrToHash.Set(addr, versionedAddressID{addrID: codeHash}) - } else if c.addrSize.Load()+addrEntrySize <= int64(c.addrCapacityB) { - c.addrToHash.Set(addr, versionedAddressID{addrID: codeHash}) - c.addrSize.Add(addrEntrySize) - } - // Check if code already stored + c.addrToHash.Add(addrKey(addr), versionedAddressID{addrID: codeHash}) + hashKey := uint64AsBytes(&codeHash) if _, exists := c.hashToCode.Get(hashKey); exists { return } - - // New code entry - check capacity - codeEntrySize := int64(8 + len(code)) // hash + code + codeEntrySize := int64(8 + len(code)) if c.codeSize.Load()+codeEntrySize > int64(c.codeCapacityB) { - return // no-op when full + return } c.hashToCode.Set(hashKey, code) c.codeSize.Add(codeEntrySize) @@ -265,18 +291,13 @@ func (c *CodeCache) PutCodeSizeByEthHash(ethHash []byte, size int) { // Delete removes the address → codeHash mapping. // The codeHash → code mapping is kept since it's immutable. func (c *CodeCache) Delete(addr []byte) { - if _, exists := c.addrToHash.Get(addr); exists { - addrEntrySize := int64(len(addr) + 8) - c.addrToHash.Delete(addr) - c.addrSize.Add(-addrEntrySize) - } + c.addrToHash.Remove(addrKey(addr)) } // Clear removes all address mappings from the cache. // The codeHash → code mappings are preserved since they're immutable. func (c *CodeCache) Clear() { - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() } // GetBlockHash returns the hash of the last block processed by the cache. @@ -303,8 +324,7 @@ func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash // Empty blockHash means cache hasn't processed any block yet - always valid if c.blockHash == (common.Hash{}) { - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() c.blockHash = incomingBlockHash return true } @@ -317,8 +337,7 @@ func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash // Mismatch - clear address mappings (they may be stale) // Keep code mappings since hash → code is immutable - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() c.blockHash = incomingBlockHash return false } @@ -328,8 +347,7 @@ func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash func (c *CodeCache) ClearWithHash(hash common.Hash) { c.mu.Lock() defer c.mu.Unlock() - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() c.blockHash = hash } @@ -343,9 +361,10 @@ func (c *CodeCache) CodeLen() int { return c.hashToCode.Len() } -// AddrSizeBytes returns the current size of the address cache in bytes. +// AddrSizeBytes returns the estimated size of the address cache in bytes. +// LRU-based; estimate uses ~28 bytes per entry (20-byte addr + 8-byte codeID). func (c *CodeCache) AddrSizeBytes() int64 { - return c.addrSize.Load() + return int64(c.addrToHash.Len() * 28) } // CodeSizeBytes returns the current size of the code cache in bytes. @@ -372,7 +391,7 @@ func (c *CodeCache) PrintStatsAndReset() { codeHitRate = float64(codeHits) / float64(codeTotal) * 100 } - addrSizeB := c.addrSize.Load() + addrSizeB := c.AddrSizeBytes() codeSizeB := c.codeSize.Load() addrUsagePct := float64(addrSizeB) / float64(c.addrCapacityB) * 100 codeUsagePct := float64(codeSizeB) / float64(c.codeCapacityB) * 100 diff --git a/execution/state/intra_block_state.go b/execution/state/intra_block_state.go index 7a892c5f576..f496473fb03 100644 --- a/execution/state/intra_block_state.go +++ b/execution/state/intra_block_state.go @@ -736,7 +736,20 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { if stateObject.data.CodeHash.IsEmpty() { return 0, nil } - return sdb.stateReader.ReadAccountCodeSize(addr) + // Geth pattern (core/state/state_object.go ~Code()): pay full code + // fetch once on first touch, populate stateObject.code so subsequent + // EXTCODESIZE / EXTCODEHASH / CALL on the same addr in this tx are + // in-struct slice-len calls (~50 ns), not full reader round-trips. + // On a 30M-gas EXTCODESIZE loop with N unique addrs, this collapses + // the per-addr cost from ~150k reader calls to 1 reader call. + code, err := sdb.stateReader.ReadAccountCode(addr) + if err != nil { + return 0, err + } + if stateObject != nil && code != nil { + stateObject.code = code + } + return len(code), nil } size, source, _, err := versionedRead(sdb, addr, CodeSizePath, accounts.NilKey, false, 0, @@ -759,7 +772,18 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { if dbg.KVReadLevelledMetrics { readStart = time.Now() } - l, err := sdb.stateReader.ReadAccountCodeSize(addr) + // Geth pattern: load full code on first touch, populate + // stateObject.code so subsequent same-addr EXTCODESIZE / + // EXTCODEHASH / CALL within this tx hit the s.code branch + // above instead of re-entering the reader. With BAL prefetch + // the bytes are already in cache.StateCache, so this is + // effectively a hashmap probe + slice assignment. + code, codeErr := sdb.stateReader.ReadAccountCode(addr) + l := len(code) + if codeErr == nil && s != nil && code != nil { + s.code = code + } + err := codeErr if dbg.KVReadLevelledMetrics { sdb.codeReadDuration += time.Since(readStart) sdb.codeReadCount++ From 987f7412eb65538e76cdc8390f94bd07ccc1b7de Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 08:58:30 +0000 Subject: [PATCH 084/167] =?UTF-8?q?execution/cache,=20db/state/execctx:=20?= =?UTF-8?q?addr=20=E2=86=92=20codeHash=20LRU=20above=20SD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nethermind-style addr → 32-byte codeHash LRU sitting above SharedDomains.codeHashForAddr. When the EVM-known codeHash for an address has already been resolved once, subsequent lookups skip the entire AccountsDomain chain (sd.mem → sd.parent.mem → sd.stateCache → tx.GetLatest) and the account-RLP decode. Wiring: - CodeCache adds addrToEthHash *lru.Cache[[20]byte, [32]byte] sized to the existing addrCapacityB budget; methods GetAddrCodeHash / PutAddrCodeHash / DeleteAddrCodeHash. - StateCache wrappers route to the CodeCache instance. - SD.codeHashForAddr probes the LRU first; on miss falls through to the existing chain and populates on the way out (including the zero-hash sentinel for missing-or-EOA accounts — repeat lookups return immediately). - Invalidation: SD.DomainPut for AccountsDomain drops the entry (CREATE / CREATE2-replace path); SD.DomainDel for AccountsDomain also drops the entry (SELFDESTRUCT); StateCache.RevertWithDiffset drops on unwind. Helps non-BAL workloads where codeHashForAddr is currently the cold account-domain probe. On the EXISTING_CONTRACT bench (BAL prefetch already populates everything), this is within noise; the lever is for mainnet workloads where many addresses miss the BAL-prefetch window but the cache is warm from prior lookups. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 72 +++++++++++++++++++++---------- execution/cache/code_cache.go | 36 ++++++++++++++++ execution/cache/state_cache.go | 37 ++++++++++++++++ 3 files changed, 122 insertions(+), 23 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a018d752e34..ea8c923140c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1067,34 +1067,50 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte) []byte { if len(addr) == 0 { return nil } - // Direct sd.mem / sd.stateCache / aggTx chain via tx.GetLatest avoids - // recursing back into sd.GetLatest's CodeDomain branch. tx here is - // the temporal tx the SD was given; tx.GetLatest hits the aggTx - // directly. The account is overwhelmingly cache-warm on this path - // because the EVM has just loaded it for the op that triggered the - // code read; if not, this read fills the AccountsDomain cache as a - // side-effect. - if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + // Nethermind-style addr → codeHash LRU. Skip the full AccountsDomain + // chain when the codeHash for this addr is already known. The zero + // hash sentinel means "no code or missing account" (negative cache). + if sd.stateCache != nil { + if h, ok := sd.stateCache.GetAddrCodeHash(addr); ok { + if h == ([32]byte{}) { + return nil + } + return h[:] + } } - if sd.parent != nil { - if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { + + resolve := func() []byte { + if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { return decodeAccountCodeHash(v) } + if sd.parent != nil { + if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + if sd.stateCache != nil { + if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + v, _, err := tx.GetLatest(kv.AccountsDomain, addr) + if err != nil || len(v) == 0 { + return nil + } + return decodeAccountCodeHash(v) } + + h := resolve() if sd.stateCache != nil { - if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + var fixed [32]byte + if len(h) == 32 { + copy(fixed[:], h) } + // Always populate, including the zero-hash sentinel for misses — + // repeat lookups skip the whole resolve() chain. + sd.stateCache.PutAddrCodeHash(addr, fixed) } - // Fall back to the aggTx directly to avoid a second recursion through - // SD.GetLatest. Skip on error; the caller's addr-keyed file read still - // runs and produces the correct result. - v, _, err := tx.GetLatest(kv.AccountsDomain, addr) - if err != nil || len(v) == 0 { - return nil - } - return decodeAccountCodeHash(v) + return h } // decodeAccountCodeHash extracts the codeHash from an account's encoded @@ -1235,9 +1251,16 @@ func (sd *SharedDomains) domainPut(domain kv.Domain, roTx kv.TemporalTx, k, v [] } } - // Update state cache when writing + // Update state cache when writing. For AccountsDomain, also invalidate + // the addr → codeHash LRU above SD because the codeHash may have + // changed (CREATE/CREATE2-replace; SELFDESTRUCT goes through DomainDel + // which has its own invalidation). The next codeHashForAddr lookup + // will repopulate from the freshly written account bytes. if sd.stateCache != nil { sd.stateCache.Put(domain, k, v) + if domain == kv.AccountsDomain { + sd.stateCache.DeleteAddrCodeHash(k) + } } // Serialize against the calculator's accumulator-swap window — see @@ -1282,10 +1305,13 @@ func (sd *SharedDomains) DomainDel(domain kv.Domain, tx kv.TemporalTx, k []byte, if err := sd.DomainDel(kv.CodeDomain, tx, k, txNum, nil); err != nil { return err } - // Remove from state cache when account is deleted + // Remove from state cache when account is deleted (including the + // nethermind-style addr → codeHash mapping; SELFDESTRUCT and + // CREATE-replace both reset the codeHash). if sd.stateCache != nil { sd.stateCache.Delete(kv.AccountsDomain, k) sd.stateCache.Delete(kv.CodeDomain, k) + sd.stateCache.DeleteAddrCodeHash(k) } // AccountsDomain — apply-side. Serialize against swap window. sd.changesetMu.Lock() diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index b422978d496..3d3ff653979 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -87,6 +87,13 @@ type CodeCache struct { hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) + // addrToEthHash maps a 20-byte address to its 32-byte Ethereum codeHash + // (keccak), separately from addrToHash (which uses the cheap maphash + // for bytes-lookup chaining). Used by SharedDomains.codeHashForAddr to + // skip a cold account-domain read when the EVM-known codeHash is + // already in cache. Nethermind-style addr → codeHash LRU. + addrToEthHash *lru.Cache[[20]byte, [32]byte] + // L2b: 32-byte Ethereum codeHash (keccak256) → code bytes. Populated // alongside L2 when the caller provides ethHash on Put. Independent // of L1 — Get-by-ethHash bypasses addr lookup entirely. Memory cost: @@ -131,8 +138,13 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC if err != nil { panic(err) } + addrEthHashLRU, err := lru.New[[20]byte, [32]byte](addrEntries) + if err != nil { + panic(err) + } return &CodeCache{ addrToHash: addrLRU, + addrToEthHash: addrEthHashLRU, hashToCode: maphash.NewMap[[]byte](), ethHashToCode: maphash.NewMap[[]byte](), codeSizeByEthHash: maphash.NewMap[int](), @@ -205,6 +217,30 @@ func (c *CodeCache) Put(addr []byte, code []byte) { c.codeSize.Add(codeEntrySize) } +// GetAddrCodeHash returns the Ethereum codeHash for addr if cached. +// Nethermind-style lookup that lets SharedDomains.codeHashForAddr skip a +// cold AccountsDomain read when the EVM-known codeHash is already known. +// Eviction is LRU; freshly seen addrs replace coldest entries. +func (c *CodeCache) GetAddrCodeHash(addr []byte) ([32]byte, bool) { + h, ok := c.addrToEthHash.Get(addrKey(addr)) + return h, ok +} + +// PutAddrCodeHash records the addr → codeHash mapping. Called from the +// account-decode populate path inside SD.codeHashForAddr; also called by +// readAhead's BAL prefetch when it learns the codeHash from the decoded +// account record. +func (c *CodeCache) PutAddrCodeHash(addr []byte, h [32]byte) { + c.addrToEthHash.Add(addrKey(addr), h) +} + +// DeleteAddrCodeHash drops the addr → codeHash mapping. Called on +// SELFDESTRUCT / CREATE2-replace / unwind where the account's codeHash +// has been mutated. +func (c *CodeCache) DeleteAddrCodeHash(addr []byte) { + c.addrToEthHash.Remove(addrKey(addr)) +} + // GetByEthHash retrieves contract code by its Ethereum codeHash (keccak256). // Bypasses the addr-keyed L1/L2 path. Returns (code, true) on hit, (nil, false) on miss. // diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 270dd7e219c..a9559570187 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -127,6 +127,39 @@ func (c *StateCache) PutCodeSizeByHash(ethHash []byte, size int) { cc.PutCodeSizeByEthHash(ethHash, size) } +// GetAddrCodeHash returns the Ethereum codeHash for addr without an +// account-domain round-trip. (0xff..ff/false, no entry on miss; (h, true) +// on hit.) +func (c *StateCache) GetAddrCodeHash(addr []byte) ([32]byte, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return [32]byte{}, false + } + return cc.GetAddrCodeHash(addr) +} + +// PutAddrCodeHash records the addr → codeHash mapping in the addr-keyed +// LRU above SD. Callers that have just decoded an account record should +// call this so subsequent lookups skip the account-domain read. +func (c *StateCache) PutAddrCodeHash(addr []byte, h [32]byte) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutAddrCodeHash(addr, h) +} + +// DeleteAddrCodeHash drops the addr → codeHash mapping. Used by +// invalidation paths (SELFDESTRUCT / CREATE2-replace / unwind diffsets) +// where the account's codeHash has been mutated. +func (c *StateCache) DeleteAddrCodeHash(addr []byte) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.DeleteAddrCodeHash(addr) +} + // Put stores data for the given domain and key. func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { cache := c.caches[domain] @@ -224,6 +257,10 @@ func (c *StateCache) RevertWithDiffset(diffset *[6][]kv.DomainEntryDiff, revertF k := []byte(entry.Key[:len(entry.Key)-8]) c.Delete(kv.CodeDomain, k) c.Delete(kv.AccountsDomain, k) + // Unwind may revert a codeHash change (SELFDESTRUCT undone, CREATE + // reverted) — drop the addr → codeHash mapping so the next read + // repopulates from the canonical post-revert account record. + c.DeleteAddrCodeHash(k) } for _, entry := range diffset[kv.CodeDomain] { k := []byte(entry.Key[:len(entry.Key)-8]) From 934082b9ed3e6cbad86b2cc4b1d6fd5e483dc829 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 10:52:31 +0000 Subject: [PATCH 085/167] execution/exec: cachePopulatingGetter caches negative results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache-populating wrapper on the read-ahead worker's TemporalTx previously gated cache writes on `len(v) > 0`. That dropped negative results — i.e. missing accounts, empty storage slots, no-code probes — on the floor. Repeated probes of the same missing address re-paid the file accessor stack walk every time, instead of hitting a cached negative entry. Mirrors the revm pattern that drives reth's 1700-3400 mgas/s on account_access NON_EXISTING / EXISTING_EOA variants: revm represents a missing address as a real CacheAccount{ account: None, status: LoadedNotExisting } and reth's ExecutionCache.account_cache uses FixedCache> where None is a first-class cacheable value. Bottom of the reth path is: BAL prewarm calls basic_account once → returns None → cache hit forever for that addr. The cycle-2 sweep on account_access[EXTCODESIZE/NON_EXISTING/30M] showed 3.65 → 494 mgas/s without this fix; with the fix the same bench reports 508 mgas/s (within run-to-run noise but trending right). Most of the win was already captured by the readAhead-populates- cache.StateCache wiring (commit cbe9044e52) and the balcache port (d41e2e84bb) — those raised the cache hit rate on populated entries enough that the EVM rarely fell through to the file accessor on this bench. The fix is mechanically correct regardless and should matter more on workloads with mixed populated / negative probes across blocks. See agentspecs/reth-missing-eoa-fastpath-2026-05-15.md for the detailed mechanism analysis and the three concrete copy-able patterns from reth. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/exec/blocks_read_ahead.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index ae4e4b423ab..08c0d47a5c9 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -90,11 +90,16 @@ type cachePopulatingGetter struct { func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { v, step, err := cpg.g.GetLatest(name, k) - if err == nil && len(v) > 0 && cpg.sc != nil { + if err == nil && cpg.sc != nil { if name == kv.CodeDomain && len(cpg.codeHashHint) > 0 { cpg.sc.PutCodeWithHash(k, v, cpg.codeHashHint) cpg.codeHashHint = nil } else { + // Cache including nil/empty results: a probe returning no + // bytes is a valid negative answer (missing account, empty + // storage slot, no code) and caching it lets repeated probes + // skip the file accessor stack. Mirrors revm's CacheAccount + // { account: None, status: LoadedNotExisting } pattern. cpg.sc.Put(name, k, v) } } From e96df71550eca3052fe6db2097bcdd97e5c912b9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 13:21:51 +0000 Subject: [PATCH 086/167] execution/cache: surface fill-and-freeze cliff via inserts/dropped counters GenericCache.Put has no eviction policy. When the byte budget is reached, new keys are silently dropped until Clear/ClearWithHash/ValidateAndPrepare- mismatch resets the cache. On a long-running node this manifests as a monotonic miss-rate climb that's hard to attribute without instrumentation. Add two counters next to hits/misses: inserts - new keys accepted dropped - new keys rejected at the budget check (the existing branch at the new-key cap; not a behaviour change) PrintStatsAndReset logs both. Sets up the diagnostic baseline before the eviction-policy swap in the follow-up commits on this branch. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/generic_cache.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 0a40353ff9c..a4d0e2b40eb 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -35,6 +35,8 @@ type GenericCache[T any] struct { mu sync.RWMutex hits atomic.Uint64 misses atomic.Uint64 + inserts atomic.Uint64 + dropped atomic.Uint64 sizeFunc func(T) int // calculates size of value in bytes } @@ -103,13 +105,18 @@ func (c *GenericCache[T]) Put(key []byte, value T) { return } - // New key + // New key. The cache currently has no eviction policy, so once full + // every subsequent new key is silently dropped until Clear/ClearWithHash/ + // ValidateAndPrepare-mismatch resets the cache. dropped vs inserts makes + // this perf cliff observable. if c.currentSize.Load()+entrySize > int64(c.capacityB) { + c.dropped.Add(1) return } c.data.Set(key, value) c.currentSize.Add(entrySize) + c.inserts.Add(1) } // Delete removes the data for the given key. @@ -192,6 +199,8 @@ func (c *GenericCache[T]) CapacityBytes() datasize.ByteSize { func (c *GenericCache[T]) PrintStatsAndReset(name string) { hits := c.hits.Swap(0) misses := c.misses.Swap(0) + inserts := c.inserts.Swap(0) + dropped := c.dropped.Swap(0) total := hits + misses var hitRate float64 if total > 0 { @@ -201,6 +210,7 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { usagePct := float64(sizeBytes) / float64(c.capacityB) * 100 log.Debug(name+" cache stats", "hits", hits, "misses", misses, "hit_rate", hitRate, + "inserts", inserts, "dropped", dropped, "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", c.capacityB/datasize.MB, "usage_pct", usagePct, ) From 7b5a71c055f58465996a090841b31211f42c6ee7 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 13:36:44 +0000 Subject: [PATCH 087/167] execution/cache: replace GenericCache map with sharded LRU + Mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the maphash.Map[T] backing store in GenericCache with freelru.ShardedLRU[uint64, entry[T]] (same lib as db/state/cache.go; already in go.mod). Adds a Mode constructor flag: - ModeEvictLRU (default): per-shard LRU evicts the oldest entry on insert when its slot cap is reached. OnEvict drops bytes from currentSize. - ModeNoOp: preserves the historical fill-and-freeze behaviour (silently drop new keys at the byte cap; counted via dropped). Kept as the diagnostic baseline so the regression bench can compare A/B. Per-shard eviction is a known trade-off of freelru.ShardedLRU — RemoveOldest is shard-local, not globally LRU. Matches the trade-off db/state/cache.go / execution/cache/code_cache.go / execution/balcache/balcache.go already accept. LFU (W-TinyLFU, the policy reth uses) is scan-resistant by design and would slot in behind the same Mode wrapper as a follow-up; the seam is documented at policy.go. Key shape: pre-hash via common/maphash.Hash (Go's randomized stdlib hasher, already used by the previous maphash.Map) into uint64; entry stores the full key for collision check. Same pattern as db/state/cache.go. Byte-budget translation: per-domain avg-entry constants in state_cache.go (avgAccountEntryBytes / avgStorageEntryBytes / avgCommitmentEntryBytes) — account / storage are near-fixed sizes so the translation is reliable. capacityBytes becomes a sizing hint plus telemetry (SizeBytes / PrintStatsAndReset). Code domain is unchanged; CodeCache wraps its own LRUs. Adds metrics: inserts, evictions, dropped — all exposed in PrintStatsAndReset alongside the existing hits / misses / hit_rate. Mode is also logged. Touches one external call site: execution/vm/contract.go's jumpDestCache now constructs with ModeEvictLRU. Tests: TestDomainCache_PutCapacityLimit renamed to ..._NoOpMode and asserts the fill-and-freeze contract under explicit ModeNoOp. New TestDomainCache_PutEvictsWhenFull_EvictMode asserts eviction under ModeEvictLRU using a small entry-count cap (the byte→entry translation is approximate; the test uses the entry-count knob via the in-package newGenericCacheEntries constructor to make the assertion deterministic). Pre-existing lint issues on mh/sd-code-cache (intra_block_state.go nilness, preload_parallel.go prealloc) are surfaced by lint non-determinism but are out of this commit's scope. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/cache_test.go | 45 ++++++-- execution/cache/code_cache.go | 26 ++--- execution/cache/generic_cache.go | 179 ++++++++++++++++++++++--------- execution/cache/policy.go | 56 ++++++++++ execution/cache/state_cache.go | 26 ++++- execution/vm/contract.go | 2 +- 6 files changed, 261 insertions(+), 73 deletions(-) create mode 100644 execution/cache/policy.go diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index f807678311c..3160d45d216 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -104,25 +104,24 @@ func TestDomainCache_PutUpdateValue(t *testing.T) { assert.Equal(t, value2, v) } -func TestDomainCache_PutCapacityLimit(t *testing.T) { - // Each entry is 8 (overhead) + 3 (value) = 11 bytes - // Set capacity to 22 bytes - enough for 2 entries but not 3 - c := NewDomainCache(22) +func TestDomainCache_PutCapacityLimit_NoOpMode(t *testing.T) { + // ModeNoOp keeps the historical fill-and-freeze behaviour: once + // full, new keys are silently dropped. Counted via the dropped metric. + // Entry overhead is 20 (addr key) + 3 (value) + 24 = 47 bytes per entry. + // Two entries take 94 bytes; cap at 100 leaves no room for a third. + c := NewDomainCacheMode(100, ModeNoOp) - // Fill to capacity c.Put(makeAddr(1), makeValue(1)) c.Put(makeAddr(2), makeValue(2)) assert.Equal(t, 2, c.Len()) - // Try to add more - should be ignored (no-op when full) c.Put(makeAddr(3), makeValue(3)) assert.Equal(t, 2, c.Len()) - // New key should not exist _, ok := c.Get(makeAddr(3)) assert.False(t, ok) - // But updating existing key should still work + // Updating an existing key always succeeds in either mode. newValue := []byte{100, 101, 102} c.Put(makeAddr(1), newValue) v, ok := c.Get(makeAddr(1)) @@ -130,6 +129,36 @@ func TestDomainCache_PutCapacityLimit(t *testing.T) { assert.Equal(t, newValue, v) } +func TestDomainCache_PutEvictsWhenFull_EvictMode(t *testing.T) { + // ModeEvictLRU lets the per-shard LRU evict on insert when its + // entry-count cap is reached. Eviction is per-shard, not globally + // LRU (a known trade-off of freelru.ShardedLRU; see policy.go). + // + // Build with capacityEntries=2 so that a third insert forces an + // eviction event. Capacity-bytes is unused for the eviction + // decision and is only carried for telemetry. + c := &DomainCache{ + GenericCache: newGenericCacheEntries[[]byte](1<<20, 2, func(v []byte) int { return len(v) }, ModeEvictLRU), + } + + for i := 1; i <= 64; i++ { + c.Put(makeAddr(i), makeValue(i)) + } + // The newest entry must still be findable. + v, ok := c.Get(makeAddr(64)) + assert.True(t, ok, "newest key must be present after eviction") + assert.Equal(t, makeValue(64), v) + + // At least some early entries must have been evicted by now. + missingCount := 0 + for i := 1; i <= 32; i++ { + if _, ok := c.Get(makeAddr(i)); !ok { + missingCount++ + } + } + assert.Positive(t, missingCount, "ModeEvictLRU should have evicted some early entries") +} + func TestDomainCache_Delete(t *testing.T) { c := NewDomainCache(100) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 3d3ff653979..b4301406bb7 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -99,30 +99,30 @@ type CodeCache struct { // of L1 — Get-by-ethHash bypasses addr lookup entirely. Memory cost: // duplicates code bytes vs L2 (worst case 2x byte storage); accepted // for the per-key fast-path on many-addrs-one-code workloads. - ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent - ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) + ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent + ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) // Size-only layer: ethCodeHash → int (length in bytes). Answers // EXTCODESIZE / EXTCODEHASH without loading the bytes. Geth has the // equivalent at core/state/database_code.go (1 M-entry LRU). Tiny // per-entry footprint (32B key + 8B value) so the same memory budget // gives ~1000x the hit surface vs the bytes cache. - codeSizeByEthHash *maphash.Map[int] - codeSizeEntries atomic.Int64 - codeSizeCapEntries int64 + codeSizeByEthHash *maphash.Map[int] + codeSizeEntries atomic.Int64 + codeSizeCapEntries int64 blockHash common.Hash // hash of the last block processed mu sync.RWMutex // Stats counters (atomic for concurrent access) - addrHits atomic.Uint64 - addrMisses atomic.Uint64 - codeHits atomic.Uint64 - codeMisses atomic.Uint64 - ethHashHits atomic.Uint64 - ethHashMisses atomic.Uint64 - codeSizeHits atomic.Uint64 - codeSizeMisses atomic.Uint64 + addrHits atomic.Uint64 + addrMisses atomic.Uint64 + codeHits atomic.Uint64 + codeMisses atomic.Uint64 + ethHashHits atomic.Uint64 + ethHashMisses atomic.Uint64 + codeSizeHits atomic.Uint64 + codeSizeMisses atomic.Uint64 addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index a4d0e2b40eb..af9e3d61222 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -17,36 +17,97 @@ package cache import ( + "bytes" "sync" "sync/atomic" "github.com/c2h5oh/datasize" + "github.com/elastic/go-freelru" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" ) -// GenericCache is a bounded concurrent cache for key-value data. +// avgBytesPerEntry is the assumption used to translate a byte budget into +// the entry-count cap that freelru.ShardedLRU is sized against. 256 B +// approximates account-record + key overhead and storage-slot value+key +// overhead in the same order of magnitude. Actual residency tracked in +// currentSize and reported via PrintStatsAndReset. +const avgBytesPerEntry = 256 + +// entry stores the full key alongside the value so callers can detect +// hash collisions (the freelru shard key is the uint64 maphash of the +// byte-string key — Go's randomized stdlib hasher, so collisions are +// rare but not impossible). size carries the byte cost of the entry so +// the OnEvict callback can update currentSize without re-running +// sizeFunc. +type entry[T any] struct { + key []byte + val T + size int +} + +// GenericCache is a sharded, LRU-evicting bounded cache for key-value +// data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { - data *maphash.Map[T] + data *freelru.ShardedLRU[uint64, entry[T]] capacityB datasize.ByteSize currentSize atomic.Int64 - blockHash common.Hash - mu sync.RWMutex - hits atomic.Uint64 - misses atomic.Uint64 - inserts atomic.Uint64 - dropped atomic.Uint64 - sizeFunc func(T) int // calculates size of value in bytes -} - -// NewGenericCache creates a new GenericCache with the specified byte capacity. -func NewGenericCache[T any](capacityBytes datasize.ByteSize, sizeFunc func(T) int) *GenericCache[T] { - return &GenericCache[T]{ - data: maphash.NewMap[T](), + mode Mode + + blockHash common.Hash + mu sync.RWMutex + + hits atomic.Uint64 + misses atomic.Uint64 + inserts atomic.Uint64 + evictions atomic.Uint64 + dropped atomic.Uint64 + + sizeFunc func(T) int +} + +func u64identity(k uint64) uint32 { return uint32(k) } + +// NewGenericCache creates a new GenericCache with the specified byte +// capacity. mode selects ModeEvictLRU (default in this tree) or ModeNoOp +// (kept for diagnostic baselines). +func NewGenericCache[T any](capacityBytes datasize.ByteSize, sizeFunc func(T) int, mode Mode) *GenericCache[T] { + capacityEntries := uint32(uint64(capacityBytes) / avgBytesPerEntry) + if capacityEntries < 1024 { + capacityEntries = 1024 + } + if capacityEntries > 1<<22 { + capacityEntries = 1 << 22 + } + return newGenericCacheEntries(capacityBytes, capacityEntries, sizeFunc, mode) +} + +// newGenericCacheEntries builds a cache against an explicit entry-count +// cap. Used by tests that want to exercise eviction with small capacities; +// production constructs via NewGenericCache. +func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntries uint32, sizeFunc func(T) int, mode Mode) *GenericCache[T] { + if capacityEntries == 0 { + capacityEntries = 1 + } + lru, err := freelru.NewSharded[uint64, entry[T]](capacityEntries, u64identity) + if err != nil { + panic(err) + } + c := &GenericCache[T]{ + data: lru, capacityB: capacityBytes, + mode: mode, sizeFunc: sizeFunc, } + // OnEvict fires for capacity-driven LRU eviction (ModeEvictLRU) and + // for explicit Remove(). In both cases we want currentSize to follow. + lru.SetOnEvict(func(_ uint64, e entry[T]) { + c.currentSize.Add(-int64(e.size)) + c.evictions.Add(1) + }) + return c } // DomainCache wraps GenericCache[[]byte] to implement the Cache interface. @@ -54,10 +115,16 @@ type DomainCache struct { *GenericCache[[]byte] } -// NewDomainCache creates a new cache for domain data. +// NewDomainCache creates a new cache for domain data. Defaults to +// ModeEvictLRU; use NewDomainCacheMode to override. func NewDomainCache(capacityBytes datasize.ByteSize) *DomainCache { + return NewDomainCacheMode(capacityBytes, ModeEvictLRU) +} + +// NewDomainCacheMode creates a new domain cache with the given mode. +func NewDomainCacheMode(capacityBytes datasize.ByteSize, mode Mode) *DomainCache { return &DomainCache{ - GenericCache: NewGenericCache(capacityBytes, func(v []byte) int { return len(v) }), + GenericCache: NewGenericCache(capacityBytes, func(v []byte) int { return len(v) }, mode), } } @@ -82,55 +149,64 @@ func (c *DomainCache) Delete(key []byte) { // Get retrieves data for the given key. func (c *GenericCache[T]) Get(key []byte) (T, bool) { - value, ok := c.data.Get(key) - if !ok { + h := maphash.Hash(key) + e, ok := c.data.Get(h) + if !ok || !bytes.Equal(e.key, key) { c.misses.Add(1) var zero T return zero, false } c.hits.Add(1) - return value, true + return e.val, true } -// Put stores data for the given key. +// Put stores data for the given key. In ModeEvictLRU the underlying +// sharded LRU evicts cold entries when its entry-count cap is reached. +// In ModeNoOp inserts that would overflow the byte budget are dropped +// (and counted via the dropped metric). func (c *GenericCache[T]) Put(key []byte, value T) { - entrySize := int64(8 + c.sizeFunc(value)) - - // Check if key already exists - if existing, ok := c.data.Get(key); ok { - oldSize := int64(8 + c.sizeFunc(existing)) - sizeDiff := entrySize - oldSize - c.data.Set(key, value) - c.currentSize.Add(sizeDiff) + h := maphash.Hash(key) + valBytes := c.sizeFunc(value) + newSize := len(key) + valBytes + 24 + + // Existing key — update in place. Reuse the stored key buffer to + // avoid an extra allocation; the freshly-decoded value replaces the + // old one. + if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { + oldSize := existing.size + c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize}) + c.currentSize.Add(int64(newSize - oldSize)) return } - // New key. The cache currently has no eviction policy, so once full - // every subsequent new key is silently dropped until Clear/ClearWithHash/ - // ValidateAndPrepare-mismatch resets the cache. dropped vs inserts makes - // this perf cliff observable. - if c.currentSize.Load()+entrySize > int64(c.capacityB) { - c.dropped.Add(1) - return + if c.mode == ModeNoOp { + if c.currentSize.Load()+int64(newSize) > int64(c.capacityB) { + c.dropped.Add(1) + return + } } + // In ModeEvictLRU the per-shard LRU evicts the oldest entry inside + // freelru.Add when its slot cap is reached; OnEvict drops the size + // from currentSize. Eviction is per-shard, not globally-LRU — same + // trade-off code_cache.go / balcache.go / db/state/cache.go accept. - c.data.Set(key, value) - c.currentSize.Add(entrySize) + keyCopy := common.Copy(key) + c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize}) + c.currentSize.Add(int64(newSize)) c.inserts.Add(1) } // Delete removes the data for the given key. func (c *GenericCache[T]) Delete(key []byte) { - if existing, ok := c.data.Get(key); ok { - entrySize := int64(8 + c.sizeFunc(existing)) - c.data.Delete(key) - c.currentSize.Add(-entrySize) + h := maphash.Hash(key) + if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { + c.data.Remove(h) } } // Clear removes all entries from the cache. func (c *GenericCache[T]) Clear() { - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) } @@ -154,7 +230,7 @@ func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlo defer c.mu.Unlock() if c.blockHash == (common.Hash{}) { - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) c.blockHash = incomingBlockHash return true @@ -165,7 +241,7 @@ func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlo return true } - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) c.blockHash = incomingBlockHash return false @@ -175,14 +251,14 @@ func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlo func (c *GenericCache[T]) ClearWithHash(hash common.Hash) { c.mu.Lock() defer c.mu.Unlock() - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) c.blockHash = hash } // Len returns the number of entries in the cache. func (c *GenericCache[T]) Len() int { - return c.data.Len() + return int(c.data.Len()) } // SizeBytes returns the current size of the cache in bytes. @@ -195,11 +271,17 @@ func (c *GenericCache[T]) CapacityBytes() datasize.ByteSize { return c.capacityB } +// Mode returns the eviction mode this cache was constructed with. +func (c *GenericCache[T]) Mode() Mode { + return c.mode +} + // PrintStatsAndReset prints cache statistics and resets counters. func (c *GenericCache[T]) PrintStatsAndReset(name string) { hits := c.hits.Swap(0) misses := c.misses.Swap(0) inserts := c.inserts.Swap(0) + evictions := c.evictions.Swap(0) dropped := c.dropped.Swap(0) total := hits + misses var hitRate float64 @@ -209,8 +291,9 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { sizeBytes := c.currentSize.Load() usagePct := float64(sizeBytes) / float64(c.capacityB) * 100 log.Debug(name+" cache stats", + "mode", c.mode.String(), "hits", hits, "misses", misses, "hit_rate", hitRate, - "inserts", inserts, "dropped", dropped, + "inserts", inserts, "evictions", evictions, "dropped", dropped, "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", c.capacityB/datasize.MB, "usage_pct", usagePct, ) diff --git a/execution/cache/policy.go b/execution/cache/policy.go new file mode 100644 index 00000000000..7efa0fb649d --- /dev/null +++ b/execution/cache/policy.go @@ -0,0 +1,56 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +// Mode selects what GenericCache does when an insert would overflow the +// capacity. The seam is the same for both modes — the in-package eviction +// pop is the only behaviour difference. +// +// ModeEvictLRU is the default and matches the policy other state caches in +// the tree use (db/state/cache.go's DomainGetFromFileCache / +// IISeekInFilesCache, execution/cache/code_cache.go's addr LRUs, +// execution/balcache). +// +// ModeNoOp preserves the historical "first writers win forever" behaviour: +// once full, every new key is silently dropped (counted via the dropped +// metric). Kept as a deliberate diagnostic baseline so the regression +// bench can compare against the pre-policy behaviour without flipping +// branches. +// +// LFU is a known follow-up. Pure LRU is scan-fragile: a flood of one-shot +// keys (cold-storage bloat workloads; mainnet's long tail of single-touch +// slots) evicts the genuinely-hot working set because every cold scan is +// "more recent" than the hot entries. W-TinyLFU keeps a frequency sketch +// so single-touch entries can't displace frequently-touched ones — which +// is why reth uses it for state caches. A ModeEvictLFU would slot in +// behind the same wrapper without touching call sites. +type Mode uint8 + +const ( + ModeEvictLRU Mode = iota + ModeNoOp +) + +func (m Mode) String() string { + switch m { + case ModeEvictLRU: + return "evict" + case ModeNoOp: + return "noop" + } + return "unknown" +} diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index a9559570187..8e07f988b49 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -33,6 +33,14 @@ const ( DefaultStorageCacheBytes = 1 * datasize.GB // DefaultCommitmentCacheBytes is the byte limit for commitment cache (128 MB) DefaultCommitmentCacheBytes = 128 * datasize.MB + + // Per-domain avg entry size used to translate the byte budget into the + // entry-count cap the underlying sharded LRU is sized against. Account + // and storage are near-fixed: addr + record or addr+slot + value plus + // entry overhead. Commitment values are smaller branch nodes. + avgAccountEntryBytes = 96 // 20 addr + ~50 account record + 24 overhead + avgStorageEntryBytes = 88 // 52 addr+slot + ~12 value + 24 overhead + avgCommitmentEntryBytes = 80 ) // StateCache is a unified cache for domain data (Account, Storage, Code). @@ -48,13 +56,25 @@ type StateCache struct { // NewStateCache creates a new StateCache with the specified byte capacities. func NewStateCache(accountBytes, storageBytes, codeBytes, addrBytes, commitmentBytes datasize.ByteSize) *StateCache { sc := &StateCache{} - sc.caches[kv.AccountsDomain] = NewDomainCache(accountBytes) - sc.caches[kv.StorageDomain] = NewDomainCache(storageBytes) + sc.caches[kv.AccountsDomain] = newDomainCacheBytes(accountBytes, avgAccountEntryBytes, ModeEvictLRU) + sc.caches[kv.StorageDomain] = newDomainCacheBytes(storageBytes, avgStorageEntryBytes, ModeEvictLRU) sc.caches[kv.CodeDomain] = NewCodeCache(codeBytes, addrBytes) - //sc.caches[kv.CommitmentDomain] = NewDomainCache(commitmentBytes) + //sc.caches[kv.CommitmentDomain] = newDomainCacheBytes(commitmentBytes, avgCommitmentEntryBytes, ModeEvictLRU) return sc } +// newDomainCacheBytes constructs a DomainCache where the entry-count cap +// is derived from the byte budget using the supplied per-domain avg. +func newDomainCacheBytes(capacityBytes datasize.ByteSize, avgBytes uint32, mode Mode) *DomainCache { + capacityEntries := uint32(uint64(capacityBytes) / uint64(avgBytes)) + if capacityEntries == 0 { + capacityEntries = 1 + } + return &DomainCache{ + GenericCache: newGenericCacheEntries(capacityBytes, capacityEntries, func(v []byte) int { return len(v) }, mode), + } +} + // NewDefaultStateCache creates a new StateCache with default byte capacities. // Account: 256MB, Storage: 512MB, Code: 512MB, Addr: 16MB func NewDefaultStateCache() *StateCache { diff --git a/execution/vm/contract.go b/execution/vm/contract.go index c6821001bb3..da7e2cb9c3b 100644 --- a/execution/vm/contract.go +++ b/execution/vm/contract.go @@ -56,7 +56,7 @@ type Contract struct { } // around 64MB cache in the worst case. -var jumpDestCache = cache.NewGenericCache[bitvec](64*datasize.MB, func(v bitvec) int { return len(v) }) +var jumpDestCache = cache.NewGenericCache[bitvec](64*datasize.MB, func(v bitvec) int { return len(v) }, cache.ModeEvictLRU) // NewContract returns a new contract environment for the execution of EVM. func NewContract(caller accounts.Address, callerAddress accounts.Address, addr accounts.Address, value uint256.Int) *Contract { From 3fec4764a22a9ff656f9f3b976833a4ce3ce9b58 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 13:42:27 +0000 Subject: [PATCH 088/167] execution/cache: STATE_CACHE_MODE env override at NewStateCache time Single env knob read once at NewStateCache. Default ModeEvictLRU, recognised override "noop" (for the regression-bench baseline so ModeEvictLRU and ModeNoOp can be compared on the same binary). Unrecognised values fall back to evict with a warn log. ModeNoOp engagement is logged at info level because the fill-and-freeze behaviour is a deliberate diagnostic state, not a production setting. Pattern matches db/state/cache.go's D_LRU_ENABLED / D_LRU knobs (dbg.EnvString from common/dbg). Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/state_cache.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 8e07f988b49..2e0d047eed6 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -18,10 +18,13 @@ package cache import ( "bytes" + "strings" "github.com/c2h5oh/datasize" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/dbg" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/execution/commitment/commitmentdb" ) @@ -54,15 +57,36 @@ type StateCache struct { } // NewStateCache creates a new StateCache with the specified byte capacities. +// Mode for the byte-budget DomainCaches (Account/Storage/Commitment) is +// read once from STATE_CACHE_MODE (evict|noop, default evict). CodeCache +// has its own LRU and is not gated by this knob. func NewStateCache(accountBytes, storageBytes, codeBytes, addrBytes, commitmentBytes datasize.ByteSize) *StateCache { + mode := stateCacheModeFromEnv() sc := &StateCache{} - sc.caches[kv.AccountsDomain] = newDomainCacheBytes(accountBytes, avgAccountEntryBytes, ModeEvictLRU) - sc.caches[kv.StorageDomain] = newDomainCacheBytes(storageBytes, avgStorageEntryBytes, ModeEvictLRU) + sc.caches[kv.AccountsDomain] = newDomainCacheBytes(accountBytes, avgAccountEntryBytes, mode) + sc.caches[kv.StorageDomain] = newDomainCacheBytes(storageBytes, avgStorageEntryBytes, mode) sc.caches[kv.CodeDomain] = NewCodeCache(codeBytes, addrBytes) - //sc.caches[kv.CommitmentDomain] = newDomainCacheBytes(commitmentBytes, avgCommitmentEntryBytes, ModeEvictLRU) + //sc.caches[kv.CommitmentDomain] = newDomainCacheBytes(commitmentBytes, avgCommitmentEntryBytes, mode) return sc } +// stateCacheModeFromEnv reads STATE_CACHE_MODE once. Unset or unrecognised +// returns ModeEvictLRU. Recognised values: "evict", "noop". Logged at +// startup so operators can see the mode pick. +func stateCacheModeFromEnv() Mode { + v := strings.ToLower(strings.TrimSpace(dbg.EnvString("STATE_CACHE_MODE", ""))) + switch v { + case "", "evict": + return ModeEvictLRU + case "noop": + log.Info("[cache] STATE_CACHE_MODE=noop — Account/Storage caches will drop new keys when full (diagnostic baseline; not for production)") + return ModeNoOp + default: + log.Warn("[cache] unrecognised STATE_CACHE_MODE; defaulting to evict", "value", v) + return ModeEvictLRU + } +} + // newDomainCacheBytes constructs a DomainCache where the entry-count cap // is derived from the byte budget using the supplied per-domain avg. func newDomainCacheBytes(capacityBytes datasize.ByteSize, avgBytes uint32, mode Mode) *DomainCache { From 1a56794176aa88e907f33e83f52521f287e2e024 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 14:13:11 +0000 Subject: [PATCH 089/167] execution/cache: correct the LFU rationale in Mode docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous comment asserted "reth uses W-TinyLFU for state caches" — that is wrong on the execution hot path. Reth's cross-block state cache is `fixed-cache` (PR #21128, v1.11.0): a lock-free direct-mapped / set-associative array with collision-evict semantics. No LRU list, no LFU sketch. Their published wins (~25% newPayload p50 / +33% gas/s) came from *removing* LRU/LFU bookkeeping, not adding LFU. Where reth uses real LRU/LFU it's deliberate and not the execution cache (schnellru::LruMap for networking; moka in precompile_cache.rs explicitly configured with eviction_policy(EvictionPolicy::lru())). The docstring now reflects two follow-up policies both real: - ModeEvictFixedCache (reth's actual choice, more interesting structural option than LFU) - ModeEvictLFU (W-TinyLFU; helps mainnet steady-state, not the cycle-2 bloat fixtures which are pure cold scans) Decision criterion (per agentspecs/lfu-vs-lru-state-cache-decision-2026-05-15.md): ship ModeEvictLFU only if a 24h mainnet replay shows current sharded-LRU hit-rate < 90 % on Account/Storage. Otter is the only credible Go W-TinyLFU library; ristretto has documented correctness bugs and is disqualified for an EL hot path. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/policy.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/execution/cache/policy.go b/execution/cache/policy.go index 7efa0fb649d..6297df9108c 100644 --- a/execution/cache/policy.go +++ b/execution/cache/policy.go @@ -31,13 +31,28 @@ package cache // bench can compare against the pre-policy behaviour without flipping // branches. // -// LFU is a known follow-up. Pure LRU is scan-fragile: a flood of one-shot -// keys (cold-storage bloat workloads; mainnet's long tail of single-touch -// slots) evicts the genuinely-hot working set because every cold scan is -// "more recent" than the hot entries. W-TinyLFU keeps a frequency sketch -// so single-touch entries can't displace frequently-touched ones — which -// is why reth uses it for state caches. A ModeEvictLFU would slot in -// behind the same wrapper without touching call sites. +// Pure LRU is scan-fragile in principle: a flood of one-shot keys (mainnet's +// long tail of single-touch slots) can evict the genuinely-hot working set +// because every cold scan is "more recent" than the hot entries. Two known +// follow-up policies sit behind this seam: +// +// - ModeEvictFixedCache (reth's choice). Reth's execution cache is +// `fixed-cache`: a lock-free direct-mapped / set-associative array +// with collision-evict semantics — no LRU list, no LFU sketch. Their +// PR #21128 (v1.11.0) quoted ~25% newPayload p50 / +33% gas/s vs the +// prior moka / quick-cache attempts; the win came from *removing* +// LRU/LFU bookkeeping, not adding LFU. +// - ModeEvictLFU. W-TinyLFU (Caffeine-style) admission policy keeps a +// small frequency sketch so single-touch entries can't displace +// frequently-touched ones. Helps mainnet steady-state in principle +// (+5-15pp on Zipfian-with-scan per Caffeine literature). Does NOT +// help the cycle-2 bloat fixtures — those are pure cold scans with +// no reuse, so admission policy has nothing to admit. +// +// See agentspecs/lfu-vs-lru-state-cache-decision-2026-05-15.md for the +// full analysis (which lib to use if we ship LFU — otter, not ristretto) +// and the decision criterion (24h mainnet replay hit-rate < 90% before +// LFU is worth a new dep). type Mode uint8 const ( From 4a512ce0d3345f166992e264c3238bf303cf0ed8 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 17 May 2026 20:25:02 +0000 Subject: [PATCH 090/167] execution/cache: reduce default cache caps to 100 MB each (bench knob) Investigation knob, NOT a permanent default. Account / Storage / Code each capped at 100 MB so the bench measures layer contributions instead of being dominated by preallocated cache memory pressure (1 GB / 1 GB / 512 MB defaults push sys past the GC/page-cache pressure band on this hardware/workload mix). Permanent defaults stay at 1 GB / 1 GB / 512 MB; this commit will be reverted or dynamically gated by relative-to-available sizing. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/code_cache.go | 4 ++-- execution/cache/state_cache.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index b4301406bb7..163172abda3 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -35,8 +35,8 @@ func uint64AsBytes(v *uint64) []byte { } const ( - // DefaultCodeCacheBytes is the byte limit for code cache (512 MB) - DefaultCodeCacheBytes = 512 * datasize.MB + // DefaultCodeCacheBytes is the byte limit for code cache (100 MB — investigation knob; permanent default returns to 512 MB) + DefaultCodeCacheBytes = 100 * datasize.MB // DefaultAddrCacheBytes is the byte limit for address cache (16 MB) DefaultAddrCacheBytes = 16 * datasize.MB // DefaultAddrCacheEntries derives from DefaultAddrCacheBytes assuming diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 2e0d047eed6..dbf72fb3114 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -30,10 +30,10 @@ import ( ) const ( - // DefaultAccountCacheBytes is the byte limit for account cache (1 GB) - DefaultAccountCacheBytes = 1 * datasize.GB - // DefaultStorageCacheBytes is the byte limit for storage cache (1 GB) - DefaultStorageCacheBytes = 1 * datasize.GB + // DefaultAccountCacheBytes is the byte limit for account cache (100 MB — investigation knob; permanent default returns to 1 GB) + DefaultAccountCacheBytes = 100 * datasize.MB + // DefaultStorageCacheBytes is the byte limit for storage cache (100 MB — investigation knob; permanent default returns to 1 GB) + DefaultStorageCacheBytes = 100 * datasize.MB // DefaultCommitmentCacheBytes is the byte limit for commitment cache (128 MB) DefaultCommitmentCacheBytes = 128 * datasize.MB From da4b091c22da0623165df886135901b6a0057e80 Mon Sep 17 00:00:00 2001 From: Mark Holt <135143369+mh0lt@users.noreply.github.com> Date: Mon, 25 May 2026 09:09:00 +0100 Subject: [PATCH 091/167] Parallel-exec correctness fixes (PR #3 of the perf stack) (#21387) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR ships the parallel-exec correctness fixes from `mh/parallel-exec-fixes` onto the perf stack, packaged as a focused PR on top of [#21386 (StateCache LRU)](https://github.com/erigontech/erigon/pull/21386) which itself stacks on [#21380 (State Cache Consolidation)](https://github.com/erigontech/erigon/pull/21380). > [!IMPORTANT] > **Stacks on #21386 → #21380.** Base is `mh/perf-statecache-lru-pr`, NOT `main`. Merge order: #21380 → #21386 → this PR. > [!IMPORTANT] > **Do not merge until CI is green on both parallel and serial.** Same gating rule as #21380 / #21386. ## Scope — 13 commits from `mh/parallel-exec-fixes` Brought in via a merge commit so the bisection trail is preserved. | sha | what it fixes | |---|---| | `25053e38e9` | parallel SD-of-pre-existing-contract — the 197-line foundational fix | | `2e2bf3ccc0` | clean exit when single-block batch already covered maxBlockNum | | `6e451f5ed2` | don't emit StoragePath=0 writes from IBS.Selfdestruct | | `616a4fa0a8` | clear calc Deleted on a non-SD account write even when zero | | `d99f2f704d` | gate known parallel-exec failures behind EXEC3_PARALLEL (#21136) | | `34e83e82b7` | install per-block changeset accumulator before any of the block's writes | | `b340d7e592` | drop stale sd.mem 'Trim old version entries' comment | | `629cc23566` | O(1) CollectorWrites fee-balance update, drop dead VersionedWrites.SetBalance | | `a0ecfc7e12` | first-match-wins in CollectorWrites BalancePath index | | `445f97e446` | emit EIP-7708 Burn log under parallel-exec when coinbase self-destructs | | `5e1f5fa901` | mirror ReadAccountData SD-revival check into versionedRead | | `a5dc83f509` | drop two stale EXEC3_PARALLEL t.Skips | | `8af901104f` | drop TestReceiptHashFromRPC unit-suite RPC integration test | ## Merge conflicts resolved 3 files, 8 regions — all resolved by keeping HEAD's typed-readset / per-path revival shape and confirming HEAD already absorbs each fix's intent. See the merge commit message (`cfc4ec1418`) for the per-region rationale. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Mark Holt From 9b153620a32bdc2a595330e052d72e25b67ccbc5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 25 May 2026 21:23:28 +0000 Subject: [PATCH 092/167] execution/state: drop tautological nil checks in code-loading paths (govet) stateObject and s are both verified non-nil earlier in their respective scopes; the secondary checks at lines 749 and 783 are redundant. govet nilness check fails on these. --- execution/state/intra_block_state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/execution/state/intra_block_state.go b/execution/state/intra_block_state.go index f496473fb03..a1c6800ce3e 100644 --- a/execution/state/intra_block_state.go +++ b/execution/state/intra_block_state.go @@ -746,7 +746,7 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { if err != nil { return 0, err } - if stateObject != nil && code != nil { + if code != nil { stateObject.code = code } return len(code), nil @@ -780,7 +780,7 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { // effectively a hashmap probe + slice assignment. code, codeErr := sdb.stateReader.ReadAccountCode(addr) l := len(code) - if codeErr == nil && s != nil && code != nil { + if codeErr == nil && code != nil { s.code = code } err := codeErr From 6b582fd88b055f038dcabe6a6cfcf2605419efd6 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 12:43:23 +0000 Subject: [PATCH 093/167] execution/execmodule: clear BranchCache between unwindToCommonCanonical and ValidatePayload (fix 6 hive re-org failures) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ValidateChain's fork-payload path runs unwindToCommonCanonical → ValidatePayload. SD.Unwind (called transitively by unwindToCommonCanonical) does selective per-key BranchCache invalidation only for entries listed in changeset[kv.CommitmentDomain]. Commitment-trie internal branches derived during the canonical flush sit in the aggregator-scope BranchCache without appearing in any changeset entry — so the selective sweep misses them and they survive the unwind as stale canonical-lineage entries that poison the fork-payload trie reads → wrong trie root → EngineNewPayload returns INVALID on fork payloads that should be VALID. The PR author's own doctrine on HexPatriciaHashed.Reset (execution/commitment/hex_patricia_hashed.go:2970-2975) explicitly states: "The BranchCache is intentionally NOT cleared here ... Callers that need to invalidate the cache (unwind, fork validation) MUST call ClearBranchCache explicitly." SetHead unwind already follows this doctrine (set_head.go:159 calls sd.ClearBranchCache()). The ValidateChain fork-payload path didn't — this commit closes that gap. Symptom this fixes: all 6 ethereum/engine hive failures on the merge-from-main run (beeyx741q) returned INVALID where VALID expected on re-org scenarios: - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs={T,F}, Invalid P{9,10} - Re-Org Back into Canonical Chain, Depth=10, Execute Side Payload on Re-Org - Withdrawals Fork on Block N - M Block Re-Org NewPayload - Withdrawals Fork on Canonical Block N / Side Block M - 10 Block Re-Org All six are EngineNewPayload* on re-org / fork-payload scenarios — exactly the cache-pollution shape this fix addresses. Symmetric across serial and parallel jobs, confirming it's not a parallel-exec regression. Reviewer context: this is the production-side fix taratorio asked about in PR comments — the test-runner ResetBranchCache call is not "AI slop" but defense for the same gap; this commit closes the production-side gap so the test-runner reset becomes belt-and-braces rather than load-bearing. --- execution/execmodule/exec_module.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index b77676216a4..986092880ab 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -543,6 +543,20 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } + // Explicit BranchCache clear between unwind and fork-payload validation. + // SD.Unwind's selective per-key invalidation covers only entries listed in + // changeset[kv.CommitmentDomain]; commitment-trie internal branches + // derived during the canonical flush can sit in the aggregator-scope + // BranchCache without ever appearing in a changeset entry. The + // HexPatriciaHashed.Reset doctrine (execution/commitment/hex_patricia_hashed.go:2970-2975) + // explicitly requires unwind/fork-validation callers to ClearBranchCache, + // otherwise stale canonical-lineage branches poison fork-payload reads + // and produce wrong trie roots → EngineNewPayload returns INVALID on + // fork payloads that should be VALID. Observed as the 6 hive ethereum/engine + // re-org failures on this branch's merge-from-main run; the cleared + // cache lets the fork-payload trie reads see post-unwind state. + doms.ClearBranchCache() + status, lvh, validationError, criticalError := e.forkValidator.ValidatePayload(ctx, doms, tx, header, body.RawBody(), e.logger) if criticalError != nil { return ValidationResult{}, criticalError From 25eb151af6fb6604d924a88967ec9bd4b1768664 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 13:09:06 +0000 Subject: [PATCH 094/167] Revert "execution/execmodule: clear BranchCache between unwindToCommonCanonical and ValidatePayload (fix 6 hive re-org failures)" This reverts commit 6b582fd88b055f038dcabe6a6cfcf2605419efd6. --- execution/execmodule/exec_module.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 986092880ab..b77676216a4 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -543,20 +543,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } - // Explicit BranchCache clear between unwind and fork-payload validation. - // SD.Unwind's selective per-key invalidation covers only entries listed in - // changeset[kv.CommitmentDomain]; commitment-trie internal branches - // derived during the canonical flush can sit in the aggregator-scope - // BranchCache without ever appearing in a changeset entry. The - // HexPatriciaHashed.Reset doctrine (execution/commitment/hex_patricia_hashed.go:2970-2975) - // explicitly requires unwind/fork-validation callers to ClearBranchCache, - // otherwise stale canonical-lineage branches poison fork-payload reads - // and produce wrong trie roots → EngineNewPayload returns INVALID on - // fork payloads that should be VALID. Observed as the 6 hive ethereum/engine - // re-org failures on this branch's merge-from-main run; the cleared - // cache lets the fork-payload trie reads see post-unwind state. - doms.ClearBranchCache() - status, lvh, validationError, criticalError := e.forkValidator.ValidatePayload(ctx, doms, tx, header, body.RawBody(), e.logger) if criticalError != nil { return ValidationResult{}, criticalError From 0021402e1b773feea1522bb9769c00de66e48ce4 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 20:54:08 +0000 Subject: [PATCH 095/167] execution/commitment, common/maphash: txN-tagged BranchCache entries + UnwindTo watermark eviction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of replacing diffset-driven BranchCache invalidation with a self-sufficient txN-tagged cache. - branchCacheEntry gains txN uint64. Tagging conventions: sd.Flush = exact sd.txNum; snapshot reads = file endTxNum; DB latest reads = lastTxNumOfStep(step) high-water. 0 means "not tracked" — entry survives any watermark (back-compat). - Put / PutIfClean / PinEntry signatures take txN alongside step. All existing call sites pass 0 to preserve today's behavior; meaningful values land in subsequent commits as the read paths are plumbed. - New UnwindTo(maxValidTxN) evicts every entry with txN > watermark across root + pinned + LRU tail. Single watermark pass, no per-key changeset required. Pinned entries are evicted by watermark — pinning protects capacity, not correctness. - common/maphash gains Map.Range / Map.DeleteByHash and LRU.DeleteByHash to support hash-keyed iteration with in-place delete (needed because Set hashes-and-discards the byte key). Tests cover: watermark eviction across all tiers, txN=0 immortality, boundary equality (txN==watermark stays, txN==watermark+1 evicts). Step 1/n of #21380 chosen-fix design — see pr-21380-txn-tagged-cache-design-2026-05-26 memory pin. --- common/maphash/maphash.go | 24 ++++ db/state/execctx/domain_shared.go | 4 +- execution/commitment/branch_cache.go | 75 +++++++++++- execution/commitment/branch_cache_test.go | 134 +++++++++++++++++++--- execution/commitment/preload.go | 2 +- execution/commitment/preload_parallel.go | 2 +- execution/commitment/trunk_pin_test.go | 10 +- 7 files changed, 221 insertions(+), 30 deletions(-) diff --git a/common/maphash/maphash.go b/common/maphash/maphash.go index 747fb45e1fd..45dfc97a00a 100644 --- a/common/maphash/maphash.go +++ b/common/maphash/maphash.go @@ -61,6 +61,24 @@ func (m *Map[V]) Clear() { m.m.Clear() } +// Range iterates over every (hash, value) pair. Iteration order is +// unspecified. Return false from fn to stop early. Concurrent +// modification during Range is permitted by the underlying xsync.Map. +// +// The original byte-key is not recoverable — Set hashes-and-discards. +// Pair with DeleteByHash to evict entries discovered via Range. +func (m *Map[V]) Range(fn func(hash uint64, v V) bool) { + m.m.Range(func(key uint64, value V) bool { + return fn(key, value) + }) +} + +// DeleteByHash removes the entry under the pre-computed hash. Use +// alongside Range when the byte-key is unknown. +func (m *Map[V]) DeleteByHash(hash uint64) { + m.m.Delete(hash) +} + // NonConcurrentMap is a non-thread-safe map that uses maphash to hash []byte keys. // Use this when you don't need concurrent access for better performance. type NonConcurrentMap[V any] struct { @@ -164,6 +182,12 @@ func (l *LRU[V]) ContainsByHash(hash uint64) bool { return l.cache.Contains(hash) } +// DeleteByHash removes the entry under the pre-computed hash. Use +// alongside Range when the byte-key is unknown. +func (l *LRU[V]) DeleteByHash(hash uint64) { + l.cache.Remove(hash) +} + // Range iterates over every (hash, value) pair without affecting LRU // recency (uses Peek under the hood). Iteration order is unspecified. // Return false from fn to stop early. diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a5ae5d0b196..265a77892cc 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -780,7 +780,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.branchCache.Invalidate(k) return } - sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + sd.branchCache.Put(k, v, uint64(step), 0, "sd.Flush") }); err != nil { return err } @@ -969,7 +969,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) sd.stateCache.Put(domain, k, v) } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { - sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") + sd.branchCache.Put(k, v, uint64(step), 0, "sd.GetLatest") } return v, step, nil diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 9edf294d242..310a8dd4199 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -270,6 +270,18 @@ type branchCacheEntry struct { // in-memory tests but real callers should always pass the step // returned by aggTx.MeteredGetLatest / tx.GetLatest. step uint64 + + // txN is the high-water txNum the cached bytes correspond to. Used by + // UnwindTo: an unwind to txNum T evicts every entry whose txN > T. + // Tagging conventions per write source: + // - sd.Flush → sd.txNum (exact, in-memory write) + // - snapshot file → file.endTxNum (step-boundary, exact under + // the no-unwind-into-snapshots invariant) + // - DB latest read → lastTxNumOfStep(step) (conservative + // high-water within the step) + // 0 means "not tracked" — entry survives any watermark (back-compat + // with callers that haven't been migrated to pass a real txN yet). + txN uint64 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -372,7 +384,11 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // eager preload of hot prefixes — e.g. the storage-trunk of big // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate // the input after the call. -func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { +// +// txN tags the entry with its high-water validity; UnwindTo evicts a +// pinned entry when its txN exceeds the watermark. Pinning protects +// against capacity-pressure eviction, not correctness eviction. +func (c *BranchCache) PinEntry(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -381,6 +397,7 @@ func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin s c.pinned.Set(prefix, &branchCacheEntry{ data: dataCopy, step: step, + txN: txN, writeSeq: c.writeSeq.Add(1), origin: origin, writeTimeNanos: time.Now().UnixNano(), @@ -487,9 +504,11 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of // caller buffer lifetime. step is the on-disk file step the bytes came -// from (0 if not tracked); origin is a short label of the write site -// captured for divergence-detection diagnostics. -func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { +// from (0 if not tracked); txN is the high-water txNum the entry is +// valid through (0 if not tracked — entry survives any UnwindTo +// watermark); origin is a short label of the write site captured for +// divergence-detection diagnostics. +func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -498,6 +517,7 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string c.store(prefix, &branchCacheEntry{ data: dataCopy, step: step, + txN: txN, origin: origin, writeSeq: c.writeSeq.Add(1), writeTimeNanos: time.Now().UnixNano(), @@ -511,11 +531,11 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step uint64, origin string) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step, txN uint64, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data, step, origin) + c.Put(prefix, data, step, txN, origin) return true } @@ -574,6 +594,49 @@ func (c *BranchCache) Invalidate(prefix []byte) { c.tail.Delete(prefix) } +// UnwindTo evicts every cache entry whose txN > maxValidTxN across all +// tiers (root, pinned, LRU tail). Returns the number of entries evicted. +// +// This is the txN-tagged-cache invalidation primitive: a single watermark +// pass over the cache, no per-key changeset required. Entries with txN=0 +// ("not tracked") are NEVER evicted by the watermark — they're treated +// as permanently valid (callers that don't tag pay no watermark cost, +// preserving back-compat for migration). +// +// Pinned entries ARE evicted by watermark — pinning protects against +// capacity-pressure eviction, not correctness eviction. +// +// Concurrency: safe to call alongside concurrent reads. Writes during +// UnwindTo may produce a slightly larger effective working set than the +// post-call watermark suggests (a Put racing with the scan may insert an +// entry the scan already passed), but every such Put carries its own txN +// and will be caught by the next UnwindTo if the entry violates the +// invariant. +func (c *BranchCache) UnwindTo(maxValidTxN uint64) (evicted int) { + // Root tier — single slot, atomic check-and-clear. + if entry := c.root.Load(); entry != nil && entry.txN > maxValidTxN { + c.root.Store(nil) + evicted++ + } + // Pinned tier — iterate and delete in place. + c.pinned.Range(func(hash uint64, entry *branchCacheEntry) bool { + if entry != nil && entry.txN > maxValidTxN { + c.pinned.DeleteByHash(hash) + evicted++ + } + return true + }) + // LRU tail — iterate and delete in place. + c.tail.Range(func(hash uint64, entry *branchCacheEntry) bool { + if entry != nil && entry.txN > maxValidTxN { + c.tail.DeleteByHash(hash) + evicted++ + } + return true + }) + return evicted +} + // Clear empties the cache and resets stats counters across ALL tiers // (root, pinned, LRU tail). Use on Reset / fork-validation paths to // ensure stale entries from one trie root are not served against a diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 29e3534e817..e8a29b8a0f1 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -31,8 +31,8 @@ func TestBranchCache_RootPinning(t *testing.T) { rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch deepKey := []byte{0x12, 0x34, 0x56} - c.Put(rootKey, []byte("root-data"), 0, "test") - c.Put(deepKey, []byte("deep-data"), 0, "test") + c.Put(rootKey, []byte("root-data"), 0, 0, "test") + c.Put(deepKey, []byte("deep-data"), 0, 0, "test") // Root reads should increment rootHits, not tailHits got, _, ok := c.Get(rootKey) @@ -55,11 +55,11 @@ func TestBranchCache_RootPinning(t *testing.T) { func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { c := NewBranchCache(10) // very small tail rootKey := []byte{0x00} - c.Put(rootKey, []byte("ROOT-PERSISTS"), 0, "test") + c.Put(rootKey, []byte("ROOT-PERSISTS"), 0, 0, "test") // Stuff the tail well past capacity for i := 0; i < 100; i++ { - c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, 0, "test") + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, 0, 0, "test") } // Root must still be there @@ -78,20 +78,20 @@ func TestBranchCache_DirtyFlag(t *testing.T) { c := NewBranchCache(100) key := []byte{0x12, 0x34} - require.True(t, c.PutIfClean(key, []byte("v1"), 0, "test")) + require.True(t, c.PutIfClean(key, []byte("v1"), 0, 0, "test")) c.MarkDirty(key) - require.False(t, c.PutIfClean(key, []byte("v2"), 0, "test"), "PutIfClean must refuse dirty entry") + require.False(t, c.PutIfClean(key, []byte("v2"), 0, 0, "test"), "PutIfClean must refuse dirty entry") got, _, _ := c.Get(key) require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") // Unconditional Put replaces - c.Put(key, []byte("v3"), 0, "test") + c.Put(key, []byte("v3"), 0, 0, "test") got, _, _ = c.Get(key) require.Equal(t, []byte("v3"), got) // New entry is clean - require.True(t, c.PutIfClean(key, []byte("v4"), 0, "test")) + require.True(t, c.PutIfClean(key, []byte("v4"), 0, 0, "test")) } // TestBranchCache_GetDecoded verifies the lazy-decode read path for a @@ -106,7 +106,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.NoError(t, err) prefix := []byte{0x12, 0x34} - c.Put(prefix, enc, 0, "test") + c.Put(prefix, enc, 0, 0, "test") // First decoded-read decodes lazily bitmap, cells, ok := c.GetDecoded(prefix) @@ -130,8 +130,8 @@ func TestBranchCache_Invalidate(t *testing.T) { c := NewBranchCache(100) rootKey := []byte{0x00} deepKey := []byte{0x12, 0x34} - c.Put(rootKey, []byte("r"), 0, "test") - c.Put(deepKey, []byte("d"), 0, "test") + c.Put(rootKey, []byte("r"), 0, 0, "test") + c.Put(deepKey, []byte("d"), 0, 0, "test") c.Invalidate(rootKey) _, _, ok := c.Get(rootKey) @@ -145,8 +145,8 @@ func TestBranchCache_Invalidate(t *testing.T) { // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("r"), 0, "test") - c.Put([]byte{0x12}, []byte("d"), 0, "test") + c.Put([]byte{0x00}, []byte("r"), 0, 0, "test") + c.Put([]byte{0x12}, []byte("d"), 0, 0, "test") _, _, _ = c.Get([]byte{0x00}) _, _, _ = c.Get([]byte{0x12}) @@ -166,8 +166,8 @@ func TestBranchCache_Clear(t *testing.T) { // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("rrr"), 0, "test") - c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, "test") + c.Put([]byte{0x00}, []byte("rrr"), 0, 0, "test") + c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, 0, "test") _, _, _ = c.Get([]byte{0x00}) _, _, _ = c.Get([]byte{0x12, 0x34}) _, _, _ = c.Get([]byte{0xff}) // tail miss @@ -185,3 +185,107 @@ func TestBranchCache_Stats(t *testing.T) { // Sanity: format doesn't blow up if we read it require.True(t, strings.HasPrefix(s, "branch-cache ")) } + +// TestBranchCache_UnwindTo_EvictsByTxNWatermark verifies that UnwindTo +// evicts every entry whose txN > watermark across root + pinned + tail +// tiers, and that entries with txN <= watermark survive untouched. +func TestBranchCache_UnwindTo_EvictsByTxNWatermark(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} + pinnedKey := []byte{0x12, 0x34, 0x56} + tailKeyKeep := []byte{0xa0, 0xb0} + tailKeyEvict := []byte{0xa0, 0xb1} + + // txN=50 entries in every tier — these should survive a watermark of 60. + c.Put(rootKey, []byte("root-keep"), 0, 50, "test") + c.PinEntry(pinnedKey, []byte("pinned-keep"), 0, 50, "test") + c.Put(tailKeyKeep, []byte("tail-keep"), 0, 50, "test") + + // txN=100 tail entry — should be evicted at watermark=60. + c.Put(tailKeyEvict, []byte("tail-evict"), 0, 100, "test") + + evicted := c.UnwindTo(60) + require.Equal(t, 1, evicted, "only the txN=100 tail entry should be evicted") + + _, _, ok := c.Get(rootKey) + require.True(t, ok, "root entry with txN=50 must survive watermark=60") + _, _, ok = c.Get(pinnedKey) + require.True(t, ok, "pinned entry with txN=50 must survive watermark=60") + _, _, ok = c.Get(tailKeyKeep) + require.True(t, ok, "tail entry with txN=50 must survive watermark=60") + _, _, ok = c.Get(tailKeyEvict) + require.False(t, ok, "tail entry with txN=100 must be evicted by watermark=60") +} + +// TestBranchCache_UnwindTo_EvictsAcrossAllTiers verifies eviction reaches +// the root + pinned tiers, not just the LRU tail. +func TestBranchCache_UnwindTo_EvictsAcrossAllTiers(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} + pinnedKey := []byte{0x12, 0x34, 0x56} + tailKey := []byte{0xa0, 0xb0} + + c.Put(rootKey, []byte("root"), 0, 100, "test") + c.PinEntry(pinnedKey, []byte("pinned"), 0, 100, "test") + c.Put(tailKey, []byte("tail"), 0, 100, "test") + + evicted := c.UnwindTo(50) + require.Equal(t, 3, evicted, "every entry with txN > watermark must be evicted") + + _, _, ok := c.Get(rootKey) + require.False(t, ok, "root entry must be evicted") + _, _, ok = c.Get(pinnedKey) + require.False(t, ok, "pinned entry must be evicted (pinning protects capacity, not correctness)") + _, _, ok = c.Get(tailKey) + require.False(t, ok, "tail entry must be evicted") +} + +// TestBranchCache_UnwindTo_TxNZeroIsImmortal verifies that entries +// tagged with txN=0 ("not tracked") are NEVER evicted by any watermark. +// Preserves back-compat with callers that haven't been migrated to +// pass real txN values. +func TestBranchCache_UnwindTo_TxNZeroIsImmortal(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} + pinnedKey := []byte{0x12, 0x34, 0x56} + tailKey := []byte{0xa0, 0xb0} + + c.Put(rootKey, []byte("root"), 0, 0, "test") + c.PinEntry(pinnedKey, []byte("pinned"), 0, 0, "test") + c.Put(tailKey, []byte("tail"), 0, 0, "test") + + evicted := c.UnwindTo(0) + require.Equal(t, 0, evicted, "no entry with txN=0 should be evicted") + evicted = c.UnwindTo(1_000_000) + require.Equal(t, 0, evicted, "no entry with txN=0 should be evicted even at huge watermark") + + _, _, ok := c.Get(rootKey) + require.True(t, ok, "txN=0 root entry must survive any watermark") + _, _, ok = c.Get(pinnedKey) + require.True(t, ok, "txN=0 pinned entry must survive any watermark") + _, _, ok = c.Get(tailKey) + require.True(t, ok, "txN=0 tail entry must survive any watermark") +} + +// TestBranchCache_UnwindTo_BoundaryEqualsWatermark verifies the +// inequality at the watermark itself: entries at exactly txN==watermark +// are NOT evicted (the rule is txN > watermark). +func TestBranchCache_UnwindTo_BoundaryEqualsWatermark(t *testing.T) { + c := NewBranchCache(100) + + atKey := []byte{0xa0, 0xb0} + aboveKey := []byte{0xa0, 0xb1} + c.Put(atKey, []byte("at"), 0, 100, "test") + c.Put(aboveKey, []byte("above"), 0, 101, "test") + + evicted := c.UnwindTo(100) + require.Equal(t, 1, evicted, "only the txN=101 entry must be evicted; txN=100 stays") + + _, _, ok := c.Get(atKey) + require.True(t, ok, "entry at txN==watermark must survive") + _, _, ok = c.Get(aboveKey) + require.False(t, ok, "entry at txN>watermark must be evicted") +} diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 88fa7cc2677..d578d43d914 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -124,7 +124,7 @@ func (p *ContractTrunkPreload) Run( break } - cache.PinEntry(prefix, v, step, "preload-trunk") + cache.PinEntry(prefix, v, step, 0, "preload-trunk") // Copy prefix because nibbles.HexToCompact may return a slice // over a reused buffer; we need a stable handle for later // Invalidate on demotion. diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 6f8374a0d32..1e8ff8a464f 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -164,7 +164,7 @@ func (p *ContractTrunkPreloadParallel) Run( budgetHit = true return false } - cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel-resumable") + cache.PinEntry(pk.key, v, 0, 0, "preload-trunk-parallel-resumable") kc := make([]byte, len(pk.key)) copy(kc, pk.key) p.pinnedPrefixes = append(p.pinnedPrefixes, kc) diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index e1c58b2dc35..a8d114c2fed 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -41,7 +41,7 @@ func fakeReader(saturated bool) CommitmentReader { func TestPinEntry_AndPinnedCount(t *testing.T) { c := NewBranchCache(8) require.Equal(t, 0, c.PinnedCount()) - c.PinEntry([]byte{0x12, 0x34, 0x56}, []byte{0xab, 0xcd}, 1, "test") + c.PinEntry([]byte{0x12, 0x34, 0x56}, []byte{0xab, 0xcd}, 1, 0, "test") require.Equal(t, 1, c.PinnedCount()) // Pinned entry is hit, not the tail. @@ -56,7 +56,7 @@ func TestPinEntry_AndPinnedCount(t *testing.T) { // deletes a pinned entry, not just LRU/root. func TestInvalidate_RemovesFromPinned(t *testing.T) { c := NewBranchCache(8) - c.PinEntry([]byte{0x12, 0x34}, []byte{0x99}, 1, "test") + c.PinEntry([]byte{0x12, 0x34}, []byte{0x99}, 1, 0, "test") require.Equal(t, 1, c.PinnedCount()) c.Invalidate([]byte{0x12, 0x34}) @@ -69,7 +69,7 @@ func TestInvalidate_RemovesFromPinned(t *testing.T) { // pinned entries (was a correctness gap pre-fix). func TestGetWithOrigin_ChecksPinnedTier(t *testing.T) { c := NewBranchCache(8) - c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 5, "preload") + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 5, 0, "preload") data, origin, _, _, ok := c.GetWithOrigin([]byte{0x12, 0x34}) require.True(t, ok) require.Equal(t, []byte{0xab}, data) @@ -80,7 +80,7 @@ func TestGetWithOrigin_ChecksPinnedTier(t *testing.T) { // pinned-stats counters (was a correctness gap pre-fix). func TestClear_ResetsPinnedTier(t *testing.T) { c := NewBranchCache(8) - c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 1, "test") + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 1, 0, "test") _, _, _ = c.Get([]byte{0x12, 0x34}) // bump pinned hit c.Clear() require.Equal(t, 0, c.PinnedCount()) @@ -99,7 +99,7 @@ func TestMissCallback_FiresOnTripleMiss(t *testing.T) { }) // Hit (pinned): callback should NOT fire. - c.PinEntry([]byte{0x12, 0x34}, []byte{0xff}, 1, "test") + c.PinEntry([]byte{0x12, 0x34}, []byte{0xff}, 1, 0, "test") _, _, _ = c.Get([]byte{0x12, 0x34}) require.Equal(t, uint64(0), fired.Load(), "pinned hit must not fire callback") From f972499e1e502e0eb77e38b0155d80c8f25f128f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 21:11:50 +0000 Subject: [PATCH 096/167] db/state, db/state/execctx: plumb real txN to BranchCache writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 2+3 of the txN-tagged BranchCache rollout. Now every write to the cache carries a meaningful txN so UnwindTo (added in Step 1) can actually evict the right entries on unwind. - AggregatorRoTx gains MeteredGetLatestWithTxN + internal getLatestWithTxN for CommitmentDomain. DB-sourced reads tag with lastTxNumOfStep(step) (conservative high-water within the step — exact write txN isn't recoverable from the step-keyed DB record). File-sourced reads tag with fileEndTxNum (step-boundary exact; unwind can't cross into snapshots, so always <= any legal watermark). - sd.GetLatest type-switches on tx.AggTx() to prefer MeteredGetterWithTxN, falling back to legacy MeteredGetter (passes txN=0, "not tracked") and finally tx.GetLatest. The captured txN flows to branchCache.Put. - sd.Flush callback tags entries with sd.txNum (the frontier at flush time). UnwindTo can evict cache entries from any unwound batch via a single watermark pass. - Non-CommitmentDomain reads return txN=0 — only CommitmentDomain participates in the txN-tagged BranchCache today. Step 2+3/n of #21380 chosen-fix design — see pr-21380-txn-tagged-cache-design-2026-05-26 memory pin. --- db/state/aggregator.go | 50 ++++++++++++++++++++++++------- db/state/execctx/domain_shared.go | 28 ++++++++++++++--- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 387e4f4a906..68f49eaf639 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2482,36 +2482,66 @@ func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, return at.getLatest(domain, k, tx, maxStep, metrics, start) } +// MeteredGetLatestWithTxN is the txN-aware variant of MeteredGetLatest. +// In addition to (value, step), it returns the high-water txNum the +// returned value corresponds to — used to tag BranchCache entries so +// UnwindTo can evict by watermark. For non-CommitmentDomain reads, +// txN=0 ("not tracked"); only CommitmentDomain participates in the +// txN-tagged BranchCache today. +// +// Tagging conventions for CommitmentDomain: +// - DB-sourced: txN = lastTxNumOfStep(step) (conservative high-water +// within the step — exact write txN isn't recoverable from the +// step-keyed DB record). +// - File-sourced: txN = fileEndTxNum (exact; can't unwind into +// snapshots, so the high-water is safe by invariant). +func (at *AggregatorRoTx) MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { + return at.getLatestWithTxN(domain, k, tx, maxStep, metrics, start) +} + func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { + v, step, _, ok, err = at.getLatestWithTxN(domain, k, tx, maxStep, metrics, start) + return v, step, ok, err +} + +// getLatestWithTxN is the txN-aware variant of getLatest. It mirrors +// the source split (DB vs. files) and returns the high-water txNum the +// returned value corresponds to. See MeteredGetLatestWithTxN for the +// tagging convention. +func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { if domain != kv.CommitmentDomain { - return at.d[domain].getLatest(k, tx, maxStep, metrics, start) + // Non-Commitment domains don't participate in the txN-tagged + // BranchCache; return txN=0 ("not tracked"). The cache layer + // treats this as immortal-by-watermark. + v, step, ok, err = at.d[domain].getLatest(k, tx, maxStep, metrics, start) + return v, step, 0, ok, err } v, step, ok, err = at.d[domain].getLatestFromDb(k, tx) if err != nil { - return nil, kv.Step(0), false, err + return nil, kv.Step(0), 0, false, err } if ok && step <= maxStep { if metrics != nil && dbg.KVReadLevelledMetrics { metrics.UpdateDbReads(domain, start) } - return v, step, true, nil + // DB-sourced: high-water within the step (the actual write + // txN isn't recoverable from the step-keyed record). + return v, step, lastTxNumOfStep(step, at.StepSize()), true, nil } v, found, fileStartTxNum, fileEndTxNum, err := at.d[domain].getLatestFromFiles(k, 0) if !found { - return nil, kv.Step(0), false, err + return nil, kv.Step(0), 0, false, err } if metrics != nil && dbg.KVReadLevelledMetrics { - // UpdateFileReadsUnique tracks both total reads and distinct - // prefixes. The ratio FileReadCount / UniqueFileReadCount is - // the read amplification factor — useful when investigating - // whether the same prefixes are being re-read from file layer - // (cache misses on hot prefixes). metrics.UpdateFileReadsUnique(domain, k, start) } v, err = at.replaceShortenedKeysInBranch(k, commitment.BranchData(v), fileStartTxNum, fileEndTxNum) - return v, kv.Step(fileEndTxNum / at.StepSize()), found, err + // File-sourced: tag with fileEndTxNum (step boundary, exact). + // Snapshots are immutable and unwind can't cross into them, so + // fileEndTxNum is always <= any legal unwind watermark. + return v, kv.Step(fileEndTxNum / at.StepSize()), fileEndTxNum, found, err } func (at *AggregatorRoTx) DebugGetLatestFromDB(domain kv.Domain, key []byte, tx kv.Tx) ([]byte, kv.Step, bool, error) { diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 265a77892cc..f0bb1506ac6 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -780,7 +780,11 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.branchCache.Invalidate(k) return } - sd.branchCache.Put(k, v, uint64(step), 0, "sd.Flush") + // sd.txNum is the frontier at flush time; all entries in + // this batch were written at or before it. Tagging with + // sd.txNum lets UnwindTo evict cache entries from any + // unwound batch via a single watermark pass. + sd.branchCache.Put(k, v, uint64(step), sd.txNum, "sd.Flush") }); err != nil { return err } @@ -912,6 +916,15 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) type MeteredGetter interface { MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } + // MeteredGetterWithTxN is the txN-aware sibling. Used to tag + // CommitmentDomain BranchCache puts so UnwindTo can evict by + // watermark. AggregatorRoTx implements both; the cache-relevant + // read path prefers this one and falls back to MeteredGetter + // (passing txN=0, "not tracked") when only the legacy interface + // is available — e.g. test stubs. + type MeteredGetterWithTxN interface { + MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) + } // stateCache holds in-flight values from previous transactions in the same batch // that haven't been flushed to DB yet. Early return keeps correctness AND performance. @@ -955,9 +968,16 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } - if aggTx, ok := tx.AggTx().(MeteredGetter); ok { + // Capture txN from the read for BranchCache tagging. Prefer the + // txN-aware variant; fall back to the legacy method (txN stays 0, + // "not tracked" — cache entry survives any UnwindTo watermark). + var readTxN uint64 + switch aggTx := tx.AggTx().(type) { + case MeteredGetterWithTxN: + v, step, readTxN, _, err = aggTx.MeteredGetLatestWithTxN(domain, k, tx, maxStep, &sd.metrics, start) + case MeteredGetter: v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) - } else { + default: v, step, err = tx.GetLatest(domain, k) } if err != nil { @@ -969,7 +989,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) sd.stateCache.Put(domain, k, v) } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { - sd.branchCache.Put(k, v, uint64(step), 0, "sd.GetLatest") + sd.branchCache.Put(k, v, uint64(step), readTxN, "sd.GetLatest") } return v, step, nil From b93969affe95ba362dfca5cc75a298176e714d47 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 21:18:36 +0000 Subject: [PATCH 097/167] db/state/execctx, execution/commitment: wire BranchCache.UnwindTo into SD.Unwind, retire the doctrine comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SharedDomains.Unwind now invalidates the BranchCache via a single UnwindTo(txNumUnwindTo) call — every entry whose txN > watermark is evicted across root + pinned + LRU tail in one pass. This replaces the changeset[CommitmentDomain]-walking invalidation that was load-bearing for cache correctness but silently broken at every write site that lacked an active SetChangesetAccumulator (frozen-block commits, beyond-reorg commits, statetest paths). The txN-tagged cache is correct by construction regardless of where the writes originate. HexPatriciaHashed.Reset's doctrine comment is updated: the cache no longer requires explicit ClearBranchCache calls on unwind/fork- validation paths — SharedDomains.Unwind's UnwindTo handles it via the txN-tagged eviction. Step 4+5/n of #21380 chosen-fix design — see pr-21380-txn-tagged-cache-design-2026-05-26 memory pin. --- db/state/execctx/domain_shared.go | 20 +++++++++++--------- execution/commitment/hex_patricia_hashed.go | 5 +++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index f0bb1506ac6..5aa0cb4beb5 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -568,15 +568,17 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) - // After unwind the canonical commitment-domain values are rolled - // back; any cached entries for keys touched in the unwind window - // now hold stale bytes vs the post-unwind canonical state. Drop - // those specifically — the rest of the cache (including pinned - // branches whose keys weren't in the changeset) stays warm. - if sd.branchCache != nil && changeset != nil { - for _, diff := range changeset[kv.CommitmentDomain] { - sd.branchCache.Invalidate([]byte(diff.Key)) - } + // Drop every BranchCache entry whose txN > txNumUnwindTo via a + // single watermark pass. Self-sufficient invalidation — no + // dependency on changeset[CommitmentDomain] being correctly + // populated by upstream writers. The diffset-driven invalidation + // this replaced was load-bearing for cache correctness but silently + // broken at every write site that lacked an active + // SetChangesetAccumulator (frozen-block commits, beyond-reorg + // commits, statetest paths). The txN-tagged cache is correct by + // construction regardless of where the writes originate. + if sd.branchCache != nil { + sd.branchCache.UnwindTo(txNumUnwindTo) } } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 060057af560..7de86533cf7 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2971,8 +2971,9 @@ func (hph *HexPatriciaHashed) SetLeaveDeferredForCaller(leave bool) { // // The BranchCache is intentionally NOT cleared here: with aggregator-scope // lifetime the cache must survive between commitment calculations to deliver -// cross-block hits. Callers that need to invalidate the cache (unwind, fork -// validation) MUST call ClearBranchCache explicitly. +// cross-block hits. Unwind correctness is handled by the cache's own +// txN-tagged eviction in SharedDomains.Unwind — no explicit +// invalidation is required on the commitment path. func (hph *HexPatriciaHashed) Reset() { hph.root.reset() hph.rootTouched = false From c9a3deb6a305d323e79d059415246f49ed320225 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 23:48:08 +0000 Subject: [PATCH 098/167] execution/commitment, db/state, execmodule: trim comments per project policy + drop redundant SetHead cache clear MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cleanups: 1. SharedDomains.Unwind invokes branchCache.UnwindTo (added earlier in this PR), which evicts entries by txN watermark. The explicit sd.ClearBranchCache() in ExecModule.SetHead is now redundant — the unwind path already invalidates the right entries, and KeyCommitmentState is filtered out of the cache at Put time so it can't accumulate stale entries. Removed. With this change the BranchCache is never wholesale-cleared in production code paths; all invalidation flows through UnwindTo's watermark eviction. 2. Trim verbose comments per project policy ("default to writing no comment", keep WHY-only, drop restatements / task references / over-explanations of standard Go). Net -430 lines across 14 files, mostly multi-line doc blocks that restated what the code does or narrated PR context. No code logic changed; the WHY-comments that capture non-obvious invariants, workarounds, or surprising edge cases are preserved. --- db/state/aggregator.go | 34 +--- db/state/execctx/domain_shared.go | 25 +-- execution/commitment/adaptive_pin.go | 166 +++--------------- execution/commitment/branch_cache.go | 54 ++---- execution/commitment/branch_decode.go | 41 ++--- execution/commitment/commitment.go | 53 ++---- .../commitmentdb/commitment_context.go | 50 ++---- execution/commitment/hex_patricia_hashed.go | 8 +- execution/commitment/preload.go | 91 +++------- execution/commitment/preload_parallel.go | 162 ++++------------- execution/commitment/preload_ranges.go | 9 +- execution/commitment/trunk_pin_metrics.go | 21 +-- execution/commitment/warmuper.go | 37 +--- execution/execmodule/set_head.go | 9 - 14 files changed, 165 insertions(+), 595 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 68f49eaf639..9d8bc40b0ae 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2482,19 +2482,9 @@ func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, return at.getLatest(domain, k, tx, maxStep, metrics, start) } -// MeteredGetLatestWithTxN is the txN-aware variant of MeteredGetLatest. -// In addition to (value, step), it returns the high-water txNum the -// returned value corresponds to — used to tag BranchCache entries so -// UnwindTo can evict by watermark. For non-CommitmentDomain reads, -// txN=0 ("not tracked"); only CommitmentDomain participates in the -// txN-tagged BranchCache today. -// -// Tagging conventions for CommitmentDomain: -// - DB-sourced: txN = lastTxNumOfStep(step) (conservative high-water -// within the step — exact write txN isn't recoverable from the -// step-keyed DB record). -// - File-sourced: txN = fileEndTxNum (exact; can't unwind into -// snapshots, so the high-water is safe by invariant). +// MeteredGetLatestWithTxN returns the high-water txN alongside (value, +// step) for tagging BranchCache entries so UnwindTo can evict by +// watermark. Non-CommitmentDomain reads return txN=0. func (at *AggregatorRoTx) MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { return at.getLatestWithTxN(domain, k, tx, maxStep, metrics, start) } @@ -2504,15 +2494,8 @@ func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxSte return v, step, ok, err } -// getLatestWithTxN is the txN-aware variant of getLatest. It mirrors -// the source split (DB vs. files) and returns the high-water txNum the -// returned value corresponds to. See MeteredGetLatestWithTxN for the -// tagging convention. func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { if domain != kv.CommitmentDomain { - // Non-Commitment domains don't participate in the txN-tagged - // BranchCache; return txN=0 ("not tracked"). The cache layer - // treats this as immortal-by-watermark. v, step, ok, err = at.d[domain].getLatest(k, tx, maxStep, metrics, start) return v, step, 0, ok, err } @@ -2525,8 +2508,8 @@ func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, if metrics != nil && dbg.KVReadLevelledMetrics { metrics.UpdateDbReads(domain, start) } - // DB-sourced: high-water within the step (the actual write - // txN isn't recoverable from the step-keyed record). + // DB-sourced: tag with the step's high-water; the exact write + // txN isn't recoverable from the step-keyed record. return v, step, lastTxNumOfStep(step, at.StepSize()), true, nil } @@ -2535,12 +2518,13 @@ func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, return nil, kv.Step(0), 0, false, err } if metrics != nil && dbg.KVReadLevelledMetrics { + // UpdateFileReadsUnique tracks total + distinct prefixes; the + // ratio is the read amplification factor (hot prefixes re-read). metrics.UpdateFileReadsUnique(domain, k, start) } v, err = at.replaceShortenedKeysInBranch(k, commitment.BranchData(v), fileStartTxNum, fileEndTxNum) - // File-sourced: tag with fileEndTxNum (step boundary, exact). - // Snapshots are immutable and unwind can't cross into them, so - // fileEndTxNum is always <= any legal unwind watermark. + // File-sourced: tag with fileEndTxNum; snapshots are immutable and + // unwind can't cross them, so this is always <= any legal watermark. return v, kv.Step(fileEndTxNum / at.StepSize()), fileEndTxNum, found, err } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 5aa0cb4beb5..bc7e356106b 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -568,15 +568,6 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) - // Drop every BranchCache entry whose txN > txNumUnwindTo via a - // single watermark pass. Self-sufficient invalidation — no - // dependency on changeset[CommitmentDomain] being correctly - // populated by upstream writers. The diffset-driven invalidation - // this replaced was load-bearing for cache correctness but silently - // broken at every write site that lacked an active - // SetChangesetAccumulator (frozen-block commits, beyond-reorg - // commits, statetest paths). The txN-tagged cache is correct by - // construction regardless of where the writes originate. if sd.branchCache != nil { sd.branchCache.UnwindTo(txNumUnwindTo) } @@ -782,10 +773,6 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.branchCache.Invalidate(k) return } - // sd.txNum is the frontier at flush time; all entries in - // this batch were written at or before it. Tagging with - // sd.txNum lets UnwindTo evict cache entries from any - // unwound batch via a single watermark pass. sd.branchCache.Put(k, v, uint64(step), sd.txNum, "sd.Flush") }); err != nil { return err @@ -918,12 +905,9 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) type MeteredGetter interface { MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } - // MeteredGetterWithTxN is the txN-aware sibling. Used to tag - // CommitmentDomain BranchCache puts so UnwindTo can evict by - // watermark. AggregatorRoTx implements both; the cache-relevant - // read path prefers this one and falls back to MeteredGetter - // (passing txN=0, "not tracked") when only the legacy interface - // is available — e.g. test stubs. + // MeteredGetterWithTxN exposes the txN of the read so the + // BranchCache entry can be tagged; falls back to MeteredGetter + // when only the legacy interface is implemented (test stubs). type MeteredGetterWithTxN interface { MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) } @@ -970,9 +954,6 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } - // Capture txN from the read for BranchCache tagging. Prefer the - // txN-aware variant; fall back to the legacy method (txN stays 0, - // "not tracked" — cache entry survives any UnwindTo watermark). var readTxN uint64 switch aggTx := tx.AggTx().(type) { case MeteredGetterWithTxN: diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index c3668e01e7e..baff8961978 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -18,44 +18,17 @@ import ( ) // AdaptivePinControllerConfig sets the policy knobs for the adaptive -// trunk-pin controller. Defaults are tuned for the SSTORE-bloat -// workload class (single contract dominating storage reads); other -// workload classes may want different values. +// trunk-pin controller. Defaults target the SSTORE-bloat workload class +// (single contract dominating storage reads). type AdaptivePinControllerConfig struct { - // PromoteThresholdMisses — minimum cache misses (per block) for a - // contract to be promoted. Default 100. - PromoteThresholdMisses uint64 - - // MaxPromotedContracts — hard cap on simultaneously-pinned - // contracts. Bounds total pin RAM = MaxPromotedContracts × - // PerContractMaxBudgetBytes. Default 8 (× 64 MiB = 512 MiB max). - MaxPromotedContracts int - - // DemoteCooldownBlocks — number of consecutive blocks with zero - // misses for a promoted contract before it gets demoted (and its - // pin set invalidated). Default 5. - DemoteCooldownBlocks int - - // InitialViewBudgetBytes — RAM budget for the synchronous - // initial-view preload at promotion. Default 4 MiB (covers - // d=64-67 with headroom; ~1.4 s on cold disk). - InitialViewBudgetBytes int - - // ExtensionBudgetBytes — RAM budget for the per-block extension - // step on already-promoted contracts. Default 8 MiB (~25 K - // branches per block at typical entry cost). - ExtensionBudgetBytes int - - // PerContractMaxBudgetBytes — hard ceiling on the cumulative pin - // budget per contract. Extensions stop when this is reached even - // if the BFS queue isn't empty. Default 64 MiB (matches the - // per-contract max enforced in the env hook). + PromoteThresholdMisses uint64 + MaxPromotedContracts int + DemoteCooldownBlocks int + InitialViewBudgetBytes int + ExtensionBudgetBytes int PerContractMaxBudgetBytes int } -// DefaultAdaptivePinControllerConfig returns the production defaults. -// Conservative across the board so default-on shipping doesn't pin -// memory speculatively. func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { return AdaptivePinControllerConfig{ PromoteThresholdMisses: 100, @@ -69,52 +42,31 @@ func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { // AdaptivePinController watches per-contract miss pressure on a // BranchCache and decides which contracts to pin (with a sync initial -// view) and grow (per-block extension) or demote (invalidate the pin +// view), grow (per-block extension), or demote (invalidate the pin // set after sustained inactivity). -// -// Lifecycle: -// - Construct with NewAdaptivePinController, wire to a cache via Bind -// - On every triple-miss, the cache calls back; the controller -// records a per-contract miss count -// - The host (SD or stage loop) calls OnBlockComplete at block -// boundaries with a CommitmentReader factory; the controller -// then promotes / extends / demotes based on the per-block -// miss snapshot -// - Optional: host installs a parallel-mode resolver factory via -// SetParallelMode so promote/extend uses the wave-BFS parallel -// preload (file-only batch resolver + MDBX-overlay dbBranches) -// instead of the serial CommitmentReader BFS. type AdaptivePinController struct { cache *BranchCache cfg AdaptivePinControllerConfig logger log.Logger - // misses holds per-contract miss counts since last OnBlockComplete. - // sync.Map is appropriate: writes are frequent (every triple-miss), - // reads happen once per block boundary. misses sync.Map // [32]byte → *atomic.Uint64 - // states holds per-promoted-contract bookkeeping. Protected by mu. mu sync.Mutex states map[[32]byte]*adaptiveContractState - // Parallel-mode plumbing. nil = serial-BFS path (CommitmentReader). - // Set together via SetParallelMode; both required for parallel mode. parallelResolverFactory ParallelResolverFactory dbBranchesProvider DbBranchesProvider } // ParallelResolverFactory builds a fresh BatchBranchResolver for one // OnBlockComplete call. release() is invoked after the controller is done -// with the resolver (e.g. to tear down per-call worker txs). The factory -// may return (nil, nil, err) to indicate the controller should fall back -// to the serial-BFS path for this block. +// with the resolver. Returning (nil, nil, err) makes the controller fall +// back to the serial-BFS path for this block. type ParallelResolverFactory func() (resolve BatchBranchResolver, release func(), err error) // DbBranchesProvider returns the MDBX-resident branch overlay for one // contract — values shadow file values in the parallel preload's wave. -// Empty/nil result is valid (means "no MDBX overlay; resolver is -// authoritative"); the controller treats it as a non-error. +// Empty/nil result is valid (no overlay; resolver is authoritative). type DbBranchesProvider func(contractHash []byte) map[string][]byte type adaptiveContractState struct { @@ -125,7 +77,6 @@ type adaptiveContractState struct { coldBlocksInARow int } -// pinnedTotal returns the cumulative pinned-count regardless of preload mode. func (s *adaptiveContractState) pinnedTotal() int { if s.parallel != nil { return s.parallel.PinnedTotal() @@ -133,7 +84,6 @@ func (s *adaptiveContractState) pinnedTotal() int { return s.preload.PinnedTotal() } -// usedBytes returns the cumulative used-bytes regardless of preload mode. func (s *adaptiveContractState) usedBytes() int { if s.parallel != nil { return s.parallel.UsedBytes() @@ -141,7 +91,6 @@ func (s *adaptiveContractState) usedBytes() int { return s.preload.UsedBytes() } -// queueRemaining returns the un-processed queue size regardless of mode. func (s *adaptiveContractState) queueRemaining() int { if s.parallel != nil { return s.parallel.QueueRemaining() @@ -149,7 +98,6 @@ func (s *adaptiveContractState) queueRemaining() int { return s.preload.QueueRemaining() } -// pinnedPrefixes returns the pin set for cache invalidation on demote. func (s *adaptiveContractState) pinnedPrefixes() [][]byte { if s.parallel != nil { return s.parallel.PinnedPrefixes() @@ -157,10 +105,6 @@ func (s *adaptiveContractState) pinnedPrefixes() [][]byte { return s.preload.PinnedPrefixes() } -// NewAdaptivePinController constructs a controller bound to the given -// cache. Use Bind to install the cache miss-callback (separate so a -// caller can keep a controller for telemetry without wiring it into -// the read path). func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfig, logger log.Logger) *AdaptivePinController { if cfg.InitialViewBudgetBytes <= 0 { cfg.InitialViewBudgetBytes = 4 << 20 @@ -188,31 +132,16 @@ func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfi } } -// Bind installs the controller's miss-callback on the cache. After -// Bind, every triple-miss attributable to a storage-trunk prefix -// (length >= 33 B) is counted toward the corresponding contract. -// -// Safe to call multiple times — Bind replaces any prior callback. -// Call SetMissCallback(nil) on the cache directly to unbind. +// Bind installs the controller's miss-callback on the cache. +// Safe to call multiple times — replaces any prior callback. func (c *AdaptivePinController) Bind() { c.cache.SetMissCallback(c.onCacheMiss) } -// SetParallelMode switches the controller to the wave-BFS parallel preload -// (ContractTrunkPreloadParallel) for both initial-view and per-block -// extension. The factory builds a tx-scoped resolver per OnBlockComplete -// (file-only); the provider returns the MDBX-resident branch overlay per -// contract so the freshest values shadow file values. -// -// Either argument may be nil to clear it. When factory==nil the controller -// uses the serial-BFS CommitmentReader path (existing default behavior). -// When factory is set but the factory call returns nil resolver/err, the -// controller falls back to serial-BFS for that block — the saved parallel -// state survives the block and resumes on the next successful factory call. -// -// Must be called before the first OnBlockComplete; calling it later is -// allowed but changes behavior at the next block boundary only — already- -// promoted contracts keep their existing serial/parallel state. +// SetParallelMode switches promote/extend to the wave-BFS parallel preload. +// Either argument may be nil to clear; with factory==nil the controller uses +// the serial-BFS CommitmentReader path. Already-promoted contracts keep +// their existing serial/parallel state until next demote. func (c *AdaptivePinController) SetParallelMode(factory ParallelResolverFactory, provider DbBranchesProvider) { c.mu.Lock() defer c.mu.Unlock() @@ -230,28 +159,15 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { } // OnBlockComplete consumes the per-block miss snapshot and decides -// promotions, extensions, and demotions. Called by the host at block -// boundaries (after SD.Flush). The reader is the CommitmentReader -// for the just-committed state, used by initial-view preload and -// per-block extension. -// -// Synchronous: initial-view preloads run inline so the new pin set -// is available for the NEXT block's reads. Extensions also run -// inline; sized so per-block work fits within typical inter-block -// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). -// -// Logs a [adaptive-pin] line per block when any state changes or -// promoted contracts exist. +// promotions, extensions, and demotions. Synchronous — preloads run +// inline so the new pin set is available for the next block's reads. func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { misses := c.snapshotMisses() c.mu.Lock() defer c.mu.Unlock() - // Build the per-call parallel resolver up-front (one factory call per - // OnBlockComplete shared across all contracts this block). nil when - // parallel mode isn't installed, or the factory failed (we fall back to - // the serial-BFS reader path). + // One factory call per block, shared across all contracts. nil falls back to serial. var parallelResolve BatchBranchResolver var releaseParallel func() if c.parallelResolverFactory != nil { @@ -269,13 +185,11 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui var promoted, extended, demoted int - // Already-promoted contracts: extend on hot, demote on cold. for hash, state := range c.states { n, hadMisses := misses[hash] if hadMisses && n > 0 { state.coldBlocksInARow = 0 delete(misses, hash) - // Extend if budget remains and queue not empty. if state.queueRemaining() > 0 && state.usedBytes() < c.cfg.PerContractMaxBudgetBytes { remaining := c.cfg.PerContractMaxBudgetBytes - state.usedBytes() step := c.cfg.ExtensionBudgetBytes @@ -298,9 +212,6 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui } } - // New promotion candidates: contracts whose miss count crossed the - // threshold. Cap at MaxPromotedContracts; pick the highest-miss - // candidates first (greedy, simple, sufficient for v1). if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) for _, hash := range candidates { @@ -336,8 +247,6 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui } } -// snapshotMisses atomically reads + zeros every per-contract miss -// counter. Called once per OnBlockComplete. func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { out := make(map[[32]byte]uint64) c.misses.Range(func(k, v any) bool { @@ -351,8 +260,7 @@ func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { return out } -// demoteLocked invalidates every pinned prefix for the contract. -// Caller must hold c.mu. +// demoteLocked: caller must hold c.mu. func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { for _, prefix := range state.pinnedPrefixes() { c.cache.Invalidate(prefix) @@ -366,10 +274,7 @@ func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContr } } -// promoteLocked runs the initial-view preload for a freshly-promoted -// contract. Uses the parallel wave-BFS when parallelResolve != nil, -// otherwise falls back to the serial CommitmentReader BFS. On error the -// partial pin set is rolled back. Caller must hold c.mu. +// promoteLocked: caller must hold c.mu. On error the partial pin set is rolled back. func (c *AdaptivePinController) promoteLocked( ctx context.Context, hash [32]byte, @@ -398,7 +303,6 @@ func (c *AdaptivePinController) promoteLocked( parallel: p, }, nil } - // Serial BFS fallback (no parallel resolver this block). p, err := NewContractTrunkPreload(hash[:]) if err != nil { return nil, err @@ -416,14 +320,9 @@ func (c *AdaptivePinController) promoteLocked( }, nil } -// runExtensionLocked extends a promoted contract's preload by stepBudget -// bytes. Uses the saved state's mode (parallel vs serial). Caller must -// hold c.mu. -// -// When the saved state is serial (state.preload != nil) but parallelResolve -// is available this block, we keep using serial — switching mid-contract -// would lose the queue position. Future work: convert serial-state to -// parallel-state at extension time. +// runExtensionLocked: caller must hold c.mu. Uses the saved state's mode +// (parallel vs serial); a serial state with a parallel resolver available +// keeps using serial — switching mid-contract would lose the queue position. func (c *AdaptivePinController) runExtensionLocked( ctx context.Context, state *adaptiveContractState, @@ -433,8 +332,6 @@ func (c *AdaptivePinController) runExtensionLocked( ) error { if state.parallel != nil { if parallelResolve == nil { - // Parallel state but no resolver this block — skip the - // extension; the saved state survives for the next block. return nil } var dbBranches map[string][]byte @@ -444,14 +341,10 @@ func (c *AdaptivePinController) runExtensionLocked( _, _, err := state.parallel.Run(stepBudget, dbBranches, parallelResolve, c.cache, c.logger) return err } - // Serial-BFS state. _, _, err := state.preload.Run(stepBudget, reader, c.cache, c.logger) return err } -// PromotedContracts returns the hashes of currently-pinned contracts. -// Snapshot — the slice may be stale by the time the caller uses it. -// Used by metrics + debug diagnostics. func (c *AdaptivePinController) PromotedContracts() [][32]byte { c.mu.Lock() defer c.mu.Unlock() @@ -462,10 +355,6 @@ func (c *AdaptivePinController) PromotedContracts() [][32]byte { return out } -// pickPromotionCandidates selects up to maxN contract hashes from -// misses whose count exceeds threshold, preferring highest-miss first. -// Linear scan — adequate for the small N we expect (typically ≤16 -// candidates per block; sort is cheap). func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN int) [][32]byte { if maxN <= 0 { return nil @@ -481,7 +370,6 @@ func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN } } if len(pool) > maxN { - // Partial sort: O(N×maxN) for small maxN; avoids importing sort. for i := 0; i < maxN; i++ { best := i for j := i + 1; j < len(pool); j++ { @@ -506,6 +394,4 @@ func (c *AdaptivePinController) warnf(msg string, kv ...any) { } } -// ctx unused for now; reserved for future cancellation of in-flight -// preloads (R3 bulk preload + cancellable extension). -var _ = context.Background +var _ = context.Background // reserved for cancellation of in-flight preloads diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 310a8dd4199..2a33510bff8 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -271,16 +271,9 @@ type branchCacheEntry struct { // returned by aggTx.MeteredGetLatest / tx.GetLatest. step uint64 - // txN is the high-water txNum the cached bytes correspond to. Used by - // UnwindTo: an unwind to txNum T evicts every entry whose txN > T. - // Tagging conventions per write source: - // - sd.Flush → sd.txNum (exact, in-memory write) - // - snapshot file → file.endTxNum (step-boundary, exact under - // the no-unwind-into-snapshots invariant) - // - DB latest read → lastTxNumOfStep(step) (conservative - // high-water within the step) - // 0 means "not tracked" — entry survives any watermark (back-compat - // with callers that haven't been migrated to pass a real txN yet). + // txN is the high-water txN this entry is valid through. UnwindTo + // evicts every entry whose txN > watermark. 0 means "not tracked" + // (entry survives any watermark). txN uint64 } @@ -383,11 +376,9 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // (subject to Put updates from SD.Flush refreshing the bytes). Use for // eager preload of hot prefixes — e.g. the storage-trunk of big // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate -// the input after the call. -// -// txN tags the entry with its high-water validity; UnwindTo evicts a -// pinned entry when its txN exceeds the watermark. Pinning protects -// against capacity-pressure eviction, not correctness eviction. +// the input after the call. UnwindTo still evicts pinned entries when +// txN > watermark — pinning protects against capacity pressure, not +// correctness. func (c *BranchCache) PinEntry(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return @@ -503,11 +494,8 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // Put stores branch data in the cache, replacing any existing entry // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of -// caller buffer lifetime. step is the on-disk file step the bytes came -// from (0 if not tracked); txN is the high-water txNum the entry is -// valid through (0 if not tracked — entry survives any UnwindTo -// watermark); origin is a short label of the write site captured for -// divergence-detection diagnostics. +// caller buffer lifetime. origin is captured for divergence-detection +// diagnostics. See entry.txN for the txN tagging semantics. func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return @@ -594,31 +582,16 @@ func (c *BranchCache) Invalidate(prefix []byte) { c.tail.Delete(prefix) } -// UnwindTo evicts every cache entry whose txN > maxValidTxN across all -// tiers (root, pinned, LRU tail). Returns the number of entries evicted. -// -// This is the txN-tagged-cache invalidation primitive: a single watermark -// pass over the cache, no per-key changeset required. Entries with txN=0 -// ("not tracked") are NEVER evicted by the watermark — they're treated -// as permanently valid (callers that don't tag pay no watermark cost, -// preserving back-compat for migration). -// -// Pinned entries ARE evicted by watermark — pinning protects against -// capacity-pressure eviction, not correctness eviction. -// -// Concurrency: safe to call alongside concurrent reads. Writes during -// UnwindTo may produce a slightly larger effective working set than the -// post-call watermark suggests (a Put racing with the scan may insert an -// entry the scan already passed), but every such Put carries its own txN -// and will be caught by the next UnwindTo if the entry violates the -// invariant. +// UnwindTo evicts every cache entry whose txN > maxValidTxN across +// root, pinned, and LRU tail. Returns the number of entries evicted. +// Safe to call alongside concurrent reads; a Put racing with the scan +// may insert an entry the scan already passed, but the next UnwindTo +// will catch it. func (c *BranchCache) UnwindTo(maxValidTxN uint64) (evicted int) { - // Root tier — single slot, atomic check-and-clear. if entry := c.root.Load(); entry != nil && entry.txN > maxValidTxN { c.root.Store(nil) evicted++ } - // Pinned tier — iterate and delete in place. c.pinned.Range(func(hash uint64, entry *branchCacheEntry) bool { if entry != nil && entry.txN > maxValidTxN { c.pinned.DeleteByHash(hash) @@ -626,7 +599,6 @@ func (c *BranchCache) UnwindTo(maxValidTxN uint64) (evicted int) { } return true }) - // LRU tail — iterate and delete in place. c.tail.Range(func(hash uint64, entry *branchCacheEntry) bool { if entry != nil && entry.txN > maxValidTxN { c.tail.DeleteByHash(hash) diff --git a/execution/commitment/branch_decode.go b/execution/commitment/branch_decode.go index 0e4c6b5ccc9..10093d92111 100644 --- a/execution/commitment/branch_decode.go +++ b/execution/commitment/branch_decode.go @@ -22,42 +22,23 @@ import ( "math/bits" ) -// BranchMaps captures the three branch-level bitmasks decoded alongside the -// per-cell payload. Held separately from the cells slice so callers can apply -// them into trie state without struct-assigning the whole [16]cell array. +// BranchMaps captures the three branch-level bitmasks decoded alongside +// the per-cell payload. type BranchMaps struct { - Bitmap uint16 // present-children bitmap (the canonical map encoded in the branch) - TouchMap uint16 // children that were touched in this branch (set by caller per deleted flag) + Bitmap uint16 // present-children bitmap (canonical encoded map) + TouchMap uint16 // children touched in this branch (per the deleted flag) AfterMap uint16 // children present after this commitment step } -// DecodeBranchInto parses the on-disk encoded form of a branch (as returned -// by ctx.Branch / readBranchAndCheckForFlushing with the leading 2-byte -// touch-map stripped) and populates the supplied cells array. +// DecodeBranchInto parses the on-disk encoded form of a branch into cells. +// branchData must already have the leading 2-byte touch-map prefix stripped. // -// branchData must already have the leading touch-map prefix stripped — the -// raw bytes returned from Branch() include it; callers strip it before -// invoking this function (matching the unfoldBranchNode call pattern). +// deleted=true → touchMap=bitmap, afterMap=0 (touched-but-not-present-after). +// deleted=false → touchMap=0, afterMap=bitmap (present-after). // -// deleted controls the touchMap/afterMap convention: -// - deleted=true → all decoded cells are touched-but-not-present-after -// (touchMap = bitmap, afterMap = 0) -// - deleted=false → all decoded cells are present-after (touchMap = 0, -// afterMap = bitmap) -// -// Returns the three branch-level maps for the caller to apply into their -// own state machine. The cells array is mutated in place. -// -// This is a PURE decode — it does NOT call deriveHashedKeys on the cells. -// Trie callers (HexPatriciaHashed.unfoldBranchNode) follow this with their -// own loop calling deriveHashedKeys per cell, since they have the keccak -// scratch buffer. Cache callers can skip deriveHashedKeys entirely until -// the cell is consumed by the trie. -// -// This function is the canonical decoder for branch data. Both -// unfoldBranchNode (which feeds the in-memory grid for fold/unfold) and -// future cache populators consume branches via this same path so the -// encoded→cells transformation lives in one place. +// Pure decode — does NOT call deriveHashedKeys on the cells. Trie callers do +// that themselves (they have the keccak scratch buffer); cache callers can +// defer it until the cell is consumed by the trie. func DecodeBranchInto( branchData []byte, deleted bool, diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 612b06080f9..741f2dea700 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -100,11 +100,9 @@ type Trie interface { SetCapture(capture []string) GetCapture(truncate bool) []string EnableCsvMetrics(filePathPrefix string) - // SetBranchCache attaches a BranchCache instance for branch read/write - // caching across the trie + branchEncoder. ConcurrentPatriciaHashed - // implementations propagate the same instance to all mounts so the - // concurrency contract documented in branch_cache.go is respected - // (single shared cache, partitioned-by-prefix writers). + // SetBranchCache attaches the shared cache used by trie + branchEncoder. + // ConcurrentPatriciaHashed propagates the same instance to all mounts — + // see branch_cache.go for the concurrency contract. SetBranchCache(*BranchCache) // Variant returns commitment trie variant @@ -154,12 +152,9 @@ const ( VariantConcurrentHexPatricia TrieVariant = "hex-concurrent-patricia-hashed" ) -// InitializeTrieAndUpdates constructs the trie + updates buffer for a -// SharedDomainsCommitmentContext. The branchCache argument is the -// long-lived (aggregator-scope) cache shared across SDs — pass the cache -// returned by BranchCacheProvider on the AggregatorRoTx. When nil (test -// helpers, isolated benchmarks), a fresh per-init cache is created so the -// trie still has a valid cache to read/write through. +// InitializeTrieAndUpdates constructs the trie + updates buffer. branchCache +// should be the aggregator-scope shared cache; nil falls back to a fresh +// per-init cache (test helpers, isolated benchmarks). func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string, branchCache *BranchCache) (Trie, *Updates) { if branchCache == nil { branchCache = NewBranchCache(DefaultBranchCacheTailCapacity) @@ -364,10 +359,7 @@ type BranchEncoder struct { deferUpdates bool deferred []*DeferredBranchUpdate pendingPrefixes *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates - // branchCache, when set, receives mark-dirty-before-encode + Put-after- - // canonical-write so the cache stays consistent with ctx.PutBranch. - // Set via HexPatriciaHashed.SetBranchCache (which propagates here). - branchCache *BranchCache + branchCache *BranchCache // set via HexPatriciaHashed.SetBranchCache } func NewBranchEncoder(sz uint64) *BranchEncoder { @@ -583,11 +575,9 @@ func (be *BranchEncoder) CollectUpdate( var prev []byte var err error - // Mark the BranchCache entry dirty BEFORE doing the encode work. Any - // concurrent warmer-style writer that races into PutIfClean for this - // prefix between now and our final Put below will see the dirty flag - // and skip — preventing a stale read from overwriting our fresh - // canonical write. See branch_cache.go's Concurrency Contract. + // MarkDirty BEFORE encode so any concurrent PutIfClean for this prefix + // skips — preventing a stale read from overwriting our canonical write. + // See branch_cache.go's Concurrency Contract. if be.branchCache != nil { be.branchCache.MarkDirty(prefix) } @@ -617,13 +607,9 @@ func (be *BranchEncoder) CollectUpdate( if err = ctx.PutBranch(prefixCopy, updateCopy, prev); err != nil { return err } - // BranchCache update happens lazily: ctx.PutBranch writes only to sd.mem, - // which masks any cached value at this prefix for the rest of the block. - // On sd.Flush, sd.mem is moved to MDBX and BranchCache is invalidated + - // repopulated for the affected prefixes (see SD.Flush). The MarkDirty - // above protects against concurrent warmer-style writers between now and - // that flush. Single writer per prefix per fold invariant means no - // concurrent Put races on this key. + // No cache Put here: ctx.PutBranch writes to sd.mem which masks this + // prefix for the rest of the block; BranchCache is invalidated and + // repopulated at SD.Flush. if be.metrics != nil { be.metrics.updateBranch.Add(1) } @@ -657,13 +643,9 @@ func (be *BranchEncoder) CollectDeferredUpdate( be.ClearDeferred() } - // Mark BranchCache entry dirty BEFORE the deferred enqueue. Mirrors - // the CollectUpdate path so that both immediate and deferred encoder - // entries consistently invalidate the cache. Concurrent warmer-style - // writers calling PutIfClean for this prefix between now and the - // eventual ApplyDeferredBranchUpdates write see the dirty flag and - // skip — preventing a stale read from racing the deferred write. - // See branch_cache.go's Concurrency Contract. + // MarkDirty as in CollectUpdate — the deferred write happens later via + // ApplyDeferredBranchUpdates but the cache must skip concurrent + // PutIfClean writers in the interim. if be.branchCache != nil { be.branchCache.MarkDirty(prefix) } @@ -1823,9 +1805,6 @@ func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, var prevKey []byte err := t.etl.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error { - // (previously called warmuper.Cache().EvictPlainKey(v) here to drop - // stale account/storage entries; the warmer's cache was removed - // alongside WarmupCache so there's nothing to evict.) // Copy into arena since ETL may reuse buffers hk := t.arenaAlloc(k) pk := t.arenaAlloc(v) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 874c5f651fa..4c50d6bad3a 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -42,17 +42,10 @@ type sd interface { Trace() bool CommitmentCapture() bool - // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) - // for one key. Used by the BranchCache divergence-detection probe - // to localise which state layer holds bytes that diverge from - // cache. Read-only. + // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one + // key — BranchCache divergence-detection probe. Read-only. ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) - // Metrics exposes the per-SD DomainMetrics so callers can read - // per-domain (cache, db, file) read counters. Used by the - // cache-fp log line to break the aggregate `files=N` count down - // per domain (Storage value loads vs Commitment branch reads - // vs Account loads). Metrics() *changeset.DomainMetrics } @@ -190,11 +183,9 @@ func (sdc *SharedDomainsCommitmentContext) EnableCsvMetrics(filePathPrefix strin sdc.patriciaTrie.EnableCsvMetrics(filePathPrefix) } -// NewSharedDomainsCommitmentContext constructs a per-SharedDomains -// commitment context. branchCache is the aggregator-scope cache shared -// across all SDs derived from the same Aggregator; pass nil from test -// helpers that don't have an aggregator (a fresh per-init cache will be -// created inside InitializeTrieAndUpdates as a fallback). +// NewSharedDomainsCommitmentContext: branchCache is the aggregator-scope +// shared cache; nil from test helpers falls back to a per-init cache inside +// InitializeTrieAndUpdates. func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant commitment.TrieVariant, tmpDir string, branchCache *commitment.BranchCache) *SharedDomainsCommitmentContext { ctx := &SharedDomainsCommitmentContext{ sharedDomains: sd, @@ -326,7 +317,6 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context keysPerSec = uint64(float64(updateCount) / took.Seconds()) } log.Debug("[commitment] processed", "block", blockNum, "txNum", txNum, "keys", common.PrettyCounter(updateCount), "keys/s", common.PrettyCounter(keysPerSec), "mode", sdc.updates.Mode(), "spent", took, "rootHash", hex.EncodeToString(rootHash)) - }() if updateCount == 0 { rootHash, err = sdc.patriciaTrie.RootHash() @@ -410,11 +400,6 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context }() } - // Note: a previous EnableWarmupCache(false)+defer(true) guard lived here - // to ensure RecordingContext saw every read during trace capture. Now - // that the Go-side WarmupCache is gone, every read already goes through - // ctx.* (observable to the recorder by construction); no guard needed. - var warmupConfig commitment.WarmupConfig var drainCollectors func() []*etl.Collector if sdc.paraTrieDB != nil { @@ -590,9 +575,8 @@ func (e *errorTrieContext) Storage(plainKey []byte) (*commitment.Update, error) return nil, e.err } -// KeyCommitmentState is the commitment-domain key under which the latest -// root hash and tree state are stored. Single definition lives in the -// commitment package so BranchCache can exclude it by construction. +// KeyCommitmentState aliases commitment.KeyCommitmentState — single source of +// truth so BranchCache can exclude it by construction. var KeyCommitmentState = commitment.KeyCommitmentState var ErrBehindCommitment = errors.New("behind commitment") @@ -796,11 +780,7 @@ type TrieContext struct { stateReader StateReader localCollector *etl.Collector // per-goroutine collector for concurrent PutBranch - // probeSd / probeTx feed ProbeStateLayers — diagnostics-only fields - // populated when this TrieContext is constructed from a - // SharedDomainsCommitmentContext that has both a SharedDomains - // and a tx in scope. Both nil for read-only / test contexts where - // the probe is not available. + // Diagnostics-only — both nil for read-only / test contexts. probeSd sd probeTx kv.TemporalTx } @@ -824,11 +804,9 @@ func (sdc *TrieContext) Branch(pref []byte) ([]byte, kv.Step, error) { return common.Copy(enc), step, nil } -// ProbeStateLayers samples the underlying state layers for one key — -// sd.mem, sd.parent.mem (if any), and tx-direct (MDBX) — for -// divergence-detection diagnostics. Returns empty / not-ok when -// constructed without a probe-capable SharedDomains (e.g. -// NewTrieContextRo). Read-only. +// ProbeStateLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one +// key — divergence diagnostics. Returns empty / not-ok when constructed +// without a probe-capable SharedDomains (e.g. NewTrieContextRo). func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { if sdc.probeSd == nil { return @@ -836,10 +814,8 @@ func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, par return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) } -// SiteIdentity returns a stable string identifying the underlying -// SharedDomains pointer. Used by cache write sites to tag entries with -// the SD lineage that produced them, so divergence diagnostics can tell -// parent-SD writes apart from fork-SD writes when the cache is shared. +// SiteIdentity tags cache entries with the SD lineage that produced them so +// divergence diagnostics can tell parent-SD writes from fork-SD writes. func (sdc *TrieContext) SiteIdentity() string { return fmt.Sprintf("sd=%p", sdc.probeSd) } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 7de86533cf7..464f10995ed 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2968,12 +2968,8 @@ func (hph *HexPatriciaHashed) SetLeaveDeferredForCaller(leave bool) { } // Reset allows HexPatriciaHashed instance to be reused for the new commitment calculation. -// -// The BranchCache is intentionally NOT cleared here: with aggregator-scope -// lifetime the cache must survive between commitment calculations to deliver -// cross-block hits. Unwind correctness is handled by the cache's own -// txN-tagged eviction in SharedDomains.Unwind — no explicit -// invalidation is required on the commitment path. +// The aggregator-scope BranchCache is intentionally not cleared here; +// SharedDomains.Unwind handles correctness via txN-tagged eviction. func (hph *HexPatriciaHashed) Reset() { hph.root.reset() hph.rootTouched = false diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index d578d43d914..4d573ceecf7 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -16,37 +16,22 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// CommitmentReader is the read interface ContractTrunkPreload needs: -// GetLatest on the CommitmentDomain. Returns (value, step, found, err). -// Decoupled from any specific tx/aggregator type so callers can plug -// the reader in at preload time without dragging in db/state imports. +// CommitmentReader does GetLatest on the CommitmentDomain. Decoupled from +// tx/aggregator types to keep this package free of db/state imports. type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) -// estimatedEntryOverheadBytes accounts for the per-entry RAM cost -// beyond the encoded value itself: the branchCacheEntry struct -// (~80 B), maphash slot + hash (~40 B), prefix slice (~24 B header + -// content), value slice header (~24 B). Used to predict whether -// pinning the next entry would exceed the configured budget. +// estimatedEntryOverheadBytes is the per-entry RAM cost beyond the encoded +// value itself: branchCacheEntry (~80 B), maphash slot + hash (~40 B), +// prefix slice (~24 B header + content), value slice header (~24 B). const estimatedEntryOverheadBytes = 168 -// pathDepth pairs a nibble path with the trie depth it terminates at. -// Used as the BFS queue element. type pathDepth struct { path []byte depth int } -// ContractTrunkPreload holds the resumable state of a BFS preload for -// one contract's storage subtree. Created by NewContractTrunkPreload -// and advanced incrementally via Run; this lets callers split the -// preload into an initial sync view (small budget for d=64-67) and -// per-block extensions (additional budget while the contract stays -// hot), instead of burst-loading the full budget at promotion time. -// -// Tracks its pinned prefixes so callers can Invalidate the contract's -// pin set on demotion (PinnedPrefixes returns the slice). -// -// Caller MUST NOT use the same instance from multiple goroutines. +// ContractTrunkPreload holds the resumable state of a BFS preload for one +// contract's storage subtree. Not goroutine-safe. type ContractTrunkPreload struct { contractHash []byte queue []pathDepth @@ -56,9 +41,8 @@ type ContractTrunkPreload struct { maxDepthReached int } -// NewContractTrunkPreload constructs a preload state seeded at depth 64 -// (storage subtree root) for the given contract. The contract's -// storage subtree path is keccak256(address) — 32 bytes / 64 nibbles. +// NewContractTrunkPreload seeds a preload state at depth 64 (storage +// subtree root: keccak256(address), 32 bytes / 64 nibbles). func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) { if len(contractHash) != 32 { return nil, fmt.Errorf("NewContractTrunkPreload: contractHash must be 32 bytes, got %d", len(contractHash)) @@ -75,19 +59,10 @@ func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) }, nil } -// Run advances the BFS preload, pinning branches into cache until -// either the additional budget is exhausted or the queue is empty. -// Returns the number of entries pinned during THIS call (not the -// cumulative total — see PinnedTotal), whether the queue is now -// empty (preload fully complete), and any error from the reader. -// -// Calling Run repeatedly with small budgets implements the phased -// load: an initial Run with the InitialBudget covers depths 64-67; -// each subsequent Run extends BFS one chunk further. Per-block -// callers should size the chunk to fit within inter-block idle time. -// -// On reader error the partial pin set survives; the queue position -// is preserved so a retry on the next Run picks up where it left off. +// Run advances the BFS one chunk, pinning branches until additionalBudgetBytes +// is exhausted or the queue is empty. Returns entries pinned THIS call, whether +// the preload is now complete, and any reader error. On error the partial pin +// set and queue position are preserved for a retry on the next Run. func (p *ContractTrunkPreload) Run( additionalBudgetBytes int, reader CommitmentReader, @@ -119,15 +94,12 @@ func (p *ContractTrunkPreload) Run( entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) if chunkUsedBytes+entryCost > additionalBudgetBytes { - // Re-queue the head so a future Run picks it up. p.queue = append([]pathDepth{head}, p.queue...) break } cache.PinEntry(prefix, v, step, 0, "preload-trunk") - // Copy prefix because nibbles.HexToCompact may return a slice - // over a reused buffer; we need a stable handle for later - // Invalidate on demotion. + // HexToCompact may alias a reused buffer; copy for a stable Invalidate handle. prefixCopy := make([]byte, len(prefix)) copy(prefixCopy, prefix) p.pinnedPrefixes = append(p.pinnedPrefixes, prefixCopy) @@ -142,9 +114,7 @@ func (p *ContractTrunkPreload) Run( "used_mb", (p.usedBytes+chunkUsedBytes)/(1<<20)) } - // Decode bitmap and queue children. Branch encoding: - // 2-byte touchMap || 2-byte bitmap || per-child data. Only - // queue children that actually exist in the trie (bit set). + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. if len(v) < 4 { continue } @@ -165,35 +135,16 @@ func (p *ContractTrunkPreload) Run( return chunkPinned, len(p.queue) == 0, nil } -// PinnedTotal returns the cumulative number of branches pinned by -// this preload state across all Run calls. -func (p *ContractTrunkPreload) PinnedTotal() int { return p.pinned } - -// UsedBytes returns the cumulative byte cost of this preload's pin set. -func (p *ContractTrunkPreload) UsedBytes() int { return p.usedBytes } - -// QueueRemaining returns the number of paths still queued for descent. -// Zero means the preload is complete; further Run calls are no-ops. -func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } - -// MaxDepthReached returns the deepest trie depth pinned so far. +func (p *ContractTrunkPreload) PinnedTotal() int { return p.pinned } +func (p *ContractTrunkPreload) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } -// PinnedPrefixes returns the prefix slices this preload has added to -// the cache so far. The adaptive controller uses this to Invalidate -// the contract's pin set on demotion. The returned slice aliases -// internal storage; do not mutate. +// PinnedPrefixes returns slices aliasing internal storage — do not mutate. func (p *ContractTrunkPreload) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } +func (p *ContractTrunkPreload) ContractHash() []byte { return p.contractHash } -// ContractHash returns the contract this preload targets. -func (p *ContractTrunkPreload) ContractHash() []byte { return p.contractHash } - -// PreloadContractTrunk runs the full BFS preload for a contract in a -// single call, bounded by ramBudgetBytes. Convenience wrapper around -// NewContractTrunkPreload + Run for callers that don't need phased -// loading. Existing one-shot trigger paths (PIN_CONTRACT_TRUNKS env -// hook) use this; the adaptive layer holds a *ContractTrunkPreload -// and calls Run incrementally. +// PreloadContractTrunk is the one-shot wrapper around NewContractTrunkPreload + Run. func PreloadContractTrunk( contractHash []byte, ramBudgetBytes int, diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 1e8ff8a464f..abf7f937278 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -19,38 +19,28 @@ import ( ) // BatchBranchResolver resolves a batch of compact-encoded CommitmentDomain -// keys to their (already-dereferenced) branch-node values, reading from the -// file layer only — no MDBX cursor, hence no per-key cgo boundary crossing. -// The implementation is expected to fan the batch out across worker RoTxs; -// keys are passed in ascending key order so a contiguous-slice partition maps -// to contiguous file regions (drives page-cache readahead). Results MUST be -// returned in the same order as keys, with vals[i] == nil for keys not present -// in the file layer. -// -// Decoupled (callback form) so the commitment package needn't import db/state. +// keys to branch-node values, reading from the file layer only (no MDBX cursor, +// no per-key cgo crossing). Keys are passed in ascending key order so a +// contiguous-slice partition maps to contiguous file regions (page-cache +// readahead). Results MUST be returned in the same order as keys, with +// vals[i] == nil for keys not present. type BatchBranchResolver func(keys [][]byte) (vals [][]byte, err error) -// estimatedEntryCost is the predicted RAM cost of pinning a (key, value) branch -// entry — see estimatedEntryOverheadBytes in preload.go. func estimatedEntryCost(key, value []byte) int { return estimatedEntryOverheadBytes + len(key) + len(value) } -// minEntryBytes is a true lower bound on estimatedEntryCost for a storage-trunk +// minEntryBytes: true lower bound on estimatedEntryCost for a storage-trunk // branch (33 = shortest HexToCompact key at depth >= 64; value may be empty). -// Used to bound a wave's *file fetch*: capping at remaining/minEntryBytes+1 -// never under-fetches, so the budget is guaranteed exhausted inside the wave. +// Bounds a wave's file fetch so the budget is guaranteed exhausted inside it. const minEntryBytes = estimatedEntryOverheadBytes + 33 -// maxStorageTrunkDepth is the deepest total nibble path a storage branch can -// have: 64 (account path) + 64 (keccak256(slot)) = 128. +// maxStorageTrunkDepth: 64 (account path) + 64 (keccak256(slot)) = 128. const maxStorageTrunkDepth = 128 -// pathKey is a single frontier entry: the nibble path (1 byte / nibble) and the -// HexToCompact-encoded CommitmentDomain key. type pathKey struct { path []byte // nibble path (1 byte / nibble) - key []byte // HexToCompact(path) — the CommitmentDomain key + key []byte // HexToCompact(path) } func toPathKey(path []byte) pathKey { @@ -60,37 +50,17 @@ func toPathKey(path []byte) pathKey { return pathKey{path: path, key: kc} } -// ContractTrunkPreloadParallel is the resumable analogue of ContractTrunkPreload -// (the serial BFS via CommitmentReader). It walks the contract's storage subtree -// breadth-first one depth-level ("wave") at a time and resolves missing branches -// through a batched, file-only BatchBranchResolver — no MDBX boundary crossings -// in the hot path. Each Run advances zero or more complete waves bounded by a -// per-call step budget; partial waves can be truncated by the file-fetch cap to -// fit the budget, but the truncated tail's siblings are skipped (BFS at the -// frontier is then resumed at depth+1 from the children of whatever was pinned). -// -// Mirrors ContractTrunkPreload's API surface (Run/PinnedTotal/UsedBytes/ -// QueueRemaining/MaxDepthReached/PinnedPrefixes/ContractHash) so it slots in to -// adaptive_pin.go as a drop-in replacement without changing the controller's -// promote/extend/demote loop. +// ContractTrunkPreloadParallel is the wave-BFS analogue of ContractTrunkPreload. +// It walks one depth-level per wave and resolves missing branches through a +// batched, file-only BatchBranchResolver (no MDBX in the hot path). Each Run +// advances zero or more waves bounded by stepBudgetBytes; partial waves are +// truncated to fit the budget and resumed on the next Run. // -// Correctness: dbBranches values shadow file values for the same key (DB is -// authoritative for steps not yet flushed to files), so callers extending on a -// live node should pass the freshest MDBX-resident set per Run. With an empty -// dbBranches Run is a pure file-only preload — correct only from a cold snapshot -// or when MDBX/SD's flush-callback has already populated the cache with any -// freshly-written branches that would otherwise shadow the file values. +// dbBranches shadows file values for the same key — DB is authoritative for +// steps not yet flushed to files. Pass nil for cold-snapshot / file-only mode. // -// Caller MUST NOT use the same instance from multiple goroutines. The -// BatchBranchResolver is passed PER Run rather than held on the struct so -// callers can supply a fresh tx-scoped resolver each call (e.g. adaptive -// controller building a resolver from the in-flight Flush tx every block). -// -// Resumability: when a step budget cuts a wave mid-iteration, the un-processed -// items at the current depth are preserved in `frontier`; the children of -// the items that did get pinned this wave land in `pendingChildren` at depth -// nextDepth+1. The next Run finishes the current depth first, then advances. -// Once a wave fully completes, frontier := pendingChildren and nextDepth++. +// Not goroutine-safe. The resolver is passed per-Run (not held) so callers +// can supply a fresh tx-scoped resolver each block. type ContractTrunkPreloadParallel struct { contractHash []byte frontier []pathKey // paths to process at depth = nextDepth @@ -103,10 +73,7 @@ type ContractTrunkPreloadParallel struct { dbHitsPinned int } -// NewContractTrunkPreloadParallel seeds a resumable preload state at depth 64 -// (storage subtree root) for the given contract. The resolver is passed -// per-Run (not stored on the state) so callers can hand a fresh tx-scoped -// resolver to each call. +// NewContractTrunkPreloadParallel seeds a preload at depth 64 (storage subtree root). func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreloadParallel, error) { if len(contractHash) != 32 { return nil, fmt.Errorf("NewContractTrunkPreloadParallel: contractHash must be 32 bytes, got %d", len(contractHash)) @@ -121,20 +88,9 @@ func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreload }, nil } -// Run advances the wave-BFS, pinning branches into cache until either the step -// budget is exhausted, the frontier is empty (preload complete), or -// maxStorageTrunkDepth is reached. Returns newlyPinned during THIS call (not -// cumulative — see PinnedTotal), whether the preload is now complete, and any -// resolver error. -// -// dbBranches is consulted per wave (MDBX-resident overlays — values shadow file -// values, freshest). Pass nil for file-only mode (correct from a cold snapshot -// or when SD.Flush's cache callback has already populated freshly-written -// branches into the cache that would otherwise need to come from dbBranches). -// -// On resolver error the partial pin set survives; the wave position is -// preserved so a retry on the next Run picks up where it left off (at the same -// p.nextDepth). +// Run advances the wave-BFS until stepBudgetBytes is exhausted, the frontier +// is empty, or maxStorageTrunkDepth is reached. On resolver error the partial +// pin set and wave position survive for retry on the next Run. func (p *ContractTrunkPreloadParallel) Run( stepBudgetBytes int, dbBranches map[string][]byte, @@ -156,8 +112,7 @@ func (p *ContractTrunkPreloadParallel) Run( chunkPinned := 0 budgetHit := false - // pin records the entry, advances accounting, queues the branch's - // children. Returns false if the entry didn't fit (step budget exhausted). + // pin records the entry and queues its children. Returns false on budget hit. pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { cost := estimatedEntryCost(pk.key, v) if p.usedBytes+cost > stepCap { @@ -195,14 +150,9 @@ func (p *ContractTrunkPreloadParallel) Run( for !budgetHit && p.nextDepth <= maxStorageTrunkDepth && len(p.frontier) > 0 { depth := p.nextDepth - // Ascending key order so a contiguous-slice partition of the file batch - // is contiguous-in-file (page-cache readahead). BFS append order is - // already key-sorted (HexToCompact packing is monotone for same-length - // paths), but sort defensively — a few k entries / wave, cheap. + // Ascending key order so the file-batch partition is contiguous-in-file. sort.Slice(p.frontier, func(i, j int) bool { return bytes.Compare(p.frontier[i].key, p.frontier[j].key) < 0 }) - // Split this wave into DB-resident hits (have the value) and file-misses - // (need a fetch). Both stay sorted (sub-sequences of the sorted frontier). var dbHits []pathKey var dbVals [][]byte var fileMiss []pathKey @@ -217,10 +167,7 @@ func (p *ContractTrunkPreloadParallel) Run( } } - // Cap the *file fetch* at what the step budget can absorb after the - // DB-hits (which we pin first). minEntryBytes is a true lower bound, - // so this never under-fetches; if the cap truncates the wave, the - // un-fetched tail is preserved in p.frontier for the next Run. + // Cap the file fetch by what the budget can absorb after dbHits. var fileMissDeferred []pathKey if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget <= 0 { fileMissDeferred = fileMiss @@ -238,7 +185,6 @@ func (p *ContractTrunkPreloadParallel) Run( } fileVals, err = resolve(keys) if err != nil { - // State unchanged — retry on next Run picks up at same depth. return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", depth, err) } if len(fileVals) != len(keys) { @@ -246,10 +192,6 @@ func (p *ContractTrunkPreloadParallel) Run( } } - // Pin in dbHits-then-fileMiss order. Children of pinned items accumulate - // in p.pendingChildren (depth nextDepth+1). On budget hit mid-iteration, - // the un-processed remainder of dbHits/fileMiss + the deferred - // fileMissDeferred all become the new frontier (still at nextDepth). dbHitStop := len(dbHits) for i, pk := range dbHits { if !pin(pk, dbVals[i], depth, &p.pendingChildren) { @@ -263,7 +205,7 @@ func (p *ContractTrunkPreloadParallel) Run( for i, pk := range fileMiss { v := fileVals[i] if v == nil { - continue // not in the file layer either + continue } if !pin(pk, v, depth, &p.pendingChildren) { fileMissStop = i @@ -273,26 +215,18 @@ func (p *ContractTrunkPreloadParallel) Run( } if budgetHit { - // Preserve un-pinned items at the current depth for the next Run. - // Resolved-but-not-pinned file values are discarded (callable resolver - // will be re-invoked on the next Run for these keys). + // Preserve un-pinned items at current depth; pendingChildren stays + // at depth+1 for when this depth is drained on a future Run. rest := make([]pathKey, 0, len(dbHits)-dbHitStop+len(fileMiss)-fileMissStop+len(fileMissDeferred)) rest = append(rest, dbHits[dbHitStop:]...) rest = append(rest, fileMiss[fileMissStop:]...) rest = append(rest, fileMissDeferred...) p.frontier = rest - // Don't advance nextDepth — pendingChildren stays at depth+1 for - // when this depth is fully drained on a future Run. break } - // Wave fully processed. Anything in fileMissDeferred (won't have any - // since !budgetHit ⇒ no truncation) goes back to frontier; pending - // children promote to the next wave. if len(fileMissDeferred) > 0 { - // Shouldn't happen — if we got here without budgetHit, no truncation - // occurred. Defensive: re-queue them at the current depth so we - // don't lose data. + // Defensive: !budgetHit should mean no truncation. Re-queue at current depth. p.frontier = fileMissDeferred } else { p.frontier = p.pendingChildren @@ -318,43 +252,21 @@ func (p *ContractTrunkPreloadParallel) Run( return chunkPinned, queueEmpty, nil } -// PinnedTotal returns the cumulative number of branches pinned by this preload -// state across all Run calls. -func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } - -// UsedBytes returns the cumulative byte cost of this preload's pin set. -func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } +func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } +func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } +func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } -// QueueRemaining returns the number of paths still queued for descent -// (un-processed at the current depth + accumulated children at the next). -// Zero means the preload is complete; further Run calls are no-ops. func (p *ContractTrunkPreloadParallel) QueueRemaining() int { return len(p.frontier) + len(p.pendingChildren) } -// MaxDepthReached returns the deepest trie depth pinned so far. -func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } - -// PinnedPrefixes returns the prefix slices this preload has added to the cache -// so far. The adaptive controller uses this to Invalidate the contract's pin -// set on demotion. The returned slice aliases internal storage; do not mutate. +// PinnedPrefixes returns slices aliasing internal storage — do not mutate. func (p *ContractTrunkPreloadParallel) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } -// ContractHash returns the contract this preload targets. -func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } - -// DbHitsPinned returns the cumulative number of branches that were pinned from -// the dbBranches overlay (rather than via the file resolver). Diagnostic only. -func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } - -// PreloadContractTrunkParallel pins contract H's storage-trunk branches into -// cache in a single call, bounded by ramBudgetBytes. Convenience wrapper around -// NewContractTrunkPreloadParallel + Run for callers that don't need phased -// loading. The startup PIN_CONTRACT_TRUNKS hook uses this; the adaptive layer -// holds a *ContractTrunkPreloadParallel and calls Run incrementally. -// -// dbBranches is the MDBX-resident set — values shadow file values for the same -// key. Pass nil/empty for file-only mode (correct only from a cold snapshot). +// PreloadContractTrunkParallel is the one-shot wrapper around +// NewContractTrunkPreloadParallel + Run. func PreloadContractTrunkParallel( contractHash []byte, ramBudgetBytes int, diff --git a/execution/commitment/preload_ranges.go b/execution/commitment/preload_ranges.go index 7883ad95681..3e2f9b8406d 100644 --- a/execution/commitment/preload_ranges.go +++ b/execution/commitment/preload_ranges.go @@ -10,10 +10,8 @@ package commitment import "github.com/erigontech/erigon/execution/commitment/nibbles" -// NextSubtree returns the smallest key K' that is greater than every key having -// `in` as a prefix — i.e. the exclusive upper bound for a prefix-range scan over -// `in`. Returns nil if `in` is all 0xff (no such K'). Mirrors db/kv.NextSubtree; -// inlined to keep this package's import set minimal. +// NextSubtree: exclusive upper bound of a prefix-range scan over `in`. Returns +// nil if `in` is all 0xff. Mirrors db/kv.NextSubtree (inlined to keep imports minimal). func NextSubtree(in []byte) []byte { r := make([]byte, len(in)) copy(r, in) @@ -50,8 +48,7 @@ func ContractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, return evenFrom, evenTo, oddFrom, oddTo } -// ContractNibbles expands a 32-byte contract hash to its 64-nibble path -// (one nibble per byte, high nibble first). +// ContractNibbles expands a 32-byte hash to its 64-nibble path (high nibble first). func ContractNibbles(contractHash []byte) []byte { out := make([]byte, len(contractHash)*2) for i, b := range contractHash { diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go index 82c6b2e26fc..b16996278ee 100644 --- a/execution/commitment/trunk_pin_metrics.go +++ b/execution/commitment/trunk_pin_metrics.go @@ -12,38 +12,25 @@ import ( "github.com/erigontech/erigon/diagnostics/metrics" ) -// Prometheus metrics for the storage-trunk pin layer. Per-contract -// labels intentionally omitted — cardinality risk is real (mainnet -// could see hundreds of promoted contracts over a process lifetime) -// and the structured [adaptive-pin] log line gives per-contract -// detail when needed for debugging. +// Per-contract labels omitted to keep cardinality bounded; per-contract +// detail is in the [adaptive-pin] structured log line. var ( - // BranchCache pinned-tier counters. Surface the existing - // PinnedStats hits/misses/entries so dashboards can reason about - // pin effectiveness without scraping logs. mxPinnedHits = metrics.GetOrCreateCounter("commitment_branchcache_pinned_hits_total") mxPinnedMisses = metrics.GetOrCreateCounter("commitment_branchcache_pinned_misses_total") mxPinnedEntries = metrics.GetOrCreateGauge("commitment_branchcache_pinned_entries") - // AdaptivePinController lifecycle counters. mxAdaptivePromoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_promoted_total") mxAdaptiveExtended = metrics.GetOrCreateCounter("commitment_adaptive_pin_extended_total") mxAdaptiveDemoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_demoted_total") mxAdaptiveActive = metrics.GetOrCreateGauge("commitment_adaptive_pin_active_contracts") - // Preload-side counters for diagnostics. Per-contract preload - // duration is too high-cardinality for a histogram label; total - // duration spent in preload is the aggregate proxy. mxPreloadDurationSecondsTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_duration_seconds_total") mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") ) -// PublishMetrics snapshots the cache's current counters into the -// Prometheus registry. Counters are monotonic-add so the cache tracks -// last-published values internally and emits deltas. Gauges Set -// absolute. Call periodically from the host (e.g. once per SD.Flush); -// the once-per-batch cadence avoids hot-path cost. +// PublishMetrics emits counter deltas (last-published tracked internally) and +// sets gauges absolute. Call once per SD.Flush — once-per-batch avoids hot-path cost. func (c *BranchCache) PublishMetrics() { hits := c.pinnedHits.Load() misses := c.pinnedMisses.Load() diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index f71a73fa312..2e334ca6833 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -30,21 +30,16 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// Warmer outcome counters — measurement scaffolding to size the -// hit-rate of warmer branch reads. Hit means branchFromCacheOrDB -// returned >= 4 bytes (a real branch). Empty means it returned -// nothing or too short to parse (no branch at this depth in the -// on-disk trie). Used to bound the value of bypassing the xorfilter -// for the warmer specifically — high hit ratio implies the filter -// is mostly overhead in this call path. +// Warmer branch-read outcome counters. Hit: branchFromCacheOrDB returned +// >= 4 bytes; Empty: returned nothing or unparseable. Used to size the +// value of bypassing the xorfilter in this call path. var ( warmerBranchHitCount atomic.Uint64 warmerBranchEmptyCount atomic.Uint64 ) -// WarmerBranchOutcomeStats returns process-cumulative counts of -// branch-read outcomes from the warmer's depth walk. To get a -// per-block delta, snapshot before/after. +// WarmerBranchOutcomeStats returns process-cumulative counts. Snapshot +// before/after for per-block deltas. func WarmerBranchOutcomeStats() (hit, empty uint64) { return warmerBranchHitCount.Load(), warmerBranchEmptyCount.Load() } @@ -111,11 +106,6 @@ func NewWarmuper(ctx context.Context, cfg WarmupConfig) *Warmuper { } } -// branchFromCacheOrDB reads branch data via ctx.Branch. The Go-side -// warmup cache that previously sat above this has been removed; the -// aggregator-scope BranchCache (reached through SD.GetLatest) is the -// only branch cache. The function is kept as a thin wrapper to -// preserve the warmer-side error handling shape. func (w *Warmuper) branchFromCacheOrDB(trieCtx PatriciaContext, prefix []byte) ([]byte, error) { branchData, _, err := trieCtx.Branch(prefix) return branchData, err @@ -181,16 +171,6 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep } nextNibble := int(hashedKey[depth]) - // Account/storage prefetch removed: the warmer's scope is branch - // warm-up only. If a leaf hash needs underlying account/storage - // data during fold, that data is either (a) memoized as the - // cell's stateHash, (b) carried in the Updates buffer from the - // executor, or (c) faulted on the trie's own slow path — - // signalling a memoization gap that wants fixing at the trie - // layer, not papered over by a separate prefetcher. Counters - // disk_sto / disk_acc on the [commitment][cache-fp] log line - // expose any such fall-through. - branchData = branchData[2:] // skip touch map bitmap := binary.BigEndian.Uint16(branchData[0:2]) @@ -220,11 +200,8 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep fieldBits := branchData[pos] pos++ - // If the child cell is a leaf (holds a plain key), there is no - // branch below it — stop walking. Without this check the warmer - // pays one extra recsplit + xorfilter lookup per leaf-terminating - // path, all of which return empty. Mirrors what unfold does - // implicitly by traversing the actual trie structure. + // Leaf cell — no branch below. Without this the warmer pays one + // extra recsplit + xorfilter lookup per leaf-terminating path. if cellFields(fieldBits)&(fieldAccountAddr|fieldStorageAddr) != 0 { break } diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 0623b5bcaa6..91da37f13e1 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -149,15 +149,6 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { return fmt.Errorf("failed to flush shared domains: %w", err) } - // SetHead unwinds commitment history but the BranchCache (aggregator- - // scope, shared across SDs) still holds entries from the pre-unwind - // state — notably KeyCommitmentState pointing at the original head - // blockNum. The next FCU's NewSharedDomains.SeekCommitment would hit - // the cache, see the original-head blockNum, and trip - // ErrBehindCommitment against the now-truncated TxNums index. Clear - // the cache so subsequent reads see the post-unwind MDBX state. - sd.ClearBranchCache() - if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } From 4618add1c09a8c76d2726774a0b967bf98cd521a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 27 May 2026 07:09:04 +0000 Subject: [PATCH 099/167] execution/engineapi, db/state/execctx, execution/commitment: remove orphaned cache-clear API surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SharedDomains.Unwind already evicts BranchCache entries by txN watermark via the UnwindTo path. After dropping ExecModule.SetHead's redundant clear in the previous commit, none of these stayed live; they're all removed now so callers can't reach in to wipe the cache. - EngineXTestRunner.Run: drop tester.ResetBranchCache(). The tester is reused across (fork, preAllocHash) groups; the next test's NewPayload+FCU triggers a reorg-back-to-genesis through SD.Unwind, which evicts every entry with txN above the unwound watermark. - EngineApiTester.ResetBranchCache: orphan after the above. Removed. - SharedDomains.ClearBranchCache: orphan after the SetHead removal. Removed — SD owns cache lifecycle; callers must not bypass. - HexPatriciaHashed.ClearBranchCache: orphan. Removed. BranchCache.Clear (the underlying primitive) stays — still used by the older rawdbreset.ResetExec path (#21138) for table-wipe scenarios. --- db/state/execctx/domain_shared.go | 12 -------- execution/commitment/hex_patricia_hashed.go | 11 ------- .../engineapitester/engine_api_tester.go | 29 ------------------- .../engineapitester/engine_x_test_runner.go | 5 ---- 4 files changed, 57 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index bc7e356106b..180d2d9044e 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -851,18 +851,6 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return sd.mem.Flush(ctx, tx) } -// ClearBranchCache empties the aggregator-scope commitment BranchCache. -// Use after operations that mutate commitment state outside the normal -// sd.Flush callback path — notably SetHead unwind, which truncates -// commitment domain history but does not re-write all the keys whose -// cached values are now stale. Without this call, the next FCU sees -// the pre-unwind KeyCommitmentState and trips ErrBehindCommitment. -func (sd *SharedDomains) ClearBranchCache() { - if sd.branchCache != nil { - sd.branchCache.Clear() - } -} - // TemporalDomain satisfaction func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { if tx == nil { diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 464f10995ed..2ad57cc885f 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2977,17 +2977,6 @@ func (hph *HexPatriciaHashed) Reset() { hph.rootPresent = true } -// ClearBranchCache drops every entry in the attached BranchCache. Use on -// unwind or fork-validation paths where the cached branches diverge from -// the canonical store. No-op when no cache is attached or when this is a -// mounted subtrie (the root trie owns the clear, mounts share the same -// cache pointer). -func (hph *HexPatriciaHashed) ClearBranchCache() { - if hph.branchCache != nil && !hph.mounted { - hph.branchCache.Clear() - } -} - func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } diff --git a/execution/engineapi/engineapitester/engine_api_tester.go b/execution/engineapi/engineapitester/engine_api_tester.go index 1d9cd14a47d..407d1583b1f 100644 --- a/execution/engineapi/engineapitester/engine_api_tester.go +++ b/execution/engineapi/engineapitester/engine_api_tester.go @@ -41,7 +41,6 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" - dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/chain/networkname" @@ -403,34 +402,6 @@ func InitialiseEngineApiTester(ctx context.Context, args EngineApiTesterInitArgs }, nil } -// ResetBranchCache drops the aggregator's in-memory branchCache. The cache is -// aggregator-scoped and survives a tx rollback, so a tester reused across -// independent EEST cases would otherwise serve commitment-trie entries -// populated by the previous test's pre-state — producing a `wrong trie root -// of block 1` on the next test's genesis→block-1 boundary. Same mechanism the -// rawdbreset.ResetExec path uses after wiping the commitment table (#21138); -// here the trigger is "moving to a new test" instead of "wiping the table". -// Safe to call at any time the tester is between tests; no-op if no -// aggregator-backed db is attached. -func (eat EngineApiTester) ResetBranchCache() { - if eat.ChainDB == nil { - return - } - hasAgg, ok := eat.ChainDB.(dbstate.HasAgg) - if !ok { - return - } - agg, ok := hasAgg.Agg().(*dbstate.Aggregator) - if !ok { - return - } - aggTx := agg.BeginFilesRo() - if bc := aggTx.BranchCache(); bc != nil { - bc.Clear() - } - aggTx.Close() -} - type EngineApiTesterInitArgs struct { Logger log.Logger DataDir string diff --git a/execution/engineapi/engineapitester/engine_x_test_runner.go b/execution/engineapi/engineapitester/engine_x_test_runner.go index 0a01116a946..aaebe201e2d 100644 --- a/execution/engineapi/engineapitester/engine_x_test_runner.go +++ b/execution/engineapi/engineapitester/engine_x_test_runner.go @@ -222,11 +222,6 @@ func (extr *EngineXTestRunner) Run(ctx context.Context, test EngineXTestDefiniti if err != nil { return err } - // Reuse the cached tester for the (fork, preAllocHash) group but reset the - // aggregator-scope branchCache so commitment-trie entries from the prior - // test don't leak into this test's genesis→block-1 boundary. Cheaper than - // evicting the whole tester (which would rebuild the genesis + node). - tester.ResetBranchCache() return extr.execute(ctx, tester, test) } From 0dc7b1bec5c23720849a15b30d2f2b39e47680ff Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 27 May 2026 08:55:11 +0000 Subject: [PATCH 100/167] execution/commitment, db/state/execctx, cmd/evm: fix 4 Copilot review issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - branch_cache: ContractHashFromPrefix mis-attributed odd-length nibble paths. HexToCompact stores the first nibble in the flag byte's low half when the path has odd parity, so prefix[1:33] read the contract hash off by one nibble. Decoder now handles both parities; the adaptive pin controller's per-contract miss tracking is now correct for storage-trunk depths >= 65 with odd parity. - branch_cache: PutIfClean and MarkDirty went through lookup(), which bumps miss counters and fires the onMiss callback. Both are write-path operations — counting them as triple-misses inflated the promotion signal for the adaptive pin controller. Added a peek() helper that walks the same tiers without side effects; PutIfClean and MarkDirty use it instead. - domain_shared: EnableParaTrieDB's once-per-cache TryClaimPreload guard also gated the per-SD adaptive controller bind. After the first SD, subsequent SDs' controllers never installed their onMiss callback, leaving the cache pointing at a closed predecessor's handler. Split: TryClaimPreload still gates the env-driven PIN_CONTRACT_TRUNKS preload (intentional fire-once), but Bind() runs on every SD. - staterunner bench mode: RunNoVerify reused the SharedDomains the preceding test.Run had already mutated, so the timed work started from a dirty post-state. Bench now opens its own tx + SD so each iteration starts from the same pre-state as the test under measure. Tests: TestContractHashFromPrefix_EvenAndOddNibblePaths covers depth 64/65/66 round-trip. TestBranchCache_PutIfClean_DoesNotCountAsMiss pins the write-path-no-fire invariant for both PutIfClean and MarkDirty. --- cmd/evm/staterunner.go | 20 ++++++- db/state/execctx/domain_shared.go | 35 +++++------- execution/commitment/branch_cache.go | 42 +++++++++++---- execution/commitment/branch_cache_test.go | 65 +++++++++++++++++++++++ 4 files changed, 128 insertions(+), 34 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index b4f34b2bec4..67c1ab494b2 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -250,8 +250,26 @@ func runStateTest(ctx *cli.Context, cfg vm.Config, fname string) ([]testResult, } } if bench { + // Run the benchmark on a fresh tx + SD so the timed work + // starts from the same pre-state every iteration; the + // preceding test.Run mutated sd in place (pre-state + + // tx execution + commitment), which would otherwise + // skew the gas/throughput numbers. + benchTx, err := db.BeginTemporalRw(context.Background()) + if err != nil { + result.Pass, result.Error = false, err.Error() + return + } + defer benchTx.Rollback() + benchSD, err := execctx.NewSharedDomains(context.Background(), benchTx, log.New()) + if err != nil { + result.Pass, result.Error = false, err.Error() + return + } + defer benchSD.Close() + _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { - _, _, gasUsed, _ := test.RunNoVerify(nil, sd, tx, st, cfg, dirs) + _, _, gasUsed, _ := test.RunNoVerify(nil, benchSD, benchTx, st, cfg, dirs) return nil, gasUsed, nil }) result.Stats = &stats diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 180d2d9044e..8136724198d 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1273,36 +1273,25 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) - // Stage-init is the right firing point for both the env-driven - // forced preload AND the adaptive controller's bind: - // (a) we have a kv.TemporalRoDB to spawn an own-tx preload goroutine - // with — avoids the cgo-pointer panic that the earlier shared-tx - // attempt produced. - // (b) we run from the stage-init path, not an engine request handler, - // so we don't block engine HTTP responses during the 3-4s preload - // window — the synchronous-from-NewSharedDomains shape caused the - // bench's first NewPayload to be dropped, breaking block validation. - // TryClaimPreload guards fire-once-per-BranchCache-lifetime. - if sd.branchCache == nil || !sd.branchCache.TryClaimPreload() { + if sd.branchCache == nil { return } - // Operator-override path: PIN_CONTRACT_TRUNKS forces specific - // contracts to be pinned regardless of adaptive policy. Runs - // first so its pin set is in place before the controller binds. - if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { - triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) + // PIN_CONTRACT_TRUNKS preload is fire-once-per-BranchCache-lifetime: + // the cache is aggregator-scope and the preload's IO cost is paid up + // front; subsequent SDs reuse the already-populated pins. + if sd.branchCache.TryClaimPreload() { + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) + } } - // Bind the adaptive controller to the cache so it starts - // observing miss pressure for un-forced contracts. The reader - // for promote/extend preloads is constructed per-call from the - // in-flight Flush tx (see SD.Flush). + // Bind the adaptive controller on every SD: the controller is + // per-SD and Bind installs the cache's onMiss callback. Without + // this, SDs created after the first have inert controllers and + // the cache's onMiss may point at a closed predecessor. if sd.adaptivePinController != nil { sd.adaptivePinController.Bind() - sd.logger.Info("[adaptive-pin] enabled", - "max_promoted", commitment.DefaultAdaptivePinControllerConfig().MaxPromotedContracts, - "per_contract_max_mb", commitment.DefaultAdaptivePinControllerConfig().PerContractMaxBudgetBytes/(1<<20)) } } diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 2a33510bff8..9767ed54dfc 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -330,8 +330,6 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { c.rootHits.Add(1) return entry, true } - // Check pinned tier before tail. Pinned entries never evict and - // are fed by PinEntry (preload) + Put updates that preserve pin. if entry, ok := c.pinned.Get(prefix); ok { c.pinnedHits.Add(1) return entry, true @@ -347,6 +345,21 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { return entry, true } +// peek is the write-path equivalent of lookup: same tier walk, but does +// not bump hit/miss counters and does not fire the onMiss callback. +// Used by PutIfClean / MarkDirty so write traffic doesn't masquerade as +// read-miss pressure for the adaptive pin controller. +func (c *BranchCache) peek(prefix []byte) (*branchCacheEntry, bool) { + if isRootPrefix(prefix) { + entry := c.root.Load() + return entry, entry != nil + } + if entry, ok := c.pinned.Get(prefix); ok { + return entry, true + } + return c.tail.Get(prefix) +} + // fireOnMiss invokes the registered miss callback (if any). Hot path — // the no-callback case is a single atomic load and a nil check. func (c *BranchCache) fireOnMiss(prefix []byte) { @@ -422,18 +435,27 @@ func (c *BranchCache) SetMissCallback(cb MissCallback) { // ContractHashFromPrefix extracts the 32-byte contract hash from a // commitment-trunk prefix. Returns (hash, true) if the prefix is a -// storage-trunk prefix (depth >= 64, encoded as compact-hex with the -// contract's 64-nibble keccak prefix), or (_, false) for shorter +// storage-trunk prefix (depth >= 64, encoded as hex-prefix compact with +// the contract's 64-nibble keccak prefix), or (_, false) for shorter // prefixes (account-trie branches at depth < 64). // -// Compact-hex encoding: 1-byte HP flag, then nibble-pairs packed -// 2-per-byte. The contract's 64-nibble path occupies 32 bytes after -// the flag byte, so a storage-trunk prefix is at least 33 bytes. +// Hex-prefix encoding (see nibbles.HexToCompact): bit 4 of byte 0 is the +// odd-length flag. When set, the first nibble of the path lives in the +// low nibble of byte 0 and the remaining nibbles are packed 2-per-byte +// in buf[1:]; reading prefix[1:33] directly mis-attributes the hash by +// one nibble. Decoding via the high/low-nibble split here handles both +// parities. func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { if len(prefix) < 33 { return hash, false } - copy(hash[:], prefix[1:33]) + if prefix[0]&0x10 == 0 { + copy(hash[:], prefix[1:33]) + return hash, true + } + for i := 0; i < 32; i++ { + hash[i] = ((prefix[i] & 0x0f) << 4) | (prefix[i+1] >> 4) + } return hash, true } @@ -520,7 +542,7 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64, origin s // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step, txN uint64, origin string) bool { - if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { + if existing, ok := c.peek(prefix); ok && existing.dirty.Load() { return false } c.Put(prefix, data, step, txN, origin) @@ -558,7 +580,7 @@ func (c *BranchCache) GetWithOrigin(prefix []byte) (data []byte, origin string, // // No-op if no entry exists at prefix. func (c *BranchCache) MarkDirty(prefix []byte) { - if entry, ok := c.lookup(prefix); ok { + if entry, ok := c.peek(prefix); ok { entry.dirty.Store(true) } } diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index e8a29b8a0f1..9d43cde4306 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -18,9 +18,12 @@ package commitment import ( "strings" + "sync/atomic" "testing" "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/execution/commitment/nibbles" ) // TestBranchCache_RootPinning verifies the root branch lands in the pinned @@ -289,3 +292,65 @@ func TestBranchCache_UnwindTo_BoundaryEqualsWatermark(t *testing.T) { _, _, ok = c.Get(aboveKey) require.False(t, ok, "entry at txN>watermark must be evicted") } + +// TestContractHashFromPrefix_EvenAndOddNibblePaths verifies that the +// contract-hash extraction handles both parities of hex-prefix compact +// encoding (the odd-length-path flag puts the first nibble in byte 0, +// which a naive prefix[1:33] read mis-attributes by one nibble). +func TestContractHashFromPrefix_EvenAndOddNibblePaths(t *testing.T) { + var contract [32]byte + for i := range contract { + contract[i] = byte((i * 31) & 0xff) + } + contractNibbles := make([]byte, 64) + for i, b := range contract { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + + // Even-length path: depth 64 (no storage-trunk nibble). Encodes as + // flag byte (0x00) followed by the 32 contract bytes. + even := nibbles.HexToCompact(contractNibbles) + got, ok := ContractHashFromPrefix(even) + require.True(t, ok) + require.Equal(t, contract, got, "even-length path must round-trip") + + // Odd-length path: depth 65 (contract + one storage nibble). + odd := nibbles.HexToCompact(append(append([]byte(nil), contractNibbles...), 0x0a)) + got, ok = ContractHashFromPrefix(odd) + require.True(t, ok) + require.Equal(t, contract, got, "odd-length path must extract the contract hash, not a nibble-shifted slice") + + // Even-length path at depth 66 (contract + two storage nibbles). + even66 := nibbles.HexToCompact(append(append([]byte(nil), contractNibbles...), 0x0a, 0x05)) + got, ok = ContractHashFromPrefix(even66) + require.True(t, ok) + require.Equal(t, contract, got, "depth-66 even path must extract the contract hash") +} + +// TestBranchCache_PutIfClean_DoesNotCountAsMiss verifies that the +// write-path dirty check does not bump miss counters or fire the +// onMiss callback — write traffic must not masquerade as read pressure +// for the adaptive pin controller. +func TestBranchCache_PutIfClean_DoesNotCountAsMiss(t *testing.T) { + c := NewBranchCache(100) + var fired atomic.Uint64 + c.SetMissCallback(func(prefix []byte) { fired.Add(1) }) + + key := []byte{0x12, 0x34} + // PutIfClean on a cold prefix must succeed without registering as a + // triple-miss (no fire, no tail-miss counter bump). + require.True(t, c.PutIfClean(key, []byte("v1"), 0, 0, "test")) + require.Equal(t, uint64(0), fired.Load(), "PutIfClean must not fire onMiss on the write path") + require.Equal(t, uint64(0), c.tailMisses.Load(), "PutIfClean must not bump tail miss counter") + + // MarkDirty on a different cold prefix: also write-path, also must not count. + c.MarkDirty([]byte{0xff, 0xee}) + require.Equal(t, uint64(0), fired.Load(), "MarkDirty must not fire onMiss on the write path") + require.Equal(t, uint64(0), c.tailMisses.Load(), "MarkDirty must not bump tail miss counter") + + // A real read miss should still count. + _, _, ok := c.Get([]byte{0xaa, 0xbb}) + require.False(t, ok) + require.Equal(t, uint64(1), fired.Load(), "Get on cold prefix must fire onMiss") +} From fd74979033ce44924f451f288fdcb2916511ad43 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Thu, 4 Jun 2026 16:13:32 +0000 Subject: [PATCH 101/167] db/state/execctx: decode AccountsDomain codeHash with DeserialiseV3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codeHashForAddr resolves an account's codeHash from the AccountsDomain so the CodeDomain ethHash bypass can serve shared bytecode without an addr-keyed file read. decodeAccountCodeHash decoded the account value with acc.DecodeForStorage, but AccountsDomain values are SerialiseV3-encoded. DecodeForStorage is the legacy MDBX bitmask format with an incompatible binary layout; applied to V3 bytes it silently misparses and leaves CodeHash empty. As a result codeHashForAddr returned nil for every account and the ethHash bypass never engaged for any contract — every CodeDomain read that missed the addr cache fell through to a file read. This is a decoding-correctness bug: the wrong decoder is applied to the stored encoding. Use accounts.DeserialiseV3, the matching decoder for AccountsDomain values. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/execctx/domain_shared.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index ea8c923140c..2b660f4a580 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1121,7 +1121,12 @@ func decodeAccountCodeHash(enc []byte) []byte { return nil } var acc accounts.Account - if err := acc.DecodeForStorage(enc); err != nil { + // AccountsDomain values are SerialiseV3-encoded, so they must be decoded + // with DeserialiseV3. DecodeForStorage is the legacy MDBX bitmask format + // with an incompatible binary layout; applied to V3 bytes it silently + // misparses and leaves CodeHash empty, so codeHashForAddr returned nil for + // every contract and the CodeDomain ethHash bypass never fired. + if err := accounts.DeserialiseV3(&acc, enc); err != nil { return nil } if acc.CodeHash.IsEmpty() { From ba6c67a18c81858066f91f637ae6a3b9e9806251 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 17:10:32 +0000 Subject: [PATCH 102/167] =?UTF-8?q?db/state,=20db/kv:=20unify=20FlushWithC?= =?UTF-8?q?allback=20=E2=80=94=20flush-only=20refresh=20for=20branchCache?= =?UTF-8?q?=20+=20stateCache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The optimization stack and the cache branch had grown two divergent FlushWithCallback methods — one single-domain for the BranchCache (commitment), one all-domains for the StateCache flush-only fix. Merge them into one. FlushWithCallback is now all-domains: it invokes cb for every (domain, key, latest-value, step) across all domains (sd.domains + sd.storage), then drains the mem-batch — callback, MDBX flush and drain in one latestStateLock window. SharedDomains.Flush passes a single callback that routes CommitmentDomain → BranchCache and Accounts/Storage/Code → StateCache. The per-write stateCache.Put/Delete in domainPut/DomainDel are removed: a write is in-flight, fork-specific state living in sd.mem; mirroring it into the process-wide cache let a sibling fork's re-execution read another fork's uncommitted bytes. The cache is now refreshed only on flush, so it mirrors committed, fork-agnostic state. drainLocked empties sd.mem as part of the flush so a child SD chained as parent reads through to the refreshed cache / DB instead of stale bytes. This folds the cache fix (was 527cb23077 on the cache branch) into the optimization stack so the local group-test exercises the final code. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/kv/kv_interface.go | 2 +- db/state/execctx/domain_shared.go | 208 ++++++++++++++++-------------- db/state/temporal_mem_batch.go | 79 +++++++----- 3 files changed, 165 insertions(+), 124 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 16a76e0ff4e..e757a32effd 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,7 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error - FlushWithCallback(ctx context.Context, tx RwTx, domain Domain, cb func(k []byte, v []byte, step Step)) error + FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, step Step)) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 2b660f4a580..3f9ebd930b5 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -780,91 +780,127 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - // Update the cache with the bytes about to land in MDBX, atomically + // Refresh the caches with the bytes about to land in MDBX, atomically // with the flush. FlushWithCallback holds the latestState write lock // for the full window so a concurrent DomainPut cannot interleave // between the cache update and the MDBX write — readers see either // the local sd.mem value (during the lock) or the cached/MDBX value - // (after release). No window where cache is stale vs MDBX. - if sd.branchCache != nil { - if err := sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { - if len(v) == 0 { - sd.branchCache.Invalidate(k) - return + // (after release). The caches are refreshed ONLY here, on flush — + // never per-write — so they mirror committed, fork-agnostic state: + // CommitmentDomain → BranchCache, Accounts/Storage/Code → StateCache. + if sd.branchCache != nil || sd.stateCache != nil { + if err := sd.mem.FlushWithCallback(ctx, tx, func(domain kv.Domain, k []byte, v []byte, step kv.Step) { + switch domain { + case kv.CommitmentDomain: + if sd.branchCache != nil { + if len(v) == 0 { + sd.branchCache.Invalidate(k) + } else { + sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + } + } + case kv.AccountsDomain: + if sd.stateCache != nil { + if len(v) == 0 { + sd.stateCache.Delete(kv.AccountsDomain, k) + sd.stateCache.Delete(kv.CodeDomain, k) + sd.stateCache.DeleteAddrCodeHash(k) + } else { + sd.stateCache.Put(kv.AccountsDomain, k, v) + sd.stateCache.DeleteAddrCodeHash(k) + } + } + case kv.StorageDomain: + if sd.stateCache != nil { + if len(v) == 0 { + sd.stateCache.Delete(kv.StorageDomain, k) + } else { + sd.stateCache.Put(kv.StorageDomain, k, v) + } + } + case kv.CodeDomain: + if sd.stateCache != nil { + if len(v) == 0 { + sd.stateCache.Delete(kv.CodeDomain, k) + } else { + sd.stateCache.Put(kv.CodeDomain, k, v) + } + } } - sd.branchCache.Put(k, v, uint64(step), "sd.Flush") }); err != nil { return err } - // Adaptive controller hook: now that the cache reflects this - // batch's writes, decide promotions / extensions / demotions - // for the contracts whose miss pressure crossed thresholds. - // The reader uses the in-flight Flush tx so pre-cache reads - // see the just-committed bytes (the tx's own writes are - // visible to itself). - if sd.adaptivePinController != nil { - if ttx, ok := tx.(kv.TemporalTx); ok { - reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) - if err != nil { - return nil, 0, false, err + if sd.branchCache != nil { + // Adaptive controller hook: now that the cache reflects this + // batch's writes, decide promotions / extensions / demotions + // for the contracts whose miss pressure crossed thresholds. + // The reader uses the in-flight Flush tx so pre-cache reads + // see the just-committed bytes (the tx's own writes are + // visible to itself). + if sd.adaptivePinController != nil { + if ttx, ok := tx.(kv.TemporalTx); ok { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil } - return v, uint64(step), len(v) > 0, nil - } - // Install per-call parallel-mode plumbing so the - // controller's promote/extend uses the wave-BFS file-only - // resolver. The factory captures ttx; cleared after the - // call so a future OnBlockComplete can't reach a stale tx. - factory := func() (commitment.BatchBranchResolver, func(), error) { - resolve := func(keys [][]byte) ([][]byte, error) { - d := ttx.Debug() - vals := make([][]byte, len(keys)) - for i, k := range keys { - v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) - if err != nil { - return nil, err - } - if found { - vc := make([]byte, len(v)) - copy(vc, v) - vals[i] = vc + // Install per-call parallel-mode plumbing so the + // controller's promote/extend uses the wave-BFS file-only + // resolver. The factory captures ttx; cleared after the + // call so a future OnBlockComplete can't reach a stale tx. + factory := func() (commitment.BatchBranchResolver, func(), error) { + resolve := func(keys [][]byte) ([][]byte, error) { + d := ttx.Debug() + vals := make([][]byte, len(keys)) + for i, k := range keys { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) + if err != nil { + return nil, err + } + if found { + vc := make([]byte, len(v)) + copy(vc, v) + vals[i] = vc + } } + return vals, nil } - return vals, nil + return resolve, nil, nil } - return resolve, nil, nil - } - provider := func(contractHash []byte) map[string][]byte { - m := map[string][]byte{} - c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) - if cerr != nil { - return m - } - defer c.Close() - evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) - scan := func(from, to []byte) { - for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { - if bytes.Compare(k, to) >= 0 { - return - } - if len(v) < 8 { - continue + provider := func(contractHash []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) + scan := func(from, to []byte) { + for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { + if bytes.Compare(k, to) >= 0 { + return + } + if len(v) < 8 { + continue + } + m[string(common.Copy(k))] = common.Copy(v[8:]) } - m[string(common.Copy(k))] = common.Copy(v[8:]) } + scan(evenFrom, evenTo) + scan(oddFrom, oddTo) + return m } - scan(evenFrom, evenTo) - scan(oddFrom, oddTo) - return m + sd.adaptivePinController.SetParallelMode(factory, provider) + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + sd.adaptivePinController.SetParallelMode(nil, nil) } - sd.adaptivePinController.SetParallelMode(factory, provider) - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) - sd.adaptivePinController.SetParallelMode(nil, nil) } + // Refresh pinned-tier gauges once per Flush — once-per-batch + // cadence avoids per-lookup cost in the hot read path. + sd.branchCache.PublishMetrics() } - // Refresh pinned-tier gauges once per Flush — once-per-batch - // cadence avoids per-lookup cost in the hot read path. - sd.branchCache.PublishMetrics() return nil } return sd.mem.Flush(ctx, tx) @@ -1256,17 +1292,12 @@ func (sd *SharedDomains) domainPut(domain kv.Domain, roTx kv.TemporalTx, k, v [] } } - // Update state cache when writing. For AccountsDomain, also invalidate - // the addr → codeHash LRU above SD because the codeHash may have - // changed (CREATE/CREATE2-replace; SELFDESTRUCT goes through DomainDel - // which has its own invalidation). The next codeHashForAddr lookup - // will repopulate from the freshly written account bytes. - if sd.stateCache != nil { - sd.stateCache.Put(domain, k, v) - if domain == kv.AccountsDomain { - sd.stateCache.DeleteAddrCodeHash(k) - } - } + // The state cache is NOT updated here. This write goes into sd.mem and + // is served from there (checked first on every read, fork-isolated via + // the parent chain); the shared cache is refreshed only on flush + // (SharedDomains.Flush → FlushWithCallback), so it mirrors committed, + // fork-agnostic state. A per-write update would leak non-flushed, + // fork-specific bytes into a sibling fork's reads. // Serialize against the calculator's accumulator-swap window — see // changesetMu doc on the SharedDomains struct. Skipped when the caller @@ -1310,31 +1341,20 @@ func (sd *SharedDomains) DomainDel(domain kv.Domain, tx kv.TemporalTx, k []byte, if err := sd.DomainDel(kv.CodeDomain, tx, k, txNum, nil); err != nil { return err } - // Remove from state cache when account is deleted (including the - // nethermind-style addr → codeHash mapping; SELFDESTRUCT and - // CREATE-replace both reset the codeHash). - if sd.stateCache != nil { - sd.stateCache.Delete(kv.AccountsDomain, k) - sd.stateCache.Delete(kv.CodeDomain, k) - sd.stateCache.DeleteAddrCodeHash(k) - } + // State cache is refreshed on flush only — see DomainPut. The flush + // callback handles the empty-value (delete) case for accounts, code + // and the addr→codeHash mapping. // AccountsDomain — apply-side. Serialize against swap window. sd.changesetMu.Lock() defer sd.changesetMu.Unlock() return sd.mem.DomainDel(kv.AccountsDomain, ks, txNum, prevVal) case kv.StorageDomain: - // Remove from state cache when storage is deleted - if sd.stateCache != nil { - sd.stateCache.Delete(kv.StorageDomain, k) - } + // State cache refreshed on flush only — see DomainPut. case kv.CodeDomain: if prevVal == nil { return nil } - // Remove from state cache when code is deleted - if sd.stateCache != nil { - sd.stateCache.Delete(kv.CodeDomain, k) - } + // State cache refreshed on flush only — see DomainPut. default: //noop } diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 5ce7e1c8158..fa67b0bd1f5 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -762,47 +762,68 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { return nil } -// FlushWithCallback updates a downstream cache via cb for each -// (key, value, step) tuple in domain, then flushes the mem-batch to tx. -// Cache-update is ordered BEFORE the MDBX flush so that during the -// MDBX write window the cache still holds the entry — a reader that -// goes (sd write buffer → cache → MDBX) never observes a key missing -// from all three sources. The MDBX commit provides db-side atomicity -// independently. +// FlushWithCallback flushes the mem-batch to tx, but first invokes cb for +// every (domain, key, latest-value, step) tuple so downstream caches can be +// refreshed, then drains the in-memory latest/history state. // -// Used by SharedDomains.Flush to keep an external BranchCache in sync -// with commitment-domain writes. +// The callback, the MDBX flush and the drain run under latestStateLock as +// one atomic window: a concurrent reader sees either the full pre-flush mem +// state or the fully-drained post-flush state, never a partial mix. cb runs +// before the flush so that during the MDBX write window the cache already +// holds the entry — a reader going mem → cache → MDBX never sees a key +// missing from all three. data may be nil/empty for deletes — cb +// distinguishes on len(v)==0. // -// cb is called under latestStateLock so the snapshot it sees matches -// the in-memory state at flush time. Flush itself runs under the same -// lock to prevent concurrent DomainPut from interleaving with the -// snapshot/cache-update/flush sequence. -// -// If Flush fails, the cache has already been updated for this batch. -// That's the same invariant the cache maintains for any other write -// path — values may exist in the cache before they reach disk; a -// retry of Flush re-applies them. +// Used by SharedDomains.Flush to keep the BranchCache (commitment domain) +// and the StateCache (accounts/storage/code) in sync — refreshed only on +// flush, so they hold committed, fork-agnostic state. func (sd *TemporalMemBatch) FlushWithCallback( - ctx context.Context, tx kv.RwTx, domain kv.Domain, - cb func(k []byte, v []byte, step kv.Step), + ctx context.Context, tx kv.RwTx, + cb func(domain kv.Domain, k []byte, v []byte, step kv.Step), ) error { sd.latestStateLock.Lock() defer sd.latestStateLock.Unlock() - // dir is unset on entries written via DomainPut/DomainDel (always - // default zero); the latest history entry per key is authoritative. - // data may be nil/empty for deletes — caller's cb is expected to - // distinguish (see SharedDomains.Flush, which Invalidate's on - // len(v)==0 and Put's otherwise). - for keyStr, history := range sd.domains[domain] { + for d := range sd.domains { + for keyStr, history := range sd.domains[d] { + if len(history) == 0 { + continue + } + latest := history[len(history)-1] + cb(kv.Domain(d), []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + } + } + // StorageDomain entries live in sd.storage (a btree), not sd.domains. + sd.storage.Scan(func(keyStr string, history []dataWithTxNum) bool { if len(history) == 0 { - continue + return true } latest := history[len(history)-1] - cb([]byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + cb(kv.StorageDomain, []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + return true + }) + + if err := sd.flushLocked(ctx, tx); err != nil { + return err } + sd.drainLocked() + return nil +} - return sd.flushLocked(ctx, tx) +// drainLocked clears the in-memory latest/history state after a flush. The +// caller must hold latestStateLock. Mirrors the data-clearing half of +// ClearRam (without the metrics reset): once flushed, the mem-batch holds +// no data — values now live in the DB and the refreshed caches, and a child +// SD chained to this one as parent reads through instead of being shadowed +// by stale, fork-specific bytes. +func (sd *TemporalMemBatch) drainLocked() { + for i := range sd.domains { + sd.domains[i] = map[string][]dataWithTxNum{} + } + sd.storage = btree2.NewMap[string, []dataWithTxNum](128) + sd.unwindToTxNum = 0 + sd.unwindChangeset = nil + sd.unwindChangesetRaw = nil } // flushLocked is the body of Flush, factored so FlushWithCallback can From a5f973bfcba672b807c528ab5ae62b07ac388198 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 16:35:39 +0000 Subject: [PATCH 103/167] execution/execmodule: FCU semaphore decouple + foreground-priority bg-commit worker (gate item 2, core) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The post-FCU flush+commit+prune held the ExecModule semaphore for its whole duration, so the next FCU blocked on the previous commit — async commit was a throughput regression. Decouple it: the commit runs on a background worker, the semaphore is released the moment updateForkChoice returns. Base intent: foreground execution is never blocked on the DB. bg_commit.go adds a foreground-priority coordinator — every foreground op (FCU / ValidateChain / InsertBlocks / AssembleBlock / SetHead) registers via the fgTryAcquire/fgAcquire/fgRelease semaphore wrappers; the single FIFO commit worker waits for a foreground-free window (waitForegroundIdle) before each commit, so the commit RwTx never overlaps a foreground roTx and never pins MDBX pages. An in-flight commit is never aborted — a foreground op that arrives mid-commit just makes the next commit wait. Completed FCU generations are added to a chain; the next FCU's SharedDomains SetParent()s to the newest generation so it reads not-yet-committed domain state instead of a stale DB. drainCommittedGens, run under the semaphore at FCU entry, closes the chain as a unit once every generation has committed. Gated by fcuBackgroundCommit (default false) — the foreground-commit path is byte-for-byte unchanged. WIP: split-commit (sync block-data commit) and ValidateChain/InsertBlocks chain wiring still to land before the engineapi async-commit tests pass. See project_fcu_semaphore_decouple_plan. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/execmodule/bg_commit.go | 191 +++++++++++++++++++++++++ execution/execmodule/block_building.go | 8 +- execution/execmodule/exec_module.go | 35 +++-- execution/execmodule/forkchoice.go | 63 ++++---- execution/execmodule/inserters.go | 4 +- execution/execmodule/set_head.go | 4 +- 6 files changed, 261 insertions(+), 44 deletions(-) create mode 100644 execution/execmodule/bg_commit.go diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go new file mode 100644 index 00000000000..c183e81e265 --- /dev/null +++ b/execution/execmodule/bg_commit.go @@ -0,0 +1,191 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execmodule + +import ( + "context" + "errors" + + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" +) + +// Background commit / foreground-priority coordination (gate item 2). +// +// Base intent: foreground execution (FCU / ValidateChain / InsertBlocks / +// AssembleBlock / SetHead) is NEVER blocked on the DB. The post-FCU +// flush+commit+prune runs on a single background goroutine that yields to +// foreground: a commit starts only in a foreground-free window, so the +// commit RwTx never overlaps a foreground roTx and never pins MDBX pages +// (which would grow the freelist / DB file). An in-flight commit is never +// aborted — a foreground op that arrives mid-commit simply makes the *next* +// commit wait. See project_fcu_semaphore_decouple_plan. + +// commitGen is one in-flight background-commit unit: the SharedDomains of a +// completed FCU whose domain state must be flushed + committed off the +// foreground path. +type commitGen struct { + sd *execctx.SharedDomains + roTx kv.TemporalTx + finishProgressBefore uint64 + isSynced bool + initialCycle bool + committed bool // guarded by ExecModule.fgMu +} + +// fgTryAcquire acquires the foreground semaphore non-blocking and, on +// success, registers the caller as an active foreground op. Pair with +// fgRelease. Replaces direct e.semaphore.TryAcquire so the foreground +// counter and the semaphore can never drift. +func (e *ExecModule) fgTryAcquire() bool { + if !e.semaphore.TryAcquire(1) { + return false + } + e.enterForeground() + return true +} + +// fgAcquire is the blocking counterpart of fgTryAcquire. +func (e *ExecModule) fgAcquire(ctx context.Context) error { + if err := e.semaphore.Acquire(ctx, 1); err != nil { + return err + } + e.enterForeground() + return nil +} + +// fgRelease releases the foreground semaphore and deregisters the foreground +// op, waking the commit worker if this was the last one. +func (e *ExecModule) fgRelease() { + e.leaveForeground() + e.semaphore.Release(1) +} + +// enterForeground marks a foreground operation as active. The background +// commit worker will not start a commit while any foreground op is active. +func (e *ExecModule) enterForeground() { + e.fgMu.Lock() + e.fgCount++ + e.fgMu.Unlock() +} + +// leaveForeground marks a foreground operation as finished, waking the +// commit worker when the last one leaves. +func (e *ExecModule) leaveForeground() { + e.fgMu.Lock() + e.fgCount-- + if e.fgCount == 0 { + e.fgIdle.Broadcast() + } + e.fgMu.Unlock() +} + +// waitForegroundIdle blocks until no foreground operation is active. The +// commit worker calls it before each commit so the commit RwTx runs in a +// foreground-free window — no foreground roTx open to pin MDBX pages. +func (e *ExecModule) waitForegroundIdle() { + e.fgMu.Lock() + for e.fgCount > 0 { + e.fgIdle.Wait() + } + e.fgMu.Unlock() +} + +// enqueueCommit hands a completed generation to the background commit +// worker. Non-blocking by design (buffered channel) so the foreground FCU +// is never blocked handing off its commit. +func (e *ExecModule) enqueueCommit(gen *commitGen) { + e.commitCh <- gen +} + +// commitWorker is the single FIFO background-commit goroutine. It pulls +// completed generations and commits each one's domain delta — but only once +// foreground execution is idle, so the commit never overlaps a foreground +// roTx. An in-flight commit is never aborted; a foreground op that arrives +// mid-commit makes the next iteration wait. +func (e *ExecModule) commitWorker() { + for gen := range e.commitCh { + e.waitForegroundIdle() + err := e.runPostForkchoice(gen.sd, gen.roTx, gen.finishProgressBefore, gen.isSynced, gen.initialCycle) + if err != nil && !errors.Is(err, context.Canceled) { + e.logger.Error("background commit failed", "err", err) + } + // roTx is rolled back inside runForkchoiceFlushCommit between Flush + // and Commit; this is a safety net (Rollback is idempotent). + gen.roTx.Rollback() + e.markGenCommitted(gen) + if dispatcher := e.pipelineExecutor.Dispatcher(); dispatcher != nil { + dispatcher.PublishOverlay(nil) + } + } +} + +// markGenCommitted records that a generation's commit has landed. The +// generation's SharedDomains is NOT closed here — a foreground op may still +// hold it via the parent chain. It is closed as a unit by drainCommittedGens +// once the whole chain is committed. +func (e *ExecModule) markGenCommitted(gen *commitGen) { + e.fgMu.Lock() + gen.committed = true + e.fgMu.Unlock() +} + +// latestGen returns the newest in-flight generation — the parent a new +// foreground SharedDomains chains to so its reads see not-yet-committed +// domain state. nil when the chain is empty. +func (e *ExecModule) latestGen() *execctx.SharedDomains { + e.fgMu.Lock() + defer e.fgMu.Unlock() + if len(e.gens) == 0 { + return nil + } + return e.gens[len(e.gens)-1].sd +} + +// addGen appends a new in-flight generation to the chain. +func (e *ExecModule) addGen(gen *commitGen) { + e.fgMu.Lock() + e.gens = append(e.gens, gen) + e.fgMu.Unlock() +} + +// drainCommittedGens closes and clears the generation chain when every +// generation in it has committed. Called by a foreground op while it holds +// the semaphore — no concurrent reader exists, so closing the SharedDomains +// is race-free. If any generation is still uncommitted the chain is left +// intact. Returns true when the chain was drained. +func (e *ExecModule) drainCommittedGens() bool { + e.fgMu.Lock() + defer e.fgMu.Unlock() + if len(e.gens) == 0 { + return true + } + for _, g := range e.gens { + if !g.committed { + return false + } + } + gens := e.gens + e.gens = nil + // Detach + close oldest→newest; each generation's data is in the DB, + // so a reader that still walked the chain would get the same bytes. + for _, g := range gens { + g.sd.SetParent(nil) + g.sd.Close() + } + return true +} diff --git a/execution/execmodule/block_building.go b/execution/execmodule/block_building.go index 40d6d00601c..74b64d1c87a 100644 --- a/execution/execmodule/block_building.go +++ b/execution/execmodule/block_building.go @@ -49,10 +49,10 @@ func (e *ExecModule) evictOldBuilders() { } func (e *ExecModule) AssembleBlock(ctx context.Context, params *builder.Parameters) (AssembleBlockResult, error) { - if !e.semaphore.TryAcquire(1) { + if !e.fgTryAcquire() { return AssembleBlockResult{Busy: true}, nil } - defer e.semaphore.Release(1) + defer e.fgRelease() if err := e.checkWithdrawalsPresence(params.Timestamp, params.Withdrawals); err != nil { return AssembleBlockResult{}, err @@ -97,10 +97,10 @@ func blockValue(br *types.BlockWithReceipts, baseFee *uint256.Int) *uint256.Int } func (e *ExecModule) GetAssembledBlock(_ context.Context, payloadID uint64) (AssembledBlockResult, error) { - if !e.semaphore.TryAcquire(1) { + if !e.fgTryAcquire() { return AssembledBlockResult{Busy: true}, nil } - defer e.semaphore.Release(1) + defer e.fgRelease() bldr, ok := e.builders[payloadID] if !ok { diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 8a46de7e839..434f05d77a6 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -224,6 +224,16 @@ type ExecModule struct { currentContext *execctx.SharedDomains publishedSD func() *execctx.SharedDomains // fallback for background commit + // Background-commit / foreground-priority coordination (gate item 2 — + // see bg_commit.go). fgMu guards fgCount + gens; fgIdle is signalled + // when fgCount reaches zero so the commit worker can run in a + // foreground-free window. + fgMu sync.Mutex + fgCount int + fgIdle *sync.Cond + gens []*commitGen + commitCh chan *commitGen + // stateCache is a cache for state data (accounts, storage, code) stateCache *cache.StateCache readAheader *exec.BlockReadAheader @@ -289,16 +299,25 @@ func NewExecModule( if stateCache != nil { stateCache.execModule = em } + + // Start the background-commit worker (gate item 2). It pulls completed + // generations off commitCh and commits each in a foreground-free + // window. The buffered channel keeps the foreground FCU's hand-off + // non-blocking. + em.fgIdle = sync.NewCond(&em.fgMu) + em.commitCh = make(chan *commitGen, 1024) + go em.commitWorker() + return em } // WaitIdle blocks until any in-flight updateForkChoice goroutine finishes. // Call before closing the database to avoid waitTxsAllDoneOnClose hangs. func (e *ExecModule) WaitIdle(ctx context.Context) { - if err := e.semaphore.Acquire(ctx, 1); err != nil { + if err := e.fgAcquire(ctx); err != nil { return // context cancelled — best effort } - e.semaphore.Release(1) + e.fgRelease() } // ForkValidator returns the fork validator owned by this module. @@ -390,13 +409,13 @@ func (e *ExecModule) unwindToCommonCanonical(sd *execctx.SharedDomains, tx kv.Te } func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, blockNumber uint64) (ValidationResult, error) { - if !e.semaphore.TryAcquire(1) { + if !e.fgTryAcquire() { e.logger.Trace("ethereumExecutionModule.ValidateChain: ExecutionStatus_Busy") return ValidationResult{ ValidationStatus: ExecutionStatusBusy, }, nil } - defer e.semaphore.Release(1) + defer e.fgRelease() e.hook.LastNewBlockSeen(blockNumber) // used by eth_syncing e.currentContext.ResetPendingUpdates() @@ -641,13 +660,13 @@ func (e *ExecModule) purgeBadChain(ctx context.Context, tx kv.RwTx, latestValidH } func (e *ExecModule) Start(ctx context.Context, hook *stageloop.Hook) { - if err := e.semaphore.Acquire(ctx, 1); err != nil { + if err := e.fgAcquire(ctx); err != nil { if !errors.Is(err, context.Canceled) { e.logger.Error("Could not start execution service", "err", err) } return } - defer e.semaphore.Release(1) + defer e.fgRelease() if err := e.pipelineExecutor.ProcessFrozenBlocks(ctx, hook, e.onlySnapDownloadOnStart); err != nil { if !errors.Is(err, context.Canceled) { @@ -693,11 +712,11 @@ func (e *ExecModule) Ready(ctx context.Context) (bool, error) { return false, err } - if !e.semaphore.TryAcquire(1) { + if !e.fgTryAcquire() { e.logger.Trace("ethereumExecutionModule.Ready: ExecutionStatus_Busy") return false, nil } - defer e.semaphore.Release(1) + defer e.fgRelease() return true, nil } diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 532c5f29355..8dec7ebcde0 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -155,7 +155,7 @@ func writeForkChoiceHashes(tx kv.RwTx, blockHash, safeHash, finalizedHash common } func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, safeHash, finalizedHash common.Hash, outcomeCh chan forkchoiceOutcome) (err error) { - if !e.semaphore.TryAcquire(1) { + if !e.fgTryAcquire() { e.logger.Trace("ethereumExecutionModule.updateForkChoice: ExecutionStatus_Busy") sendForkchoiceResultWithoutWaiting(outcomeCh, ForkChoiceResult{ LatestValidHash: common.Hash{}, @@ -163,12 +163,15 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa }, false) return fmt.Errorf("semaphore timeout") } - shouldReleaseSema := true - defer func() { - if shouldReleaseSema { - e.semaphore.Release(1) - } - }() + // The semaphore is always released when updateForkChoice returns — the + // background commit no longer holds it (gate item 2: the commit runs on + // the bg worker, decoupled from the foreground semaphore). + defer e.fgRelease() + + // Retire the in-flight commit-generation chain if every generation has + // committed (gate item 2). Safe here: the foreground semaphore is held, + // so no concurrent op is reading a generation's SharedDomains. + e.drainCommittedGens() defer UpdateForkChoiceDuration(time.Now()) defer e.forkValidator.ClearWithUnwind() @@ -223,6 +226,14 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // ValidateChain (fork validation, exec_module.go) set this, leaving // the canonical execution path running uncached against the aggTx. currentContext.SetStateCache(e.stateCache) + // Chain to the newest in-flight commit generation (gate item 2): + // if a previous FCU's commit has not yet landed, its domain state + // lives only in that generation's sd.mem — SetParent lets this FCU + // read through to it instead of a stale DB. nil when the chain is + // empty (all prior commits landed → DB is current). + if parent := e.latestGen(); parent != nil { + currentContext.SetParent(parent) + } } defer func() { @@ -654,31 +665,27 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // fcuBackgroundCommit is explicitly enabled. var commitTimings []any if e.fcuBackgroundCommit { - shouldReleaseSema = false - // Transfer roTx ownership to the goroutine so the outer defer - // (roTx.Rollback) becomes a no-op on nil. + // Hand the commit to the background worker (gate item 2). + // shouldReleaseSema stays true → the outer defer fgRelease()s + // the semaphore the moment updateForkChoice returns, so the + // next FCU is never blocked on this commit. The worker commits + // in a foreground-free window (commitWorker → waitForegroundIdle) + // so the commit RwTx never overlaps a foreground roTx. The + // generation is added to the chain so the next FCU's SD can + // read its not-yet-committed domain state via SetParent. bgRoTx := roTx roTx = nil bgSD := currentContext currentContext = nil - dispatcher := e.pipelineExecutor.Dispatcher() - go func() { - defer e.semaphore.Release(1) - defer bgSD.Close() - // bgRoTx is rolled back inside runForkchoiceFlushCommit between - // Flush and Commit so the commit sees openTxs=1 in MDBX. This - // defer is a safety net — Rollback is idempotent. - defer bgRoTx.Rollback() - err := e.runPostForkchoice(bgSD, bgRoTx, finishProgressBefore, isSynced, initialCycle) - if err != nil && !errors.Is(err, context.Canceled) { - e.logger.Error("Error running background post forkchoice", "err", err) - } - // Signal that the DB commit is done — RPC consumers can - // drop their SD reference and read from committed DB. - if dispatcher != nil { - dispatcher.PublishOverlay(nil) - } - }() + gen := &commitGen{ + sd: bgSD, + roTx: bgRoTx, + finishProgressBefore: finishProgressBefore, + isSynced: isSynced, + initialCycle: initialCycle, + } + e.addGen(gen) + e.enqueueCommit(gen) } else { // Foreground commit: pass the outer roTx so it gets released // between Flush and Commit (same openTxs=2→1 optimization as diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index f4a90d71201..87c03514e2c 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -53,11 +53,11 @@ func (e *ExecModule) flushBlockOverlayToDB(ctx context.Context, sd *execctx.Shar } func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) (ExecutionStatus, error) { - if !e.semaphore.TryAcquire(1) { + if !e.fgTryAcquire() { e.logger.Trace("ethereumExecutionModule.InsertBlocks: ExecutionStatus_Busy") return ExecutionStatusBusy, nil } - defer e.semaphore.Release(1) + defer e.fgRelease() e.forkValidator.ClearWithUnwind() frozenBlocks := e.blockReader.FrozenBlocks() diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 0623b5bcaa6..aac8206e5ff 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -53,10 +53,10 @@ func getLatestBlockNumber(tx kv.Tx) (uint64, error) { func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { acquireCtx, acquireCancel := context.WithTimeout(ctx, 5*time.Second) defer acquireCancel() - if err := e.semaphore.Acquire(acquireCtx, 1); err != nil { + if err := e.fgAcquire(acquireCtx); err != nil { return fmt.Errorf("execution module is busy: %w", err) } - defer e.semaphore.Release(1) + defer e.fgRelease() tx, err := e.db.BeginTemporalRw(ctx) if err != nil { From f54eb60dc1b5351f69915262d0902db5408eef1b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 16:57:50 +0000 Subject: [PATCH 104/167] execution/execmodule: gate-2 overlay-continuity wiring (FCU/ValidateChain/InsertBlocks) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the commit-generation chain into the three SharedDomains-creation sites so block data and domain state stay visible across the async-commit window without a split commit — the block overlay carries continuity: - FCU currentContext, ValidateChain doms, InsertBlocks sd: SetParent to the newest in-flight generation (domain reads) and InitBlockOverlay backed by that generation's overlay temporal read view (block-data reads cascade through it instead of a stale DB). - ValidateChain: guard the now-nillable e.currentContext; read block data through latestGen's overlay when currentContext is nil. Builds + vet clean; gated by fcuBackgroundCommit (default false) so the foreground-commit path is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/execmodule/exec_module.go | 67 +++++++++++++++++------------ execution/execmodule/forkchoice.go | 14 +++++- execution/execmodule/inserters.go | 17 +++++++- 3 files changed, 68 insertions(+), 30 deletions(-) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 434f05d77a6..d76cda4792c 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -418,7 +418,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b defer e.fgRelease() e.hook.LastNewBlockSeen(blockNumber) // used by eth_syncing - e.currentContext.ResetPendingUpdates() + // currentContext is nil while a background commit holds the previous + // FCU's SD (gate item 2) — guard the access. + if e.currentContext != nil { + e.currentContext.ResetPendingUpdates() + } e.forkValidator.ClearWithUnwind() e.logger.Debug("[execmodule] validating chain", "number", blockNumber, "hash", blockHash) var ( @@ -449,22 +453,31 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) currentBlockNumber = rawdb.ReadCurrentBlockNumber(overlay) } else { - if err := e.db.View(ctx, func(tx kv.Tx) error { - header, err = e.blockReader.Header(ctx, tx, blockHash, blockNumber) - if err != nil { - return err - } - - body, err = e.blockReader.BodyWithTransactions(ctx, tx, blockHash, blockNumber) - if err != nil { - return err + // currentContext is nil — read block data through the newest + // in-flight commit generation's overlay when one exists (gate item + // 2), so the previous FCU's not-yet-committed headers/bodies/TDs + // are visible; otherwise a plain DB read. + roTx, err := e.db.BeginTemporalRo(ctx) + if err != nil { + return ValidationResult{}, err + } + defer roTx.Rollback() + var src kv.TemporalTx = roTx + if parent := e.latestGen(); parent != nil { + if v := parent.BlockOverlayTemporalTx(roTx); v != nil { + src = v } - e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) - currentBlockNumber = rawdb.ReadCurrentBlockNumber(tx) - return nil - }); err != nil { + } + header, err = e.blockReader.Header(ctx, src, blockHash, blockNumber) + if err != nil { + return ValidationResult{}, err + } + body, err = e.blockReader.BodyWithTransactions(ctx, src, blockHash, blockNumber) + if err != nil { return ValidationResult{}, err } + e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) + currentBlockNumber = rawdb.ReadCurrentBlockNumber(src) } if header == nil || body == nil { return ValidationResult{ @@ -504,20 +517,21 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // We Close explicitly only on the early-return error paths below. doms.SetInMemHistoryReads(inMemHistoryReads) - if err := doms.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { + // Back the validation overlay by the newest in-flight commit generation + // (gate item 2) so block-data reads cascade through its overlay instead + // of a stale DB while a background commit is in flight. + valOverlayBase := kv.TemporalTx(roTx) + if parent := e.latestGen(); parent != nil { + if v := parent.BlockOverlayTemporalTx(roTx); v != nil { + valOverlayBase = v + } + } + if err := doms.InitBlockOverlay(valOverlayBase, roTx.Debug().Dirs().Tmp); err != nil { doms.Close() return ValidationResult{}, fmt.Errorf("ValidateChain: init block overlay: %w", err) } var tx kv.TemporalRwTx = doms.BlockOverlay() - // DO NOT CHANGE THIS WITHOUT WORKING THROUGH THE UNWIND CACHING SCENARIOS. - // The earlier `header.ParentHash == ReadHeadBlockHash(tx)` head-extending- - // only gate has been intentionally widened back to "chain whenever a - // currentContext exists" because fork-payload caching needs the parent - // link too — see the two-role breakdown below. The narrower gate was - // merged from main during the post-#21017 rebase and is the WRONG choice - // for this branch; keep the wider gate. - // // Chain the validation SD to the latest in-memory canonical generation: // e.currentContext when present, otherwise the newest in-flight commit // generation (gate item 2 — the prior FCU cleared currentContext and @@ -539,13 +553,10 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // unwindToCommonCanonical has run, doms.mem.unwindChangeset holds every // key the unwound canonical blocks touched, and TemporalMemBatch.getLatest // resolves those from the unwind set before ever consulting the parent. - // - // Cherry-pick note: the upstream commit also chained to e.latestGen() - // (the gate-2 in-flight commit generation) when currentContext is nil; - // that generation chain is not on this branch, so currentContext is the - // only canonical generation here. if e.currentContext != nil { doms.SetParent(e.currentContext) + } else if parent := e.latestGen(); parent != nil { + doms.SetParent(parent) } // Flush block overlay data (headers, bodies, TDs from InsertBlocks) into diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 8dec7ebcde0..2736a62c868 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -252,7 +252,19 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // hashes, stage progress, forkchoice markers, TxNums, etc.). All // pipeline reads cascade through the overlay to the RO tx; writes // stay in memory until commit. - if err := currentContext.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { + // Back the block overlay by the newest in-flight commit generation's + // overlay when one exists (gate item 2): block data — headers, bodies, + // TDs, receipts — written by a not-yet-committed FCU lives in that + // generation's overlay. A concurrent-safe temporal read view lets this + // FCU read through to it instead of a stale DB. The generation chain + // (closed as a unit by drainCommittedGens) keeps the backing valid. + overlayBase := kv.TemporalTx(roTx) + if parent := e.latestGen(); parent != nil { + if v := parent.BlockOverlayTemporalTx(roTx); v != nil { + overlayBase = v + } + } + if err := currentContext.InitBlockOverlay(overlayBase, roTx.Debug().Dirs().Tmp); err != nil { return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, fmt.Errorf("updateForkChoice: init block overlay: %w", err), false) } var tx kv.TemporalRwTx = currentContext.BlockOverlay() diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index 87c03514e2c..08df512809c 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -23,6 +23,7 @@ import ( "github.com/holiman/uint256" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/commitment/commitmentdb" @@ -80,12 +81,26 @@ func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) } e.logger.Info("ethereumExecutionModule.InsertBlocks: state ahead of blocks, proceeding with catch-up", "err", err) } + // Chain to the newest in-flight commit generation (gate item 2) so + // the parent-TD read below sees a previous FCU's not-yet-committed + // block data instead of a stale DB. + if sd != nil { + if parent := e.latestGen(); parent != nil { + sd.SetParent(parent) + } + } e.lock.Lock() e.currentContext = sd e.lock.Unlock() } if sd.BlockOverlay() == nil { - if err := sd.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { + overlayBase := kv.TemporalTx(roTx) + if parent := e.latestGen(); parent != nil { + if v := parent.BlockOverlayTemporalTx(roTx); v != nil { + overlayBase = v + } + } + if err := sd.InitBlockOverlay(overlayBase, roTx.Debug().Dirs().Tmp); err != nil { return 0, fmt.Errorf("ethereumExecutionModule.InsertBlocks: %w", err) } } else { From 980898df8eeedbc7e2a94b00d90d6d905dc374ab Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 17:15:04 +0000 Subject: [PATCH 105/167] =?UTF-8?q?execution/execmodule:=20commitWorker=20?= =?UTF-8?q?lifecycle=20=E2=80=94=20drain=20+=20exit=20before=20DB-close?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The background commit worker ran `for range commitCh` on a channel that was never closed, so it never terminated; a worker mid-commit at teardown deadlocked DB-close (waitTxsAllDoneOnClose). The engineapi async-commit test hung 7/10. Add a commitWorkerStop quit channel and a commitWg WaitGroup. The worker selects over commitCh and the quit channel; on quit it drains any queued generations (so a shutdown never drops a not-yet-landed commit) and exits. WaitIdle — the documented pre-DB-close hook — now stops the worker and waits for it (and its in-flight commit tx) to finish. commitCh is never closed, so a late enqueue can never panic. Hangs eliminated (7/10 → 0/10). Residual: the engineapi async-commit test now fails cleanly with "nonce too low" — a state-continuity bug tracked separately. Gated by fcuBackgroundCommit; default path unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/execmodule/bg_commit.go | 48 +++++++++++++++++++++-------- execution/execmodule/exec_module.go | 20 +++++++++++- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index c183e81e265..e53e7d7b67f 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -117,23 +117,47 @@ func (e *ExecModule) enqueueCommit(gen *commitGen) { // foreground execution is idle, so the commit never overlaps a foreground // roTx. An in-flight commit is never aborted; a foreground op that arrives // mid-commit makes the next iteration wait. +// +// On commitWorkerStop it drains any still-queued generations (so a shutdown +// never drops a not-yet-landed commit) and exits; commitWg lets WaitIdle +// block until the worker — and its in-flight commit tx — are done before +// DB-close. func (e *ExecModule) commitWorker() { - for gen := range e.commitCh { - e.waitForegroundIdle() - err := e.runPostForkchoice(gen.sd, gen.roTx, gen.finishProgressBefore, gen.isSynced, gen.initialCycle) - if err != nil && !errors.Is(err, context.Canceled) { - e.logger.Error("background commit failed", "err", err) - } - // roTx is rolled back inside runForkchoiceFlushCommit between Flush - // and Commit; this is a safety net (Rollback is idempotent). - gen.roTx.Rollback() - e.markGenCommitted(gen) - if dispatcher := e.pipelineExecutor.Dispatcher(); dispatcher != nil { - dispatcher.PublishOverlay(nil) + defer e.commitWg.Done() + for { + select { + case gen := <-e.commitCh: + e.runCommit(gen) + case <-e.commitWorkerStop: + for { + select { + case gen := <-e.commitCh: + e.runCommit(gen) + default: + return + } + } } } } +// runCommit flushes + commits one generation's delta in a foreground-free +// window. Called only by commitWorker. +func (e *ExecModule) runCommit(gen *commitGen) { + e.waitForegroundIdle() + err := e.runPostForkchoice(gen.sd, gen.roTx, gen.finishProgressBefore, gen.isSynced, gen.initialCycle) + if err != nil && !errors.Is(err, context.Canceled) { + e.logger.Error("background commit failed", "err", err) + } + // roTx is rolled back inside runForkchoiceFlushCommit between Flush and + // Commit; this is a safety net (Rollback is idempotent). + gen.roTx.Rollback() + e.markGenCommitted(gen) + if dispatcher := e.pipelineExecutor.Dispatcher(); dispatcher != nil { + dispatcher.PublishOverlay(nil) + } +} + // markGenCommitted records that a generation's commit has landed. The // generation's SharedDomains is NOT closed here — a foreground op may still // hold it via the parent chain. It is closed as a unit by drainCommittedGens diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index d76cda4792c..bac5f961f50 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -233,6 +233,12 @@ type ExecModule struct { fgIdle *sync.Cond gens []*commitGen commitCh chan *commitGen + // commitWorker lifecycle: commitWorkerStop signals the worker to drain + // and exit; commitWg tracks it so shutdown (WaitIdle) waits for the + // worker — and its in-flight commit txs — to finish before DB-close. + commitWorkerStop chan struct{} + commitStopOnce sync.Once + commitWg sync.WaitGroup // stateCache is a cache for state data (accounts, storage, code) stateCache *cache.StateCache @@ -303,9 +309,11 @@ func NewExecModule( // Start the background-commit worker (gate item 2). It pulls completed // generations off commitCh and commits each in a foreground-free // window. The buffered channel keeps the foreground FCU's hand-off - // non-blocking. + // non-blocking. WaitIdle stops the worker before DB-close. em.fgIdle = sync.NewCond(&em.fgMu) em.commitCh = make(chan *commitGen, 1024) + em.commitWorkerStop = make(chan struct{}) + em.commitWg.Add(1) go em.commitWorker() return em @@ -318,6 +326,16 @@ func (e *ExecModule) WaitIdle(ctx context.Context) { return // context cancelled — best effort } e.fgRelease() + // Stop the background-commit worker and wait for it to drain its queue + // (including any in-flight commit tx) so DB-close does not race it. + e.stopCommitWorker() +} + +// stopCommitWorker signals the background-commit worker to drain and exit, +// then waits for it. Idempotent. +func (e *ExecModule) stopCommitWorker() { + e.commitStopOnce.Do(func() { close(e.commitWorkerStop) }) + e.commitWg.Wait() } // ForkValidator returns the fork validator owned by this module. From a1dbe0b6a09248a19601159125df95064bd8fbcc Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 21:51:09 +0000 Subject: [PATCH 106/167] execution, rpc, txpool: route latest-state reads through the in-flight SD under async commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate item 2 (FCU async background commit) decouples block execution from the DB commit, so between an FCU and its commit landing the latest block's state lives only in the in-flight SharedDomains, not the committed DB. Several read paths still observed the committed DB and so saw stale state — a freshly-built block's nonce/headers — producing "nonce too low" rejections and "txns not found in block". - rpchelper._GetBlockNumber: the PendingBlockNumber case hardcoded latest=false, forcing latest-state queries down the historical read path (uncommitted history) — now resolves latest from the overlay-aware plainStateBlockNumber, matching the LatestBlockNumber case. - 13 rpc read sites (eth/erigon/debug/bor block-number + stage-progress lookups) now wrap the tx with filters.WithOverlay so an in-flight block is not mistaken for a future block. - membatchwithdb overlay read views route domain GetLatest/HasPrefix through the owning SharedDomains, so a consumer reading the published overlay sees sd.mem + the generation chain, not only committed DB. - txpool shares the execmodule.Cache SD-wrapper instead of its own notification-fed SimpleCache, so txpool validation and rpc reads agree. - LatestSD publish is foreground-owned: the background commit worker no longer republishes (it nil'd LatestSD whenever it caught up); drainCommittedGens keeps the newest generation alive as LatestSD and retires only superseded generations; closeAllGens releases it at shutdown. WIP checkpoint on the gate-2 continuity series — engineapi async-commit test improved (0/10 -> 1/10 pass); a residual nonce-too-low timing race and the newPayload SYNCING hang remain under investigation. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/kv/membatchwithdb/memory_mutation.go | 62 +++++++++++++++++++++---- db/state/execctx/domain_shared.go | 7 +++ execution/execmodule/bg_commit.go | 58 ++++++++++++++++------- execution/execmodule/exec_module.go | 4 ++ execution/execmodule/forkchoice.go | 38 ++++++++++----- node/eth/backend.go | 13 ++++-- rpc/jsonrpc/bor_api_impl.go | 4 +- rpc/jsonrpc/debug_api.go | 8 ++-- rpc/jsonrpc/debug_execution_witness.go | 2 +- rpc/jsonrpc/erigon_receipts.go | 10 ++-- rpc/jsonrpc/erigon_system.go | 2 +- rpc/jsonrpc/eth_block.go | 4 +- rpc/jsonrpc/eth_call.go | 4 +- rpc/jsonrpc/eth_receipts.go | 10 ++-- rpc/jsonrpc/eth_simulation.go | 2 +- rpc/jsonrpc/eth_system.go | 4 +- rpc/jsonrpc/overlay_api.go | 2 +- rpc/rpchelper/helper.go | 8 +++- 18 files changed, 179 insertions(+), 63 deletions(-) diff --git a/db/kv/membatchwithdb/memory_mutation.go b/db/kv/membatchwithdb/memory_mutation.go index b5166c64a58..bcddc3e0f6b 100644 --- a/db/kv/membatchwithdb/memory_mutation.go +++ b/db/kv/membatchwithdb/memory_mutation.go @@ -52,6 +52,20 @@ type MemoryMutation struct { clearedTables map[string]struct{} db kv.TemporalTx statelessCursors map[string]kv.RwCursor + + // domainGetterFor, when set on a base overlay, produces a temporal domain + // getter bound to a given backing tx. The block overlay's owner + // (SharedDomains) installs it so that domain reads (GetLatest/HasPrefix) + // on an overlay *read view* see not-yet-committed SharedDomains state — + // sd.mem plus the in-flight async-commit generation chain — instead of + // only the committed DB. Without this, a consumer reading "latest" state + // through the overlay while a background commit is still in flight gets a + // value that lags by >=1 block. nil for plain mem-batches. + domainGetterFor func(tx kv.TemporalTx) kv.TemporalGetter + // domainGetter is the factory's result, resolved against this view's + // backing tx. Set only on read views created via newReadViewMut; nil on + // the base overlay. + domainGetter kv.TemporalGetter } // NewMemoryBatch creates a pure Go in-memory batch with no OS-thread affinity. @@ -906,6 +920,9 @@ func (m *MemoryMutation) AggTx() any { } func (m *MemoryMutation) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { + if m.domainGetter != nil { + return m.domainGetter.GetLatest(name, k) + } if m.db == nil { return nil, 0, fmt.Errorf("MemoryMutation: domain read requires backing tx (detached overlay)") } @@ -920,6 +937,9 @@ func (m *MemoryMutation) GetAsOf(name kv.Domain, k []byte, ts uint64) (v []byte, } func (m *MemoryMutation) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + if m.domainGetter != nil { + return m.domainGetter.HasPrefix(name, prefix) + } if m.db == nil { return nil, nil, false, nil } @@ -1053,17 +1073,37 @@ func (m *MemoryMutation) newReadViewMut(tx kv.Tx) *MemoryMutation { if t, ok := tx.(kv.TemporalTx); ok { dbTx = t } - return &MemoryMutation{ - mu: m.mu, // share parent's mutex for synchronization - memTx: m.memTx, - memDb: nil, // caller doesn't own the memDb - deletedEntries: m.deletedEntries, - deletedDups: m.deletedDups, - clearedTables: m.clearedTables, - db: dbTx, + rv := &MemoryMutation{ + mu: m.mu, // share parent's mutex for synchronization + memTx: m.memTx, + memDb: nil, // caller doesn't own the memDb + deletedEntries: m.deletedEntries, + deletedDups: m.deletedDups, + clearedTables: m.clearedTables, + db: dbTx, + domainGetterFor: m.domainGetterFor, + } + // Resolve the SharedDomains-backed domain getter against this view's own + // backing tx so domain reads see uncommitted (in-flight) state. + if m.domainGetterFor != nil && dbTx != nil { + rv.domainGetter = m.domainGetterFor(dbTx) } + return rv } +// SetDomainGetterFactory installs the factory used to route domain reads on +// read views through a SharedDomains-backed getter. See domainGetterFor. +func (m *MemoryMutation) SetDomainGetterFactory(f func(tx kv.TemporalTx) kv.TemporalGetter) { + m.domainGetterFor = f +} + +// HasOverlayDomainGetter reports whether this read view routes domain reads +// through a SharedDomains-backed getter — i.e. it is an overlay view of an +// in-flight async-commit generation. Callers use this to bypass caches whose +// coherency is keyed to the committed DB and would otherwise serve a value +// lagging a not-yet-landed background commit. +func (m *MemoryMutation) HasOverlayDomainGetter() bool { return m.domainGetter != nil } + // OverlayTemporalReadView extends an overlay read view with kv.TemporalTx // support. It embeds a *MemoryMutation for all overlay-aware KV methods // (GetOne, Cursor, etc.) and delegates temporal methods (GetLatest, GetAsOf, @@ -1111,9 +1151,15 @@ func (v *OverlayTemporalReadView) Apply(_ context.Context, f func(tx kv.Tx) erro // Temporal methods — delegate to the independent temporal tx. func (v *OverlayTemporalReadView) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { + if v.MemoryMutation.domainGetter != nil { + return v.MemoryMutation.domainGetter.GetLatest(name, k) + } return v.temporalTx.GetLatest(name, k) } func (v *OverlayTemporalReadView) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + if v.MemoryMutation.domainGetter != nil { + return v.MemoryMutation.domainGetter.HasPrefix(name, prefix) + } return v.temporalTx.HasPrefix(name, prefix) } func (v *OverlayTemporalReadView) StepsInFiles(entitySet ...kv.Domain) kv.Step { diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 3f9ebd930b5..60999ac5a59 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -643,6 +643,13 @@ func (sd *SharedDomains) InitBlockOverlay(tx kv.TemporalTx, tmpDir string) error if err != nil { return fmt.Errorf("init block overlay: %w", err) } + // Route domain reads on overlay read views through this SharedDomains so + // consumers (RPC, txpool) reading "latest" state via the published overlay + // see sd.mem and the in-flight async-commit generation chain — not a DB + // that lags behind a not-yet-landed background commit. + overlay.SetDomainGetterFactory(func(roTx kv.TemporalTx) kv.TemporalGetter { + return sd.AsGetter(roTx) + }) sd.blockOverlay.Store(overlay) return nil } diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index e53e7d7b67f..4fe9b555323 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -153,9 +153,12 @@ func (e *ExecModule) runCommit(gen *commitGen) { // Commit; this is a safety net (Rollback is idempotent). gen.roTx.Rollback() e.markGenCommitted(gen) - if dispatcher := e.pipelineExecutor.Dispatcher(); dispatcher != nil { - dispatcher.PublishOverlay(nil) - } + // NOTE: the worker does NOT touch the published overlay (Events.LatestSD). + // Publishing the latest executed block's SharedDomains is a foreground + // concern, owned by updateForkChoice (dispatchNotificationsFromOverlay). + // A background commit landing does not change "what the latest block is", + // so it must not republish — doing so previously nil'd LatestSD whenever + // the worker caught up, leaving the block builder / RPC caches blind. } // markGenCommitted records that a generation's commit has landed. The @@ -187,29 +190,48 @@ func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Unlock() } -// drainCommittedGens closes and clears the generation chain when every -// generation in it has committed. Called by a foreground op while it holds -// the semaphore — no concurrent reader exists, so closing the SharedDomains -// is race-free. If any generation is still uncommitted the chain is left -// intact. Returns true when the chain was drained. -func (e *ExecModule) drainCommittedGens() bool { +// drainCommittedGens retires fully-committed generations, but ALWAYS keeps +// the newest one alive. The newest generation's SharedDomains is what the +// foreground published as Events.LatestSD — the latest executed block's +// state — and the block builder + RPC caches read through it until a newer +// block supersedes it. Retiring it here would make LatestSD point at a +// closed SD (the root cause of the builder reading a stale DB). +// +// Older generations are detached + closed only once EVERY generation has +// committed: each closed generation's delta is durably in the DB, and the +// kept newest generation reads its own committed delta via the stateCache / +// files, so detaching the parent chain loses nothing. Called by a foreground +// op holding the semaphore, so closing is race-free. +func (e *ExecModule) drainCommittedGens() { e.fgMu.Lock() defer e.fgMu.Unlock() - if len(e.gens) == 0 { - return true + if len(e.gens) <= 1 { + return // nothing to retire — keep the (at most one) newest generation } for _, g := range e.gens { if !g.committed { - return false + return // a commit is still in flight — leave the chain intact } } - gens := e.gens - e.gens = nil - // Detach + close oldest→newest; each generation's data is in the DB, - // so a reader that still walked the chain would get the same bytes. - for _, g := range gens { + newest := e.gens[len(e.gens)-1] + old := e.gens[:len(e.gens)-1] + e.gens = []*commitGen{newest} + newest.sd.SetParent(nil) + for _, g := range old { g.sd.SetParent(nil) g.sd.Close() } - return true +} + +// closeAllGens detaches + closes every remaining generation. Called only at +// shutdown (after the commit worker has stopped) to release the newest +// generation that drainCommittedGens deliberately keeps alive. +func (e *ExecModule) closeAllGens() { + e.fgMu.Lock() + defer e.fgMu.Unlock() + for _, g := range e.gens { + g.sd.SetParent(nil) + g.sd.Close() + } + e.gens = nil } diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index bac5f961f50..5308b962aa8 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -329,6 +329,10 @@ func (e *ExecModule) WaitIdle(ctx context.Context) { // Stop the background-commit worker and wait for it to drain its queue // (including any in-flight commit tx) so DB-close does not race it. e.stopCommitWorker() + // Close the generation(s) still held — drainCommittedGens deliberately + // keeps the newest alive as Events.LatestSD; release it now that the + // worker has stopped and no foreground/RPC reader can be in flight. + e.closeAllGens() } // stopCommitWorker signals the background-commit worker to drain and exit, diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 2736a62c868..8fad17f5554 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -664,12 +664,38 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa e.lock.Unlock() } + // Background commit (gate item 2): register the generation in the + // in-flight chain BEFORE dispatching notifications / publishing the + // overlay. The commit worker republishes the overlay as it commits + // each generation (bg_commit.go: latestUncommittedGenSD); if this + // generation were not yet in e.gens at that point, the worker would + // publish a stale (older or nil) overlay and clobber this FCU's. + var bgGen *commitGen + if e.fcuBackgroundCommit { + bgGen = &commitGen{ + sd: currentContext, + roTx: roTx, + finishProgressBefore: finishProgressBefore, + isSynced: isSynced, + initialCycle: initialCycle, + } + e.addGen(bgGen) + } + // Dispatch notifications from the SD overlay (before flush/commit). // After this, all consumers have the data — the semaphore can be // released and flush/commit/prune can proceed without blocking the // next FCU. e.logger.Debug("[updateForkChoice] dispatching notifications", "head", blockHash, "bgCommit", e.fcuBackgroundCommit) if err := e.dispatchNotificationsFromOverlay(currentContext, finishProgressBefore); err != nil { + // The generation is already registered; enqueue it so its commit + // still lands and the chain can drain — otherwise an uncommitted + // generation would block drainCommittedGens forever. + if bgGen != nil { + roTx = nil + currentContext = nil + e.enqueueCommit(bgGen) + } return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, fmt.Errorf("fcu: dispatch notifications: %w", err), stateFlushingInParallel) } @@ -685,19 +711,9 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // so the commit RwTx never overlaps a foreground roTx. The // generation is added to the chain so the next FCU's SD can // read its not-yet-committed domain state via SetParent. - bgRoTx := roTx roTx = nil - bgSD := currentContext currentContext = nil - gen := &commitGen{ - sd: bgSD, - roTx: bgRoTx, - finishProgressBefore: finishProgressBefore, - isSynced: isSynced, - initialCycle: initialCycle, - } - e.addGen(gen) - e.enqueueCommit(gen) + e.enqueueCommit(bgGen) } else { // Foreground commit: pass the outer roTx so it gets released // between Flush and Commit (same openTxs=2→1 optimization as diff --git a/node/eth/backend.go b/node/eth/backend.go index 53bac782138..7d59139be52 100644 --- a/node/eth/backend.go +++ b/node/eth/backend.go @@ -733,6 +733,15 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger go sentry_multi_client.NewBALDownloader(backend.sentryProvider.Client, backend.chainDB, logger).Run(backend.sentryCtx) } + // SD-wrapper state cache, shared by the txpool and the embedded + // rpcdaemon. Both read account state from the authoritative in-flight + // SharedDomains rather than an async-notification-fed cache that can + // diverge from it during a background commit (gate item 2). execModule + // is wired later by NewExecModule; until then Cache.View falls back to + // the published SD via SetPublishedSD. + execmoduleCache := &execmodule.Cache{} + execmoduleCache.SetPublishedSD(backend.notifications.Events.LatestSD) + var txnProvider txnprovider.TxnProvider if config.TxPool.Disable { backend.txPoolGrpcServer = &txpool.GrpcDisabled{} @@ -748,7 +757,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger ctx, config.TxPool, backend.chainDB, - kvcache.NewSimple(), + execmoduleCache, sentries, backend.stateDiffClient, blockBuilderNotifyNewTxns, @@ -762,8 +771,6 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger txnProvider = backend.txPool } - execmoduleCache := &execmodule.Cache{} - execmoduleCache.SetPublishedSD(backend.notifications.Events.LatestSD) httpRpcCfg := stack.Config().Http httpRpcCfg.StateCache.LocalCache = execmoduleCache ethRpcClient, txPoolRpcClient, miningRpcClient, rpcDaemonStateCache, rpcFilters := rpcdaemoncli.EmbeddedServices( diff --git a/rpc/jsonrpc/bor_api_impl.go b/rpc/jsonrpc/bor_api_impl.go index 33ba68f8e0f..1c1edf4c341 100644 --- a/rpc/jsonrpc/bor_api_impl.go +++ b/rpc/jsonrpc/bor_api_impl.go @@ -102,7 +102,7 @@ func (api *BorImpl) GetAuthor(blockNrOrHash *rpc.BlockNumberOrHash) (accounts.Ad //nolint:nestif if blockNrOrHash == nil { - latestBlockNum, err2 := rpchelper.GetLatestBlockNumber(tx) + latestBlockNum, err2 := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err2 != nil { return accounts.NilAddress, err2 } @@ -298,7 +298,7 @@ func (api *BorImpl) getLatestBlockNum(ctx context.Context) (uint64, error) { } defer tx.Rollback() - return rpchelper.GetLatestBlockNumber(tx) + return rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) } // GetSnapshotProposer retrieves the in-turn signer at a given block. diff --git a/rpc/jsonrpc/debug_api.go b/rpc/jsonrpc/debug_api.go index 9780ec91a02..2f95f472d76 100644 --- a/rpc/jsonrpc/debug_api.go +++ b/rpc/jsonrpc/debug_api.go @@ -105,7 +105,7 @@ func (api *DebugAPIImpl) SetHead(ctx context.Context, number hexutil.Uint64) err } defer tx.Rollback() - currentHead, err := rpchelper.GetLatestBlockNumber(tx) + currentHead, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return err } @@ -225,7 +225,7 @@ func (api *DebugAPIImpl) AccountRange(ctx context.Context, blockNrOrHash rpc.Blo if number == rpc.LatestBlockNumber { var err error - blockNumber, err = stages.GetStageProgress(tx, stages.Execution) + blockNumber, err = stages.GetStageProgress(api.filters.WithOverlay(tx), stages.Execution) if err != nil { return state.IteratorDump{}, fmt.Errorf("last block has not found: %w", err) } @@ -290,7 +290,7 @@ func (api *DebugAPIImpl) GetModifiedAccountsByNumber(ctx context.Context, startN } defer tx.Rollback() - latestBlock, err := stages.GetStageProgress(tx, stages.Execution) + latestBlock, err := stages.GetStageProgress(api.filters.WithOverlay(tx), stages.Execution) if err != nil { return nil, err } @@ -492,7 +492,7 @@ func (api *DebugAPIImpl) GetModifiedAccountsByHash(ctx context.Context, startHas } defer tx.Rollback() - latestBlock, err := stages.GetStageProgress(tx, stages.Execution) + latestBlock, err := stages.GetStageProgress(api.filters.WithOverlay(tx), stages.Execution) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/debug_execution_witness.go b/rpc/jsonrpc/debug_execution_witness.go index a1735b9536a..efab22ba941 100644 --- a/rpc/jsonrpc/debug_execution_witness.go +++ b/rpc/jsonrpc/debug_execution_witness.go @@ -1113,7 +1113,7 @@ func (api *DebugAPIImpl) buildExpectedPostState( postSdCtx.SetDeferBranchUpdates(false) // Set up to read state at current block (after execution) - latestBlock, err := rpchelper.GetLatestBlockNumber(tx) + latestBlock, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, nil, fmt.Errorf("failed to get latest block: %w", err) } diff --git a/rpc/jsonrpc/erigon_receipts.go b/rpc/jsonrpc/erigon_receipts.go index 80e25c05d19..c4ddaf48f96 100644 --- a/rpc/jsonrpc/erigon_receipts.go +++ b/rpc/jsonrpc/erigon_receipts.go @@ -97,8 +97,9 @@ func (api *ErigonImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) end = header.Number.Uint64() } else { - // Convert the RPC block numbers into internal representations - latest, err := rpchelper.GetLatestBlockNumber(tx) + // Convert the RPC block numbers into internal representations. + // Overlay-wrap so an in-flight (not-yet-committed) FCU block is the head. + latest, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } @@ -192,8 +193,9 @@ func (api *ErigonImpl) GetLatestLogs(ctx context.Context, crit filters.FilterCri begin = header.Number.Uint64() end = header.Number.Uint64() } else { - // Convert the RPC block numbers into internal representations - latest, err := rpchelper.GetLatestBlockNumber(tx) + // Convert the RPC block numbers into internal representations. + // Overlay-wrap so an in-flight (not-yet-committed) FCU block is the head. + latest, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/erigon_system.go b/rpc/jsonrpc/erigon_system.go index ed7eb3751e7..0c109c0ec07 100644 --- a/rpc/jsonrpc/erigon_system.go +++ b/rpc/jsonrpc/erigon_system.go @@ -88,7 +88,7 @@ func (api *ErigonImpl) BlockNumber(ctx context.Context, rpcBlockNumPtr *rpc.Bloc return 0, err } default: - blockNum, err = rpchelper.GetLatestExecutedBlockNumber(tx) + blockNum, err = rpchelper.GetLatestExecutedBlockNumber(overlayTx) if err != nil { return 0, err } diff --git a/rpc/jsonrpc/eth_block.go b/rpc/jsonrpc/eth_block.go index 65db83c6ba0..0a6dba5867a 100644 --- a/rpc/jsonrpc/eth_block.go +++ b/rpc/jsonrpc/eth_block.go @@ -376,7 +376,9 @@ func (api *APIImpl) GetBlockTransactionCountByNumber(ctx context.Context, blockN return nil, err } - latestBlockNumber, err := rpchelper.GetLatestBlockNumber(tx) + // Overlay-wrap so a freshly-FCU'd block whose background commit is still + // in flight is not mistaken for a future block (which would return null). + latestBlockNumber, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_call.go b/rpc/jsonrpc/eth_call.go index 96b90566e34..a337017ab61 100644 --- a/rpc/jsonrpc/eth_call.go +++ b/rpc/jsonrpc/eth_call.go @@ -467,7 +467,7 @@ func (api *APIImpl) getProof(ctx context.Context, roTx kv.TemporalTx, address co sdCtx := domains.GetCommitmentContext() sdCtx.SetDeferBranchUpdates(false) - latestBlock, err := rpchelper.GetLatestBlockNumber(roTx) + latestBlock, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(roTx)) if err != nil { return nil, err } @@ -674,7 +674,7 @@ func (api *BaseAPI) getWitness(ctx context.Context, db kv.TemporalRoDB, blockNrO return nil, fmt.Errorf("transaction index out of bounds: %d", txIndex) } - latestBlock, err := rpchelper.GetLatestBlockNumber(roTx) + latestBlock, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(roTx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_receipts.go b/rpc/jsonrpc/eth_receipts.go index c39e8acdd88..b6c80d0f55f 100644 --- a/rpc/jsonrpc/eth_receipts.go +++ b/rpc/jsonrpc/eth_receipts.go @@ -128,7 +128,10 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t end = num } else { // Convert the RPC block numbers into internal representations - latest, _, _, err := rpchelper.GetBlockNumber(ctx, rpc.BlockNumberOrHashWithNumber(rpc.LatestExecutedBlockNumber), tx, api._blockReader, nil) + // Pass api.filters (not nil) so LatestExecutedBlockNumber resolves + // through the block overlay — otherwise a freshly-FCU'd block whose + // background commit is still in flight is invisible here. + latest, _, _, err := rpchelper.GetBlockNumber(ctx, rpc.BlockNumberOrHashWithNumber(rpc.LatestExecutedBlockNumber), tx, api._blockReader, api.filters) if err != nil { return nil, err } @@ -173,7 +176,7 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t return nil, &rpc.CustomError{Message: errInvalidBlockRange, Code: rpc.ErrCodeInvalidParams} } if end > roaring.MaxUint32 { - latest, err := rpchelper.GetLatestBlockNumber(tx) + latest, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } @@ -185,7 +188,8 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t // Check if the requested blocks have been executed. // This prevents returning empty results when blocks exist but haven't been executed yet. - latestExecuted, err := rpchelper.GetLatestExecutedBlockNumber(tx) + // Overlay-wrap so an in-flight (not-yet-committed) FCU block counts as executed. + latestExecuted, err := rpchelper.GetLatestExecutedBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_simulation.go b/rpc/jsonrpc/eth_simulation.go index 3a377700d39..2dc9f4b1630 100644 --- a/rpc/jsonrpc/eth_simulation.go +++ b/rpc/jsonrpc/eth_simulation.go @@ -128,7 +128,7 @@ func (api *APIImpl) SimulateV1(ctx context.Context, req SimulationRequest, block if err != nil { return nil, err } - latestBlockNumber, err := rpchelper.GetLatestBlockNumber(tx) + latestBlockNumber, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_system.go b/rpc/jsonrpc/eth_system.go index fa05a8d3bae..312f81c0f4f 100644 --- a/rpc/jsonrpc/eth_system.go +++ b/rpc/jsonrpc/eth_system.go @@ -553,7 +553,7 @@ func (b *GasPriceOracleBackend) ChainConfig() *chain.Config { } func (b *GasPriceOracleBackend) GetLatestBlockNumber() (uint64, error) { - return rpchelper.GetLatestBlockNumber(b.tx) + return rpchelper.GetLatestBlockNumber(b.baseApi.filters.WithOverlay(b.tx)) } func (b *GasPriceOracleBackend) GetReceipts(ctx context.Context, block *types.Block) (types.Receipts, error) { @@ -575,7 +575,7 @@ func (b *GasPriceOracleBackend) PendingBlockAndReceipts() (*types.Block, types.R if block := b.baseApi.pendingBlock(); block != nil { return block, nil } - latestNum, err := rpchelper.GetLatestBlockNumber(b.tx) + latestNum, err := rpchelper.GetLatestBlockNumber(b.baseApi.filters.WithOverlay(b.tx)) if err != nil { return nil, nil } diff --git a/rpc/jsonrpc/overlay_api.go b/rpc/jsonrpc/overlay_api.go index 97e09af02f5..cb01ddcfbbd 100644 --- a/rpc/jsonrpc/overlay_api.go +++ b/rpc/jsonrpc/overlay_api.go @@ -629,7 +629,7 @@ func getBeginEnd(ctx context.Context, tx kv.Tx, api *OverlayAPIImpl, crit filter return 0, 0, fmt.Errorf("end (%d) < begin (%d)", end, begin) } if end > roaring.MaxUint32 { - latest, err := rpchelper.GetLatestBlockNumber(tx) + latest, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return 0, 0, err } diff --git a/rpc/rpchelper/helper.go b/rpc/rpchelper/helper.go index 0811d7d36e6..8cf598b307a 100644 --- a/rpc/rpchelper/helper.go +++ b/rpc/rpchelper/helper.go @@ -124,7 +124,13 @@ func _GetBlockNumber(ctx context.Context, requireCanonical bool, blockNrOrHash r if pendingBlock == nil { blockNumber = plainStateBlockNumber } else { - return pendingBlock.NumberU64(), pendingBlock.Hash(), false, true, nil + // latest must be true when the pending block coincides with the + // in-flight execution progress (plainStateBlockNumber is read via + // overlayTx, so it includes a not-yet-committed block). Returning + // latest=false there forces the historical read path, whose + // history is not committed during a background commit — yielding + // a stale/zero nonce. Mirrors the LatestBlockNumber handling. + return pendingBlock.NumberU64(), pendingBlock.Hash(), pendingBlock.NumberU64() == plainStateBlockNumber, true, nil } case rpc.LatestExecutedBlockNumber: blockNumber = plainStateBlockNumber From c2ee3fc13a4ab8369100b4b98ecaaa4657812806 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 23:35:48 +0000 Subject: [PATCH 107/167] execution: published-leaf SD reads for the outer layer under async commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the previous commit. The outer application layer (RPC, txpool, block builder) must read a stable, completed-block "leaf" SharedDomains — never the executor's in-progress, mid-mutation SD, and never an SD that has lost snapshot stability by being drained. - Cache.View now returns the latest *published* SharedDomains, never e.currentContext. currentContext is the block-being-executed's half-written SD (its coinbase nonce is already bumped mid-block); a consumer that read it saw a not-yet-final block, so eth_getTransactionCount and txpool validateTx in one request disagreed by a nonce ("nonce too low"). The published leaf only advances at block completion. - A committed generation keeps its mem-batch (SharedDomains.FlushKeepMem, TemporalMemBatch.FlushWithCallback drain=false). It stays Events.LatestSD — the latest executed block's snapshot — so it must answer reads from its own immutable delta, not fall through to the shared, concurrently-mutated caches. runForkchoiceFlushCommit uses FlushKeepMem. - drainCommittedGens is now a no-op: generations are retained for the run and released as a unit at shutdown (closeAllGens, called from WaitIdle). Freeing one mid-run safely needs a refcount/scoped-read proving no consumer still reads it — the explicit memory-coordination follow-up. - The block builder's filterSd (bad-txn filter) is chained to the parent generation. Without it, filterBadTransactions resolved the sender nonce from committed files only — stale during a background commit — and dropped valid spill-over txns as nonce-gapped, building blocks short of txns. engineapi single-txn async-commit test now 10/10; the two builder multi-txn tests pass. Residual: an intermittent package-run "behind commitment" desync (commitment domain one block ahead of the TxNums index) — next item. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/kv/kv_interface.go | 2 +- db/state/execctx/domain_shared.go | 18 ++++++++++- db/state/temporal_mem_batch.go | 26 +++++++++++----- execution/builder/builder.go | 2 +- execution/builder/exec.go | 12 +++++++- execution/execmodule/bg_commit.go | 47 ++++++++++------------------- execution/execmodule/exec_module.go | 17 ++++++----- execution/execmodule/forkchoice.go | 2 +- 8 files changed, 74 insertions(+), 52 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index e757a32effd..ad7cf7284ca 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,7 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error - FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, step Step)) error + FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, step Step), drain bool) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 60999ac5a59..6cfcd57d811 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -772,7 +772,23 @@ func (sd *SharedDomains) Close() { sd.sdCtx = nil } +// Flush writes the mem-batch to tx, refreshes the caches, and drains the +// mem-batch (frees its in-memory delta). func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { + return sd.flush(ctx, tx, true) +} + +// FlushKeepMem is Flush without the post-flush drain: the mem-batch is kept +// intact so this SharedDomains stays a stable, self-contained snapshot after +// commit. Used for a published commit generation (gate item 2) — it remains +// Events.LatestSD and must answer reads from its own immutable delta, not +// fall through to the shared concurrently-mutated caches. The mem is freed +// at Close. +func (sd *SharedDomains) FlushKeepMem(ctx context.Context, tx kv.RwTx) error { + return sd.flush(ctx, tx, false) +} + +func (sd *SharedDomains) flush(ctx context.Context, tx kv.RwTx, drain bool) error { defer mxFlushTook.ObserveDuration(time.Now()) if sd.sdCtx.HasPendingUpdate() { @@ -834,7 +850,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } } } - }); err != nil { + }, drain); err != nil { return err } if sd.branchCache != nil { diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index fa67b0bd1f5..eb44ec5f95b 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -764,22 +764,30 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { // FlushWithCallback flushes the mem-batch to tx, but first invokes cb for // every (domain, key, latest-value, step) tuple so downstream caches can be -// refreshed, then drains the in-memory latest/history state. +// refreshed. When drain is true it then drains the in-memory latest/history +// state; when false the mem-batch is kept intact after the flush. // -// The callback, the MDBX flush and the drain run under latestStateLock as -// one atomic window: a concurrent reader sees either the full pre-flush mem -// state or the fully-drained post-flush state, never a partial mix. cb runs -// before the flush so that during the MDBX write window the cache already -// holds the entry — a reader going mem → cache → MDBX never sees a key -// missing from all three. data may be nil/empty for deletes — cb +// The callback, the MDBX flush and the (optional) drain run under +// latestStateLock as one atomic window: a concurrent reader sees either the +// full pre-flush mem state or the fully-drained post-flush state, never a +// partial mix. cb runs before the flush so that during the MDBX write window +// the cache already holds the entry — a reader going mem → cache → MDBX never +// sees a key missing from all three. data may be nil/empty for deletes — cb // distinguishes on len(v)==0. // +// drain=false is used for a published generation's commit (gate item 2): the +// generation's SharedDomains remains Events.LatestSD — the latest executed +// block's stable snapshot — so its mem must survive the flush and answer +// reads from its own immutable delta rather than falling through to the +// shared, concurrently-mutated caches. Such a mem-batch is freed at Close. +// // Used by SharedDomains.Flush to keep the BranchCache (commitment domain) // and the StateCache (accounts/storage/code) in sync — refreshed only on // flush, so they hold committed, fork-agnostic state. func (sd *TemporalMemBatch) FlushWithCallback( ctx context.Context, tx kv.RwTx, cb func(domain kv.Domain, k []byte, v []byte, step kv.Step), + drain bool, ) error { sd.latestStateLock.Lock() defer sd.latestStateLock.Unlock() @@ -806,7 +814,9 @@ func (sd *TemporalMemBatch) FlushWithCallback( if err := sd.flushLocked(ctx, tx); err != nil { return err } - sd.drainLocked() + if drain { + sd.drainLocked() + } return nil } diff --git a/execution/builder/builder.go b/execution/builder/builder.go index 388b8f96b49..68febc97d4e 100644 --- a/execution/builder/builder.go +++ b/execution/builder/builder.go @@ -170,7 +170,7 @@ func (b *Builder) Build(param *Parameters, interrupt *atomic.Bool) (result *type if err := createBlock(b.ctx, sd, compositeTx, executionAt, createCfg, b.logger); err != nil { return nil, err } - if err := execBlock(b.ctx, sd, compositeTx, executionAt, execCfg, b.executeBlockCfg, b.logger); err != nil { + if err := execBlock(b.ctx, sd, parentSD, compositeTx, executionAt, execCfg, b.executeBlockCfg, b.logger); err != nil { return nil, err } if err := finishBlock(compositeTx, finishCfg, b.logger); err != nil { diff --git a/execution/builder/exec.go b/execution/builder/exec.go index 52530e451bf..9cff907ea0f 100644 --- a/execution/builder/exec.go +++ b/execution/builder/exec.go @@ -97,7 +97,7 @@ func StageBuilderExecCfg( // // TODO: // - resubmitAdjustCh - variable is not implemented -func execBlock(ctx context0.Context, sd *execctx.SharedDomains, tx kv.TemporalTx, executionAt uint64, cfg BuilderExecCfg, execCfg stagedsync.ExecuteBlockCfg, logger log.Logger) (err error) { +func execBlock(ctx context0.Context, sd *execctx.SharedDomains, parentSD *execctx.SharedDomains, tx kv.TemporalTx, executionAt uint64, cfg BuilderExecCfg, execCfg stagedsync.ExecuteBlockCfg, logger log.Logger) (err error) { const logPrefix = "BuilderExec" // Copy vmConfig to avoid mutating the shared struct across concurrent Build calls. @@ -134,6 +134,16 @@ func execBlock(ctx context0.Context, sd *execctx.SharedDomains, tx kv.TemporalTx return err } defer filterSd.Close() + // Chain filterSd to the parent generation (gate item 2): like the main + // sd, the filter must read the latest executed block's not-yet-committed + // domain state. Without this, filterBadTransactions resolves the sender + // nonce from committed files only — stale during a background commit — + // and drops valid spill-over txns as nonce-gapped (block built empty). + // filterSd's own speculative writes stay in filterSd.mem; the parent is + // read-only here, so sd's commitment is unaffected. + if parentSD != nil { + filterSd.SetParent(parentSD) + } filterWriter := state.NewWriter(filterSd.AsPutDel(filterMb), nil, txNum) filterReader := state.NewReaderV3(filterSd.AsGetter(filterMb)) diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 4fe9b555323..191cf730e62 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -190,42 +190,27 @@ func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Unlock() } -// drainCommittedGens retires fully-committed generations, but ALWAYS keeps -// the newest one alive. The newest generation's SharedDomains is what the -// foreground published as Events.LatestSD — the latest executed block's -// state — and the block builder + RPC caches read through it until a newer -// block supersedes it. Retiring it here would make LatestSD point at a -// closed SD (the root cause of the builder reading a stale DB). +// drainCommittedGens is intentionally a no-op for now (model A). // -// Older generations are detached + closed only once EVERY generation has -// committed: each closed generation's delta is durably in the DB, and the -// kept newest generation reads its own committed delta via the stateCache / -// files, so detaching the parent chain loses nothing. Called by a foreground -// op holding the semaphore, so closing is race-free. +// Each generation's SharedDomains.mem is kept after commit (FlushKeepMem) so +// the generation stays a stable, self-contained snapshot — it may still be +// Events.LatestSD, or a parent in some reader's chain. Freeing one mid-run +// would need to prove no consumer (block builder, RPC cache, txpool) is still +// reading it; LatestSD hands out a bare pointer, so that proof needs a +// refcount / scoped-read primitive we have not built yet. Until then, +// generations are released as a unit at shutdown by closeAllGens. +// +// Precise per-generation freeing — a refcounted/scoped-read lifetime, or a +// shallow COW snapshot decoupled from the generation — is the explicit +// memory-coordination follow-up (next optimization phase). The call site is +// kept as the documented hook for that work. func (e *ExecModule) drainCommittedGens() { - e.fgMu.Lock() - defer e.fgMu.Unlock() - if len(e.gens) <= 1 { - return // nothing to retire — keep the (at most one) newest generation - } - for _, g := range e.gens { - if !g.committed { - return // a commit is still in flight — leave the chain intact - } - } - newest := e.gens[len(e.gens)-1] - old := e.gens[:len(e.gens)-1] - e.gens = []*commitGen{newest} - newest.sd.SetParent(nil) - for _, g := range old { - g.sd.SetParent(nil) - g.sd.Close() - } + // no-op — see closeAllGens and the memory-coordination follow-up } // closeAllGens detaches + closes every remaining generation. Called only at -// shutdown (after the commit worker has stopped) to release the newest -// generation that drainCommittedGens deliberately keeps alive. +// shutdown (after the commit worker has stopped) to release the generations +// that drainCommittedGens deliberately keeps alive for the lifetime of the run. func (e *ExecModule) closeAllGens() { e.fgMu.Lock() defer e.fgMu.Unlock() diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 5308b962aa8..ca6b4aa634a 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -120,15 +120,16 @@ var _ kvcache.Cache = (*Cache)(nil) // compile-time interface check var _ kvcache.CacheView = (*CacheView)(nil) // compile-time interface check func (c *Cache) View(_ context.Context, tx kv.TemporalTx) (kvcache.CacheView, error) { + // Read the latest *published* SharedDomains — the most recently executed + // block's stable leaf snapshot. Deliberately NOT e.currentContext: while + // an FCU is in progress currentContext is that block's half-written SD, + // actively mutated by the executing pipeline (the coinbase nonce is + // already bumped mid-block). A consumer that grabbed currentContext would + // see a not-yet-final block — e.g. GetTransactionCount and txpool + // validateTx in the same SubmitTransfer disagreeing by one nonce. The + // published leaf only advances at block completion, so it is stable. var context *execctx.SharedDomains - if c.execModule != nil { - c.execModule.lock.RLock() - context = c.execModule.currentContext - c.execModule.lock.RUnlock() - } - // Fall back to the published SD from Events during background commits - // (currentContext is nil but the SD is still valid in memory). - if context == nil && c.publishedSD != nil { + if c.publishedSD != nil { context = c.publishedSD() } diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 8fad17f5554..cb8eb1a9a67 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -853,7 +853,7 @@ func (e *ExecModule) runForkchoiceFlushCommit(sd *execctx.SharedDomains, roTxToC } defer rwTx.Rollback() flushStart := time.Now() - if err := sd.Flush(e.bacgroundCtx, rwTx); err != nil { + if err := sd.FlushKeepMem(e.bacgroundCtx, rwTx); err != nil { return nil, err } timings = append(timings, "flush", common.Round(time.Since(flushStart), 0)) From 2a4119265a09133df4397d86b84fb358c17bae03 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 5 Jun 2026 16:54:16 +0000 Subject: [PATCH 108/167] execution/cache, state, stagedsync: txNum/epoch cache invalidation; remove the schedule-time ValidateAndPrepare purge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state cache carried a per-domain blockHash and was scrubbed by ValidateAndPrepare before every block. In the parallel executor that call sits in processRequest — a *schedule* step, not an apply step — copied from the serial path (#18868). With a 32-deep pipeline and heavy retry traffic the single blockHash almost never equals the next call's parentHash, so it took the wipe branch ~100% of the time: measured storage-cache purge_rate ~100%, hit ~35% during catch-up, the cache wiped every block. Make the cache what it should be — a SharedDomains implementation detail, populated only at flush (committed, fork-agnostic state) and invalidated only on unwind. Coherence is now txNum/epoch based, no block awareness and no diffset: - Each GenericCache entry carries (txNum, epoch). A read is valid iff it was written in the current epoch OR its txNum is at/below unwindFloor (predates every unwind). - Unwind(txNum) bumps the epoch and lowers the floor — O(1), no scan. Stale entries are dropped lazily on their next read. txNum slots are reused across forks, so the epoch (not the txNum) tells a dead fork's write from the live fork's at the same txNum. - CodeCache clears its small mutable addr layers on unwind; immutable content-addressed code is kept. Wiring: FlushWithCallback delivers txNum (cache stamps it; branchCache derives step); read-population and read-ahead stamp the step's txNum upper bound. The three exec-flow ValidateAndPrepare calls are removed; the unwind path calls stateCache.Unwind(txNum) unconditionally (diffset-free, matching the overlay's maxtx prune — diffsets aren't generated below the reorg window, so the old changeSet-gated cache revert left a stale gap). RevertWithDiffset/blockHash/ClearWithHash and the fork-validation cache scrub are removed. (DB-level diffset retirement is a follow-on.) Co-Authored-By: Claude Opus 4.8 (1M context) --- db/kv/kv_interface.go | 2 +- db/state/execctx/domain_shared.go | 20 +- db/state/temporal_mem_batch.go | 6 +- execution/cache/cache.go | 24 +- execution/cache/cache_test.go | 501 +++++-------------------- execution/cache/code_cache.go | 66 +--- execution/cache/generic_cache.go | 122 +++--- execution/cache/state_cache.go | 79 +--- execution/exec/blocks_read_ahead.go | 9 +- execution/execmodule/exec_module.go | 10 +- execution/stagedsync/exec3.go | 7 - execution/stagedsync/exec3_parallel.go | 18 +- execution/stagedsync/exec3_serial.go | 6 +- execution/stagedsync/stage_execute.go | 17 +- execution/vm/contract.go | 3 +- 15 files changed, 243 insertions(+), 647 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index e757a32effd..7aee2feaee0 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,7 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error - FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, step Step)) error + FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, txNum uint64)) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 3f9ebd930b5..a184e56219c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -789,14 +789,14 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { // never per-write — so they mirror committed, fork-agnostic state: // CommitmentDomain → BranchCache, Accounts/Storage/Code → StateCache. if sd.branchCache != nil || sd.stateCache != nil { - if err := sd.mem.FlushWithCallback(ctx, tx, func(domain kv.Domain, k []byte, v []byte, step kv.Step) { + if err := sd.mem.FlushWithCallback(ctx, tx, func(domain kv.Domain, k []byte, v []byte, txNum uint64) { switch domain { case kv.CommitmentDomain: if sd.branchCache != nil { if len(v) == 0 { sd.branchCache.Invalidate(k) } else { - sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + sd.branchCache.Put(k, v, txNum/sd.StepSize(), "sd.Flush") } } case kv.AccountsDomain: @@ -806,7 +806,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.stateCache.Delete(kv.CodeDomain, k) sd.stateCache.DeleteAddrCodeHash(k) } else { - sd.stateCache.Put(kv.AccountsDomain, k, v) + sd.stateCache.Put(kv.AccountsDomain, k, v, txNum) sd.stateCache.DeleteAddrCodeHash(k) } } @@ -815,7 +815,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { if len(v) == 0 { sd.stateCache.Delete(kv.StorageDomain, k) } else { - sd.stateCache.Put(kv.StorageDomain, k, v) + sd.stateCache.Put(kv.StorageDomain, k, v, txNum) } } case kv.CodeDomain: @@ -823,7 +823,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { if len(v) == 0 { sd.stateCache.Delete(kv.CodeDomain, k) } else { - sd.stateCache.Put(kv.CodeDomain, k, v) + sd.stateCache.Put(kv.CodeDomain, k, v, txNum) } } } @@ -1029,12 +1029,18 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) return nil, 0, fmt.Errorf("storage %x read error: %w", k, err) } - // Populate state cache on successful storage read + // Populate state cache on successful read. Stamp with an upper bound on the + // value's write txNum (the read only gives us the file/step it came from): + // the last txNum of that step. An unwind below this bound can't leave the + // value stale, so frozen-step reads stay warm across unwinds while + // recent-step reads are dropped. (CodeCache addr layers are cleared wholesale + // on unwind, so PutCodeWithHash needs no txNum.) if sd.stateCache != nil { if domain == kv.CodeDomain && len(codeEthHash) > 0 && len(v) > 0 { sd.stateCache.PutCodeWithHash(k, v, codeEthHash) } else { - sd.stateCache.Put(domain, k, v) + readTxNum := (uint64(step)+1)*sd.StepSize() - 1 + sd.stateCache.Put(domain, k, v, readTxNum) } } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index fa67b0bd1f5..d68afa04828 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -779,7 +779,7 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { // flush, so they hold committed, fork-agnostic state. func (sd *TemporalMemBatch) FlushWithCallback( ctx context.Context, tx kv.RwTx, - cb func(domain kv.Domain, k []byte, v []byte, step kv.Step), + cb func(domain kv.Domain, k []byte, v []byte, txNum uint64), ) error { sd.latestStateLock.Lock() defer sd.latestStateLock.Unlock() @@ -790,7 +790,7 @@ func (sd *TemporalMemBatch) FlushWithCallback( continue } latest := history[len(history)-1] - cb(kv.Domain(d), []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + cb(kv.Domain(d), []byte(keyStr), latest.data, latest.txNum) } } // StorageDomain entries live in sd.storage (a btree), not sd.domains. @@ -799,7 +799,7 @@ func (sd *TemporalMemBatch) FlushWithCallback( return true } latest := history[len(history)-1] - cb(kv.StorageDomain, []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + cb(kv.StorageDomain, []byte(keyStr), latest.data, latest.txNum) return true }) diff --git a/execution/cache/cache.go b/execution/cache/cache.go index 170ab31dee7..2171137a0ca 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -16,16 +16,15 @@ package cache -import "github.com/erigontech/erigon/common" - // Cache is the interface for domain caches. // Implementations: GenericCache (for Account/Storage), CodeCache (for Code). type Cache interface { // Get retrieves data for the given key. Get(key []byte) ([]byte, bool) - // Put stores data for the given key. - Put(key []byte, value []byte) + // Put stores data for the given key, stamped with the txNum the value + // reflects (used for txNum/epoch unwind invalidation). + Put(key []byte, value []byte, txNum uint64) // Delete removes the data for the given key. Delete(key []byte) @@ -33,18 +32,11 @@ type Cache interface { // Clear removes all mutable entries from the cache. Clear() - // GetBlockHash returns the hash of the last block processed. - GetBlockHash() common.Hash - - // SetBlockHash sets the hash of the current block being processed. - SetBlockHash(hash common.Hash) - - // ValidateAndPrepare checks if parentHash matches the cache's blockHash. - // If mismatch, clears mutable caches. Returns true if valid, false if cleared. - ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool - - // ClearWithHash clears the cache and sets the block hash. - ClearWithHash(hash common.Hash) + // Unwind invalidates entries that reflect state above unwindToTxNum on a + // now-dead fork. Diffset-free: GenericCache bumps an epoch and lowers a + // floor (lazy evict on read); CodeCache clears its small mutable addr + // layers. Immutable content-addressed code layers are untouched. + Unwind(unwindToTxNum uint64) // Len returns the number of entries in the cache. Len() int diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 3160d45d216..ca83eb8b5a9 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -53,13 +53,6 @@ func makeValue(i int) []byte { // DomainCache Tests // ============================================================================= -func TestDomainCache_NewDomainCache(t *testing.T) { - c := NewDomainCache(100) - require.NotNil(t, c) - assert.Equal(t, 0, c.Len()) - assert.Equal(t, common.Hash{}, c.GetBlockHash()) -} - func TestDomainCache_NewWithByteCapacity(t *testing.T) { c := NewDomainCache(1 * datasize.MB) // 1MB require.NotNil(t, c) @@ -80,7 +73,7 @@ func TestDomainCache_GetPut(t *testing.T) { assert.Nil(t, v) // Put and Get - c.Put(addr, value) + c.Put(addr, value, 0) v, ok = c.Get(addr) assert.True(t, ok) assert.Equal(t, value, v) @@ -94,10 +87,10 @@ func TestDomainCache_PutUpdateValue(t *testing.T) { value1 := []byte{1, 2, 3, 4, 5, 6, 7, 8} // 8 bytes value2 := []byte{9, 10, 11} // 3 bytes - c.Put(addr, value1) + c.Put(addr, value1, 0) // Update with different value - c.Put(addr, value2) + c.Put(addr, value2, 0) v, ok := c.Get(addr) assert.True(t, ok) @@ -111,11 +104,11 @@ func TestDomainCache_PutCapacityLimit_NoOpMode(t *testing.T) { // Two entries take 94 bytes; cap at 100 leaves no room for a third. c := NewDomainCacheMode(100, ModeNoOp) - c.Put(makeAddr(1), makeValue(1)) - c.Put(makeAddr(2), makeValue(2)) + c.Put(makeAddr(1), makeValue(1), 0) + c.Put(makeAddr(2), makeValue(2), 0) assert.Equal(t, 2, c.Len()) - c.Put(makeAddr(3), makeValue(3)) + c.Put(makeAddr(3), makeValue(3), 0) assert.Equal(t, 2, c.Len()) _, ok := c.Get(makeAddr(3)) @@ -123,7 +116,7 @@ func TestDomainCache_PutCapacityLimit_NoOpMode(t *testing.T) { // Updating an existing key always succeeds in either mode. newValue := []byte{100, 101, 102} - c.Put(makeAddr(1), newValue) + c.Put(makeAddr(1), newValue, 0) v, ok := c.Get(makeAddr(1)) assert.True(t, ok) assert.Equal(t, newValue, v) @@ -142,7 +135,7 @@ func TestDomainCache_PutEvictsWhenFull_EvictMode(t *testing.T) { } for i := 1; i <= 64; i++ { - c.Put(makeAddr(i), makeValue(i)) + c.Put(makeAddr(i), makeValue(i), 0) } // The newest entry must still be findable. v, ok := c.Get(makeAddr(64)) @@ -163,7 +156,7 @@ func TestDomainCache_Delete(t *testing.T) { c := NewDomainCache(100) addr := makeAddr(1) - c.Put(addr, makeValue(1)) + c.Put(addr, makeValue(1), 0) assert.Equal(t, 1, c.Len()) c.Delete(addr) @@ -176,89 +169,19 @@ func TestDomainCache_Delete(t *testing.T) { func TestDomainCache_Clear(t *testing.T) { c := NewDomainCache(100) - c.Put(makeAddr(1), makeValue(1)) - c.Put(makeAddr(2), makeValue(2)) + c.Put(makeAddr(1), makeValue(1), 0) + c.Put(makeAddr(2), makeValue(2), 0) assert.Equal(t, 2, c.Len()) c.Clear() assert.Equal(t, 0, c.Len()) } -func TestDomainCache_BlockHash(t *testing.T) { - c := NewDomainCache(100) - - assert.Equal(t, common.Hash{}, c.GetBlockHash()) - - hash := makeHash(1) - c.SetBlockHash(hash) - assert.Equal(t, hash, c.GetBlockHash()) -} - -func TestDomainCache_ValidateAndPrepare_EmptyCache(t *testing.T) { - c := NewDomainCache(100) - - parentHash := makeHash(1) - incomingHash := makeHash(2) - - // Empty cache should always be valid - valid := c.ValidateAndPrepare(parentHash, incomingHash) - assert.True(t, valid) - assert.Equal(t, incomingHash, c.GetBlockHash()) -} - -func TestDomainCache_ValidateAndPrepare_MatchingParent(t *testing.T) { - c := NewDomainCache(100) - - block1 := makeHash(1) - block2 := makeHash(2) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeValue(1)) - - // Parent matches current block hash - valid := c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) - assert.Equal(t, block2, c.GetBlockHash()) - // Data should be preserved - assert.Equal(t, 1, c.Len()) -} - -func TestDomainCache_ValidateAndPrepare_Mismatch(t *testing.T) { - c := NewDomainCache(100) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeValue(1)) - - // Parent doesn't match - should clear - valid := c.ValidateAndPrepare(block2, block3) - assert.False(t, valid) - assert.Equal(t, block3, c.GetBlockHash()) - // Data should be cleared - assert.Equal(t, 0, c.Len()) -} - -func TestDomainCache_ClearWithHash(t *testing.T) { - c := NewDomainCache(100) - - c.Put(makeAddr(1), makeValue(1)) - c.SetBlockHash(makeHash(1)) - - newHash := makeHash(2) - c.ClearWithHash(newHash) - - assert.Equal(t, 0, c.Len()) - assert.Equal(t, newHash, c.GetBlockHash()) -} - func TestDomainCache_PrintStatsAndReset(t *testing.T) { c := NewDomainCache(100) // Generate some hits and misses - c.Put(makeAddr(1), makeValue(1)) + c.Put(makeAddr(1), makeValue(1), 0) c.Get(makeAddr(1)) // hit c.Get(makeAddr(1)) // hit c.Get(makeAddr(2)) // miss @@ -284,14 +207,6 @@ func TestDomainCache_ImplementsInterface(t *testing.T) { // CodeCache Tests // ============================================================================= -func TestCodeCache_NewCodeCache(t *testing.T) { - c := NewCodeCache(100, 200) - require.NotNil(t, c) - assert.Equal(t, 0, c.Len()) - assert.Equal(t, 0, c.CodeLen()) - assert.Equal(t, common.Hash{}, c.GetBlockHash()) -} - func TestCodeCache_NewDefaultCodeCache(t *testing.T) { c := NewDefaultCodeCache() require.NotNil(t, c) @@ -311,7 +226,7 @@ func TestCodeCache_GetPut(t *testing.T) { assert.Nil(t, v) // Put and Get - c.Put(addr, code) + c.Put(addr, code, 0) v, ok = c.Get(addr) assert.True(t, ok) assert.Equal(t, code, v) @@ -323,7 +238,7 @@ func TestCodeCache_PutEmptyCode(t *testing.T) { c := NewCodeCache(100, 200) addr := makeAddr(1) - c.Put(addr, []byte{}) + c.Put(addr, []byte{}, 0) // Should not store empty code assert.Equal(t, 0, c.Len()) @@ -339,9 +254,9 @@ func TestCodeCache_CodeDeduplication(t *testing.T) { addr3 := makeAddr(3) // Three addresses with same code - c.Put(addr1, code) - c.Put(addr2, code) - c.Put(addr3, code) + c.Put(addr1, code, 0) + c.Put(addr2, code, 0) + c.Put(addr3, code, 0) // Should have 3 address mappings but only 1 code entry assert.Equal(t, 3, c.Len()) @@ -374,7 +289,7 @@ func TestCodeCache_AddrCapacityLimit(t *testing.T) { c := NewCodeCache(1024*1024, 1024*28) // 1MB code, ~1024 addr LRU entries for i := 0; i < 1100; i++ { - c.Put(wideAddr(i), wideCode(i)) + c.Put(wideAddr(i), wideCode(i), 0) } // Len should be exactly the LRU cap (1024), not silently truncated to 0. @@ -396,7 +311,7 @@ func TestCodeCache_AddrCapacityLimit(t *testing.T) { assert.Equal(t, 1100, c.CodeLen()) // Updating an existing addr re-writes the entry (LRU promotes to MRU). - c.Put(wideAddr(1099), wideCode(4242)) + c.Put(wideAddr(1099), wideCode(4242), 0) v, ok = c.Get(wideAddr(1099)) assert.True(t, ok) assert.Equal(t, wideCode(4242), v) @@ -408,12 +323,12 @@ func TestCodeCache_CodeCapacityLimit(t *testing.T) { c := NewCodeCache(25, 1024*1024) // 25 bytes code, 1MB addr // Fill code capacity - c.Put(makeAddr(1), makeCode(1)) - c.Put(makeAddr(2), makeCode(2)) + c.Put(makeAddr(1), makeCode(1), 0) + c.Put(makeAddr(2), makeCode(2), 0) assert.Equal(t, 2, c.CodeLen()) // Try to add more code - addr mapping added, but code not stored - c.Put(makeAddr(3), makeCode(3)) + c.Put(makeAddr(3), makeCode(3), 0) assert.Equal(t, 3, c.Len()) // addr mapping added assert.Equal(t, 2, c.CodeLen()) // code not added (at capacity) @@ -427,7 +342,7 @@ func TestCodeCache_Delete(t *testing.T) { addr := makeAddr(1) code := makeCode(1) - c.Put(addr, code) + c.Put(addr, code, 0) c.Delete(addr) assert.Equal(t, 0, c.Len()) @@ -441,8 +356,8 @@ func TestCodeCache_Delete(t *testing.T) { func TestCodeCache_Clear(t *testing.T) { c := NewCodeCache(100, 200) - c.Put(makeAddr(1), makeCode(1)) - c.Put(makeAddr(2), makeCode(2)) + c.Put(makeAddr(1), makeCode(1), 0) + c.Put(makeAddr(2), makeCode(2), 0) c.Clear() assert.Equal(t, 0, c.Len()) @@ -450,81 +365,10 @@ func TestCodeCache_Clear(t *testing.T) { assert.Equal(t, 2, c.CodeLen()) } -func TestCodeCache_BlockHash(t *testing.T) { - c := NewCodeCache(100, 200) - - assert.Equal(t, common.Hash{}, c.GetBlockHash()) - - hash := makeHash(1) - c.SetBlockHash(hash) - assert.Equal(t, hash, c.GetBlockHash()) -} - -func TestCodeCache_ValidateAndPrepare_EmptyCache(t *testing.T) { - c := NewCodeCache(100, 200) - - parentHash := makeHash(1) - incomingHash := makeHash(2) - - valid := c.ValidateAndPrepare(parentHash, incomingHash) - assert.True(t, valid) - assert.Equal(t, incomingHash, c.GetBlockHash()) -} - -func TestCodeCache_ValidateAndPrepare_MatchingParent(t *testing.T) { - c := NewCodeCache(100, 200) - - block1 := makeHash(1) - block2 := makeHash(2) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeCode(1)) - - valid := c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) - assert.Equal(t, block2, c.GetBlockHash()) - // Addr mapping should be preserved - assert.Equal(t, 1, c.Len()) -} - -func TestCodeCache_ValidateAndPrepare_Mismatch(t *testing.T) { - c := NewCodeCache(100, 200) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeCode(1)) - - valid := c.ValidateAndPrepare(block2, block3) - assert.False(t, valid) - assert.Equal(t, block3, c.GetBlockHash()) - // Addr mapping should be cleared - assert.Equal(t, 0, c.Len()) - // Code should still exist (immutable) - assert.Equal(t, 1, c.CodeLen()) -} - -func TestCodeCache_ClearWithHash(t *testing.T) { - c := NewCodeCache(100, 200) - - c.Put(makeAddr(1), makeCode(1)) - c.SetBlockHash(makeHash(1)) - - newHash := makeHash(2) - c.ClearWithHash(newHash) - - assert.Equal(t, 0, c.Len()) - assert.Equal(t, newHash, c.GetBlockHash()) - // Code should still exist - assert.Equal(t, 1, c.CodeLen()) -} - func TestCodeCache_PrintStatsAndReset(t *testing.T) { c := NewCodeCache(100, 200) - c.Put(makeAddr(1), makeCode(1)) + c.Put(makeAddr(1), makeCode(1), 0) c.Get(makeAddr(1)) // hit c.Get(makeAddr(2)) // miss @@ -544,7 +388,7 @@ func TestCodeCache_GetMissingCode(t *testing.T) { // Manually set addr mapping without code (simulates capacity limit scenario) addr := makeAddr(1) code := makeCode(1) - c.Put(addr, code) + c.Put(addr, code, 0) // Clear the code cache but keep addr mapping c.hashToCode.Clear() @@ -598,7 +442,7 @@ func TestStateCache_GetPut_Account(t *testing.T) { assert.Nil(t, v) // Put and Get - c.Put(kv.AccountsDomain, addr, value) + c.Put(kv.AccountsDomain, addr, value, 0) v, ok = c.Get(kv.AccountsDomain, addr) assert.True(t, ok) assert.Equal(t, value, v) @@ -612,7 +456,7 @@ func TestStateCache_GetPut_Storage(t *testing.T) { key[51] = 1 value := makeValue(1) - c.Put(kv.StorageDomain, key, value) + c.Put(kv.StorageDomain, key, value, 0) v, ok := c.Get(kv.StorageDomain, key) assert.True(t, ok) assert.Equal(t, value, v) @@ -624,7 +468,7 @@ func TestStateCache_GetPut_Code(t *testing.T) { addr := makeAddr(1) code := makeCode(1) - c.Put(kv.CodeDomain, addr, code) + c.Put(kv.CodeDomain, addr, code, 0) v, ok := c.Get(kv.CodeDomain, addr) assert.True(t, ok) assert.Equal(t, code, v) @@ -634,7 +478,7 @@ func TestStateCache_GetPut_UnsupportedDomain(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) // ReceiptDomain is not supported - c.Put(kv.ReceiptDomain, makeAddr(1), makeValue(1)) + c.Put(kv.ReceiptDomain, makeAddr(1), makeValue(1), 0) v, ok := c.Get(kv.ReceiptDomain, makeAddr(1)) assert.False(t, ok) assert.Nil(t, v) @@ -644,7 +488,7 @@ func TestStateCache_Delete(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) addr := makeAddr(1) - c.Put(kv.AccountsDomain, addr, makeValue(1)) + c.Put(kv.AccountsDomain, addr, makeValue(1), 0) c.Delete(kv.AccountsDomain, addr) _, ok := c.Get(kv.AccountsDomain, addr) @@ -661,7 +505,7 @@ func TestStateCache_PutEmpty_ThenGet_IsCacheHit(t *testing.T) { key[0] = 0x1d key[51] = 0xa2 - c.Put(kv.StorageDomain, key, nil) + c.Put(kv.StorageDomain, key, nil, 0) v, ok := c.Get(kv.StorageDomain, key) assert.True(t, ok, "Get after Put(nil) must be a cache hit, not a miss") @@ -676,7 +520,7 @@ func TestStateCache_PutEmptySlice_ThenGet_IsCacheHit(t *testing.T) { key[0] = 0x1d key[51] = 0xa2 - c.Put(kv.StorageDomain, key, []byte{}) + c.Put(kv.StorageDomain, key, []byte{}, 0) v, ok := c.Get(kv.StorageDomain, key) assert.True(t, ok, "Get after Put([]byte{}) must be a cache hit") @@ -693,9 +537,9 @@ func TestStateCache_Delete_UnsupportedDomain(t *testing.T) { func TestStateCache_Clear(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.StorageDomain, makeAddr(2), makeValue(2)) - c.Put(kv.CodeDomain, makeAddr(3), makeCode(3)) + c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1), 0) + c.Put(kv.StorageDomain, makeAddr(2), makeValue(2), 0) + c.Put(kv.CodeDomain, makeAddr(3), makeCode(3), 0) c.Clear() @@ -708,51 +552,6 @@ func TestStateCache_Clear(t *testing.T) { assert.False(t, ok3) } -func TestStateCache_ValidateAndPrepare_AllValid(t *testing.T) { - c := NewStateCache(100, 100, 100, 100, 100) - - block1 := makeHash(1) - block2 := makeHash(2) - - // Set same block hash on all caches - c.GetCache(kv.AccountsDomain).SetBlockHash(block1) - c.GetCache(kv.StorageDomain).SetBlockHash(block1) - c.GetCache(kv.CodeDomain).SetBlockHash(block1) - - valid := c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) -} - -func TestStateCache_ValidateAndPrepare_SomeMismatch(t *testing.T) { - c := NewStateCache(100, 100, 100, 100, 100) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - // Set different block hashes - c.GetCache(kv.AccountsDomain).SetBlockHash(block1) - c.GetCache(kv.StorageDomain).SetBlockHash(block2) // mismatch - c.GetCache(kv.CodeDomain).SetBlockHash(block1) - - valid := c.ValidateAndPrepare(block1, block3) - assert.False(t, valid) -} - -func TestStateCache_ClearWithHash(t *testing.T) { - c := NewStateCache(100, 100, 100, 100, 100) - - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.StorageDomain, makeAddr(2), makeValue(2)) - - newHash := makeHash(5) - c.ClearWithHash(newHash) - - assert.Equal(t, newHash, c.GetCache(kv.AccountsDomain).GetBlockHash()) - assert.Equal(t, newHash, c.GetCache(kv.StorageDomain).GetBlockHash()) - assert.Equal(t, newHash, c.GetCache(kv.CodeDomain).GetBlockHash()) -} - func TestStateCache_GetCache_OutOfBounds(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) @@ -776,7 +575,7 @@ func TestDomainCache_ConcurrentAccess(t *testing.T) { // Writer goroutine go func() { for i := 0; i < 100; i++ { - c.Put(makeAddr(i), makeValue(i)) + c.Put(makeAddr(i), makeValue(i), 0) } done <- true }() @@ -801,7 +600,7 @@ func TestCodeCache_ConcurrentAccess(t *testing.T) { // Writer goroutine go func() { for i := 0; i < 100; i++ { - c.Put(makeAddr(i), makeCode(i)) + c.Put(makeAddr(i), makeCode(i), 0) } done <- true }() @@ -830,9 +629,9 @@ func TestStateCache_DomainIsolation(t *testing.T) { storageData := []byte("storage") codeData := []byte{0x60, 0x00, 0x60, 0x00} // valid code - c.Put(kv.AccountsDomain, addr, accountData) - c.Put(kv.StorageDomain, addr, storageData) - c.Put(kv.CodeDomain, addr, codeData) + c.Put(kv.AccountsDomain, addr, accountData, 0) + c.Put(kv.StorageDomain, addr, storageData, 0) + c.Put(kv.CodeDomain, addr, codeData, 0) v1, ok1 := c.Get(kv.AccountsDomain, addr) v2, ok2 := c.Get(kv.StorageDomain, addr) @@ -851,50 +650,13 @@ func TestStateCache_DomainIsolation(t *testing.T) { // Block Continuity Tests // ============================================================================= -func TestBlockContinuity_Sequential(t *testing.T) { - c := NewDefaultStateCache() - - block0 := common.Hash{} - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - // Block 1 (parent = empty) - valid := c.ValidateAndPrepare(block0, block1) - assert.True(t, valid) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - - // Block 2 (parent = block1) - valid = c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) - // Data from block 1 should still be there - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.True(t, ok) - - // Block 3 (parent = block2) - valid = c.ValidateAndPrepare(block2, block3) - assert.True(t, valid) -} - -func TestBlockContinuity_Reorg(t *testing.T) { - c := NewDefaultStateCache() - - block1 := makeHash(1) - block2 := makeHash(2) - block2prime := makeHash(22) // fork - - // Build on block 1 - c.ValidateAndPrepare(common.Hash{}, block1) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - - // Continue to block 2 - c.ValidateAndPrepare(block1, block2) - - // Reorg: new block with different parent - valid := c.ValidateAndPrepare(block1, block2prime) - assert.False(t, valid) // Should detect mismatch -} - +// Fork-validation (engine_newPayload) of a block building on the canonical tip +// must NOT purge the hot cache when that block is subsequently applied +// canonically. Regression for the tip purge_rate bug: fork-validation advancing +// blockHash to the speculative block made the canonical apply mismatch & purge. +// Fork-validation of a block on a DIFFERENT parent (reorg proposal) must still +// purge, since cache-as-of-canonical-tip would serve incoherent reads, but it +// must not advance blockHash (canonical continues cleanly afterward). // ============================================================================= // RevertWithDiffset Tests // ============================================================================= @@ -916,134 +678,75 @@ func makeDiffKey(baseKey []byte, step uint64) string { return string(k) } -func TestRevertWithDiffset_SurgicalEviction(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - blockTip := makeHash(3) - blockTarget := makeHash(1) - - // Simulate cache state at the tip block. - c.ClearWithHash(blockTip) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) // touched by unwind - c.Put(kv.AccountsDomain, makeAddr(2), makeValue(2)) // untouched - c.Put(kv.StorageDomain, makeAddr(3), makeValue(3)) // touched by unwind - c.Put(kv.StorageDomain, makeAddr(4), makeValue(4)) // untouched - - // Build diffset: addr(1) changed in accounts, addr(3) changed in storage. - var diffset [kv.DomainLen][]kv.DomainEntryDiff - diffset[kv.AccountsDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(1), 0), Value: makeValue(10)}, - } - diffset[kv.StorageDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(3), 0), Value: makeValue(30)}, - } +// --- txNum/epoch unwind invalidation (replaces the blockHash/diffset model) --- - c.RevertWithDiffset(&diffset, blockTip, blockTarget) +// Entries stamped at/below the unwind point survive (warm hot set kept); entries +// above it from the now-dead epoch are dropped lazily on read. +func TestUnwind_KeepsBelowFloor_EvictsAbove(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + below := makeAddr(1) + above := makeAddr(2) + c.Put(below, makeValue(1), 50) // predates the unwind + c.Put(above, makeValue(2), 150) // written in the unwound range - // Touched keys should be evicted (cache miss). - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.False(t, ok, "touched account key should be evicted") - _, ok = c.Get(kv.StorageDomain, makeAddr(3)) - assert.False(t, ok, "touched storage key should be evicted") + c.Unwind(100) // keep <=100, drop >100 - // Untouched keys should be preserved. - v, ok := c.Get(kv.AccountsDomain, makeAddr(2)) - assert.True(t, ok, "untouched account key should be preserved") - assert.Equal(t, makeValue(2), v) - v, ok = c.Get(kv.StorageDomain, makeAddr(4)) - assert.True(t, ok, "untouched storage key should be preserved") - assert.Equal(t, makeValue(4), v) + v, ok := c.Get(below) + assert.True(t, ok, "entry below the unwind point must stay warm") + assert.Equal(t, makeValue(1), v) - // Block hash should be updated to the unwind target. - assert.Equal(t, blockTarget, c.GetCache(kv.AccountsDomain).GetBlockHash()) - assert.Equal(t, blockTarget, c.GetCache(kv.StorageDomain).GetBlockHash()) + _, ok = c.Get(above) + assert.False(t, ok, "entry above the unwind point must be invalidated") + assert.Equal(t, 1, c.Len(), "the stale entry is evicted lazily on its read") } -func TestRevertWithDiffset_HashMismatch_ClearsAll(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - blockTip := makeHash(3) - blockStale := makeHash(99) // doesn't match any cache - blockTarget := makeHash(1) +// The reused-txNum case: after an unwind, the live fork re-writes a key at the +// SAME txNum as the dead fork's write. The epoch — not the txNum — distinguishes +// them, so the dead entry reads stale and the re-written one reads valid. +func TestUnwind_ReusedTxNumDisambiguatedByEpoch(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + k := makeAddr(1) + c.Put(k, makeValue(1), 150) // dead fork, epoch 0 - c.ClearWithHash(blockTip) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.StorageDomain, makeAddr(2), makeValue(2)) + c.Unwind(100) // epoch -> 1, floor -> 100 - var diffset [kv.DomainLen][]kv.DomainEntryDiff + _, ok := c.Get(k) + assert.False(t, ok, "dead-fork entry (old epoch, above floor) reads stale") - // revertFromHash doesn't match the cache's block hash — full clear. - c.RevertWithDiffset(&diffset, blockStale, blockTarget) - - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.False(t, ok, "all keys should be cleared on hash mismatch") - _, ok = c.Get(kv.StorageDomain, makeAddr(2)) - assert.False(t, ok, "all keys should be cleared on hash mismatch") - - assert.Equal(t, blockTarget, c.GetCache(kv.AccountsDomain).GetBlockHash()) + c.Put(k, makeValue(2), 150) // live fork re-writes at the same txNum, epoch 1 + v, ok := c.Get(k) + assert.True(t, ok, "live-fork entry at the same txNum is valid (current epoch)") + assert.Equal(t, makeValue(2), v) } -func TestRevertWithDiffset_AccountChange_EvictsCode(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - blockTip := makeHash(5) - blockTarget := makeHash(3) +// A straggler the live fork never re-writes must not resurrect: it stays in a +// dead epoch above the floor and reads stale no matter how far execution +// advances afterwards (there is no rising high-water mark to re-validate it). +func TestUnwind_StragglerNeverResurrects(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + straggler := makeAddr(1) + c.Put(straggler, makeValue(1), 150) // epoch 0 - c.ClearWithHash(blockTip) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.CodeDomain, makeAddr(1), makeCode(1)) - c.Put(kv.CodeDomain, makeAddr(2), makeCode(2)) // untouched + c.Unwind(100) // epoch 1 - var diffset [kv.DomainLen][]kv.DomainEntryDiff - diffset[kv.AccountsDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(1), 0), Value: makeValue(10)}, + // Advance the live fork far past the straggler's txNum (no re-write of it). + for i := 2; i < 50; i++ { + c.Put(makeAddr(i), makeValue(i), uint64(200+i)) } - - c.RevertWithDiffset(&diffset, blockTip, blockTarget) - - // Account change evicts both the account and its code. - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.False(t, ok, "touched account should be evicted") - _, ok = c.Get(kv.CodeDomain, makeAddr(1)) - assert.False(t, ok, "code for touched account should be evicted") - - // Unrelated code entry preserved. - v, ok := c.Get(kv.CodeDomain, makeAddr(2)) - assert.True(t, ok, "untouched code should be preserved") - assert.Equal(t, makeCode(2), v) + _, ok := c.Get(straggler) + assert.False(t, ok, "straggler in a dead epoch must stay stale, never resurrect") } -func TestRevertWithDiffset_ThenValidateAndPrepare_Continuity(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - block2new := makeHash(22) // new block 2 after re-execution - - // Execute blocks 1, 2, 3. - c.ValidateAndPrepare(common.Hash{}, block1) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.ValidateAndPrepare(block1, block2) - c.Put(kv.AccountsDomain, makeAddr(2), makeValue(2)) - c.ValidateAndPrepare(block2, block3) - c.Put(kv.AccountsDomain, makeAddr(3), makeValue(3)) - - // Unwind to block 1: revert blocks 2-3. - var diffset [kv.DomainLen][]kv.DomainEntryDiff - diffset[kv.AccountsDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(2), 0), Value: nil}, - {Key: makeDiffKey(makeAddr(3), 0), Value: nil}, - } - c.RevertWithDiffset(&diffset, block3, block1) +// A second, shallower unwind must not resurrect entries a deeper earlier unwind +// invalidated (floor only moves down). +func TestUnwind_FloorOnlyMovesDown(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + k := makeAddr(1) + c.Put(k, makeValue(1), 70) // epoch 0 - // Cache should now have blockHash = block1. - // Executing a new block 2 with parent=block1 should preserve surviving cache data. - valid := c.ValidateAndPrepare(block1, block2new) - assert.True(t, valid, "ValidateAndPrepare should succeed after surgical unwind") + c.Unwind(50) // floor 50, epoch 1 — k(70>50, epoch0) now stale + c.Unwind(100) // shallower; floor must stay 50, not rise to 100 - // addr(1) was not in the diffset — should survive both the revert and ValidateAndPrepare. - v, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.True(t, ok, "untouched key should survive revert + ValidateAndPrepare") - assert.Equal(t, makeValue(1), v) + _, ok := c.Get(k) + assert.False(t, ok, "deeper unwind's floor must not be raised by a later shallower one") } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 163172abda3..fc94bbe04c6 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -17,14 +17,12 @@ package cache import ( - "sync" "sync/atomic" "unsafe" "github.com/c2h5oh/datasize" lru "github.com/hashicorp/golang-lru/v2" - "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" ) @@ -111,9 +109,6 @@ type CodeCache struct { codeSizeEntries atomic.Int64 codeSizeCapEntries int64 - blockHash common.Hash // hash of the last block processed - mu sync.RWMutex - // Stats counters (atomic for concurrent access) addrHits atomic.Uint64 addrMisses atomic.Uint64 @@ -196,7 +191,7 @@ func (c *CodeCache) Get(addr []byte) ([]byte, bool) { // Uses fast maphash to compute the code identifier. // addrToHash is an LRU (auto-evicts oldest when full); hashToCode is byte-capped // and no-ops on new writes when full (code is immutable, so existing entries stay). -func (c *CodeCache) Put(addr []byte, code []byte) { +func (c *CodeCache) Put(addr []byte, code []byte, _ uint64) { if len(code) == 0 { return } @@ -272,7 +267,7 @@ func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { } if len(addr) > 0 { - c.Put(addr, code) + c.Put(addr, code, 0) // addr layer is cleared on unwind, not txNum-tracked } // Populate the size-only layer alongside the bytes layer — every time @@ -334,57 +329,18 @@ func (c *CodeCache) Delete(addr []byte) { // The codeHash → code mappings are preserved since they're immutable. func (c *CodeCache) Clear() { c.addrToHash.Purge() + c.addrToEthHash.Purge() } -// GetBlockHash returns the hash of the last block processed by the cache. -func (c *CodeCache) GetBlockHash() common.Hash { - c.mu.RLock() - defer c.mu.RUnlock() - return c.blockHash -} - -// SetBlockHash sets the hash of the current block being processed. -func (c *CodeCache) SetBlockHash(hash common.Hash) { - c.mu.Lock() - c.blockHash = hash - c.mu.Unlock() -} - -// ValidateAndPrepare checks if the given parentHash matches the cache's current blockHash. -// If there's a mismatch (indicating non-sequential block processing), the address cache is cleared. -// The code cache (hash → code) is preserved since code hashes are immutable. -// Returns true if the cache was valid (hashes matched or cache was empty), false if cleared. -func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool { - c.mu.Lock() - defer c.mu.Unlock() - - // Empty blockHash means cache hasn't processed any block yet - always valid - if c.blockHash == (common.Hash{}) { - c.addrToHash.Purge() - c.blockHash = incomingBlockHash - return true - } - - // Check if we're continuing from the expected block - if c.blockHash == parentHash { - c.blockHash = incomingBlockHash - return true - } - - // Mismatch - clear address mappings (they may be stale) - // Keep code mappings since hash → code is immutable - c.addrToHash.Purge() - c.blockHash = incomingBlockHash - return false -} - -// ClearWithHash clears the address cache and sets the block hash. -// Used during unwind to reset the cache to a known state. -func (c *CodeCache) ClearWithHash(hash common.Hash) { - c.mu.Lock() - defer c.mu.Unlock() +// Unwind drops the mutable address → code mappings, which may now reflect a +// dead fork's code for an address. The content-addressed code layers +// (hash → code, ethHash → code, codeSize) are immutable and kept. The addr +// layers are small and clearing them on the rare unwind is cheap, so unlike +// GenericCache they don't need epoch tracking. unwindToTxNum is accepted for +// interface symmetry; the addr layers don't carry per-entry txNums. +func (c *CodeCache) Unwind(_ uint64) { c.addrToHash.Purge() - c.blockHash = hash + c.addrToEthHash.Purge() } // Len returns the number of entries in the address cache. diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index af9e3d61222..0119e855d68 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -18,7 +18,7 @@ package cache import ( "bytes" - "sync" + "math" "sync/atomic" "github.com/c2h5oh/datasize" @@ -43,9 +43,11 @@ const avgBytesPerEntry = 256 // the OnEvict callback can update currentSize without re-running // sizeFunc. type entry[T any] struct { - key []byte - val T - size int + key []byte + val T + size int + txNum uint64 // commit/read txNum the cached value reflects (upper bound) + epoch uint32 // unwind generation the entry was written in } // GenericCache is a sharded, LRU-evicting bounded cache for key-value @@ -56,14 +58,22 @@ type GenericCache[T any] struct { currentSize atomic.Int64 mode Mode - blockHash common.Hash - mu sync.RWMutex - - hits atomic.Uint64 - misses atomic.Uint64 - inserts atomic.Uint64 - evictions atomic.Uint64 - dropped atomic.Uint64 + // Cache coherence across unwinds is txNum/epoch based — no block awareness, + // no diffset. An entry is valid iff it was written in the current epoch OR + // its txNum is at/below unwindFloor (so it predates every unwind seen and + // can't be stale). Unwind bumps the epoch and lowers the floor (O(1), no + // scan); stale entries are dropped lazily on their next read. txNum slots + // are reused across forks, so the epoch — not the txNum — is what tells a + // dead fork's write apart from the live fork's at the same txNum. + epoch atomic.Uint32 + unwindFloor atomic.Uint64 + + hits atomic.Uint64 + misses atomic.Uint64 + inserts atomic.Uint64 + evictions atomic.Uint64 + dropped atomic.Uint64 + staleEvicted atomic.Uint64 // entries dropped lazily on read after an unwind sizeFunc func(T) int } @@ -101,6 +111,9 @@ func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntr mode: mode, sizeFunc: sizeFunc, } + // Before any unwind every entry predates the (nonexistent) floor, so all + // reads are valid; the floor only drops once an unwind happens. + c.unwindFloor.Store(math.MaxUint64) // OnEvict fires for capacity-driven LRU eviction (ModeEvictLRU) and // for explicit Remove(). In both cases we want currentSize to follow. lru.SetOnEvict(func(_ uint64, e entry[T]) { @@ -138,8 +151,8 @@ func (c *DomainCache) Get(key []byte) ([]byte, bool) { } // Put stores data for the given key, implementing the Cache interface. -func (c *DomainCache) Put(key []byte, value []byte) { - c.GenericCache.Put(key, value) +func (c *DomainCache) Put(key []byte, value []byte, txNum uint64) { + c.GenericCache.Put(key, value, txNum) } // Delete removes the data for the given key, delegating to GenericCache. @@ -156,6 +169,16 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { var zero T return zero, false } + // Lazy unwind invalidation: an entry from a superseded epoch whose txNum is + // above the unwind floor reflects dead-fork state — drop it and miss so the + // read falls through to the reverted domain and repopulates. + if e.epoch != c.epoch.Load() && e.txNum > c.unwindFloor.Load() { + c.data.Remove(h) + c.staleEvicted.Add(1) + c.misses.Add(1) + var zero T + return zero, false + } c.hits.Add(1) return e.val, true } @@ -164,17 +187,18 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { // sharded LRU evicts cold entries when its entry-count cap is reached. // In ModeNoOp inserts that would overflow the byte budget are dropped // (and counted via the dropped metric). -func (c *GenericCache[T]) Put(key []byte, value T) { +func (c *GenericCache[T]) Put(key []byte, value T, txNum uint64) { h := maphash.Hash(key) valBytes := c.sizeFunc(value) newSize := len(key) + valBytes + 24 + ep := c.epoch.Load() // Existing key — update in place. Reuse the stored key buffer to // avoid an extra allocation; the freshly-decoded value replaces the // old one. if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { oldSize := existing.size - c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize}) + c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize - oldSize)) return } @@ -191,7 +215,7 @@ func (c *GenericCache[T]) Put(key []byte, value T) { // trade-off code_cache.go / balcache.go / db/state/cache.go accept. keyCopy := common.Copy(key) - c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize}) + c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize)) c.inserts.Add(1) } @@ -210,50 +234,24 @@ func (c *GenericCache[T]) Clear() { c.currentSize.Store(0) } -// GetBlockHash returns the hash of the last block processed by the cache. -func (c *GenericCache[T]) GetBlockHash() common.Hash { - c.mu.RLock() - defer c.mu.RUnlock() - return c.blockHash -} - -// SetBlockHash sets the hash of the current block being processed. -func (c *GenericCache[T]) SetBlockHash(hash common.Hash) { - c.mu.Lock() - c.blockHash = hash - c.mu.Unlock() -} - -// ValidateAndPrepare checks if the given parentHash matches the cache's current blockHash. -func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool { - c.mu.Lock() - defer c.mu.Unlock() - - if c.blockHash == (common.Hash{}) { - c.data.Purge() - c.currentSize.Store(0) - c.blockHash = incomingBlockHash - return true - } - - if c.blockHash == parentHash { - c.blockHash = incomingBlockHash - return true +// Unwind invalidates entries that reflect state above unwindToTxNum on a +// now-dead fork. O(1): bump the epoch (so entries written in the new, live +// epoch stay valid) and lower the floor to the unwind point (so entries at or +// below it — which predate the unwind and can't be stale — stay warm). Stale +// entries (old epoch, txNum above the floor) are dropped lazily on their next +// read. The floor only ever moves down, to the deepest unwind seen, so a later +// shallower unwind can't resurrect entries an earlier deeper one invalidated. +func (c *GenericCache[T]) Unwind(unwindToTxNum uint64) { + c.epoch.Add(1) + for { + cur := c.unwindFloor.Load() + if unwindToTxNum >= cur { + break + } + if c.unwindFloor.CompareAndSwap(cur, unwindToTxNum) { + break + } } - - c.data.Purge() - c.currentSize.Store(0) - c.blockHash = incomingBlockHash - return false -} - -// ClearWithHash clears the cache and sets the block hash. -func (c *GenericCache[T]) ClearWithHash(hash common.Hash) { - c.mu.Lock() - defer c.mu.Unlock() - c.data.Purge() - c.currentSize.Store(0) - c.blockHash = hash } // Len returns the number of entries in the cache. @@ -283,6 +281,7 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { inserts := c.inserts.Swap(0) evictions := c.evictions.Swap(0) dropped := c.dropped.Swap(0) + staleEvicted := c.staleEvicted.Swap(0) total := hits + misses var hitRate float64 if total > 0 { @@ -290,10 +289,11 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { } sizeBytes := c.currentSize.Load() usagePct := float64(sizeBytes) / float64(c.capacityB) * 100 - log.Debug(name+" cache stats", + log.Info(name+" cache stats", "mode", c.mode.String(), "hits", hits, "misses", misses, "hit_rate", hitRate, "inserts", inserts, "evictions", evictions, "dropped", dropped, + "stale_evicted", staleEvicted, "epoch", c.epoch.Load(), "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", c.capacityB/datasize.MB, "usage_pct", usagePct, ) diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index dbf72fb3114..da5a6a53fe7 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -32,8 +32,11 @@ import ( const ( // DefaultAccountCacheBytes is the byte limit for account cache (100 MB — investigation knob; permanent default returns to 1 GB) DefaultAccountCacheBytes = 100 * datasize.MB - // DefaultStorageCacheBytes is the byte limit for storage cache (100 MB — investigation knob; permanent default returns to 1 GB) - DefaultStorageCacheBytes = 100 * datasize.MB + // DefaultStorageCacheBytes is the byte limit for storage cache. 150 MB: the + // measured mainnet-tip storage working set is ~106 MB for 95% of reads + // (top 1% of keys = 48%, ~10 MB for 80%); 150 MB leaves headroom over the + // 95% set so eviction pressure doesn't push the hot set out. + DefaultStorageCacheBytes = 150 * datasize.MB // DefaultCommitmentCacheBytes is the byte limit for commitment cache (128 MB) DefaultCommitmentCacheBytes = 128 * datasize.MB @@ -204,8 +207,9 @@ func (c *StateCache) DeleteAddrCodeHash(addr []byte) { cc.DeleteAddrCodeHash(addr) } -// Put stores data for the given domain and key. -func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { +// Put stores data for the given domain and key, stamped with the txNum the +// value reflects (for txNum/epoch unwind invalidation). +func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte, txNum uint64) { cache := c.caches[domain] if cache == nil { return @@ -213,7 +217,7 @@ func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { if domain == kv.CommitmentDomain && bytes.Equal(key, commitmentdb.KeyCommitmentState) { return } - cache.Put(key, common.Copy(value)) + cache.Put(key, common.Copy(value), txNum) } // Delete removes the data for the given domain and key. @@ -234,25 +238,15 @@ func (c *StateCache) Clear() { } } -// ValidateAndPrepare validates and prepares all caches for a new block. -// Returns true if all caches were valid, false if any were cleared. -func (c *StateCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool { - allValid := true +// Unwind invalidates, across all caches, entries reflecting state above +// unwindToTxNum on a now-dead fork. Diffset-free and O(1) (GenericCache bumps +// an epoch + lowers a floor; CodeCache clears its mutable addr layers). This is +// the sole cache-invalidation path on unwind — the executor never touches the +// cache during forward execution. +func (c *StateCache) Unwind(unwindToTxNum uint64) { for _, cache := range c.caches { if cache != nil { - if !cache.ValidateAndPrepare(parentHash, incomingBlockHash) { - allValid = false - } - } - } - return allValid -} - -// ClearWithHash clears all caches and sets their block hash. -func (c *StateCache) ClearWithHash(hash common.Hash) { - for _, cache := range c.caches { - if cache != nil { - cache.ClearWithHash(hash) + cache.Unwind(unwindToTxNum) } } } @@ -284,44 +278,3 @@ func (c *StateCache) PrintStatsAndReset() { code.PrintStatsAndReset() } } - -func (c *StateCache) RevertWithDiffset(diffset *[6][]kv.DomainEntryDiff, revertFromHash, newBlockHash common.Hash) { - // If the cache's block hash doesn't match the block we're unwinding from, - // the cache was modified by a rolled-back tx (e.g. ValidatePayload). - // Clear everything and set the new hash — surgical eviction can't fix stale data - // from a different execution path. - for _, cache := range c.caches { - if cache != nil && cache.GetBlockHash() != revertFromHash { - c.ClearWithHash(newBlockHash) - return - } - } - - for _, entry := range diffset[kv.AccountsDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.CodeDomain, k) - c.Delete(kv.AccountsDomain, k) - // Unwind may revert a codeHash change (SELFDESTRUCT undone, CREATE - // reverted) — drop the addr → codeHash mapping so the next read - // repopulates from the canonical post-revert account record. - c.DeleteAddrCodeHash(k) - } - for _, entry := range diffset[kv.CodeDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.CodeDomain, k) - } - for _, entry := range diffset[kv.StorageDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.StorageDomain, k) - } - for _, entry := range diffset[kv.CommitmentDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.CommitmentDomain, k) - } - // Update block hash after successful surgical eviction - for _, cache := range c.caches { - if cache != nil { - cache.SetBlockHash(newBlockHash) - } - } -} diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 08c0d47a5c9..57931161f85 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -85,6 +85,7 @@ func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { type cachePopulatingGetter struct { g kv.TemporalGetter sc *cache.StateCache + stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) codeHashHint []byte // valid only for the next CodeDomain read; cleared after use } @@ -100,7 +101,9 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // storage slot, no code) and caching it lets repeated probes // skip the file accessor stack. Mirrors revm's CacheAccount // { account: None, status: LoadedNotExisting } pattern. - cpg.sc.Put(name, k, v) + // Stamp with an upper bound on the value's write txNum (last txNum + // of the step it came from) so unwind invalidation is correct. + cpg.sc.Put(name, k, v, (uint64(step)+1)*cpg.stepSize-1) } } return v, step, err @@ -229,7 +232,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} getter = cpg } stateReader := state.NewReaderV3(getter) @@ -308,7 +311,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} getter = cpg } stateReader := state.NewReaderV3(getter) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 8a46de7e839..f20e02a2125 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -555,11 +555,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, criticalError } - // Clear state cache on invalid block - isInvalid := status == engine_types.InvalidStatus || status == engine_types.InvalidBlockHashStatus || validationError != nil - if e.stateCache != nil && isInvalid { - e.stateCache.ClearWithHash(header.ParentHash) - } + // No cache invalidation needed on an invalid payload: the state cache is + // populated only at flush (committed, fork-agnostic state) and this + // validation path never flushes, so a rejected payload leaves nothing + // fork-specific in the cache. Reads during validation only add canonical + // committed bytes. (Cache invalidation happens solely on unwind.) // Validation tx is the SD's BlockOverlay; defer doms.Close() above handles // its rollback. By design we do not persist validation-run writes — there diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 5d2e668794c..e57af3c0c53 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -600,13 +600,6 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m } go warmTxsHashes(b) - // Bug 2 fix from commit 43a64a0b66 (parallel executor: fix two cache bugs): - // validate stateCache entries against this block's parent hash so stale - // values from a prior fork-validation are cleared before reads. - if stateCache := te.doms.GetStateCache(); stateCache != nil { - stateCache.ValidateAndPrepare(b.ParentHash(), b.Hash()) - } - var dbBAL types.BlockAccessList // Read BAL through blockTx (overlay or execRoTx) — do NOT open // a separate db.View() as it can deadlock with the stageloop's diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 0b5a18f89b8..1eca858a1f2 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -728,6 +728,9 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U func (pe *parallelExecutor) LogExecution() { pe.progress.LogExecution(pe.rs.StateV3, pe) + if stateCache := pe.doms.GetStateCache(); stateCache != nil { + stateCache.PrintStatsAndReset() + } if domainMetrics := pe.domains().LogMetrics(); len(domainMetrics) > 0 { pe.logger.Info(fmt.Sprintf("[%s] domain reads", pe.logPrefix), domainMetrics...) } @@ -1055,17 +1058,10 @@ func (pe *parallelExecutor) execLoop(ctx context.Context) (err error) { } func (pe *parallelExecutor) processRequest(ctx context.Context, execRequest *execRequest) (err error) { - // Validate state cache before processing block - ensures cache is cleared after reorgs - // This matches the behavior in serial execution (exec3_serial.go) - if len(execRequest.tasks) > 0 { - if txTask, ok := execRequest.tasks[0].(*exec.TxTask); ok && txTask.Header != nil { - parentHash := txTask.Header.ParentHash - blockHash := execRequest.blockHash - if stateCache := pe.doms.GetStateCache(); stateCache != nil { - stateCache.ValidateAndPrepare(parentHash, blockHash) - } - } - } + // The state cache is a SharedDomain implementation detail: it is populated + // only at flush (committed, fork-agnostic state) and invalidated only on + // unwind (txNum/epoch — see StateCache.Unwind). The executor does not touch + // it during forward execution. prevSenderTx := map[accounts.Address]int{} var scheduleable *blockExecutor diff --git a/execution/stagedsync/exec3_serial.go b/execution/stagedsync/exec3_serial.go index a2bff285df0..7c53ad1b82a 100644 --- a/execution/stagedsync/exec3_serial.go +++ b/execution/stagedsync/exec3_serial.go @@ -82,7 +82,7 @@ func (se *serialExecutor) exec(ctx context.Context, execStage *StageState, u Unw "initialCycle", initialCycle, "isForkValidation", se.isForkValidation) } - stateCache := se.doms.GetStateCache() + stateCache := se.doms.GetStateCache() // for periodic PrintStatsAndReset only for ; blockNum <= maxBlockNum; blockNum++ { shouldGenerateChangesets := shouldGenerateChangeSets(se.cfg, blockNum, maxBlockNum) @@ -114,10 +114,6 @@ func (se *serialExecutor) exec(ctx context.Context, execStage *StageState, u Unw } go warmTxsHashes(b) - if stateCache != nil { - stateCache.ValidateAndPrepare(b.ParentHash(), b.Hash()) - } - txs := b.Transactions() header := b.HeaderNoCopy() getHashFnMutex := sync.Mutex{} diff --git a/execution/stagedsync/stage_execute.go b/execution/stagedsync/stage_execute.go index dccf76087fa..2b01df83d3e 100644 --- a/execution/stagedsync/stage_execute.go +++ b/execution/stagedsync/stage_execute.go @@ -212,8 +212,7 @@ func unwindExec3(u *UnwindState, s *StageState, doms *execctx.SharedDomains, rwT } } } - // Get the hash of the last executed block (the tip we're unwinding from) - // so RevertWithDiffset can detect if the cache was modified by a rolled-back tx. + // Get the hash of the last executed block (the tip we're unwinding from). lastExecHash, _, err := br.CanonicalHash(ctx, rwTx, u.CurrentBlockNumber) if err != nil { lastExecHash = common.Hash{} @@ -221,14 +220,12 @@ func unwindExec3(u *UnwindState, s *StageState, doms *execctx.SharedDomains, rwT if err := unwindExec3State(ctx, doms, rwTx, u.UnwindPoint, txNum, accumulator, changeSet, lastExecHash, logger); err != nil { return fmt.Errorf("unwindExec3State(%d->%d): %w, took %s", s.BlockNumber, u.UnwindPoint, err, time.Since(t)) } - // Surgically evict keys touched by the unwound blocks from the state cache. - // Keys not in the diffset remain cached (they weren't modified by the unwound range). - if stateCache := doms.GetStateCache(); stateCache != nil && changeSet != nil { - unwindTargetHash, _, err := br.CanonicalHash(ctx, rwTx, u.UnwindPoint) - if err != nil { - unwindTargetHash = common.Hash{} - } - stateCache.RevertWithDiffset(changeSet, lastExecHash, unwindTargetHash) + // Invalidate the state cache for everything above the unwind point. txNum/epoch + // based and diffset-free (see StateCache.Unwind), so it runs unconditionally — + // independent of whether changesets were generated for the unwound range, which + // they are not below the reorg window. Matches the domain overlay's maxtx prune. + if stateCache := doms.GetStateCache(); stateCache != nil { + stateCache.Unwind(txNum) } if err := rawdb.DeleteNewerEpochs(rwTx, u.UnwindPoint+1); err != nil { return fmt.Errorf("delete newer epochs: %w", err) diff --git a/execution/vm/contract.go b/execution/vm/contract.go index da7e2cb9c3b..b80bb53e912 100644 --- a/execution/vm/contract.go +++ b/execution/vm/contract.go @@ -105,7 +105,8 @@ func (c *Contract) isCode(udest uint64) bool { c.analysis = codeBitmap(c.Code) if !isCodeHashZero { - jumpDestCache.Put(codeHash[:], c.analysis) + // content-addressed by codeHash and never unwound, so txNum is irrelevant + jumpDestCache.Put(codeHash[:], c.analysis, 0) } return c.analysis.codeSegment(udest) From 42a2aafd32458127e2f16f1adece8f041225ded7 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 5 Jun 2026 18:01:58 +0000 Subject: [PATCH 109/167] execution: wire state cache into SetHead unwind (fixes BadBlock on re-exec) SetHead (debug_setHead) built its unwind SharedDomains via NewSharedDomains but never called SetStateCache, so unwindExec3's doms.GetStateCache() returned nil and stateCache.Unwind(txNum) was skipped. The shared singleton cache kept pre-unwind values at the old epoch; the next UpdateForkChoice re-execution read them and computed a stale state root, returning ExecutionStatusBadBlock. Wire the shared cache into the SetHead SD, mirroring ValidateChain and the forkchoice path, so the epoch bump + floor lowering runs. TestSetHeadCanonicalCleanup covers this (failed cache-on, passes now). Co-Authored-By: Claude Opus 4.8 (1M context) --- execution/execmodule/set_head.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 0623b5bcaa6..75aef16ff7a 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -103,6 +103,12 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { } defer sd.Close() + // Wire the shared state cache so unwindExec3 invalidates it (epoch bump + + // floor lower). Without this, GetStateCache() returns nil during the unwind, + // the cache keeps pre-unwind values, and the next FCU re-execution reads them + // and computes a stale state root (BadBlock). Mirrors ValidateChain/forkchoice. + sd.SetStateCache(e.stateCache) + // Set the unwind point and run the unwind if err := e.pipelineExecutor.UnwindTo(targetBlock, stagedsync.StagedUnwind, tx); err != nil { return fmt.Errorf("failed to set unwind point: %w", err) From 26838656780069d46aecb73a6d7ed8b690c28dd5 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 5 Jun 2026 18:38:31 +0000 Subject: [PATCH 110/167] commitment: LatestStateReader.Clone keeps its source tx, not the compute tx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ComputeCommitment clones a custom state reader with its own compute tx (commitment_context.go trieContext). When that reader is a LatestStateReader bound to a different database (e.g. recomputing commitment in an empty db while reading committed state from the source db, as TouchChangedKeysFromHistory does), Clone rebuilt sd.AsGetter against the foreign compute tx and read the wrong database — returning empty account/storage and thus a wrong (empty-leaf) root. This was masked until FlushWithCallback began draining sd.mem on flush: the in-memory batch previously still held the source values. Store the reader's source tx and rebind Clone to it. The default commitment path never routes through LatestStateReader.Clone (it constructs fresh readers), and the production custom readers build their own sub-readers, so this only affects readers explicitly bound to a foreign source. Co-Authored-By: Claude Opus 4.8 (1M context) --- execution/commitment/commitmentdb/reader.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/execution/commitment/commitmentdb/reader.go b/execution/commitment/commitmentdb/reader.go index d2950323f53..805931727db 100644 --- a/execution/commitment/commitmentdb/reader.go +++ b/execution/commitment/commitmentdb/reader.go @@ -16,10 +16,11 @@ type StateReader interface { type LatestStateReader struct { sharedDomains sd getter kv.TemporalGetter + srcTx kv.TemporalTx } func NewLatestStateReader(tx kv.TemporalTx, sd sd) *LatestStateReader { - return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx)} + return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx), srcTx: tx} } func (r *LatestStateReader) WithHistory() bool { @@ -43,7 +44,14 @@ func (r *LatestStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) } func (r *LatestStateReader) Clone(tx kv.TemporalTx) StateReader { - return NewLatestStateReader(tx, r.sharedDomains) + // Keep reading the source this reader was bound to. The tx passed by + // clone/warmup callers targets the *compute* database, which may differ + // from this reader's source — e.g. recomputing commitment in an empty db + // while reading committed state from the source db (TouchChangedKeysFromHistory). + // Before flush drained sd.mem this was masked because the in-memory batch + // still held the source values; rebinding sd.AsGetter to the foreign compute + // tx reads the wrong database and yields empty state (wrong root). + return NewLatestStateReader(r.srcTx, r.sharedDomains) } // HistoryStateReader reads *full* historical state at specified txNum. From 44355fd1fba362c60da48a087142ad51e168d797 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 19:45:10 +0000 Subject: [PATCH 111/167] db/state, commitment: lock-free per-worker read metrics (+ tx-aware BranchCache) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the DomainMetrics write lock from the hot state-read path. Every GetLatest previously took DomainMetrics.Lock() — a write lock with no fast path — to bump read counters; under concurrent trie-warmup / parallel-exec workers that was a hard serialization point, so KV read metrics had to be left off. Replace it with a lock-free per-worker accumulator (changeset.WorkerMetrics) that each goroutine owns exclusively, combined into the shared DomainMetrics once per task via DomainMetrics.Merge — no lock, no atomics, no shared cache line on the hot path. The accumulator is carried via context (const-int ctx key, matching node/app) and read through getter.GetLatestContext / SharedDomains.GetLatestContext (mirroring the existing optional MeteredGetLatest). Commitment/warmup worker readers carry a per-worker ctx and merge at teardown; the main goroutine meters into sd.mainWM, flushed into the global on metrics read. Parallel-exec state readers use AsGetterNoMetrics for now (no metering); wiring their per-worker metrics is a follow-up on the state-read path. This also removes a data race those concurrent workers would otherwise hit on a shared accumulator once metrics are enabled. Also carries the tx-aware BranchCache (txNum/epoch unwind invalidation + maxStep-bounded reads) this sits on top of. Verified: build, make lint (0 issues), and -race with KV_READ_METRICS=true clean across commitmentdb / changeset / execmodule parallel block-assembly. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/aggregator.go | 4 +- db/state/changeset/state_changeset.go | 282 ++++++++++++------ db/state/domain.go | 2 +- db/state/execctx/domain_shared.go | 159 ++++++++-- execution/commitment/branch_cache.go | 94 +++++- .../commitmentdb/commitment_context.go | 26 +- .../commitmentdb/commitment_context_test.go | 3 + execution/commitment/commitmentdb/reader.go | 85 +++++- execution/commitment/preload.go | 6 +- execution/exec/state.go | 10 +- execution/execmodule/forkchoice.go | 2 + execution/stagedsync/committer.go | 19 +- execution/stagedsync/exec3_parallel.go | 4 +- rpc/jsonrpc/eth_simulation.go | 7 + 14 files changed, 555 insertions(+), 148 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 387e4f4a906..c83c8568dbd 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2478,11 +2478,11 @@ func (at *AggregatorRoTx) GetLatest(domain kv.Domain, k []byte, tx kv.Tx) (v []b return at.getLatest(domain, k, tx, math.MaxUint64, nil, time.Time{}) } -func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { return at.getLatest(domain, k, tx, maxStep, metrics, start) } -func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { if domain != kv.CommitmentDomain { return at.d[domain].getLatest(k, tx, maxStep, metrics, start) } diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 0e30f687897..60bbd70a3d4 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -17,6 +17,7 @@ package changeset import ( + "context" "encoding/binary" "fmt" "math" @@ -514,79 +515,189 @@ type DomainMetrics struct { seenFileReads sync.Map } -func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.CacheReadCount++ - readDuration := time.Since(start) - dm.CacheReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.CacheReadCount++ - d.CacheReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - CacheReadCount: 1, - CacheReadDuration: readDuration, - } +// WorkerMetrics is a per-worker, lock-free I/O metrics accumulator. Each +// goroutine on the hot read path (the main commitment/exec goroutine and each +// trie-warmup worker) owns one exclusively, so updates need no synchronization +// — no mutex, no atomics, no shared cache line. The per-worker accumulators are +// combined into the shared DomainMetrics once per task via DomainMetrics.Merge. +// +// This replaces the previous design where every read called a DomainMetrics +// method that took the global write lock (dm.Lock()); under concurrent warmup +// workers that write lock was a hard serialization point on the hottest path. +type WorkerMetrics struct { + total DomainIOMetrics + domains [kv.DomainLen]DomainIOMetrics + + // uniqueSeen records the first local sighting of each (domain,prefix) that + // fell through to a file read, so cross-worker uniqueness can be resolved + // once at Merge time (against DomainMetrics.seenFileReads) instead of + // touching a shared structure on every read. Value is the length bucket. + // Nil until the first UpdateFileReadsUnique call (workers that never read + // from files pay nothing). + uniqueSeen map[uniqueKey]int +} + +type uniqueKey struct { + domain kv.Domain + prefix string +} + +// NewWorkerMetrics returns a fresh lock-free accumulator. A nil *WorkerMetrics +// is valid and makes every Update* a no-op, so callers that don't collect +// metrics can pass nil. +func NewWorkerMetrics() *WorkerMetrics { return &WorkerMetrics{} } + +// ctxKey is the typed-int context-key namespace for this package (the const-int +// model used by node/app), avoiding per-key struct types. +type ctxKey int + +const ( + ckWorkerMetrics ctxKey = iota +) + +// ContextWithWorkerMetrics returns a child context carrying a per-worker, +// lock-free metrics accumulator. Concurrent workers (e.g. trie-warmup +// goroutines) each run with their own, so reads never contend on a shared +// metrics structure (and take no lock). +func ContextWithWorkerMetrics(ctx context.Context, wm *WorkerMetrics) context.Context { + return context.WithValue(ctx, ckWorkerMetrics, wm) +} + +// WorkerMetricsFromContext extracts the per-worker accumulator, or nil if none +// was attached (in which case reads collect no metrics — a valid no-op). +func WorkerMetricsFromContext(ctx context.Context) *WorkerMetrics { + if ctx == nil { + return nil } + wm, _ := ctx.Value(ckWorkerMetrics).(*WorkerMetrics) + return wm } -func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.DbReadCount++ - readDuration := time.Since(start) - dm.DbReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.DbReadCount++ - d.DbReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - DbReadCount: 1, - DbReadDuration: readDuration, - } +// Reset zeroes the accumulator so it can be reused after its counts have been +// merged into the shared DomainMetrics. Used for the long-lived main-goroutine +// accumulator, which is merged-and-reset on each metrics read. +func (w *WorkerMetrics) Reset() { + if w == nil { + return } + w.total = DomainIOMetrics{} + for i := range w.domains { + w.domains[i] = DomainIOMetrics{} + } + w.uniqueSeen = nil } -func (dm *DomainMetrics) UpdateStateCacheHit(domain kv.Domain) { - dm.Lock() - defer dm.Unlock() - dm.StateCacheHitCount++ - if d, ok := dm.Domains[domain]; ok { - d.StateCacheHitCount++ - } else { - dm.Domains[domain] = &DomainIOMetrics{StateCacheHitCount: 1} +func (w *WorkerMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { + if w == nil { + return } + d := time.Since(start) + w.total.CacheReadCount++ + w.total.CacheReadDuration += d + w.domains[domain].CacheReadCount++ + w.domains[domain].CacheReadDuration += d } -func (dm *DomainMetrics) UpdateStateCacheMiss(domain kv.Domain) { - dm.Lock() - defer dm.Unlock() - dm.StateCacheMissCount++ - if d, ok := dm.Domains[domain]; ok { - d.StateCacheMissCount++ - } else { - dm.Domains[domain] = &DomainIOMetrics{StateCacheMissCount: 1} +func (w *WorkerMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { + if w == nil { + return + } + d := time.Since(start) + w.total.DbReadCount++ + w.total.DbReadDuration += d + w.domains[domain].DbReadCount++ + w.domains[domain].DbReadDuration += d +} + +func (w *WorkerMetrics) UpdateStateCacheHit(domain kv.Domain) { + if w == nil { + return } + w.total.StateCacheHitCount++ + w.domains[domain].StateCacheHitCount++ } -func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { +func (w *WorkerMetrics) UpdateStateCacheMiss(domain kv.Domain) { + if w == nil { + return + } + w.total.StateCacheMissCount++ + w.domains[domain].StateCacheMissCount++ +} + +func (w *WorkerMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { + if w == nil { + return + } + d := time.Since(start) + w.total.FileReadCount++ + w.total.FileReadDuration += d + w.domains[domain].FileReadCount++ + w.domains[domain].FileReadDuration += d +} + +// Merge folds a worker's accumulated counters into the shared DomainMetrics +// under a single lock acquisition (once per task, not once per read). Unique +// file-read prefixes are deduplicated across workers here, against the global +// seenFileReads set, so the cross-worker uniqueness count stays correct without +// any shared structure on the hot path. +func (dm *DomainMetrics) Merge(w *WorkerMetrics) { + if w == nil { + return + } dm.Lock() defer dm.Unlock() - dm.FileReadCount++ - readDuration := time.Since(start) - dm.FileReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.FileReadCount++ - d.FileReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - FileReadCount: 1, - FileReadDuration: readDuration, + addIOMetrics(&dm.DomainIOMetrics, &w.total) + for d := range w.domains { + src := &w.domains[d] + if src.isZero() { + continue + } + dst, ok := dm.Domains[kv.Domain(d)] + if !ok { + dst = &DomainIOMetrics{} + dm.Domains[kv.Domain(d)] = dst + } + addIOMetrics(dst, src) + } + for uk, bucket := range w.uniqueSeen { + domainKey := uk.domain.String() + ":" + uk.prefix + if _, already := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}); already { + continue + } + dm.UniqueFileReadCount++ + dm.UniqueLenBuckets[bucket]++ + if dst, ok := dm.Domains[uk.domain]; ok { + dst.UniqueFileReadCount++ + dst.UniqueLenBuckets[bucket]++ + } else { + nd := &DomainIOMetrics{UniqueFileReadCount: 1} + nd.UniqueLenBuckets[bucket] = 1 + dm.Domains[uk.domain] = nd } } } +// addIOMetrics adds src's cumulative read-side counters into dst. CachePut* +// fields are intentionally excluded: those track mem-batch buffer occupancy +// (functional state, reset on flush), not cumulative read metrics, and are +// accounted separately off this path. +func addIOMetrics(dst, src *DomainIOMetrics) { + dst.CacheReadCount += src.CacheReadCount + dst.CacheReadDuration += src.CacheReadDuration + dst.DbReadCount += src.DbReadCount + dst.DbReadDuration += src.DbReadDuration + dst.FileReadCount += src.FileReadCount + dst.FileReadDuration += src.FileReadDuration + dst.StateCacheHitCount += src.StateCacheHitCount + dst.StateCacheMissCount += src.StateCacheMissCount +} + +func (m *DomainIOMetrics) isZero() bool { + return m.CacheReadCount == 0 && m.DbReadCount == 0 && m.FileReadCount == 0 && + m.StateCacheHitCount == 0 && m.StateCacheMissCount == 0 +} + // lenBucket maps a prefix byte-length to its UniqueLenBuckets index. // See DomainIOMetrics.UniqueLenBuckets for the bucket layout. Buckets // 5-8 sub-divide the storage subtree (depths 64-127) so per-contract @@ -616,45 +727,26 @@ func lenBucket(n int) int { } } -// UpdateFileReadsUnique records a file read while also tracking whether -// the prefix has been seen before for this domain. Same gate as -// UpdateFileReads (dbg.KVReadLevelledMetrics); call this instead when -// the read-amplification ratio (FileReadCount / UniqueFileReadCount) -// is wanted on the metric output. The key bytes are copied into the -// internal dedup map; do not mutate after the call. -func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { - // Composite ":" so two domains can hold the same prefix - // shape (commitment compact-encoded paths vs accounts plain addresses) - // without colliding. - domainKey := domain.String() + ":" + string(key) - _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) - bucket := lenBucket(len(key)) - - dm.Lock() - defer dm.Unlock() - dm.FileReadCount++ - readDuration := time.Since(start) - dm.FileReadDuration += readDuration - if !alreadySeen { - dm.UniqueFileReadCount++ - dm.UniqueLenBuckets[bucket]++ - } - if d, ok := dm.Domains[domain]; ok { - d.FileReadCount++ - d.FileReadDuration += readDuration - if !alreadySeen { - d.UniqueFileReadCount++ - d.UniqueLenBuckets[bucket]++ - } - } else { - newD := &DomainIOMetrics{ - FileReadCount: 1, - FileReadDuration: readDuration, - } - if !alreadySeen { - newD.UniqueFileReadCount = 1 - newD.UniqueLenBuckets[bucket] = 1 - } - dm.Domains[domain] = newD +// UpdateFileReadsUnique records a file read in the worker-local accumulator, +// remembering the (domain,prefix) first-sighting so cross-worker uniqueness can +// be resolved at Merge time. Lock-free: the key bytes are copied into a +// per-worker map; nothing shared is touched on the hot path. The key bytes are +// copied; do not mutate after the call. +func (w *WorkerMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { + if w == nil { + return + } + d := time.Since(start) + w.total.FileReadCount++ + w.total.FileReadDuration += d + w.domains[domain].FileReadCount++ + w.domains[domain].FileReadDuration += d + + if w.uniqueSeen == nil { + w.uniqueSeen = make(map[uniqueKey]int) + } + uk := uniqueKey{domain: domain, prefix: string(key)} + if _, ok := w.uniqueSeen[uk]; !ok { + w.uniqueSeen[uk] = lenBucket(len(key)) } } diff --git a/db/state/domain.go b/db/state/domain.go index d9158f5ce42..e512287659e 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -1611,7 +1611,7 @@ func (dt *DomainRoTx) GetLatest(key []byte, roTx kv.Tx) ([]byte, kv.Step, bool, return dt.getLatest(key, roTx, math.MaxInt64, nil, time.Time{}) } -func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) ([]byte, kv.Step, bool, error) { +func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) ([]byte, kv.Step, bool, error) { if dt.d.Disable { return nil, 0, false, nil } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a184e56219c..073ecf3b0d9 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "math" + "os" "runtime" "strings" "sync" @@ -108,6 +109,11 @@ type SharedDomains struct { disableInlineTouchKey bool mem kv.TemporalMemBatch metrics changeset.DomainMetrics + // mainWM is the lock-free metrics accumulator for the main goroutine's + // reads (direct GetLatest + the main commitment/exec getter). Trie-warmup + // workers get their own via AsGetterWorker; all are combined into metrics + // at task boundaries. Keeps the global metrics write-lock off the hot path. + mainWM *changeset.WorkerMetrics // blockOverlay is an in-memory overlay for block-level metadata writes (headers, bodies, // canonical hashes, TD, stage progress, forkchoice markers). It allows execution to @@ -197,6 +203,7 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg logger: logger, //trace: true, metrics: changeset.DomainMetrics{Domains: map[kv.Domain]*changeset.DomainIOMetrics{}}, + mainWM: changeset.NewWorkerMetrics(), stepSize: tx.Debug().StepSize(), } @@ -433,10 +440,26 @@ func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k type temporalGetter struct { sd *SharedDomains tx kv.TemporalTx + // wm is the accumulator GetLatest records into: sd.mainWM for the main + // goroutine (AsGetter), or nil to collect nothing (AsGetterNoMetrics) for + // concurrent callers whose per-worker metering isn't wired yet (parallel + // exec workers) — they must not write the shared main accumulator (a race). + wm *changeset.WorkerMetrics } func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.GetLatest(name, gt.tx, k) + return gt.sd.getLatestMetered(name, gt.tx, k, gt.wm) +} + +// GetLatestContext is the context-aware read: it records into the per-worker, +// lock-free accumulator carried by ctx (a nil ctx-value collects no metrics). +// Concurrent workers (trie-warmup goroutines) pass their own accumulator via +// ctx, so they neither share metrics state with the main goroutine nor take any +// lock. Optional method — callers type-assert for it (mirrors the existing +// AggregatorRoTx.MeteredGetLatest pattern). The plan is to migrate GetLatest +// callers onto this ctx-carrying form in a follow-up, then drop the split. +func (gt *temporalGetter) GetLatestContext(ctx context.Context, name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { + return gt.sd.getLatestMetered(name, gt.tx, k, changeset.WorkerMetricsFromContext(ctx)) } // GetCodeSize returns the length of the code at addr without loading the @@ -472,7 +495,22 @@ func (up *unmarkedPutter) Put(num kv.Num, v []byte) error { } func (sd *SharedDomains) AsGetter(tx kv.TemporalTx) kv.TemporalGetter { - return &temporalGetter{sd, tx} + return &temporalGetter{sd: sd, tx: tx, wm: sd.mainWM} +} + +// AsGetterNoMetrics returns a getter whose GetLatest collects no metrics. Use +// for concurrent callers that share this SD but don't yet have per-worker +// metrics wiring (parallel exec workers): metering them into sd.mainWM would be +// a data race. Their per-worker metering is a follow-up (the state-read path). +func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { + return &temporalGetter{sd: sd, tx: tx, wm: nil} +} + +// MergeWorkerMetrics folds a worker's lock-free accumulator into the shared +// DomainMetrics under a single lock. Called once when a worker finishes (not +// per read), so the global metrics write-lock stays off the hot path. +func (sd *SharedDomains) MergeWorkerMetrics(wm *changeset.WorkerMetrics) { + sd.metrics.Merge(wm) } // LockChangesetAccumulator and UnlockChangesetAccumulator bracket a @@ -580,14 +618,21 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) - // After unwind the canonical commitment-domain values are rolled - // back; any cached entries for keys touched in the unwind window - // now hold stale bytes vs the post-unwind canonical state. Drop - // those specifically — the rest of the cache (including pinned - // branches whose keys weren't in the changeset) stays warm. - if sd.branchCache != nil && changeset != nil { - for _, diff := range changeset[kv.CommitmentDomain] { - sd.branchCache.Invalidate([]byte(diff.Key)) + // Tx-aware unwind of the commitment BranchCache: every cached branch + // whose bytes belong to the rolled-back window (txNum above the unwind + // point, current epoch) is now stale vs the post-unwind canonical state. + // The changeset-gated Invalidate below only covers keys the unwound + // blocks explicitly wrote — it misses entries seeded by the read-pop and + // the trunk preload, which is what left stale committed branches that a + // fork-validation then read as a wrong trie root. Unwind(txNum) drops all + // of them (lazily, by epoch+floor) while keeping at/below-floor entries + // — including frozen preloads — warm. + if sd.branchCache != nil { + sd.branchCache.Unwind(txNumUnwindTo) + if changeset != nil { + for _, diff := range changeset[kv.CommitmentDomain] { + sd.branchCache.Invalidate([]byte(diff.Key)) + } } } } @@ -796,7 +841,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { if len(v) == 0 { sd.branchCache.Invalidate(k) } else { - sd.branchCache.Put(k, v, txNum/sd.StepSize(), "sd.Flush") + sd.branchCache.Put(k, v, txNum, "sd.Flush") } } case kv.AccountsDomain: @@ -918,8 +963,39 @@ func (sd *SharedDomains) ClearBranchCache() { } } -// TemporalDomain satisfaction +// DetachBranchCache makes this SharedDomains ignore the aggregator-scope +// BranchCache: commitment branch reads go straight to sd.mem/overlay/MDBX and +// no read populates the shared cache. Used for fork-validation SDs, which read +// transient fork state — sharing the canonical BranchCache let them read stale +// committed branches (wrong trie root → INVALID payload) and pollute the cache +// with fork-transient branches. Only sd.branchCache (the read/populate path, +// domain_shared GetLatest) is consulted, so nil-ing it fully detaches; the +// canonical SDs keep their warm cache. +func (sd *SharedDomains) DetachBranchCache() { + sd.branchCache = nil +} + +// TemporalDomain satisfaction. Reads on behalf of the main goroutine, recording +// metrics into the shared lock-free main accumulator. func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { + return sd.getLatestMetered(domain, tx, k, sd.mainWM) +} + +// GetLatestContext is the context-aware read for callers that read on behalf of +// a concurrent worker: metrics go to the per-worker, lock-free accumulator +// carried by ctx (nil ctx-value => no metrics). Lets a worker's reader meter +// without touching sd.mainWM (a race) or any lock. Mirrors +// temporalGetter.GetLatestContext for readers that hold the SD directly (e.g. +// the committer's asOfStateReader). +func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { + return sd.getLatestMetered(domain, tx, k, changeset.WorkerMetricsFromContext(ctx)) +} + +// getLatestMetered is the read implementation. wm is the caller's lock-free +// metrics accumulator (the main goroutine's sd.mainWM, or a warmup worker's +// private one); a nil wm disables metrics for the call. No global metrics lock +// is taken on this hot path — accumulators are combined later via Merge. +func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.WorkerMetrics) (v []byte, step kv.Step, err error) { if tx == nil { return nil, 0, errors.New("sd.GetLatest: unexpected nil tx") } @@ -934,7 +1010,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // so the value is already accessible without caching it again. if v, step, ok := sd.mem.GetLatest(domain, k); ok { if dbg.KVReadLevelledMetrics { - sd.metrics.UpdateCacheReads(domain, start) + wm.UpdateCacheReads(domain, start) } return v, step, nil } else { @@ -947,7 +1023,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) if sd.parent != nil { if v, step, ok := sd.parent.mem.GetLatest(domain, k); ok { if dbg.KVReadLevelledMetrics { - sd.metrics.UpdateCacheReads(domain, start) + wm.UpdateCacheReads(domain, start) } return v, step, nil } else { @@ -958,7 +1034,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } type MeteredGetter interface { - MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) + MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } // stateCache holds in-flight values from previous transactions in the same batch @@ -967,9 +1043,9 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) v, ok := sd.stateCache.Get(domain, k) if dbg.KVReadLevelledMetrics { if ok { - sd.metrics.UpdateStateCacheHit(domain) + wm.UpdateStateCacheHit(domain) } else { - sd.metrics.UpdateStateCacheMiss(domain) + wm.UpdateStateCacheMiss(domain) } } if ok { @@ -979,7 +1055,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // backing tx is the single source of truth for this key at this point. var vDB []byte if aggTx, okAgg := tx.AggTx().(MeteredGetter); okAgg { - vDB, _, _, _ = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) + vDB, _, _, _ = aggTx.MeteredGetLatest(domain, k, tx, maxStep, wm, start) } else { vDB, _, _ = tx.GetLatest(domain, k) } @@ -998,8 +1074,29 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // equivalent to reading from MDBX. Cleared by sd.Flush so a new MDBX // state never coexists with stale cache entries. if domain == kv.CommitmentDomain && sd.branchCache != nil { - if cv, cstep, ok := sd.branchCache.Get(k); ok { - return cv, kv.Step(cstep), nil + if cv, cTxNum, ok := sd.branchCache.Get(k); ok { + // Respect maxStep. sd.mem / sd.parent.mem lowered maxStep above when an + // unwound key reports its restored step via unwindChangeset (the per-key + // in-mem unwind signal). The cache's global epoch/floor is coarser than + // that per-key signal, so a cached entry below the floor can still belong + // to a step the unwind re-bound away. Serving it then diverges from the + // (maxStep-bounded) DB read the cache-disabled path takes. Fall through to + // the bounded read in that case; the Put below refreshes the entry. + cStep := kv.Step(cTxNum / sd.StepSize()) + if cStep <= maxStep { + if dbg.EnvBool("DBG_BC", false) { + var vDB []byte + if aggTx, okAgg := tx.AggTx().(MeteredGetter); okAgg { + vDB, _, _, _ = aggTx.MeteredGetLatest(domain, k, tx, maxStep, wm, start) + } else { + vDB, _, _ = tx.GetLatest(domain, k) + } + if !bytes.Equal(cv, vDB) { + fmt.Fprintf(os.Stderr, "[BC-DIVERGE] key=%x cStep=%d maxStep=%d cTxNum=%d sdTxNum=%d cached=%x db=%x\n", k, cStep, maxStep, cTxNum, sd.txNum, cv, vDB) + } + } + return cv, cStep, nil + } } } @@ -1021,7 +1118,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } if aggTx, ok := tx.AggTx().(MeteredGetter); ok { - v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) + v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, wm, start) } else { v, step, err = tx.GetLatest(domain, k) } @@ -1044,7 +1141,10 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { - sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") + // Stamp the upper-bound txNum of the step the bytes came from, so an + // unwind into a recent step drops this entry (frozen-step reads, with + // a low txNum, stay warm). Mirrors the stateCache read-pop above. + sd.branchCache.Put(k, v, (uint64(step)+1)*sd.StepSize()-1, "sd.GetLatest") } return v, step, nil @@ -1178,13 +1278,28 @@ func decodeAccountCodeHash(enc []byte) []byte { return h[:] } +// flushMainWM folds the main goroutine's lock-free accumulator into the shared +// DomainMetrics and resets it, so a subsequent read of sd.metrics reflects the +// main goroutine's reads. Must be called from the main goroutine (the sole +// writer of mainWM). Warmup workers merge their own accumulators independently +// via MergeWorkerMetrics; both serialize on sd.metrics' lock. +func (sd *SharedDomains) flushMainWM() { + if sd.mainWM == nil { + return + } + sd.metrics.Merge(sd.mainWM) + sd.mainWM.Reset() +} + func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { + sd.flushMainWM() return &sd.metrics } func (sd *SharedDomains) LogMetrics() []any { var metrics []any + sd.flushMainWM() sd.metrics.RLock() defer sd.metrics.RUnlock() diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 9edf294d242..302914e6cee 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -19,6 +19,8 @@ package commitment import ( "bytes" "fmt" + "math" + "os" "sync" "sync/atomic" "time" @@ -225,6 +227,18 @@ type BranchCache struct { // exactly once per cache lifetime, even though many SharedDomains // instances may be created over a process's lifetime. preloadClaimed atomic.Bool + + // Tx-aware unwind coherence — same discipline as the state-cache + // GenericCache. Each entry carries the txNum its bytes are valid as of + // and the epoch it was written in. Unwind(txNum) bumps the epoch and + // lowers unwindFloor; an entry is valid iff it was written in the + // current epoch OR its txNum is at/below the floor. Without this the + // cache cannot be unwound: a reorg leaves recent canonical branches + // that a later fork-validation reads as a wrong trie root. Step + // granularity is too coarse here — an unwind to a txNum inside the + // latest step needs txNum precision. + epoch atomic.Uint32 + unwindFloor atomic.Uint64 } // TryClaimPreload returns true the first time it's called on a given @@ -264,12 +278,18 @@ type branchCacheEntry struct { writeSeq uint64 writeTimeNanos int64 - // step is the on-disk file step the cached bytes came from. Returned - // by Get so callers (e.g. CheckDataAvailable) can validate against - // the latest visible step. 0 means "step not tracked" — fine for - // in-memory tests but real callers should always pass the step - // returned by aggTx.MeteredGetLatest / tx.GetLatest. - step uint64 + // txNum is an upper bound on the txNum the cached bytes are valid as of + // (the last txNum of the file/step they came from). Used to gate reads + // after an unwind: an entry whose txNum is above the unwind floor and + // whose epoch is superseded is stale. 0 means frozen/untracked — always + // at/below any unwind floor, so kept (correct for frozen-file preloads, + // which can never be unwound into). Real callers pass the value's txNum. + txNum uint64 + + // epoch is the unwind generation the entry was written in. Disambiguates + // a txNum reused across forks: an entry from a superseded epoch with a + // txNum above the floor is dropped lazily on its next Get. + epoch uint32 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -300,10 +320,14 @@ func NewBranchCache(tailCapacity int) *BranchCache { if err != nil { panic(fmt.Sprintf("BranchCache: NewLRU: %s", err)) } - return &BranchCache{ + bc := &BranchCache{ tail: tail, pinned: maphash.NewMap[*branchCacheEntry](), } + // No unwind seen yet: every entry's txNum is at/below the floor, so the + // epoch check never strands a valid entry. + bc.unwindFloor.Store(math.MaxUint64) + return bc } // isRootPrefix reports whether prefix targets the pinned root slot. The @@ -372,7 +396,7 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // eager preload of hot prefixes — e.g. the storage-trunk of big // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate // the input after the call. -func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { +func (c *BranchCache) PinEntry(prefix []byte, data []byte, txNum uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -380,7 +404,8 @@ func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin s copy(dataCopy, data) c.pinned.Set(prefix, &branchCacheEntry{ data: dataCopy, - step: step, + txNum: txNum, + epoch: c.epoch.Load(), writeSeq: c.writeSeq.Add(1), origin: origin, writeTimeNanos: time.Now().UnixNano(), @@ -440,8 +465,47 @@ func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { if !ok { return nil, 0, false } + // Tx-aware unwind invalidation: an entry from a superseded epoch whose + // txNum is above the unwind floor is stale (its bytes belong to a chain + // segment that was unwound). Drop it lazily on read. Entries at/below + // the floor (incl. frozen-file preloads stamped txNum 0) stay warm. + if entry.epoch != c.epoch.Load() && entry.txNum > c.unwindFloor.Load() { + c.Invalidate(prefix) + if dbgBC { + fmt.Fprintf(os.Stderr, "[BC-EVICT] prefix=%x txNum=%d floor=%d eEpoch=%d cur=%d\n", prefix, entry.txNum, c.unwindFloor.Load(), entry.epoch, c.epoch.Load()) + } + return nil, 0, false + } + if dbgBC && entry.epoch != c.epoch.Load() { + fmt.Fprintf(os.Stderr, "[BC-SERVE-OLD] prefix=%x txNum=%d floor=%d eEpoch=%d cur=%d\n", prefix, entry.txNum, c.unwindFloor.Load(), entry.epoch, c.epoch.Load()) + } c.bytesServed.Add(uint64(len(entry.data))) - return entry.data, entry.step, true + return entry.data, entry.txNum, true +} + +var dbgBC = os.Getenv("DBG_BC") != "" + +// Unwind invalidates cache entries whose bytes belong to a chain segment +// above unwindToTxNum. O(1): bump the epoch (so entries written in the new, +// live epoch stay valid) and lower the floor to the unwind point (so entries +// at/below it survive); stale entries (superseded epoch, txNum above the +// floor) are dropped lazily on their next Get. Mirrors GenericCache.Unwind. +// Driven from SharedDomains.Unwind so a reorg can't leave stale committed +// branches that a fork-validation then reads as a wrong trie root. +func (c *BranchCache) Unwind(unwindToTxNum uint64) { + c.epoch.Add(1) + if dbgBC { + fmt.Fprintf(os.Stderr, "[BC-UNWIND] txNum=%d newEpoch=%d\n", unwindToTxNum, c.epoch.Load()) + } + for { + cur := c.unwindFloor.Load() + if unwindToTxNum >= cur { + break + } + if c.unwindFloor.CompareAndSwap(cur, unwindToTxNum) { + break + } + } } // GetDecoded retrieves the cached branch in decoded form. Lazy-decodes on @@ -489,7 +553,7 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // caller buffer lifetime. step is the on-disk file step the bytes came // from (0 if not tracked); origin is a short label of the write site // captured for divergence-detection diagnostics. -func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { +func (c *BranchCache) Put(prefix []byte, data []byte, txNum uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -497,7 +561,8 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string copy(dataCopy, data) c.store(prefix, &branchCacheEntry{ data: dataCopy, - step: step, + txNum: txNum, + epoch: c.epoch.Load(), origin: origin, writeSeq: c.writeSeq.Add(1), writeTimeNanos: time.Now().UnixNano(), @@ -511,11 +576,11 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step uint64, origin string) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, txNum uint64, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data, step, origin) + c.Put(prefix, data, txNum, origin) return true } @@ -584,6 +649,7 @@ func (c *BranchCache) Clear() { c.root.Store(nil) c.pinned = maphash.NewMap[*branchCacheEntry]() c.tail.Purge() + c.unwindFloor.Store(math.MaxUint64) c.rootHits.Store(0) c.rootMisses.Store(0) c.pinnedHits.Store(0) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 874c5f651fa..ed62f5769fd 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -38,6 +38,9 @@ type sd interface { SetTxNum(blockNum uint64) AsGetter(tx kv.TemporalTx) kv.TemporalGetter AsPutDel(tx kv.TemporalTx) kv.TemporalPutDel + // MergeWorkerMetrics folds a finished worker's lock-free metrics accumulator + // into the shared metrics (once, not per read). + MergeWorkerMetrics(wm *changeset.WorkerMetrics) StepSize() uint64 Trace() bool CommitmentCapture() bool @@ -500,6 +503,14 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex if err != nil { return &errorTrieContext{err: err}, func() {} } + // Warmup workers run concurrently with the main commitment goroutine. + // Each gets its own lock-free metrics accumulator carried in a per-worker + // context, so reads neither share metrics state with the main goroutine + // (a data race) nor take the global metrics write lock (the prior hard + // serialization point). The accumulator is folded into the shared metrics + // once at teardown. + wm := changeset.NewWorkerMetrics() + workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -507,11 +518,12 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex txNum: txNum, } if sdc.stateReader != nil { - warmupCtx.stateReader = sdc.stateReader.Clone(roTx) + warmupCtx.stateReader = sdc.stateReader.CloneForWorker(workerCtx, roTx) } else { - warmupCtx.stateReader = NewLatestStateReader(roTx, sdc.sharedDomains) + warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { + sdc.sharedDomains.MergeWorkerMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup @@ -539,6 +551,11 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont collectors = append(collectors, collector) mu.Unlock() + // Concurrent mounts run in parallel; each gets its own lock-free metrics + // accumulator via a per-worker context so they don't share metrics state + // (a race) or take the global metrics lock. Folded in at teardown. + wm := changeset.NewWorkerMetrics() + workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -547,11 +564,12 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont localCollector: collector, } if sdc.stateReader != nil { - warmupCtx.stateReader = sdc.stateReader.Clone(roTx) + warmupCtx.stateReader = sdc.stateReader.CloneForWorker(workerCtx, roTx) } else { - warmupCtx.stateReader = NewLatestStateReader(roTx, sdc.sharedDomains) + warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { + sdc.sharedDomains.MergeWorkerMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup diff --git a/execution/commitment/commitmentdb/commitment_context_test.go b/execution/commitment/commitmentdb/commitment_context_test.go index 868443018e1..11d04181f11 100644 --- a/execution/commitment/commitmentdb/commitment_context_test.go +++ b/execution/commitment/commitmentdb/commitment_context_test.go @@ -1,6 +1,7 @@ package commitmentdb import ( + "context" "math/rand" "testing" @@ -55,6 +56,8 @@ func (r *testStateReader) Read(d kv.Domain, key []byte, stepSize uint64) ([]byte func (r *testStateReader) Clone(kv.TemporalTx) StateReader { return r } +func (r *testStateReader) CloneForWorker(context.Context, kv.TemporalTx) StateReader { return r } + func Test_TrieContext_BranchCopiesData(t *testing.T) { t.Parallel() diff --git a/execution/commitment/commitmentdb/reader.go b/execution/commitment/commitmentdb/reader.go index 805931727db..fc9500c424d 100644 --- a/execution/commitment/commitmentdb/reader.go +++ b/execution/commitment/commitmentdb/reader.go @@ -1,6 +1,7 @@ package commitmentdb import ( + "context" "fmt" "github.com/erigontech/erigon/db/kv" @@ -11,18 +12,43 @@ type StateReader interface { CheckDataAvailable(d kv.Domain, step kv.Step) error Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) Clone(tx kv.TemporalTx) StateReader + // CloneForWorker clones the reader for a concurrent worker (trie-warmup / + // concurrent-commitment mount). Behaves like Clone except reads are metered + // into the per-worker accumulator carried by workerCtx, so concurrent + // workers never touch the main goroutine's lock-free accumulator (a race) + // or take the global metrics lock. + CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader +} + +// ctxGetter is the optional context-aware read method (see +// temporalGetter.GetLatestContext). Worker readers type-assert for it so reads +// meter into the per-worker accumulator carried by the worker context. +type ctxGetter interface { + GetLatestContext(ctx context.Context, name kv.Domain, k []byte) (v []byte, step kv.Step, err error) } type LatestStateReader struct { sharedDomains sd getter kv.TemporalGetter srcTx kv.TemporalTx + // workerCtx, when non-nil, carries this worker's lock-free metrics + // accumulator; reads route through getter.GetLatestContext(workerCtx, …) so + // concurrent workers don't share the main accumulator. Nil on the main + // reader, which meters into sd.mainWM via the plain GetLatest. + workerCtx context.Context } func NewLatestStateReader(tx kv.TemporalTx, sd sd) *LatestStateReader { return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx), srcTx: tx} } +// NewLatestStateReaderForWorker is like NewLatestStateReader but reads meter +// into the per-worker accumulator carried by workerCtx (for concurrent +// workers). See LatestStateReader.workerCtx. +func NewLatestStateReaderForWorker(workerCtx context.Context, tx kv.TemporalTx, sd sd) *LatestStateReader { + return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx), srcTx: tx, workerCtx: workerCtx} +} + func (r *LatestStateReader) WithHistory() bool { return false } @@ -36,6 +62,15 @@ func (r *LatestStateReader) CheckDataAvailable(d kv.Domain, step kv.Step) error } func (r *LatestStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) { + if r.workerCtx != nil { + if cg, ok := r.getter.(ctxGetter); ok { + enc, step, err = cg.GetLatestContext(r.workerCtx, d, plainKey) + if err != nil { + return nil, 0, fmt.Errorf("LatestStateReader(GetLatestContext) %q: %w", d, err) + } + return enc, step, nil + } + } enc, step, err = r.getter.GetLatest(d, plainKey) if err != nil { return nil, 0, fmt.Errorf("LatestStateReader(GetLatest) %q: %w", d, err) @@ -51,7 +86,13 @@ func (r *LatestStateReader) Clone(tx kv.TemporalTx) StateReader { // Before flush drained sd.mem this was masked because the in-memory batch // still held the source values; rebinding sd.AsGetter to the foreign compute // tx reads the wrong database and yields empty state (wrong root). - return NewLatestStateReader(r.srcTx, r.sharedDomains) + return &LatestStateReader{sharedDomains: r.sharedDomains, getter: r.sharedDomains.AsGetter(r.srcTx), srcTx: r.srcTx, workerCtx: r.workerCtx} +} + +// CloneForWorker clones into a worker reader that meters into workerCtx's +// per-worker accumulator. Source tx preserved, same as Clone. +func (r *LatestStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return NewLatestStateReaderForWorker(workerCtx, r.srcTx, r.sharedDomains) } // HistoryStateReader reads *full* historical state at specified txNum. @@ -88,6 +129,12 @@ func (r *HistoryStateReader) Clone(tx kv.TemporalTx) StateReader { return NewHistoryStateReader(tx, r.limitReadAsOfTxNum) } +// CloneForWorker: history reads go straight to roTx.GetAsOf (no shared metrics +// accumulator), so it's identical to Clone. +func (r *HistoryStateReader) CloneForWorker(_ context.Context, tx kv.TemporalTx) StateReader { + return NewHistoryStateReader(tx, r.limitReadAsOfTxNum) +} + // FilesOnlyStateReader reads from .kv files only, capped at limitTxNum. // On miss (key not present in any frozen .kv file ≤ limitTxNum), returns nil // without any fallback. This is the right semantic for integrity checks and @@ -123,6 +170,12 @@ func (r *FilesOnlyStateReader) Clone(tx kv.TemporalTx) StateReader { return NewFilesOnlyStateReader(tx, r.limitTxNum) } +// CloneForWorker: files-only reads go straight to the tx (no shared metrics +// accumulator), so it's identical to Clone. +func (r *FilesOnlyStateReader) CloneForWorker(_ context.Context, tx kv.TemporalTx) StateReader { + return NewFilesOnlyStateReader(tx, r.limitTxNum) +} + // SplitStateReader implements commitmentdb.StateReader using (potentially) different state readers for commitment // data and account/storage/code data. type SplitStateReader struct { @@ -163,6 +216,13 @@ func (r *SplitStateReader) Clone(tx kv.TemporalTx) StateReader { return NewCommitmentSplitStateReader(r.commitmentReader.Clone(tx), r.plainStateReader.Clone(tx), r.withHistory) } +// CloneForWorker propagates the worker clone to sub-readers so an embedded +// LatestStateReader (the commitment reader) meters into the per-worker +// accumulator instead of the shared one. +func (r *SplitStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return NewCommitmentSplitStateReader(r.commitmentReader.CloneForWorker(workerCtx, tx), r.plainStateReader.CloneForWorker(workerCtx, tx), r.withHistory) +} + func NewCommitmentSplitStateReader(commitmentReader StateReader, plainStateReader StateReader, withHistory bool) *SplitStateReader { return &SplitStateReader{ commitmentReader: commitmentReader, @@ -197,6 +257,18 @@ func (crsr *CommitmentReplayStateReader) Clone(tx kv.TemporalTx) StateReader { } } +// CloneForWorker mirrors Clone but meters the commitment (Latest) reader into +// the per-worker accumulator carried by workerCtx. +func (crsr *CommitmentReplayStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return &CommitmentReplayStateReader{ + SplitStateReader: NewCommitmentSplitStateReader( + crsr.commitmentReader.CloneForWorker(workerCtx, tx), + crsr.plainStateReader, + false, + ), + } +} + // RebuildStateReader creates a StateReader for building commitment from scratch, block-by-block. // Commitment is read from SharedDomains' in-memory batch (LatestStateReader) because we are generating // it incrementally - prior commitment state lives in the MemBatch, not yet on disk. @@ -241,3 +313,14 @@ func (r *RebuildStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) func (r *RebuildStateReader) Clone(tx kv.TemporalTx) StateReader { return NewRebuildStateReader(tx, r.sd, r.plainStateAsOf) } + +// CloneForWorker mirrors Clone but the commitment (Latest) reader meters into +// the per-worker accumulator carried by workerCtx. +func (r *RebuildStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return &RebuildStateReader{ + commitmentReader: NewLatestStateReaderForWorker(workerCtx, tx, r.sd), + plainStateReader: NewHistoryStateReader(tx, r.plainStateAsOf), + plainStateAsOf: r.plainStateAsOf, + sd: r.sd, + } +} diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 88fa7cc2677..7d0584b5050 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -124,7 +124,11 @@ func (p *ContractTrunkPreload) Run( break } - cache.PinEntry(prefix, v, step, "preload-trunk") + // Trunk preload reads frozen .kv files only (below any unwind point), + // so the bytes can never go stale on a reorg — stamp txNum 0 ("frozen, + // always valid") so the tx-aware Get keeps them warm across unwinds. + _ = step + cache.PinEntry(prefix, v, 0, "preload-trunk") // Copy prefix because nibbles.HexToCompact may return a slice // over a reused buffer; we need a stable handle for later // Invalidate on demotion. diff --git a/execution/exec/state.go b/execution/exec/state.go index 1fc34bc96e3..d519578b0f0 100644 --- a/execution/exec/state.go +++ b/execution/exec/state.go @@ -226,7 +226,7 @@ func (rw *Worker) ResetState(rs *state.StateV3Buffered, chainTx kv.TemporalTx, s } else { var getter kv.TemporalGetter if chainTx != nil { - getter = rs.Domains().AsGetter(chainTx) + getter = rs.Domains().AsGetterNoMetrics(chainTx) } // Use CachedReaderV3 for parallel workers — caches account data // on first read per block, providing a stable pre-block committed @@ -292,7 +292,7 @@ func (rw *Worker) resetTx(chainTx kv.TemporalTx) error { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetter(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) case historic: typedReader.SetTx(rw.chainTx) default: @@ -415,7 +415,7 @@ func (rw *Worker) SetReader(reader state.StateReader) { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetter(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) case historic: typedReader.SetTx(rw.chainTx) } @@ -447,7 +447,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask Task) *TxResult { // the coinbase race investigation). rw.SetReader(state.NewHistoryReaderV3WithSharedDomains(rw.chainTx, rw.rs.Domains(), txTask.Version().TxNum)) } else if !txTask.IsHistoric() && (rw.stateReader == nil || rw.historyMode) { - rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetter(rw.chainTx), nil)) + rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx), nil)) } // Set the per-block committed state cache from the task. @@ -533,7 +533,7 @@ func NewWorkersPool(ctx context.Context, accumulator *shards.Accumulator, backgr reader := stateReader if reader == nil { - reader = state.NewReaderV3(rs.Domains().AsGetter(nil)) + reader = state.NewReaderV3(rs.Domains().AsGetterNoMetrics(nil)) } if err = reconWorkers[i].ResetState(rs, nil, reader, stateWriter, accumulator); err != nil { diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 532c5f29355..5767cfae91b 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -403,6 +403,8 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa err = fmt.Errorf("updateForkChoice: %w", err) return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, err, false) } + // SD.Unwind (inside RunUnwind) tx-aware-invalidates the BranchCache by + // the unwound txNum, so no whole-cache clear is needed here. UpdateForkChoiceDepth(fcuHeader.Number.Uint64() - 1 - unwindTarget) diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 56e6c5fde18..e2900758b2b 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -528,6 +528,11 @@ type asOfStateReader struct { sd *execctx.SharedDomains roTx kv.TemporalTx txNum uint64 + // workerCtx, when non-nil, carries this worker's lock-free metrics + // accumulator; the CommitmentDomain read routes through GetLatestContext so + // a concurrent trie-warmup worker doesn't write the shared main accumulator + // (a race) or take the global metrics lock. Nil on the main reader. + workerCtx context.Context } func (r *asOfStateReader) WithHistory() bool { return false } @@ -539,7 +544,11 @@ func (r *asOfStateReader) CheckDataAvailable(d kv.Domain, step kv.Step) error { func (r *asOfStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) { if d == kv.CommitmentDomain { // Branches: use GetLatest — written only by this calculator, sequential. - enc, step, err = r.sd.GetLatest(d, r.roTx, plainKey) + if r.workerCtx != nil { + enc, step, err = r.sd.GetLatestContext(r.workerCtx, d, r.roTx, plainKey) + } else { + enc, step, err = r.sd.GetLatest(d, r.roTx, plainKey) + } } else { // Account/storage/code: use GetAsOf to avoid reading future state. // Check sd.mem first (in-memory data from current batch), then @@ -570,5 +579,13 @@ func (r *asOfStateReader) Clone(tx kv.TemporalTx) commitmentdb.StateReader { return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum} } +// CloneForWorker meters the worker's CommitmentDomain reads into the per-worker +// accumulator carried by workerCtx (this reader is used as the commitment +// reader during block assembly, where trie-warmup runs concurrently — so it +// must not write the shared main accumulator). +func (r *asOfStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) commitmentdb.StateReader { + return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum, workerCtx: workerCtx} +} + // Keep imports used. var _ = commitment.CommitProgress{} diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 1eca858a1f2..7e1bf7c6424 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -2540,7 +2540,7 @@ func (be *blockExecutor) nextResult(ctx context.Context, pe *parallelExecutor, r // BlockStateCache write buffer. This ensures the // system TX sees all accumulated state from prior // TXs in the block, not stale sd.mem values. - stateReader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetter(applyTx), be.blockStateCache) + stateReader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetterNoMetrics(applyTx), be.blockStateCache) } } @@ -2803,7 +2803,7 @@ func (be *blockExecutor) nextResult(ctx context.Context, pe *parallelExecutor, r // the pre-block balance and stomped tx 28's in-block update. reader = state.NewHistoryReaderV3WithBlockCache(applyTx, pe.rs.Domains(), be.blockStateCache, finalVersion.TxNum) } else { - reader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetter(applyTx), be.blockStateCache) + reader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetterNoMetrics(applyTx), be.blockStateCache) } pe.RUnlock() diff --git a/rpc/jsonrpc/eth_simulation.go b/rpc/jsonrpc/eth_simulation.go index 3a377700d39..948650d07f5 100644 --- a/rpc/jsonrpc/eth_simulation.go +++ b/rpc/jsonrpc/eth_simulation.go @@ -999,6 +999,13 @@ func (r *simulationStateReader) Clone(tx kv.TemporalTx) commitmentdb.StateReader return newHistoryCommitmentOnlyReader(tx, r.sd, r.commitmentAsOfTxNum, r.plainStateAsOfTxNum) } +// CloneForWorker mirrors Clone. eth_simulation runs commitment single-threaded +// (no concurrent warmup), so worker metering isn't needed here; deferred to the +// state-read-path migration. +func (r *simulationStateReader) CloneForWorker(_ context.Context, tx kv.TemporalTx) commitmentdb.StateReader { + return newHistoryCommitmentOnlyReader(tx, r.sd, r.commitmentAsOfTxNum, r.plainStateAsOfTxNum) +} + func newHistoryCommitmentOnlyReader(roTx kv.TemporalTx, sd *execctx.SharedDomains, commitmentAsOfTxNum uint64, plainStateAsOfTxNum uint64) commitmentdb.StateReader { return &simulationStateReader{sd: sd, roTx: roTx, commitmentAsOfTxNum: commitmentAsOfTxNum, plainStateAsOfTxNum: plainStateAsOfTxNum} } From a0270020108c18a1604c7617500000591d58d9c8 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 20:25:33 +0000 Subject: [PATCH 112/167] db/state: drop process-wide metrics accumulator (metrics-on safety) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit kept a single SharedDomains.mainWM that every AsGetter caller recorded into. But AsGetter is used by many concurrent goroutines on a live node (RPC, engine API, txpool, exec) — so a single lock-free accumulator was both raced and unbounded (its uniqueSeen map grew without limit and was written from every read). With KV_READ_METRICS=true this degraded a long-running node to a crawl (cancun 225/226 failing on insufficient-funds / timeouts); metrics-off was unaffected since the accumulator was never touched. Remove mainWM (and flushMainWM): GetLatest now collects no metrics. Read metrics are gathered only per-task / per-worker via GetLatestContext (the commitment + trie-warmup readers), each accumulator bounded to one task and merged into the shared DomainMetrics at teardown. AsGetterNoMetrics is kept as an explicit-intent alias. Wiring the exec/RPC read paths onto per-task contexts is a follow-up. Verified: KV_READ_METRICS=true cancun back to the 3 known failures (was 225); -race + metrics-on clean; build/gofmt clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/execctx/domain_shared.go | 62 ++++++++++--------------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 073ecf3b0d9..6464a7dd057 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -109,11 +109,6 @@ type SharedDomains struct { disableInlineTouchKey bool mem kv.TemporalMemBatch metrics changeset.DomainMetrics - // mainWM is the lock-free metrics accumulator for the main goroutine's - // reads (direct GetLatest + the main commitment/exec getter). Trie-warmup - // workers get their own via AsGetterWorker; all are combined into metrics - // at task boundaries. Keeps the global metrics write-lock off the hot path. - mainWM *changeset.WorkerMetrics // blockOverlay is an in-memory overlay for block-level metadata writes (headers, bodies, // canonical hashes, TD, stage progress, forkchoice markers). It allows execution to @@ -203,7 +198,6 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg logger: logger, //trace: true, metrics: changeset.DomainMetrics{Domains: map[kv.Domain]*changeset.DomainIOMetrics{}}, - mainWM: changeset.NewWorkerMetrics(), stepSize: tx.Debug().StepSize(), } @@ -440,15 +434,16 @@ func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k type temporalGetter struct { sd *SharedDomains tx kv.TemporalTx - // wm is the accumulator GetLatest records into: sd.mainWM for the main - // goroutine (AsGetter), or nil to collect nothing (AsGetterNoMetrics) for - // concurrent callers whose per-worker metering isn't wired yet (parallel - // exec workers) — they must not write the shared main accumulator (a race). - wm *changeset.WorkerMetrics } +// GetLatest collects no read metrics. There is no process-wide accumulator: +// AsGetter is used by many concurrent goroutines (RPC, engine, exec) and a +// shared lock-free accumulator would be raced/unbounded. Metrics are collected +// only per-task via GetLatestContext (the commitment/warmup worker readers), +// merged into the shared DomainMetrics at task end. Wiring the exec/RPC read +// paths onto per-task contexts is a follow-up. func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.getLatestMetered(name, gt.tx, k, gt.wm) + return gt.sd.getLatestMetered(name, gt.tx, k, nil) } // GetLatestContext is the context-aware read: it records into the per-worker, @@ -495,15 +490,14 @@ func (up *unmarkedPutter) Put(num kv.Num, v []byte) error { } func (sd *SharedDomains) AsGetter(tx kv.TemporalTx) kv.TemporalGetter { - return &temporalGetter{sd: sd, tx: tx, wm: sd.mainWM} + return &temporalGetter{sd: sd, tx: tx} } -// AsGetterNoMetrics returns a getter whose GetLatest collects no metrics. Use -// for concurrent callers that share this SD but don't yet have per-worker -// metrics wiring (parallel exec workers): metering them into sd.mainWM would be -// a data race. Their per-worker metering is a follow-up (the state-read path). +// AsGetterNoMetrics is retained as an explicit-intent alias of AsGetter (which +// already collects no metrics) for concurrent callers (parallel exec workers) +// where the no-metrics choice is deliberate pending per-task wiring. func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { - return &temporalGetter{sd: sd, tx: tx, wm: nil} + return &temporalGetter{sd: sd, tx: tx} } // MergeWorkerMetrics folds a worker's lock-free accumulator into the shared @@ -975,26 +969,25 @@ func (sd *SharedDomains) DetachBranchCache() { sd.branchCache = nil } -// TemporalDomain satisfaction. Reads on behalf of the main goroutine, recording -// metrics into the shared lock-free main accumulator. +// TemporalDomain satisfaction. Collects no read metrics — see +// temporalGetter.GetLatest for why there is no process-wide accumulator. func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { - return sd.getLatestMetered(domain, tx, k, sd.mainWM) + return sd.getLatestMetered(domain, tx, k, nil) } // GetLatestContext is the context-aware read for callers that read on behalf of // a concurrent worker: metrics go to the per-worker, lock-free accumulator // carried by ctx (nil ctx-value => no metrics). Lets a worker's reader meter -// without touching sd.mainWM (a race) or any lock. Mirrors -// temporalGetter.GetLatestContext for readers that hold the SD directly (e.g. -// the committer's asOfStateReader). +// without any shared accumulator or lock. Mirrors temporalGetter.GetLatestContext +// for readers that hold the SD directly (e.g. the committer's asOfStateReader). func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { return sd.getLatestMetered(domain, tx, k, changeset.WorkerMetricsFromContext(ctx)) } // getLatestMetered is the read implementation. wm is the caller's lock-free -// metrics accumulator (the main goroutine's sd.mainWM, or a warmup worker's -// private one); a nil wm disables metrics for the call. No global metrics lock -// is taken on this hot path — accumulators are combined later via Merge. +// per-task/per-worker metrics accumulator (nil disables metrics for the call). +// No global metrics lock is taken on this hot path — accumulators are combined +// into the shared DomainMetrics later via Merge. func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.WorkerMetrics) (v []byte, step kv.Step, err error) { if tx == nil { return nil, 0, errors.New("sd.GetLatest: unexpected nil tx") @@ -1278,28 +1271,13 @@ func decodeAccountCodeHash(enc []byte) []byte { return h[:] } -// flushMainWM folds the main goroutine's lock-free accumulator into the shared -// DomainMetrics and resets it, so a subsequent read of sd.metrics reflects the -// main goroutine's reads. Must be called from the main goroutine (the sole -// writer of mainWM). Warmup workers merge their own accumulators independently -// via MergeWorkerMetrics; both serialize on sd.metrics' lock. -func (sd *SharedDomains) flushMainWM() { - if sd.mainWM == nil { - return - } - sd.metrics.Merge(sd.mainWM) - sd.mainWM.Reset() -} - func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { - sd.flushMainWM() return &sd.metrics } func (sd *SharedDomains) LogMetrics() []any { var metrics []any - sd.flushMainWM() sd.metrics.RLock() defer sd.metrics.RUnlock() From 943968dde6b83d45e0e3f050d4f1a09d4108e29f Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 21:06:15 +0000 Subject: [PATCH 113/167] commitment: meter the main-fold reads per-ComputeCommitment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metrics-on safety fix left the main (root-fold) commitment reads unmetered — only trie-warmup workers collected, giving a partial read profile. Give the main fold its own per-ComputeCommitment lock-free accumulator (carried via the read context, merged into the shared metrics when commitment finishes). The fold is single-goroutine so it owns the accumulator exclusively; warmup / concurrent-mount workers keep their own. Now KV_READ_METRICS captures the full commitment read profile without any process-wide accumulator. Verified: -race + KV_READ_METRICS=true clean (3x) on execmodule parallel block-assembly; build/gofmt clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../commitmentdb/commitment_context.go | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index ed62f5769fd..fe082edd1df 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -207,7 +207,11 @@ func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant return ctx } -func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNum, txNum uint64) *TrieContext { +// trieContext builds the main (root-fold) trie read context. readCtx carries +// the per-ComputeCommitment lock-free metrics accumulator (nil-value => no +// metrics); the main fold is single-goroutine so it owns that accumulator +// exclusively. Warmup/concurrent-mount readers get their own via the factories. +func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNum, txNum uint64, readCtx context.Context) *TrieContext { mainTtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(tx), putter: sdc.sharedDomains.AsPutDel(tx), @@ -218,9 +222,9 @@ func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNu probeTx: tx, } if sdc.stateReader != nil { - mainTtx.stateReader = sdc.stateReader.Clone(tx) + mainTtx.stateReader = sdc.stateReader.CloneForWorker(readCtx, tx) } else { - mainTtx.stateReader = NewLatestStateReader(tx, sdc.sharedDomains) + mainTtx.stateReader = NewLatestStateReaderForWorker(readCtx, tx, sdc.sharedDomains) } sdc.patriciaTrie.ResetContext(mainTtx) return mainTtx @@ -346,7 +350,15 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context } } - trieContext := sdc.trieContext(tx, blockNum, txNum) + // Per-ComputeCommitment metrics accumulator for the main (root-fold) reads. + // The fold is single-goroutine, so this lock-free accumulator is owned + // exclusively here; merged into the shared metrics when commitment finishes. + // Warmup/concurrent-mount workers accumulate + merge independently. + commitMetrics := changeset.NewWorkerMetrics() + defer sdc.sharedDomains.MergeWorkerMetrics(commitMetrics) + readCtx := changeset.ContextWithWorkerMetrics(ctx, commitMetrics) + + trieContext := sdc.trieContext(tx, blockNum, txNum, readCtx) // If trie trace is configured, wrap the context with a recorder. // Block-targeted: when TrieTraceBlock is set, only record that specific block. @@ -659,7 +671,7 @@ func (sdc *SharedDomainsCommitmentContext) enableConcurrentCommitmentIfPossible( // SeekCommitment searches for last encoded state from DomainCommitted // and if state found, sets it up to current domain func (sdc *SharedDomainsCommitmentContext) SeekCommitment(ctx context.Context, tx kv.TemporalTx) (txNum, blockNum uint64, err error) { - trieContext := sdc.trieContext(tx, 0, 0) // blockNum/txNum not yet known; trieContext only used for reading here + trieContext := sdc.trieContext(tx, 0, 0, ctx) // blockNum/txNum not yet known; trieContext only used for reading here _, _, state, err := sdc.LatestCommitmentState(trieContext) if err != nil { From f375aa842784ee4220ce23d513477fbc1a45c68e Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 22:55:20 +0000 Subject: [PATCH 114/167] db/state, commitment: collapse metrics to per-worker DomainMetrics instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces changeset.WorkerMetrics (added in 44355fd) with plain per-worker *DomainMetrics instances — removing both the name collision with the unrelated execution/exec.WorkerMetrics and an extra type. The model is now: - Update* are lock-free and run on a single-owner per-worker instance (one per trie-warmup worker, one per ComputeCommitment for the main fold). - Merge folds a finished worker instance into the shared aggregate under the aggregate's lock — the only protected write. A separate goroutine reading the aggregate under RLock always sees a consistent snapshot. So: updates unprotected, aggregation/snapshot protected — no per-read lock, no atomics, no process-wide accumulator. Also drops the UniqueFileReadCount / UniqueLenBuckets read-amplification tracking: it had no consumer (only a comment referenced it) and cost a string-alloc + map-insert on every file read — a needless hot-path cost. The struct fields remain (unpopulated) to avoid churn. Update* gain nil receiver guards: a nil *DomainMetrics is the valid "collect no metrics" sentinel for reads with no per-worker context (e.g. SeekCommitment, plain AsGetter on RPC/exec paths). Verified: build, make lint (0 issues), gofmt, -race + KV_READ_METRICS=true clean (3x), and hive cancun with KV_READ_METRICS=true back to the 3 known fails (node healthy). Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/aggregator.go | 4 +- db/state/changeset/state_changeset.go | 284 ++++++------------ db/state/domain.go | 2 +- db/state/execctx/domain_shared.go | 12 +- .../commitmentdb/commitment_context.go | 22 +- 5 files changed, 112 insertions(+), 212 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index c83c8568dbd..387e4f4a906 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2478,11 +2478,11 @@ func (at *AggregatorRoTx) GetLatest(domain kv.Domain, k []byte, tx kv.Tx) (v []b return at.getLatest(domain, k, tx, math.MaxUint64, nil, time.Time{}) } -func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { return at.getLatest(domain, k, tx, maxStep, metrics, start) } -func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { if domain != kv.CommitmentDomain { return at.d[domain].getLatest(k, tx, maxStep, metrics, start) } diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 60bbd70a3d4..c6b9e0962b7 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -501,190 +501,126 @@ type DomainIOMetrics struct { StateCacheMissCount int64 } +// DomainMetrics is used in two roles. As the shared aggregate, it is read for a +// snapshot under RLock and written only by Merge under Lock — so a separate +// goroutine always sees a consistent snapshot. As a per-worker instance (one +// per goroutine, via NewDomainMetrics), its Update* run lock-free because the +// instance is single-owner; the worker's counts are folded into the aggregate +// via Merge when the worker's task finishes. The embedded mutex therefore +// guards aggregation/snapshot only — never the per-read Update* path. type DomainMetrics struct { sync.RWMutex DomainIOMetrics Domains map[kv.Domain]*DomainIOMetrics - - // seenFileReads is a process-cumulative dedup set for - // UpdateFileReadsUnique. Keys are (domain.String() + prefixBytes); - // LoadOrStore on each read decides whether the prefix is new and - // therefore contributes to UniqueFileReadCount. Held outside the - // mutex (sync.Map handles its own concurrency) so the unique check - // doesn't serialise with other metric updates. - seenFileReads sync.Map } -// WorkerMetrics is a per-worker, lock-free I/O metrics accumulator. Each -// goroutine on the hot read path (the main commitment/exec goroutine and each -// trie-warmup worker) owns one exclusively, so updates need no synchronization -// — no mutex, no atomics, no shared cache line. The per-worker accumulators are -// combined into the shared DomainMetrics once per task via DomainMetrics.Merge. -// -// This replaces the previous design where every read called a DomainMetrics -// method that took the global write lock (dm.Lock()); under concurrent warmup -// workers that write lock was a hard serialization point on the hottest path. -type WorkerMetrics struct { - total DomainIOMetrics - domains [kv.DomainLen]DomainIOMetrics - - // uniqueSeen records the first local sighting of each (domain,prefix) that - // fell through to a file read, so cross-worker uniqueness can be resolved - // once at Merge time (against DomainMetrics.seenFileReads) instead of - // touching a shared structure on every read. Value is the length bucket. - // Nil until the first UpdateFileReadsUnique call (workers that never read - // from files pay nothing). - uniqueSeen map[uniqueKey]int +// NewDomainMetrics returns a fresh instance (used for the aggregate and for +// per-worker instances). +func NewDomainMetrics() *DomainMetrics { + return &DomainMetrics{Domains: map[kv.Domain]*DomainIOMetrics{}} } -type uniqueKey struct { - domain kv.Domain - prefix string -} - -// NewWorkerMetrics returns a fresh lock-free accumulator. A nil *WorkerMetrics -// is valid and makes every Update* a no-op, so callers that don't collect -// metrics can pass nil. -func NewWorkerMetrics() *WorkerMetrics { return &WorkerMetrics{} } - -// ctxKey is the typed-int context-key namespace for this package (the const-int -// model used by node/app), avoiding per-key struct types. -type ctxKey int - -const ( - ckWorkerMetrics ctxKey = iota -) - -// ContextWithWorkerMetrics returns a child context carrying a per-worker, -// lock-free metrics accumulator. Concurrent workers (e.g. trie-warmup -// goroutines) each run with their own, so reads never contend on a shared -// metrics structure (and take no lock). -func ContextWithWorkerMetrics(ctx context.Context, wm *WorkerMetrics) context.Context { - return context.WithValue(ctx, ckWorkerMetrics, wm) -} - -// WorkerMetricsFromContext extracts the per-worker accumulator, or nil if none -// was attached (in which case reads collect no metrics — a valid no-op). -func WorkerMetricsFromContext(ctx context.Context) *WorkerMetrics { - if ctx == nil { - return nil +// domainEntry returns the per-domain counters, creating them on first use. +// Lock-free: only called on a single-owner per-worker instance (the per-domain +// roll-up into the aggregate happens in Merge, under the aggregate's lock). +func (dm *DomainMetrics) domainEntry(domain kv.Domain) *DomainIOMetrics { + d, ok := dm.Domains[domain] + if !ok { + d = &DomainIOMetrics{} + dm.Domains[domain] = d } - wm, _ := ctx.Value(ckWorkerMetrics).(*WorkerMetrics) - return wm + return d } -// Reset zeroes the accumulator so it can be reused after its counts have been -// merged into the shared DomainMetrics. Used for the long-lived main-goroutine -// accumulator, which is merged-and-reset on each metrics read. -func (w *WorkerMetrics) Reset() { - if w == nil { - return - } - w.total = DomainIOMetrics{} - for i := range w.domains { - w.domains[i] = DomainIOMetrics{} - } - w.uniqueSeen = nil -} - -func (w *WorkerMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { - if w == nil { +// Update* record one read into this (single-owner) instance, lock-free. Do NOT +// call them on the shared aggregate from multiple goroutines — use a per-worker +// instance and Merge it in. +func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { + if dm == nil { return } d := time.Since(start) - w.total.CacheReadCount++ - w.total.CacheReadDuration += d - w.domains[domain].CacheReadCount++ - w.domains[domain].CacheReadDuration += d + dm.CacheReadCount++ + dm.CacheReadDuration += d + de := dm.domainEntry(domain) + de.CacheReadCount++ + de.CacheReadDuration += d } -func (w *WorkerMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { - if w == nil { +func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { + if dm == nil { return } d := time.Since(start) - w.total.DbReadCount++ - w.total.DbReadDuration += d - w.domains[domain].DbReadCount++ - w.domains[domain].DbReadDuration += d + dm.DbReadCount++ + dm.DbReadDuration += d + de := dm.domainEntry(domain) + de.DbReadCount++ + de.DbReadDuration += d } -func (w *WorkerMetrics) UpdateStateCacheHit(domain kv.Domain) { - if w == nil { +func (dm *DomainMetrics) UpdateStateCacheHit(domain kv.Domain) { + if dm == nil { return } - w.total.StateCacheHitCount++ - w.domains[domain].StateCacheHitCount++ + dm.StateCacheHitCount++ + dm.domainEntry(domain).StateCacheHitCount++ } -func (w *WorkerMetrics) UpdateStateCacheMiss(domain kv.Domain) { - if w == nil { +func (dm *DomainMetrics) UpdateStateCacheMiss(domain kv.Domain) { + if dm == nil { return } - w.total.StateCacheMissCount++ - w.domains[domain].StateCacheMissCount++ + dm.StateCacheMissCount++ + dm.domainEntry(domain).StateCacheMissCount++ } -func (w *WorkerMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { - if w == nil { +func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { + if dm == nil { return } d := time.Since(start) - w.total.FileReadCount++ - w.total.FileReadDuration += d - w.domains[domain].FileReadCount++ - w.domains[domain].FileReadDuration += d + dm.FileReadCount++ + dm.FileReadDuration += d + de := dm.domainEntry(domain) + de.FileReadCount++ + de.FileReadDuration += d } -// Merge folds a worker's accumulated counters into the shared DomainMetrics -// under a single lock acquisition (once per task, not once per read). Unique -// file-read prefixes are deduplicated across workers here, against the global -// seenFileReads set, so the cross-worker uniqueness count stays correct without -// any shared structure on the hot path. -func (dm *DomainMetrics) Merge(w *WorkerMetrics) { - if w == nil { +// UpdateFileReadsUnique records a file read. The distinct-prefix (read +// amplification) tracking it used to do was removed: it had no consumer and +// cost a string alloc + map insert on every file read. Thin alias so call sites +// are unchanged. +func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { + dm.UpdateFileReads(domain, start) +} + +// Merge folds a finished per-worker instance into this aggregate under the +// aggregate's lock — the join. This and the RLock'd snapshot are the only +// protected operations. +func (dm *DomainMetrics) Merge(src *DomainMetrics) { + if src == nil { return } dm.Lock() defer dm.Unlock() - addIOMetrics(&dm.DomainIOMetrics, &w.total) - for d := range w.domains { - src := &w.domains[d] - if src.isZero() { - continue - } - dst, ok := dm.Domains[kv.Domain(d)] - if !ok { - dst = &DomainIOMetrics{} - dm.Domains[kv.Domain(d)] = dst - } - addIOMetrics(dst, src) - } - for uk, bucket := range w.uniqueSeen { - domainKey := uk.domain.String() + ":" + uk.prefix - if _, already := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}); already { - continue - } - dm.UniqueFileReadCount++ - dm.UniqueLenBuckets[bucket]++ - if dst, ok := dm.Domains[uk.domain]; ok { - dst.UniqueFileReadCount++ - dst.UniqueLenBuckets[bucket]++ - } else { - nd := &DomainIOMetrics{UniqueFileReadCount: 1} - nd.UniqueLenBuckets[bucket] = 1 - dm.Domains[uk.domain] = nd - } + addIOMetrics(&dm.DomainIOMetrics, &src.DomainIOMetrics) + for d, sd := range src.Domains { + addIOMetrics(dm.domainEntry(d), sd) } } -// addIOMetrics adds src's cumulative read-side counters into dst. CachePut* -// fields are intentionally excluded: those track mem-batch buffer occupancy -// (functional state, reset on flush), not cumulative read metrics, and are -// accounted separately off this path. func addIOMetrics(dst, src *DomainIOMetrics) { dst.CacheReadCount += src.CacheReadCount dst.CacheReadDuration += src.CacheReadDuration + dst.CacheGetCount += src.CacheGetCount + dst.CachePutCount += src.CachePutCount + dst.CacheGetSize += src.CacheGetSize + dst.CacheGetKeySize += src.CacheGetKeySize + dst.CacheGetValueSize += src.CacheGetValueSize + dst.CachePutSize += src.CachePutSize + dst.CachePutKeySize += src.CachePutKeySize + dst.CachePutValueSize += src.CachePutValueSize dst.DbReadCount += src.DbReadCount dst.DbReadDuration += src.DbReadDuration dst.FileReadCount += src.FileReadCount @@ -693,60 +629,24 @@ func addIOMetrics(dst, src *DomainIOMetrics) { dst.StateCacheMissCount += src.StateCacheMissCount } -func (m *DomainIOMetrics) isZero() bool { - return m.CacheReadCount == 0 && m.DbReadCount == 0 && m.FileReadCount == 0 && - m.StateCacheHitCount == 0 && m.StateCacheMissCount == 0 -} +// ctxKey is this package's typed-int context-key namespace (the node/app model). +type ctxKey int -// lenBucket maps a prefix byte-length to its UniqueLenBuckets index. -// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. Buckets -// 5-8 sub-divide the storage subtree (depths 64-127) so per-contract -// trunk vs deep leaf-parent reads are distinguishable. -func lenBucket(n int) int { - switch { - case n <= 1: - return 0 - case n <= 4: - return 1 - case n <= 8: - return 2 - case n <= 16: - return 3 - case n <= 32: - return 4 - case n == 33: - return 5 - case n <= 36: - return 6 - case n <= 44: - return 7 - case n <= 64: - return 8 - default: - return 9 - } -} +const ckMetrics ctxKey = iota -// UpdateFileReadsUnique records a file read in the worker-local accumulator, -// remembering the (domain,prefix) first-sighting so cross-worker uniqueness can -// be resolved at Merge time. Lock-free: the key bytes are copied into a -// per-worker map; nothing shared is touched on the hot path. The key bytes are -// copied; do not mutate after the call. -func (w *WorkerMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { - if w == nil { - return - } - d := time.Since(start) - w.total.FileReadCount++ - w.total.FileReadDuration += d - w.domains[domain].FileReadCount++ - w.domains[domain].FileReadDuration += d +// ContextWithMetrics attaches a per-worker metrics instance to ctx; the read +// path extracts it via MetricsFromContext so a concurrent worker accumulates +// into its own instance and never touches shared state. +func ContextWithMetrics(ctx context.Context, m *DomainMetrics) context.Context { + return context.WithValue(ctx, ckMetrics, m) +} - if w.uniqueSeen == nil { - w.uniqueSeen = make(map[uniqueKey]int) - } - uk := uniqueKey{domain: domain, prefix: string(key)} - if _, ok := w.uniqueSeen[uk]; !ok { - w.uniqueSeen[uk] = lenBucket(len(key)) +// MetricsFromContext returns the per-worker instance, or nil (reads then collect +// no metrics — a valid no-op). +func MetricsFromContext(ctx context.Context) *DomainMetrics { + if ctx == nil { + return nil } + m, _ := ctx.Value(ckMetrics).(*DomainMetrics) + return m } diff --git a/db/state/domain.go b/db/state/domain.go index e512287659e..d9158f5ce42 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -1611,7 +1611,7 @@ func (dt *DomainRoTx) GetLatest(key []byte, roTx kv.Tx) ([]byte, kv.Step, bool, return dt.getLatest(key, roTx, math.MaxInt64, nil, time.Time{}) } -func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) ([]byte, kv.Step, bool, error) { +func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) ([]byte, kv.Step, bool, error) { if dt.d.Disable { return nil, 0, false, nil } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 6464a7dd057..16d2d3a1186 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -454,7 +454,7 @@ func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv // AggregatorRoTx.MeteredGetLatest pattern). The plan is to migrate GetLatest // callers onto this ctx-carrying form in a follow-up, then drop the split. func (gt *temporalGetter) GetLatestContext(ctx context.Context, name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.getLatestMetered(name, gt.tx, k, changeset.WorkerMetricsFromContext(ctx)) + return gt.sd.getLatestMetered(name, gt.tx, k, changeset.MetricsFromContext(ctx)) } // GetCodeSize returns the length of the code at addr without loading the @@ -500,10 +500,10 @@ func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { return &temporalGetter{sd: sd, tx: tx} } -// MergeWorkerMetrics folds a worker's lock-free accumulator into the shared +// MergeMetrics folds a worker's lock-free accumulator into the shared // DomainMetrics under a single lock. Called once when a worker finishes (not // per read), so the global metrics write-lock stays off the hot path. -func (sd *SharedDomains) MergeWorkerMetrics(wm *changeset.WorkerMetrics) { +func (sd *SharedDomains) MergeMetrics(wm *changeset.DomainMetrics) { sd.metrics.Merge(wm) } @@ -981,14 +981,14 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // without any shared accumulator or lock. Mirrors temporalGetter.GetLatestContext // for readers that hold the SD directly (e.g. the committer's asOfStateReader). func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { - return sd.getLatestMetered(domain, tx, k, changeset.WorkerMetricsFromContext(ctx)) + return sd.getLatestMetered(domain, tx, k, changeset.MetricsFromContext(ctx)) } // getLatestMetered is the read implementation. wm is the caller's lock-free // per-task/per-worker metrics accumulator (nil disables metrics for the call). // No global metrics lock is taken on this hot path — accumulators are combined // into the shared DomainMetrics later via Merge. -func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.WorkerMetrics) (v []byte, step kv.Step, err error) { +func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.DomainMetrics) (v []byte, step kv.Step, err error) { if tx == nil { return nil, 0, errors.New("sd.GetLatest: unexpected nil tx") } @@ -1027,7 +1027,7 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k } type MeteredGetter interface { - MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) + MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } // stateCache holds in-flight values from previous transactions in the same batch diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index fe082edd1df..791a26450f4 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -38,9 +38,9 @@ type sd interface { SetTxNum(blockNum uint64) AsGetter(tx kv.TemporalTx) kv.TemporalGetter AsPutDel(tx kv.TemporalTx) kv.TemporalPutDel - // MergeWorkerMetrics folds a finished worker's lock-free metrics accumulator + // MergeMetrics folds a finished worker's lock-free metrics accumulator // into the shared metrics (once, not per read). - MergeWorkerMetrics(wm *changeset.WorkerMetrics) + MergeMetrics(wm *changeset.DomainMetrics) StepSize() uint64 Trace() bool CommitmentCapture() bool @@ -354,9 +354,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // The fold is single-goroutine, so this lock-free accumulator is owned // exclusively here; merged into the shared metrics when commitment finishes. // Warmup/concurrent-mount workers accumulate + merge independently. - commitMetrics := changeset.NewWorkerMetrics() - defer sdc.sharedDomains.MergeWorkerMetrics(commitMetrics) - readCtx := changeset.ContextWithWorkerMetrics(ctx, commitMetrics) + commitMetrics := changeset.NewDomainMetrics() + defer sdc.sharedDomains.MergeMetrics(commitMetrics) + readCtx := changeset.ContextWithMetrics(ctx, commitMetrics) trieContext := sdc.trieContext(tx, blockNum, txNum, readCtx) @@ -521,8 +521,8 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex // (a data race) nor take the global metrics write lock (the prior hard // serialization point). The accumulator is folded into the shared metrics // once at teardown. - wm := changeset.NewWorkerMetrics() - workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) + wm := changeset.NewDomainMetrics() + workerCtx := changeset.ContextWithMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -535,7 +535,7 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { - sdc.sharedDomains.MergeWorkerMetrics(wm) + sdc.sharedDomains.MergeMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup @@ -566,8 +566,8 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont // Concurrent mounts run in parallel; each gets its own lock-free metrics // accumulator via a per-worker context so they don't share metrics state // (a race) or take the global metrics lock. Folded in at teardown. - wm := changeset.NewWorkerMetrics() - workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) + wm := changeset.NewDomainMetrics() + workerCtx := changeset.ContextWithMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -581,7 +581,7 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { - sdc.sharedDomains.MergeWorkerMetrics(wm) + sdc.sharedDomains.MergeMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup From cd1fb8114f496593fab563bb467919a790d3e907 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 23:09:38 +0000 Subject: [PATCH 115/167] exec: meter parallel-exec worker reads (per-worker DomainMetrics, per-task merge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Un-defers the exec-path read metrics that the metrics refactor had left collecting nothing (AsGetterNoMetrics), which made DomainMetrics undercount on a syncing node (exec reads dominate). Each exec Worker now owns a private *changeset.DomainMetrics; its state reader records into it via AsGetterMetered (lock-free, single-owner). The instance is merged into the shared SharedDomains metrics and reset once per task — right after the result is queued, off the hot path. That replaces a lock-per-read with a lock-per-task (a task has many reads), so contention is far lower; skipped entirely when KV_READ_METRICS is off. So exec3_metrics' per-domain Account/Storage/Code cache-layer profile is now populated, complementing exec.WorkerMetrics' per-worker read counts. Verified: build, make lint (0 issues), gofmt, -race + KV_READ_METRICS=true clean. Hive metrics-on validation pending (next). Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/changeset/state_changeset.go | 13 +++++++++++++ db/state/execctx/domain_shared.go | 27 ++++++++++++++++---------- execution/exec/state.go | 28 ++++++++++++++++++++------- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index c6b9e0962b7..48ae1dc0260 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -520,6 +520,19 @@ func NewDomainMetrics() *DomainMetrics { return &DomainMetrics{Domains: map[kv.Domain]*DomainIOMetrics{}} } +// Reset zeroes a per-worker instance after its counts have been merged into the +// aggregate, so it can be reused for the next task. Lock-free: called by the +// single owning worker. The Domains map is cleared but kept allocated. +func (dm *DomainMetrics) Reset() { + if dm == nil { + return + } + dm.DomainIOMetrics = DomainIOMetrics{} + for k := range dm.Domains { + delete(dm.Domains, k) + } +} + // domainEntry returns the per-domain counters, creating them on first use. // Lock-free: only called on a single-owner per-worker instance (the per-domain // roll-up into the aggregate happens in Merge, under the aggregate's lock). diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 16d2d3a1186..61fcb201cae 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -434,16 +434,16 @@ func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k type temporalGetter struct { sd *SharedDomains tx kv.TemporalTx + // m is an optional per-worker metrics instance to record reads into. nil + // (the AsGetter default) collects nothing — there is no process-wide + // accumulator, since AsGetter is used by many concurrent goroutines (RPC, + // engine) where a shared one would be raced/unbounded. Exec workers pass + // their own instance via AsGetterMetered and merge it at task end. + m *changeset.DomainMetrics } -// GetLatest collects no read metrics. There is no process-wide accumulator: -// AsGetter is used by many concurrent goroutines (RPC, engine, exec) and a -// shared lock-free accumulator would be raced/unbounded. Metrics are collected -// only per-task via GetLatestContext (the commitment/warmup worker readers), -// merged into the shared DomainMetrics at task end. Wiring the exec/RPC read -// paths onto per-task contexts is a follow-up. func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.getLatestMetered(name, gt.tx, k, nil) + return gt.sd.getLatestMetered(name, gt.tx, k, gt.m) } // GetLatestContext is the context-aware read: it records into the per-worker, @@ -493,13 +493,20 @@ func (sd *SharedDomains) AsGetter(tx kv.TemporalTx) kv.TemporalGetter { return &temporalGetter{sd: sd, tx: tx} } -// AsGetterNoMetrics is retained as an explicit-intent alias of AsGetter (which -// already collects no metrics) for concurrent callers (parallel exec workers) -// where the no-metrics choice is deliberate pending per-task wiring. +// AsGetterNoMetrics is an explicit-intent alias of AsGetter (collects no +// metrics), for concurrent callers (RPC/engine) where that is deliberate. func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { return &temporalGetter{sd: sd, tx: tx} } +// AsGetterMetered returns a getter that records reads into the caller's own +// per-worker metrics instance m. m must be single-owner (one goroutine); the +// caller merges it into the shared metrics via MergeMetrics at task end (a lock +// per task, not per read) and Resets it. Used by parallel-exec workers. +func (sd *SharedDomains) AsGetterMetered(tx kv.TemporalTx, m *changeset.DomainMetrics) kv.TemporalGetter { + return &temporalGetter{sd: sd, tx: tx, m: m} +} + // MergeMetrics folds a worker's lock-free accumulator into the shared // DomainMetrics under a single lock. Called once when a worker finishes (not // per read), so the global metrics write-lock stays off the hot path. diff --git a/execution/exec/state.go b/execution/exec/state.go index d519578b0f0..c24532439d4 100644 --- a/execution/exec/state.go +++ b/execution/exec/state.go @@ -32,6 +32,7 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol" @@ -126,6 +127,11 @@ type Worker struct { dirs datadir.Dirs metrics *WorkerMetrics + // readMetrics is this worker's private domain read-metrics accumulator + // (lock-free, single-owner). The worker's state reader records into it; it + // is merged into the shared SharedDomains metrics + reset once per task (a + // lock per task, not per read), at the point the result is queued. + readMetrics *changeset.DomainMetrics } // installWorkerGetHash replaces the EVM's GetHash function with one that @@ -177,8 +183,9 @@ func NewWorker(ctx context.Context, background bool, metrics *WorkerMetrics, cha evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), - dirs: dirs, - metrics: metrics, + dirs: dirs, + metrics: metrics, + readMetrics: changeset.NewDomainMetrics(), } w.runnable.Store(true) w.ibs = state.New(w.stateReader) @@ -226,7 +233,7 @@ func (rw *Worker) ResetState(rs *state.StateV3Buffered, chainTx kv.TemporalTx, s } else { var getter kv.TemporalGetter if chainTx != nil { - getter = rs.Domains().AsGetterNoMetrics(chainTx) + getter = rs.Domains().AsGetterMetered(chainTx, rw.readMetrics) } // Use CachedReaderV3 for parallel workers — caches account data // on first read per block, providing a stable pre-block committed @@ -292,7 +299,7 @@ func (rw *Worker) resetTx(chainTx kv.TemporalTx) error { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterMetered(rw.chainTx, rw.readMetrics)) case historic: typedReader.SetTx(rw.chainTx) default: @@ -361,6 +368,13 @@ func (rw *Worker) Run() (err error) { if err := rw.results.Add(rw.ctx, result); err != nil { return err } + // Fold this task's reads into the shared metrics and reset. Off the hot + // path (the result is already queued): a single lock per task replacing + // the old lock-per-read. Skipped entirely when read metrics are off. + if dbg.KVReadLevelledMetrics && rw.rs != nil { + rw.rs.Domains().MergeMetrics(rw.readMetrics) + rw.readMetrics.Reset() + } } return nil } @@ -415,7 +429,7 @@ func (rw *Worker) SetReader(reader state.StateReader) { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterMetered(rw.chainTx, rw.readMetrics)) case historic: typedReader.SetTx(rw.chainTx) } @@ -447,7 +461,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask Task) *TxResult { // the coinbase race investigation). rw.SetReader(state.NewHistoryReaderV3WithSharedDomains(rw.chainTx, rw.rs.Domains(), txTask.Version().TxNum)) } else if !txTask.IsHistoric() && (rw.stateReader == nil || rw.historyMode) { - rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx), nil)) + rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetterMetered(rw.chainTx, rw.readMetrics), nil)) } // Set the per-block committed state cache from the task. @@ -533,7 +547,7 @@ func NewWorkersPool(ctx context.Context, accumulator *shards.Accumulator, backgr reader := stateReader if reader == nil { - reader = state.NewReaderV3(rs.Domains().AsGetterNoMetrics(nil)) + reader = state.NewReaderV3(rs.Domains().AsGetterMetered(nil, reconWorkers[i].readMetrics)) } if err = reconWorkers[i].ResetState(rs, nil, reader, stateWriter, accumulator); err != nil { From bd6097ec8f61aed3a294bdd57eb72d7aa10ed70e Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sun, 7 Jun 2026 09:19:59 +0000 Subject: [PATCH 116/167] execution/cache: fix off-by-one in state-cache unwind floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state-cache unwind set its invalidation floor to unwindToTxNum = Min(UnwindPoint+1) — the first txNum of the first rolled-back block — but the staleness check dropped entries only when txNum strictly exceeded the floor (txNum > floor). An entry stamped at exactly the floor therefore survived, even though that txNum belongs to a dead (unwound) block. This is the EIP-4788 beacon-root case: the beacon-root storage write runs in a block's begin-system-tx, stamped at the block's first txNum. When that block is the first one unwound by a reorg, its txNum equals the floor, so the dead-fork value was served stale during sidechain re-execution, yielding a wrong trie root (hive engine-cancun "Sidechain Reorg"). Drop entries at-or-above the floor (txNum >= floor). The surviving block's last txNum is floor-1, so this never evicts a live entry. Add a boundary regression test (TestUnwind_EvictsEntryAtFloor) covering txNum == floor. Co-Authored-By: Claude Opus 4.8 (1M context) --- execution/cache/cache_test.go | 26 ++++++++++++++++++++++++- execution/cache/generic_cache.go | 33 +++++++++++++++++++------------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index ca83eb8b5a9..7c3bd977e13 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -689,7 +689,7 @@ func TestUnwind_KeepsBelowFloor_EvictsAbove(t *testing.T) { c.Put(below, makeValue(1), 50) // predates the unwind c.Put(above, makeValue(2), 150) // written in the unwound range - c.Unwind(100) // keep <=100, drop >100 + c.Unwind(100) // floor=100 (first unwound txNum): keep <100, drop >=100 v, ok := c.Get(below) assert.True(t, ok, "entry below the unwind point must stay warm") @@ -700,6 +700,30 @@ func TestUnwind_KeepsBelowFloor_EvictsAbove(t *testing.T) { assert.Equal(t, 1, c.Len(), "the stale entry is evicted lazily on its read") } +// Boundary regression: unwindToTxNum is the FIRST rolled-back txNum +// (Min(UnwindPoint+1)), so an entry stamped at exactly that txNum belongs to the +// first dead block and must be evicted — the drop rule is txNum>=floor, not +// txNum>floor. This reproduces the Sidechain Reorg wrong-root bug: an EIP-4788 +// beacon-root storage write runs in a block's begin-system-tx, stamped at the +// block's first txNum; when that block is the first unwound one, txNum==floor and +// a strict `>` left the dead-fork value served stale. +func TestUnwind_EvictsEntryAtFloor(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + atFloor := makeAddr(1) + belowFloor := makeAddr(2) + c.Put(atFloor, makeValue(1), 100) // first txNum of the first unwound block + c.Put(belowFloor, makeValue(2), 99) // last txNum of the surviving block + + c.Unwind(100) // floor=100, epoch->1 + + _, ok := c.Get(atFloor) + assert.False(t, ok, "entry at txNum==floor is on the dead fork and must be evicted") + + v, ok := c.Get(belowFloor) + assert.True(t, ok, "entry at txNum==floor-1 predates the unwind and must stay warm") + assert.Equal(t, makeValue(2), v) +} + // The reused-txNum case: after an unwind, the live fork re-writes a key at the // SAME txNum as the dead fork's write. The epoch — not the txNum — distinguishes // them, so the dead entry reads stale and the re-written one reads valid. diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 0119e855d68..7280f74302f 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -60,9 +60,10 @@ type GenericCache[T any] struct { // Cache coherence across unwinds is txNum/epoch based — no block awareness, // no diffset. An entry is valid iff it was written in the current epoch OR - // its txNum is at/below unwindFloor (so it predates every unwind seen and - // can't be stale). Unwind bumps the epoch and lowers the floor (O(1), no - // scan); stale entries are dropped lazily on their next read. txNum slots + // its txNum is strictly below unwindFloor (the first unwound txNum), so it + // predates every unwind seen and can't be stale. Unwind bumps the epoch and + // lowers the floor (O(1), no scan); stale entries are dropped lazily on their + // next read. txNum slots // are reused across forks, so the epoch — not the txNum — is what tells a // dead fork's write apart from the live fork's at the same txNum. epoch atomic.Uint32 @@ -170,9 +171,14 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { return zero, false } // Lazy unwind invalidation: an entry from a superseded epoch whose txNum is - // above the unwind floor reflects dead-fork state — drop it and miss so the - // read falls through to the reverted domain and repopulates. - if e.epoch != c.epoch.Load() && e.txNum > c.unwindFloor.Load() { + // at or above the unwind floor reflects dead-fork state — drop it and miss so + // the read falls through to the reverted domain and repopulates. The floor is + // the first unwound txNum (Min(UnwindPoint+1), the first txNum of the first + // rolled-back block), so an entry stamped exactly at the floor belongs to a + // dead block — e.g. an EIP-4788 beacon-root write in the block-begin system + // tx — and must be dropped; >= not > (the surviving block's last txNum is + // floor-1, so this never drops a live entry). + if e.epoch != c.epoch.Load() && e.txNum >= c.unwindFloor.Load() { c.data.Remove(h) c.staleEvicted.Add(1) c.misses.Add(1) @@ -234,13 +240,14 @@ func (c *GenericCache[T]) Clear() { c.currentSize.Store(0) } -// Unwind invalidates entries that reflect state above unwindToTxNum on a -// now-dead fork. O(1): bump the epoch (so entries written in the new, live -// epoch stay valid) and lower the floor to the unwind point (so entries at or -// below it — which predate the unwind and can't be stale — stay warm). Stale -// entries (old epoch, txNum above the floor) are dropped lazily on their next -// read. The floor only ever moves down, to the deepest unwind seen, so a later -// shallower unwind can't resurrect entries an earlier deeper one invalidated. +// Unwind invalidates entries that reflect dead-fork state. unwindToTxNum is the +// first rolled-back txNum (Min(UnwindPoint+1)); every entry at or above it is on +// the dead fork. O(1): bump the epoch (so entries written in the new, live epoch +// stay valid) and lower the floor to unwindToTxNum (so entries strictly below it +// — which predate the unwind and can't be stale — stay warm). Stale entries (old +// epoch, txNum >= floor) are dropped lazily on their next read. The floor only +// ever moves down, to the deepest unwind seen, so a later shallower unwind can't +// resurrect entries an earlier deeper one invalidated. func (c *GenericCache[T]) Unwind(unwindToTxNum uint64) { c.epoch.Add(1) for { From 67d0f33b552800cbc657db5036aafb6216e89ed9 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sun, 7 Jun 2026 19:52:03 +0000 Subject: [PATCH 117/167] execution: address #21380 review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engineapitester: remove the now-orphaned EngineApiTester.ChainDB field (its only consumer — the EngineXTestRunner cache-clear — was removed in 4618add1c0) plus its now-unused kv import - stagedsync/exec3, exec3_parallel: drop comments that describe code removed from main (the warmup-cache-disable workaround), per review - execmodule: add TestValidateForkPayloadOffNonTipCanonicalBlockWithCache, ExecModuleTester coverage for the unwind/fork-payload caching path — a fork off a NON-tip canonical block exercises the partial unwind + parent-link diffset masking of the BranchCache, round-tripped in both directions. The existing side-fork test only branches at genesis (full unwind). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../engineapitester/engine_api_tester.go | 10 +-- execution/execmodule/exec_module_test.go | 73 +++++++++++++++++++ execution/stagedsync/exec3.go | 8 -- execution/stagedsync/exec3_parallel.go | 8 -- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/execution/engineapi/engineapitester/engine_api_tester.go b/execution/engineapi/engineapitester/engine_api_tester.go index 407d1583b1f..15f05b82b3f 100644 --- a/execution/engineapi/engineapitester/engine_api_tester.go +++ b/execution/engineapi/engineapitester/engine_api_tester.go @@ -39,7 +39,6 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain" @@ -397,7 +396,6 @@ func InitialiseEngineApiTester(ctx context.Context, args EngineApiTesterInitArgs TxnInclusionVerifier: NewTxnInclusionVerifier(rpcApiClient), Node: ethNode, NodeKey: nodeKey, - ChainDB: ethBackend.ChainKV(), cleanup: cleanup, }, nil } @@ -428,13 +426,7 @@ type EngineApiTester struct { TxnInclusionVerifier TxnInclusionVerifier Node *node.Node NodeKey *ecdsa.PrivateKey - // ChainDB is the running backend's temporal DB. Retained on the tester so - // callers (e.g. EngineXTestRunner.Run between tests in the same group) can - // reach the aggregator's BranchCache and clear it without rebuilding the - // whole tester. The reference must NOT be Closed by callers — the tester's - // cleanup owns the lifecycle. - ChainDB kv.RwDB - cleanup *cleanupHandle + cleanup *cleanupHandle } func (eat EngineApiTester) Run(t *testing.T, test func(ctx context.Context, t *testing.T, eat EngineApiTester)) { diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 1deb5178cd6..00ae54e4eb5 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -178,6 +178,79 @@ func TestValidateChainAndUpdateForkChoiceWithSideForksThatGoBackAndForwardInHeig require.NoError(t, err) } +// TestValidateForkPayloadOffNonTipCanonicalBlockWithCache exercises the +// unwind/fork-payload caching path in ExecModule.ValidateChain. A fork that +// branches off a NON-tip canonical block forces validateChain to +// unwindToCommonCanonical (unwinding only the canonical blocks above the branch +// point), chain the validation SharedDomains to e.currentContext so the unwound +// blocks' diffsets are reachable via the parent link, and mask the BranchCache +// with the resulting unwind set. If any of that is wrong the re-executed fork +// produces a wrong trie root, so a Success validation status is the correctness +// assertion. The existing side-fork test only branches at genesis (full unwind); +// this covers the partial-unwind round trip the ValidateChain parent-link +// comment guards. +func TestValidateForkPayloadOffNonTipCanonicalBlockWithCache(t *testing.T) { + privKey, err := crypto.GenerateKey() + require.NoError(t, err) + senderAddr := crypto.PubkeyToAddress(privKey.PublicKey) + genesis := &types.Genesis{ + Config: chain.AllProtocolChanges, + Alloc: types.GenesisAlloc{ + senderAddr: {Balance: new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)}, + }, + } + m := execmoduletester.New(t, execmoduletester.WithGenesisSpec(genesis), execmoduletester.WithKey(privKey)) + to := common.Address{0xaa} + baseFee := m.Genesis.BaseFee().Uint64() + mkTx := func(nonce, amount uint64) types.Transaction { + tx, err := types.SignTx( + types.NewTransaction(nonce, to, uint256.NewInt(amount), 50000, uint256.NewInt(baseFee), nil), + *types.LatestSignerForChainID(nil), + privKey, + ) + require.NoError(t, err) + return tx + } + + // blockgen reads the latest committed DB state, so each segment must be + // generated while the chain head sits at its intended parent. Build and + // finalize the shared prefix genesis → 1 → 2 first (nonces 0,1), each block + // carrying a state-touching txn so the BranchCache + commitment hold real + // entries. After this the head is block 2. + prefix, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 2, func(i int, b *blockgen.BlockGen) { + b.AddTx(mkTx(uint64(i), 1_000)) + }) + require.NoError(t, err) + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, prefix.Blocks)) + block2 := prefix.Blocks[1] + + // With the head at block 2 (signer nonce 2), generate both continuations off + // block 2 before inserting either: the canonical tip (block 3, nonce 2) and a + // longer fork (blocks 3',4' at nonces 2,3) that branches off the same non-tip + // block 2. + canonicalTip, err := blockgen.GenerateChain(m.ChainConfig, block2, m.Engine, m.DB, 1, func(i int, b *blockgen.BlockGen) { + b.AddTx(mkTx(2, 1_000)) + }) + require.NoError(t, err) + fork, err := blockgen.GenerateChain(m.ChainConfig, block2, m.Engine, m.DB, 2, func(i int, b *blockgen.BlockGen) { + b.AddTx(mkTx(uint64(2+i), 2_000)) + }) + require.NoError(t, err) + + // Finalize the canonical tip (head → block 3). + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, canonicalTip.Blocks)) + + // Validating fork.Blocks[0] (height 3, parent = block 2) must unwind canonical + // block 3 back to block 2; fork.Blocks[1] then head-extends and the FCU reorgs + // onto the longer fork. + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, fork.Blocks)) + + // Reorg back onto the original canonical block 3 (same common ancestor, + // block 2) to exercise the BranchCache masking in the other direction: + // unwind the fork blocks and re-validate canonical block 3 off block 2. + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, canonicalTip.Blocks)) +} + func addTwoTxnsToPool(ctx context.Context, startingNonce uint64, t *testing.T, m *execmoduletester.ExecModuleTester, txpool txpoolproto.TxpoolServer, baseFee uint64) { tx2, err := types.SignTx(types.NewTransaction(startingNonce, common.Address{1}, uint256.NewInt(10_000), params.TxGas, uint256.NewInt(baseFee), nil), *types.LatestSignerForChainID(m.ChainConfig.ChainID), m.Key) require.NoError(t, err) diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 5d2e668794c..af559768993 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -205,14 +205,6 @@ func ExecV3(ctx context.Context, doms.EnableParaTrieDB(cfg.db) doms.EnableTrieWarmup(true) - // DO NOT re-introduce a warmup-cache disable here without working through - // the unwind caching scenarios. main carried a short-term workaround - // (`doms.EnableWarmupCache(!isApplyingBlocks && !parallel)`) that disabled - // the trie warmuper's value cache on the parallel path to dodge a - // stale-cache wrong-root. THIS PR is the proper fix for that wrong-root: - // caching stays enabled, the BranchCache + SD chain handle multi-block - // uncommitted batches and fork validation correctly. The re-org tests are - // the gate that confirms this. doms.SetDeferCommitmentUpdates(false) // Enable deferred commitment updates for fork validation and parallel initial sync. // Deferred updates batch commitment calculations to block boundaries rather than diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 0b5a18f89b8..77530161b86 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -217,14 +217,6 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U pe.rs.Domains().SetInMemHistoryReads(true) defer pe.rs.Domains().SetInMemHistoryReads(prevInMemHistoryReads) - // Trie warmup left enabled for the parallel path. Original disable was - // based on a calculator/warmer interaction concern that turned out to be - // overly conservative — the Warmuper's reads are independent of the - // calculator's SetUpdates call. Removing the disable produced an 8× - // throughput improvement on the perf-devnet-3 SSTORE-bloated benchmark - // (block 24358306) by letting the Warmuper pre-fetch branch data while - // EVM execution runs. See #20920 for the canonical perf measurement. - // Skip step-boundary commitment — the calculator handles this. pe.rs.StateV3.SetSkipStepBoundaryCommitment(true) defer pe.rs.StateV3.SetSkipStepBoundaryCommitment(false) From 12e9a88855a19821088d7000aa08862e2ac049a6 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 10:23:00 +0000 Subject: [PATCH 118/167] execmodule, cl/beacon: route bg-commit reads via published SD (WIP) - execmoduletester.InsertChain verifies header/stage-progress/head-hash through the published SharedDomains overlay (BlockOverlayTemporalTx) instead of raw DB, so harness postconditions hold under async background commit. Benefits all harness-based packages. - cl/beacon block_production_test tip blockgen calls pass m.PublishedSD(). - cl/beacon/handler now green under fcuBackgroundCommit=true default. WIP toward full-codebase bg-commit readiness; NOT pushed (branch still red: remaining blockgen tip callers, deeper execmodule FCU tests, rpcdaemon/hive). --- cl/beacon/handler/block_production_test.go | 4 ++-- .../execmoduletester/exec_module_tester.go | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cl/beacon/handler/block_production_test.go b/cl/beacon/handler/block_production_test.go index a0e8399a118..9b30d97a17c 100644 --- a/cl/beacon/handler/block_production_test.go +++ b/cl/beacon/handler/block_production_test.go @@ -284,7 +284,7 @@ func TestCaplinBlockProductionWithWithdrawalRequest(t *testing.T) { ) require.NoError(t, err) gen.AddTx(tx) - }) + }, m.PublishedSD()) require.NoError(t, err) err = m.InsertChain(chainPack) require.NoError(t, err) @@ -416,7 +416,7 @@ func TestCaplinBlockProductionGlamsterdamSlotNumber(t *testing.T) { ) require.NoError(t, err) gen.AddTx(tx) - }) + }, m.PublishedSD()) require.NoError(t, err) err = m.InsertChain(chainPack) require.NoError(t, err) diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 66ad426c0a5..3d84d210d98 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -925,11 +925,20 @@ func (emt *ExecModuleTester) InsertChain(chain *blockgen.ChainPack) error { if err := emt.insertPoSBlocks(chain); err != nil { return err } - roTx, err := emt.DB.BeginRo(emt.Ctx) + baseTx, err := emt.DB.BeginTemporalRo(emt.Ctx) if err != nil { return err } - defer roTx.Rollback() + defer baseTx.Rollback() + // Under background commit the just-inserted block metadata (header, stage + // progress, head hash) lives in the published SharedDomains overlay before + // it lands in the raw DB, so read through the overlay when one is published. + var roTx kv.Tx = baseTx + if sd := emt.PublishedSD(); sd != nil { + if v := sd.BlockOverlayTemporalTx(baseTx); v != nil { + roTx = v + } + } // Check if the latest header was imported or rolled back if rawdb.ReadHeader(roTx, chain.TopBlock.Hash(), chain.TopBlock.NumberU64()) == nil { return fmt.Errorf("did not import block %d %x", chain.TopBlock.NumberU64(), chain.TopBlock.Hash()) From d1350b77175c5cdfd00c74132bb0f797563fd49a Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 12:19:52 +0000 Subject: [PATCH 119/167] execution, rpc, cmd/rpcdaemon: route bg-commit test reads via published SD Add ExecModuleTester.OverlayDB(): a TemporalRoDB whose txns route consensus-table reads through the published block overlay and latest domain-state reads through the published SharedDomains (AsGetter/GetLatest), so RPC-daemon and SimulatedBackend readers observe the tip under background commit instead of a lagging raw DB. Route the rpc/jsonrpc, gasprice and rpcdaemontest API constructors + SimulatedBackend reads through it, and pass PublishedSD() to tip-build blockgen.GenerateChain calls. Fix an intermittent harness hang: ExecModuleTester.Close now drains and stops the background-commit worker (WaitIdle) before closing the DB, so DB.Close can no longer race an in-flight commit. --- cmd/rpcdaemon/rpcdaemontest/test_util.go | 12 +- execution/abi/bind/backends/simulated.go | 36 +++--- .../execmoduletester/exec_module_tester.go | 7 ++ .../execmoduletester/overlay_read.go | 110 ++++++++++++++++++ rpc/gasprice/bench_test.go | 2 +- rpc/gasprice/feehistory_test.go | 4 +- rpc/gasprice/gasprice_test.go | 12 +- rpc/jsonrpc/call_traces_test.go | 18 +-- rpc/jsonrpc/corner_cases_support_test.go | 4 +- rpc/jsonrpc/debug_api_test.go | 42 +++---- rpc/jsonrpc/erigon_receipts_test.go | 28 ++--- rpc/jsonrpc/eth_api_test.go | 46 ++++---- rpc/jsonrpc/eth_block_test.go | 26 ++--- rpc/jsonrpc/eth_callMany_test.go | 4 +- rpc/jsonrpc/eth_call_test.go | 22 ++-- rpc/jsonrpc/eth_fill_transaction_test.go | 28 ++--- rpc/jsonrpc/eth_filters_test.go | 14 +-- rpc/jsonrpc/eth_simulation_test.go | 2 +- rpc/jsonrpc/eth_subscribe_test.go | 4 +- rpc/jsonrpc/eth_system_test.go | 24 ++-- rpc/jsonrpc/gen_traces_test.go | 6 +- .../otterscan_contract_creator_test.go | 2 +- rpc/jsonrpc/otterscan_search_backward_test.go | 2 +- rpc/jsonrpc/otterscan_search_forward_test.go | 2 +- ...an_transaction_by_sender_and_nonce_test.go | 2 +- rpc/jsonrpc/overlay_race_test.go | 10 +- rpc/jsonrpc/receipts/handler_test.go | 2 +- rpc/jsonrpc/send_transaction_test.go | 6 +- rpc/jsonrpc/trace_adhoc_test.go | 30 ++--- rpc/jsonrpc/trace_filtering_test.go | 26 ++--- rpc/jsonrpc/txpool_api_test.go | 4 +- 31 files changed, 328 insertions(+), 209 deletions(-) create mode 100644 execution/execmodule/execmoduletester/overlay_read.go diff --git a/cmd/rpcdaemon/rpcdaemontest/test_util.go b/cmd/rpcdaemon/rpcdaemontest/test_util.go index c47e3d1f014..0492be6c8fc 100644 --- a/cmd/rpcdaemon/rpcdaemontest/test_util.go +++ b/cmd/rpcdaemon/rpcdaemontest/test_util.go @@ -37,6 +37,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/u256" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/abi/bind" "github.com/erigontech/erigon/execution/abi/bind/backends" "github.com/erigontech/erigon/execution/builder" @@ -113,11 +114,11 @@ func genTestChainOnce(t *testing.T) { defer contractBackend.Close() var err error - testOrphanedChain, err = blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 5, func(i int, block *blockgen.BlockGen) {}) + testOrphanedChain, err = blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 5, func(i int, block *blockgen.BlockGen) {}, m.PublishedSD()) if err != nil { t.Fatalf("rpcdaemontest: failed to generate orphaned chain: %v", err) } - testChain, err = generateChain(&addresses, m.ChainConfig, m.Genesis, m.Engine, m.DB, contractBackend) + testChain, err = generateChain(&addresses, m.ChainConfig, m.Genesis, m.Engine, m.DB, contractBackend, m.PublishedSD()) if err != nil { t.Fatalf("rpcdaemontest: failed to generate chain: %v", err) } @@ -156,6 +157,7 @@ func generateChain( engine rules.Engine, db kv.TemporalRwDB, contractBackend *backends.SimulatedBackend, + readSD ...*execctx.SharedDomains, ) (*blockgen.ChainPack, error) { var ( key = addresses.key @@ -339,7 +341,7 @@ func generateChain( block.AddTx(txn) } contractBackend.Commit() - }) + }, readSD...) } func computeMappingStorageKey(addr common.Address, slot uint64) common.Hash { @@ -559,7 +561,7 @@ func CreateTestExecModuleForTraces(t *testing.T) *execmoduletester.ExecModuleTes tx, _ := types.SignTx(types.NewTransaction(0, a2, &u256.Num0, 50000, &u256.Num1, []byte{0x01, 0x00, 0x01, 0x00}), *types.LatestSignerForChainID(nil), key) b.AddTx(tx) - }) + }, m.PublishedSD()) if err != nil { t.Fatalf("generate blocks: %v", err) } @@ -668,7 +670,7 @@ func CreateTestExecModuleForTracesCollision(t *testing.T) *execmoduletester.Exec tx, _ = types.SignTx(types.NewTransaction(2, bb, &u256.Num0, 100000, &u256.Num1, nil), *types.LatestSignerForChainID(nil), key) b.AddTx(tx) - }) + }, m.PublishedSD()) if err != nil { t.Fatalf("generate blocks: %v", err) } diff --git a/execution/abi/bind/backends/simulated.go b/execution/abi/bind/backends/simulated.go index db981b30b35..8d5e65c4e6f 100644 --- a/execution/abi/bind/backends/simulated.go +++ b/execution/abi/bind/backends/simulated.go @@ -109,7 +109,7 @@ func NewSimulatedBackendWithConfig(t *testing.T, alloc types.GenesisAlloc, confi m: m, prependBlock: m.Genesis, getHeader: func(hash common.Hash, number uint64) (h *types.Header, err error) { - err = m.DB.View(context.Background(), func(tx kv.Tx) error { + err = m.OverlayDB().View(context.Background(), func(tx kv.Tx) error { h, err = m.BlockReader.Header(context.Background(), tx, hash, number) return nil }) @@ -173,7 +173,7 @@ func (b *SimulatedBackend) Rollback() { } func (b *SimulatedBackend) emptyPendingBlock() { - blockChain, _ := blockgen.GenerateChain(b.m.ChainConfig, b.prependBlock, b.m.Engine, b.m.DB, 1, func(int, *blockgen.BlockGen) {}) + blockChain, _ := blockgen.GenerateChain(b.m.ChainConfig, b.prependBlock, b.m.Engine, b.m.DB, 1, func(int, *blockgen.BlockGen) {}, b.m.PublishedSD()) b.pendingBlock = blockChain.Blocks[0] b.pendingReceipts = blockChain.Receipts[0] b.pendingHeader = blockChain.Headers[0] @@ -182,7 +182,7 @@ func (b *SimulatedBackend) emptyPendingBlock() { if b.pendingReaderTx != nil { b.pendingReaderTx.Rollback() } - tx, err := b.m.DB.BeginTemporalRo(context.Background()) //nolint:gocritic + tx, err := b.m.OverlayDB().BeginTemporalRo(context.Background()) //nolint:gocritic if err != nil { panic(err) } @@ -203,7 +203,7 @@ func (b *SimulatedBackend) stateByBlockNumber(db kv.TemporalTx, blockNumber *uin func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *uint256.Int) ([]byte, error) { b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginTemporalRo(context.Background()) + tx, err := b.m.OverlayDB().BeginTemporalRo(context.Background()) if err != nil { return nil, err } @@ -216,7 +216,7 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address, func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Address, blockNumber *uint256.Int) (*uint256.Int, error) { b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginTemporalRo(context.Background()) + tx, err := b.m.OverlayDB().BeginTemporalRo(context.Background()) if err != nil { return nil, err } @@ -230,7 +230,7 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address, blockNumber *uint256.Int) (uint64, error) { b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginTemporalRo(context.Background()) + tx, err := b.m.OverlayDB().BeginTemporalRo(context.Background()) if err != nil { return 0, err } @@ -244,7 +244,7 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address, func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Address, key common.Hash, blockNumber *uint256.Int) ([]byte, error) { b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginTemporalRo(context.Background()) + tx, err := b.m.OverlayDB().BeginTemporalRo(context.Background()) if err != nil { return nil, err } @@ -263,7 +263,7 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginTemporalRo(context.Background()) + tx, err := b.m.OverlayDB().BeginTemporalRo(context.Background()) if err != nil { return nil, err } @@ -307,7 +307,7 @@ func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common. b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginRo(ctx) + tx, err := b.m.OverlayDB().BeginRo(ctx) if err != nil { return nil, false, err } @@ -354,7 +354,7 @@ func (b *SimulatedBackend) BlockByHash(ctx context.Context, hash common.Hash) (* if hash == b.pendingBlock.Hash() { return b.pendingBlock, nil } - tx, err := b.m.DB.BeginRo(ctx) + tx, err := b.m.OverlayDB().BeginRo(ctx) if err != nil { return nil, err } @@ -388,7 +388,7 @@ func (b *SimulatedBackend) blockByNumberNoLock(ctx context.Context, number *uint return b.prependBlock, nil } - tx, err := b.m.DB.BeginRo(context.Background()) + tx, err := b.m.OverlayDB().BeginRo(context.Background()) if err != nil { return nil, err } @@ -413,7 +413,7 @@ func (b *SimulatedBackend) HeaderByHash(ctx context.Context, hash common.Hash) ( if hash == b.pendingBlock.Hash() { return b.pendingBlock.Header(), nil } - tx, err := b.m.DB.BeginRo(context.Background()) + tx, err := b.m.OverlayDB().BeginRo(context.Background()) if err != nil { return nil, err } @@ -439,7 +439,7 @@ func (b *SimulatedBackend) HeaderByHash(ctx context.Context, hash common.Hash) ( func (b *SimulatedBackend) HeaderByNumber(ctx context.Context, number *uint256.Int) (*types.Header, error) { b.mu.Lock() defer b.mu.Unlock() - tx, err := b.m.DB.BeginRo(context.Background()) + tx, err := b.m.OverlayDB().BeginRo(context.Background()) if err != nil { return nil, err } @@ -464,7 +464,7 @@ func (b *SimulatedBackend) TransactionCount(ctx context.Context, blockHash commo if blockHash == b.pendingBlock.Hash() { return uint(b.pendingBlock.Transactions().Len()), nil } - tx, err := b.m.DB.BeginRo(context.Background()) + tx, err := b.m.OverlayDB().BeginRo(context.Background()) if err != nil { return 0, err } @@ -498,7 +498,7 @@ func (b *SimulatedBackend) TransactionInBlock(ctx context.Context, blockHash com return transactions[index], nil } - tx, err := b.m.DB.BeginRo(context.Background()) + tx, err := b.m.OverlayDB().BeginRo(context.Background()) if err != nil { return nil, err } @@ -572,7 +572,7 @@ func (b *SimulatedBackend) CallContract(ctx context.Context, call bind.CallMsg, return nil, errBlockNumberUnsupported } var res *evmtypes.ExecutionResult - if err := b.m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) (err error) { + if err := b.m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) (err error) { s := state.New(b.m.NewStateReader(tx)) res, err = b.callContract(ctx, call, b.pendingBlock, s) if err != nil { @@ -814,7 +814,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, txn types.Transa block.AddTxWithChain(b.getHeader, b.m.Engine, txn) } block.AddTxWithChain(b.getHeader, b.m.Engine, txn) - }) + }, b.m.PublishedSD()) if err != nil { return err } @@ -859,7 +859,7 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { block.AddTxWithChain(b.getHeader, b.m.Engine, txn) } block.OffsetTime(int64(adjustment.Seconds())) - }) + }, b.m.PublishedSD()) if err != nil { return err } diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 3d84d210d98..8ae0b988f05 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -148,6 +148,13 @@ type ExecModuleTester struct { } func (emt *ExecModuleTester) Close() { + // Drain and stop the background-commit worker (and its in-flight commit tx) + // before closing the DB, mirroring production shutdown. Otherwise DB.Close + // can race a commit still writing, which deadlocks. Must run before cancel() + // so WaitIdle can still acquire the foreground semaphore. + if emt.ExecModule != nil { + emt.ExecModule.WaitIdle(emt.Ctx) + } emt.cancel() if err := emt.bgComponentsEg.Wait(); err != nil && emt.tb != nil { require.Equal(emt.tb, context.Canceled, err) // upon waiting for clean exit we should get ctx cancelled diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go new file mode 100644 index 00000000000..7b899050d50 --- /dev/null +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -0,0 +1,110 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execmoduletester + +import ( + "context" + + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" +) + +// OverlayDB returns a read-only DB whose transactions route reads through the +// latest published SharedDomains: consensus-table reads see the block overlay +// and domain-state reads see the in-flight (not-yet-committed) values. Pass it +// to RPC-daemon APIs so their reads observe the tip under background commit +// instead of a lagging raw DB. +func (emt *ExecModuleTester) OverlayDB() kv.TemporalRoDB { + return &sdRoDB{TemporalRoDB: emt.DB, publishedSD: emt.PublishedSD} +} + +type sdRoDB struct { + kv.TemporalRoDB + publishedSD func() *execctx.SharedDomains +} + +func (d *sdRoDB) BeginTemporalRo(ctx context.Context) (kv.TemporalTx, error) { + base, err := d.TemporalRoDB.BeginTemporalRo(ctx) + if err != nil { + return nil, err + } + return wrapSDTx(base, d.publishedSD()), nil +} + +func (d *sdRoDB) BeginRo(ctx context.Context) (kv.Tx, error) { + return d.BeginTemporalRo(ctx) +} + +func (d *sdRoDB) ViewTemporal(ctx context.Context, f func(tx kv.TemporalTx) error) error { + tx, err := d.BeginTemporalRo(ctx) + if err != nil { + return err + } + defer tx.Rollback() + return f(tx) +} + +func (d *sdRoDB) View(ctx context.Context, f func(tx kv.Tx) error) error { + tx, err := d.BeginTemporalRo(ctx) + if err != nil { + return err + } + defer tx.Rollback() + return f(tx) +} + +// wrapSDTx layers domain- and table-level read-through onto base for the given +// published SD. With no published SD it returns base unchanged. +func wrapSDTx(base kv.TemporalTx, sd *execctx.SharedDomains) kv.TemporalTx { + if sd == nil { + return base + } + read := base + if v := sd.BlockOverlayTemporalTx(base); v != nil { + read = v + } + return &sdRoTx{TemporalTx: read, base: base, sd: sd} +} + +// sdRoTx serves consensus-table reads from the block overlay (the embedded +// TemporalTx) and latest domain-state reads from the published SD's in-flight +// chain. Historical (GetAsOf) reads try the in-flight mem first, then fall back +// to committed state on the overlay tx. +type sdRoTx struct { + kv.TemporalTx + base kv.TemporalTx + sd *execctx.SharedDomains +} + +func (t *sdRoTx) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { + return t.sd.GetLatest(name, t.base, k) +} + +func (t *sdRoTx) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + return t.sd.HasPrefix(name, prefix, t.base) +} + +func (t *sdRoTx) GetAsOf(name kv.Domain, k []byte, ts uint64) ([]byte, bool, error) { + if v, ok, err := t.sd.GetAsOf(name, k, ts); err != nil || ok { + return v, ok, err + } + return t.TemporalTx.GetAsOf(name, k, ts) +} + +func (t *sdRoTx) Rollback() { + t.base.Rollback() +} diff --git a/rpc/gasprice/bench_test.go b/rpc/gasprice/bench_test.go index e623faf67b2..11a21e84182 100644 --- a/rpc/gasprice/bench_test.go +++ b/rpc/gasprice/bench_test.go @@ -73,7 +73,7 @@ func newTestBackendN(tb testing.TB, n int) *execmoduletester.ExecModuleTester { } b.AddTx(tx) } - }) + }, m.PublishedSD()) require.NoError(tb, err) require.NoError(tb, m.InsertChain(ch)) return m diff --git a/rpc/gasprice/feehistory_test.go b/rpc/gasprice/feehistory_test.go index a25f6c8883a..ed76df17da8 100644 --- a/rpc/gasprice/feehistory_test.go +++ b/rpc/gasprice/feehistory_test.go @@ -81,12 +81,12 @@ func TestFeeHistory(t *testing.T) { defer m.Close() baseApi := jsonrpc.NewBaseApi(nil, kvcache.NewSimple(), m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - tx, err := m.DB.BeginTemporalRo(m.Ctx) + tx, err := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() cache := jsonrpc.NewGasPriceCache() - oracle := gasprice.NewOracle(jsonrpc.NewGasPriceOracleBackend(m.DB, tx, baseApi), config, cache, gasprice.NewFeeHistoryCache(), log.New()) + oracle := gasprice.NewOracle(jsonrpc.NewGasPriceOracleBackend(m.OverlayDB(), tx, baseApi), config, cache, gasprice.NewFeeHistoryCache(), log.New()) first, reward, baseFee, ratio, blobBaseFee, blobBaseFeeRatio, err := oracle.FeeHistory(context.Background(), c.count, c.last, c.percent) diff --git a/rpc/gasprice/gasprice_test.go b/rpc/gasprice/gasprice_test.go index d57c80cb253..7f0ff8c8abd 100644 --- a/rpc/gasprice/gasprice_test.go +++ b/rpc/gasprice/gasprice_test.go @@ -66,7 +66,7 @@ func newTestBackend(t *testing.T) *execmoduletester.ExecModuleTester { t.Fatalf("failed to create tx: %v", txErr) } b.AddTx(tx) - }) + }, m.PublishedSD()) if err != nil { t.Error(err) } @@ -90,7 +90,7 @@ func TestSuggestPrice(t *testing.T) { m := newTestBackend(t) //, big.NewInt(16), c.pending) baseApi := jsonrpc.NewBaseApi(nil, kvcache.NewSimple(), m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - tx, err := m.DB.BeginTemporalRo(m.Ctx) + tx, err := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() @@ -384,7 +384,7 @@ func TestSuggestTipCap_SparseBlocks(t *testing.T) { } b.AddTx(tx) } - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(ch)) @@ -396,7 +396,7 @@ func TestSuggestTipCap_SparseBlocks(t *testing.T) { baseApi := jsonrpc.NewBaseApi(nil, kvcache.NewSimple(), m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - dbTx, txErr := m.DB.BeginTemporalRo(m.Ctx) + dbTx, txErr := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, txErr) defer dbTx.Rollback() @@ -422,7 +422,7 @@ func TestSuggestTipCap_AllEmptyBlocks(t *testing.T) { ch, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, totalBlocks, func(_ int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) // no transactions — all blocks are empty - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(ch)) @@ -434,7 +434,7 @@ func TestSuggestTipCap_AllEmptyBlocks(t *testing.T) { baseApi := jsonrpc.NewBaseApi(nil, kvcache.NewSimple(), m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - dbTx, txErr := m.DB.BeginTemporalRo(m.Ctx) + dbTx, txErr := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, txErr) defer dbTx.Rollback() diff --git a/rpc/jsonrpc/call_traces_test.go b/rpc/jsonrpc/call_traces_test.go index eec2ad44d0c..3e44997c332 100644 --- a/rpc/jsonrpc/call_traces_test.go +++ b/rpc/jsonrpc/call_traces_test.go @@ -69,12 +69,12 @@ func TestCallTraceOneByOne(t *testing.T) { m := execmoduletester.New(t) chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 10, func(i int, gen *blockgen.BlockGen) { gen.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) if err != nil { t.Fatalf("generate chain: %v", err) } - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Insert blocks 1 by 1 to trigger possible "off by one" errors for i := 0; i < chain.Length(); i++ { if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { @@ -104,7 +104,7 @@ func TestCallTraceUnwind(t *testing.T) { var err error chainA, err = blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 10, func(i int, gen *blockgen.BlockGen) { gen.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) if err != nil { t.Fatalf("generate chainA: %v", err) } @@ -114,12 +114,12 @@ func TestCallTraceUnwind(t *testing.T) { } else { gen.SetCoinbase(common.Address{2}) } - }) + }, m.PublishedSD()) if err != nil { t.Fatalf("generate chainB: %v", err) } - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) if err = m.InsertChain(chainA); err != nil { t.Fatalf("inserting chainA: %v", err) @@ -179,11 +179,11 @@ func TestFilterNoAddresses(t *testing.T) { m := execmoduletester.New(t) chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 10, func(i int, gen *blockgen.BlockGen) { gen.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) if err != nil { t.Fatalf("generate chain: %v", err) } - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Insert blocks 1 by 1 to trigger possible "off by one" errors for i := 0; i < chain.Length(); i++ { if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { @@ -207,7 +207,7 @@ func TestFilterNoAddresses(t *testing.T) { func TestFilterAddressIntersection(t *testing.T) { m := execmoduletester.New(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) toAddress1, toAddress2, other := common.Address{1}, common.Address{2}, common.Address{3} @@ -230,7 +230,7 @@ func TestFilterAddressIntersection(t *testing.T) { t.Fatal(err) } block.AddTx(txn) - }) + }, m.PublishedSD()) require.NoError(t, err, "generate chain") err = m.InsertChain(chain) diff --git a/rpc/jsonrpc/corner_cases_support_test.go b/rpc/jsonrpc/corner_cases_support_test.go index 1642fb42d3c..7edfb05b3e3 100644 --- a/rpc/jsonrpc/corner_cases_support_test.go +++ b/rpc/jsonrpc/corner_cases_support_test.go @@ -35,7 +35,7 @@ func TestNotFoundMustReturnNil(t *testing.T) { } assertions := require.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) ctx := context.Background() a, err := api.GetTransactionByBlockNumberAndIndex(ctx, 10_000, 1) @@ -80,7 +80,7 @@ func TestNotFoundMustReturnNil(t *testing.T) { func TestNotFoundMustReturnError(t *testing.T) { assertions := require.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) ctx := context.Background() a, err := api.GetBalance(ctx, common.Address{}, bnhPtr(rpc.BlockNumberOrHashWithNumber(10_000))) diff --git a/rpc/jsonrpc/debug_api_test.go b/rpc/jsonrpc/debug_api_test.go index 57b069d1eb5..454ae650631 100644 --- a/rpc/jsonrpc/debug_api_test.go +++ b/rpc/jsonrpc/debug_api_test.go @@ -92,8 +92,8 @@ func TestTraceBlockByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) baseApi := NewBaseApi(nil, stateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - ethApi := newEthApiForTest(baseApi, m.DB, nil, nil) - api := NewPrivateDebugAPI(baseApi, m.DB, nil, 0, false) + ethApi := newEthApiForTest(baseApi, m.OverlayDB(), nil, nil) + api := NewPrivateDebugAPI(baseApi, m.OverlayDB(), nil, 0, false) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer s := jsonstream.New(jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096)) @@ -143,8 +143,8 @@ func TestTraceBlockByNumber(t *testing.T) { func TestTraceBlockByHash(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - ethApi := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + ethApi := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer s := jsonstream.New(jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096)) @@ -175,7 +175,7 @@ func TestTraceBlockByHash(t *testing.T) { func TestTraceTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer s := jsonstream.New(jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096)) @@ -204,7 +204,7 @@ func TestTraceTransaction(t *testing.T) { func TestTraceTransactionNotFound(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) var buf bytes.Buffer s := jsonstream.New(jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096)) @@ -217,7 +217,7 @@ func TestTraceTransactionNotFound(t *testing.T) { // the "result" field when the stream is untouched, producing {error:...} without result:null. func TestTraceErrorPathsWriteNoStream(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) newStream := func() (*bytes.Buffer, jsonstream.Stream) { var buf bytes.Buffer @@ -301,7 +301,7 @@ func TestTraceErrorPathsWriteNoStream(t *testing.T) { } func (c *baseFeeTestChain) debugAPI() *DebugAPIImpl { - return NewPrivateDebugAPI(newBaseApiForTest(c.m), c.m.DB, nil, 0, false) + return NewPrivateDebugAPI(newBaseApiForTest(c.m), c.m.OverlayDB(), nil, 0, false) } // callDebugTraceCall invokes debug_traceCall with the given args and @@ -401,8 +401,8 @@ func TestTxResultFieldStreamLazy(t *testing.T) { // before any write (inner.Written stays false): each tx object has "error" but no "result" field. func TestTraceBlockErrorBeforeWrite(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) - ethApi := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) + ethApi := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) tx, err := ethApi.GetTransactionByHash(m.Ctx, common.HexToHash(debugTraceTransactionTests[0].txHash)) require.NoError(t, err) @@ -476,7 +476,7 @@ func TestTraceBlockErrorAfterWrite(t *testing.T) { func TestTraceTransactionNoRefund(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) for _, tt := range debugTraceTransactionNoRefundTests { var buf bytes.Buffer s := jsonstream.New(jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096)) @@ -506,7 +506,7 @@ func TestTraceTransactionNoRefund(t *testing.T) { func TestStorageRangeAt(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) t.Run("invalid addr", func(t *testing.T) { var block4 *types.Block var err error @@ -600,7 +600,7 @@ func TestStorageRangeAt(t *testing.T) { func TestStorageRangeAtGethCompat(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, true) // gethCompatibility=true + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, true) // gethCompatibility=true t.Run("block latest, addr 1", func(t *testing.T) { var latestBlock *types.Block err := m.DB.View(m.Ctx, func(tx kv.Tx) (err error) { @@ -666,7 +666,7 @@ func TestStorageRangeAtGethCompat(t *testing.T) { func TestAccountRange(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) t.Run("valid account", func(t *testing.T) { addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf55") @@ -765,7 +765,7 @@ func TestAccountRange(t *testing.T) { func TestGetModifiedAccountsByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) t.Run("correct input", func(t *testing.T) { n, n2 := rpc.BlockNumber(1), rpc.BlockNumber(2) @@ -880,7 +880,7 @@ func TestMapTxNum2BlockNum(t *testing.T) { func TestAccountAt(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) var blockHash0, blockHash1, blockHash3, blockHash10, blockHashNonExistent common.Hash _ = m.DB.View(m.Ctx, func(tx kv.Tx) error { @@ -943,7 +943,7 @@ func TestAccountAt(t *testing.T) { func TestGetBadBlocks(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 5000000, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 5000000, false) ctx := context.Background() require := require.New(t) @@ -1016,7 +1016,7 @@ func TestGetBadBlocks(t *testing.T) { func TestGetRawTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 5000000, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 5000000, false) ctx := context.Background() require := require.New(t) @@ -1057,7 +1057,7 @@ func TestGetRawTransaction(t *testing.T) { func TestGetRawReceipts(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 5000000, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 5000000, false) ctx := context.Background() require := require.New(t) @@ -1098,7 +1098,7 @@ func TestExecutionWitness(t *testing.T) { }) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, nil, 0, false) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) ctx := context.Background() // Write the DB flag so that debug_executionWitness accepts the request. @@ -1261,7 +1261,7 @@ func TestSetHead(t *testing.T) { backendServer := privateapi.NewEthBackendServer(ctx, mock, m.DB, m.Notifications, m.BlockReader, nil, logger, builder.NewLatestBlockBuiltStore(), nil) backendClient := direct.NewEthBackendClientDirect(backendServer) backend := rpcservices.NewRemoteBackend(backendClient, m.DB, m.BlockReader) - return NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, backend, 0, false) + return NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), backend, 0, false) } // Rewinding one block below the current head is the simplest valid rewind. diff --git a/rpc/jsonrpc/erigon_receipts_test.go b/rpc/jsonrpc/erigon_receipts_test.go index 7c359cf2920..9eeaa535dae 100644 --- a/rpc/jsonrpc/erigon_receipts_test.go +++ b/rpc/jsonrpc/erigon_receipts_test.go @@ -45,7 +45,7 @@ func TestGetLogs(t *testing.T) { assert, require := assert.New(t), require.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) { - ethApi := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + ethApi := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)}) require.NoError(err) @@ -77,7 +77,7 @@ func TestErigonGetLatestLogs(t *testing.T) { } assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - db := m.DB + db := m.OverlayDB() api := NewErigonAPI(newBaseApiForTest(m), db, nil) expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) @@ -117,7 +117,7 @@ func TestErigonGetLatestLogs(t *testing.T) { func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - db := m.DB + db := m.OverlayDB() api := NewErigonAPI(newBaseApiForTest(m), db, nil) expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) @@ -197,7 +197,7 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) { } // Assemble the test environment m := mockWithGenerator(t, 4, generator) - api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) expect := map[uint64]string{ 0: `[]`, @@ -228,7 +228,7 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) { // requested block range exceeds rangeLimit. func TestGetLogs_RangeLimitExceeded(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - ethApi := newEthApiForTest(newBaseApiWithLimits(m, 5, 0), m.DB, nil, nil) + ethApi := newEthApiForTest(newBaseApiWithLimits(m, 5, 0), m.OverlayDB(), nil, nil) _, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(10), @@ -245,7 +245,7 @@ func TestGetLogs_RangeLimitExceeded(t *testing.T) { // range is within rangeLimit. func TestGetLogs_RangeLimitOk(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - ethApi := newEthApiForTest(newBaseApiWithLimits(m, 11, 0), m.DB, nil, nil) + ethApi := newEthApiForTest(newBaseApiWithLimits(m, 11, 0), m.OverlayDB(), nil, nil) logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(10), @@ -258,7 +258,7 @@ func TestGetLogs_RangeLimitOk(t *testing.T) { // unlimited (0). func TestGetLogs_MaxResultsOk(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - ethApi := newEthApiForTest(newBaseApiWithLimits(m, 0, 0), m.DB, nil, nil) + ethApi := newEthApiForTest(newBaseApiWithLimits(m, 0, 0), m.OverlayDB(), nil, nil) logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(10), @@ -271,7 +271,7 @@ func TestGetLogs_MaxResultsOk(t *testing.T) { // when the matching log count exceeds maxResults. func TestGetLogs_MaxResultsExceeded(t *testing.T) { m, _, contractAddr, _ := chainWithDeployedContract(t) - ethApi := newEthApiForTest(newBaseApiWithLimits(m, 0, 1), m.DB, nil, nil) + ethApi := newEthApiForTest(newBaseApiWithLimits(m, 0, 1), m.OverlayDB(), nil, nil) _, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), @@ -289,7 +289,7 @@ func TestGetLogs_MaxResultsExceeded(t *testing.T) { // returns an error when the requested logCount exceeds getLogsMaxResults. func TestGetLatestLogs_LogCountExceedsMaxResults(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewErigonAPI(newBaseApiWithLimits(m, 0, 5), m.DB, nil) + api := NewErigonAPI(newBaseApiWithLimits(m, 0, 5), m.OverlayDB(), nil) _, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{}, filters.LogFilterOptions{ LogCount: 10, }) @@ -301,7 +301,7 @@ func TestGetLatestLogs_LogCountExceedsMaxResults(t *testing.T) { // returns an error when the requested blockCount exceeds rangeLimit. func TestGetLatestLogs_BlockCountExceedsRangeLimit(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.DB, nil) + api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.OverlayDB(), nil) _, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{}, filters.LogFilterOptions{ BlockCount: 10, }) @@ -314,7 +314,7 @@ func TestGetLatestLogs_BlockCountExceedsRangeLimit(t *testing.T) { // no logCount/blockCount is specified. func TestGetLatestLogs_ExplicitRangeExceedsLimit(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.DB, nil) + api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.OverlayDB(), nil) _, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(10), @@ -329,7 +329,7 @@ func TestGetLatestLogs_ExplicitRangeExceedsLimit(t *testing.T) { func TestGetLatestLogs_ExplicitRangeWithLogCount_NoRangeCheck(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) // rangeLimit=5, but logCount is set so the range check should be skipped. - api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.DB, nil) + api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.OverlayDB(), nil) _, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(10), @@ -345,7 +345,7 @@ func TestGetLatestLogs_ExplicitRangeWithLogCount_NoRangeCheck(t *testing.T) { func TestGetLatestLogs_ExplicitRangeWithBlockCount_NoRangeCheck(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) // rangeLimit=5, blockCount=3 (≤5 so ok), explicit range 0-10 is skipped. - api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.DB, nil) + api := NewErigonAPI(newBaseApiWithLimits(m, 5, 0), m.OverlayDB(), nil) _, err := api.GetLatestLogs(context.Background(), filters.FilterCriteria{ FromBlock: big.NewInt(0), ToBlock: big.NewInt(10), @@ -367,7 +367,7 @@ func mockWithGenerator(t *testing.T, blocks int, generator func(int, *blockgen.B execmoduletester.WithKey(testKey), ) if blocks > 0 { - chain, _ := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator) + chain, _ := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator, m.PublishedSD()) err := m.InsertChain(chain) require.NoError(t, err) } diff --git a/rpc/jsonrpc/eth_api_test.go b/rpc/jsonrpc/eth_api_test.go index cdc7ef8fc9c..cf4dc19aac6 100644 --- a/rpc/jsonrpc/eth_api_test.go +++ b/rpc/jsonrpc/eth_api_test.go @@ -68,7 +68,7 @@ func TestGetBalanceChangesInBlock(t *testing.T) { assert := assert.New(t) myBlockNum := rpc.BlockNumberOrHashWithNumber(0) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - db := m.DB + db := m.OverlayDB() api := NewErigonAPI(newBaseApiForTest(m), db, nil) balances, err := api.GetBalanceChangesInBlock(context.Background(), myBlockNum) if err != nil { @@ -89,7 +89,7 @@ func TestGetBalanceChangesInBlock(t *testing.T) { func TestGetTransactionReceipt(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := newEthApiForTest(NewBaseApi(nil, stateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0), m.DB, nil, nil) + api := newEthApiForTest(NewBaseApi(nil, stateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0), m.OverlayDB(), nil, nil) // Call GetTransactionReceipt for transaction which is not in the database if _, err := api.GetTransactionReceipt(context.Background(), common.Hash{}); err != nil { t.Errorf("calling GetTransactionReceipt with empty hash: %v", err) @@ -98,7 +98,7 @@ func TestGetTransactionReceipt(t *testing.T) { func TestGetTransactionReceiptUnprotected(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) // Call GetTransactionReceipt for un-protected transaction if _, err := api.GetTransactionReceipt(context.Background(), common.HexToHash("0x3f3cb8a0e13ed2481f97f53f7095b9cbc78b6ffb779f2d3e565146371a8830ea")); err != nil { t.Errorf("calling GetTransactionReceipt for unprotected tx: %v", err) @@ -110,7 +110,7 @@ func TestGetTransactionReceiptUnprotected(t *testing.T) { func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", bnhPtr(rpc.BlockNumberOrHashWithNumber(0))) @@ -124,7 +124,7 @@ func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", bnhPtr(rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), false))) @@ -138,7 +138,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", bnhPtr(rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), true))) @@ -154,11 +154,11 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError t.Skip("slow test") } m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *blockgen.BlockGen) { - }) + }, m.PublishedSD()) if err != nil { t.Fatal(err) } @@ -175,11 +175,11 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *blockgen.BlockGen) { - }) + }, m.PublishedSD()) if err != nil { t.Fatal(err) } @@ -197,7 +197,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testing.T) { assert := assert.New(t) m, _, orphanedChain := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -216,7 +216,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock( func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -232,7 +232,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t * func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -258,7 +258,7 @@ func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testi func TestCall_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -292,7 +292,7 @@ func (m mockBridgeReader) EventTxnLookup(context.Context, common.Hash) (uint64, func TestGetStorageValues_HappyPath(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") slot0 := common.Hash{} @@ -316,7 +316,7 @@ func TestGetStorageValues_HappyPath(t *testing.T) { func TestGetStorageValues_MultipleAddresses(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") addr2 := common.HexToAddress("0x1000000000000000000000000000000000000001") addr3 := common.HexToAddress("0x2000000000000000000000000000000000000002") @@ -349,7 +349,7 @@ func TestGetStorageValues_MultipleAddresses(t *testing.T) { func TestGetStorageValues_MissingSlotReturnsZero(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) @@ -367,7 +367,7 @@ func TestGetStorageValues_MissingSlotReturnsZero(t *testing.T) { func TestGetStorageValues_EmptyRequestReturnsError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) @@ -382,7 +382,7 @@ func TestGetStorageValues_EmptyRequestReturnsError(t *testing.T) { func TestGetStorageValues_ExceedingSlotLimitReturnsError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) @@ -402,7 +402,7 @@ func TestGetStorageValues_ExceedingSlotLimitReturnsError(t *testing.T) { func TestGetStorageValues_ByBlockHash_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -423,7 +423,7 @@ func TestGetStorageValues_ByBlockHash_NonCanonicalBlock(t *testing.T) { func TestGetStorageValues_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -444,7 +444,7 @@ func TestGetStorageValues_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock func TestGetStorageValues_PrunedBlockReturnsError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) addr1 := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") @@ -470,7 +470,7 @@ func bnhPtr(b rpc.BlockNumberOrHash) *rpc.BlockNumberOrHash { return &b } func TestStateMethods_OmittedBlockDefaultsToLatest(t *testing.T) { a := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) ctx := context.Background() addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) diff --git a/rpc/jsonrpc/eth_block_test.go b/rpc/jsonrpc/eth_block_test.go index ccdf9beda3e..dfda7ef1614 100644 --- a/rpc/jsonrpc/eth_block_test.go +++ b/rpc/jsonrpc/eth_block_test.go @@ -43,7 +43,7 @@ import ( // Gets the latest block number with the latest tag func TestGetBlockByNumberWithLatestTag(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) b, err := api.GetBlockByNumber(context.Background(), rpc.LatestBlockNumber, false) expected := common.HexToHash("0x9c47d5780744fa24ccdb1543a9b715e53431d5560b9e460b8b7a68f7c58310ae") if err != nil { @@ -73,7 +73,7 @@ func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { } tx.Commit() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) block, err := api.GetBlockByNumber(ctx, rpc.LatestBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -103,7 +103,7 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { RplBlock: rlpBlock, }) - api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil) b, err := api.GetBlockByNumber(context.Background(), rpc.PendingBlockNumber, false) if err != nil { t.Errorf("error getting block number with pending tag: %s", err) @@ -115,7 +115,7 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { func TestGetBlockByNumber_WithFinalizedTag_NoFinalizedBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) _, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false) if err != nil { var customErr *rpc.CustomError @@ -147,7 +147,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) } tx.Commit() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) block, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -159,7 +159,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) func TestGetBlockByNumber_WithSafeTag_NoSafeBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) _, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false) if err != nil { var customErr *rpc.CustomError @@ -191,7 +191,7 @@ func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { } tx.Commit() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) block, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -204,7 +204,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -236,7 +236,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") tx, err := m.DB.BeginRw(ctx) @@ -268,7 +268,7 @@ func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { func TestGetBlockTransactionCountByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -300,7 +300,7 @@ func TestGetBlockTransactionCountByNumber(t *testing.T) { func TestGetBlockTransactionCountByNumber_ZeroTx(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") @@ -342,7 +342,7 @@ func TestGetBlockByNumber_BlockPruneGating(t *testing.T) { setup := func(t *testing.T, pm prune.Mode) *APIImpl { t.Helper() m := execmoduletester.New(t, execmoduletester.WithPruneMode(pm)) - c, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainSize, func(_ int, _ *blockgen.BlockGen) {}) + c, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainSize, func(_ int, _ *blockgen.BlockGen) {}, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(c)) @@ -354,7 +354,7 @@ func TestGetBlockByNumber_BlockPruneGating(t *testing.T) { require.NoError(t, err) require.NoError(t, tx.Commit()) - return newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + return newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) } legacyFull := prune.Mode{ diff --git a/rpc/jsonrpc/eth_callMany_test.go b/rpc/jsonrpc/eth_callMany_test.go index 710a449a967..5376535ae54 100644 --- a/rpc/jsonrpc/eth_callMany_test.go +++ b/rpc/jsonrpc/eth_callMany_test.go @@ -67,8 +67,8 @@ func TestSetupEVMTimeoutCancelsEVMStoredAfterExpiry(t *testing.T) { func TestCallManyEmptyBundles(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) baseApi := newBaseApiForTest(m) - api := newEthApiForTest(baseApi, m.DB, nil, nil) - debugApi := NewPrivateDebugAPI(baseApi, m.DB, nil, 5000000, false) + api := newEthApiForTest(baseApi, m.OverlayDB(), nil, nil) + debugApi := NewPrivateDebugAPI(baseApi, m.OverlayDB(), nil, 5000000, false) ctx := context.Background() txIndex := -1 diff --git a/rpc/jsonrpc/eth_call_test.go b/rpc/jsonrpc/eth_call_test.go index fa1f3f4a7c1..cdfe623a3d1 100644 --- a/rpc/jsonrpc/eth_call_test.go +++ b/rpc/jsonrpc/eth_call_test.go @@ -215,7 +215,7 @@ func newTestEthAPIWithFilters(t *testing.T, m *execmoduletester.ExecModuleTester ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, execmoduletester.New(t)) mining := txpoolproto.NewMiningClient(conn) filters := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, nil, mining, func() {}, m.Log, nil) - return newEthApiForTest(newBaseApiWithFiltersForTest(filters, stateCache, m), m.DB, nil, nil) + return newEthApiForTest(newBaseApiWithFiltersForTest(filters, stateCache, m), m.OverlayDB(), nil, nil) } type stubTxPoolClient struct{ txpoolproto.TxpoolClient } @@ -226,7 +226,7 @@ func (stubTxPoolClient) Nonce(context.Context, *txpoolproto.NonceRequest, ...grp func TestCreateAccessListContractCreationWithoutFromDoesNotPanic(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var ( res *accessListResult @@ -242,7 +242,7 @@ func TestCreateAccessListContractCreationWithoutFromDoesNotPanic(t *testing.T) { func TestEthCallNonCanonical(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := newEthApiForTest(newBaseApiWithFiltersForTest(nil, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(nil, stateCache, m), m.OverlayDB(), nil, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") blockNumberOrHash := rpc.BlockNumberOrHashWithHash(common.HexToHash("0x3fcb7c0d4569fddc89cbea54b42f163e0c789351d98810a513895ab44b47020b"), true) @@ -261,7 +261,7 @@ func TestEthCallToPrunedBlock(t *testing.T) { m, bankAddress, contractAddress, _ := chainWithDeployedContract(t) doPrune(t, m.DB, pruneTo) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) callData := hexutil.MustDecode("0x2e64cec1") callDataBytes := hexutil.Bytes(callData) @@ -291,7 +291,7 @@ func TestGetProof(t *testing.T) { RpcTxSyncDefaultTimeout: 20 * time.Second, RpcTxSyncMaxTimeout: 1 * time.Minute, } - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, cfg, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.OverlayDB(), nil, nil, nil, cfg, log.New()) key := func(b byte) hexutil.Bytes { result := common.Hash{} @@ -435,7 +435,7 @@ func TestGetBlockByTimestampLatestTime(t *testing.T) { tx, err := m.DB.BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() - api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) latestBlock, err := m.BlockReader.CurrentBlock(tx) require.NoError(t, err) @@ -462,7 +462,7 @@ func TestGetBlockByTimestampOldestTime(t *testing.T) { tx, err := m.DB.BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() - api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) oldestBlock, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 0) require.NoError(t, err) @@ -490,7 +490,7 @@ func TestGetBlockByTimeHigherThanLatestBlock(t *testing.T) { tx, err := m.DB.BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() - api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) latestBlock, err := m.BlockReader.CurrentBlock(tx) require.NoError(t, err) @@ -518,7 +518,7 @@ func TestGetBlockByTimeMiddle(t *testing.T) { tx, err := m.DB.BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() - api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) currentHeader := rawdb.ReadCurrentHeader(tx) oldestHeader, err := api._blockReader.HeaderByNumber(ctx, tx, 0) @@ -551,7 +551,7 @@ func TestGetBlockByTimestamp(t *testing.T) { tx, err := m.DB.BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() - api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) highestBlockNumber := rawdb.ReadCurrentHeader(tx).Number pickedBlock, err := m.BlockReader.BlockByNumber(m.Ctx, tx, highestBlockNumber.Uint64()/3) @@ -827,7 +827,7 @@ func chainWithDeployedContractAndConfig(t *testing.T, cfg *chain.Config) (*execm case 5: // empty block } - }) + }, m.PublishedSD()) require.NoError(t, err) err = m.InsertChain(chain) diff --git a/rpc/jsonrpc/eth_fill_transaction_test.go b/rpc/jsonrpc/eth_fill_transaction_test.go index 17777ba9c76..6dd009605f6 100644 --- a/rpc/jsonrpc/eth_fill_transaction_test.go +++ b/rpc/jsonrpc/eth_fill_transaction_test.go @@ -73,12 +73,12 @@ func newLondonApiForTest(t *testing.T) *APIImpl { Ethash: new(chain.EthashConfig), } m := execmoduletester.New(t, execmoduletester.WithChainConfig(londonCfg)) - return newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + return newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) } func TestFillTransactionFillsDefaults(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -97,7 +97,7 @@ func TestFillTransactionFillsDefaults(t *testing.T) { func TestFillTransactionConflictingFees(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -116,7 +116,7 @@ func TestFillTransactionConflictingFees(t *testing.T) { func TestFillTransactionChainIDMismatch(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -133,7 +133,7 @@ func TestFillTransactionChainIDMismatch(t *testing.T) { func TestFillTransactionContractCreationNoData(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") @@ -146,7 +146,7 @@ func TestFillTransactionContractCreationNoData(t *testing.T) { func TestFillTransactionNoFrom(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -160,7 +160,7 @@ func TestFillTransactionNoFrom(t *testing.T) { func TestFillTransactionExplicitNoncePreserved(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -177,7 +177,7 @@ func TestFillTransactionExplicitNoncePreserved(t *testing.T) { func TestFillTransactionExplicitGasPreserved(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -196,7 +196,7 @@ func TestFillTransactionBlobPreCancun(t *testing.T) { // TestChainBerlinConfig has no Cancun (ExcessBlobGas == nil on head). // A blob tx request must return a clear error, not panic. m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -214,7 +214,7 @@ func TestFillTransactionBlobPreCancun(t *testing.T) { func TestFillTransactionBlobPreCancunExplicitBlobFee(t *testing.T) { // Even with an explicit maxFeePerBlobGas, blob txs on a pre-Cancun chain must error. m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -234,7 +234,7 @@ func TestFillTransactionBlobPreCancunExplicitBlobFee(t *testing.T) { func TestFillTransactionPoolErrorPropagates(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, errPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), errPoolClient{}, nil) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -264,7 +264,7 @@ func TestFillTransactionGasPriceWithAccessListIsTypeOne(t *testing.T) { func TestFillTransactionUserGasAboveCapPreserved(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -283,7 +283,7 @@ func TestFillTransactionUserGasAboveCapPreserved(t *testing.T) { func TestFillTransactionPoolNonce(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) const pendingNonce = uint64(5) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, fixedNoncePoolClient{nonce: pendingNonce}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), fixedNoncePoolClient{nonce: pendingNonce}, nil) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -388,7 +388,7 @@ func TestFillTransactionBlobFeeUsesHeadExcess(t *testing.T) { } m := execmoduletester.New(t, execmoduletester.WithGenesisSpec(gspec)) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, stubTxPoolClient{}, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), stubTxPoolClient{}, nil) to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") gas := hexutil.Uint64(21000) diff --git a/rpc/jsonrpc/eth_filters_test.go b/rpc/jsonrpc/eth_filters_test.go index 00457bf426a..eee0d2aa4ca 100644 --- a/rpc/jsonrpc/eth_filters_test.go +++ b/rpc/jsonrpc/eth_filters_test.go @@ -53,8 +53,8 @@ func TestSubscriptionsRequireFiltersAndNotifier(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) apis := map[string]*APIImpl{ - "withFilters": newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil), - "nilFilters": newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil), + "withFilters": newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil), + "nilFilters": newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil), } for apiName, api := range apis { // ctx carries no rpc notifier, so every subscription method must refuse @@ -85,7 +85,7 @@ func TestNewFilters(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, execmoduletester.New(t)) mining := txpoolproto.NewMiningClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, nil, mining, func() {}, m.Log, nil) - api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil) ptf, err := api.NewPendingTransactionFilter(ctx) assert.NoError(err) @@ -167,7 +167,7 @@ func TestBlockFilterGetFilterChangesInitiallyEmpty(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, execmoduletester.New(t)) mining := txpoolproto.NewMiningClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, nil, mining, func() {}, m.Log, nil) - api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil) // Create a new block filter bf, err := api.NewBlockFilter(ctx) @@ -195,7 +195,7 @@ func TestCompositeFiltersGetFilterChangesInitiallyEmpty(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, execmoduletester.New(t)) mining := txpoolproto.NewMiningClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, nil, mining, func() {}, m.Log, nil) - api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil) // Create all three filter types ptf, err := api.NewPendingTransactionFilter(ctx) @@ -250,7 +250,7 @@ func TestPendingTxsFilterChangesReturnsAllBatches(t *testing.T) { mining := txpoolproto.NewMiningClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, nil, mining, func() {}, m.Log, nil) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil) ptf, err := api.NewPendingTransactionFilter(ctx) require.NoError(t, err) @@ -287,7 +287,7 @@ func TestGetFilterChangesReturnsFilterNotFoundForUnknownID(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, execmoduletester.New(t)) mining := txpoolproto.NewMiningClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, nil, mining, func() {}, m.Log, nil) - api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiWithFiltersForTest(ff, stateCache, m), m.OverlayDB(), nil, nil) // Use a bogus id that does not correspond to any subscription _, err := api.GetFilterChanges(ctx, "0xdeadbeefcafebabe") diff --git a/rpc/jsonrpc/eth_simulation_test.go b/rpc/jsonrpc/eth_simulation_test.go index a53fde5b7c6..e20a4992f54 100644 --- a/rpc/jsonrpc/eth_simulation_test.go +++ b/rpc/jsonrpc/eth_simulation_test.go @@ -525,7 +525,7 @@ func TestSimulationRequestTypes(t *testing.T) { func TestSimulateV1PopulatesMaxUsedGas(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0000000000000000000000000000000000000001") diff --git a/rpc/jsonrpc/eth_subscribe_test.go b/rpc/jsonrpc/eth_subscribe_test.go index dd4c1943596..9171f4138aa 100644 --- a/rpc/jsonrpc/eth_subscribe_test.go +++ b/rpc/jsonrpc/eth_subscribe_test.go @@ -47,7 +47,7 @@ func TestEthSubscribe(t *testing.T) { m := execmoduletester.New(t) chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 7, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) backendServer := privateapi.NewEthBackendServer(ctx, nil, m.DB, m.Notifications, m.BlockReader, nil, logger, builder.NewLatestBlockBuiltStore(), nil) backendClient := direct.NewEthBackendClientDirect(backendServer) @@ -82,7 +82,7 @@ func TestEthSubscribeReceipts(t *testing.T) { tx, err := types.SignTx(types.NewTransaction(uint64(i), m.Address, uint256.NewInt(1), params.TxGas, uint256.NewInt(1), nil), *types.LatestSignerForChainID(m.ChainConfig.ChainID), m.Key) require.NoError(t, err) b.AddTx(tx) - }) + }, m.PublishedSD()) require.NoError(t, err) backendServer := privateapi.NewEthBackendServer(ctx, nil, m.DB, m.Notifications, m.BlockReader, nil, logger, builder.NewLatestBlockBuiltStore(), nil) backendClient := direct.NewEthBackendClientDirect(backendServer) diff --git a/rpc/jsonrpc/eth_system_test.go b/rpc/jsonrpc/eth_system_test.go index 24b0a004638..b4b741b122a 100644 --- a/rpc/jsonrpc/eth_system_test.go +++ b/rpc/jsonrpc/eth_system_test.go @@ -87,7 +87,7 @@ func TestCapabilities(t *testing.T) { t.Fatal(txErr) } b.AddTx(tx) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(c)) @@ -111,7 +111,7 @@ func TestCapabilities(t *testing.T) { head, err := stages.GetStageProgress(roTx, stages.Execution) require.NoError(t, err) - return newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil), head + return newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil), head } setupAPIWithMerge := func(t *testing.T, mergeAt uint64, persistReceipts bool) *APIImpl { @@ -134,7 +134,7 @@ func TestCapabilities(t *testing.T) { t.Fatal(txErr) } b.AddTx(tx) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(c)) ctx := t.Context() @@ -148,7 +148,7 @@ func TestCapabilities(t *testing.T) { require.NoError(t, kvcfg.PersistReceipts.ForceWrite(dbTx, true)) } require.NoError(t, dbTx.Commit()) - return newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + return newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) } oldest := func(t *testing.T, f CapabilityField) uint64 { @@ -409,7 +409,7 @@ func TestCapabilities(t *testing.T) { require.NoError(t, rawdb.WriteDBCommitmentHistoryEnabled(dbTx, false)) require.NoError(t, dbTx.Commit()) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) result, err := api.Capabilities(ctx) require.NoError(t, err) require.Equal(t, uint64(0), uint64(result.Head.Number)) @@ -446,7 +446,7 @@ func TestGasPrice(t *testing.T) { t.Run(testCase.description, func(t *testing.T) { m := createGasPriceTestKV(t, testCase.chainSize) defer m.DB.Close() - eth := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + eth := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) ctx := context.Background() result, err := eth.GasPrice(ctx) @@ -562,7 +562,7 @@ func TestEthConfig(t *testing.T) { require.NoError(t, err) m := execmoduletester.New(t, execmoduletester.WithGenesisSpec(&genesis), execmoduletester.WithKey(key)) defer m.Close() - eth := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + eth := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) if test.head != nil { tx, err := m.DB.BeginTemporalRw(ctx) require.NoError(t, err) @@ -600,7 +600,7 @@ func TestBaseFee(t *testing.T) { Config: chain.TestChainBerlinConfig, Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, }), execmoduletester.WithKey(key)) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) result, err := api.BaseFee(context.Background()) require.NoError(t, err) require.Nil(t, result) @@ -612,7 +612,7 @@ func TestBaseFee(t *testing.T) { Config: chain.TestChainOsakaConfig, Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, }), execmoduletester.WithKey(key)) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) result, err := api.BaseFee(context.Background()) require.NoError(t, err) require.NotNil(t, result) @@ -631,7 +631,7 @@ func TestBlobBaseFee(t *testing.T) { Config: chain.TestChainBerlinConfig, Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, }), execmoduletester.WithKey(key)) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) result, err := api.BlobBaseFee(context.Background()) require.NoError(t, err) require.Nil(t, result) @@ -643,7 +643,7 @@ func TestBlobBaseFee(t *testing.T) { Config: chain.TestChainOsakaConfig, Alloc: types.GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, }), execmoduletester.WithKey(key)) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) result, err := api.BlobBaseFee(context.Background()) require.NoError(t, err) require.NotNil(t, result) @@ -670,7 +670,7 @@ func createGasPriceTestKV(t *testing.T, chainSize int) *execmoduletester.ExecMod t.Fatalf("failed to create tx: %v", txErr) } b.AddTx(tx) - }) + }, m.PublishedSD()) if err != nil { t.Error(err) } diff --git a/rpc/jsonrpc/gen_traces_test.go b/rpc/jsonrpc/gen_traces_test.go index b78161bf252..9a51cff39a3 100644 --- a/rpc/jsonrpc/gen_traces_test.go +++ b/rpc/jsonrpc/gen_traces_test.go @@ -47,7 +47,7 @@ func TestGeneratedDebugApi(t *testing.T) { m := rpcdaemontest.CreateTestExecModuleForTraces(t) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) baseApi := NewBaseApi(nil, stateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - api := NewPrivateDebugAPI(baseApi, m.DB, nil, 0, false) + api := NewPrivateDebugAPI(baseApi, m.OverlayDB(), nil, 0, false) var buf bytes.Buffer stream := jsonstream.New(jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096)) callTracer := "callTracer" @@ -134,7 +134,7 @@ func TestGeneratedTraceApi(t *testing.T) { m := rpcdaemontest.CreateTestExecModuleForTraces(t) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) baseApi := NewBaseApi(nil, stateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - api := NewTraceAPI(baseApi, m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(baseApi, m.OverlayDB(), &httpcfg.HttpCfg{}) traces, err := api.Block(context.Background(), rpc.BlockNumber(1), new(bool), nil) if err != nil { t.Errorf("trace_block %d: %v", 0, err) @@ -289,7 +289,7 @@ func TestGeneratedTraceApi(t *testing.T) { func TestGeneratedTraceApiCollision(t *testing.T) { m := rpcdaemontest.CreateTestExecModuleForTracesCollision(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) traces, err := api.Transaction(context.Background(), common.HexToHash("0xb2b9fa4c999c1c8370ce1fbd1c4315a9ce7f8421fe2ebed8a9051ff2e4e7e3da"), new(bool), nil) if err != nil { t.Errorf("trace_block %d: %v", 0, err) diff --git a/rpc/jsonrpc/otterscan_contract_creator_test.go b/rpc/jsonrpc/otterscan_contract_creator_test.go index 0bdbc033cc4..77f2540bd88 100644 --- a/rpc/jsonrpc/otterscan_contract_creator_test.go +++ b/rpc/jsonrpc/otterscan_contract_creator_test.go @@ -27,7 +27,7 @@ import ( func TestGetContractCreator(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewOtterscanAPI(newBaseApiForTest(m), m.DB, 25) + api := NewOtterscanAPI(newBaseApiForTest(m), m.OverlayDB(), 25) addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") expectCreator := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") diff --git a/rpc/jsonrpc/otterscan_search_backward_test.go b/rpc/jsonrpc/otterscan_search_backward_test.go index 26ffcf23ff0..304eff15495 100644 --- a/rpc/jsonrpc/otterscan_search_backward_test.go +++ b/rpc/jsonrpc/otterscan_search_backward_test.go @@ -167,7 +167,7 @@ func TestBackwardBlockProviderWithMultipleChunksBlockNotFound(t *testing.T) { func TestSearchTransactionsBefore(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewOtterscanAPI(newBaseApiForTest(m), m.DB, 25) + api := NewOtterscanAPI(newBaseApiForTest(m), m.OverlayDB(), 25) addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") t.Run("small page size", func(t *testing.T) { diff --git a/rpc/jsonrpc/otterscan_search_forward_test.go b/rpc/jsonrpc/otterscan_search_forward_test.go index 0579c46aa8b..cf5f7ee6e83 100644 --- a/rpc/jsonrpc/otterscan_search_forward_test.go +++ b/rpc/jsonrpc/otterscan_search_forward_test.go @@ -166,7 +166,7 @@ func TestForwardBlockProviderWithMultipleChunksBlockNotFound(t *testing.T) { func TestSearchTransactionsAfter(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewOtterscanAPI(newBaseApiForTest(m), m.DB, 25) + api := NewOtterscanAPI(newBaseApiForTest(m), m.OverlayDB(), 25) addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") t.Run("small page size", func(t *testing.T) { diff --git a/rpc/jsonrpc/otterscan_transaction_by_sender_and_nonce_test.go b/rpc/jsonrpc/otterscan_transaction_by_sender_and_nonce_test.go index b5bb4bd4498..30f3aba78b6 100644 --- a/rpc/jsonrpc/otterscan_transaction_by_sender_and_nonce_test.go +++ b/rpc/jsonrpc/otterscan_transaction_by_sender_and_nonce_test.go @@ -31,7 +31,7 @@ func TestGetTransactionBySenderAndNonce(t *testing.T) { t.Skip("slow test") } m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewOtterscanAPI(NewBaseApi(nil, nil, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0), m.DB, 25) + api := NewOtterscanAPI(NewBaseApi(nil, nil, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0), m.OverlayDB(), 25) addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") expectCreator := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") diff --git a/rpc/jsonrpc/overlay_race_test.go b/rpc/jsonrpc/overlay_race_test.go index b37b3133fa1..173adb62361 100644 --- a/rpc/jsonrpc/overlay_race_test.go +++ b/rpc/jsonrpc/overlay_race_test.go @@ -68,7 +68,7 @@ func newOverlayAheadTestAPI(t *testing.T) (base *BaseAPI, m *execmoduletester.Ex c, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, overlayRaceChainSize, func(i int, gen *blockgen.BlockGen) { gen.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(c)) @@ -150,7 +150,7 @@ func marshalOverlayRaceTestTx(t *testing.T, txn types.Transaction) []byte { func TestGetBlockByTimestamp_SeesOverlayHead(t *testing.T) { t.Parallel() base, m, overlayHeader := newOverlayAheadTestAPI(t) - api := NewErigonAPI(base, m.DB, nil) + api := NewErigonAPI(base, m.OverlayDB(), nil) resp, err := api.GetBlockByTimestamp(m.Ctx, rpc.Timestamp(overlayHeader.Time), false) require.NoError(t, err) @@ -171,7 +171,7 @@ func TestGetTransactionByHash_PendingTx_UsesOverlayHead(t *testing.T) { pool := &overlayRaceTxPoolClient{ transactionsReply: &txpoolproto.TransactionsReply{RlpTxs: [][]byte{marshalOverlayRaceTestTx(t, pendingTxn)}}, } - api := newEthApiForTest(base, m.DB, pool, nil) + api := newEthApiForTest(base, m.OverlayDB(), pool, nil) got, err := api.GetTransactionByHash(m.Ctx, pendingTxn.Hash()) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestTxPoolContent_UsesOverlayHead(t *testing.T) { t.Parallel() base, m, overlayHeader := newOverlayAheadTestAPI(t) pool, txn := newOverlayRacePendingPool(t, m) - api := NewTxPoolAPI(base, m.DB, pool) + api := NewTxPoolAPI(base, m.OverlayDB(), pool) content, err := api.Content(m.Ctx) require.NoError(t, err) @@ -217,7 +217,7 @@ func TestTxPoolContentFrom_UsesOverlayHead(t *testing.T) { t.Parallel() base, m, overlayHeader := newOverlayAheadTestAPI(t) pool, txn := newOverlayRacePendingPool(t, m) - api := NewTxPoolAPI(base, m.DB, pool) + api := NewTxPoolAPI(base, m.OverlayDB(), pool) content, err := api.ContentFrom(m.Ctx, m.Address) require.NoError(t, err) diff --git a/rpc/jsonrpc/receipts/handler_test.go b/rpc/jsonrpc/receipts/handler_test.go index ae56fd81803..e8d31617fe0 100644 --- a/rpc/jsonrpc/receipts/handler_test.go +++ b/rpc/jsonrpc/receipts/handler_test.go @@ -356,7 +356,7 @@ func mockWithGenerator(t *testing.T, blocks int, generator func(int, *blockgen.B execmoduletester.WithKey(testKey), ) if blocks > 0 { - chain, _ := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator) + chain, _ := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator, m.PublishedSD()) err := m.InsertChain(chain) require.NoError(t, err) } diff --git a/rpc/jsonrpc/send_transaction_test.go b/rpc/jsonrpc/send_transaction_test.go index eb88075f5a7..f00929b500e 100644 --- a/rpc/jsonrpc/send_transaction_test.go +++ b/rpc/jsonrpc/send_transaction_test.go @@ -39,7 +39,7 @@ import ( func oneBlockStep(m *execmoduletester.ExecModuleTester, require *require.Assertions) { chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1 /*number of blocks:*/, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(err) err = m.InsertChain(chain) require.NoError(err) @@ -55,7 +55,7 @@ func TestSendRawTransaction(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) txPool := txpoolproto.NewTxpoolClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, txPool, txpoolproto.NewMiningClient(conn), func() {}, m.Log, nil) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, txPool, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), txPool, nil) buf := bytes.NewBuffer(nil) err = txn.MarshalBinary(buf) require.NoError(err) @@ -96,7 +96,7 @@ func TestSendRawTransactionUnprotected(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) txPool := txpoolproto.NewTxpoolClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, txPool, txpoolproto.NewMiningClient(conn), func() {}, m.Log, nil) - api := newEthApiForTest(newBaseApiForTest(m), m.DB, txPool, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), txPool, nil) // Enable unprotected txs flag api.AllowUnprotectedTxs = true buf := bytes.NewBuffer(nil) diff --git a/rpc/jsonrpc/trace_adhoc_test.go b/rpc/jsonrpc/trace_adhoc_test.go index 6960f5f9014..61ac26da06e 100644 --- a/rpc/jsonrpc/trace_adhoc_test.go +++ b/rpc/jsonrpc/trace_adhoc_test.go @@ -55,7 +55,7 @@ import ( func TestEmptyQuery(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage("[]"), &rpc.BlockNumberOrHash{BlockNumber: &latest}, nil) @@ -71,7 +71,7 @@ func TestEmptyQuery(t *testing.T) { } func TestCoinbaseBalance(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage(` @@ -101,7 +101,7 @@ func internedAddress(addr string) accounts.Address { func TestSwapBalance(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage(` @@ -186,7 +186,7 @@ func TestSwapBalance(t *testing.T) { func TestCorrectStateDiff(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage(` @@ -318,7 +318,7 @@ func TestCorrectStateDiff(t *testing.T) { func TestReplayTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) var txnHash common.Hash if err := m.DB.View(context.Background(), func(tx kv.Tx) error { b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 6) @@ -345,7 +345,7 @@ func TestReplayTransaction(t *testing.T) { func TestReplayBlockTransactions(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database n := rpc.BlockNumber(6) @@ -499,7 +499,7 @@ func rawTxFromBlock(t *testing.T, m *execmoduletester.ExecModuleTester, blockNum func TestRawTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) encoded, _, _ := rawTxFromBlock(t, m, 6) result, err := api.RawTransaction(context.Background(), encoded, []string{"trace"}) @@ -510,7 +510,7 @@ func TestRawTransaction(t *testing.T) { func TestRawTransactionStateDiff(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) encoded, from, to := rawTxFromBlock(t, m, 6) @@ -563,7 +563,7 @@ func TestRawTransactionStateDiff(t *testing.T) { func TestRawTransactionVmTrace(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) encoded, _, _ := rawTxFromBlock(t, m, 6) @@ -578,7 +578,7 @@ func TestRawTransactionVmTrace(t *testing.T) { func TestRawTransactionAllTraceTypes(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) encoded, _, _ := rawTxFromBlock(t, m, 6) @@ -607,7 +607,7 @@ func TestParseOeTracerConfigToleratesEmptyTracer(t *testing.T) { func TestTraceCallRejectsCustomTracer(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) tracer := "callTracer" var latest = rpc.LatestBlockNumber @@ -619,7 +619,7 @@ func TestTraceCallRejectsCustomTracer(t *testing.T) { func TestRawTransactionInvalidType(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) encoded, _, _ := rawTxFromBlock(t, m, 6) @@ -634,7 +634,7 @@ func TestTraceCallBlockOverridesBaseFeeAffectsGasPrice(t *testing.T) { } m, bankAddr, contractAddr, _ := chainWithDeployedContractAndConfig(t, chain.AllProtocolChanges) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // EVM bytecode: GASPRICE (0x3a), PUSH1 0x00, MSTORE, PUSH1 0x20, PUSH1 0x00, RETURN gasPriceCode := hexutil.Bytes{0x3a, 0x60, 0x00, 0x52, 0x60, 0x20, 0x60, 0x00, 0xf3} @@ -714,7 +714,7 @@ func (c *baseFeeTestChain) mineBlock(t *testing.T, gen func(*blockgen.BlockGen)) chainB, err := blockgen.GenerateChain(c.m.ChainConfig, c.head, c.m.Engine, c.m.DB, 1, func(_ int, block *blockgen.BlockGen) { gen(block) - }) + }, c.m.PublishedSD()) require.NoError(t, err) require.NoError(t, c.m.InsertChain(chainB)) c.head = chainB.TopBlock @@ -797,7 +797,7 @@ func traceConfigWithBaseFeeOverride(baseFee *uint256.Int) *config.TraceConfig { } func (c *baseFeeTestChain) traceAPI() *TraceAPIImpl { - return NewTraceAPI(newBaseApiForTest(c.m), c.m.DB, &httpcfg.HttpCfg{}) + return NewTraceAPI(newBaseApiForTest(c.m), c.m.OverlayDB(), &httpcfg.HttpCfg{}) } // setupBaseFeeOverrideCall deploys an opcode-emitting contract, mines a diff --git a/rpc/jsonrpc/trace_filtering_test.go b/rpc/jsonrpc/trace_filtering_test.go index e0504590450..72edbb0bf7c 100644 --- a/rpc/jsonrpc/trace_filtering_test.go +++ b/rpc/jsonrpc/trace_filtering_test.go @@ -50,7 +50,7 @@ import ( // worker-pool path of doCallBlockParallel. func TestCallBlockParallelMatchesSequential(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) ctx := context.Background() const blockNum = uint64(6) // block 6 has 32 txs (case i=5 in test chain generation) @@ -180,7 +180,7 @@ func chainWithWithdrawal(t *testing.T, withdrawalAddr common.Address, withdrawal Address: withdrawalAddr, Amount: withdrawalGwei, }) - }) + }, m.PublishedSD()) require.NoError(t, err) err = m.InsertChain(generated) require.NoError(t, err) @@ -194,7 +194,7 @@ func TestBlockWithdrawalTraceEntries(t *testing.T) { withdrawalAddr := common.HexToAddress("0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") m, blk := chainWithWithdrawal(t, withdrawalAddr, withdrawalGwei) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(blk.NumberU64()) traces, err := api.Block(context.Background(), n, new(bool), traceConfigWithWithdrawals()) @@ -231,7 +231,7 @@ func TestBlockWithdrawalNoEntriesWithoutFlag(t *testing.T) { withdrawalAddr := common.HexToAddress("0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") m, blk := chainWithWithdrawal(t, withdrawalAddr, withdrawalGwei) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(blk.NumberU64()) traces, err := api.Block(context.Background(), n, new(bool), nil) @@ -244,7 +244,7 @@ func TestBlockWithdrawalNoEntriesWithoutFlag(t *testing.T) { func TestBlockWithdrawalNoEntriesPreShanghai(t *testing.T) { // Default test chain uses TestChainBerlinConfig which has no ShanghaiTime. m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) // Block 1 is pre-Shanghai on the default test chain. n := rpc.BlockNumber(1) @@ -261,7 +261,7 @@ func TestReplayBlockTransactionsWithdrawalStateDiff(t *testing.T) { withdrawalAddr := common.HexToAddress("0xcafecafecafecafecafecafecafecafecafecafe") m, blk := chainWithWithdrawal(t, withdrawalAddr, withdrawalGwei) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(blk.NumberU64()) bnOrHash := rpc.BlockNumberOrHash{BlockNumber: &n} @@ -311,13 +311,13 @@ func TestReplayBlockTransactionsMultiWithdrawalSameAddr(t *testing.T) { generated, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(_ int, b *blockgen.BlockGen) { b.AddWithdrawal(&types.Withdrawal{Index: 0, Validator: 42, Address: withdrawalAddr, Amount: wd1Gwei}) b.AddWithdrawal(&types.Withdrawal{Index: 1, Validator: 43, Address: withdrawalAddr, Amount: wd2Gwei}) - }) + }, m.PublishedSD()) require.NoError(t, err) err = m.InsertChain(generated) require.NoError(t, err) blk := generated.Blocks[0] - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(blk.NumberU64()) bnOrHash := rpc.BlockNumberOrHash{BlockNumber: &n} results, err := api.ReplayBlockTransactions(context.Background(), bnOrHash, []string{"stateDiff"}, new(bool), traceConfigWithWithdrawals()) @@ -357,11 +357,11 @@ func TestReplayBlockTransactionsWithdrawalNewAddress(t *testing.T) { m := execmoduletester.New(t, execmoduletester.WithGenesisSpec(gspec)) generated, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(_ int, b *blockgen.BlockGen) { b.AddWithdrawal(&types.Withdrawal{Index: 0, Validator: 42, Address: newAddr, Amount: withdrawalGwei}) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(generated)) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(generated.Blocks[0].NumberU64()) bnOrHash := rpc.BlockNumberOrHash{BlockNumber: &n} results, err := api.ReplayBlockTransactions(context.Background(), bnOrHash, []string{"stateDiff"}, new(bool), traceConfigWithWithdrawals()) @@ -402,11 +402,11 @@ func TestReplayBlockTransactionsMultiWithdrawalNewAddress(t *testing.T) { generated, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(_ int, b *blockgen.BlockGen) { b.AddWithdrawal(&types.Withdrawal{Index: 0, Validator: 1, Address: newAddr, Amount: wd1Gwei}) b.AddWithdrawal(&types.Withdrawal{Index: 1, Validator: 2, Address: newAddr, Amount: wd2Gwei}) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(generated)) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(generated.Blocks[0].NumberU64()) bnOrHash := rpc.BlockNumberOrHash{BlockNumber: &n} results, err := api.ReplayBlockTransactions(context.Background(), bnOrHash, []string{"stateDiff"}, new(bool), traceConfigWithWithdrawals()) @@ -437,7 +437,7 @@ func TestReplayBlockTransactionsWithdrawalNoEntriesWithoutFlag(t *testing.T) { withdrawalAddr := common.HexToAddress("0xcafecafecafecafecafecafecafecafecafecafe") m, blk := chainWithWithdrawal(t, withdrawalAddr, withdrawalGwei) - api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) n := rpc.BlockNumber(blk.NumberU64()) bnOrHash := rpc.BlockNumberOrHash{BlockNumber: &n} diff --git a/rpc/jsonrpc/txpool_api_test.go b/rpc/jsonrpc/txpool_api_test.go index 741e1e34651..3677457bbeb 100644 --- a/rpc/jsonrpc/txpool_api_test.go +++ b/rpc/jsonrpc/txpool_api_test.go @@ -41,7 +41,7 @@ func TestTxPoolContent(t *testing.T) { require := require.New(t) chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(err) err = m.InsertChain(chain) require.NoError(err) @@ -49,7 +49,7 @@ func TestTxPoolContent(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) txPool := txpoolproto.NewTxpoolClient(conn) ff := rpchelper.New(ctx, rpchelper.DefaultFiltersConfig, nil, txPool, txpoolproto.NewMiningClient(conn), func() {}, m.Log, nil) - api := NewTxPoolAPI(NewBaseApi(ff, kvcache.New(kvcache.DefaultCoherentConfig), m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0), m.DB, txPool) + api := NewTxPoolAPI(NewBaseApi(ff, kvcache.New(kvcache.DefaultCoherentConfig), m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0), m.OverlayDB(), txPool) expectValue := uint64(1234) txn, err := types.SignTx(types.NewTransaction(0, common.Address{1}, uint256.NewInt(expectValue), params.TxGas, uint256.NewInt(10*common.GWei), nil), *types.LatestSignerForChainID(m.ChainConfig.ChainID), m.Key) From dbea3a286b240b7db47a244db5cf75387ce273f2 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 13:49:10 +0000 Subject: [PATCH 120/167] db/state, execmodule: SharedDomains range-as-of merging in-mem history + committed Add SharedDomains.RangeAsOf (delegating to TemporalMemBatch.RangeAsOf): merges the in-flight in-memory domain history with the committed DB+files range via a newest-wins union, resolving each key as of the requested txNum and masking in-memory deletes. Only wired from the RPC test harness (OverlayDB), which now routes RangeAsOf so debug/trace range reads (AccountRange, StorageRangeAt, AccountAt) observe the uncommitted tip under background commit. Route the debug test-body block/state lookups through OverlayDB too. --- db/kv/kv_interface.go | 1 + db/state/execctx/domain_shared.go | 7 + db/state/temporal_mem_batch.go | 138 ++++++++++++++++++ .../execmoduletester/overlay_read.go | 6 + rpc/jsonrpc/debug_api_test.go | 36 ++--- 5 files changed, 170 insertions(+), 18 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 78882dbf7d5..19e3dfba1e8 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -560,6 +560,7 @@ type TemporalMemBatch interface { ClearRam() IndexAdd(table InvertedIdx, key []byte, txNum uint64) (err error) IteratePrefix(domain Domain, prefix []byte, roTx Tx, it func(k []byte, v []byte) (cont bool, err error)) error + RangeAsOf(ctx context.Context, domain Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int, roTx Tx) (stream.KV, error) HasPrefix(domain Domain, prefix []byte, roTx Tx) ([]byte, []byte, bool, error) HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 10a348d8132..92730bbf902 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/db/kv/membatchwithdb" "github.com/erigontech/erigon/db/kv/order" "github.com/erigontech/erigon/db/kv/rawdbv3" + "github.com/erigontech/erigon/db/kv/stream" "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/kvmetrics" "github.com/erigontech/erigon/db/state/statecfg" @@ -1572,6 +1573,12 @@ func (sd *SharedDomains) GetAsOf(domain kv.Domain, key []byte, ts uint64) (v []b return sd.mem.GetAsOf(domain, key, ts) } +// RangeAsOf returns domain values over [fromKey, toKey) as of ts, merging the +// in-flight in-memory history with committed DB+files. +func (sd *SharedDomains) RangeAsOf(ctx context.Context, domain kv.Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int, roTx kv.Tx) (stream.KV, error) { + return sd.mem.RangeAsOf(ctx, domain, fromKey, toKey, ts, asc, limit, roTx) +} + // DomainPut // Optimizations: // - user can provide `prevVal != nil` - then it will not read prev value from storage diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index dd56cabb688..c94f1e82e98 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -17,6 +17,7 @@ package state import ( + "bytes" "context" "encoding/binary" "errors" @@ -30,6 +31,8 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/order" + "github.com/erigontech/erigon/db/kv/stream" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/execctx" @@ -321,6 +324,141 @@ func (sd *TemporalMemBatch) GetAsOf(domain kv.Domain, key []byte, ts uint64) (v return unwoundLatest(domain, keyS) } +// RangeAsOf returns domain values over [fromKey, toKey) as of txNum ts, merging +// the in-memory (not-yet-committed) history with committed DB+files. In-memory +// values take precedence per key; keys deleted or not yet created as of ts are +// omitted. Only used by the RPC test harness to read the in-flight tip. +func (sd *TemporalMemBatch) RangeAsOf(ctx context.Context, domain kv.Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int, roTx kv.Tx) (stream.KV, error) { + committed, err := AggTx(roTx).RangeAsOf(ctx, roTx, domain, fromKey, toKey, ts, asc, limit) + if err != nil { + return nil, err + } + if !sd.inMemHistoryReads { + return committed, nil + } + mem := sd.memRangeAsOf(domain, fromKey, toKey, ts, asc) + if len(mem) == 0 { + return committed, nil + } + merged := stream.UnionKV(&memKVIter{pairs: mem}, committed, -1) + return &liveLimitKV{inner: merged, limit: limit}, nil +} + +// memRangeAsOf collects the in-memory latest history for domain over +// [fromKey, toKey), resolved as of ts. Tombstones (empty value) are kept so +// they mask committed rows; keys with no in-memory entry at or before ts are +// omitted so committed state supplies them. +func (sd *TemporalMemBatch) memRangeAsOf(domain kv.Domain, fromKey, toKey []byte, ts uint64, asc order.By) []memKVPair { + sd.latestStateLock.RLock() + defer sd.latestStateLock.RUnlock() + inRange := func(k string) bool { + if fromKey != nil && k < string(fromKey) { + return false + } + if toKey != nil && k >= string(toKey) { + return false + } + return true + } + var pairs []memKVPair + collect := func(k string, entries []dataWithTxNum) { + if !inRange(k) { + return + } + if v, ok := asOfEntry(entries, ts); ok { + pairs = append(pairs, memKVPair{k: []byte(k), v: v}) + } + } + if domain == kv.StorageDomain { + sd.storage.Scan(func(k string, entries []dataWithTxNum) bool { + collect(k, entries) + return true + }) + } else { + for k, entries := range sd.domains[domain] { + collect(k, entries) + } + } + sort.Slice(pairs, func(i, j int) bool { + if asc == order.Asc { + return bytes.Compare(pairs[i].k, pairs[j].k) < 0 + } + return bytes.Compare(pairs[i].k, pairs[j].k) > 0 + }) + return pairs +} + +// asOfEntry returns the value in effect at ts from a txNum-ascending history +// (the entry with the greatest txNum < ts). ok is false when ts precedes the +// first entry, i.e. the key did not exist yet. +func asOfEntry(entries []dataWithTxNum, ts uint64) ([]byte, bool) { + for i := range entries { + if ts > entries[i].txNum && (i == len(entries)-1 || ts <= entries[i+1].txNum) { + return entries[i].data, true + } + } + return nil, false +} + +type memKVPair struct{ k, v []byte } + +type memKVIter struct { + pairs []memKVPair + i int +} + +func (m *memKVIter) HasNext() bool { return m.i < len(m.pairs) } +func (m *memKVIter) Close() {} +func (m *memKVIter) Next() ([]byte, []byte, error) { + p := m.pairs[m.i] + m.i++ + return p.k, p.v, nil +} + +// liveLimitKV drops tombstone (empty-value) rows from the merged stream and +// stops after limit live rows (limit < 0 means unlimited). +type liveLimitKV struct { + inner stream.KV + limit int + seen int + k, v []byte + ready bool + err error +} + +func (m *liveLimitKV) advance() { + if m.ready || m.err != nil { + return + } + if m.limit >= 0 && m.seen >= m.limit { + return + } + for m.inner.HasNext() { + k, v, err := m.inner.Next() + if err != nil { + m.err = err + return + } + if len(v) == 0 { + continue + } + m.k, m.v, m.ready = k, v, true + return + } +} + +func (m *liveLimitKV) HasNext() bool { m.advance(); return m.ready || m.err != nil } +func (m *liveLimitKV) Close() { m.inner.Close() } +func (m *liveLimitKV) Next() ([]byte, []byte, error) { + m.advance() + if m.err != nil { + return nil, nil, m.err + } + m.ready = false + m.seen++ + return m.k, m.v, nil +} + func (sd *TemporalMemBatch) SizeEstimate() uint64 { // CachePutSize is guarded by the metrics lock — the put path, Merge and the // reset all mutate it under sd.metrics.Lock(), and MergeMetrics folds in a diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index 7b899050d50..c2d8b4de67e 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -20,6 +20,8 @@ import ( "context" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/order" + "github.com/erigontech/erigon/db/kv/stream" "github.com/erigontech/erigon/db/state/execctx" ) @@ -105,6 +107,10 @@ func (t *sdRoTx) GetAsOf(name kv.Domain, k []byte, ts uint64) ([]byte, bool, err return t.TemporalTx.GetAsOf(name, k, ts) } +func (t *sdRoTx) RangeAsOf(name kv.Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int) (stream.KV, error) { + return t.sd.RangeAsOf(context.Background(), name, fromKey, toKey, ts, asc, limit, t.base) +} + func (t *sdRoTx) Rollback() { t.base.Rollback() } diff --git a/rpc/jsonrpc/debug_api_test.go b/rpc/jsonrpc/debug_api_test.go index 454ae650631..a9d87736233 100644 --- a/rpc/jsonrpc/debug_api_test.go +++ b/rpc/jsonrpc/debug_api_test.go @@ -234,7 +234,7 @@ func TestTraceErrorPathsWriteNoStream(t *testing.T) { t.Run("TraceBlockByHash_genesis", func(t *testing.T) { var genesisHash common.Hash - require.NoError(t, m.DB.View(m.Ctx, func(tx kv.Tx) error { + require.NoError(t, m.OverlayDB().View(m.Ctx, func(tx kv.Tx) error { genesisHash, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 0) return nil })) @@ -510,7 +510,7 @@ func TestStorageRangeAt(t *testing.T) { t.Run("invalid addr", func(t *testing.T) { var block4 *types.Block var err error - err = m.DB.View(m.Ctx, func(tx kv.Tx) error { + err = m.OverlayDB().View(m.Ctx, func(tx kv.Tx) error { block4, err = m.BlockReader.BlockByNumber(m.Ctx, tx, 4) return err }) @@ -523,7 +523,7 @@ func TestStorageRangeAt(t *testing.T) { }) t.Run("block 4, addr 1", func(t *testing.T) { var block4 *types.Block - err := m.DB.View(m.Ctx, func(tx kv.Tx) error { + err := m.OverlayDB().View(m.Ctx, func(tx kv.Tx) error { block4, _ = m.BlockReader.BlockByNumber(m.Ctx, tx, 4) return nil }) @@ -544,7 +544,7 @@ func TestStorageRangeAt(t *testing.T) { }) t.Run("block latest, addr 1", func(t *testing.T) { var latestBlock *types.Block - err := m.DB.View(m.Ctx, func(tx kv.Tx) (err error) { + err := m.OverlayDB().View(m.Ctx, func(tx kv.Tx) (err error) { latestBlock, err = m.BlockReader.CurrentBlock(tx) return err }) @@ -603,7 +603,7 @@ func TestStorageRangeAtGethCompat(t *testing.T) { api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, true) // gethCompatibility=true t.Run("block latest, addr 1", func(t *testing.T) { var latestBlock *types.Block - err := m.DB.View(m.Ctx, func(tx kv.Tx) (err error) { + err := m.OverlayDB().View(m.Ctx, func(tx kv.Tx) (err error) { latestBlock, err = m.BlockReader.CurrentBlock(tx) return err }) @@ -841,7 +841,7 @@ func TestMapTxNum2BlockNum(t *testing.T) { } } t.Run("descend", func(t *testing.T) { - tx, err := m.DB.BeginTemporalRo(m.Ctx) + tx, err := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() @@ -853,7 +853,7 @@ func TestMapTxNum2BlockNum(t *testing.T) { checkIter(t, expectTxNums, txNumsIter) }) t.Run("ascend", func(t *testing.T) { - tx, err := m.DB.BeginTemporalRo(m.Ctx) + tx, err := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() @@ -865,7 +865,7 @@ func TestMapTxNum2BlockNum(t *testing.T) { checkIter(t, expectTxNums, txNumsIter) }) t.Run("ascend limit", func(t *testing.T) { - tx, err := m.DB.BeginTemporalRo(m.Ctx) + tx, err := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() @@ -883,7 +883,7 @@ func TestAccountAt(t *testing.T) { api := NewPrivateDebugAPI(newBaseApiForTest(m), m.OverlayDB(), nil, 0, false) var blockHash0, blockHash1, blockHash3, blockHash10, blockHashNonExistent common.Hash - _ = m.DB.View(m.Ctx, func(tx kv.Tx) error { + _ = m.OverlayDB().View(m.Ctx, func(tx kv.Tx) error { blockHash0, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 0) blockHash1, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 1) blockHash3, _, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 3) @@ -998,7 +998,7 @@ func TestGetBadBlocks(t *testing.T) { tx.Commit() // Reset the global bad block cache so it reads only from this test's DB - tx2, err := m.DB.BeginRo(ctx) + tx2, err := m.OverlayDB().BeginRo(ctx) require.NoError(err) defer tx2.Rollback() require.NoError(rawdb.ResetBadBlockCache(tx2, 100)) @@ -1033,7 +1033,7 @@ func TestGetRawTransaction(t *testing.T) { } var testedOnce = false for i := uint64(0); i < number; i++ { - tx, err := m.DB.BeginRo(ctx) + tx, err := m.OverlayDB().BeginRo(ctx) require.NoError(err) defer tx.Rollback() block, err := api._blockReader.BlockByNumber(ctx, tx, i) @@ -1061,7 +1061,7 @@ func TestGetRawReceipts(t *testing.T) { ctx := context.Background() require := require.New(t) - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(err) defer tx.Rollback() number := *rawdb.ReadCurrentBlockNumber(tx) @@ -1109,7 +1109,7 @@ func TestExecutionWitness(t *testing.T) { // Get the latest block number var latestBlockNum uint64 - err = m.DB.View(ctx, func(tx kv.Tx) error { + err = m.OverlayDB().View(ctx, func(tx kv.Tx) error { latestBlockNum, _ = stages.GetStageProgress(tx, stages.Execution) return nil }) @@ -1151,7 +1151,7 @@ func TestExecutionWitness(t *testing.T) { t.Run("by block hash", func(t *testing.T) { var blockHash common.Hash - err := m.DB.View(ctx, func(tx kv.Tx) error { + err := m.OverlayDB().View(ctx, func(tx kv.Tx) error { blockHash, _, _ = m.BlockReader.CanonicalHash(ctx, tx, 1) return nil }) @@ -1248,7 +1248,7 @@ func TestSetHead(t *testing.T) { logger := log.New() // Determine the canonical head of the test chain. - roTx, err := m.DB.BeginRo(ctx) + roTx, err := m.OverlayDB().BeginRo(ctx) require.NoError(t, err) defer roTx.Rollback() head, err := rpchelper.GetLatestBlockNumber(roTx) @@ -1339,7 +1339,7 @@ func TestSetHeadCanonicalCleanup(t *testing.T) { ctx := m.Ctx // Snapshot canonical state before the unwind. - roTx, err := m.DB.BeginRo(ctx) + roTx, err := m.OverlayDB().BeginRo(ctx) require.NoError(t, err) defer roTx.Rollback() head, err := rpchelper.GetLatestBlockNumber(roTx) @@ -1368,7 +1368,7 @@ func TestSetHeadCanonicalCleanup(t *testing.T) { require.NoError(t, err) // --- Verify DB state after unwind --- - roTx, err = m.DB.BeginRo(ctx) + roTx, err = m.OverlayDB().BeginRo(ctx) require.NoError(t, err) defer roTx.Rollback() @@ -1409,7 +1409,7 @@ func TestSetHeadCanonicalCleanup(t *testing.T) { "UpdateForkChoice back to original head should succeed after SetHead") // Verify the chain is back at the original head. - roTx, err = m.DB.BeginRo(ctx) + roTx, err = m.OverlayDB().BeginRo(ctx) require.NoError(t, err) defer roTx.Rollback() From aaac43ff7cd76fe4f9f6c824090134e355168a2a Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 13:58:03 +0000 Subject: [PATCH 121/167] db/state, execmodule: SharedDomains.HistoryRange merging in-mem + committed history Mirror RangeAsOf for changeset scans: SharedDomains.HistoryRange (via TemporalMemBatch.HistoryRange) returns keys changed in [fromTs, toTs) with their pre-range value, merging the in-flight in-memory history with committed. Wired from the RPC test harness OverlayDB so debug GetModifiedAccountsByNumber sees uncommitted changes under background commit. --- db/kv/kv_interface.go | 1 + db/state/execctx/domain_shared.go | 6 ++ db/state/temporal_mem_batch.go | 63 +++++++++++++++++++ .../execmoduletester/overlay_read.go | 4 ++ 4 files changed, 74 insertions(+) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 19e3dfba1e8..0e8fba3bd16 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -561,6 +561,7 @@ type TemporalMemBatch interface { IndexAdd(table InvertedIdx, key []byte, txNum uint64) (err error) IteratePrefix(domain Domain, prefix []byte, roTx Tx, it func(k []byte, v []byte) (cont bool, err error)) error RangeAsOf(ctx context.Context, domain Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int, roTx Tx) (stream.KV, error) + HistoryRange(ctx context.Context, domain Domain, fromTs, toTs int, asc order.By, limit int, roTx Tx) (stream.KV, error) HasPrefix(domain Domain, prefix []byte, roTx Tx) ([]byte, []byte, bool, error) HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 92730bbf902..f73c6317869 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1579,6 +1579,12 @@ func (sd *SharedDomains) RangeAsOf(ctx context.Context, domain kv.Domain, fromKe return sd.mem.RangeAsOf(ctx, domain, fromKey, toKey, ts, asc, limit, roTx) } +// HistoryRange returns keys changed in [fromTs, toTs) with their pre-range +// value, merging in-flight in-memory history with committed. +func (sd *SharedDomains) HistoryRange(ctx context.Context, domain kv.Domain, fromTs, toTs int, asc order.By, limit int, roTx kv.Tx) (stream.KV, error) { + return sd.mem.HistoryRange(ctx, domain, fromTs, toTs, asc, limit, roTx) +} + // DomainPut // Optimizations: // - user can provide `prevVal != nil` - then it will not read prev value from storage diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index c94f1e82e98..28d4a60037d 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -400,6 +400,69 @@ func asOfEntry(entries []dataWithTxNum, ts uint64) ([]byte, bool) { return nil, false } +// HistoryRange returns one entry per key changed in [fromTs, toTs), each with +// its value just before the range, merging in-memory history with committed. +// In-memory keys take precedence. Only used by the RPC test harness. +func (sd *TemporalMemBatch) HistoryRange(ctx context.Context, domain kv.Domain, fromTs, toTs int, asc order.By, limit int, roTx kv.Tx) (stream.KV, error) { + committed, err := AggTx(roTx).HistoryRange(domain, fromTs, toTs, asc, limit, roTx) + if err != nil { + return nil, err + } + if !sd.inMemHistoryReads { + return committed, nil + } + mem := sd.memHistoryRange(domain, fromTs, toTs, asc, roTx) + if len(mem) == 0 { + return committed, nil + } + return stream.UnionKV(&memKVIter{pairs: mem}, committed, limit), nil +} + +// memHistoryRange collects keys changed in [fromTs, toTs) from the in-memory +// history, each paired with its pre-range value (in-memory value just before +// fromTs, falling back to committed state when the key predates its in-memory +// history). Empty pre-values are kept — they mean "did not exist before". +func (sd *TemporalMemBatch) memHistoryRange(domain kv.Domain, fromTs, toTs int, asc order.By, roTx kv.Tx) []memKVPair { + sd.latestStateLock.RLock() + defer sd.latestStateLock.RUnlock() + from, to := uint64(fromTs), uint64(toTs) + var pairs []memKVPair + collect := func(k string, entries []dataWithTxNum) { + changed := false + for i := range entries { + if entries[i].txNum >= from && entries[i].txNum < to { + changed = true + break + } + } + if !changed { + return + } + pre, ok := asOfEntry(entries, from) + if !ok { + pre, _, _ = AggTx(roTx).GetAsOf(domain, common.ToBytesZeroCopy(k), from, roTx) + } + pairs = append(pairs, memKVPair{k: []byte(k), v: pre}) + } + if domain == kv.StorageDomain { + sd.storage.Scan(func(k string, entries []dataWithTxNum) bool { + collect(k, entries) + return true + }) + } else { + for k, entries := range sd.domains[domain] { + collect(k, entries) + } + } + sort.Slice(pairs, func(i, j int) bool { + if asc == order.Asc { + return bytes.Compare(pairs[i].k, pairs[j].k) < 0 + } + return bytes.Compare(pairs[i].k, pairs[j].k) > 0 + }) + return pairs +} + type memKVPair struct{ k, v []byte } type memKVIter struct { diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index c2d8b4de67e..b55c13b4887 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -111,6 +111,10 @@ func (t *sdRoTx) RangeAsOf(name kv.Domain, fromKey, toKey []byte, ts uint64, asc return t.sd.RangeAsOf(context.Background(), name, fromKey, toKey, ts, asc, limit, t.base) } +func (t *sdRoTx) HistoryRange(name kv.Domain, fromTs, toTs int, asc order.By, limit int) (stream.KV, error) { + return t.sd.HistoryRange(context.Background(), name, fromTs, toTs, asc, limit, t.base) +} + func (t *sdRoTx) Rollback() { t.base.Rollback() } From 2de5052d1a52689df19d8feb6a854f9c5276cbc3 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:05:32 +0000 Subject: [PATCH 122/167] db/state, execmodule: SharedDomains.IndexRange over in-mem history Add SharedDomains.IndexRange (via TemporalMemBatch.IndexRange): for domain-backed inverted indices, derive the in-flight change txNums from the in-memory domain history and union them with the committed index; standalone indices fall back to committed. Wired from the RPC test harness OverlayDB. --- db/kv/kv_interface.go | 1 + db/state/execctx/domain_shared.go | 6 ++ db/state/temporal_mem_batch.go | 61 +++++++++++++++++++ .../execmoduletester/overlay_read.go | 4 ++ 4 files changed, 72 insertions(+) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 0e8fba3bd16..036ff5617c3 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -562,6 +562,7 @@ type TemporalMemBatch interface { IteratePrefix(domain Domain, prefix []byte, roTx Tx, it func(k []byte, v []byte) (cont bool, err error)) error RangeAsOf(ctx context.Context, domain Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int, roTx Tx) (stream.KV, error) HistoryRange(ctx context.Context, domain Domain, fromTs, toTs int, asc order.By, limit int, roTx Tx) (stream.KV, error) + IndexRange(name InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int, roTx Tx) (stream.U64, error) HasPrefix(domain Domain, prefix []byte, roTx Tx) ([]byte, []byte, bool, error) HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index f73c6317869..1d1ecfae1a3 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1585,6 +1585,12 @@ func (sd *SharedDomains) HistoryRange(ctx context.Context, domain kv.Domain, fro return sd.mem.HistoryRange(ctx, domain, fromTs, toTs, asc, limit, roTx) } +// IndexRange returns the txNums at which key k changed in [fromTs, toTs), +// merging in-flight in-memory history with the committed inverted index. +func (sd *SharedDomains) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int, roTx kv.Tx) (stream.U64, error) { + return sd.mem.IndexRange(name, k, fromTs, toTs, asc, limit, roTx) +} + // DomainPut // Optimizations: // - user can provide `prevVal != nil` - then it will not read prev value from storage diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 28d4a60037d..96aec8b8031 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -463,6 +463,67 @@ func (sd *TemporalMemBatch) memHistoryRange(domain kv.Domain, fromTs, toTs int, return pairs } +// IndexRange returns the txNums at which key k changed in [fromTs, toTs), +// merging the in-flight in-memory history with the committed inverted index. +// Only domain-backed indices contribute in-memory txNums; standalone indices +// (logs, traces) fall back to committed. Only used by the RPC test harness. +func (sd *TemporalMemBatch) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int, roTx kv.Tx) (stream.U64, error) { + committed, err := AggTx(roTx).IndexRange(name, k, fromTs, toTs, asc, limit, roTx) + if err != nil { + return nil, err + } + if !sd.inMemHistoryReads { + return committed, nil + } + at := AggTx(roTx) + var domain kv.Domain + found := false + for i := range at.d { + if at.d[i].d.HistoryIdx == name { + domain = kv.Domain(i) + found = true + break + } + } + if !found { + return committed, nil + } + memTs := sd.memIndexTxNums(domain, k, fromTs, toTs, asc) + if len(memTs) == 0 { + return committed, nil + } + return stream.Union[uint64](stream.Array(memTs), committed, asc, limit), nil +} + +func (sd *TemporalMemBatch) memIndexTxNums(domain kv.Domain, k []byte, fromTs, toTs int, asc order.By) []uint64 { + sd.latestStateLock.RLock() + defer sd.latestStateLock.RUnlock() + ks := common.ToStringZeroCopy(k) + var entries []dataWithTxNum + if domain == kv.StorageDomain { + entries, _ = sd.storage.Get(ks) + } else { + entries = sd.domains[domain][ks] + } + out := make([]uint64, 0, len(entries)) + for i := range entries { + tn := entries[i].txNum + if fromTs >= 0 && tn < uint64(fromTs) { + continue + } + if toTs >= 0 && tn >= uint64(toTs) { + continue + } + out = append(out, tn) + } + if asc == order.Desc { + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + } + return out +} + type memKVPair struct{ k, v []byte } type memKVIter struct { diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index b55c13b4887..6cac3be0227 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -115,6 +115,10 @@ func (t *sdRoTx) HistoryRange(name kv.Domain, fromTs, toTs int, asc order.By, li return t.sd.HistoryRange(context.Background(), name, fromTs, toTs, asc, limit, t.base) } +func (t *sdRoTx) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int) (stream.U64, error) { + return t.sd.IndexRange(name, k, fromTs, toTs, asc, limit, t.base) +} + func (t *sdRoTx) Rollback() { t.base.Rollback() } From ce06873be1e690cf0a7a652d75b2c937b27a068e Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:11:10 +0000 Subject: [PATCH 123/167] rpc/jsonrpc: route test state cache + block reads via published SD (bg-commit) newBaseApiWithFiltersForTest uses the SD-wired m.StateCache so EstimateGas/EthCall state reads observe the in-flight tip; chainWithDeployedContractAndConfig and the ReplayTransaction block lookup read via m.OverlayDB() so historical/tip blocks resolve under background commit instead of a nil raw-DB block. --- rpc/jsonrpc/eth_call_test.go | 2 +- rpc/jsonrpc/eth_filters_test.go | 6 ++++-- rpc/jsonrpc/trace_adhoc_test.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rpc/jsonrpc/eth_call_test.go b/rpc/jsonrpc/eth_call_test.go index cdfe623a3d1..9104fd70a9a 100644 --- a/rpc/jsonrpc/eth_call_test.go +++ b/rpc/jsonrpc/eth_call_test.go @@ -725,7 +725,7 @@ func chainWithDeployedContractAndConfig(t *testing.T, cfg *chain.Config) (*execm _, fillerPublicKeys, err := generatePseudoRandomECDSAKeyPairs(rng, nFillerAccounts) require.NoError(t, err) - db := m.DB + db := m.OverlayDB() var contractAddr common.Address diff --git a/rpc/jsonrpc/eth_filters_test.go b/rpc/jsonrpc/eth_filters_test.go index eee0d2aa4ca..82c98184997 100644 --- a/rpc/jsonrpc/eth_filters_test.go +++ b/rpc/jsonrpc/eth_filters_test.go @@ -41,8 +41,10 @@ import ( "github.com/erigontech/erigon/rpc/rpchelper" ) -func newBaseApiWithFiltersForTest(f *rpchelper.Filters, stateCache *kvcache.Coherent, m *execmoduletester.ExecModuleTester) *BaseAPI { - return NewBaseApi(f, stateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) +func newBaseApiWithFiltersForTest(f *rpchelper.Filters, _ *kvcache.Coherent, m *execmoduletester.ExecModuleTester) *BaseAPI { + // Use the SD-wired state cache so reads observe the in-flight tip under + // background commit, not a plain coherent cache fed by stale notifications. + return NewBaseApi(f, m.StateCache, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) } func TestSubscriptionsRequireFiltersAndNotifier(t *testing.T) { diff --git a/rpc/jsonrpc/trace_adhoc_test.go b/rpc/jsonrpc/trace_adhoc_test.go index 61ac26da06e..f67c780235d 100644 --- a/rpc/jsonrpc/trace_adhoc_test.go +++ b/rpc/jsonrpc/trace_adhoc_test.go @@ -320,7 +320,7 @@ func TestReplayTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) api := NewTraceAPI(newBaseApiForTest(m), m.OverlayDB(), &httpcfg.HttpCfg{}) var txnHash common.Hash - if err := m.DB.View(context.Background(), func(tx kv.Tx) error { + if err := m.OverlayDB().View(context.Background(), func(tx kv.Tx) error { b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 6) if err != nil { return err From aa69f558e8fdb90613a7c518add48adb6c8fdfa1 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:27:38 +0000 Subject: [PATCH 124/167] rpc/jsonrpc: route remaining test-body block/state reads via published SD Sweep the test-body raw m.DB.View/BeginTemporalRo/BeginRo lookups (block, log and state fetches for setup/assertions) to m.OverlayDB() so they resolve the in-flight tip under background commit instead of a nil/stale raw DB. Removes the nil-deref panics that aborted the suite and fixes the block-by-timestamp reads. --- rpc/jsonrpc/erigon_receipts_test.go | 2 +- rpc/jsonrpc/eth_call_test.go | 12 ++++++------ rpc/jsonrpc/eth_system_test.go | 2 +- rpc/jsonrpc/overlay_api_test.go | 2 +- rpc/jsonrpc/overlay_race_test.go | 2 +- rpc/jsonrpc/trace_adhoc_test.go | 2 +- rpc/jsonrpc/trace_filtering_test.go | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rpc/jsonrpc/erigon_receipts_test.go b/rpc/jsonrpc/erigon_receipts_test.go index 9eeaa535dae..6a1f05a929b 100644 --- a/rpc/jsonrpc/erigon_receipts_test.go +++ b/rpc/jsonrpc/erigon_receipts_test.go @@ -206,7 +206,7 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) { 3: `[]`, 4: `[]`, } - err := m.DB.View(m.Ctx, func(tx kv.Tx) error { + err := m.OverlayDB().View(m.Ctx, func(tx kv.Tx) error { for i := uint64(0); i <= rawdb.ReadCurrentHeader(tx).Number.Uint64(); i++ { block := rawdb.ReadHeaderByNumber(tx, i) diff --git a/rpc/jsonrpc/eth_call_test.go b/rpc/jsonrpc/eth_call_test.go index 9104fd70a9a..f54dee595cf 100644 --- a/rpc/jsonrpc/eth_call_test.go +++ b/rpc/jsonrpc/eth_call_test.go @@ -398,7 +398,7 @@ func TestGetProof(t *testing.T) { require.NoError(t, err) require.NotNil(t, proof) - tx, err := m.DB.BeginTemporalRo(context.Background()) + tx, err := m.OverlayDB().BeginTemporalRo(context.Background()) require.NoError(t, err) defer tx.Rollback() header, err := api.headerByNumber(context.Background(), rpc.BlockNumber(tt.blockNum), tx) @@ -432,7 +432,7 @@ func TestGetProof(t *testing.T) { func TestGetBlockByTimestampLatestTime(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestExecModule(t) - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) @@ -459,7 +459,7 @@ func TestGetBlockByTimestampLatestTime(t *testing.T) { func TestGetBlockByTimestampOldestTime(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestExecModule(t) - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) @@ -487,7 +487,7 @@ func TestGetBlockByTimestampOldestTime(t *testing.T) { func TestGetBlockByTimeHigherThanLatestBlock(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestExecModule(t) - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) @@ -515,7 +515,7 @@ func TestGetBlockByTimeHigherThanLatestBlock(t *testing.T) { func TestGetBlockByTimeMiddle(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestExecModule(t) - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) @@ -548,7 +548,7 @@ func TestGetBlockByTimeMiddle(t *testing.T) { func TestGetBlockByTimestamp(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestExecModule(t) - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() api := NewErigonAPI(newBaseApiForTest(m), m.OverlayDB(), nil) diff --git a/rpc/jsonrpc/eth_system_test.go b/rpc/jsonrpc/eth_system_test.go index b4b741b122a..1e3b04c73cb 100644 --- a/rpc/jsonrpc/eth_system_test.go +++ b/rpc/jsonrpc/eth_system_test.go @@ -105,7 +105,7 @@ func TestCapabilities(t *testing.T) { } require.NoError(t, tx.Commit()) - roTx, err := m.DB.BeginTemporalRo(ctx) + roTx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer roTx.Rollback() head, err := stages.GetStageProgress(roTx, stages.Execution) diff --git a/rpc/jsonrpc/overlay_api_test.go b/rpc/jsonrpc/overlay_api_test.go index 7f7f517d807..c8d377ad161 100644 --- a/rpc/jsonrpc/overlay_api_test.go +++ b/rpc/jsonrpc/overlay_api_test.go @@ -31,7 +31,7 @@ import ( func TestOverlayGetBeginEnd(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) api := &OverlayAPIImpl{BaseAPI: newBaseApiForTest(m)} - tx, err := m.DB.BeginTemporalRo(m.Ctx) + tx, err := m.OverlayDB().BeginTemporalRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() diff --git a/rpc/jsonrpc/overlay_race_test.go b/rpc/jsonrpc/overlay_race_test.go index 173adb62361..849be59133c 100644 --- a/rpc/jsonrpc/overlay_race_test.go +++ b/rpc/jsonrpc/overlay_race_test.go @@ -73,7 +73,7 @@ func newOverlayAheadTestAPI(t *testing.T) (base *BaseAPI, m *execmoduletester.Ex require.NoError(t, m.InsertChain(c)) ctx := m.Ctx - overlayRoTx, err := m.DB.BeginTemporalRo(ctx) + overlayRoTx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) t.Cleanup(overlayRoTx.Rollback) doms, err := execctx.NewSharedDomains(ctx, overlayRoTx, m.Log) diff --git a/rpc/jsonrpc/trace_adhoc_test.go b/rpc/jsonrpc/trace_adhoc_test.go index f67c780235d..98777cac412 100644 --- a/rpc/jsonrpc/trace_adhoc_test.go +++ b/rpc/jsonrpc/trace_adhoc_test.go @@ -471,7 +471,7 @@ func TestOeTracer(t *testing.T) { // returns its binary encoding together with the sender and recipient addresses. func rawTxFromBlock(t *testing.T, m *execmoduletester.ExecModuleTester, blockNum uint64) (encoded []byte, from, to accounts.Address) { t.Helper() - if err := m.DB.View(context.Background(), func(tx kv.Tx) error { + if err := m.OverlayDB().View(context.Background(), func(tx kv.Tx) error { b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, blockNum) if err != nil { return err diff --git a/rpc/jsonrpc/trace_filtering_test.go b/rpc/jsonrpc/trace_filtering_test.go index 72edbb0bf7c..8b76fa2072b 100644 --- a/rpc/jsonrpc/trace_filtering_test.go +++ b/rpc/jsonrpc/trace_filtering_test.go @@ -56,7 +56,7 @@ func TestCallBlockParallelMatchesSequential(t *testing.T) { const blockNum = uint64(6) // block 6 has 32 txs (case i=5 in test chain generation) traceTypes := []string{TraceTypeTrace} - tx, err := m.DB.BeginTemporalRo(ctx) + tx, err := m.OverlayDB().BeginTemporalRo(ctx) require.NoError(t, err) defer tx.Rollback() From f239ba6f05c82deab0c8f91b7f891c7c7e6cf2b2 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:29:33 +0000 Subject: [PATCH 125/167] execmodule/execmoduletester: retry InsertChain FCU on ExecutionStatusBusy Under background commit the FCU returns Busy when the commit backlog is full; the real CL retries. Mirror that in the test harness (bounded retry yielding to the commit worker) instead of failing the insert. --- .../execmoduletester/exec_module_tester.go | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 8ae0b988f05..4c44bf84fb5 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -885,9 +885,23 @@ func (emt *ExecModuleTester) insertPoSBlocks(chain *blockgen.ChainPack) error { tipHash := chain.TopBlock.Hash() - status, verr, _, err := wr.UpdateForkChoice(emt.Ctx, tipHash, tipHash, tipHash) - if err != nil { - return err + // Busy means the background-commit backlog is full; the real CL retries the + // FCU, giving the commit worker a foreground-idle window to drain. Mirror + // that here instead of failing the insert. + var status execmodule.ExecutionStatus + var verr *string + for { + status, verr, _, err = wr.UpdateForkChoice(emt.Ctx, tipHash, tipHash, tipHash) + if err != nil { + return err + } + if status != execmodule.ExecutionStatusBusy { + break + } + if emt.Ctx.Err() != nil { + return emt.Ctx.Err() + } + time.Sleep(time.Millisecond) } if status != execmodule.ExecutionStatusSuccess { From 04ab2535322917775816b8e332b6c4f7432c60e0 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:46:51 +0000 Subject: [PATCH 126/167] rpc/jsonrpc: head-pointer tests read raw DB, not published-SD overlay GetBlockByNumber latest/finalized/safe-tag tests and the eth_config head subtest write a committed forkchoice/head pointer directly to the raw DB (no in-flight FCU), so they must resolve via m.DB; the published-SD overlay only holds the genesis head. Revert their API db arg from m.OverlayDB() to m.DB. --- rpc/jsonrpc/eth_block_test.go | 6 +++--- rpc/jsonrpc/eth_system_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rpc/jsonrpc/eth_block_test.go b/rpc/jsonrpc/eth_block_test.go index dfda7ef1614..ed380b3361a 100644 --- a/rpc/jsonrpc/eth_block_test.go +++ b/rpc/jsonrpc/eth_block_test.go @@ -73,7 +73,7 @@ func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { } tx.Commit() - api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) block, err := api.GetBlockByNumber(ctx, rpc.LatestBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -147,7 +147,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) } tx.Commit() - api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) block, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -191,7 +191,7 @@ func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { } tx.Commit() - api := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) + api := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) block, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) diff --git a/rpc/jsonrpc/eth_system_test.go b/rpc/jsonrpc/eth_system_test.go index 1e3b04c73cb..c70354f44c2 100644 --- a/rpc/jsonrpc/eth_system_test.go +++ b/rpc/jsonrpc/eth_system_test.go @@ -562,7 +562,7 @@ func TestEthConfig(t *testing.T) { require.NoError(t, err) m := execmoduletester.New(t, execmoduletester.WithGenesisSpec(&genesis), execmoduletester.WithKey(key)) defer m.Close() - eth := newEthApiForTest(newBaseApiForTest(m), m.OverlayDB(), nil, nil) + eth := newEthApiForTest(newBaseApiForTest(m), m.DB, nil, nil) if test.head != nil { tx, err := m.DB.BeginTemporalRw(ctx) require.NoError(t, err) From 25b379bdad9831d6d4749b97e0760475c99c3f60 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:50:35 +0000 Subject: [PATCH 127/167] db/state, execmodule: SharedDomains.HistorySeek over in-mem history Add SharedDomains.HistorySeek (via TemporalMemBatch.HistorySeek): return the in-flight in-memory historical value of a key as of ts (value before the first recorded change at/after ts; creation yields non-nil empty), falling back to committed history on miss. Wired from the RPC test harness OverlayDB so otterscan GetContractCreator/GetTransactionBySenderAndNonce resolve under background commit. --- db/kv/kv_interface.go | 1 + db/state/execctx/domain_shared.go | 5 ++++ db/state/temporal_mem_batch.go | 29 +++++++++++++++++++ .../execmoduletester/overlay_read.go | 7 +++++ 4 files changed, 42 insertions(+) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 036ff5617c3..275adea9492 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -563,6 +563,7 @@ type TemporalMemBatch interface { RangeAsOf(ctx context.Context, domain Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int, roTx Tx) (stream.KV, error) HistoryRange(ctx context.Context, domain Domain, fromTs, toTs int, asc order.By, limit int, roTx Tx) (stream.KV, error) IndexRange(name InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int, roTx Tx) (stream.U64, error) + HistorySeek(domain Domain, key []byte, ts uint64) (v []byte, ok bool, err error) HasPrefix(domain Domain, prefix []byte, roTx Tx) ([]byte, []byte, bool, error) HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 1d1ecfae1a3..081b5749b16 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1591,6 +1591,11 @@ func (sd *SharedDomains) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs return sd.mem.IndexRange(name, k, fromTs, toTs, asc, limit, roTx) } +// HistorySeek returns the in-flight in-memory historical value of key as of ts. +func (sd *SharedDomains) HistorySeek(domain kv.Domain, key []byte, ts uint64) ([]byte, bool, error) { + return sd.mem.HistorySeek(domain, key, ts) +} + // DomainPut // Optimizations: // - user can provide `prevVal != nil` - then it will not read prev value from storage diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 96aec8b8031..7145c9ba3d2 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -388,6 +388,35 @@ func (sd *TemporalMemBatch) memRangeAsOf(domain kv.Domain, fromKey, toKey []byte return pairs } +// HistorySeek returns the in-memory historical value of key as of ts: the value +// just before the first recorded change at or after ts. ok is true when the key +// has such a change in memory; a creation event yields (non-nil empty, true). +// Returns (nil, false) when ts is past all in-memory changes so the caller can +// fall back to committed history. +func (sd *TemporalMemBatch) HistorySeek(domain kv.Domain, key []byte, ts uint64) ([]byte, bool, error) { + if !sd.inMemHistoryReads { + return nil, false, nil + } + sd.latestStateLock.RLock() + defer sd.latestStateLock.RUnlock() + ks := common.ToStringZeroCopy(key) + var entries []dataWithTxNum + if domain == kv.StorageDomain { + entries, _ = sd.storage.Get(ks) + } else { + entries = sd.domains[domain][ks] + } + for i := range entries { + if entries[i].txNum >= ts { + if i == 0 { + return []byte{}, true, nil + } + return entries[i-1].data, true, nil + } + } + return nil, false, nil +} + // asOfEntry returns the value in effect at ts from a txNum-ascending history // (the entry with the greatest txNum < ts). ok is false when ts precedes the // first entry, i.e. the key did not exist yet. diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index 6cac3be0227..431ece2e68a 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -119,6 +119,13 @@ func (t *sdRoTx) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc return t.sd.IndexRange(name, k, fromTs, toTs, asc, limit, t.base) } +func (t *sdRoTx) HistorySeek(name kv.Domain, k []byte, ts uint64) ([]byte, bool, error) { + if v, ok, err := t.sd.HistorySeek(name, k, ts); err != nil || ok { + return v, ok, err + } + return t.TemporalTx.HistorySeek(name, k, ts) +} + func (t *sdRoTx) Rollback() { t.base.Rollback() } From 3081f3856482195410a540b775d36fe9f23989f0 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:51:33 +0000 Subject: [PATCH 128/167] rpc/jsonrpc: route ListStorageKeys parity tests via published SD NewParityAPIImpl was missed by the API-constructor rollout (name ends APIImpl); route its db arg to m.OverlayDB() so ListStorageKeys resolves the account and storage range on the in-flight tip under background commit. --- rpc/jsonrpc/parity_api_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rpc/jsonrpc/parity_api_test.go b/rpc/jsonrpc/parity_api_test.go index 097f3d115d1..dbb95e44212 100644 --- a/rpc/jsonrpc/parity_api_test.go +++ b/rpc/jsonrpc/parity_api_test.go @@ -38,7 +38,7 @@ func TestParityAPIImpl_ListStorageKeys_NoOffset(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) baseApi := NewBaseApi(nil, nil, m.BlockReader, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, 0, 0) - api := NewParityAPIImpl(baseApi, m.DB) + api := NewParityAPIImpl(baseApi, m.OverlayDB()) answers := []string{ "0000000000000000000000000000000000000000000000000000000000000000", "0000000000000000000000000000000000000000000000000000000000000002", @@ -60,7 +60,7 @@ func TestParityAPIImpl_ListStorageKeys_NoOffset(t *testing.T) { func TestParityAPIImpl_ListStorageKeys_WithOffset_ExistingPrefix(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.OverlayDB()) answers := []string{ "29d05770ca9ee7088a64e18c8e5160fc62c3c2179dc8ef9b4dbc970c9e51b4d8", "29edc84535d98b29835079d685b97b41ee8e831e343cc80793057e462353a26d", @@ -84,7 +84,7 @@ func TestParityAPIImpl_ListStorageKeys_WithOffset_ExistingPrefix(t *testing.T) { func TestParityAPIImpl_ListStorageKeys_WithOffset_NonExistingPrefix(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.OverlayDB()) answers := []string{ "4644be453c81744b6842ddf615d7fca0e14a23b09734be63d44c23452de95631", "4974416255391052161ba8184fe652f3bf8c915592c65f7de127af8e637dce5d", @@ -105,7 +105,7 @@ func TestParityAPIImpl_ListStorageKeys_WithOffset_NonExistingPrefix(t *testing.T func TestParityAPIImpl_ListStorageKeys_WithOffset_EmptyResponse(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.OverlayDB()) addr := common.HexToAddress("0x920fd5070602feaea2e251e9e7238b6c376bcae5") offset := common.Hex2Bytes("ff") b := hexutil.Bytes(offset) @@ -119,7 +119,7 @@ func TestParityAPIImpl_ListStorageKeys_WithOffset_EmptyResponse(t *testing.T) { func TestParityAPIImpl_ListStorageKeys_AccNotFound(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestExecModule(t) - api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.OverlayDB()) addr := common.HexToAddress("0x920fd5070602feaea2e251e9e7238b6c376bcaef") _, err := api.ListStorageKeys(context.Background(), addr, 2, nil, latestBlock) assert.Error(err, errors.New("acc not found")) From 751459009af359066037df41338cefe802fb33bc Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 14:57:55 +0000 Subject: [PATCH 129/167] rpc/jsonrpc: resolve head-pointer test blocks via overlay before raw writes The finalized/safe/latest-tag tests read the tip block through the raw RW tx to get its number, which nil-derefs when the tip is not yet committed under background commit (timing-dependent panic). Resolve the block via m.OverlayDB() first, then open the raw RW tx only for the head-pointer writes. --- rpc/jsonrpc/eth_block_test.go | 43 ++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/rpc/jsonrpc/eth_block_test.go b/rpc/jsonrpc/eth_block_test.go index ed380b3361a..794a967d5b0 100644 --- a/rpc/jsonrpc/eth_block_test.go +++ b/rpc/jsonrpc/eth_block_test.go @@ -29,6 +29,7 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/db/kv/kvcache" "github.com/erigontech/erigon/db/kv/prune" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/execmoduletester" "github.com/erigontech/erigon/execution/rlp" @@ -55,16 +56,18 @@ func TestGetBlockByNumberWithLatestTag(t *testing.T) { func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() + latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") + var latestBlock *types.Block + require.NoError(t, m.OverlayDB().View(ctx, func(rtx kv.Tx) (err error) { + latestBlock, err = m.BlockReader.BlockByHash(ctx, rtx, latestBlockHash) + return err + })) + require.NotNil(t, latestBlock, "couldn't retrieve latest block") + tx, err := m.DB.BeginRw(ctx) require.NoError(t, err) defer tx.Rollback() - latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") - latestBlock, err := m.BlockReader.BlockByHash(ctx, tx, latestBlockHash) - if err != nil { - tx.Rollback() - t.Errorf("couldn't retrieve latest block") - } rawdb.WriteHeaderNumber(tx, latestBlockHash, latestBlock.NonceU64()) rawdb.WriteForkchoiceHead(tx, latestBlockHash) if safedHeadBlock := rawdb.ReadForkchoiceHead(tx); safedHeadBlock == (common.Hash{}) { @@ -129,16 +132,18 @@ func TestGetBlockByNumber_WithFinalizedTag_NoFinalizedBlockInDb(t *testing.T) { func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() + latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") + var latestBlock *types.Block + require.NoError(t, m.OverlayDB().View(ctx, func(rtx kv.Tx) (err error) { + latestBlock, err = m.BlockReader.BlockByHash(ctx, rtx, latestBlockHash) + return err + })) + require.NotNil(t, latestBlock, "couldn't retrieve latest block") + tx, err := m.DB.BeginRw(ctx) require.NoError(t, err) defer tx.Rollback() - latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") - latestBlock, err := m.BlockReader.BlockByHash(ctx, tx, latestBlockHash) - if err != nil { - tx.Rollback() - t.Errorf("couldn't retrieve latest block") - } rawdb.WriteHeaderNumber(tx, latestBlockHash, latestBlock.NonceU64()) rawdb.WriteForkchoiceFinalized(tx, latestBlockHash) if safedFinalizedBlock := rawdb.ReadForkchoiceFinalized(tx); safedFinalizedBlock == (common.Hash{}) { @@ -173,16 +178,18 @@ func TestGetBlockByNumber_WithSafeTag_NoSafeBlockInDb(t *testing.T) { func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestExecModule(t) ctx := context.Background() + latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") + var latestBlock *types.Block + require.NoError(t, m.OverlayDB().View(ctx, func(rtx kv.Tx) (err error) { + latestBlock, err = m.BlockReader.BlockByHash(ctx, rtx, latestBlockHash) + return err + })) + require.NotNil(t, latestBlock, "couldn't retrieve latest block") + tx, err := m.DB.BeginRw(ctx) require.NoError(t, err) defer tx.Rollback() - latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") - latestBlock, err := m.BlockReader.BlockByHash(ctx, tx, latestBlockHash) - if err != nil { - tx.Rollback() - t.Errorf("couldn't retrieve latest block") - } rawdb.WriteHeaderNumber(tx, latestBlockHash, latestBlock.NonceU64()) rawdb.WriteForkchoiceSafe(tx, latestBlockHash) if safedSafeBlock := rawdb.ReadForkchoiceSafe(tx); safedSafeBlock == (common.Hash{}) { From 25ede7e95dfa16d83f6f2299b24e80055f04b3c8 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 16:25:53 +0000 Subject: [PATCH 130/167] rpc/jsonrpc: eth_getProof builds tip proof against in-flight commitment For a latest-block proof under background commit, chain the fresh SharedDomains to the published SharedDomains (exposed via the overlay tx) and seek commitment through it, so Witness builds the account-proof trie against the in-flight commitment root instead of the stale committed one. --- .../execmodule/execmoduletester/overlay_read.go | 5 +++++ rpc/jsonrpc/eth_call.go | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index 431ece2e68a..e93a50f55ad 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -92,6 +92,11 @@ type sdRoTx struct { sd *execctx.SharedDomains } +// PublishedSharedDomains exposes the published SharedDomains this tx reads +// through, so a consumer building a fresh SharedDomains (e.g. eth_getProof) can +// chain it as a parent to see the in-flight tip under background commit. +func (t *sdRoTx) PublishedSharedDomains() *execctx.SharedDomains { return t.sd } + func (t *sdRoTx) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { return t.sd.GetLatest(name, t.base, k) } diff --git a/rpc/jsonrpc/eth_call.go b/rpc/jsonrpc/eth_call.go index 9be27296e01..c8130b1e9de 100644 --- a/rpc/jsonrpc/eth_call.go +++ b/rpc/jsonrpc/eth_call.go @@ -504,6 +504,19 @@ func (api *APIImpl) getProof(ctx context.Context, roTx kv.TemporalTx, address co if _, _, err := domains.SeekCommitment(context.Background(), roTx); err != nil { return nil, err } + } else if p, ok := tx.(interface { + PublishedSharedDomains() *execctx.SharedDomains + }); ok { + // Requested block is the tip. Under background commit its commitment + // still lives in the published SharedDomains, so chain the fresh domains + // to it and seek commitment through the chain to build the proof against + // the in-flight trie root. + if psd := p.PublishedSharedDomains(); psd != nil { + domains.SetParent(psd) + if _, _, err := domains.SeekCommitment(context.Background(), roTx); err != nil { + return nil, err + } + } } // touch account From 2f205c324c08be2cbe31c389ecf51802a05d3686 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 16:29:32 +0000 Subject: [PATCH 131/167] execution/execmodule: clear published overlay after SetHead unwind SetHead commits the unwind to the raw DB but the last FCU's published SharedDomains still points at the unwound-away tip, so overlay-routed readers served stale canonical hashes. Publish a nil overlay after the unwind so readers fall through to the raw DB. --- execution/execmodule/set_head.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 8b348b94ed0..c0723f16159 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -163,6 +163,13 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { return fmt.Errorf("failed to commit shared domains: %w", err) } + // Clear the published overlay: the last FCU's SharedDomains now points at the + // unwound-away tip, so readers must fall through to the raw DB this unwind + // just committed instead of serving stale overlay state. + if dispatcher := e.pipelineExecutor.Dispatcher(); dispatcher != nil { + dispatcher.PublishOverlay(nil) + } + e.logger.Info("SetHead: successfully rewound chain", "targetBlock", targetBlock, "previousHead", currentHead) return nil } From 957e42e56fad2420a09f8b5e282704bf2d1b7f68 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 16:32:18 +0000 Subject: [PATCH 132/167] execution/execmodule: reset gen chain + context on SetHead unwind SetHead unwinds directly to the raw DB, bypassing the background-commit generations and current context which still describe the unwound-away tip. Drop them so the next UpdateForkChoice rebuilds from the committed raw DB instead of treating the stale tip as current (which no-op'd the forward re-org). --- execution/execmodule/set_head.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index c0723f16159..620ace9fdf0 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -163,9 +163,12 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { return fmt.Errorf("failed to commit shared domains: %w", err) } - // Clear the published overlay: the last FCU's SharedDomains now points at the - // unwound-away tip, so readers must fall through to the raw DB this unwind - // just committed instead of serving stale overlay state. + // The direct unwind above superseded any in-flight background-commit + // generations and the current context (they describe the unwound-away tip), + // so drop them and clear the published overlay. Readers and the next FCU + // then rebuild from the raw DB this unwind just committed. + e.closeAllGens() + e.closeModuleContext() if dispatcher := e.pipelineExecutor.Dispatcher(); dispatcher != nil { dispatcher.PublishOverlay(nil) } From dd04431fcd0fcbb5c61865dc48a913fb4e5df7aa Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 16:50:50 +0000 Subject: [PATCH 133/167] db/state: keep standalone inverted-index writes in a local collection The domain writes go to a queryable in-mem collection (sd.domains/storage) as well as the etl writer, which is what lets IndexRange/RangeAsOf read the in-flight tip. Standalone inverted indices (logs, traces) only wrote to the write-only etl buffer, so IndexRange couldn't answer in-flight queries for them (trace_filter/getLogs by address, otterscan search missed the uncommitted tip under background commit). Add iiMem: a local key->txNums collection populated by IndexAdd (gated on inMemHistoryReads, like the domain history), unwound and cleared with the other collections, and read by IndexRange alongside committed. Fix the range-bound handling to honor asc=[from,to) / desc=(to,from]. --- db/state/temporal_mem_batch.go | 101 ++++++++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 15 deletions(-) diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 7145c9ba3d2..001b8220647 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -70,6 +70,13 @@ type TemporalMemBatch struct { domainWriters [kv.DomainLen]*DomainBufferedWriter iiWriters []*InvertedIndexBufferedWriter + // iiMem is the queryable local copy of standalone inverted-index writes + // (index -> key -> ascending txNums). The ii writers are write-only etl + // buffers, so without this IndexRange can't answer in-flight queries for + // standalone indices (traces, logs) the way it can for domain-backed ones + // via the domain history. Populated by IndexAdd only when inMemHistoryReads + // is set — the same gate under which the domain history above is retained. + iiMem map[kv.InvertedIdx]map[string][]uint64 pastDomainWriters [kv.DomainLen][]*DomainBufferedWriter pastIIWriters []*InvertedIndexBufferedWriter @@ -106,6 +113,7 @@ func NewTemporalMemBatch(tx kv.TemporalTx, ioMetrics any) *TemporalMemBatch { storage: btree2.NewMap[string, []dataWithTxNum](128), metrics: ioMetrics.(*kvmetrics.DomainMetrics), inMemHistoryReads: true, + iiMem: map[kv.InvertedIdx]map[string][]uint64{}, } aggTx := AggTx(tx) sd.stepSize = aggTx.StepSize() @@ -492,10 +500,10 @@ func (sd *TemporalMemBatch) memHistoryRange(domain kv.Domain, fromTs, toTs int, return pairs } -// IndexRange returns the txNums at which key k changed in [fromTs, toTs), -// merging the in-flight in-memory history with the committed inverted index. -// Only domain-backed indices contribute in-memory txNums; standalone indices -// (logs, traces) fall back to committed. Only used by the RPC test harness. +// IndexRange returns the txNums at which key k appears in [fromTs, toTs), +// merging the in-flight in-memory index with the committed inverted index. +// Domain-backed indices derive their in-memory txNums from the domain history; +// standalone indices (logs, traces) read them from the local ii collection. func (sd *TemporalMemBatch) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int, roTx kv.Tx) (stream.U64, error) { committed, err := AggTx(roTx).IndexRange(name, k, fromTs, toTs, asc, limit, roTx) if err != nil { @@ -505,25 +513,48 @@ func (sd *TemporalMemBatch) IndexRange(name kv.InvertedIdx, k []byte, fromTs, to return committed, nil } at := AggTx(roTx) - var domain kv.Domain + var memTs []uint64 found := false for i := range at.d { if at.d[i].d.HistoryIdx == name { - domain = kv.Domain(i) + memTs = sd.memIndexTxNums(kv.Domain(i), k, fromTs, toTs, asc) found = true break } } if !found { - return committed, nil + memTs = sd.memIndexTxNumsII(name, k, fromTs, toTs, asc) } - memTs := sd.memIndexTxNums(domain, k, fromTs, toTs, asc) if len(memTs) == 0 { return committed, nil } return stream.Union[uint64](stream.Array(memTs), committed, asc, limit), nil } +// memIndexTxNumsII reads the local standalone inverted-index collection for the +// txNums of key k in [fromTs, toTs), deduplicated and ordered per asc. +func (sd *TemporalMemBatch) memIndexTxNumsII(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc order.By) []uint64 { + sd.latestStateLock.RLock() + defer sd.latestStateLock.RUnlock() + txNums := sd.iiMem[name][common.ToStringZeroCopy(k)] + out := make([]uint64, 0, len(txNums)) + for _, tn := range txNums { + if !idxTxNumInRange(tn, fromTs, toTs, asc) { + continue + } + if len(out) > 0 && out[len(out)-1] == tn { + continue // ascending append order makes duplicates consecutive + } + out = append(out, tn) + } + if asc == order.Desc { + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + } + return out +} + func (sd *TemporalMemBatch) memIndexTxNums(domain kv.Domain, k []byte, fromTs, toTs int, asc order.By) []uint64 { sd.latestStateLock.RLock() defer sd.latestStateLock.RUnlock() @@ -536,14 +567,9 @@ func (sd *TemporalMemBatch) memIndexTxNums(domain kv.Domain, k []byte, fromTs, t } out := make([]uint64, 0, len(entries)) for i := range entries { - tn := entries[i].txNum - if fromTs >= 0 && tn < uint64(fromTs) { - continue - } - if toTs >= 0 && tn >= uint64(toTs) { - continue + if tn := entries[i].txNum; idxTxNumInRange(tn, fromTs, toTs, asc) { + out = append(out, tn) } - out = append(out, tn) } if asc == order.Desc { for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { @@ -553,6 +579,15 @@ func (sd *TemporalMemBatch) memIndexTxNums(domain kv.Domain, k []byte, fromTs, t return out } +// idxTxNumInRange applies the inverted-index range bounds: asc = [fromTs, toTs), +// desc = (toTs, fromTs]. A negative bound is unbounded. +func idxTxNumInRange(tn uint64, fromTs, toTs int, asc order.By) bool { + if asc == order.Asc { + return (fromTs < 0 || tn >= uint64(fromTs)) && (toTs < 0 || tn < uint64(toTs)) + } + return (fromTs < 0 || tn <= uint64(fromTs)) && (toTs < 0 || tn > uint64(toTs)) +} + type memKVPair struct{ k, v []byte } type memKVIter struct { @@ -630,6 +665,7 @@ func (sd *TemporalMemBatch) ClearRam() { } sd.storage = btree2.NewMap[string, []dataWithTxNum](128) + sd.iiMem = map[kv.InvertedIdx]map[string][]uint64{} sd.unwindToTxNum = 0 sd.unwindChangeset = nil sd.unwindChangesetRaw = nil @@ -865,6 +901,24 @@ func (sd *TemporalMemBatch) Unwind(unwindToTxNum uint64, changeset *[kv.DomainLe sd.storage.Set(e.key, e.kept) } } + for table, byKey := range sd.iiMem { + for k, txNums := range byKey { + kept := txNums[:0] + for _, tn := range txNums { + if tn <= unwindToTxNum { + kept = append(kept, tn) + } + } + if len(kept) == 0 { + delete(byKey, k) + } else { + byKey[k] = kept + } + } + if len(byKey) == 0 { + delete(sd.iiMem, table) + } + } var unwindChangeset *[kv.DomainLen]map[string]kv.DomainEntryDiff var unwindChangesetRaw *[kv.DomainLen][]kv.DomainEntryDiff @@ -895,12 +949,29 @@ func (sd *TemporalMemBatch) Unwind(unwindToTxNum uint64, changeset *[kv.DomainLe func (sd *TemporalMemBatch) IndexAdd(table kv.InvertedIdx, key []byte, txNum uint64) (err error) { for _, writer := range sd.iiWriters { if writer.name == table { + if sd.inMemHistoryReads { + sd.iiMemAdd(table, key, txNum) + } return writer.Add(key, txNum) } } panic(fmt.Errorf("unknown index %s", table)) } +func (sd *TemporalMemBatch) iiMemAdd(table kv.InvertedIdx, key []byte, txNum uint64) { + sd.latestStateLock.Lock() + defer sd.latestStateLock.Unlock() + byKey := sd.iiMem[table] + if byKey == nil { + byKey = map[string][]uint64{} + sd.iiMem[table] = byKey + } + ks := string(key) + // IndexAdd is called in ascending txNum order during execution, so append + // keeps each key's txNum slice sorted. + byKey[ks] = append(byKey[ks], txNum) +} + func (sd *TemporalMemBatch) Close() { for _, d := range sd.domainWriters { if d != nil { From c83386a0bbf17872a245334baf3120c09123dafe Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 17:14:50 +0000 Subject: [PATCH 134/167] execution/execmodule: roll back generation read-tx in closeAllGens closeAllGens closed each generation's SharedDomains but not its read tx. A never-committed generation (runCommit rolls back committed ones) then leaked an open MDBX reader, wedging DB.Close on shutdown. Roll it back here (idempotent). --- execution/execmodule/bg_commit.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 1095d78c5f5..86283fdcddd 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -242,6 +242,12 @@ func (e *ExecModule) closeAllGens() { for _, g := range e.gens { g.sd.SetParent(nil) g.sd.Close() + // Roll back the generation's read tx: committed gens already had it + // rolled back in runCommit (Rollback is idempotent), but a never-committed + // gen would otherwise leak an open MDBX reader and wedge DB.Close. + if g.roTx != nil { + g.roTx.Rollback() + } } e.gens = nil e.uncommittedGens = 0 From 2472857a0989e9c11d12ef741d113acb9b8875c9 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Mon, 13 Jul 2026 17:17:24 +0000 Subject: [PATCH 135/167] execution/execmodule: route unwind-gap test blockgen via published SD The two blockgen.GenerateChain calls in exec_module_unwind_gap_test built on genesis without the published SD, so under background commit they read the genesis alloc as absent (insufficient funds panic). Pass m.PublishedSD(). --- execution/execmodule/exec_module_unwind_gap_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/execution/execmodule/exec_module_unwind_gap_test.go b/execution/execmodule/exec_module_unwind_gap_test.go index 6d1acff3a93..00454899730 100644 --- a/execution/execmodule/exec_module_unwind_gap_test.go +++ b/execution/execmodule/exec_module_unwind_gap_test.go @@ -76,7 +76,7 @@ func TestUpdateForkChoiceBadBlockMidBatchThenRecovery(t *testing.T) { ) require.NoError(t, err) b.AddTx(tx) - }) + }, m.PublishedSD()) require.NoError(t, err) require.Len(t, chainPack.Blocks, chainLen) @@ -196,7 +196,7 @@ func TestUpdateForkChoiceBadBlockAtLongBatchTailThenRecovery(t *testing.T) { } chainPack, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainLen+1, func(i int, b *blockgen.BlockGen) { b.AddTx(mkTx(i, 1_000)) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, insertValidateAndUfc1By1(ctx, m.ExecModule, chainPack.Blocks[:committedTo])) From 511b8ec259e2adec64134c8b4d673f04f06a0d72 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 11:08:35 +0000 Subject: [PATCH 136/167] execution: fix bg-commit test flake and route readers via published SD Under fcuBackgroundCommit, the test harness InsertChain returned before its commit landed, so a following operation (a reorg's unwind, a raw-DB read) could race the still-in-flight commit of the chain just inserted. A reorg's sd.Unwind (branchCache epoch bump + DB revert) racing a superseded-fork block's commitment on the shared branchCache produced "empty branch data read during unfold" / wrong-root flakes under parallel exec + concurrent test load. - execmoduletester.InsertChain now drains the background commit before returning, serializing bg-commit operations in tests. - bg_commit.go: latestGen returns only the newest not-yet-committed generation (a committed gen's state is already in the DB and would mask direct-DB reads); add WaitCommitsDrained() as a repeatable drain barrier for tests. - Route test readers through the published SharedDomains (OverlayDB / PublishedSD) instead of the raw DB, which lags under background commit. - SimulatedBackend generates its pending block on committed state with a fresh cache (drains after Commit), avoiding the shared-branchCache race. --- db/snapshotsync/freezeblocks/dump_test.go | 2 +- execution/abi/bind/backends/simulated.go | 10 ++- execution/execmodule/bg_commit.go | 27 ++++++-- execution/execmodule/exec_module_test.go | 11 ++-- .../execmodule/exec_module_unwind_gap_test.go | 2 + .../execmoduletester/exec_module_tester.go | 4 ++ .../exec_module_tester_test.go | 12 ++-- .../execmoduletester/overlay_read.go | 2 +- execution/state/database_test.go | 65 ++++++++++--------- execution/verify/history_verify_test.go | 4 +- rpc/jsonrpc/eth_block_test.go | 2 +- 11 files changed, 88 insertions(+), 53 deletions(-) diff --git a/db/snapshotsync/freezeblocks/dump_test.go b/db/snapshotsync/freezeblocks/dump_test.go index cfe6572946f..8ec4325029f 100644 --- a/db/snapshotsync/freezeblocks/dump_test.go +++ b/db/snapshotsync/freezeblocks/dump_test.go @@ -291,7 +291,7 @@ func createDumpTestKV(t *testing.T, chainConfig *chain.Config, chainSize int) *e t.Fatalf("failed to create tx: %v", txErr) } b.AddTx(tx) - }) + }, m.PublishedSD()) if err != nil { t.Fatal(err) } diff --git a/execution/abi/bind/backends/simulated.go b/execution/abi/bind/backends/simulated.go index 8d5e65c4e6f..73cfdd235ae 100644 --- a/execution/abi/bind/backends/simulated.go +++ b/execution/abi/bind/backends/simulated.go @@ -154,6 +154,10 @@ func (b *SimulatedBackend) Commit() { }); err != nil { panic(err) } + // Make the just-committed block durable before generating the next pending + // block on it: the generation below reads prependBlock's committed state + // with a fresh SharedDomains, so it must not race the background commit. + b.m.ExecModule.WaitCommitsDrained() //nolint:prealloc var allLogs []*types.Log for _, r := range b.pendingReceipts { @@ -173,7 +177,7 @@ func (b *SimulatedBackend) Rollback() { } func (b *SimulatedBackend) emptyPendingBlock() { - blockChain, _ := blockgen.GenerateChain(b.m.ChainConfig, b.prependBlock, b.m.Engine, b.m.DB, 1, func(int, *blockgen.BlockGen) {}, b.m.PublishedSD()) + blockChain, _ := blockgen.GenerateChain(b.m.ChainConfig, b.prependBlock, b.m.Engine, b.m.DB, 1, func(int, *blockgen.BlockGen) {}) b.pendingBlock = blockChain.Blocks[0] b.pendingReceipts = blockChain.Receipts[0] b.pendingHeader = blockChain.Headers[0] @@ -814,7 +818,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, txn types.Transa block.AddTxWithChain(b.getHeader, b.m.Engine, txn) } block.AddTxWithChain(b.getHeader, b.m.Engine, txn) - }, b.m.PublishedSD()) + }) if err != nil { return err } @@ -859,7 +863,7 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { block.AddTxWithChain(b.getHeader, b.m.Engine, txn) } block.OffsetTime(int64(adjustment.Seconds())) - }, b.m.PublishedSD()) + }) if err != nil { return err } diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 86283fdcddd..7d4325dfc91 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -177,6 +177,19 @@ func (e *ExecModule) markGenCommitted(gen *commitGen) { e.fgMu.Lock() gen.committed = true e.uncommittedGens-- + if e.uncommittedGens == 0 { + e.fgIdle.Broadcast() + } + e.fgMu.Unlock() +} + +// WaitCommitsDrained blocks until every enqueued background commit has landed; +// unlike WaitIdle it leaves the commit worker running, so it is repeatable. +func (e *ExecModule) WaitCommitsDrained() { + e.fgMu.Lock() + for e.uncommittedGens > 0 { + e.fgIdle.Wait() + } e.fgMu.Unlock() } @@ -195,16 +208,22 @@ func (e *ExecModule) commitBacklogFull() bool { return e.uncommittedGens >= maxInFlightCommits } -// latestGen returns the newest in-flight generation — the parent a new -// foreground SharedDomains chains to so its reads see not-yet-committed -// domain state. nil when the chain is empty. +// latestGen returns the newest not-yet-committed generation to chain a new +// SharedDomains onto — a committed generation's state is already in the raw DB, +// so chaining to it adds nothing and masks direct-DB reads. The single FIFO +// commit worker commits in enqueue order, so a committed newest generation +// means all are committed: read straight from the DB. func (e *ExecModule) latestGen() *execctx.SharedDomains { e.fgMu.Lock() defer e.fgMu.Unlock() if len(e.gens) == 0 { return nil } - return e.gens[len(e.gens)-1].sd + last := e.gens[len(e.gens)-1] + if last.committed { + return nil + } + return last.sd } // addGen appends a new in-flight generation to the chain. diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 1cdaf4f776f..c40a239207a 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -417,6 +417,7 @@ func TestUpdateForkChoiceForwardExecutesAfterStateAheadRecovery(t *testing.T) { res2, err := updateForkChoice(ctx, m.ExecModule, tip) require.NoError(t, err) require.Equal(t, execmodule.ExecutionStatusSuccess, res2.Status, "second FCU should execute forward to the tip") + m.ExecModule.WaitCommitsDrained() require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { execProg, err := stages.GetStageProgress(tx, stages.Execution) require.NoError(t, err) @@ -493,7 +494,7 @@ func TestReorgBackAndForwardIntoCanonicalChain(t *testing.T) { // Let the last FCU's commit and prune (foreground or background) // settle before reading the committed head. - m.ExecModule.WaitIdle(ctx) + m.ExecModule.WaitCommitsDrained() require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { require.Equal(t, headerAt(chainLen).Hash(), rawdb.ReadHeadBlockHash(tx), "head must be at canonical tip") return nil @@ -2041,6 +2042,7 @@ func TestInsertBlocksWithBatchedFCU(t *testing.T) { // After each batch's FCU, TxNums + execution must have advanced to // the batch tip. The next batch's first block reads its parent's TD // from this committed state. + m.ExecModule.WaitCommitsDrained() require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { lastTxNumBlock, _, err := rawdbv3.TxNums.Last(tx) require.NoError(t, err) @@ -2317,6 +2319,7 @@ func TestUpdateForkChoiceShallowReorgAfterLargeBatchExec(t *testing.T) { "shallow reorg of %d blocks after a %d-block batch must succeed; status=%s validationError=%q", reorgDepth, chainLen, fcuRes.Status, fcuRes.ValidationError) + m.ExecModule.WaitCommitsDrained() require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { execProg, err := stages.GetStageProgress(tx, stages.Execution) require.NoError(t, err) @@ -2439,7 +2442,7 @@ func runBALFoldAheadChangeset(t *testing.T, foldAhead, shadow bool) balFoldResul require.Equal(t, execmodule.ExecutionStatusSuccess, fcuRes.Status, "batch with fold-ahead=%v must execute cleanly; validationError=%q", foldAhead, fcuRes.ValidationError) - m.ExecModule.WaitIdle(ctx) + m.ExecModule.WaitCommitsDrained() res := balFoldResult{ windowCommitmentKeys: map[uint64][]string{}, @@ -2483,7 +2486,7 @@ func runBALFoldAheadChangeset(t *testing.T, foldAhead, shadow bool) balFoldResul back, err := updateForkChoice(ctx, m.ExecModule, canonical.Blocks[reorgBackTo-1].Header()) require.NoError(t, err) require.Equal(t, execmodule.ExecutionStatusSuccess, back.Status, "reorg back must succeed") - m.ExecModule.WaitIdle(ctx) + m.ExecModule.WaitCommitsDrained() require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { execProg, err := stages.GetStageProgress(tx, stages.Execution) require.NoError(t, err) @@ -2496,7 +2499,7 @@ func runBALFoldAheadChangeset(t *testing.T, foldAhead, shadow bool) balFoldResul require.Equal(t, execmodule.ExecutionStatusSuccess, fwd.Status, "forward re-exec after unwind must reach the correct root (fold-ahead=%v); validationError=%q", foldAhead, fwd.ValidationError) - m.ExecModule.WaitIdle(ctx) + m.ExecModule.WaitCommitsDrained() require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { execProg, err := stages.GetStageProgress(tx, stages.Execution) require.NoError(t, err) diff --git a/execution/execmodule/exec_module_unwind_gap_test.go b/execution/execmodule/exec_module_unwind_gap_test.go index 00454899730..638e1260cd4 100644 --- a/execution/execmodule/exec_module_unwind_gap_test.go +++ b/execution/execmodule/exec_module_unwind_gap_test.go @@ -153,6 +153,7 @@ func TestUpdateForkChoiceBadBlockMidBatchThenRecovery(t *testing.T) { _, err = updateForkChoice(ctx, m.ExecModule, block13.Header()) require.NoError(t, err) + m.ExecModule.WaitCommitsDrained() var acc accounts.Account require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { v, _, err := tx.GetLatest(kv.AccountsDomain, senderAddr[:]) @@ -243,6 +244,7 @@ func TestUpdateForkChoiceBadBlockAtLongBatchTailThenRecovery(t *testing.T) { _, err = updateForkChoice(ctx, m.ExecModule, forkTip.Header()) require.NoError(t, err) + m.ExecModule.WaitCommitsDrained() var acc accounts.Account require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error { v, _, err := tx.GetLatest(kv.AccountsDomain, senderAddr[:]) diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 4c44bf84fb5..f947c612952 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -974,6 +974,10 @@ func (emt *ExecModuleTester) InsertChain(chain *blockgen.ChainPack) error { if rawdb.ReadHeadBlockHash(roTx) != chain.TopBlock.Hash() { return fmt.Errorf("did not import block %d %x", chain.TopBlock.NumberU64(), chain.TopBlock.Hash()) } + // Under background commit InsertChain returns before its commit lands. Drain + // it before returning so a following operation (a reorg's unwind, a raw-DB + // read) cannot race the still-in-flight commit of the chain just inserted. + emt.ExecModule.WaitCommitsDrained() return nil } diff --git a/execution/execmodule/execmoduletester/exec_module_tester_test.go b/execution/execmodule/execmoduletester/exec_module_tester_test.go index 901cc1d25a3..f417d0a4a28 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester_test.go +++ b/execution/execmodule/execmoduletester/exec_module_tester_test.go @@ -40,7 +40,7 @@ func TestInsertChain(t *testing.T) { m := execmoduletester.New(t) chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 100, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) err = m.InsertChain(chain) require.NoError(t, err) @@ -51,7 +51,7 @@ func TestReorgsWithInsertChain(t *testing.T) { m := execmoduletester.New(t) chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 10, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) // insert initial chain err = m.InsertChain(chain) @@ -59,16 +59,16 @@ func TestReorgsWithInsertChain(t *testing.T) { // Now generate three competing branches, one short and two longer ones short, err := blockgen.GenerateChain(m.ChainConfig, chain.TopBlock, m.Engine, m.DB, 2, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) long1, err := blockgen.GenerateChain(m.ChainConfig, chain.TopBlock, m.Engine, m.DB, 10, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{2}) // Need to make headers different from short branch - }) + }, m.PublishedSD()) require.NoError(t, err) // Second long chain needs to be slightly shorter than the first long chain long2, err := blockgen.GenerateChain(m.ChainConfig, chain.TopBlock, m.Engine, m.DB, 9, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{3}) // Need to make headers different from short branch and another long branch - }) + }, m.PublishedSD()) require.NoError(t, err) // insert short chain err = m.InsertChain(short) @@ -79,7 +79,7 @@ func TestReorgsWithInsertChain(t *testing.T) { // another short chain short2, err := blockgen.GenerateChain(m.ChainConfig, long1.TopBlock, m.Engine, m.DB, 2, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) // insert long2 chain err = m.InsertChain(long2) diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index e93a50f55ad..1f8e798f416 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -40,7 +40,7 @@ type sdRoDB struct { } func (d *sdRoDB) BeginTemporalRo(ctx context.Context) (kv.TemporalTx, error) { - base, err := d.TemporalRoDB.BeginTemporalRo(ctx) + base, err := d.TemporalRoDB.BeginTemporalRo(ctx) //nolint:gocritic // base is wrapped and returned; caller owns Rollback if err != nil { return nil, err } diff --git a/execution/state/database_test.go b/execution/state/database_test.go index f6937ee4bb0..14fec3bc824 100644 --- a/execution/state/database_test.go +++ b/execution/state/database_test.go @@ -128,7 +128,7 @@ func TestCreate2Revive(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { t.Error(err) @@ -149,7 +149,7 @@ func TestCreate2Revive(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -167,7 +167,7 @@ func TestCreate2Revive(t *testing.T) { var key2 accounts.StorageKey var check2 uint256.Int - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -189,7 +189,7 @@ func TestCreate2Revive(t *testing.T) { if err = m.InsertChain(chain.Slice(2, 3)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -204,7 +204,7 @@ func TestCreate2Revive(t *testing.T) { if err = m.InsertChain(chain.Slice(3, 4)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -347,7 +347,7 @@ func TestCreate2Polymorth(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { @@ -369,7 +369,7 @@ func TestCreate2Polymorth(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -385,7 +385,7 @@ func TestCreate2Polymorth(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -416,7 +416,7 @@ func TestCreate2Polymorth(t *testing.T) { if err = m.InsertChain(chain.Slice(2, 3)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -431,7 +431,7 @@ func TestCreate2Polymorth(t *testing.T) { if err = m.InsertChain(chain.Slice(3, 4)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -462,7 +462,7 @@ func TestCreate2Polymorth(t *testing.T) { if err = m.InsertChain(chain.Slice(4, 5)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(create2address); err != nil { t.Error(err) @@ -582,7 +582,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { t.Fatalf("generate long blocks") } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { @@ -605,7 +605,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { var key0 = accounts.ZeroKey var correctValueX uint256.Int - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -626,7 +626,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -640,7 +640,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { if err = m.InsertChain(longerChain.Slice(1, 4)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -739,7 +739,7 @@ func TestReorgOverStateChange(t *testing.T) { t.Fatalf("generate longer blocks: %v", err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { t.Error(err) @@ -763,7 +763,7 @@ func TestReorgOverStateChange(t *testing.T) { var key0 = accounts.ZeroKey var correctValueX uint256.Int - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -788,7 +788,7 @@ func TestReorgOverStateChange(t *testing.T) { if err = m.InsertChain(longerChain.Slice(1, 3)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -881,7 +881,7 @@ func TestCreateOnExistingStorage(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { t.Error(err) @@ -902,7 +902,7 @@ func TestCreateOnExistingStorage(t *testing.T) { var key0 = accounts.ZeroKey var check0 uint256.Int - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -1031,7 +1031,7 @@ func TestEip2200Gas(t *testing.T) { } var balanceBefore uint256.Int - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { t.Error(err) @@ -1053,7 +1053,7 @@ func TestEip2200Gas(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -1131,7 +1131,7 @@ func TestWrongIncarnation(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { t.Error(err) @@ -1152,7 +1152,7 @@ func TestWrongIncarnation(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { stateReader := m.NewStateReader(tx) acc, err := stateReader.ReadAccountData(accounts.InternAddress(contractAddress)) if err != nil { @@ -1180,7 +1180,7 @@ func TestWrongIncarnation(t *testing.T) { if err = m.InsertChain(chain.Slice(1, 2)); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { stateReader := m.NewStateReader(tx) acc, err := stateReader.ReadAccountData(accounts.InternAddress(contractAddress)) if err != nil { @@ -1291,7 +1291,7 @@ func TestWrongIncarnation2(t *testing.T) { t.Fatalf("generate longer blocks: %v", err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(address); err != nil { t.Error(err) @@ -1312,7 +1312,7 @@ func TestWrongIncarnation2(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(contractAddress)); err != nil { t.Error(err) @@ -1339,7 +1339,7 @@ func TestWrongIncarnation2(t *testing.T) { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { stateReader := m.NewStateReader(tx) acc, err := stateReader.ReadAccountData(accounts.InternAddress(contractAddress)) if err != nil { @@ -1634,7 +1634,7 @@ func TestRecreateAndRewind(t *testing.T) { var key0 = accounts.ZeroKey var check0 uint256.Int - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(phoenixAddress)); err != nil { t.Error(err) @@ -1654,7 +1654,7 @@ func TestRecreateAndRewind(t *testing.T) { if err = m.InsertChain(chain.Slice(2, chain.Length())); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(phoenixAddress)); err != nil { @@ -1676,7 +1676,7 @@ func TestRecreateAndRewind(t *testing.T) { if err = m.InsertChain(longerChain); err != nil { t.Fatal(err) } - err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { + err = m.OverlayDB().ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { st := state.New(m.NewStateReader(tx)) if exist, err := st.Exist(accounts.InternAddress(phoenixAddress)); err != nil { t.Error(err) @@ -1747,6 +1747,9 @@ func TestTxLookupUnwind(t *testing.T) { if err = m.InsertChain(chain2); err != nil { t.Fatal(err) } + // Count is unsupported on the block-overlay tx; drain the background commit + // and count on the raw DB instead. + m.ExecModule.WaitCommitsDrained() var count uint64 if err = m.DB.ViewTemporal(context.Background(), func(tx kv.TemporalTx) error { var e error diff --git a/execution/verify/history_verify_test.go b/execution/verify/history_verify_test.go index 63e25c3934a..9f2329c8796 100644 --- a/execution/verify/history_verify_test.go +++ b/execution/verify/history_verify_test.go @@ -63,7 +63,7 @@ func TestHistoryVerification_SimpleBlocks(t *testing.T) { batchEnd := min(batchStart+batchSize, numBlocks) chainResult, err := blockgen.GenerateChain(m.ChainConfig, parent, m.Engine, m.DB, batchEnd-batchStart, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(chainResult)) parent = chainResult.TopBlock @@ -159,7 +159,7 @@ func TestHistoryVerification_WithUserTransactions(t *testing.T) { b.AddTx(signed) nonce++ } - }) + }, m.PublishedSD()) require.NoError(t, err) require.NoError(t, m.InsertChain(chainResult)) t.Logf("Inserted %d blocks with 2 user txs each", numBlocks) diff --git a/rpc/jsonrpc/eth_block_test.go b/rpc/jsonrpc/eth_block_test.go index 794a967d5b0..4e0b60dbafe 100644 --- a/rpc/jsonrpc/eth_block_test.go +++ b/rpc/jsonrpc/eth_block_test.go @@ -27,9 +27,9 @@ import ( "github.com/erigontech/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/hexutil" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/kvcache" "github.com/erigontech/erigon/db/kv/prune" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/execmoduletester" "github.com/erigontech/erigon/execution/rlp" From 4670f5cab5d6d7bc493d3be797000d8aa0599762 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 11:09:20 +0000 Subject: [PATCH 137/167] execution/types, execution/engineapi, execution/stagedsync: carry the block access list as a first-class block object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EIP-7928 Block Access Lists are part of the payload and should be consumed by execution from the block it is handed, the same model as headers and bodies. Previously the BAL only existed transiently on RawBlock during InsertBlocks, was written to the kv.BlockAccessList table, and RawBlock.AsBlock dropped it — so the block object execution processes never carried its own BAL, and exec re-read it from the DB. - types.Block gains an unexported blockAccessList sidecar (BlockAccessList / SetBlockAccessList). It is carried out-of-band and must never enter the block RLP encoding or hash; RawBlock.AsBlock/Copy/WithSeal transfer it. Added a regression test pinning it out of RLP/hash. - newPayload attaches the payload's BAL to the block; InsertBlocks still writes it into the block overlay (flushed at commit) as secondary storage. - Execution consumes the BAL from the block, falling back to the DB sidecar for snapshot/forward-sync blocks that don't carry it (ProcessBAL compute+validate remains the ultimate fallback). - rawdb.ReadBlock populates the BAL sidecar as secondary storage. - Rework GetPayloadBodies*V2 tests to exercise the unified insert path. --- db/rawdb/accessors_chain.go | 12 +++- execution/engineapi/engine_server.go | 4 ++ execution/engineapi/engine_server_test.go | 76 +++++------------------ execution/stagedsync/exec3.go | 19 ++++-- execution/types/block.go | 32 +++++++--- execution/types/block_test.go | 32 ++++++++++ 6 files changed, 98 insertions(+), 77 deletions(-) diff --git a/db/rawdb/accessors_chain.go b/db/rawdb/accessors_chain.go index d9393403df0..6464dc4afdc 100644 --- a/db/rawdb/accessors_chain.go +++ b/db/rawdb/accessors_chain.go @@ -609,7 +609,10 @@ func ReadBlockAccessListBytes(db kv.Getter, hash common.Hash, number uint64) ([] return data, nil } -// WriteBlockAccessListBytes stores the RLP-encoded block access list sidecar for a block. +// WriteBlockAccessListBytes stores the RLP-encoded block access list sidecar for +// a block. This is secondary storage (serving, backfill, unwind); the primary +// carry into execution is Block.BlockAccessList(), written via the block overlay +// in InsertBlocks and flushed at commit. func WriteBlockAccessListBytes(db kv.Putter, hash common.Hash, number uint64, data []byte) error { if err := db.Put(kv.BlockAccessList, dbutils.BlockBodyKey(number, hash), data); err != nil { return fmt.Errorf("failed to store block access list: %w", err) @@ -807,6 +810,13 @@ func ReadBlock(tx kv.Getter, hash common.Hash, number uint64) *types.Block { return nil } block := types.NewBlockFromStorage(hash, header, body.Transactions, body.Uncles, body.Withdrawals) + // Carry the BAL sidecar (secondary storage) so a block reconstructed from the + // DB carries its BAL like its header/body. Only Amsterdam+ blocks have one. + if header.BlockAccessListHash != nil { + if bal, err := ReadBlockAccessListBytes(tx, hash, number); err == nil && len(bal) > 0 { + block.SetBlockAccessList(bal) + } + } return block } diff --git a/execution/engineapi/engine_server.go b/execution/engineapi/engine_server.go index a8b2b141193..326febe83db 100644 --- a/execution/engineapi/engine_server.go +++ b/execution/engineapi/engine_server.go @@ -488,6 +488,10 @@ func (s *EngineServer) newPayload(ctx context.Context, req *engine_types.Executi // via rlp.EncodeToBytes. Both slices reference the same underlying // byte buffers from req.Transactions. block := types.NewBlockFromStorageWithBinaryTxs(blockHash, &header, transactions, txs, nil /* uncles */, withdrawals) + // Carry the payload's BAL on the block so execution consumes it from the + // in-memory payload; InsertBlocks still stores it in the overlay (flushed at + // commit) as secondary storage. + block.SetBlockAccessList(blockAccessListBytes) payloadStatus, err := s.HandleNewPayload(ctx, "NewPayload", block, expectedBlobHashes, blockAccessListBytes) if err != nil { if errors.Is(err, rules.ErrInvalidBlock) { diff --git a/execution/engineapi/engine_server_test.go b/execution/engineapi/engine_server_test.go index 81f557108f0..8a185d08744 100644 --- a/execution/engineapi/engine_server_test.go +++ b/execution/engineapi/engine_server_test.go @@ -36,7 +36,6 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/kvcache" - "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/execmodule/execmoduletester" "github.com/erigontech/erigon/execution/tests/blockgen" @@ -50,13 +49,14 @@ import ( ) // Do 1 step to start txPool -func oneBlockSteps(m *execmoduletester.ExecModuleTester, require *require.Assertions, blocks int) { +func oneBlockSteps(m *execmoduletester.ExecModuleTester, require *require.Assertions, blocks int) *blockgen.ChainPack { chain, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, func(i int, b *blockgen.BlockGen) { b.SetCoinbase(common.Address{1}) }) require.NoError(err) err = m.InsertChain(chain) require.NoError(err) + return chain } // Do 1 step to start txPool @@ -290,64 +290,34 @@ func TestGetBlobsV3(t *testing.T) { } } -func canonicalHashAt(t *testing.T, db kv.TemporalRoDB, blockNum uint64) common.Hash { - t.Helper() - var hash common.Hash - err := db.View(context.Background(), func(tx kv.Tx) error { - var err error - hash, err = rawdb.ReadCanonicalHash(tx, blockNum) - return err - }) - require.NoError(t, err) - return hash -} - -func writeBlockAccessListBytes(t *testing.T, db kv.TemporalRwDB, blockHash common.Hash, blockNum uint64, balBytes []byte) { - t.Helper() - err := db.Update(context.Background(), func(tx kv.RwTx) error { - return rawdb.WriteBlockAccessListBytes(tx, blockHash, blockNum, balBytes) - }) - require.NoError(t, err) -} - func TestGetPayloadBodiesByHashV2(t *testing.T) { mockSentry := execmoduletester.New(t, execmoduletester.WithTxPool(), execmoduletester.WithChainConfig(chain.AllProtocolChanges)) req := require.New(t) - oneBlockStep(mockSentry, req) + // Insert a block carrying its BAL through the unified InsertChain path (BAL + // stored in the overlay, flushed at commit); serving must reflect that BAL. + chain := oneBlockSteps(mockSentry, req, 1) executionRpc := mockSentry.ExecModule maxReorgDepth := ethconfig.Defaults.MaxReorgDepth engineServer := NewEngineServer(mockSentry.Log, mockSentry.ChainConfig, executionRpc, nil, false, false, false, true, nil, nil, ethconfig.Defaults.FcuTimeout, maxReorgDepth) - const blockNum = 1 - blockHash := canonicalHashAt(t, mockSentry.DB, blockNum) + blockHash := chain.Blocks[0].Hash() req.NotEqual(common.Hash{}, blockHash) + req.NotEmpty(chain.BlockAccessLists[0], "Amsterdam block must carry a BAL from GenerateChain") ctx := context.Background() - // Amsterdam-enabled chains always have a BAL written by GenerateChain bodies, err := engineServer.GetPayloadBodiesByHashV2(ctx, []common.Hash{blockHash}) req.NoError(err) req.Len(bodies, 1) req.NotNil(bodies[0]) - req.NotEmpty(bodies[0].BlockAccessList) - - // Overwrite with a non-empty BAL and verify it's returned - balBytes := []byte{0x01, 0x02, 0x03} - writeBlockAccessListBytes(t, mockSentry.DB, blockHash, blockNum, balBytes) - - bodies, err = engineServer.GetPayloadBodiesByHashV2(ctx, []common.Hash{blockHash}) - req.NoError(err) - req.Len(bodies, 1) - req.NotNil(bodies[0]) - req.NotNil(bodies[0].BlockAccessList) - req.Equal(hexutil.Bytes(balBytes), bodies[0].BlockAccessList) + req.Equal(hexutil.Bytes(chain.BlockAccessLists[0]), bodies[0].BlockAccessList) } func TestGetPayloadBodiesByRangeV2(t *testing.T) { mockSentry := execmoduletester.New(t, execmoduletester.WithTxPool(), execmoduletester.WithChainConfig(chain.AllProtocolChanges)) req := require.New(t) - oneBlockSteps(mockSentry, req, 2) + chain := oneBlockSteps(mockSentry, req, 2) executionRpc := mockSentry.ExecModule maxReorgDepth := ethconfig.Defaults.MaxReorgDepth @@ -357,35 +327,17 @@ func TestGetPayloadBodiesByRangeV2(t *testing.T) { start = 1 count = 2 ) - blockHash1 := canonicalHashAt(t, mockSentry.DB, start) - blockHash2 := canonicalHashAt(t, mockSentry.DB, start+1) - req.NotEqual(common.Hash{}, blockHash1) - req.NotEqual(common.Hash{}, blockHash2) + req.NotEmpty(chain.BlockAccessLists[0], "Amsterdam block must carry a BAL from GenerateChain") + req.NotEmpty(chain.BlockAccessLists[1], "Amsterdam block must carry a BAL from GenerateChain") ctx := context.Background() - // Amsterdam-enabled chains always have a BAL written by GenerateChain + // Serving must reflect the BALs inserted with the blocks (unified path). bodies, err := engineServer.GetPayloadBodiesByRangeV2(ctx, start, count) req.NoError(err) req.Len(bodies, 2) req.NotNil(bodies[0]) req.NotNil(bodies[1]) - req.NotEmpty(bodies[0].BlockAccessList) - req.NotEmpty(bodies[1].BlockAccessList) - - // Overwrite with non-empty BALs and verify they're returned - balBytes1 := []byte{0x01, 0x02, 0x03} - balBytes2 := []byte{0x04, 0x05, 0x06} - writeBlockAccessListBytes(t, mockSentry.DB, blockHash1, start, balBytes1) - writeBlockAccessListBytes(t, mockSentry.DB, blockHash2, start+1, balBytes2) - - bodies, err = engineServer.GetPayloadBodiesByRangeV2(ctx, start, count) - req.NoError(err) - req.Len(bodies, 2) - req.NotNil(bodies[0]) - req.NotNil(bodies[1]) - req.NotNil(bodies[0].BlockAccessList) - req.NotNil(bodies[1].BlockAccessList) - req.Equal(hexutil.Bytes(balBytes1), bodies[0].BlockAccessList) - req.Equal(hexutil.Bytes(balBytes2), bodies[1].BlockAccessList) + req.Equal(hexutil.Bytes(chain.BlockAccessLists[0]), bodies[0].BlockAccessList) + req.Equal(hexutil.Bytes(chain.BlockAccessLists[1]), bodies[1].BlockAccessList) } diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 5998badb9f6..857fb793840 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -612,12 +612,19 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m go warmTxsHashes(b) var dbBAL types.BlockAccessList - // Read BAL through blockTx (overlay or execRoTx) — do NOT open - // a separate db.View() as it can deadlock with the stageloop's - // RW transaction when BlockOverlay is active. - data, err := rawdb.ReadBlockAccessListBytes(blockTx, b.Hash(), blockNum) - if err != nil { - return err + // Prefer the BAL carried on the block (the payload) — the newPayload / + // backward-sync paths attach it, so no read is needed. Fall back to the + // BAL sidecar in the DB (via blockTx: overlay or execRoTx) for blocks + // that don't carry it (snapshot / forward-sync); do NOT open a separate + // db.View() as it can deadlock with the stageloop's RW transaction when + // BlockOverlay is active. ProcessBAL still computes+validates the BAL + // from the write-set as the ultimate fallback. + data := b.BlockAccessList() + if len(data) == 0 { + data, err = rawdb.ReadBlockAccessListBytes(blockTx, b.Hash(), blockNum) + if err != nil { + return err + } } if len(data) > 0 && !dbg.IgnoreBAL { dbBAL, err = types.DecodeBlockAccessListBytes(data) diff --git a/execution/types/block.go b/execution/types/block.go index 45392e171f3..b34399c8475 100644 --- a/execution/types/block.go +++ b/execution/types/block.go @@ -792,6 +792,7 @@ func (r RawBlock) AsBlock() (*Block, error) { } } b.transactions = txs + b.blockAccessList = r.BlockAccessList return b, nil } @@ -803,6 +804,12 @@ type Block struct { transactions Transactions withdrawals []*Withdrawal + // blockAccessList is the RLP-encoded EIP-7928 Block Access List sidecar + // carried with the payload (nil pre-Amsterdam). It is NOT part of the block's + // RLP/consensus encoding or hash — never add it to EncodeRLP/DecodeRLP/ + // payloadSize. The header's BlockAccessListHash is the consensus commitment. + blockAccessList []byte + // binaryTransactions optionally caches the transactions' encodings (e.g. from // an engine_newPayload payload) so RawBody() can skip re-encoding them. binaryTransactions BinaryTransactions @@ -1365,6 +1372,13 @@ func (b *Block) ParentBeaconBlockRoot() *common.Hash { return b.header.ParentBea func (b *Block) RequestsHash() *common.Hash { return b.header.RequestsHash } func (b *Block) BlockAccessListHash() *common.Hash { return b.header.BlockAccessListHash } +// BlockAccessList returns the RLP-encoded EIP-7928 BAL sidecar carried with the +// payload (nil when absent). It is not part of the block's RLP encoding or hash. +func (b *Block) BlockAccessList() []byte { return b.blockAccessList } + +// SetBlockAccessList attaches the RLP-encoded BAL sidecar to the block. +func (b *Block) SetBlockAccessList(bal []byte) { b.blockAccessList = bal } + // Header returns a deep-copy of the entire block header using CopyHeader() func (b *Block) Header() *Header { return CopyHeader(b.header) } func (b *Block) HeaderNoCopy() *Header { return b.header } @@ -1525,10 +1539,11 @@ func (b *Block) Copy() *Block { } newB := &Block{ - header: CopyHeader(b.header), - uncles: uncles, - transactions: CopyTxs(b.transactions), - withdrawals: withdrawals, + header: CopyHeader(b.header), + uncles: uncles, + transactions: CopyTxs(b.transactions), + withdrawals: withdrawals, + blockAccessList: common.Copy(b.blockAccessList), } szCopy := b.size.Load() newB.size.Store(szCopy) @@ -1542,10 +1557,11 @@ func (b *Block) WithSeal(header *Header) *Block { headerCopy.mutable = false headerCopy.hash.Store(nil) // invalidate cached hash return &Block{ - header: headerCopy, - transactions: b.transactions, - uncles: b.uncles, - withdrawals: b.withdrawals, + header: headerCopy, + transactions: b.transactions, + uncles: b.uncles, + withdrawals: b.withdrawals, + blockAccessList: b.blockAccessList, } } diff --git a/execution/types/block_test.go b/execution/types/block_test.go index 4ead6e6cd63..6d1ef3cf30d 100644 --- a/execution/types/block_test.go +++ b/execution/types/block_test.go @@ -161,6 +161,38 @@ func TestBlockEncoding(t *testing.T) { } } +// TestBlockAccessListNotInEncoding pins the invariant that the BAL sidecar is +// carried out-of-band: it must never enter the block's RLP encoding or hash. +func TestBlockAccessListNotInEncoding(t *testing.T) { + t.Parallel() + blockEnc := common.FromHex("f90260f901f9a083cafc574e1f51ba9dc0568fc617a08ea2429fb384059c972f13b19fa1c8dd55a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a05fe50b260da6308036625b850b5d6ced6d0a9f814c0688bc91ffb7b7a3a54b67a0bc37d79753ad738a6dac4921e57392f145d8887476de3f783dfa7edae9283e52b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd8825208845506eb0780a0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4f861f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1c0") + var block Block + if err := rlp.DecodeBytes(blockEnc, &block); err != nil { + t.Fatal("decode error: ", err) + } + + hashBefore := block.Hash() + block.SetBlockAccessList([]byte{0x01, 0x02, 0x03}) + if got := block.Hash(); got != hashBefore { + t.Errorf("BAL changed block hash: got %x want %x", got, hashBefore) + } + enc, err := rlp.EncodeToBytes(&block) + if err != nil { + t.Fatal("encode error: ", err) + } + if !bytes.Equal(enc, blockEnc) { + t.Errorf("BAL leaked into block RLP:\ngot: %x\nwant: %x", enc, blockEnc) + } + + var decoded Block + if err := rlp.DecodeBytes(enc, &decoded); err != nil { + t.Fatal("decode error: ", err) + } + if decoded.BlockAccessList() != nil { + t.Errorf("BAL survived RLP round-trip (must be a non-encoded sidecar): %x", decoded.BlockAccessList()) + } +} + func TestEIP1559BlockEncoding(t *testing.T) { t.Parallel() blockEnc := common.FromHex("f9030bf901fea083cafc574e1f51ba9dc0568fc617a08ea2429fb384059c972f13b19fa1c8dd55a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a05fe50b260da6308036625b850b5d6ced6d0a9f814c0688bc91ffb7b7a3a54b67a0bc37d79753ad738a6dac4921e57392f145d8887476de3f783dfa7edae9283e52b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd8825208845506eb0780a0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4843b9aca00f90106f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1b8a302f8a0018080843b9aca008301e24194095e7baea6a6c7c4c2dfeb977efac326af552d878080f838f7940000000000000000000000000000000000000001e1a0000000000000000000000000000000000000000000000000000000000000000080a0fe38ca4e44a30002ac54af7cf922a6ac2ba11b7d22f548e8ecb3f51f41cb31b0a06de6a5cbae13c0c856e33acf021b51819636cfc009d39eafb9f606d546e305a8c0") From 3b0a19c0ac717362f4c4a422f09dedf56b98a0cd Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 13:16:14 +0000 Subject: [PATCH 138/167] cmd/utils/app, db/snapshotsync: make the import tool bg-commit-durable Under fcuBackgroundCommit the import command read the raw DB between files before the previous file's forkchoice commit had landed, so the side-chain TD reorg decision and the post-import head were computed from stale state (head never advanced), and BlockReader.CurrentBlock dereferenced a nil head number when the head hash's header was still only in the in-flight overlay. - import InsertChain drains the background commit after each file's UpdateForkChoice so the next file's TD/head reads and the final head write observe durable state. - BlockReader.CurrentBlock returns (nil, nil) instead of panicking when the head hash has no header number yet; missingBlocks tolerates a nil head. --- cmd/utils/app/import_cmd.go | 11 +++++++++-- db/snapshotsync/freezeblocks/block_reader.go | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/utils/app/import_cmd.go b/cmd/utils/app/import_cmd.go index 838a5c202e8..9837c04d663 100644 --- a/cmd/utils/app/import_cmd.go +++ b/cmd/utils/app/import_cmd.go @@ -268,8 +268,10 @@ func missingBlocks(chainDB kv.RwDB, blocks []*types.Block, blockReader services. }) for i, block := range blocks { - // If we're behind the chain head, only check block, state is available at head - if headBlock.NumberU64() > block.NumberU64() { + // No durable head yet (e.g. only genesis, or the head's commit is still + // in flight under background commit): nothing is behind us, so fall + // through to the per-block presence check. + if headBlock != nil && headBlock.NumberU64() > block.NumberU64() { if !ChainHasBlock(chainDB, block) { return blocks[i:] } @@ -445,6 +447,11 @@ func InsertChain(ethereum *eth.Ethereum, chain *blockgen.ChainPack, setHead bool } } + // UpdateForkChoice commits in the background; drain it so this file's + // header/TD/canonical state is durable in the raw DB before the next file's + // side-chain TD/head reads (and the final head write) observe it. + ethereum.ExecutionModule().WaitCommitsDrained() + return ethereum.ChainDB().Update(ethereum.SentryCtx(), func(tx kv.RwTx) error { rawdb.WriteHeadBlockHash(tx, lvh) return nil diff --git a/db/snapshotsync/freezeblocks/block_reader.go b/db/snapshotsync/freezeblocks/block_reader.go index 23a96ef3c07..371ee9823c1 100644 --- a/db/snapshotsync/freezeblocks/block_reader.go +++ b/db/snapshotsync/freezeblocks/block_reader.go @@ -1447,6 +1447,9 @@ func (r *BlockReader) CurrentBlock(db kv.Tx) (*types.Block, error) { if err != nil { return nil, fmt.Errorf("failed HeaderNumber: %w", err) } + if headNumber == nil { + return nil, nil + } block, _, err := r.blockWithSenders(context.Background(), db, headHash, *headNumber, true) return block, err } From b7f352b2807edabb8110da64bb51ba1113f940c8 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 13:21:34 +0000 Subject: [PATCH 139/167] execution/builder, db/state/execctx: isolate the block builder from the shared branch cache The block builder assembled a speculative block on a SharedDomains that shared the aggregator-scope commitment BranchCache with the live node. The build runs in a background goroutine (AssembleBlock releases the foreground semaphore), so its commitment computation raced the node's concurrent background commit on that cache, producing a wrong state root under load, and also polluted the cache with speculative branches. Add execctx.WithoutSharedBranchCache and use it for the builder's SharedDomains so the discarded build reads through sd.mem / the parent generation / MDBX instead of the shared cache. The built root stays correct (validated by the assemble tests) and no longer races or pollutes the live cache. --- db/state/execctx/domain_shared.go | 17 +++++++++++------ db/state/execctx/options.go | 12 +++++++++++- execution/builder/builder.go | 6 +++++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 2e9bd8e6e29..5a346bc715b 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -208,11 +208,13 @@ func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger, // aggregator). The duck-typed BranchCacheProvider lookup avoids // importing db/state directly — db/state already imports execctx, so // the reverse import would create a cycle. - var branchCache *commitment.BranchCache - if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { - branchCache = p.BranchCache() + if !o.disableSharedBranchCache { + var branchCache *commitment.BranchCache + if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { + branchCache = p.BranchCache() + } + sd.branchCache = branchCache } - sd.branchCache = branchCache if p, ok := tx.AggTx().(kvmetrics.MetricsCollectorProvider); ok { sd.collector = p.MetricsCollector() } @@ -220,8 +222,11 @@ func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger, // The pin controller is aggregator-scoped (co-located with branchCache) so pin // residency ages by block-access recency across all SharedDomains, not per-SD. - if p, ok := tx.AggTx().(commitment.AdaptivePinControllerProvider); ok { - sd.adaptivePinController = p.AdaptivePinController() + // Skip it too when the shared cache is detached (speculative builder domains). + if !o.disableSharedBranchCache { + if p, ok := tx.AggTx().(commitment.AdaptivePinControllerProvider); ok { + sd.adaptivePinController = p.AdaptivePinController() + } } _, blockNum, err := sd.SeekCommitment(ctx, tx) diff --git a/db/state/execctx/options.go b/db/state/execctx/options.go index 1c48ebbb8b6..04d12d7d679 100644 --- a/db/state/execctx/options.go +++ b/db/state/execctx/options.go @@ -19,12 +19,22 @@ package execctx import "github.com/erigontech/erigon/execution/commitment" type sharedDomainOptions struct { - trieCfg commitment.TrieConfig + trieCfg commitment.TrieConfig + disableSharedBranchCache bool } // SharedDomainOption configures NewSharedDomains. type SharedDomainOption func(*sharedDomainOptions) +// WithoutSharedBranchCache detaches this SharedDomains from the aggregator-scope +// commitment BranchCache. Use for speculative, discarded computations (e.g. the +// block builder) that run concurrently with the live node: sharing the cache +// would both race the node's writers and pollute it with speculative branches. +// Reads fall through to sd.mem / the parent chain / MDBX instead. +func WithoutSharedBranchCache() SharedDomainOption { + return func(o *sharedDomainOptions) { o.disableSharedBranchCache = true } +} + // WithTrieConfig replaces the trie configuration wholesale; the caller owns Variant. func WithTrieConfig(cfg commitment.TrieConfig) SharedDomainOption { return func(o *sharedDomainOptions) { o.trieCfg = cfg } diff --git a/execution/builder/builder.go b/execution/builder/builder.go index c3f7e233e2b..2c613d0c34a 100644 --- a/execution/builder/builder.go +++ b/execution/builder/builder.go @@ -145,7 +145,11 @@ func (b *Builder) Build(param *Parameters, interrupt *atomic.Bool) (result *type } } - sd, err := execctx.NewSharedDomains(b.ctx, compositeTx, b.logger, execctx.WithoutDeferredBranchUpdates()) + // The build is speculative and discarded; detach it from the aggregator's + // shared commitment BranchCache so its commitment computation neither races + // the live node's concurrent background commit nor pollutes the cache with + // speculative branches. Reads resolve through sd.mem / parentSD / MDBX. + sd, err := execctx.NewSharedDomains(b.ctx, compositeTx, b.logger, execctx.WithoutDeferredBranchUpdates(), execctx.WithoutSharedBranchCache()) if err != nil { return nil, err } From 8f8d941d3e2bd6786dbb1cd3e76f1b2552c1e0df Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 14:40:17 +0000 Subject: [PATCH 140/167] execution/builder, execution/execmodule: pin block builder to a scoped by-block read view Under background commit the raw DB lags the tip and the published SharedDomains is a mutable global, so the builder's old habit of opening its own db.BeginTemporalRo and reading Events.LatestSD could straddle a commit and build on inconsistent (or the wrong) state. AssembleBlock now captures a ScopedReadView pinned to params.ParentHash while holding the foreground semaphore (state settled, no commit mid-flight): a fresh roTx plus the head SharedDomains when its block is ParentHash, else the committed DB verified canonical. If the head is not yet at ParentHash it returns Busy so the CL retries rather than building on the wrong block. The build reads only through the view and releases it (idempotent) when the goroutine exits; evicted builders are cancelled so their view is not leaked. The builder no longer opens its own DB snapshot or reads the global published SD. --- execution/builder/block_builder.go | 13 ++ execution/builder/builder.go | 30 ++-- execution/builder/builder_test.go | 25 +--- execution/builder/parameters.go | 18 +++ execution/execmodule/bg_commit.go | 3 + execution/execmodule/block_building.go | 25 +++- execution/execmodule/exec_module_test.go | 43 ++++++ .../execmoduletester/exec_module_tester.go | 1 - execution/execmodule/forkchoice.go | 2 + execution/execmodule/scoped_read.go | 133 ++++++++++++++++++ node/eth/backend.go | 1 - 11 files changed, 254 insertions(+), 40 deletions(-) create mode 100644 execution/execmodule/scoped_read.go diff --git a/execution/builder/block_builder.go b/execution/builder/block_builder.go index 859278a875c..d520384b3bf 100644 --- a/execution/builder/block_builder.go +++ b/execution/builder/block_builder.go @@ -35,15 +35,28 @@ type BlockBuilder struct { syncCond *sync.Cond result *types.BlockWithReceipts err error + view ScopedReadView // pinned by-block read snapshot; released when the build goroutine exits } +// Cancel signals the build to stop; the goroutine then exits and releases its +// scoped read view. Non-blocking (used on eviction). +func (b *BlockBuilder) Cancel() { b.interrupt.Store(true) } + func NewBlockBuilder(build BlockBuilderFunc, param *Parameters, maxBuildTimeSecs uint64) *BlockBuilder { builder := new(BlockBuilder) builder.syncCond = sync.NewCond(new(sync.Mutex)) + builder.view = param.ScopedView terminated := make(chan struct{}) go func() { defer close(terminated) + // Release the pinned read snapshot (its roTx) when the build goroutine + // exits — on success, error, timeout-driven Stop, or panic. Idempotent. + defer func() { + if builder.view != nil { + builder.view.Release() + } + }() var result *types.BlockWithReceipts var err error diff --git a/execution/builder/builder.go b/execution/builder/builder.go index 2c613d0c34a..f81f6ded841 100644 --- a/execution/builder/builder.go +++ b/execution/builder/builder.go @@ -37,10 +37,6 @@ import ( "github.com/erigontech/erigon/txnprovider" ) -// SDProvider returns the latest published SharedDomains from FCU, or nil if none. -// Used by the builder to read uncommitted state during background commits. -type SDProvider func() *execctx.SharedDomains - // Builder runs the three block-building steps (createBlock, execBlock, finishBlock) directly // without staged-sync machinery. Its Build method satisfies BlockBuilderFunc and can // be passed directly to ExecModule. @@ -59,7 +55,6 @@ type Builder struct { txnProvider txnprovider.TxnProvider sealCancel chan struct{} latestBlockBuiltStore *LatestBlockBuiltStore - sdProvider SDProvider logger log.Logger } @@ -77,7 +72,6 @@ func NewBuilder( txnProvider txnprovider.TxnProvider, sealCancel chan struct{}, latestBlockBuiltStore *LatestBlockBuiltStore, - sdProvider SDProvider, logger log.Logger, ) *Builder { return &Builder{ @@ -95,7 +89,6 @@ func NewBuilder( txnProvider: txnProvider, sealCancel: sealCancel, latestBlockBuiltStore: latestBlockBuiltStore, - sdProvider: sdProvider, logger: logger, } } @@ -124,21 +117,22 @@ func (b *Builder) Build(param *Parameters, interrupt *atomic.Bool) (result *type BuiltBlock: &exec.AssembledBlock{}, } - tx, err := b.db.BeginTemporalRo(b.ctx) - if err != nil { - return nil, err + // Read only through the scoped, by-block read view captured by AssembleBlock + // (pinned to param.ParentHash): its roTx is the consistent committed-state + // snapshot and its head SharedDomains carries the in-flight tip state. The + // builder never opens its own db.BeginTemporalRo nor reads the mutable global + // published SD. The view's roTx is released by the BlockBuilder goroutine. + view := param.ScopedView + if view == nil { + return nil, fmt.Errorf("builder: nil scoped read view") } - defer tx.Rollback() + tx := view.Tx() - // When a published SD is available (background commit in progress), create - // a child SD that reads domain state from the parent's mem batch and table - // data from the parent's overlay. The child's own writes are local and + // Create a child SD that reads domain state from the head's mem batch and + // table data from its overlay. The child's own writes are local and // discarded after block construction. var compositeTx kv.TemporalTx = tx - var parentSD *execctx.SharedDomains - if b.sdProvider != nil { - parentSD = b.sdProvider() - } + parentSD := view.HeadSD() if parentSD != nil { if overlay := parentSD.BlockOverlay(); overlay != nil { compositeTx = overlay.NewReadView(tx) diff --git a/execution/builder/builder_test.go b/execution/builder/builder_test.go index f7c758522fe..8c7560e1856 100644 --- a/execution/builder/builder_test.go +++ b/execution/builder/builder_test.go @@ -18,45 +18,32 @@ package builder import ( "context" - "errors" "sync/atomic" "testing" "github.com/stretchr/testify/require" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/types" ) -// errDB is a minimal kv.TemporalRoDB stub whose BeginTemporalRo always fails. -// Other methods are never reached because Build returns at the first error. -type errDB struct { - kv.TemporalRoDB // nil embed satisfies the interface; other methods must not be called - err error -} - -func (e *errDB) BeginTemporalRo(_ context.Context) (kv.TemporalTx, error) { - return nil, e.err -} - -// TestBuilder_Build_DBError verifies that Build propagates a BeginTemporalRo error -// immediately, without panicking or hanging on a channel read. -func TestBuilder_Build_DBError(t *testing.T) { +// TestBuilder_Build_NilScopedView verifies that Build fails fast (no panic, no +// hang) when AssembleBlock attached no scoped read view — the builder reads only +// through that view and must not fall back to opening its own DB snapshot. +func TestBuilder_Build_NilScopedView(t *testing.T) { t.Parallel() - want := errors.New("db open failed") b := &Builder{ ctx: context.Background(), - db: &errDB{err: want}, builderCfg: &buildercfg.BuilderConfig{}, pendingBlockCh: make(chan *types.Block, 1), logger: log.New(), } _, err := b.Build(&Parameters{}, &atomic.Bool{}) - require.ErrorIs(t, err, want) + require.Error(t, err) + require.Contains(t, err.Error(), "scoped read view") } // TestBuilder_PendingBlockCh verifies that NewBuilder initialises a non-nil buffered diff --git a/execution/builder/parameters.go b/execution/builder/parameters.go index 10d063fd37d..8059e100692 100644 --- a/execution/builder/parameters.go +++ b/execution/builder/parameters.go @@ -18,13 +18,31 @@ package builder import ( "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/txnprovider" ) +// ScopedReadView is a consistent, by-block read snapshot handed to the builder +// so it reads the exact block it builds on (never the raw DB directly nor a +// mutable global). Implemented by execmodule.ScopedReadView. The build owns it +// and releases it when it finishes. +type ScopedReadView interface { + Tx() kv.TemporalTx + HeadSD() *execctx.SharedDomains + BlockHash() common.Hash + BlockNum() uint64 + Release() +} + // Parameters for PoS block building // See also https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#payloadattributesv4 type Parameters struct { + // ScopedView is the consistent read snapshot pinned to ParentHash. Set by + // AssembleBlock; the build reads only through it and releases it on finish. + ScopedView ScopedReadView + PayloadId uint64 ParentHash common.Hash Timestamp uint64 diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 7d4325dfc91..3d88945bc11 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -20,6 +20,7 @@ import ( "context" "errors" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/execctx" ) @@ -41,6 +42,8 @@ import ( type commitGen struct { sd *execctx.SharedDomains roTx kv.TemporalTx + blockHash common.Hash // the FCU head this generation's state is for + blockNum uint64 finishProgressBefore uint64 isSynced bool initialCycle bool diff --git a/execution/execmodule/block_building.go b/execution/execmodule/block_building.go index 74b64d1c87a..ce5ed9da65b 100644 --- a/execution/execmodule/block_building.go +++ b/execution/execmodule/block_building.go @@ -18,6 +18,7 @@ package execmodule import ( "context" + "errors" "reflect" "github.com/holiman/uint256" @@ -44,6 +45,11 @@ func (e *ExecModule) evictOldBuilders() { // remove old builders so that at most MaxBuilders - 1 remain for i := 0; i <= len(e.builders)-engine_helpers.MaxBuilders; i++ { + if bldr := e.builders[ids[i]]; bldr != nil { + // Cancel so the build goroutine exits and releases its scoped read + // view (pinned roTx) rather than leaking it past eviction. + bldr.Cancel() + } delete(e.builders, ids[i]) } } @@ -67,6 +73,19 @@ func (e *ExecModule) AssembleBlock(ctx context.Context, params *builder.Paramete } } + // Pin a consistent, by-block read snapshot for the requested parent while we + // hold the foreground semaphore (settled state, no commit mid-flight). If the + // head is not yet at ParentHash, signal Busy so the CL retries rather than + // building on the wrong block. The build reads only through this view — never + // the raw DB directly nor the mutable global published SD. + view, err := e.captureScopedReadView(ctx, params.ParentHash) + if err != nil { + if errors.Is(err, errHeadMismatch) { + return AssembleBlockResult{Busy: true}, nil + } + return AssembleBlockResult{}, err + } + // Initiate payload building e.evictOldBuilders() @@ -74,7 +93,11 @@ func (e *ExecModule) AssembleBlock(ctx context.Context, params *builder.Paramete params.PayloadId = e.nextPayloadId e.lastParameters = params - e.builders[e.nextPayloadId] = builder.NewBlockBuilder(e.builderFunc, params, e.config.SecondsPerSlot()/4) + // Carry the view on a per-build copy so it never enters e.lastParameters + // (which the duplicate-request check DeepEquals). + buildParams := *params + buildParams.ScopedView = view + e.builders[e.nextPayloadId] = builder.NewBlockBuilder(e.builderFunc, &buildParams, e.config.SecondsPerSlot()/4) e.logger.Info("[ForkChoiceUpdated] BlockBuilder added", "payload", e.nextPayloadId) return AssembleBlockResult{PayloadID: e.nextPayloadId}, nil diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index baa22e64f98..ff520d751fd 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -847,6 +847,49 @@ func TestAssembleBlockWithStateVerification(t *testing.T) { require.NoError(t, err) } +// TestAssembleBlockRejectsWrongParent verifies AssembleBlock pins to the +// requested ParentHash: a request whose parent is not the current head returns +// Busy (the CL retries) rather than building on the wrong block, while a request +// on the real head builds a block extending exactly that parent. +func TestAssembleBlockRejectsWrongParent(t *testing.T) { + t.Parallel() + ctx := t.Context() + m := execmoduletester.New(t, execmoduletester.WithTxPool(), execmoduletester.WithChainConfig(chain.AllProtocolChanges)) + exec := m.ExecModule + + chainPack, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, gen *blockgen.BlockGen) { + gen.SetCoinbase(common.Address{1}) + }, m.PublishedSD()) + require.NoError(t, err) + require.NoError(t, m.InsertChain(chainPack)) + head := chainPack.TopBlock + + mkParams := func(parent common.Hash) *builder.Parameters { + return &builder.Parameters{ + ParentHash: parent, + Timestamp: head.Time() + 1, + PrevRandao: head.Header().MixDigest, + SuggestedFeeRecipient: common.Address{1}, + Withdrawals: make([]*types.Withdrawal, 0), + ParentBeaconBlockRoot: func() *common.Hash { h := randomHash(); return &h }(), + } + } + + // Wrong parent → Busy, no block built (call directly, not via the retryBusy helper). + res, err := exec.AssembleBlock(ctx, mkParams(randomHash())) + require.NoError(t, err) + require.True(t, res.Busy, "assemble on a non-head parent must return Busy") + + // Correct parent → builds a block extending exactly that parent. + payloadId, err := assembleBlock(ctx, exec, mkParams(head.Hash())) + require.NoError(t, err) + block, err := getAssembledBlock(ctx, exec, payloadId) + require.NoError(t, err) + require.NotNil(t, block) + require.Equal(t, head.Hash(), block.ParentHash(), "assembled block must extend the requested parent") + require.Equal(t, head.NumberU64()+1, block.NumberU64()) +} + func TestAssembleBlockWithContractCreation(t *testing.T) { t.Parallel() ctx := t.Context() diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 6f78b13139e..b2a154772ca 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -700,7 +700,6 @@ func New(tb testing.TB, opts ...Option) *ExecModuleTester { mock.TxPool, miningCancel, latestBlockBuiltStore, - nil, /*sdProvider*/ logger, ) diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index ffd2dde6166..cd9c41f3352 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -736,6 +736,8 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa bgGen = &commitGen{ sd: currentContext, roTx: roTx, + blockHash: blockHash, + blockNum: fcuHeader.Number.Uint64(), finishProgressBefore: finishProgressBefore, isSynced: isSynced, initialCycle: initialCycle, diff --git a/execution/execmodule/scoped_read.go b/execution/execmodule/scoped_read.go new file mode 100644 index 00000000000..1005c8d791e --- /dev/null +++ b/execution/execmodule/scoped_read.go @@ -0,0 +1,133 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execmodule + +import ( + "context" + "errors" + "sync/atomic" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" +) + +// errHeadMismatch means the current head is not the block the caller asked to +// read/build on — the published head has not (yet) reached that block, or a +// reorg moved it. The caller signals Busy rather than reading the wrong block. +var errHeadMismatch = errors.New("head is not at the requested block") + +// ScopedReadView is a by-block-consistent read snapshot pinned to a specific +// block under background commit: a base MDBX roTx captured at a settled instant +// (while the caller holds the foreground semaphore, so no commit is mid-flight) +// plus the head SharedDomains whose in-flight mem the snapshot layers on (nil +// when the committed DB is already current). getLatest reads sd.mem → the head +// chain → the roTx, and mem shadows the DB, so the two are one coherent view of +// the block regardless of whether its commit has landed. +// +// Single-owner: exactly one consumer holds it and MUST call Release exactly +// once. Release is idempotent; failing to call it leaks an open MDBX reader and +// wedges DB close. +type ScopedReadView struct { + roTx kv.TemporalTx + headSD *execctx.SharedDomains + blockHash common.Hash + blockNum uint64 + released atomic.Bool +} + +// Tx returns the pinned base roTx. Readers pass it to sd.AsGetter / +// BlockOverlay().NewReadView / BlockOverlayTemporalTx. +func (v *ScopedReadView) Tx() kv.TemporalTx { return v.roTx } + +// HeadSD returns the head SharedDomains this view is pinned to, or nil when the +// committed DB is already current (all generations committed). +func (v *ScopedReadView) HeadSD() *execctx.SharedDomains { return v.headSD } + +// BlockHash returns the block this view reads. +func (v *ScopedReadView) BlockHash() common.Hash { return v.blockHash } + +// BlockNum returns the number of the block this view reads. +func (v *ScopedReadView) BlockNum() uint64 { return v.blockNum } + +// Release rolls back the pinned roTx. Idempotent; safe on nil. +func (v *ScopedReadView) Release() { + if v == nil || v.released.Swap(true) { + return + } + if v.roTx != nil { + v.roTx.Rollback() + } +} + +// captureScopedReadView pins a consistent read snapshot for wantHash. The caller +// MUST hold the foreground semaphore (mutually exclusive with the background +// commit worker) so the snapshot is settled. The head is the newest +// not-yet-committed generation's SharedDomains when its block is wantHash, else +// the committed DB. Returns errHeadMismatch when the head is not wantHash so the +// caller can return Busy instead of reading the wrong block. The returned view +// owns the roTx — the caller must Release it. +func (e *ExecModule) captureScopedReadView(ctx context.Context, wantHash common.Hash) (*ScopedReadView, error) { + e.fgMu.Lock() + var head *execctx.SharedDomains + var headHash common.Hash + var headNum uint64 + if n := len(e.gens); n > 0 { + if last := e.gens[n-1]; !last.committed { + head, headHash, headNum = last.sd, last.blockHash, last.blockNum + } + } + e.fgMu.Unlock() + + roTx, err := e.db.BeginTemporalRo(ctx) //nolint:gocritic // handed to the returned view, which owns Rollback; guard below covers all error/panic paths before handoff + if err != nil { + return nil, err + } + handedOff := false + defer func() { + if !handedOff { + roTx.Rollback() + } + }() + + if head != nil { + if headHash != wantHash { + return nil, errHeadMismatch + } + handedOff = true + return &ScopedReadView{roTx: roTx, headSD: head, blockHash: wantHash, blockNum: headNum}, nil + } + + // No uncommitted generation: the committed DB is the head. Confirm wantHash + // is canonical at its number in this snapshot. + num, err := e.blockReader.HeaderNumber(ctx, roTx, wantHash) + if err != nil { + return nil, err + } + if num == nil { + return nil, errHeadMismatch + } + canon, err := e.canonicalHash(ctx, roTx, *num) + if err != nil { + return nil, err + } + if canon != wantHash { + return nil, errHeadMismatch + } + handedOff = true + return &ScopedReadView{roTx: roTx, blockHash: wantHash, blockNum: *num}, nil +} diff --git a/node/eth/backend.go b/node/eth/backend.go index 987db41694a..b20c5ab716c 100644 --- a/node/eth/backend.go +++ b/node/eth/backend.go @@ -875,7 +875,6 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger txnProvider, backend.miningSealingQuit, latestBlockBuiltStore, - backend.notifications.Events.LatestSD, logger, ) backend.pendingBlocks = blkBuilder.PendingBlockCh() From 56c0f3ca9486d6b75a6a79ddbc6c378a3a252b2f Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 15:48:54 +0000 Subject: [PATCH 141/167] execution/execmodule: bind scoped read view roTx to the background context AssembleBlock captures the view synchronously but the build runs asynchronously and outlives the request, so opening the roTx with the request-scoped ctx cancelled the build's reads on return ("get payload error: context canceled"). Bind it to the long-lived background context, matching the builder's prior roTx lifetime. --- execution/execmodule/scoped_read.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/execution/execmodule/scoped_read.go b/execution/execmodule/scoped_read.go index 1005c8d791e..94e6a2062b7 100644 --- a/execution/execmodule/scoped_read.go +++ b/execution/execmodule/scoped_read.go @@ -93,7 +93,10 @@ func (e *ExecModule) captureScopedReadView(ctx context.Context, wantHash common. } e.fgMu.Unlock() - roTx, err := e.db.BeginTemporalRo(ctx) //nolint:gocritic // handed to the returned view, which owns Rollback; guard below covers all error/panic paths before handoff + // Bind the roTx to the long-lived background context, not the request ctx: + // the view is handed to the async build, which outlives this call — a + // request-scoped ctx would be cancelled on return and fail the build's reads. + roTx, err := e.db.BeginTemporalRo(e.bacgroundCtx) //nolint:gocritic // handed to the returned view, which owns Rollback; guard below covers all error/panic paths before handoff if err != nil { return nil, err } From 1a7ecd2cb9b4f088a4ba06b09c6b83009f59577a Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 18:12:03 +0000 Subject: [PATCH 142/167] execution/stagedsync: don't save a genesis changeset in serial exec Serial exec gated SetChangesetAccumulator with `blockNum > 0` but gated the matching SavePastChangesetAccumulator only on shouldGenerateChangeSets, which is true for the genesis exec (blockNum==maxBlockNum==0, so 0+MaxReorgDepth >= 0). That saved an empty block-0 changeset; under background commit it flushes to ChangeSets3, dragging ReadLowestUnwindableBlock to 0. Genesis is never unwindable, so match the two guards. --- execution/stagedsync/exec3_serial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution/stagedsync/exec3_serial.go b/execution/stagedsync/exec3_serial.go index 01da451dbd8..d7f08fa9397 100644 --- a/execution/stagedsync/exec3_serial.go +++ b/execution/stagedsync/exec3_serial.go @@ -195,7 +195,7 @@ func (se *serialExecutor) exec(ctx context.Context, execStage *StageState, u Unw return nil, rwTx, err } - if shouldGenerateChangesets { + if shouldGenerateChangesets && blockNum > 0 { se.doms.SavePastChangesetAccumulator(b.Hash(), blockNum, changeSet) } se.doms.SetChangesetAccumulator(nil) From 582b08d7dcb83ebeccc4919c47ad4e1ac0d0c1d5 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 14 Jul 2026 21:46:36 +0000 Subject: [PATCH 143/167] execution/stagedsync: treat empty apply-loop close as clean, not more-work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the parallel apply loop's channel closes with no stop cause and the loop executed nothing (no tx-results, no blockResult), the requested range was already applied before this call — under background commit the async commit can advance execution progress to a single-block fork-validation target before its StateStep runs. The fallback classified that empty close as a partial batch and returned ErrLoopExhausted, which the stage loop reported as "unexpected state step has more work". Add applyLoopCloseIsClean so an empty loop is a clean end. --- execution/stagedsync/exec3_parallel.go | 20 ++++++++++-- .../exec3_parallel_robustness_test.go | 32 +++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 3563374657a..bc9113a735a 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -595,9 +595,11 @@ func (pe *parallelExecutor) execImpl(ctx context.Context, execStage *StageState, } // Fallback for exit paths that publish no cause: a single-block // fork-validation batch exits via execLoopExitCheck (no cause), and - // real shutdown cancels with context.Canceled. A fully-applied - // requested range is a clean end; otherwise there is more work. - if lastBlockResult.BlockNum >= pe.maxBlockNum { + // real shutdown cancels with context.Canceled. A fully-applied range + // — or an empty loop that executed nothing because the range was + // already applied (async background commit advanced progress) — is a + // clean end; otherwise there is more work. + if applyLoopCloseIsClean(lastBlockResult.BlockNum, pe.maxBlockNum, len(txResultBlocks)) { return nil } return &ErrLoopExhausted{From: startBlockNum, To: lastBlockResult.BlockNum, Reason: "block batch is full"} @@ -1476,6 +1478,18 @@ func execLoopShouldExit(blockResult *blockResult, sizeEst, batchLimit, maxBlockN return execLoopContinue } +// applyLoopCloseIsClean reports whether an apply-loop close with no published +// stop cause is a clean end rather than a partial batch to resume. It is clean +// when the requested range was fully applied (lastBlockNum >= maxBlockNum) or +// when the loop executed nothing at all (no tx-results and no blockResult) — +// the range was already applied before this call, so there is no pending work. +func applyLoopCloseIsClean(lastBlockNum, maxBlockNum uint64, txResultCount int) bool { + if lastBlockNum >= maxBlockNum { + return true + } + return txResultCount == 0 && lastBlockNum == 0 +} + // closeApplyChannels closes the apply-loop-bound channels in the order // the calculator and apply loop require: commitResults FIRST so the // calculator drains and closes rootResults, then applyResults so the diff --git a/execution/stagedsync/exec3_parallel_robustness_test.go b/execution/stagedsync/exec3_parallel_robustness_test.go index 236d8cc1aca..53b2e4a687a 100644 --- a/execution/stagedsync/exec3_parallel_robustness_test.go +++ b/execution/stagedsync/exec3_parallel_robustness_test.go @@ -726,6 +726,38 @@ func TestExecLoopShouldExitPriority(t *testing.T) { } } +// TestApplyLoopCloseIsClean pins the no-stop-cause apply-loop close +// classification. The load-bearing case is the empty loop +// (txResultCount==0, lastBlockNum==0): under background commit the async +// commit can advance execution progress to the validation target before a +// single-block fork-validation step runs, so the exec loop executes nothing +// and produces no blockResult. Treating that as pending work returns a +// spurious ErrLoopExhausted, which the stage loop reports as +// "unexpected state step has more work". +func TestApplyLoopCloseIsClean(t *testing.T) { + cases := []struct { + name string + lastBlockNum uint64 + maxBlockNum uint64 + txResults int + want bool + }{ + {name: "fully applied", lastBlockNum: 5, maxBlockNum: 5, txResults: 3, want: true}, + {name: "past target", lastBlockNum: 6, maxBlockNum: 5, txResults: 3, want: true}, + {name: "partial batch is not clean", lastBlockNum: 3, maxBlockNum: 5, txResults: 2, want: false}, + {name: "empty loop, nothing executed", lastBlockNum: 0, maxBlockNum: 21, txResults: 0, want: true}, + {name: "tx-results without blockResult is not clean", lastBlockNum: 0, maxBlockNum: 21, txResults: 4, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := applyLoopCloseIsClean(tc.lastBlockNum, tc.maxBlockNum, tc.txResults) + if got != tc.want { + t.Fatalf("applyLoopCloseIsClean(%d,%d,%d) = %v, want %v", tc.lastBlockNum, tc.maxBlockNum, tc.txResults, got, tc.want) + } + }) + } +} + // TestShouldMarkExhaustedAtBlock exercises the production // shouldMarkExhaustedAtBlock helper directly. The helper is the gate // that decides whether executeBlocks stamps a dispatched block with From ffacdd0e497f95662f375e6d04caa8a7339b6278 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 08:54:54 +0000 Subject: [PATCH 144/167] db/state, execution: one coordinated generation-chain lookup path for exec/validation readers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under background commit, in-flight state lives in a chain of in-memory SharedDomains generations and the raw DB lags. Exec/validation readers built their lookup path (the SetParent chain + block-overlay base) inconsistently, so a datum could be reachable via one capture but not the one the reader actually walked — surfacing as "nil block N" and spurious "state step has more work" under load. Two halves: - Committed-frame visibility: SharedDomains gains a readCoordinator (BeginCoordinatedRo), supplied by ExecModule.beginCoordinatedRo which opens the base roTx under fgMu. markGenCommitted runs under fgMu strictly after the DB commit, so a generation dropped from the parent chain as committed is guaranteed visible in that roTx — never in neither the chain nor the DB. Injected on currentContext / validation doms / InsertBlocks SD and inherited through SetParent; executeBlocks opens its base via BeginCoordinatedRo. - Uncommitted-frame reachability: ValidateChain now builds one lookup path. It refreshes the in-progress top's parent to the current in-flight tip, sets the SD parent first, then derives the block-overlay base from the SD's own parent chain (ParentBlockOverlayTemporalTx) — so block-metadata reads and domain-state reads traverse the identical generations instead of two separately-captured, divergent chains. Fork-payload unwind is unaffected: extending the parent only adds reachability below currentContext, and own mem/diffsets are consulted before the parent. --- db/state/execctx/domain_shared.go | 43 ++++++++++++++++++++++++++++- execution/execmodule/bg_commit.go | 12 ++++++++ execution/execmodule/exec_module.go | 39 +++++++++++++++----------- execution/execmodule/forkchoice.go | 1 + execution/execmodule/inserters.go | 1 + execution/stagedsync/exec3.go | 22 +++++++++------ 6 files changed, 93 insertions(+), 25 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 5a346bc715b..9bc5b53e793 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -125,6 +125,14 @@ type SharedDomains struct { // to read from the FCU's published SD without writing to it. parent *SharedDomains + // readCoordinator, when set, opens a base RO tx coordinated with the + // background-commit generation set: opened while holding the commit mutex so + // the tx's committed snapshot reflects every generation the parent chain has + // dropped as committed. Readers that need an underlying-DB base call + // BeginCoordinatedRo instead of opening an ad-hoc BeginTemporalRo, so a datum + // is always in the mem chain or this tx — never neither. + readCoordinator func(context.Context) (kv.TemporalTx, error) + // stateCache is an optional cache for state data (accounts, storage, code) stateCache *cache.StateCache @@ -684,7 +692,29 @@ func (sd *SharedDomains) InMemHistoryReads() bool { return sd.mem.InMem // SetParent sets a parent SD for read-through domain chaining. Domain reads // that miss in the local mem batch will check the parent's mem batch before // falling through to the underlying tx/aggregator. -func (sd *SharedDomains) SetParent(parent *SharedDomains) { sd.parent = parent } +func (sd *SharedDomains) SetParent(parent *SharedDomains) { + sd.parent = parent + // Inherit the parent's read coordinator so a child SD built for a read pass + // opens coordinated base txns without the caller re-wiring it. + if parent != nil && sd.readCoordinator == nil { + sd.readCoordinator = parent.readCoordinator + } +} + +// SetReadCoordinator wires the coordinated base-tx opener (see readCoordinator). +func (sd *SharedDomains) SetReadCoordinator(fn func(context.Context) (kv.TemporalTx, error)) { + sd.readCoordinator = fn +} + +// BeginCoordinatedRo opens a base RO tx coordinated with the generation set (see +// readCoordinator). Falls back to a plain tx from db when no coordinator is set +// (non-bg-commit setups), so callers can use it unconditionally. +func (sd *SharedDomains) BeginCoordinatedRo(ctx context.Context, db kv.TemporalRoDB) (kv.TemporalTx, error) { + if sd.readCoordinator != nil { + return sd.readCoordinator(ctx) + } + return db.BeginTemporalRo(ctx) +} // BlockOverlay returns the in-memory overlay for block-level metadata (headers, bodies, // canonical hashes, TD, stage progress, forkchoice markers). Callers can use this @@ -723,6 +753,17 @@ func (sd *SharedDomains) BlockOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalT return overlay.NewTemporalReadView(base) } +// ParentBlockOverlayTemporalTx returns the block-overlay read view of the parent +// chain only (excluding this SD's own overlay), or nil when there is no parent. +// A reader derives its overlay base from this so the base and the chain it later +// walks via BlockOverlayTemporalTx are the same generations — they cannot diverge. +func (sd *SharedDomains) ParentBlockOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { + if sd.parent == nil { + return nil + } + return sd.parent.BlockOverlayTemporalTx(roTx) +} + // InitBlockOverlay creates (or replaces) the block-level metadata overlay backed by // the given base transaction. Writes to the overlay are visible to subsequent reads // and are flushed atomically alongside domain state via Flush(). diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 3d88945bc11..4346d4ad39d 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -229,6 +229,18 @@ func (e *ExecModule) latestGen() *execctx.SharedDomains { return last.sd } +// beginCoordinatedRo opens a base RO tx under fgMu so its committed snapshot +// reflects every generation markGenCommitted has recorded — markGenCommitted runs +// under fgMu strictly after the DB commit, so a datum dropped from the parent +// chain as committed is guaranteed visible in this tx. Handed to SharedDomains as +// their read coordinator; exec/validation readers open through it instead of an +// ad-hoc BeginTemporalRo that could straddle a background commit. +func (e *ExecModule) beginCoordinatedRo(ctx context.Context) (kv.TemporalTx, error) { + e.fgMu.Lock() + defer e.fgMu.Unlock() + return e.db.BeginTemporalRo(ctx) +} + // addGen appends a new in-flight generation to the chain. func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Lock() diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 7553a2301d9..cb419f674bf 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -602,27 +602,15 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // Do not defer doms.Close(): on the success path ownership transfers to // forkValidator.sharedDom inside ValidatePayload and later phases close it, // so we Close explicitly only on the early-return error paths below. + doms.SetReadCoordinator(e.beginCoordinatedRo) doms.SetInMemHistoryReads(inMemHistoryReads) - // Back the validation overlay by the newest in-flight commit generation - // (gate item 2) so block-data reads cascade through its overlay instead - // of a stale DB while a background commit is in flight. - valOverlayBase := kv.TemporalTx(roTx) - if parent := e.latestGen(); parent != nil { - if v := parent.BlockOverlayTemporalTx(roTx); v != nil { - valOverlayBase = v - } - } - if err := doms.InitBlockOverlay(valOverlayBase, roTx.Debug().Dirs().Tmp); err != nil { - doms.Close() - return ValidationResult{}, fmt.Errorf("ValidateChain: init block overlay: %w", err) - } - var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain the validation SD to the latest in-memory canonical generation: // e.currentContext when present, otherwise the newest in-flight commit // generation (gate item 2 — the prior FCU cleared currentContext and - // handed its SD to the background commit). + // handed its SD to the background commit). This parent link is the single + // lookup path: the overlay base below is derived from it so block-data reads + // and domain-state reads traverse the identical generation chain. // // The parent link serves two roles: // @@ -641,11 +629,30 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // key the unwound canonical blocks touched, and TemporalMemBatch.getLatest // resolves those from the unwind set before ever consulting the parent. if e.currentContext != nil { + // Refresh the in-progress top's parent to the current in-flight tip so + // its chain reaches generations pushed since currentContext was created; + // otherwise its stale/nil parent leaves in-flight block data unreachable. + if parent := e.latestGen(); parent != nil { + e.currentContext.SetParent(parent) + } doms.SetParent(e.currentContext) } else if parent := e.latestGen(); parent != nil { doms.SetParent(parent) } + // Back the validation overlay by the SD's OWN parent chain (gate item 2) so + // block-data reads cascade through the same generations that domain-state + // reads do — never a separate, divergent capture. + valOverlayBase := kv.TemporalTx(roTx) + if v := doms.ParentBlockOverlayTemporalTx(roTx); v != nil { + valOverlayBase = v + } + if err := doms.InitBlockOverlay(valOverlayBase, roTx.Debug().Dirs().Tmp); err != nil { + doms.Close() + return ValidationResult{}, fmt.Errorf("ValidateChain: init block overlay: %w", err) + } + var tx kv.TemporalRwTx = doms.BlockOverlay() + // Flush block overlay data (headers, bodies, TDs from InsertBlocks) into // the validation overlay so unwindToCommonCanonical and ValidatePayload — // and the parallel exec goroutine via NewReadView — see this block data. diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index cd9c41f3352..2e37faba615 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -416,6 +416,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa } } if currentContext != nil { + currentContext.SetReadCoordinator(e.beginCoordinatedRo) currentContext.SetInMemHistoryReads(inMemHistoryReads) // Wire the state cache so canonical execution reads benefit from // the per-execution Account/Storage/Code cache. Previously only diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index fe7f7d8c58a..49efccda7c0 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -89,6 +89,7 @@ func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) // the parent-TD read below sees a previous FCU's not-yet-committed // block data instead of a stale DB. if sd != nil { + sd.SetReadCoordinator(e.beginCoordinatedRo) if parent := e.latestGen(); parent != nil { sd.SetParent(parent) } diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 857fb793840..4fa4d3b08b4 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -555,19 +555,25 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m defer close(blockRequests) } - // Open a thread-local roTx for block metadata and StepsInFiles. - // Must NOT use the stageloop's rwTx — it's thread-bound. - execRoTx, err := te.cfg.db.BeginTemporalRo(ctx) + // Open a thread-local base roTx for block metadata and StepsInFiles, + // coordinated with the background-commit generation set so a generation + // dropped from the parent chain as committed is visible here (never + // neither). Must NOT use the stageloop's rwTx — it's thread-bound. + execRoTx, err := te.doms.BeginCoordinatedRo(ctx, te.cfg.db) if err != nil { return fmt.Errorf("executeBlocks: open roTx: %w", err) } defer execRoTx.Rollback() - var blockTx kv.Tx - if overlay := te.doms.BlockOverlay(); overlay != nil { - blockTx = overlay.NewReadView(execRoTx) - } else { - blockTx = execRoTx + // Resolve block metadata memory-first: walk the block-overlay chain + // (local overlay + parent generations' overlays), falling through to the + // goroutine-local execRoTx for the underlying DB read only on a miss — + // mirroring how domain-state reads chain sd.mem → parents → tx. The parent + // chain must be layered ahead of the tx so an uncommitted ancestor + // generation's block data stays visible until its commit lands. + var blockTx kv.Tx = execRoTx + if v := te.doms.BlockOverlayTemporalTx(execRoTx); v != nil { + blockTx = v } // Use the max of all state domain steps (not just commitment) to From 4605dcc191b45520b8cd4aa95246167f2000384d Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 11:50:41 +0000 Subject: [PATCH 145/167] execution/builder, execution/execmodule: coordinate the builder's scoped read with the published head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The block builder read the parent block's state inconsistently with the txpool under background commit: captureScopedReadView dropped to the committed-DB path (headSD=nil) the moment the parent's generation committed, and the builder read block metadata through parentSD's local overlay only. The txpool, meanwhile, reads the retained published SD and its best() gates txns on lastSeenBlock >= ParentBlockNum. So the pool could yield txns valid at the parent block while the builder's state reader still saw the pre-parent nonce, filtering every txn as nonce-gapped and building an empty block. Coordinate both readers on the same block: - captureScopedReadView opens its base roTx under fgMu (atomic with the generation-set read) and uses the newest generation's retained SD as the head regardless of commit status — the same SD the txpool reads via the published SD — so the builder and pool process identical block state. - The builder reads block metadata (and executionAt) through the full parent generation chain (BlockOverlayTemporalTx) instead of parentSD's local overlay, so parent-block data held in an ancestor generation is visible. --- execution/builder/builder.go | 8 ++++++-- execution/execmodule/scoped_read.go | 25 ++++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/execution/builder/builder.go b/execution/builder/builder.go index 270bfe2a574..bb7ea99f8cc 100644 --- a/execution/builder/builder.go +++ b/execution/builder/builder.go @@ -134,8 +134,12 @@ func (b *Builder) Build(param *Parameters, interrupt *atomic.Bool) (result *type var compositeTx kv.TemporalTx = tx parentSD := view.HeadSD() if parentSD != nil { - if overlay := parentSD.BlockOverlay(); overlay != nil { - compositeTx = overlay.NewReadView(tx) + // Read block metadata (and stage progress) through the full parent + // generation chain, not just parentSD's local overlay — otherwise block + // data held in an ancestor generation is missed and executionAt reads + // stale, which desyncs the builder from the txpool's head. + if v := parentSD.BlockOverlayTemporalTx(tx); v != nil { + compositeTx = v } } diff --git a/execution/execmodule/scoped_read.go b/execution/execmodule/scoped_read.go index 94e6a2062b7..b27589bf9e9 100644 --- a/execution/execmodule/scoped_read.go +++ b/execution/execmodule/scoped_read.go @@ -82,21 +82,28 @@ func (v *ScopedReadView) Release() { // caller can return Busy instead of reading the wrong block. The returned view // owns the roTx — the caller must Release it. func (e *ExecModule) captureScopedReadView(ctx context.Context, wantHash common.Hash) (*ScopedReadView, error) { + // Capture the head generation and the base roTx atomically under fgMu, so the + // roTx's committed snapshot is coordinated with the captured generation set: + // markGenCommitted runs under fgMu strictly after the DB commit, so a datum is + // either in the captured head's mem chain or already visible in this roTx — + // never in neither. Binding to the background context (not the request ctx) + // keeps the roTx valid for the async build, which outlives this call. e.fgMu.Lock() var head *execctx.SharedDomains var headHash common.Hash var headNum uint64 if n := len(e.gens); n > 0 { - if last := e.gens[n-1]; !last.committed { - head, headHash, headNum = last.sd, last.blockHash, last.blockNum - } + // Use the newest generation's retained SD as the head regardless of + // commit status: its mem is kept after commit (drainCommittedGens no-op) + // and it is the same SD the txpool reads via the published SD, so the + // builder and the pool process the identical block state. Reads resolve + // mem-first over the coordinated roTx, so a committed generation's mem + // simply shadows the (equal) committed DB. + last := e.gens[n-1] + head, headHash, headNum = last.sd, last.blockHash, last.blockNum } - e.fgMu.Unlock() - - // Bind the roTx to the long-lived background context, not the request ctx: - // the view is handed to the async build, which outlives this call — a - // request-scoped ctx would be cancelled on return and fail the build's reads. roTx, err := e.db.BeginTemporalRo(e.bacgroundCtx) //nolint:gocritic // handed to the returned view, which owns Rollback; guard below covers all error/panic paths before handoff + e.fgMu.Unlock() if err != nil { return nil, err } @@ -115,7 +122,7 @@ func (e *ExecModule) captureScopedReadView(ctx context.Context, wantHash common. return &ScopedReadView{roTx: roTx, headSD: head, blockHash: wantHash, blockNum: headNum}, nil } - // No uncommitted generation: the committed DB is the head. Confirm wantHash + // No in-flight generation: the committed DB is the head. Confirm wantHash // is canonical at its number in this snapshot. num, err := e.blockReader.HeaderNumber(ctx, roTx, wantHash) if err != nil { From beec531a0c4b527de41decfe00aaa7b10e8859de Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 14:19:17 +0000 Subject: [PATCH 146/167] execution/execmodule: harden bg-commit generation lifecycle + builder head check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three P1 review items on the background-commit machinery: - Retire a generation only when its durable commit actually succeeds. runCommit previously called markGenCommitted unconditionally — even when fgAcquire failed or runPostForkchoice errored — silently dropping the generation's delta while a later generation committed only its own. Now a shutdown-cancelled commit leaves the generation un-retired (re-derived on restart) and a non-shutdown failure is fatal rather than pretended-committed. - Skip a generation whose set was discarded wholesale (SetHead unwind) while it sat queued. closeAllGens bumps a genEpoch; runCommit compares the gen's epoch under the semaphore and skips a superseded one instead of committing an already-closed SharedDomains (panic) and underflowing the reset counter. - captureScopedReadView's no-in-flight-generation path now requires wantHash to BE the current head (ReadHeadBlockHash), not merely canonical at its own height — an old canonical ancestor would otherwise pass and the async builder would fail its parent check later. Returns Busy so the caller retries. Tests: TestAssembleBlockRejectsOldCanonicalParent; existing suite race-clean in both exec modes. --- execution/execmodule/bg_commit.go | 47 +++++++++++++++++++----- execution/execmodule/exec_module.go | 7 +++- execution/execmodule/exec_module_test.go | 40 ++++++++++++++++++++ execution/execmodule/scoped_read.go | 18 ++++----- 4 files changed, 93 insertions(+), 19 deletions(-) diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 4346d4ad39d..e1eaf1ac195 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -47,7 +47,8 @@ type commitGen struct { finishProgressBefore uint64 isSynced bool initialCycle bool - committed bool // guarded by ExecModule.fgMu + committed bool // guarded by ExecModule.fgMu + epoch uint64 // genEpoch at enqueue; a stale epoch means superseded+closed } // fgTryAcquire acquires the foreground semaphore non-blocking and, on @@ -147,22 +148,46 @@ func (e *ExecModule) commitWorker() { // runCommit flushes + commits one generation's delta in a foreground-free // window. Called only by commitWorker. func (e *ExecModule) runCommit(gen *commitGen) { - // Hold the foreground semaphore for the whole flush+commit+prune, not just - // wait for idle: prune physically collates/removes domain history, and a - // foreground FCU acquiring mid-prune could run a reorg unwind against - // half-pruned state and re-execute to a wrong (empty) root. + // Hold the foreground semaphore across flush+commit+prune so it is + // coordinated with foreground tx opening — a foreground unwind then always + // opens against a consistent db/file view. This full exclusion is the current + // coordination mechanism; making the background pausable so foreground has + // priority (rather than blocking it) is tracked follow-up work. if err := e.fgAcquire(e.bacgroundCtx); err != nil { - e.markGenCommitted(gen) + // Could not acquire (shutdown-cancelled ctx): the commit did not run, so + // do NOT mark it committed — pretending it landed would drop the delta. + // The FCU's state is re-derivable on restart. return } defer e.fgRelease() - err := e.runPostForkchoice(gen.sd, gen.roTx, gen.finishProgressBefore, gen.isSynced, gen.initialCycle) - if err != nil && !errors.Is(err, context.Canceled) { - e.logger.Error("background commit failed", "err", err) + + // A stale epoch means the generation set was discarded (SetHead unwind) while + // this gen sat queued — its SharedDomains is closed, so skip it rather than + // commit a closed SD. The epoch bump and this check both hold the semaphore. + e.fgMu.Lock() + stale := gen.epoch != e.genEpoch + e.fgMu.Unlock() + if stale { + return } + + err := e.runPostForkchoice(gen.sd, gen.roTx, gen.finishProgressBefore, gen.isSynced, gen.initialCycle) // roTx is rolled back inside runForkchoiceFlushCommit between Flush and // Commit; this is a safety net (Rollback is idempotent). gen.roTx.Rollback() + if err != nil { + if errors.Is(err, context.Canceled) { + // Shutdown cancelled the commit; leave the gen un-retired (re-derived + // on restart) rather than mark a commit that did not land. + return + } + // A durable commit failed for a non-shutdown reason: the block was already + // reported VALID to the CL, so silently dropping its state would diverge + // from consensus. This is unrecoverable in-process — fail hard rather than + // retire the generation as if it committed. + e.logger.Crit("background commit failed; cannot retire generation", "block", gen.blockNum, "hash", gen.blockHash, "err", err) + return + } e.markGenCommitted(gen) // NOTE: the worker does NOT touch the published overlay (Events.LatestSD). // Publishing the latest executed block's SharedDomains is a foreground @@ -244,6 +269,7 @@ func (e *ExecModule) beginCoordinatedRo(ctx context.Context) (kv.TemporalTx, err // addGen appends a new in-flight generation to the chain. func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Lock() + gen.epoch = e.genEpoch e.gens = append(e.gens, gen) e.uncommittedGens++ e.fgMu.Unlock() @@ -273,6 +299,9 @@ func (e *ExecModule) drainCommittedGens() { func (e *ExecModule) closeAllGens() { e.fgMu.Lock() defer e.fgMu.Unlock() + // Invalidate any generation still queued for commit: the worker compares its + // gen's epoch against this and skips a superseded (now-closed) one. + e.genEpoch++ for _, g := range e.gens { g.sd.SetParent(nil) g.sd.Close() diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index c3b95e5968d..4563a02e02c 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -237,7 +237,12 @@ type ExecModule struct { fgIdle *sync.Cond gens []*commitGen uncommittedGens int - commitCh chan *commitGen + // genEpoch is bumped (under fgMu) whenever the generation set is discarded + // wholesale (closeAllGens, e.g. a SetHead unwind). A commit worker holding a + // generation from an earlier epoch skips it rather than committing an + // already-closed SharedDomains. + genEpoch uint64 + commitCh chan *commitGen // commitWorker lifecycle: commitWorkerStop signals the worker to drain // and exit; commitWg tracks it so shutdown (WaitIdle) waits for the // worker — and its in-flight commit txs — to finish before DB-close. diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 8374723de06..9d4de623dbc 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -890,6 +890,46 @@ func TestAssembleBlockRejectsWrongParent(t *testing.T) { require.Equal(t, head.NumberU64()+1, block.NumberU64()) } +// TestAssembleBlockRejectsOldCanonicalParent verifies AssembleBlock rejects a +// parent that is canonical at its own height but is not the current head: it +// must return Busy rather than build on a stale ancestor (which the async +// builder would then fail its parent check against). +func TestAssembleBlockRejectsOldCanonicalParent(t *testing.T) { + t.Parallel() + ctx := t.Context() + m := execmoduletester.New(t, execmoduletester.WithTxPool(), execmoduletester.WithChainConfig(chain.AllProtocolChanges)) + exec := m.ExecModule + + chainPack, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 2, func(i int, gen *blockgen.BlockGen) { + gen.SetCoinbase(common.Address{1}) + }, m.PublishedSD()) + require.NoError(t, err) + require.NoError(t, insertValidateAndUfc1By1(ctx, exec, chainPack.Blocks)) + head := chainPack.TopBlock // block 2 — the head + oldParent := chainPack.Blocks[0] // block 1 — canonical, but not the head + + mkParams := func(parent common.Hash) *builder.Parameters { + return &builder.Parameters{ + ParentHash: parent, + Timestamp: head.Time() + 1, + PrevRandao: head.Header().MixDigest, + SuggestedFeeRecipient: common.Address{1}, + Withdrawals: make([]*types.Withdrawal, 0), + ParentBeaconBlockRoot: func() *common.Hash { h := randomHash(); return &h }(), + } + } + + res, err := exec.AssembleBlock(ctx, mkParams(oldParent.Hash())) + require.NoError(t, err) + require.True(t, res.Busy, "assemble on an old canonical ancestor must return Busy") + + payloadId, err := assembleBlock(ctx, exec, mkParams(head.Hash())) + require.NoError(t, err) + block, err := getAssembledBlock(ctx, exec, payloadId) + require.NoError(t, err) + require.Equal(t, head.Hash(), block.ParentHash(), "assembled block must extend the head") +} + func TestAssembleBlockWithContractCreation(t *testing.T) { t.Parallel() ctx := t.Context() diff --git a/execution/execmodule/scoped_read.go b/execution/execmodule/scoped_read.go index b27589bf9e9..a37f68b1716 100644 --- a/execution/execmodule/scoped_read.go +++ b/execution/execmodule/scoped_read.go @@ -23,6 +23,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/state/execctx" ) @@ -122,20 +123,19 @@ func (e *ExecModule) captureScopedReadView(ctx context.Context, wantHash common. return &ScopedReadView{roTx: roTx, headSD: head, blockHash: wantHash, blockNum: headNum}, nil } - // No in-flight generation: the committed DB is the head. Confirm wantHash - // is canonical at its number in this snapshot. - num, err := e.blockReader.HeaderNumber(ctx, roTx, wantHash) - if err != nil { - return nil, err - } - if num == nil { + // No in-flight generation: the committed DB is the head. Require wantHash to + // BE the current head, not merely canonical at its own height — an older + // canonical ancestor is canonical but is not the tip, and building on it + // would only fail the async builder's parent check later. Signal Busy so the + // caller retries against the real head instead. + if headHash := rawdb.ReadHeadBlockHash(roTx); headHash != wantHash { return nil, errHeadMismatch } - canon, err := e.canonicalHash(ctx, roTx, *num) + num, err := e.blockReader.HeaderNumber(ctx, roTx, wantHash) if err != nil { return nil, err } - if canon != wantHash { + if num == nil { return nil, errHeadMismatch } handedOff = true From f592b6d52fd6c9808e62c90dfe77792b7c8d4cd3 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 14:21:37 +0000 Subject: [PATCH 147/167] rpc/jsonrpc: guard nil header in getProof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HeaderByNumber can return (nil, nil); getProof used header.Root unchecked and would panic during the publish-to-commit window when the raw tx lags the head. Return an error instead. The broader fix — getProof reading the tip's state via the published SharedDomains (consumer holds its own coordinated tx via BeginCoordinatedRo, no SD-aware tx) — is tracked in #21314. --- rpc/jsonrpc/eth_call.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rpc/jsonrpc/eth_call.go b/rpc/jsonrpc/eth_call.go index c8130b1e9de..af459a93acb 100644 --- a/rpc/jsonrpc/eth_call.go +++ b/rpc/jsonrpc/eth_call.go @@ -474,6 +474,9 @@ func (api *APIImpl) getProof(ctx context.Context, roTx kv.TemporalTx, address co if err != nil { return nil, err } + if header == nil { + return nil, fmt.Errorf("header not found for block %d", blockNrOrHash.BlockNumber.Uint64()) + } domains, err := execctx.NewSharedDomains(ctx, tx, log.New(), execctx.WithoutDeferredBranchUpdates(), execctx.WithSequentialCommitment()) if err != nil { From 6f0d543c15336ac63342ac6ace694222ba37d9cd Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 14:51:36 +0000 Subject: [PATCH 148/167] rpc/jsonrpc: wrap latest-block tip reads with the block overlay Under background commit the raw DB lags the tip, so RPC reads of the latest block number / current header must go through the published SD block overlay to see in-flight blocks. Several sites read the tip from a bare tx while their siblings wrapped it, so under bg-commit they reported a stale head. Route them through filters.WithOverlay like the rest. Latest-state readers (parity storage keys) need the SD-aware-consumer treatment tracked separately, not a metadata overlay wrap. --- rpc/jsonrpc/bor_api_impl.go | 12 ++++++------ rpc/jsonrpc/erigon_receipts.go | 4 ++-- rpc/jsonrpc/eth_api.go | 2 +- rpc/jsonrpc/eth_call.go | 2 +- rpc/jsonrpc/eth_receipts.go | 4 ++-- rpc/jsonrpc/eth_simulation.go | 2 +- rpc/jsonrpc/graphql_api.go | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/rpc/jsonrpc/bor_api_impl.go b/rpc/jsonrpc/bor_api_impl.go index 2508f0efd67..70cdf864f89 100644 --- a/rpc/jsonrpc/bor_api_impl.go +++ b/rpc/jsonrpc/bor_api_impl.go @@ -60,7 +60,7 @@ func (api *BorImpl) GetSnapshot(number *rpc.BlockNumber) (*Snapshot, error) { // Retrieve the requested block number (or current if none requested) var header *types.Header if number == nil || *number == rpc.LatestBlockNumber { - header = rawdb.ReadCurrentHeader(tx) + header = rawdb.ReadCurrentHeader(api.filters.WithOverlay(tx)) } else { header, _ = api.headerByNumber(ctx, *number, tx) } @@ -174,7 +174,7 @@ func (api *BorImpl) GetSigners(number *rpc.BlockNumber) ([]common.Address, error // Retrieve the requested block number (or current if none requested) var header *types.Header if number == nil || *number == rpc.LatestBlockNumber { - header = rawdb.ReadCurrentHeader(tx) + header = rawdb.ReadCurrentHeader(api.filters.WithOverlay(tx)) } else { header, _ = api.headerByNumber(ctx, *number, tx) } @@ -314,11 +314,11 @@ func (api *BorImpl) GetSnapshotProposer(blockNrOrHash *rpc.BlockNumberOrHash) (c var header *types.Header //nolint:nestif if blockNrOrHash == nil { - header = rawdb.ReadCurrentHeader(tx) + header = rawdb.ReadCurrentHeader(api.filters.WithOverlay(tx)) } else { if blockNr, ok := blockNrOrHash.Number(); ok { if blockNr == rpc.LatestBlockNumber { - header = rawdb.ReadCurrentHeader(tx) + header = rawdb.ReadCurrentHeader(api.filters.WithOverlay(tx)) } else { header, err = api.headerByNumber(ctx, blockNr, tx) } @@ -352,11 +352,11 @@ func (api *BorImpl) GetSnapshotProposerSequence(blockNrOrHash *rpc.BlockNumberOr // Retrieve the requested block number (or current if none requested) var header *types.Header if blockNrOrHash == nil { - header = rawdb.ReadCurrentHeader(tx) + header = rawdb.ReadCurrentHeader(api.filters.WithOverlay(tx)) } else { if blockNr, ok := blockNrOrHash.Number(); ok { if blockNr == rpc.LatestBlockNumber { - header = rawdb.ReadCurrentHeader(tx) + header = rawdb.ReadCurrentHeader(api.filters.WithOverlay(tx)) } else { header, err = api.headerByNumber(ctx, blockNr, tx) } diff --git a/rpc/jsonrpc/erigon_receipts.go b/rpc/jsonrpc/erigon_receipts.go index 8fbd5828eae..a9e2ddc330b 100644 --- a/rpc/jsonrpc/erigon_receipts.go +++ b/rpc/jsonrpc/erigon_receipts.go @@ -131,7 +131,7 @@ func (api *ErigonImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) } else { var err error - begin, end, err = logRangeLatestOnly(tx, crit) + begin, end, err = logRangeLatestOnly(api.filters.WithOverlay(tx), crit) if err != nil { return nil, err } @@ -208,7 +208,7 @@ func (api *ErigonImpl) GetLatestLogs(ctx context.Context, crit filters.FilterCri begin = header.Number.Uint64() end = header.Number.Uint64() } else { - begin, end, err = logRangeLatestOnly(tx, crit) + begin, end, err = logRangeLatestOnly(api.filters.WithOverlay(tx), crit) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_api.go b/rpc/jsonrpc/eth_api.go index b72af9adaa5..340e3d4ec71 100644 --- a/rpc/jsonrpc/eth_api.go +++ b/rpc/jsonrpc/eth_api.go @@ -439,7 +439,7 @@ func (api *BaseAPI) checkPruneField(tx kv.Tx, block uint64, field func(*prune.Mo if !amount.Enabled() { return nil } - latest, err := rpchelper.GetLatestBlockNumber(tx) + latest, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return err } diff --git a/rpc/jsonrpc/eth_call.go b/rpc/jsonrpc/eth_call.go index af459a93acb..76955fd9421 100644 --- a/rpc/jsonrpc/eth_call.go +++ b/rpc/jsonrpc/eth_call.go @@ -485,7 +485,7 @@ func (api *APIImpl) getProof(ctx context.Context, roTx kv.TemporalTx, address co defer domains.Close() sdCtx := domains.GetCommitmentContext() - latestBlock, err := rpchelper.GetLatestBlockNumber(roTx) + latestBlock, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(roTx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_receipts.go b/rpc/jsonrpc/eth_receipts.go index 5332e4dda11..560b14c7b86 100644 --- a/rpc/jsonrpc/eth_receipts.go +++ b/rpc/jsonrpc/eth_receipts.go @@ -228,7 +228,7 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t return nil, &rpc.CustomError{Message: errInvalidBlockRange, Code: rpc.ErrCodeInvalidParams} } if end > roaring.MaxUint32 { - latest, err := rpchelper.GetLatestBlockNumber(tx) + latest, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } @@ -240,7 +240,7 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t // Check if the requested blocks have been executed. // This prevents returning empty results when blocks exist but haven't been executed yet. - latestExecuted, err := rpchelper.GetLatestExecutedBlockNumber(tx) + latestExecuted, err := rpchelper.GetLatestExecutedBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/eth_simulation.go b/rpc/jsonrpc/eth_simulation.go index be67abb6cb2..5400b6a1b6c 100644 --- a/rpc/jsonrpc/eth_simulation.go +++ b/rpc/jsonrpc/eth_simulation.go @@ -129,7 +129,7 @@ func (api *APIImpl) SimulateV1(ctx context.Context, req SimulationRequest, block if err != nil { return nil, err } - latestBlockNumber, err := rpchelper.GetLatestBlockNumber(tx) + latestBlockNumber, err := rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) if err != nil { return nil, err } diff --git a/rpc/jsonrpc/graphql_api.go b/rpc/jsonrpc/graphql_api.go index 8b88d2f1926..e275ce1b20d 100644 --- a/rpc/jsonrpc/graphql_api.go +++ b/rpc/jsonrpc/graphql_api.go @@ -89,7 +89,7 @@ func (api *GraphQLAPIImpl) GetLatestBlockNumber(ctx context.Context) (uint64, er return 0, err } defer tx.Rollback() - return rpchelper.GetLatestBlockNumber(tx) + return rpchelper.GetLatestBlockNumber(api.filters.WithOverlay(tx)) } func (api *GraphQLAPIImpl) GetBlockNumberForTx(ctx context.Context, hash common.Hash) (uint64, bool, error) { From 2fe4547b549c914d40ecb1a52360a244dd459d82 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 14:51:42 +0000 Subject: [PATCH 149/167] execution/execmodule, db/kv: dedup overlay-base, drop dead code, add gen-stack tests Extract overlayBaseFor to replace three copies of the latestGen block-overlay base derivation, and genSuperseded for the runCommit epoch-skip check. Remove the unused HasOverlayDomainGetter. Trim the generation-stack comments to the load-bearing invariants and correct a stale one describing worker-republish behaviour that no longer exists. Add unit tests for the generation-stack counters, latestGen, and the epoch guard that skips a superseded generation. --- db/kv/membatchwithdb/memory_mutation.go | 7 -- execution/execmodule/bg_commit.go | 76 ++++++++++----------- execution/execmodule/bg_commit_gens_test.go | 71 +++++++++++++++++++ execution/execmodule/exec_module.go | 21 ++---- execution/execmodule/forkchoice.go | 31 +++------ execution/execmodule/inserters.go | 11 +-- 6 files changed, 124 insertions(+), 93 deletions(-) create mode 100644 execution/execmodule/bg_commit_gens_test.go diff --git a/db/kv/membatchwithdb/memory_mutation.go b/db/kv/membatchwithdb/memory_mutation.go index 6c78f827a30..cab6e0159b9 100644 --- a/db/kv/membatchwithdb/memory_mutation.go +++ b/db/kv/membatchwithdb/memory_mutation.go @@ -1098,13 +1098,6 @@ func (m *MemoryMutation) SetDomainGetterFactory(f func(tx kv.TemporalTx) kv.Temp m.domainGetterFor = f } -// HasOverlayDomainGetter reports whether this read view routes domain reads -// through a SharedDomains-backed getter — i.e. it is an overlay view of an -// in-flight async-commit generation. Callers use this to bypass caches whose -// coherency is keyed to the committed DB and would otherwise serve a value -// lagging a not-yet-landed background commit. -func (m *MemoryMutation) HasOverlayDomainGetter() bool { return m.domainGetter != nil } - // OverlayTemporalReadView extends an overlay read view with kv.TemporalTx // support. It embeds a *MemoryMutation for all overlay-aware KV methods // (GetOne, Cursor, etc.) and delegates temporal methods (GetLatest, GetAsOf, diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index e1eaf1ac195..a40a0c4e600 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -25,16 +25,11 @@ import ( "github.com/erigontech/erigon/db/state/execctx" ) -// Background commit / foreground-priority coordination (gate item 2). -// -// Base intent: foreground execution (FCU / ValidateChain / InsertBlocks / -// AssembleBlock / SetHead) is NEVER blocked on the DB. The post-FCU -// flush+commit+prune runs on a single background goroutine that yields to -// foreground: a commit starts only in a foreground-free window, so the -// commit RwTx never overlaps a foreground roTx and never pins MDBX pages -// (which would grow the freelist / DB file). An in-flight commit is never -// aborted — a foreground op that arrives mid-commit simply makes the *next* -// commit wait. See project_fcu_semaphore_decouple_plan. +// Background commit coordination: the post-FCU flush+commit+prune runs on a +// single background goroutine in a foreground-free window, so the commit RwTx +// never overlaps a foreground roTx (which would pin MDBX pages and grow the +// freelist). An in-flight commit is never aborted; a foreground op arriving +// mid-commit makes the next commit wait. // commitGen is one in-flight background-commit unit: the SharedDomains of a // completed FCU whose domain state must be flushed + committed off the @@ -161,13 +156,9 @@ func (e *ExecModule) runCommit(gen *commitGen) { } defer e.fgRelease() - // A stale epoch means the generation set was discarded (SetHead unwind) while - // this gen sat queued — its SharedDomains is closed, so skip it rather than - // commit a closed SD. The epoch bump and this check both hold the semaphore. - e.fgMu.Lock() - stale := gen.epoch != e.genEpoch - e.fgMu.Unlock() - if stale { + // A superseded generation had its set discarded (SetHead unwind) while it sat + // queued — its SharedDomains is closed, so skip it rather than commit a closed SD. + if e.genSuperseded(gen) { return } @@ -189,12 +180,9 @@ func (e *ExecModule) runCommit(gen *commitGen) { return } e.markGenCommitted(gen) - // NOTE: the worker does NOT touch the published overlay (Events.LatestSD). - // Publishing the latest executed block's SharedDomains is a foreground - // concern, owned by updateForkChoice (dispatchNotificationsFromOverlay). - // A background commit landing does not change "what the latest block is", - // so it must not republish — doing so previously nil'd LatestSD whenever - // the worker caught up, leaving the block builder / RPC caches blind. + // The worker must not touch the published SD (Events.LatestSD): what the + // latest block is stays a foreground concern (updateForkChoice), and a commit + // landing does not change it. } // markGenCommitted records that a generation's commit has landed. The @@ -254,6 +242,18 @@ func (e *ExecModule) latestGen() *execctx.SharedDomains { return last.sd } +// overlayBaseFor wraps roTx with the newest in-flight generation's block overlay +// when one exists, so block-data reads (headers, bodies, TDs) see a not-yet-committed +// FCU's writes instead of a stale DB. Returns roTx unchanged when there is none. +func (e *ExecModule) overlayBaseFor(roTx kv.TemporalTx) kv.TemporalTx { + if parent := e.latestGen(); parent != nil { + if v := parent.BlockOverlayTemporalTx(roTx); v != nil { + return v + } + } + return roTx +} + // beginCoordinatedRo opens a base RO tx under fgMu so its committed snapshot // reflects every generation markGenCommitted has recorded — markGenCommitted runs // under fgMu strictly after the DB commit, so a datum dropped from the parent @@ -266,6 +266,14 @@ func (e *ExecModule) beginCoordinatedRo(ctx context.Context) (kv.TemporalTx, err return e.db.BeginTemporalRo(ctx) } +// genSuperseded reports whether gen's set was discarded (via closeAllGens bumping +// genEpoch) since it was enqueued. The epoch bump and this check both hold fgMu. +func (e *ExecModule) genSuperseded(gen *commitGen) bool { + e.fgMu.Lock() + defer e.fgMu.Unlock() + return gen.epoch != e.genEpoch +} + // addGen appends a new in-flight generation to the chain. func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Lock() @@ -275,23 +283,11 @@ func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Unlock() } -// drainCommittedGens is intentionally a no-op for now (model A). -// -// Each generation's SharedDomains.mem is kept after commit (FlushKeepMem) so -// the generation stays a stable, self-contained snapshot — it may still be -// Events.LatestSD, or a parent in some reader's chain. Freeing one mid-run -// would need to prove no consumer (block builder, RPC cache, txpool) is still -// reading it; LatestSD hands out a bare pointer, so that proof needs a -// refcount / scoped-read primitive we have not built yet. Until then, -// generations are released as a unit at shutdown by closeAllGens. -// -// Precise per-generation freeing — a refcounted/scoped-read lifetime, or a -// shallow COW snapshot decoupled from the generation — is the explicit -// memory-coordination follow-up (next optimization phase). The call site is -// kept as the documented hook for that work. -func (e *ExecModule) drainCommittedGens() { - // no-op — see closeAllGens and the memory-coordination follow-up -} +// drainCommittedGens is a no-op: committed generations' mem is retained (they +// may still be a parent in a reader's chain) and freed as a unit at shutdown by +// closeAllGens. Precise per-generation freeing needs a refcount keyed on a +// publication id — tracked in erigontech/erigon#22494. +func (e *ExecModule) drainCommittedGens() {} // closeAllGens detaches + closes every remaining generation. Called only at // shutdown (after the commit worker has stopped) to release the generations diff --git a/execution/execmodule/bg_commit_gens_test.go b/execution/execmodule/bg_commit_gens_test.go new file mode 100644 index 00000000000..226e80d32de --- /dev/null +++ b/execution/execmodule/bg_commit_gens_test.go @@ -0,0 +1,71 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execmodule + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/db/state/execctx" +) + +func newGenTestModule() *ExecModule { + em := &ExecModule{} + em.fgIdle = sync.NewCond(&em.fgMu) + return em +} + +func TestGenStackCountersAndLatestGen(t *testing.T) { + em := newGenTestModule() + a := &commitGen{sd: new(execctx.SharedDomains)} + b := &commitGen{sd: new(execctx.SharedDomains)} + + em.addGen(a) + em.addGen(b) + require.Equal(t, 2, em.uncommittedGens) + require.True(t, em.latestGen() == b.sd, "latestGen is the newest uncommitted generation") + + em.markGenCommitted(b) + require.Equal(t, 1, em.uncommittedGens) + require.Nil(t, em.latestGen(), "a committed newest generation means read straight from the DB") + + em.markGenCommitted(a) + require.Equal(t, 0, em.uncommittedGens) + em.WaitCommitsDrained() // returns immediately once the backlog is empty +} + +func TestGenEpochGuardSkipsSupersededGen(t *testing.T) { + em := newGenTestModule() + a := &commitGen{sd: new(execctx.SharedDomains)} + em.addGen(a) + require.Equal(t, uint64(0), a.epoch) + require.False(t, em.genSuperseded(a), "a freshly enqueued generation is current") + + // closeAllGens bumps genEpoch to discard the queued set (the SetHead-unwind + // path); it also closes the real SharedDomains, so exercise just the bump here. + em.fgMu.Lock() + em.genEpoch++ + em.fgMu.Unlock() + require.True(t, em.genSuperseded(a), "a generation whose set was discarded is superseded") + + c := &commitGen{sd: new(execctx.SharedDomains)} + em.addGen(c) + require.Equal(t, uint64(1), c.epoch) + require.False(t, em.genSuperseded(c), "a generation enqueued after the bump is current") +} diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 4563a02e02c..05b009d9432 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -228,10 +228,8 @@ type ExecModule struct { currentContext *execctx.SharedDomains publishedSD func() *execctx.SharedDomains // fallback for background commit - // Background-commit / foreground-priority coordination (gate item 2 — - // see bg_commit.go). fgMu guards fgCount + gens; fgIdle is signalled - // when fgCount reaches zero so the commit worker can run in a - // foreground-free window. + // fgMu guards fgCount + gens; fgIdle is signalled when fgCount reaches zero + // so the commit worker can run in a foreground-free window (see bg_commit.go). fgMu sync.Mutex fgCount int fgIdle *sync.Cond @@ -332,7 +330,7 @@ func NewExecModule( stateCache.execModule = em } - // Start the background-commit worker (gate item 2). It pulls completed + // Start the background-commit worker. It pulls completed // generations off commitCh and commits each in a foreground-free // window. The buffered channel keeps the foreground FCU's hand-off // non-blocking. WaitIdle stops the worker before DB-close. @@ -512,7 +510,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b e.hook.LastNewBlockSeen(blockNumber) // used by eth_syncing // currentContext is nil while a background commit holds the previous - // FCU's SD (gate item 2) — guard the access. + // FCU's SD — guard the access. if e.currentContext != nil { e.currentContext.ResetPendingUpdates() } @@ -555,12 +553,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } defer roTx.Rollback() - var src kv.TemporalTx = roTx - if parent := e.latestGen(); parent != nil { - if v := parent.BlockOverlayTemporalTx(roTx); v != nil { - src = v - } - } + src := e.overlayBaseFor(roTx) header, err = e.blockReader.Header(ctx, src, blockHash, blockNumber) if err != nil { return ValidationResult{}, err @@ -612,7 +605,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // Chain the validation SD to the latest in-memory canonical generation: // e.currentContext when present, otherwise the newest in-flight commit - // generation (gate item 2 — the prior FCU cleared currentContext and + // generation (the prior FCU cleared currentContext and // handed its SD to the background commit). This parent link is the single // lookup path: the overlay base below is derived from it so block-data reads // and domain-state reads traverse the identical generation chain. @@ -645,7 +638,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b doms.SetParent(parent) } - // Back the validation overlay by the SD's OWN parent chain (gate item 2) so + // Back the validation overlay by the SD's OWN parent chain so // block-data reads cascade through the same generations that domain-state // reads do — never a separate, divergent capture. valOverlayBase := kv.TemporalTx(roTx) diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 2e37faba615..ea4d387bcf8 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -357,12 +357,12 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa return fmt.Errorf("semaphore timeout") } // The semaphore is always released when updateForkChoice returns — the - // background commit no longer holds it (gate item 2: the commit runs on + // background commit no longer holds it (the commit runs on // the bg worker, decoupled from the foreground semaphore). defer e.fgRelease() // Retire the in-flight commit-generation chain if every generation has - // committed (gate item 2). Safe here: the foreground semaphore is held, + // committed. Safe here: the foreground semaphore is held, // so no concurrent op is reading a generation's SharedDomains. e.drainCommittedGens() @@ -423,7 +423,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // ValidateChain (fork validation, exec_module.go) set this, leaving // the canonical execution path running uncached against the aggTx. currentContext.SetStateCache(e.stateCache) - // Chain to the newest in-flight commit generation (gate item 2): + // Chain to the newest in-flight commit generation: // if a previous FCU's commit has not yet landed, its domain state // lives only in that generation's sd.mem — SetParent lets this FCU // read through to it instead of a stale DB. nil when the chain is @@ -453,19 +453,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // hashes, stage progress, forkchoice markers, TxNums, etc.). All // pipeline reads cascade through the overlay to the RO tx; writes // stay in memory until commit. - // Back the block overlay by the newest in-flight commit generation's - // overlay when one exists (gate item 2): block data — headers, bodies, - // TDs, receipts — written by a not-yet-committed FCU lives in that - // generation's overlay. A concurrent-safe temporal read view lets this - // FCU read through to it instead of a stale DB. The generation chain - // (closed as a unit by drainCommittedGens) keeps the backing valid. - overlayBase := kv.TemporalTx(roTx) - if parent := e.latestGen(); parent != nil { - if v := parent.BlockOverlayTemporalTx(roTx); v != nil { - overlayBase = v - } - } - if err := currentContext.InitBlockOverlay(overlayBase, roTx.Debug().Dirs().Tmp); err != nil { + if err := currentContext.InitBlockOverlay(e.overlayBaseFor(roTx), roTx.Debug().Dirs().Tmp); err != nil { return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, fmt.Errorf("updateForkChoice: init block overlay: %w", err), false) } var tx kv.TemporalRwTx = currentContext.BlockOverlay() @@ -726,12 +714,9 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa e.closeModuleContext() } - // Background commit (gate item 2): register the generation in the - // in-flight chain BEFORE dispatching notifications / publishing the - // overlay. The commit worker republishes the overlay as it commits - // each generation (bg_commit.go: latestUncommittedGenSD); if this - // generation were not yet in e.gens at that point, the worker would - // publish a stale (older or nil) overlay and clobber this FCU's. + // Register the generation in the in-flight chain before publishing the + // overlay, so a reader that observes the newly published SD can also reach + // its uncommitted state as latestGen() when chaining parents. var bgGen *commitGen if e.fcuBackgroundCommit { bgGen = &commitGen{ @@ -767,7 +752,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // fcuBackgroundCommit is explicitly enabled. No semaphore handoff — // fgRelease (the outer defer) drops the foreground lock the moment // updateForkChoice returns, so the next FCU never blocks on this - // commit (gate item 2 removes the commit-holds-the-semaphore gate). + // commit. var commitTimings []any if e.fcuBackgroundCommit { // Hand the commit to the background worker: it commits in a diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index 49efccda7c0..5b67892208b 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -24,7 +24,6 @@ import ( "github.com/holiman/uint256" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/commitment/commitmentdb" @@ -85,7 +84,7 @@ func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) } e.logger.Info("ethereumExecutionModule.InsertBlocks: state ahead of blocks, proceeding with catch-up", "err", err) } - // Chain to the newest in-flight commit generation (gate item 2) so + // Chain to the newest in-flight commit generation so // the parent-TD read below sees a previous FCU's not-yet-committed // block data instead of a stale DB. if sd != nil { @@ -99,13 +98,7 @@ func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) e.lock.Unlock() } if sd.BlockOverlay() == nil { - overlayBase := kv.TemporalTx(roTx) - if parent := e.latestGen(); parent != nil { - if v := parent.BlockOverlayTemporalTx(roTx); v != nil { - overlayBase = v - } - } - if err := sd.InitBlockOverlay(overlayBase, roTx.Debug().Dirs().Tmp); err != nil { + if err := sd.InitBlockOverlay(e.overlayBaseFor(roTx), roTx.Debug().Dirs().Tmp); err != nil { return 0, fmt.Errorf("ethereumExecutionModule.InsertBlocks: %w", err) } } else { From f7caf29a315111876ca6d10412938619556ea135 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 15 Jul 2026 16:21:12 +0000 Subject: [PATCH 150/167] execution/execmodule: remove unused waitForegroundIdle The background commit worker acquires the foreground semaphore before committing rather than waiting on a foreground-idle window, so waitForegroundIdle has no callers. Remove it and correct the stale comment that still referenced it. --- execution/execmodule/bg_commit.go | 11 ----------- execution/execmodule/forkchoice.go | 10 +++++----- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index a40a0c4e600..a1632eb9685 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -93,17 +93,6 @@ func (e *ExecModule) leaveForeground() { e.fgMu.Unlock() } -// waitForegroundIdle blocks until no foreground operation is active. The -// commit worker calls it before each commit so the commit RwTx runs in a -// foreground-free window — no foreground roTx open to pin MDBX pages. -func (e *ExecModule) waitForegroundIdle() { - e.fgMu.Lock() - for e.fgCount > 0 { - e.fgIdle.Wait() - } - e.fgMu.Unlock() -} - // enqueueCommit hands a completed generation to the background commit // worker. Non-blocking by design (buffered channel) so the foreground FCU // is never blocked handing off its commit. diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index ea4d387bcf8..b3d78a63e32 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -755,11 +755,11 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // commit. var commitTimings []any if e.fcuBackgroundCommit { - // Hand the commit to the background worker: it commits in a - // foreground-idle window (commitWorker → waitForegroundIdle) so the - // commit RwTx never overlaps a foreground roTx, and the generation is - // on the chain so the next FCU's SD reads its not-yet-committed domain - // state via SetParent instead of waiting. + // Hand the commit to the background worker: it acquires the foreground + // semaphore before committing so the commit RwTx never overlaps a + // foreground roTx, and the generation is on the chain so the next FCU's + // SD reads its not-yet-committed domain state via SetParent instead of + // waiting. roTx = nil currentContext = nil e.enqueueCommit(bgGen) From 2fe51771b89b573dc1a73f9d38345397a5485add Mon Sep 17 00:00:00 2001 From: mh0lt Date: Thu, 16 Jul 2026 10:54:42 +0000 Subject: [PATCH 151/167] execution/exec, execution/execmodule: route read-ahead prefetch through the published SD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warmBody prefetcher read the raw temporal tx and wrote the process-global state cache directly via cachePopulatingGetter — the only cache writer outside SharedDomains, and under background commit it read state behind the tip. Route each prefetch worker through the published SharedDomains (own coordinated tx, read via AsGetter) so reads see in-flight tip state and cache population happens in the SD's own read-fill. Delete cachePopulatingGetter; read the BAL through the SD block overlay. This also collapses the two cache-fill sites into one. Follow-up (#22520): an SD-owned run-task API and dropping the parallel-exec start queue. --- execution/exec/blocks_read_ahead.go | 152 +++++++++-------------- execution/exec/blocks_read_ahead_test.go | 125 ------------------- execution/execmodule/exec_module.go | 13 +- 3 files changed, 69 insertions(+), 221 deletions(-) delete mode 100644 execution/exec/blocks_read_ahead_test.go diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index a9b686d51d6..6bdd6161a88 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -10,14 +10,14 @@ import ( "golang.org/x/sync/errgroup" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/dbservices" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" - "github.com/erigontech/erigon/execution/cache" + "github.com/erigontech/erigon/db/state/execctx" + "github.com/erigontech/erigon/db/state/kvmetrics" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" @@ -34,13 +34,12 @@ type BlockReadAheader struct { warming atomic.Bool // only one warmBody can run at a time warmWg sync.WaitGroup - // stateCache is the process-global state cache that SharedDomains.GetLatest - // consults on the EVM hot path. When set, warmBody routes its prefetches - // through a cache-populating getter so the same hashmap the EVM probes is - // pre-warmed. Without it, prefetches only warm OS page cache + RoTx - // cursors — disconnected from the cache layer the EVM actually reads. - // Mirrors reth's CachedReads / ExecutionCache "same hashmap" property. - stateCache *cache.StateCache + // publishedSD returns the latest published SharedDomains (the stable tip leaf). + // When set, warmBody prefetches through it so reads see in-flight tip state and + // the SD's own read-fill warms the process-global cache the EVM probes — keeping + // cache population an SD concern. A nil provider (or nil return) falls back to a + // raw read that does not touch the cache. + publishedSD func() *execctx.SharedDomains } func NewBlockReadAheader() *BlockReadAheader { @@ -63,64 +62,33 @@ func NewBlockReadAheader() *BlockReadAheader { } } -// SetStateCache wires the process-global state cache so warmBody's -// prefetches land in the same hashmap that SharedDomains.GetLatest probes -// on the EVM hot path. Without this, prefetches warm OS page cache only — -// the EVM still pays the file accessor stack on its first per-address read. -// Idempotent; safe to call before the first AddHeaderAndBody. -func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { - bra.stateCache = sc +// SetPublishedSD wires the published-SharedDomains provider so warmBody +// prefetches read through the in-flight tip state and the SD's read-fill warms +// the process-global cache the EVM probes. Idempotent; safe to call before the +// first AddHeaderAndBody. +func (bra *BlockReadAheader) SetPublishedSD(provider func() *execctx.SharedDomains) { + bra.publishedSD = provider } -// cachePopulatingGetter wraps a kv.TemporalGetter and writes successful -// reads through to a cache.StateCache as a side effect. Used by warmBody -// to make read-ahead prefetches populate the same in-process cache layer -// that SharedDomains.GetLatest consults — eliminating the file-accessor -// stack cost on the EVM's first touch of any prefetched address. -// -// For the CodeDomain the wrapper also populates the codeHashToCode -// (codeHash→bytes) + size-cache layers via PutCodeWithHash, keyed by the -// code's own keccak hash so every cached pair is self-consistent. -type cachePopulatingGetter struct { - g kv.TemporalGetter - sc *cache.StateCache - stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) -} - -func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { - v, step, err := cpg.g.GetLatest(name, k) - if err == nil && cpg.sc != nil { - // If-absent writes only: this runs in a fire-and-forget goroutine over a - // committed snapshot, so an unconditional Put racing an FCU flush's - // cache-apply could replace the flushed value with the pre-flush one. - if name == kv.CodeDomain && len(v) > 0 { - // Key the content cache by the code's OWN hash, never a separately - // read account codeHash: under parallel/speculative exec that hash - // can be skewed or cross-account, and a (hash, code) pair that - // doesn't satisfy keccak(code)==hash poisons every account sharing - // the hash. keccak(v) makes each entry self-consistent. - cpg.sc.PutCodeWithHashIfAbsent(k, v, crypto.Keccak256(v), (uint64(step)+1)*cpg.stepSize-1) - } else { - // Cache including nil/empty results: a probe returning no - // bytes is a valid negative answer (missing account, empty - // storage slot; empty code lands here too but CodeCache drops - // zero-length puts) and caching it lets repeated probes - // skip the file accessor stack. Mirrors revm's CacheAccount - // { account: None, status: LoadedNotExisting } pattern. - // Stamp with an upper bound on the value's write txNum (last txNum - // of the step it came from) so unwind invalidation is correct. - cpg.sc.PutIfAbsent(name, k, v, (uint64(step)+1)*cpg.stepSize-1) +// warmView opens a per-goroutine read view for prefetching. With a published SD +// it reads through it — in-flight tip state, and the SD's own read-fill warms the +// process-global cache (cache population stays an SD concern). Without one it +// reads the raw tx and does not populate the cache. A throwaway per-worker +// metrics accumulator keeps concurrent workers off the SD's shared request +// metrics. +func warmView(ctx context.Context, tdb kv.TemporalRoDB, sd *execctx.SharedDomains) (kv.TemporalTx, kv.TemporalGetter, error) { + if sd != nil { + ttx, err := sd.BeginCoordinatedRo(ctx, tdb) //nolint:gocritic // tx is returned to the caller, which defers Rollback + if err != nil { + return nil, nil, err } + return ttx, sd.AsGetterMetered(ttx, kvmetrics.NewDomainMetrics()), nil } - return v, step, err -} - -func (cpg *cachePopulatingGetter) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { - return cpg.g.HasPrefix(name, prefix) -} - -func (cpg *cachePopulatingGetter) StepsInFiles(entitySet ...kv.Domain) kv.Step { - return cpg.g.StepsInFiles(entitySet...) + ttx, err := tdb.BeginTemporalRo(ctx) //nolint:gocritic // tx is returned to the caller, which defers Rollback + if err != nil { + return nil, nil, err + } + return ttx, ttx, nil } func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body) { @@ -177,16 +145,36 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t workers = 1 } + tdb, ok := db.(kv.TemporalRoDB) + if !ok { + return + } + // Capture the published tip leaf once; all workers share it (concurrent reads + // are safe on the stable published SD) and each opens its own coordinated tx. + // SetHead drains in-flight warmup before closing generations, so the captured + // leaf can't be closed out from under a worker. + var sd *execctx.SharedDomains + if bra.publishedSD != nil { + sd = bra.publishedSD() + } + var wg errgroup.Group - // If BAL exists in DB, use BAL warming (more complete) + // If a BAL exists, use it (more complete). Read it through the SD's block + // overlay so an in-flight tip BAL is visible before its commit lands. var bal types.BlockAccessList - if header != nil && db != nil { - tx, err := db.BeginRo(ctx) + if header != nil { + btx, err := tdb.BeginTemporalRo(ctx) if err != nil { log.Warn("[warmBody] failed to open tx for BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) } else { - data, err := tx.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(header.Number.Uint64(), header.Hash())) + var balSrc kv.TemporalTx = btx + if sd != nil { + if ov := sd.BlockOverlayTemporalTx(btx); ov != nil { + balSrc = ov + } + } + data, err := balSrc.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(header.Number.Uint64(), header.Hash())) if err != nil { log.Warn("[warmBody] failed to read BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) } else if len(data) > 0 { @@ -195,7 +183,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) } } - tx.Rollback() + btx.Rollback() } } @@ -217,20 +205,11 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t workerStart, workerEnd, workerID := start, end, w wg.Go(func() error { startTime := time.Now() - tx, err := db.BeginRo(ctx) + ttx, getter, err := warmView(ctx, tdb, sd) if err != nil { return err } - defer tx.Rollback() - - ttx, ok := tx.(kv.TemporalTx) - if !ok { - return nil - } - var getter kv.TemporalGetter = ttx - if bra.stateCache != nil { - getter = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} - } + defer ttx.Rollback() stateReader := state.NewReaderV3(getter) for idx := workerStart; idx < workerEnd; idx++ { @@ -287,22 +266,11 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t workerStart, workerEnd, workerID := start, end, w wg.Go(func() error { startTime := time.Now() - tx, err := db.BeginRo(ctx) + ttx, getter, err := warmView(ctx, tdb, sd) if err != nil { return err } - defer tx.Rollback() - - ttx, ok := tx.(kv.TemporalTx) - if !ok { - return nil - } - var getter kv.TemporalGetter = ttx - var cpg *cachePopulatingGetter - if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} - getter = cpg - } + defer ttx.Rollback() stateReader := state.NewReaderV3(getter) for txIdx := workerStart; txIdx < workerEnd; txIdx++ { diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go deleted file mode 100644 index f3a88e4d15c..00000000000 --- a/execution/exec/blocks_read_ahead_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package exec - -import ( - "testing" - - "github.com/c2h5oh/datasize" - "github.com/stretchr/testify/require" - - "github.com/erigontech/erigon/common/crypto" - "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/execution/cache" -) - -// stubTemporalGetter stands in for the committed-state snapshot a warmup -// goroutine reads: every GetLatest returns the same fixed value. -type stubTemporalGetter struct { - v []byte - step kv.Step -} - -func (s stubTemporalGetter) GetLatest(kv.Domain, []byte) ([]byte, kv.Step, error) { - return s.v, s.step, nil -} - -func (s stubTemporalGetter) HasPrefix(kv.Domain, []byte) ([]byte, []byte, bool, error) { - return nil, nil, false, nil -} - -func (s stubTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } - -func newTestStateCache() *cache.StateCache { - b := 1 * datasize.MB - return cache.NewStateCache(b, b, b, b) -} - -// A warmup read-through must never replace a fresher entry an authoritative -// writer (the FCU flush cache-apply) has already put: the warmup reads a -// pre-flush snapshot, so a laggard Put landing after the flush would pin stale -// state in the cache and corrupt the next block's execution. -func TestCachePopulatingGetterKeepsFresherEntry(t *testing.T) { - key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") - fresh := []byte("account-record-nonce-5") - stale := []byte("account-record-nonce-4") - for _, domain := range []kv.Domain{kv.AccountsDomain, kv.StorageDomain} { - sc := newTestStateCache() - sc.Put(domain, key, fresh, 54) - cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: stale}, sc: sc, stepSize: 1_562_500} - - v, _, err := cpg.GetLatest(domain, key) - require.NoError(t, err) - require.Equal(t, stale, v, "read-through must still return the snapshot value") - - got, ok := sc.Get(domain, key) - require.True(t, ok, "domain %s", domain) - require.Equal(t, fresh, got, "domain %s: warmup must not clobber the fresher entry", domain) - } -} - -// Same invariant for the code addr→code binding, which is rebound when an -// account's code changes and is therefore just as clobber-able as accounts. -func TestCachePopulatingGetterKeepsFresherCodeBinding(t *testing.T) { - addr := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") - freshCode := []byte{0xaa, 0x01, 0x02, 0x03} - staleCode := []byte{0xbb, 0x04, 0x05, 0x06} - sc := newTestStateCache() - sc.PutCodeWithHash(addr, freshCode, crypto.Keccak256(freshCode), 54) - cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: staleCode}, sc: sc, stepSize: 1_562_500} - - _, _, err := cpg.GetLatest(kv.CodeDomain, addr) - require.NoError(t, err) - - got, ok := sc.Get(kv.CodeDomain, addr) - require.True(t, ok) - require.Equal(t, freshCode, got, "warmup must not rebind addr to older code") -} - -// Cold keys must still be warmed — that is the prefetcher's purpose. -func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { - key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") - val := []byte("account-record") - code := []byte{0xaa, 0x01, 0x02, 0x03} - - for _, domain := range []kv.Domain{kv.AccountsDomain, kv.StorageDomain} { - sc := newTestStateCache() - cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: val}, sc: sc, stepSize: 1_562_500} - _, _, err := cpg.GetLatest(domain, key) - require.NoError(t, err) - got, ok := sc.Get(domain, key) - require.True(t, ok, "domain %s", domain) - require.Equal(t, val, got, "domain %s", domain) - } - - sc := newTestStateCache() - cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: code}, sc: sc, stepSize: 1_562_500} - _, _, err := cpg.GetLatest(kv.CodeDomain, key) - require.NoError(t, err) - got, ok := sc.Get(kv.CodeDomain, key) - require.True(t, ok) - require.Equal(t, code, got) - - // Negative results (missing account, empty slot) are cached as nil hits. - sc = newTestStateCache() - cpg = &cachePopulatingGetter{g: stubTemporalGetter{v: nil}, sc: sc, stepSize: 1_562_500} - _, _, err = cpg.GetLatest(kv.AccountsDomain, key) - require.NoError(t, err) - got, ok = sc.Get(kv.AccountsDomain, key) - require.True(t, ok) - require.Empty(t, got) -} diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 05b009d9432..b1d08869139 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -319,11 +319,16 @@ func NewExecModule( stopNode: stopNode, } - // Wire the process-global state cache into the read-ahead so its - // prefetches populate the same hashmap that SharedDomains.GetLatest - // probes on the EVM hot path. Reth's "same hashmap" pattern. + // Route the read-ahead's prefetches through the published SharedDomains so + // reads see in-flight tip state and the SD's own read-fill warms the + // process-global cache the EVM probes — keeping cache population an SD concern. if readAheader != nil { - readAheader.SetStateCache(domainCache) + readAheader.SetPublishedSD(func() *execctx.SharedDomains { + if em.publishedSD != nil { + return em.publishedSD() + } + return nil + }) } if stateCache != nil { From c6d6c06fbe51ea0a66b034cb00f2c1b45aaf6388 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Thu, 16 Jul 2026 11:42:53 +0000 Subject: [PATCH 152/167] execution/execmodule: drain commits in insertValidateAndUfc1By1 test helper The helper settled the last FCU's async commit with an extra idempotent FCU, assuming the next semaphore-acquiring op blocks until the prior commit lands. Under background commit that is racy: TestUpdateForkChoiceRecoversWhenStateAheadOfTxNums read committed state from the raw DB and intermittently saw block 9 instead of 10 under -race load. Wait for the commit backlog to drain before returning so callers read settled committed state deterministically. --- execution/execmodule/exec_module_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 9d4de623dbc..ebe59372d88 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -686,15 +686,15 @@ func insertValidateAndUfc1By1(ctx context.Context, exec *execmodule.ExecModule, return fmt.Errorf("unexpected updateForkChoice status: %s", ur.Status) } } - // UpdateForkChoice returns before the background flush+commit finishes - // (per #21444). The next semaphore-acquiring op blocks until the prior - // FCU's commit defers complete — so do one more idempotent FCU for the - // last block to ensure commitBlock has settled before the caller reads it. + // UpdateForkChoice returns before the background flush+commit finishes, so do + // one more idempotent FCU for the last block and then wait for the commit + // backlog to drain — committed state is settled before the caller reads it. if len(blocks) > 0 { if _, err := updateForkChoice(ctx, exec, blocks[len(blocks)-1].Header()); err != nil { return err } } + exec.WaitCommitsDrained() return nil } From 5c7a67f7bf18dee8a37857f65184e4055e4e498b Mon Sep 17 00:00:00 2001 From: mh0lt Date: Thu, 16 Jul 2026 12:26:27 +0000 Subject: [PATCH 153/167] execution/exec: read-ahead prefetch uses a plain RO snapshot, not a coordinated tx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prefetch is a fire-and-forget warmer — its reads are hints and the executor re-reads authoritatively — so it does not need commit-coordination. Opening a plain BeginTemporalRo instead of BeginCoordinatedRo keeps both invariants (reads via the published SD, cache populated by the SD read-fill) while removing the foreground-mutex traffic the coordinated open added on the tip path. --- execution/exec/blocks_read_ahead.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 6bdd6161a88..c1bd932348b 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -70,24 +70,21 @@ func (bra *BlockReadAheader) SetPublishedSD(provider func() *execctx.SharedDomai bra.publishedSD = provider } -// warmView opens a per-goroutine read view for prefetching. With a published SD -// it reads through it — in-flight tip state, and the SD's own read-fill warms the -// process-global cache (cache population stays an SD concern). Without one it -// reads the raw tx and does not populate the cache. A throwaway per-worker -// metrics accumulator keeps concurrent workers off the SD's shared request -// metrics. +// warmView opens a per-goroutine read view for prefetching. A prefetch is a +// fire-and-forget warmer, so it takes a plain (uncoordinated) RO snapshot — no +// foreground-mutex traffic on the tip path. With a published SD it reads through +// it (in-flight tip state, and the SD's own read-fill warms the process-global +// cache); without one it reads the raw tx and populates nothing. A throwaway +// per-worker metrics accumulator keeps concurrent workers off the SD's shared +// request metrics. func warmView(ctx context.Context, tdb kv.TemporalRoDB, sd *execctx.SharedDomains) (kv.TemporalTx, kv.TemporalGetter, error) { - if sd != nil { - ttx, err := sd.BeginCoordinatedRo(ctx, tdb) //nolint:gocritic // tx is returned to the caller, which defers Rollback - if err != nil { - return nil, nil, err - } - return ttx, sd.AsGetterMetered(ttx, kvmetrics.NewDomainMetrics()), nil - } ttx, err := tdb.BeginTemporalRo(ctx) //nolint:gocritic // tx is returned to the caller, which defers Rollback if err != nil { return nil, nil, err } + if sd != nil { + return ttx, sd.AsGetterMetered(ttx, kvmetrics.NewDomainMetrics()), nil + } return ttx, ttx, nil } @@ -150,9 +147,9 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t return } // Capture the published tip leaf once; all workers share it (concurrent reads - // are safe on the stable published SD) and each opens its own coordinated tx. - // SetHead drains in-flight warmup before closing generations, so the captured - // leaf can't be closed out from under a worker. + // are safe on the stable published SD) and each opens its own read tx. SetHead + // drains in-flight warmup before closing generations, so the captured leaf + // can't be closed out from under a worker. var sd *execctx.SharedDomains if bra.publishedSD != nil { sd = bra.publishedSD() From 758a6d7e05fbaaab11d10de19de02798b03951a7 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 17 Jul 2026 17:24:05 +0000 Subject: [PATCH 154/167] execution/stagedsync, execution/execmodule: commitment calculator owns per-block changeset; fold tip commitment from BAL The commitment calculator is now the sole producer of each owned block's changeset. It buffers the per-tx write stream and, at settled compute time, reconstructs the account/storage/code diffs (prev values read as-of, matching exec's block-end flush), installs its own changeset for the commitment compute so branch deltas land in the commitment domain, and saves it. Exec no longer installs a shared changeset accumulator or records changeset diffs. With no shared accumulator, a fold can no longer race an exec-side install, so the ownsChangeset fold gate is removed: window and chain-tip blocks fold their commitment from the BAL. A folded owned block records its commitment branch deltas ahead of the tx-result stream; its account/storage/code diffs are filled at the block boundary once the buffered writes are complete, reusing the saved changeset so the branch deltas are preserved. Containing changeset production in the calculator unfuses exec from the changeset, making the parallel executor reusable for non-validation work such as block building. --- execution/execmodule/exec_module_test.go | 61 ++++++ execution/stagedsync/changeset_builder.go | 92 +++++++++ .../stagedsync/changeset_builder_test.go | 176 +++++++++++++++++ execution/stagedsync/changeset_reconstruct.go | 185 ++++++++++++++++++ execution/stagedsync/committer.go | 104 +++++++--- execution/stagedsync/exec3_parallel.go | 25 +-- 6 files changed, 597 insertions(+), 46 deletions(-) create mode 100644 execution/stagedsync/changeset_builder.go create mode 100644 execution/stagedsync/changeset_builder_test.go create mode 100644 execution/stagedsync/changeset_reconstruct.go diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index ebe59372d88..bf511e7b214 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -2699,6 +2699,67 @@ func runBALFoldAheadChangeset(t *testing.T, foldAhead, shadow bool) balFoldResul return res } +// TestBALFoldAheadFiresOnTipValidateChain pins the goal of the changeset-ownership +// refactor: a single chain-tip newPayload (ValidateChain) must fold its commitment +// from the BAL. It fails today because ownsChangeset forces the tip onto the +// incremental path; it passes once the changeset is produced result-locally by the +// commitment calculator and the ownsChangeset fold gate is removed. +func TestBALFoldAheadFiresOnTipValidateChain(t *testing.T) { + defer func(prev bool) { dbg.BALDrivenCommitment = prev }(dbg.BALDrivenCommitment) + defer func(prev bool) { dbg.IgnoreBAL = prev }(dbg.IgnoreBAL) + dbg.BALDrivenCommitment = true + dbg.IgnoreBAL = false + + const chainLen = 6 + ctx := t.Context() + privKey, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + require.NoError(t, err) + senderAddr := crypto.PubkeyToAddress(privKey.PublicKey) + m := execmoduletester.New(t, + execmoduletester.WithKey(privKey), + execmoduletester.WithGenesisSpec(&types.Genesis{ + Config: chain.AllProtocolChanges, + Alloc: types.GenesisAlloc{senderAddr: {Balance: big.NewInt(1 * common.Ether)}}, + }), + execmoduletester.WithExperimentalBAL(), + execmoduletester.WithAlwaysGenerateChangesets(false), + ) + canonical, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainLen, + func(i int, b *blockgen.BlockGen) { + tx, txErr := types.SignTx( + types.NewTransaction(uint64(i), senderAddr, uint256.NewInt(1_000), 50000, uint256.NewInt(10_000_000_000), nil), + *types.LatestSignerForChainID(nil), privKey, + ) + require.NoError(t, txErr) + b.AddTx(tx) + }, m.PublishedSD()) + require.NoError(t, err) + + // Commit all but the tip so the tip is a single-block newPayload whose parent is + // already committed — the fold baseline comes from the prior cycle. + for i := range chainLen - 1 { + _, err := insertBlocksWithBAL(ctx, m.ExecModule, canonical.Blocks[i:i+1], canonical.BlockAccessLists[i:i+1]) + require.NoError(t, err) + _, err = validateChain(ctx, m.ExecModule, canonical.Blocks[i].Header()) + require.NoError(t, err) + _, err = updateForkChoice(ctx, m.ExecModule, canonical.Blocks[i].Header()) + require.NoError(t, err) + } + m.ExecModule.WaitCommitsDrained() + + tip := canonical.TopBlock + _, err = insertBlocksWithBAL(ctx, m.ExecModule, []*types.Block{tip}, canonical.BlockAccessLists[chainLen-1:]) + require.NoError(t, err) + + stagedsync.ResetFoldsAheadForTest() + vr, err := validateChain(ctx, m.ExecModule, tip.Header()) + require.NoError(t, err) + require.Equal(t, execmodule.ExecutionStatusSuccess, vr.ValidationStatus) + + require.Positive(t, stagedsync.FoldsAheadPerformedForTest(), + "tip newPayload (ValidateChain) must fold its commitment from the BAL") +} + // A forkchoice head at height 0 that is not the genesis (e.g. a block carrying a // corrupted number) must be rejected, not treated as a genesis reset. func TestUpdateForkChoiceToNonGenesisBlockAtHeightZero(t *testing.T) { diff --git a/execution/stagedsync/changeset_builder.go b/execution/stagedsync/changeset_builder.go new file mode 100644 index 00000000000..d05aca51244 --- /dev/null +++ b/execution/stagedsync/changeset_builder.go @@ -0,0 +1,92 @@ +package stagedsync + +import ( + "bytes" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/changeset" +) + +// prevValueReader supplies the pre-block (first-touch) value of a domain key as +// raw stored bytes, or nil when absent. Only the first write to a key in a block +// consults it; later writes chain their prev in memory. +type prevValueReader interface { + prevValue(domain kv.Domain, key []byte, txNum uint64) ([]byte, error) +} + +// changesetBuilder reconstructs a block's per-domain changeset (the ChangeSets3 +// entries) from the per-tx write stream, mirroring the domain-write recording so +// the bytes are identical to exec's: prev values chain in memory (first touch +// read as-of pre-block via the reader), no-op puts are skipped, and entries are +// keyed by step = txNum/stepSize. It is a generic encoder — domain-specific +// choices (code-delete-when-absent, account self-destruct cascade) are applied by +// the caller before record; the builder just recreates the DomainUpdate stream. +type changesetBuilder struct { + reader prevValueReader + stepSize uint64 + cs *changeset.StateChangeSet + running [kv.DomainLen]map[string][]byte + firstErr error +} + +func newChangesetBuilder(reader prevValueReader, stepSize uint64) *changesetBuilder { + b := &changesetBuilder{ + reader: reader, + stepSize: stepSize, + cs: &changeset.StateChangeSet{}, + } + for i := range b.running { + b.running[i] = map[string][]byte{} + } + return b +} + +// record folds one domain write into the changeset. newVal == nil is a delete; +// a non-nil newVal is a put (skipped when it equals the current value, exactly as +// SharedDomains.DomainPut does). txNum is the write's tx position; the entry is +// keyed by step = txNum/stepSize so a block straddling a step edge yields one +// entry per step for the same key. +func (b *changesetBuilder) record(domain kv.Domain, key []byte, newVal []byte, txNum uint64) { + ks := string(key) + running := b.running[domain] + prev, seen := running[ks] + if !seen { + p, err := b.reader.prevValue(domain, key, txNum) + if err != nil { + if b.firstErr == nil { + b.firstErr = err + } + return + } + prev = p + } + + // Put whose value already matches the current value is a no-op: no history + // entry, and the running value is unchanged. + if newVal != nil && bytes.Equal(prev, newVal) { + running[ks] = prev + return + } + + // A code delete with no prior value is a no-op (nothing to restore): mirrors + // SharedDomains.DomainDel, which returns early for CodeDomain when prevVal is + // nil. Other domains record the delete even against an absent prev. + if newVal == nil && domain == kv.CodeDomain && len(prev) == 0 { + running[ks] = nil + return + } + + step := kv.Step(txNum / b.stepSize) + b.cs.Diffs[domain].DomainUpdate(key, step, prev) + + if newVal == nil { + running[ks] = nil + } else { + running[ks] = common.Copy(newVal) + } +} + +func (b *changesetBuilder) err() error { return b.firstErr } + +func (b *changesetBuilder) result() *changeset.StateChangeSet { return b.cs } diff --git a/execution/stagedsync/changeset_builder_test.go b/execution/stagedsync/changeset_builder_test.go new file mode 100644 index 00000000000..91e13da2a8b --- /dev/null +++ b/execution/stagedsync/changeset_builder_test.go @@ -0,0 +1,176 @@ +package stagedsync + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/db/kv" +) + +// fakePrevReader serves pre-block (first-touch) values for the builder, keyed by +// domain+key. Absent keys return nil (new key). It records how many times each +// key was read so a test can assert the builder only reads pre-block state once +// per key (subsequent prevs chain in-memory). +type fakePrevReader struct { + vals map[kv.Domain]map[string][]byte + reads map[kv.Domain]map[string]int +} + +func newFakePrevReader() *fakePrevReader { + return &fakePrevReader{ + vals: map[kv.Domain]map[string][]byte{}, + reads: map[kv.Domain]map[string]int{}, + } +} + +func (r *fakePrevReader) set(d kv.Domain, key string, v []byte) { + if r.vals[d] == nil { + r.vals[d] = map[string][]byte{} + } + r.vals[d][key] = v +} + +func (r *fakePrevReader) prevValue(d kv.Domain, key []byte, _ uint64) ([]byte, error) { + ks := string(key) + if r.reads[d] == nil { + r.reads[d] = map[string]int{} + } + r.reads[d][ks]++ + return r.vals[d][ks], nil +} + +// diffEntry is a decoded (key-without-step, step, prevValue) view of one +// DomainEntryDiff, for readable assertions independent of the ^step encoding. +type diffEntry struct { + key string + step kv.Step + prev string // "" means an explicit empty ([]byte{}) restore-marker (new key) +} + +func decodeDiffs(t *testing.T, diffs []kv.DomainEntryDiff) []diffEntry { + t.Helper() + out := make([]diffEntry, 0, len(diffs)) + for _, d := range diffs { + require.GreaterOrEqual(t, len(d.Key), 8, "diff key must carry an 8-byte step suffix") + raw := []byte(d.Key) + body := raw[:len(raw)-8] + var inv uint64 + for _, b := range raw[len(raw)-8:] { + inv = inv<<8 | uint64(b) + } + out = append(out, diffEntry{key: string(body), step: kv.Step(^inv), prev: string(d.Value)}) + } + return out +} + +const testStepSize = 4 + +func TestChangesetBuilder_NewKeyRecordsEmptyPrev(t *testing.T) { + r := newFakePrevReader() + b := newChangesetBuilder(r, testStepSize) + + b.record(kv.AccountsDomain, []byte("acct"), []byte("v1"), 0) + require.NoError(t, b.err()) + + got := decodeDiffs(t, b.result().Diffs[kv.AccountsDomain].GetDiffSet()) + require.Equal(t, []diffEntry{{key: "acct", step: 0, prev: ""}}, got) +} + +func TestChangesetBuilder_FirstTouchKeepsPreBlockPrev(t *testing.T) { + r := newFakePrevReader() + r.set(kv.AccountsDomain, "acct", []byte("pre")) + b := newChangesetBuilder(r, testStepSize) + + b.record(kv.AccountsDomain, []byte("acct"), []byte("v1"), 0) + got := decodeDiffs(t, b.result().Diffs[kv.AccountsDomain].GetDiffSet()) + require.Equal(t, []diffEntry{{key: "acct", step: 0, prev: "pre"}}, got) +} + +func TestChangesetBuilder_OverwriteSameStepKeepsFirstPrev(t *testing.T) { + r := newFakePrevReader() + r.set(kv.AccountsDomain, "acct", []byte("pre")) + b := newChangesetBuilder(r, testStepSize) + + // Two writes in the same step: only the first-per-step entry survives, and + // it carries the pre-block prev. + b.record(kv.AccountsDomain, []byte("acct"), []byte("v1"), 0) + b.record(kv.AccountsDomain, []byte("acct"), []byte("v2"), 1) + got := decodeDiffs(t, b.result().Diffs[kv.AccountsDomain].GetDiffSet()) + require.Equal(t, []diffEntry{{key: "acct", step: 0, prev: "pre"}}, got) + require.Equal(t, 1, r.reads[kv.AccountsDomain]["acct"], "pre-block read must happen once; later prevs chain in memory") +} + +func TestChangesetBuilder_ABAStillRecordsEntry(t *testing.T) { + r := newFakePrevReader() + r.set(kv.AccountsDomain, "acct", []byte("A")) + b := newChangesetBuilder(r, testStepSize) + + // A -> B -> A within one step: exec's replay records the first non-noop + // (prev=A) and dedups the rest; the net-zero round-trip still yields an + // entry restoring A. + b.record(kv.AccountsDomain, []byte("acct"), []byte("B"), 0) + b.record(kv.AccountsDomain, []byte("acct"), []byte("A"), 1) + got := decodeDiffs(t, b.result().Diffs[kv.AccountsDomain].GetDiffSet()) + require.Equal(t, []diffEntry{{key: "acct", step: 0, prev: "A"}}, got) +} + +func TestChangesetBuilder_NoOpWriteRecordsNothing(t *testing.T) { + r := newFakePrevReader() + r.set(kv.StorageDomain, "slot", []byte("X")) + b := newChangesetBuilder(r, testStepSize) + + // Writing the same value the slot already holds is a no-op (mirrors + // DomainPut's bytes.Equal(prev,v) skip): no diff entry. + b.record(kv.StorageDomain, []byte("slot"), []byte("X"), 0) + require.Empty(t, b.result().Diffs[kv.StorageDomain].GetDiffSet()) +} + +func TestChangesetBuilder_StraddleRecordsPerStepEntries(t *testing.T) { + r := newFakePrevReader() + r.set(kv.AccountsDomain, "acct", []byte("A")) + b := newChangesetBuilder(r, testStepSize) + + // Block straddles a step edge (stepSize=4): txNum 3 is step 0, txNum 4 is + // step 1. A key written in both steps yields two entries — step 0 with the + // pre-block prev, step 1 with the intermediate value written in step 0. + b.record(kv.AccountsDomain, []byte("acct"), []byte("B"), 3) + b.record(kv.AccountsDomain, []byte("acct"), []byte("C"), 4) + got := decodeDiffs(t, b.result().Diffs[kv.AccountsDomain].GetDiffSet()) + require.ElementsMatch(t, []diffEntry{ + {key: "acct", step: 0, prev: "A"}, + {key: "acct", step: 1, prev: "B"}, + }, got) +} + +func TestChangesetBuilder_CodeDeleteWhenAbsentRecordsNothing(t *testing.T) { + r := newFakePrevReader() // no prior code for the key + b := newChangesetBuilder(r, testStepSize) + + // A code delete with no prior value is a no-op — the system-address code + // "write" of empty bytes during an EIP-4788/2935 system call. Mirrors + // SharedDomains.DomainDel's CodeDomain skip. + b.record(kv.CodeDomain, []byte("sysaddr"), nil, 0) + require.Empty(t, b.result().Diffs[kv.CodeDomain].GetDiffSet()) +} + +func TestChangesetBuilder_StorageDeleteWhenAbsentStillRecords(t *testing.T) { + r := newFakePrevReader() // no prior slot value + b := newChangesetBuilder(r, testStepSize) + + // Storage deletes are NOT skipped when absent (only CodeDomain is). + b.record(kv.StorageDomain, []byte("slot"), nil, 0) + got := decodeDiffs(t, b.result().Diffs[kv.StorageDomain].GetDiffSet()) + require.Equal(t, []diffEntry{{key: "slot", step: 0, prev: ""}}, got) +} + +func TestChangesetBuilder_DeleteRecordsPrev(t *testing.T) { + r := newFakePrevReader() + r.set(kv.StorageDomain, "slot", []byte("V")) + b := newChangesetBuilder(r, testStepSize) + + // A delete (empty new value) restores the prior value on unwind. + b.record(kv.StorageDomain, []byte("slot"), nil, 0) + got := decodeDiffs(t, b.result().Diffs[kv.StorageDomain].GetDiffSet()) + require.Equal(t, []diffEntry{{key: "slot", step: 0, prev: "V"}}, got) +} diff --git a/execution/stagedsync/changeset_reconstruct.go b/execution/stagedsync/changeset_reconstruct.go new file mode 100644 index 00000000000..38897448d33 --- /dev/null +++ b/execution/stagedsync/changeset_reconstruct.go @@ -0,0 +1,185 @@ +package stagedsync + +import ( + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/empty" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/changeset" + "github.com/erigontech/erigon/db/state/execctx" + "github.com/erigontech/erigon/execution/state" + "github.com/erigontech/erigon/execution/types/accounts" +) + +// csWrite is one buffered per-tx domain write (new value; nil = delete) captured +// at feed time; prevs are resolved later, at settled compute time. +type csWrite struct { + domain kv.Domain + key []byte + val []byte + txNum uint64 +} + +// bufferChangesetWrites buffers an owned block's per-tx writes (new values captured +// now, from the post-tx state) so the changeset can be replayed and its prevs +// resolved at settled compute time — matching exec's block-end flush timing rather +// than the racy per-tx moment. Non-owned blocks need no changeset. +func (cc *commitmentCalculator) bufferChangesetWrites(r *txResult) { + if !cc.ownsChangeset(r.blockNum) { + return + } + if cc.csBuilderBlock != r.blockNum { + cc.csPending = cc.csPending[:0] + cc.csBuilderBlock = r.blockNum + } + cc.feedChangeset(r.writes, r.txNum) +} + +// asOfPrevReader supplies the pre-block value of an account/storage/code key as +// raw stored bytes via GetAsOf(txNum) — the same bytes exec's DomainPut resolves +// as prevVal through GetLatest at flush time. Only the first write to a key in a +// block consults it; the builder chains later prevs in memory. +type asOfPrevReader struct { + sd *execctx.SharedDomains + roTx kv.TemporalTx +} + +func (r *asOfPrevReader) prevValue(domain kv.Domain, key []byte, txNum uint64) ([]byte, error) { + enc, ok, err := r.sd.GetAsOf(domain, key, txNum) + if err != nil { + return nil, err + } + if ok { + return enc, nil + } + enc, ok, err = r.roTx.GetAsOf(domain, key, txNum) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + return enc, nil +} + +func (cc *commitmentCalculator) appendCS(domain kv.Domain, key, val []byte, txNum uint64) { + w := csWrite{domain: domain, key: common.Copy(key), txNum: txNum} + if val != nil { + w.val = common.Copy(val) + } + cc.csPending = append(cc.csPending, w) +} + +// feedChangeset buffers one tx's account/storage/code writes, mirroring exec's +// per-tx BlockStateCache.Flush op emission: the account is serialized once per tx +// from the calc's post-tx state (a Deleted+all-zero account is a delete), +// storage/code puts carry their new bytes (empty → delete), and a self-destruct +// cascades a storage-subtree wipe. Prevs are resolved later (at settled compute +// time) so the replayed ChangeSets3 bytes match exec's. +func (cc *commitmentCalculator) feedChangeset(writes *state.WriteSet, txNum uint64) { + touched := map[accounts.Address]struct{}{} + for addr := range writes.Balances() { + touched[addr] = struct{}{} + } + for addr := range writes.Nonces() { + touched[addr] = struct{}{} + } + for addr := range writes.CodeHashes() { + touched[addr] = struct{}{} + } + for addr := range writes.Codes() { + touched[addr] = struct{}{} + } + for addr := range writes.Incarnations() { + touched[addr] = struct{}{} + } + for addr, vw := range writes.SelfDestructs() { + if vw.Val { + touched[addr] = struct{}{} + } + } + + for addr := range touched { + addrVal := addr.Value() + acc := cc.state.accounts[addr] + if acc == nil { + continue + } + if acc.Deleted && acc.Balance.IsZero() && acc.Nonce == 0 && acc.CodeHash == empty.CodeHash { + cc.appendCS(kv.AccountsDomain, addrVal[:], nil, txNum) + } else { + a := accounts.NewAccount() + a.Balance = acc.Balance + a.Nonce = acc.Nonce + a.Incarnation = acc.Incarnation + a.CodeHash = accounts.InternCodeHash(common.Hash(acc.CodeHash)) + cc.appendCS(kv.AccountsDomain, addrVal[:], accounts.SerialiseV3(&a), txNum) + } + } + + for addr, vw := range writes.Codes() { + addrVal := addr.Value() + if vw.Val.Len() == 0 { + cc.appendCS(kv.CodeDomain, addrVal[:], nil, txNum) + } else { + cc.appendCS(kv.CodeDomain, addrVal[:], vw.Val.Bytes, txNum) + } + } + + for addr, inner := range writes.Storages() { + addrVal := addr.Value() + for key, vw := range inner { + keyVal := key.Value() + composite := make([]byte, 20+32) + copy(composite, addrVal[:]) + copy(composite[20:], keyVal[:]) + vb := vw.Val.Bytes() + if len(vb) == 0 { + cc.appendCS(kv.StorageDomain, composite, nil, txNum) + } else { + cc.appendCS(kv.StorageDomain, composite, vb, txNum) + } + } + } + + // Self-destruct wipes the whole storage subtree (exec's DomainDelPrefix) and + // deletes the code — reproduce both as deletes over the as-of subtree. + for addr, vw := range writes.SelfDestructs() { + if !vw.Val { + continue + } + addrVal := addr.Value() + cc.appendCS(kv.CodeDomain, addrVal[:], nil, txNum) + if cc.state.storageEnum == nil { + continue + } + _ = cc.state.storageEnum.EachStorageSlot(addr, func(key accounts.StorageKey) error { + keyVal := key.Value() + composite := make([]byte, 20+32) + copy(composite, addrVal[:]) + copy(composite[20:], keyVal[:]) + cc.appendCS(kv.StorageDomain, composite, nil, txNum) + return nil + }) + } +} + +// buildResultLocalChangeset replays the buffered per-tx writes through a fresh +// builder AT COMPUTE TIME — resolving prevs when block N-1 is settled in sd.mem +// (matching exec's block-end flush) — and returns the account/storage/code diffs +// (domains 0..2) as a StateChangeSet. The commitment domain (3) is left empty for +// the caller's compute to fill. Returns nil if no writes were buffered for the +// block or a prev-read failed. +func (cc *commitmentCalculator) buildResultLocalChangeset(blockNum uint64) *changeset.StateChangeSet { + if cc.csBuilderBlock != blockNum { + return nil + } + b := newChangesetBuilder(cc.prevReader, cc.doms.StepSize()) + for _, w := range cc.csPending { + b.record(w.domain, w.key, w.val, w.txNum) + } + if err := b.err(); err != nil { + cc.logger.Warn("["+cc.logPrefix+"] changeset reconstruct: reader error", "block", blockNum, "err", err) + return nil + } + return b.result() +} diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 6bc27c36f4b..438837042a5 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -15,6 +15,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/order" + "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/commitment" @@ -89,6 +90,15 @@ type commitmentCalculator struct { // Values are lazy-loaded from the domain on first touch via asOfReader. state *calcState + // csPending buffers the current owned block's per-tx writes (new values); at + // settled compute time they are replayed to reconstruct the account/storage/code + // changeset result-locally and its prevs resolved. csBuilderBlock is the block + // being buffered. Only active under ASSERT_CHANGESET_RECONSTRUCT (build-alongside + // byte comparison against exec's shared accumulator). + csPending []csWrite + csBuilderBlock uint64 + prevReader *asOfPrevReader + // asOfReader is shared between calcState (lazy-load) and compute // methods (fold/unfold sibling reads). Its txNum is updated at each // block boundary. @@ -230,6 +240,7 @@ func newCommitmentCalculator( asOfReader := &asOfStateReader{sd: doms, roTx: roTx, txNum: 0} return &commitmentCalculator{ + prevReader: &asOfPrevReader{sd: doms, roTx: roTx}, doms: doms, db: db, chainConfig: chainConfig, @@ -337,6 +348,7 @@ func (cc *commitmentCalculator) handleMessage(ctx context.Context, msg applyResu if !r.writes.IsEmpty() { cc.asOfReader.txNum = r.txNum cc.state.ApplyWrites(r.writes, r.rules.IsAmsterdam) + cc.bufferChangesetWrites(r) } // A folded-ahead block already emitted its interior step checkpoints from @@ -397,6 +409,12 @@ func (cc *commitmentCalculator) handleMessage(ctx context.Context, msg applyResu cc.shadowCrossCheck(ctx, r) } else { cc.state.ResetBlockFlags() + // A folded owned block recorded its root + commitment branch deltas + // ahead of the tx-result stream; fill its account/storage/code diffs + // now that the block's writes are buffered. + if cc.ownsChangeset(r.BlockNum) { + cc.finalizeFoldedChangeset(r.BlockNum, r.BlockHash) + } } case cc.perBlockCompute(r.BlockNum): if cc.lastComputedBlock == 0 && r.isPartial { @@ -485,16 +503,17 @@ func (cc *commitmentCalculator) foldGateOpen(n uint64) bool { // gate is open, and folding it is safe. Folding overlaps the execution of // block n — the parallel-commitment win. Idempotent (foldedAhead guard). // -// Two safety restrictions, both preserving the well-tested incremental path: -// - Only pre-window blocks (!ownsChangeset) fold. A block that owns a -// changeset computes incrementally at its own boundary so its per-block -// branch deltas are captured; folding it ahead would race the exec loop's -// changeset-accumulator install. Pre-window folds run under computeIsolated -// (nil accumulator), so nothing leaks into a later window block's changeset. -// - Fold only contiguously from the batch's first block: n's baseline is read -// from the commitment domain, which only a prior fold (or the prior cycle, -// for the first block) advanced. A missing-BAL block accumulates without -// advancing the domain, so folding across it would read a stale trie. +// Safety restriction preserving the well-tested path: fold only contiguously from +// the batch's first block. n's baseline is read from the commitment domain, which +// only a prior fold (or the prior cycle, for the first block) advanced. A +// missing-BAL block accumulates without advancing the domain, so folding across it +// would read a stale trie. +// +// Owned (window/tip) blocks fold too: the calculator is the sole changeset +// producer (result-local, no shared accumulator), so a fold no longer races an +// exec-side accumulator install. A folded owned block records its commitment +// branch deltas at fold time; its account/storage/code diffs are filled at the +// block boundary by finalizeFoldedChangeset once the tx-result stream is complete. func (cc *commitmentCalculator) maybeFoldAhead(ctx context.Context, n uint64) { pb, ok := cc.pending[n] if !ok || pb.mode != calcModeBALDriven || cc.foldedAhead[n] { @@ -507,9 +526,6 @@ func (cc *commitmentCalculator) maybeFoldAhead(ctx context.Context, n uint64) { if sc, stopping := stopCauseOf(cc.signalCtx); stopping && n > sc.block { return } - if cc.ownsChangeset(n) { - return - } if !cc.foldGateOpen(n) { return } @@ -895,24 +911,60 @@ func (cc *commitmentCalculator) computeWithBlockAccumulator(ctx context.Context, // would route this block's [state] write into the next block's changeset. The // lock is required even on the cs==nil path — the internal FlushPendingUpdates // mutates the same global accumulator pointer. + // Calc-owned changeset: the calculator is the sole producer of an owned block's + // changeset. It reuses the block's saved changeset across the block's multiple + // computes — a fold records the commitment branch deltas (Diffs[3]) ahead of the + // tx-result stream, then step-edge checkpoints and the block-end compute add more + // — so those deltas accumulate rather than being lost. The account/storage/code + // diffs (0..2) are filled from the buffered per-tx writes once they are ready + // (finalized at the block boundary via finalizeFoldedChangeset for a folded + // block); a not-yet-ready csPending just leaves them for that later fill. + // + // The swap is LOAD-BEARING under changesetMu (held above): it mutates the global + // current-accumulator pointer that the deferred branch writes (flushed inside + // ComputeCommitmentLocked → FlushPendingUpdatesLocked) and the end-of-compute + // [state] marker also touch, serializing against the apply goroutine. Inside the + // lock the *Locked variants are required — the public ones re-acquire the mutex. cc.doms.LockChangesetAccumulator() defer cc.doms.UnlockChangesetAccumulator() cs := cc.doms.GetChangesetByHash(t.blockNum, t.blockHash) if cs == nil { - return cc.doms.ComputeCommitmentLocked(ctx, cc.roTx, true, t.blockNum, t.lastTxNum, cc.logPrefix, nil) - } - // LOAD-BEARING swap under the outer lock (already taken above). The - // swap below mutates the global current-accumulator pointer; the - // deferred branch writes from block N-1 (flushed inside - // ComputeCommitmentLocked → FlushPendingUpdatesLocked) AND the [state] - // marker write at end of compute also touch that same global pointer - // and the per-domain diff fields. Holding changesetMu through all of - // it serializes against the apply goroutine's DomainPut/DomainDel. - // - // Inside the lock we must use the *Locked variants — the public - // counterparts re-acquire the same Mutex and would self-deadlock. + cs = &changeset.StateChangeSet{} + } + if localAcc := cc.buildResultLocalChangeset(t.blockNum); localAcc != nil { + cs.Diffs[kv.AccountsDomain] = localAcc.Diffs[kv.AccountsDomain] + cs.Diffs[kv.StorageDomain] = localAcc.Diffs[kv.StorageDomain] + cs.Diffs[kv.CodeDomain] = localAcc.Diffs[kv.CodeDomain] + } defer cc.doms.SwapChangesetAccumulatorLocked(cs)() - return cc.doms.ComputeCommitmentLocked(ctx, cc.roTx, true, t.blockNum, t.lastTxNum, cc.logPrefix, nil) + rh, err := cc.doms.ComputeCommitmentLocked(ctx, cc.roTx, true, t.blockNum, t.lastTxNum, cc.logPrefix, nil) + if err != nil { + return nil, err + } + cc.doms.SavePastChangesetAccumulator(t.blockHash, t.blockNum, cs) + return rh, nil +} + +// finalizeFoldedChangeset fills a folded owned block's account/storage/code diffs +// (0..2) into its saved changeset at the block boundary — when the buffered per-tx +// writes are complete — preserving the commitment branch deltas (Diffs[3]) the fold +// already recorded. A fold computes the root (and Diffs[3]) ahead of the tx-result +// stream, so 0..2 cannot be built at fold time; this closes that gap. +func (cc *commitmentCalculator) finalizeFoldedChangeset(blockNum uint64, blockHash common.Hash) { + localAcc := cc.buildResultLocalChangeset(blockNum) + if localAcc == nil { + return + } + cc.doms.LockChangesetAccumulator() + defer cc.doms.UnlockChangesetAccumulator() + cs := cc.doms.GetChangesetByHash(blockNum, blockHash) + if cs == nil { + cs = &changeset.StateChangeSet{} + } + cs.Diffs[kv.AccountsDomain] = localAcc.Diffs[kv.AccountsDomain] + cs.Diffs[kv.StorageDomain] = localAcc.Diffs[kv.StorageDomain] + cs.Diffs[kv.CodeDomain] = localAcc.Diffs[kv.CodeDomain] + cc.doms.SavePastChangesetAccumulator(blockHash, blockNum, cs) } // asOfStateReader reads account/storage/code at a specific txNum via diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 20b96a8f803..ba586a298d1 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -185,26 +185,11 @@ func stopCauseOf(ctx context.Context) (*stopCause, bool) { return nil, false } -// ensureChangesetAccumulator makes pe.currentChangeSet point at a fresh, -// block-specific StateChangeSet before any of blockNum's sd.mem writes are -// applied. Idempotent. Exec-loop only — it mutates SharedDomains.mem via -// SetChangesetAccumulator, which must be single-writer (see the comment on -// currentChangeSet). -func (pe *parallelExecutor) ensureChangesetAccumulator(blockNum uint64) { - if blockNum < pe.changesetWindowStart || blockNum == 0 || blockNum > pe.maxBlockNum { - return - } - if pe.currentChangeSet != nil && pe.currentChangeSetBlock == blockNum { - return - } - // A previous block's accumulator is normally saved+cleared at its - // blockResult; if one is still installed here for a different block the - // rotation was missed — overwrite (the previous block's changeset was - // already saved at its blockResult, so nothing is lost). - pe.currentChangeSet = &changeset.StateChangeSet{} - pe.currentChangeSetBlock = blockNum - pe.domains().SetChangesetAccumulator(pe.currentChangeSet) -} +// ensureChangesetAccumulator is a no-op: the commitment calculator is the sole +// producer of each owned block's changeset — it reconstructs account/storage/code +// diffs result-locally, installs its own changeset for the commitment compute, and +// saves it. Exec installs no shared accumulator and records no changeset diffs. +func (pe *parallelExecutor) ensureChangesetAccumulator(blockNum uint64) {} // clearChangesetAccumulator detaches the current changeset accumulator after // its block's changeset has been saved. Exec-loop only. From 2e1c28aff2a673a079ab55f541932536bf7450fa Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 17 Jul 2026 17:30:20 +0000 Subject: [PATCH 155/167] execution/stagedsync: remove dead exec-side changeset accumulator machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commitment calculator now owns changeset production, so the exec loop's per-block accumulator install/save/clear is dead. Remove the currentChangeSet / currentChangeSetBlock fields, the ensureChangesetAccumulator / clearChangesetAccumulator functions, and their call sites. changesetWindowStart is kept — it still bounds the calculator's per-block compute + fold window. --- execution/stagedsync/exec3_parallel.go | 87 ++------------------------ 1 file changed, 6 insertions(+), 81 deletions(-) diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index ba586a298d1..193baaa344c 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -25,7 +25,6 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb/rawtemporaldb" dbstate "github.com/erigontech/erigon/db/state" - "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/commitment" @@ -113,25 +112,11 @@ type parallelExecutor struct { // accumulator for txpool state-diff notifications; set before execLoop // starts so that AuRa system-call nonce changes are emitted per block. accumulator *shards.Accumulator - // changesetAccumulator state owned by the exec loop. Accessing or mutating - // this is the exec loop's responsibility — putting it here (rather than on - // the apply-loop side) ensures all sd.mem mutations originate from a single - // goroutine and avoids the data race between SetChangesetAccumulator - // (apply loop) and ApplyStateWrites (exec loop, via SysCallContract for - // block-end system calls) on SharedDomains.mem. - // changesetWindowStart is the first block of the batch that must capture - // a changeset (see changesetWindowStart in exec3.go); blocks below it run - // without an accumulator. + // changesetWindowStart is the first block of the batch that owns a changeset + // (see changesetWindowStart in exec3.go); it bounds the calculator's per-block + // compute + fold window. Changeset production itself lives in the commitment + // calculator, which reconstructs and saves each owned block's changeset. changesetWindowStart uint64 - currentChangeSet *changeset.StateChangeSet - // currentChangeSetBlock is the block number currentChangeSet belongs to - // (0 == none). Tracked so ensureChangesetAccumulator can be a no-op when the - // accumulator is already installed for the block whose writes are about to - // be applied — making changeset capture robust against blocks scheduled out - // of band (e.g. processRequest scheduling the first block of a new request - // after the blockExecutors map went empty mid-batch, with no preceding - // blockResult to trigger the install at the rotation site below). - currentChangeSetBlock uint64 } // stopKind classifies why the executor was asked to stop. It maps directly @@ -185,20 +170,6 @@ func stopCauseOf(ctx context.Context) (*stopCause, bool) { return nil, false } -// ensureChangesetAccumulator is a no-op: the commitment calculator is the sole -// producer of each owned block's changeset — it reconstructs account/storage/code -// diffs result-locally, installs its own changeset for the commitment compute, and -// saves it. Exec installs no shared accumulator and records no changeset diffs. -func (pe *parallelExecutor) ensureChangesetAccumulator(blockNum uint64) {} - -// clearChangesetAccumulator detaches the current changeset accumulator after -// its block's changeset has been saved. Exec-loop only. -func (pe *parallelExecutor) clearChangesetAccumulator() { - pe.domains().SetChangesetAccumulator(nil) - pe.currentChangeSet = nil - pe.currentChangeSetBlock = 0 -} - func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u Unwinder, startBlockNum uint64, offsetFromBlockBeginning uint64, maxBlockNum uint64, blockLimit uint64, initialTxNum uint64, inputTxNum uint64, initialCycle bool, rwTx kv.TemporalRwTx, @@ -307,14 +278,10 @@ func (pe *parallelExecutor) execImpl(ctx context.Context, execStage *StageState, pe.commitResultsCh = commitResults pe.maxBlockNum = maxBlockNum - // Configure changeset capture and seed the initial accumulator BEFORE - // the exec loop / executeBlocks goroutines start touching sd.mem. The - // exec loop owns all subsequent SetChangesetAccumulator transitions - // (per-block save/clear/install) so apply-loop and exec-loop sd.mem - // writes never race on SharedDomains.mem. + // The changeset window bounds which blocks the commitment calculator computes + // per-block and folds; the calculator produces and saves their changesets. pe.changesetWindowStart = changesetWindowStart(pe.cfg.syncCfg.AlwaysGenerateChangesets, pe.cfg.syncCfg.MaxReorgDepth, pe.cfg.blockReader.FrozenBlocks(), startBlockNum, maxBlockNum) - pe.ensureChangesetAccumulator(startBlockNum) // Start the commitment calculator. It mirrors serial's per-block gate // (exec3_serial.go: `if !dbg.BatchCommitments || shouldGenerateChangesets @@ -371,11 +338,6 @@ func (pe *parallelExecutor) execImpl(ctx context.Context, execStage *StageState, } defer applyRoTx.Rollback() - // pe.changesetWindowStart and pe.currentChangeSet were set up - // before pe.run/executeBlocks launched their goroutines (above the - // calculator.Start call). Per-block accumulator save/clear/install - // transitions are driven from the exec loop's blockResult handler. - // appliedBlocks tracks blockNums that completed full apply-loop // processing (including post-block validation). Used at exit to // detect "the channel closed cleanly but a block was silently @@ -1131,32 +1093,6 @@ func (pe *parallelExecutor) execLoop(ctx context.Context) (err error) { pe.blockExecMetrics.Duration.Add(time.Since(blockExecutor.execStarted)) pe.blockExecMetrics.BlockCount.Add(1) } - // Snapshot the just-completed block's changeset BEFORE sending the - // blockResult, so that the commitment calculator (which consumes - // blockResults on a separate goroutine) can find this block's - // saved changeset via GetChangesetByBlockNum at compute time. - // In per-block compute mode (changeset window), the - // calculator switches the accumulator to this saved CS for the - // duration of ComputeCommitment (committer.go:computeWithBlockAccumulator) - // so branch writes land in block N's CS rather than whatever the - // exec loop has installed as current. If we saved AFTER sendResult, - // the calculator could race ahead and look up an unsaved CS, - // causing branch deltas to leak into the next block's CS and - // produce wrong-trie-root chains on subsequent reorg-driven - // re-execution (see TestRecreateAndRewind reproducer). Clearing - // the live accumulator and the local pointer must still happen - // here (in the exec loop) so the rotation-to-next-block install - // at line 893-895 is serialized with the exec loop's other - // sd.mem writes (system calls, finalize, ApplyStateWrites for - // the next block). - // Belt-and-braces: an empty block (no tx-results reaching - // processResults) may not have triggered the install — create - // its (empty) accumulator so it gets saved like every other block. - pe.ensureChangesetAccumulator(blockResult.BlockNum) - if pe.currentChangeSet != nil { - pe.domains().SavePastChangesetAccumulator(blockResult.BlockHash, blockResult.BlockNum, pe.currentChangeSet) - } - // Decide the stop BEFORE sending. A terminal stop publishes the // stopCause on the shared context before blockResult(M) crosses the // channel, so the calculator holds the coalesce block M by the time @@ -1203,7 +1139,6 @@ func (pe *parallelExecutor) execLoop(ctx context.Context) (err error) { if err := blockExecutor.sendResult(ctx, blockResult, terminal); err != nil { return err } - pe.clearChangesetAccumulator() // Block-validity rejection: the apply loop consumes blockResult and // returns its Err; the calculator skips the commitment compute. Exit @@ -1239,11 +1174,6 @@ func (pe *parallelExecutor) execLoop(ctx context.Context) (err error) { pe.RUnlock() if ok { - // Fast-path install of the next block's changeset accumulator, - // still in the exec loop (single-writer). If the next block's - // executor isn't in the map yet this is a no-op; processResults - // then installs it lazily on the block's first apply. - pe.ensureChangesetAccumulator(blockExecutor.blockNum) pe.onBlockStart(ctx, blockExecutor.blockNum, blockExecutor.blockHash) blockExecutor.execStarted = time.Now() blockExecutor.scheduleExecution(ctx, pe) @@ -1612,11 +1542,6 @@ func (pe *parallelExecutor) processResults(ctx context.Context, applyTx kv.Tempo return nil, fmt.Errorf("unknown block: %d", txResult.Version().BlockNum) } - // Ensure this block's changeset accumulator is installed before its - // writes are applied — covers blocks scheduled out of band (with no - // preceding blockResult to trigger the fast-path install above). - pe.ensureChangesetAccumulator(txResult.Version().BlockNum) - blockResult, err = blockExecutor.nextResult(ctx, pe, txResult, applyTx) if err != nil { From 046b3ecaa2bee1db9a349e88c2daf3ce8be8e301 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 17 Jul 2026 17:46:16 +0000 Subject: [PATCH 156/167] execution/stagedsync: trim stale/oversized changeset comments (review) Address code-review comment-policy findings: drop the stale csPending reference to the removed ASSERT_CHANGESET_RECONSTRUCT flag; fix the apply-loop comment that described exec-driven accumulator rotation (now calculator-owned); condense the computeWithBlockAccumulator docstring and in-body comment, removing incident/reproducer narration and the now-false "exec loop has installed" phrasing. No behavior change. --- execution/stagedsync/committer.go | 64 ++++++++------------------ execution/stagedsync/exec3_parallel.go | 7 +-- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 438837042a5..23dcca78352 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -92,9 +92,8 @@ type commitmentCalculator struct { // csPending buffers the current owned block's per-tx writes (new values); at // settled compute time they are replayed to reconstruct the account/storage/code - // changeset result-locally and its prevs resolved. csBuilderBlock is the block - // being buffered. Only active under ASSERT_CHANGESET_RECONSTRUCT (build-alongside - // byte comparison against exec's shared accumulator). + // changeset result-locally with prevs resolved. csBuilderBlock is the block + // being buffered. csPending []csWrite csBuilderBlock uint64 prevReader *asOfPrevReader @@ -871,30 +870,13 @@ func (cc *commitmentCalculator) publish(ctx context.Context, r commitmentResult) } } -// computeWithBlockAccumulator runs ComputeCommitment with the changeset -// accumulator switched to block N's saved changeset (looked up by hash) so -// that any branch writes during compute (mid-process inline flushes from -// `pendingPrefixes` collisions, plus the [state] write at end via -// encodeAndStoreCommitmentState) land in block N's CS rather than whatever -// the exec loop has installed as current. -// -// IMPORTANT: hash-aware lookup is mandatory here. pastChangesAccumulator -// can hold multiple changesets per block number after a fork-bounce -// (canonical block 1 + forks[i] block 1 with different hashes), and a -// number-only GetChangesetByBlockNum returns the first match in -// non-deterministic map iteration order. That non-determinism caused the -// calculator's [state] write for canonical block 1 to land in the fork's -// block 1 CS during the TestBlockchainHeaderchainReorgConsistency -// reproducer, leaving canonical block 1's CS without [state] and producing -// off-by-one wrong-trie-root chains on the next iteration's re-execution. -// -// If block N's CS hasn't been saved yet it falls through to the live -// accumulator, which — because the lookup is under changesetMu — is still N's -// own (the apply loop can't rotate it while the lock is held). -// -// Also annotates the pending deferred update (set inside ComputeCommitment -// when defer mode is on) with the block's hash, so the next call's -// FlushPendingUpdates uses the same hash-aware routing. +// computeWithBlockAccumulator computes an owned block's commitment against its +// own result-local changeset. The block's changeset must be looked up by hash, +// not number: after a fork-bounce pastChangesAccumulator holds multiple +// changesets per block number, and a number-only lookup could route this block's +// branch writes into a sibling fork's changeset — producing wrong-trie-root +// chains on the next re-execution. It also stamps the pending deferred update +// with the block hash so the next call's FlushPendingUpdates routes the same way. func (cc *commitmentCalculator) computeWithBlockAccumulator(ctx context.Context, t commitTarget) ([]byte, error) { defer func() { // Stamp the pending update (if any was set during ComputeCommitment) @@ -906,25 +888,15 @@ func (cc *commitmentCalculator) computeWithBlockAccumulator(ctx context.Context, } }() - // Look up cs AND compute under changesetMu: reading cs before the lock races - // the apply loop's SavePastChangesetAccumulator + accumulator rotation, which - // would route this block's [state] write into the next block's changeset. The - // lock is required even on the cs==nil path — the internal FlushPendingUpdates - // mutates the same global accumulator pointer. - // Calc-owned changeset: the calculator is the sole producer of an owned block's - // changeset. It reuses the block's saved changeset across the block's multiple - // computes — a fold records the commitment branch deltas (Diffs[3]) ahead of the - // tx-result stream, then step-edge checkpoints and the block-end compute add more - // — so those deltas accumulate rather than being lost. The account/storage/code - // diffs (0..2) are filled from the buffered per-tx writes once they are ready - // (finalized at the block boundary via finalizeFoldedChangeset for a folded - // block); a not-yet-ready csPending just leaves them for that later fill. - // - // The swap is LOAD-BEARING under changesetMu (held above): it mutates the global - // current-accumulator pointer that the deferred branch writes (flushed inside - // ComputeCommitmentLocked → FlushPendingUpdatesLocked) and the end-of-compute - // [state] marker also touch, serializing against the apply goroutine. Inside the - // lock the *Locked variants are required — the public ones re-acquire the mutex. + // Reuse the block's saved changeset across its multiple computes so a fold's + // commitment branch deltas (Diffs[3], recorded ahead of the tx-result stream) + // accumulate with the step-edge and block-end computes instead of being lost; + // account/storage/code diffs (0..2) are filled once the buffered writes are + // ready (later via finalizeFoldedChangeset for a folded block). The swap and + // save run under changesetMu — the swap mutates the global accumulator pointer + // that the deferred branch flush and end-of-compute marker also touch, + // serializing against the apply goroutine; the *Locked variants avoid + // re-acquiring the mutex. cc.doms.LockChangesetAccumulator() defer cc.doms.UnlockChangesetAccumulator() cs := cc.doms.GetChangesetByHash(t.blockNum, t.blockHash) diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 193baaa344c..4fc4a3b12a8 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -721,11 +721,8 @@ func (pe *parallelExecutor) execImpl(ctx context.Context, execStage *StageState, deliberateCancel() } - // SavePastChangesetAccumulator + SetChangesetAccumulator(nil) + - // rotation-to-next-block accumulator are all driven by the exec - // loop now (see execLoop's blockResult handling), so the apply - // loop must NOT touch SharedDomains.mem here. Doing so used to - // race with the exec loop's ApplyStateWrites for the next block. + // The apply loop must NOT touch SharedDomains.mem here — it would + // race the exec loop's ApplyStateWrites for the next block. if dbg.StopAfterBlock > 0 && applyResult.BlockNum == dbg.StopAfterBlock { pe.logger.Warn(fmt.Sprintf("[%s] STOP_AFTER_BLOCK reached, exiting without commit (debug mode)", pe.logPrefix), "block", applyResult.BlockNum) From b57e1ed01a56a48dae8dd897fafa500900697224 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 17 Jul 2026 19:08:56 +0000 Subject: [PATCH 157/167] execution/execmodule: assert tip newPayload carries a BAL in fold-ahead test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestBALFoldAheadFiresOnTipValidateChain now asserts the tip payload's BAL is non-empty before validating — making explicit that the payload carries a BAL, the precondition for its commitment to be folded in parallel with execution (the subsequent FoldsAheadPerformedForTest > 0 confirms the fold engaged). --- execution/execmodule/exec_module_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index bf511e7b214..68516b93abe 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -2748,7 +2748,12 @@ func TestBALFoldAheadFiresOnTipValidateChain(t *testing.T) { m.ExecModule.WaitCommitsDrained() tip := canonical.TopBlock - _, err = insertBlocksWithBAL(ctx, m.ExecModule, []*types.Block{tip}, canonical.BlockAccessLists[chainLen-1:]) + tipBAL := canonical.BlockAccessLists[chainLen-1] + // The tip payload must carry a BAL — that is the precondition that lets its + // commitment be folded in parallel with execution instead of computed + // incrementally from the per-tx result stream. + require.NotEmpty(t, tipBAL, "tip newPayload must carry a BAL") + _, err = insertBlocksWithBAL(ctx, m.ExecModule, []*types.Block{tip}, [][]byte{tipBAL}) require.NoError(t, err) stagedsync.ResetFoldsAheadForTest() @@ -2756,6 +2761,8 @@ func TestBALFoldAheadFiresOnTipValidateChain(t *testing.T) { require.NoError(t, err) require.Equal(t, execmodule.ExecutionStatusSuccess, vr.ValidationStatus) + // With a BAL present, the tip's commitment is folded from it (in parallel with + // exec) rather than computed on the incremental path. require.Positive(t, stagedsync.FoldsAheadPerformedForTest(), "tip newPayload (ValidateChain) must fold its commitment from the BAL") } From c323df04a5188b0d5bd28922d3640a4e43a7130d Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 18 Jul 2026 07:10:29 +0000 Subject: [PATCH 158/167] execution/stagedsync: post-block-consistent fold reader (fix wrong root on storage clear) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BAL fold computes a block's commitment ahead of execution, so its as-of state reader sees sd.mem BEFORE the block's writes are flushed — i.e. pre-block values. The commitment trie reads current (post-write) leaf/account values from that reader during a fold; when a block clears a storage slot (SSTORE an existing non-zero slot to 0) the cleared slot forces a storage-subtree collapse, and the fold's pre-block view (slot still present) collapses to a different storage root than the incremental path (which reads post-block state after the flush) — a wrong state root, surfaced as a BadBlock. This was masked until owned/chain-tip blocks began folding. Overlay the fold's own post-block state (balState, which already holds every key the block changes) onto its commitment reader for account/storage reads, so reader-sourced trie reads reflect post-block state and match the incremental path — while keeping the fold ahead of execution. The overlay is installed only for the commitment compute, so balState's own baseline lazy-loads still read pre-block state. Regression test reproduces the wrong root without the fix; full Amsterdam EEST blocktest corpus passes with it. --- execution/execmodule/exec_module_test.go | 61 ++++++++++++++++++++++++ execution/stagedsync/calc_state.go | 49 +++++++++++++++++++ execution/stagedsync/committer.go | 25 +++++++++- 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 68516b93abe..a196ef3c60c 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -2767,6 +2767,67 @@ func TestBALFoldAheadFiresOnTipValidateChain(t *testing.T) { "tip newPayload (ValidateChain) must fold its commitment from the BAL") } +// TestBALFoldTipStorageClearValidateChain folds a single Amsterdam block whose tx +// CLEARS a genesis-preset storage slot (SSTORE existing non-zero slot → 0) on an +// account that also receives value (balance change). This is the storage-delete / +// subtree-collapse case that a non-zero write does not exercise, and it is what +// exposed a wrong folded root: the fold runs ahead of exec, so its as-of reader +// returned pre-block state (slot still present) during the collapse. Mirrors the +// EEST blockchain_test_from_state_test shape (one block on genesis). +func TestBALFoldTipStorageClearValidateChain(t *testing.T) { + defer func(prev bool) { dbg.BALDrivenCommitment = prev }(dbg.BALDrivenCommitment) + defer func(prev bool) { dbg.IgnoreBAL = prev }(dbg.IgnoreBAL) + dbg.BALDrivenCommitment = true + dbg.IgnoreBAL = false + + privKey, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + require.NoError(t, err) + senderAddr := crypto.PubkeyToAddress(privKey.PublicKey) + contractAddr := common.HexToAddress("0x00000000000000000000000000000000c0de5501") + otherAddr := common.HexToAddress("0x00000000000000000000000000000000c0de5502") + m := execmoduletester.New(t, + execmoduletester.WithKey(privKey), + execmoduletester.WithGenesisSpec(&types.Genesis{ + Config: chain.AllProtocolChanges, + Alloc: types.GenesisAlloc{ + senderAddr: {Balance: big.NewInt(1 * common.Ether)}, + // Pre-set slot 0, then have the tx SSTORE(0,0) to CLEAR it (and send + // value, so the account leaf recomputes against the collapsed storage). + contractAddr: { + Balance: big.NewInt(0), + Code: []byte{0x60, 0x00, 0x60, 0x00, 0x55, 0x00}, // SSTORE(0,0); STOP + Storage: map[common.Hash]common.Hash{{}: common.BigToHash(big.NewInt(3))}, + }, + // A second storage-bearing account so the account trie has siblings. + otherAddr: { + Balance: big.NewInt(1), + Storage: map[common.Hash]common.Hash{{}: common.BigToHash(big.NewInt(7))}, + }, + }, + }), + execmoduletester.WithExperimentalBAL(), + execmoduletester.WithAlwaysGenerateChangesets(false), + ) + canonical, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, + func(i int, b *blockgen.BlockGen) { + tx, txErr := types.SignTx( + types.NewTransaction(0, contractAddr, uint256.NewInt(1_000), 100000, uint256.NewInt(10_000_000_000), nil), + *types.LatestSignerForChainID(nil), privKey, + ) + require.NoError(t, txErr) + b.AddTx(tx) + }, m.PublishedSD()) + require.NoError(t, err) + + require.NotEmpty(t, canonical.BlockAccessLists[0], "block must carry a BAL") + stagedsync.ResetFoldsAheadForTest() + // Insert + FCU in one step (no intervening ValidateChain), matching the EEST + // blockchain-test path: the fold runs during the FCU's execution and a wrong + // folded root surfaces as a BadBlock here. + require.NoError(t, m.InsertChain(canonical)) + require.Positive(t, stagedsync.FoldsAheadPerformedForTest(), "block must fold from its BAL") +} + // A forkchoice head at height 0 that is not the genesis (e.g. a block carrying a // corrupted number) must be rejected, not treated as a genesis reset. func TestUpdateForkChoiceToNonGenesisBlockAtHeightZero(t *testing.T) { diff --git a/execution/stagedsync/calc_state.go b/execution/stagedsync/calc_state.go index 8a9075eb44a..59581d92561 100644 --- a/execution/stagedsync/calc_state.go +++ b/execution/stagedsync/calc_state.go @@ -7,6 +7,7 @@ import ( "github.com/holiman/uint256" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" @@ -429,6 +430,54 @@ func (cs *calcState) FlushToUpdates(updates *commitment.Updates) { } } +// postValue returns the account/storage value AFTER this block's changes, in the +// serialized form the domain would store, for a key the block modified. ok is +// false for keys the block did not change (the caller should read the baseline). +// It gives the fold's commitment reader a post-block-consistent view: the fold +// runs ahead of execution, so a plain as-of read would return the pre-block value, +// which yields a wrong root when a cleared slot forces a storage-subtree collapse. +func (cs *calcState) postValue(d kv.Domain, plainKey []byte) (enc []byte, ok bool) { + switch d { + case kv.AccountsDomain: + if len(plainKey) != 20 { + return nil, false + } + acc, exists := cs.accounts[accounts.InternAddress(common.BytesToAddress(plainKey))] + if !exists || !acc.dirty { + return nil, false + } + if acc.Deleted && acc.Balance.IsZero() && acc.Nonce == 0 && acc.CodeHash == empty.CodeHash { + return nil, true // removed → absent + } + a := accounts.NewAccount() + a.Balance = acc.Balance + a.Nonce = acc.Nonce + a.Incarnation = acc.Incarnation + a.CodeHash = accounts.InternCodeHash(common.Hash(acc.CodeHash)) + return accounts.SerialiseV3(&a), true + case kv.StorageDomain: + if len(plainKey) != 20+32 { + return nil, false + } + addr := accounts.InternAddress(common.BytesToAddress(plainKey[:20])) + dirty := cs.storageDirty[addr] + if dirty == nil { + return nil, false + } + slot := accounts.InternKey(common.BytesToHash(plainKey[20:])) + if !dirty[slot] { + return nil, false + } + v := cs.storageState[addr][slot] + vb := v.Bytes() + if len(vb) == 0 { + return nil, true // cleared → absent + } + return vb, true + } + return nil, false +} + // ResetBlockFlags clears the per-block dirty flags while keeping the // accumulated state values. Called after commitment computation to // prepare for the next block. diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 23dcca78352..18f592ce732 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -628,6 +628,11 @@ func (cc *commitmentCalculator) foldBALToRoot(ctx context.Context, req *blockReq balUpdates.SetMode(commitment.ModeUpdate) } balState.FlushToUpdates(balUpdates) + // Overlay the fold's post-block state onto the reader for the commitment + // compute (set only now, so balState's own lazy-loads above read the pre-block + // baseline). This makes reader-sourced trie reads reflect POST-block state, + // matching the incremental path and fixing wrong roots on storage clears. + reader.postState = balState return cc.computeRootFromUpdates(ctx, t, balUpdates, reader) } @@ -952,6 +957,11 @@ type asOfStateReader struct { // a concurrent trie-warmup worker doesn't write the shared main accumulator // (a race) or take the global metrics lock. Nil on the main reader. workerCtx context.Context + // postState, when set, overlays this block's post-values for the account/storage + // keys it changed, so the fold's commitment reads reflect POST-block state. The + // fold runs ahead of execution, so a plain as-of read returns pre-block state, + // which produces a wrong root when a cleared slot forces a storage collapse. + postState *calcState } func (r *asOfStateReader) WithHistory() bool { return false } @@ -969,6 +979,17 @@ func (r *asOfStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (e enc, step, err = r.sd.GetLatest(d, r.roTx, plainKey) } } else { + // Post-block overlay: for a key the block changed, return its post-block + // value so the fold (running ahead of exec) sees the same state the + // incremental path reads after exec's flush. + if r.postState != nil { + if v, hit := r.postState.postValue(d, plainKey); hit { + if stepSize > 0 { + step = kv.Step(r.txNum / stepSize) + } + return v, step, nil + } + } // Account/storage/code: use GetAsOf to avoid reading future state. // Check sd.mem first (in-memory data from current batch), then // fall through to DB files for data not in the batch. @@ -995,7 +1016,7 @@ func (r *asOfStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (e } func (r *asOfStateReader) Clone(tx kv.TemporalTx) commitmentdb.StateReader { - return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum} + return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum, postState: r.postState} } // CloneForWorker meters the worker's CommitmentDomain reads into the per-worker @@ -1003,7 +1024,7 @@ func (r *asOfStateReader) Clone(tx kv.TemporalTx) commitmentdb.StateReader { // reader during block assembly, where trie-warmup runs concurrently — so it // must not write the shared main accumulator). func (r *asOfStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) commitmentdb.StateReader { - return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum, workerCtx: workerCtx} + return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum, workerCtx: workerCtx, postState: r.postState} } // asOfStorageEnumerator lists the persisted storage slots under an address via From dcc35f118723961040da9515064a0c6a5598289a Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 22 Jul 2026 15:51:11 +0000 Subject: [PATCH 159/167] execution/execmodule: scope payload-bodies BAL getter tests to a synchronous mock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Empty/PrunedHistory payload-bodies tests drove the getter through the full integration harness and mutated the raw DB behind the live tip overlay, which serves the tip under background commit — so the raw-DB writes were shadowed. Serve them from a synchronous, overlay-less ExecModule (NewGetterMockForTest, test-only) that reads the raw DB directly, and set DB state via rawdb rather than through the exec module. --- execution/execmodule/exec_module_test.go | 60 ++++++++++++++++++++---- execution/execmodule/getter_mock_test.go | 42 +++++++++++++++++ 2 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 execution/execmodule/getter_mock_test.go diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 3c53742bf30..abb28c0a173 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -1599,14 +1599,16 @@ func TestGetPayloadBodiesRegenerateBlockAccessLists(t *testing.T) { func TestGetPayloadBodiesEmptyBlockAccessList(t *testing.T) { t.Parallel() - m, chainPack := newPayloadBodiesBALTestChain(t, chain.AllProtocolChanges) - block := chainPack.Blocks[0] - require.NotNil(t, block.Header().BlockAccessListHash) - err := m.DB.Update(t.Context(), func(tx kv.RwTx) error { - return rawdb.WriteBlockAccessListBytes(tx, block.Hash(), block.NumberU64(), []byte{0xc0}) - }) + m := execmoduletester.New(t, execmoduletester.WithChainConfig(chain.AllProtocolChanges)) + chainPack, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, nil) require.NoError(t, err) - requirePayloadBodiesBlockAccessList(t, m, block, []byte{0xc0}) + block := chainPack.Blocks[0] + // A present-but-empty BAL (empty RLP list) is served verbatim, not treated + // as absent and regenerated. Write the block and its empty BAL to the DB and + // read them back through a synchronous getter that reads the DB directly. + writeCanonicalBlockFixture(t, m.DB, block, []byte{0xc0}) + em := execmodule.NewGetterMockForTest(m.DB, m.BlockReader, m.Engine, m.ChainConfig, m.Log) + requireMockGetterBAL(t, em, block, []byte{0xc0}) } func TestGetPayloadBodiesPreAmsterdamBlockAccessList(t *testing.T) { @@ -1625,8 +1627,12 @@ func TestGetPayloadBodiesPrunedHistoryBlockAccessList(t *testing.T) { m, chainPack := newPayloadBodiesBALTestChain(t, chain.AllProtocolChanges) block := chainPack.Blocks[0] require.NotNil(t, block.Header().BlockAccessListHash) + // Remove the stored BAL and the state history it could be regenerated from, + // then read through a synchronous DB-backed getter: with nothing to serve + // and no history to re-execute against, it must degrade to nil. prunePayloadBodiesBALHistory(t, m, block) - requirePayloadBodiesBlockAccessList(t, m, block, nil) + em := execmodule.NewGetterMockForTest(m.DB, m.BlockReader, m.Engine, m.ChainConfig, m.Log) + requireMockGetterBAL(t, em, block, nil) } func newPayloadBodiesBALTestChain(t *testing.T, config *chain.Config) (*execmoduletester.ExecModuleTester, *blockgen.ChainPack) { @@ -1652,6 +1658,44 @@ func requirePayloadBodiesBlockAccessList(t *testing.T, m *execmoduletester.ExecM require.Equal(t, want, byRange[0].BlockAccessList) } +// writeCanonicalBlockFixture stores a block (header, canonical marker, body) and +// optionally its BAL directly in the DB, without going through the execution +// module — for getter tests that assert DB-sourced serving deterministically. +func writeCanonicalBlockFixture(t *testing.T, db kv.TemporalRwDB, block *types.Block, bal []byte) { + t.Helper() + require.NoError(t, db.Update(t.Context(), func(tx kv.RwTx) error { + if err := rawdb.WriteHeader(tx, block.Header()); err != nil { + return err + } + if err := rawdb.WriteCanonicalHash(tx, block.Hash(), block.NumberU64()); err != nil { + return err + } + if _, err := rawdb.WriteRawBodyIfNotExists(tx, block.Hash(), block.NumberU64(), block.RawBody()); err != nil { + return err + } + if bal != nil { + return rawdb.WriteBlockAccessListBytes(tx, block.Hash(), block.NumberU64(), bal) + } + return nil + })) +} + +// requireMockGetterBAL asserts the payload-bodies getters resolve the block's +// BAL to want, reading through a synchronous DB-backed ExecModule. +func requireMockGetterBAL(t *testing.T, em *execmodule.ExecModule, block *types.Block, want []byte) { + t.Helper() + byHash, err := em.GetPayloadBodiesByHash(t.Context(), []common.Hash{block.Hash()}) + require.NoError(t, err) + require.Len(t, byHash, 1) + require.NotNil(t, byHash[0]) + require.Equal(t, want, byHash[0].BlockAccessList) + byRange, err := em.GetPayloadBodiesByRange(t.Context(), block.NumberU64(), 1) + require.NoError(t, err) + require.Len(t, byRange, 1) + require.NotNil(t, byRange[0]) + require.Equal(t, want, byRange[0].BlockAccessList) +} + func prunePayloadBodiesBALHistory(t *testing.T, m *execmoduletester.ExecModuleTester, block *types.Block) { t.Helper() err := m.DB.Update(t.Context(), func(tx kv.RwTx) error { diff --git a/execution/execmodule/getter_mock_test.go b/execution/execmodule/getter_mock_test.go new file mode 100644 index 00000000000..24705edd573 --- /dev/null +++ b/execution/execmodule/getter_mock_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execmodule + +import ( + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/dbservices" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/execution/bal" + "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/protocol/rules" +) + +// NewGetterMockForTest builds a synchronous ExecModule that serves the +// payload-bodies getters straight from the raw DB. It publishes no +// SharedDomains, so beginOverlayOrRo reads the committed database directly, and +// starts no background-commit worker — tests set DB state themselves and read +// it back deterministically. Exported from a _test.go file: usable by the +// execmodule_test binary, never compiled into a production build. +func NewGetterMockForTest(db kv.TemporalRwDB, blockReader dbservices.FullBlockReader, engine rules.Engine, config *chain.Config, logger log.Logger) *ExecModule { + return &ExecModule{ + db: db, + blockReader: blockReader, + balRegenerator: bal.NewRegenerator(blockReader, engine, logger), + config: config, + logger: logger, + } +} From 6710bcb65d1ab846104be775f90bf027f571f83f Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 08:25:38 +0000 Subject: [PATCH 160/167] execution, db: address yperbasis review round 5 (P1/P2 blockers) Revert FcuBackgroundCommit default to false (was accidentally left true; config.go and tester default both reverted). HasPrefix/IteratePrefix/GetAsOf on SharedDomains now walk the parent generation chain before falling through to the committed DB, matching getLatestMetered. OverlayTemporalReadView.GetLatest and HasPrefix now consult the installed SD (via domainLatestReader type assertion) before the committed temporal tx, so in-flight uncommitted state is visible to engine and RPC callers. Propagate changeset reconstruction errors: buildResultLocalChangeset returns (nil, error) instead of logging and returning nil; the compute caller propagates the error; finalizeFoldedChangeset logs it. EachStorageSlot errors are no longer silently discarded. Background prune goroutine now holds enterForeground/leaveForeground so the commit worker cannot open a conflicting RwTx while RunPrune shares the pipeline Sync with the next FCU's RunLoop. iiMem unwind boundary: <= corrected to < so the unwind txNum is excluded, matching domain pruning semantics. --- db/kv/membatchwithdb/memory_mutation.go | 19 +++++++++++++++++- db/state/execctx/domain_shared.go | 20 +++++++++++++++++-- db/state/temporal_mem_batch.go | 2 +- .../execmoduletester/exec_module_tester.go | 2 +- execution/execmodule/forkchoice.go | 5 +++++ execution/stagedsync/changeset_reconstruct.go | 19 +++++++++--------- execution/stagedsync/committer.go | 12 +++++++++-- node/ethconfig/config.go | 2 +- 8 files changed, 64 insertions(+), 17 deletions(-) diff --git a/db/kv/membatchwithdb/memory_mutation.go b/db/kv/membatchwithdb/memory_mutation.go index a75da5e6cfe..a1cedfcef9a 100644 --- a/db/kv/membatchwithdb/memory_mutation.go +++ b/db/kv/membatchwithdb/memory_mutation.go @@ -43,6 +43,13 @@ type DomainReader interface { HistorySeek(name kv.Domain, k []byte, ts uint64) ([]byte, bool, error) } +// domainLatestReader is the subset of SharedDomains needed for GetLatest/HasPrefix +// reads from an OverlayTemporalReadView. Implemented by SharedDomains. +type domainLatestReader interface { + GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) ([]byte, kv.Step, error) + HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]byte, []byte, bool, error) +} + type MemoryMutation struct { // mu protects concurrent access to the mutation's maps and backing tx. // Read methods (GetOne, Has) acquire RLock; write methods (Put, Delete, @@ -1134,13 +1141,23 @@ func (v *OverlayTemporalReadView) Apply(_ context.Context, f func(tx kv.Tx) erro return f(v) } -// Temporal methods — delegate to the independent temporal tx. +// Temporal methods — check the installed SD chain before the committed tx. func (v *OverlayTemporalReadView) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { + if dr, ok := v.MemoryMutation.DomainReader.(domainLatestReader); ok { + if val, step, err := dr.GetLatest(name, v.temporalTx, k); err == nil && val != nil { + return val, step, nil + } + } return v.temporalTx.GetLatest(name, k) } func (v *OverlayTemporalReadView) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + if dr, ok := v.MemoryMutation.DomainReader.(domainLatestReader); ok { + if fk, fv, ok, err := dr.HasPrefix(name, prefix, v.temporalTx); ok || err != nil { + return fk, fv, ok, err + } + } return v.temporalTx.HasPrefix(name, prefix) } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 2adfbb75e5d..ede4b50caac 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -870,7 +870,15 @@ func (sd *SharedDomains) InlineTouchKeyDisabled() bool { } func (sd *SharedDomains) HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]byte, []byte, bool, error) { - return sd.mem.HasPrefix(domain, prefix, roTx) + if k, v, ok, err := sd.mem.HasPrefix(domain, prefix, roTx); ok || err != nil { + return k, v, ok, err + } + for p := sd.parent; p != nil; p = p.parent { + if k, v, ok, err := p.mem.HasPrefix(domain, prefix, roTx); ok || err != nil { + return k, v, ok, err + } + } + return nil, nil, false, nil } func (sd *SharedDomains) IteratePrefix(domain kv.Domain, prefix []byte, roTx kv.Tx, it func(k []byte, v []byte) (cont bool, err error)) error { @@ -1583,7 +1591,15 @@ func (sd *SharedDomains) DomainLogMetrics() map[kv.Domain][]any { } func (sd *SharedDomains) GetAsOf(domain kv.Domain, key []byte, ts uint64) (v []byte, ok bool, err error) { - return sd.mem.GetAsOf(domain, key, ts) + if v, ok, err = sd.mem.GetAsOf(domain, key, ts); ok || err != nil { + return v, ok, err + } + for p := sd.parent; p != nil; p = p.parent { + if v, ok, err = p.mem.GetAsOf(domain, key, ts); ok || err != nil { + return v, ok, err + } + } + return nil, false, nil } // RangeAsOf returns domain values over [fromKey, toKey) as of ts, merging the diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 773d33330c1..6bab161c626 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -923,7 +923,7 @@ func (sd *TemporalMemBatch) Unwind(unwindToTxNum uint64, changeset *[kv.DomainLe for k, txNums := range byKey { kept := txNums[:0] for _, tn := range txNums { - if tn <= unwindToTxNum { + if tn < unwindToTxNum { kept = append(kept, tn) } } diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 0b70195f8f0..a9af27e82fc 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -414,7 +414,7 @@ func applyOptions(opts []Option) options { chainConfig: chain.TestChainBerlinConfig, experimentalBAL: false, sentryProtocol: direct.ETH68, - fcuBackgroundCommit: true, + fcuBackgroundCommit: false, } for _, o := range opts { o(&opt) diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index b3d78a63e32..aff35bec95d 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -773,8 +773,13 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa commitTimings = ct // Prune: background by default (fcuBackgroundPrune=true). + // The goroutine holds an enterForeground/leaveForeground pair so the + // commit worker cannot start a commit (and open a conflicting RwTx) + // while RunPrune shares the pipeline Sync with the next FCU's RunLoop. if e.fcuBackgroundPrune { + e.enterForeground() go func() { + defer e.leaveForeground() if _, err := e.runForkchoicePrune(initialCycle); err != nil && !errors.Is(err, context.Canceled) { e.logger.Error("Error running background prune", "err", err) } diff --git a/execution/stagedsync/changeset_reconstruct.go b/execution/stagedsync/changeset_reconstruct.go index 38897448d33..7f82457035e 100644 --- a/execution/stagedsync/changeset_reconstruct.go +++ b/execution/stagedsync/changeset_reconstruct.go @@ -152,14 +152,16 @@ func (cc *commitmentCalculator) feedChangeset(writes *state.WriteSet, txNum uint if cc.state.storageEnum == nil { continue } - _ = cc.state.storageEnum.EachStorageSlot(addr, func(key accounts.StorageKey) error { + if err := cc.state.storageEnum.EachStorageSlot(addr, func(key accounts.StorageKey) error { keyVal := key.Value() composite := make([]byte, 20+32) copy(composite, addrVal[:]) copy(composite[20:], keyVal[:]) cc.appendCS(kv.StorageDomain, composite, nil, txNum) return nil - }) + }); err != nil { + cc.logger.Warn("["+cc.logPrefix+"] changeset reconstruct: storage enumeration error", "addr", addr, "err", err) + } } } @@ -167,19 +169,18 @@ func (cc *commitmentCalculator) feedChangeset(writes *state.WriteSet, txNum uint // builder AT COMPUTE TIME — resolving prevs when block N-1 is settled in sd.mem // (matching exec's block-end flush) — and returns the account/storage/code diffs // (domains 0..2) as a StateChangeSet. The commitment domain (3) is left empty for -// the caller's compute to fill. Returns nil if no writes were buffered for the -// block or a prev-read failed. -func (cc *commitmentCalculator) buildResultLocalChangeset(blockNum uint64) *changeset.StateChangeSet { +// the caller's compute to fill. Returns (nil, nil) when no writes were buffered +// for the block; returns (nil, err) on a prev-read failure. +func (cc *commitmentCalculator) buildResultLocalChangeset(blockNum uint64) (*changeset.StateChangeSet, error) { if cc.csBuilderBlock != blockNum { - return nil + return nil, nil } b := newChangesetBuilder(cc.prevReader, cc.doms.StepSize()) for _, w := range cc.csPending { b.record(w.domain, w.key, w.val, w.txNum) } if err := b.err(); err != nil { - cc.logger.Warn("["+cc.logPrefix+"] changeset reconstruct: reader error", "block", blockNum, "err", err) - return nil + return nil, err } - return b.result() + return b.result(), nil } diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 6acc3916a2a..8268d4b4b01 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -908,7 +908,11 @@ func (cc *commitmentCalculator) computeWithBlockAccumulator(ctx context.Context, if cs == nil { cs = &changeset.StateChangeSet{} } - if localAcc := cc.buildResultLocalChangeset(t.blockNum); localAcc != nil { + localAcc, err := cc.buildResultLocalChangeset(t.blockNum) + if err != nil { + return nil, fmt.Errorf("changeset reconstruct block %d: %w", t.blockNum, err) + } + if localAcc != nil { cs.Diffs[kv.AccountsDomain] = localAcc.Diffs[kv.AccountsDomain] cs.Diffs[kv.StorageDomain] = localAcc.Diffs[kv.StorageDomain] cs.Diffs[kv.CodeDomain] = localAcc.Diffs[kv.CodeDomain] @@ -928,7 +932,11 @@ func (cc *commitmentCalculator) computeWithBlockAccumulator(ctx context.Context, // already recorded. A fold computes the root (and Diffs[3]) ahead of the tx-result // stream, so 0..2 cannot be built at fold time; this closes that gap. func (cc *commitmentCalculator) finalizeFoldedChangeset(blockNum uint64, blockHash common.Hash) { - localAcc := cc.buildResultLocalChangeset(blockNum) + localAcc, err := cc.buildResultLocalChangeset(blockNum) + if err != nil { + cc.logger.Warn("["+cc.logPrefix+"] changeset reconstruct: reader error", "block", blockNum, "err", err) + return + } if localAcc == nil { return } diff --git a/node/ethconfig/config.go b/node/ethconfig/config.go index ada1b161427..782170b4a18 100644 --- a/node/ethconfig/config.go +++ b/node/ethconfig/config.go @@ -113,7 +113,7 @@ var Defaults = Config{ }, FcuTimeout: 1 * time.Second, FcuBackgroundPrune: true, - FcuBackgroundCommit: true, + FcuBackgroundCommit: false, ExperimentalBAL: false, WarmupKzgCtxOnInit: true, } From 76478166f89ffd978b69b89f03f3141811189bca Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 12:50:30 +0000 Subject: [PATCH 161/167] db/state/execctx: revert HasPrefix parent-chain walk (false CREATE-collision) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-5 review fix added a parent-generation walk to SharedDomains.HasPrefix so an in-flight generation's uncommitted storage would be visible. That is unsafe for a prefix-existence query: HasPrefix cannot respect a child generation's storage clear when walking parents — a parent that still holds a slot returns true even though the child cleared the account. HasStorage (the EIP-7610 CREATE-collision check via stateReader.HasStorage -> kv.HasPrefix) therefore reported storage on a fork whose account has none, flipping the collision decision and the block's gas nondeterministically (INVALID payload on the enginex self-destruct / CREATE-collision / recreate fixtures). Evidence: instrumenting the collision check showed HasStorage(target) returning true on failing runs and false on passing runs for the same fork-from-genesis block; reverting the parent walk gives 0/15 failures (was ~6/12) and the whole enginex self-destruct/CREATE/create2 sample passes. GetLatest and GetAsOf keep their parent walk: they read a specific key child-mem-first, so a clear overrides an ancestor value. Prefix existence has no such override, so the parent walk must be reverted here. The reviewer's enumerate-uncommitted-parent-storage concern needs a clear-respecting approach (follow-up), not a blind prefix walk. --- db/state/execctx/domain_shared.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 06aad32ba16..e8cd3068d9a 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -870,15 +870,7 @@ func (sd *SharedDomains) InlineTouchKeyDisabled() bool { } func (sd *SharedDomains) HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]byte, []byte, bool, error) { - if k, v, ok, err := sd.mem.HasPrefix(domain, prefix, roTx); ok || err != nil { - return k, v, ok, err - } - for p := sd.parent; p != nil; p = p.parent { - if k, v, ok, err := p.mem.HasPrefix(domain, prefix, roTx); ok || err != nil { - return k, v, ok, err - } - } - return nil, nil, false, nil + return sd.mem.HasPrefix(domain, prefix, roTx) } func (sd *SharedDomains) IteratePrefix(domain kv.Domain, prefix []byte, roTx kv.Tx, it func(k []byte, v []byte) (cont bool, err error)) error { From 751e7ee3abb54caf30b073af63cca10cf1effeb7 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 14:03:05 +0000 Subject: [PATCH 162/167] execution/execmodule, db/state: fix bg-prune FCU race + clear-aware HasPrefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two nondeterministic bugs surfaced by the enginex reorg/collision fixtures. Background-prune FCU race (review P1#4, done right): the prune goroutine was holding only enterForeground/leaveForeground (fgCount), which gates the commit worker but NOT the foreground semaphore — so the next FCU's fgTryAcquire still succeeded and its RunLoop raced RunPrune on the shared pipeline Sync, leaving the committed head hash stale (BadBlock "headHash and blockHash mismatch" on the reorg-forward, ~50% flaky). Hand the foreground slot to the prune goroutine instead: it releases the semaphore + fg count when RunPrune finishes, so the next FCU stays Busy until the prune's RwTx is done. Clear-aware HasPrefix: the round-5 parent-chain walk resolved each parent key via the parent's own HasPrefix, which ignores a nearer generation's storage clear. A fork whose account was cleared by create-contract DomainDelPrefix (a tombstone in this mem) still saw an ancestor's slot, so HasStorage reported storage that isn't there and flipped the EIP-7610 CREATE-collision decision and the block gas nondeterministically (INVALID on self-destruct/CREATE-collision/ recreate). Resolve each parent-found key through the child-first chain (getLatestMetered) so a nearer clear overrides the ancestor value; the reviewer's enumerate-uncommitted-parent-storage case still works because a live (uncleared) ancestor slot resolves non-empty. --- db/state/execctx/domain_shared.go | 37 +++++++++++++++++++++++++++++- execution/execmodule/forkchoice.go | 27 ++++++++++++++-------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index e8cd3068d9a..06cd67bc205 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -869,8 +869,43 @@ func (sd *SharedDomains) InlineTouchKeyDisabled() bool { return sd.disableInlineTouchKey } +// HasPrefix reports whether any live key with the prefix exists. The local +// mem+files answer comes first; parent generations are then scanned so an +// uncommitted ancestor's keys are visible (a self-destruct in this generation +// must enumerate storage a not-yet-committed parent wrote). A key found in a +// parent is resolved through the child-first chain (getLatestMetered), so a +// clear in a nearer generation — e.g. a create-contract DomainDelPrefix +// tombstone recorded only in this mem — overrides the ancestor's value rather +// than falsely reporting the prefix as present. func (sd *SharedDomains) HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]byte, []byte, bool, error) { - return sd.mem.HasPrefix(domain, prefix, roTx) + if k, v, ok, err := sd.mem.HasPrefix(domain, prefix, roTx); ok || err != nil { + return k, v, ok, err + } + tx, _ := roTx.(kv.TemporalTx) + if sd.parent == nil || tx == nil { + return nil, nil, false, nil + } + var firstKey, firstVal []byte + for p := sd.parent; p != nil; p = p.parent { + err := p.mem.IteratePrefix(domain, prefix, roTx, func(k, v []byte) (bool, error) { + lv, _, e := sd.getLatestMetered(domain, tx, k, nil) + if e != nil { + return false, e + } + if len(lv) > 0 { + firstKey, firstVal = bytes.Clone(k), bytes.Clone(lv) + return false, nil + } + return true, nil + }) + if err != nil { + return nil, nil, false, err + } + if firstKey != nil { + return firstKey, firstVal, true, nil + } + } + return nil, nil, false, nil } func (sd *SharedDomains) IteratePrefix(domain kv.Domain, prefix []byte, roTx kv.Tx, it func(k []byte, v []byte) (cont bool, err error)) error { diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 9d3a7b133b2..e238f8862a8 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -356,10 +356,16 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa }, false) return fmt.Errorf("semaphore timeout") } - // The semaphore is always released when updateForkChoice returns — the - // background commit no longer holds it (the commit runs on - // the bg worker, decoupled from the foreground semaphore). - defer e.fgRelease() + // The foreground slot is released when updateForkChoice returns, unless a + // background prune goroutine takes it over (pruneHandedOff): RunPrune shares + // the pipeline Sync with the next FCU's RunLoop, so the next FCU must stay + // Busy until the prune finishes. In that case the prune goroutine releases. + pruneHandedOff := false + defer func() { + if !pruneHandedOff { + e.fgRelease() + } + }() // Retire the in-flight commit-generation chain if every generation has // committed. Safe here: the foreground semaphore is held, @@ -773,14 +779,15 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa } commitTimings = ct - // Prune: background by default (fcuBackgroundPrune=true). - // The goroutine holds an enterForeground/leaveForeground pair so the - // commit worker cannot start a commit (and open a conflicting RwTx) - // while RunPrune shares the pipeline Sync with the next FCU's RunLoop. + // Prune: background by default (fcuBackgroundPrune=true). The prune + // goroutine takes over the foreground slot (semaphore + fg count) and + // releases it when done, so the next FCU stays Busy — RunPrune shares + // the pipeline Sync with the next FCU's RunLoop and opens its own RwTx, + // which must not overlap either. if e.fcuBackgroundPrune { - e.enterForeground() + pruneHandedOff = true go func() { - defer e.leaveForeground() + defer e.fgRelease() if _, err := e.runForkchoicePrune(initialCycle); err != nil && !errors.Is(err, context.Canceled) { e.logger.Error("Error running background prune", "err", err) } From ad2f1f22b2714bf10fb22d5356d117d50d66f8ea Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 14:34:40 +0000 Subject: [PATCH 163/167] execution/execmodule: read e.currentContext under lock in FCU cleanup The cleanupBeforeSemaRelease closure read e.currentContext without e.lock, while InsertBlocks writes it under e.lock. The bg-prune semaphore handoff lets this cleanup run on the FCU goroutine while a later InsertBlocks proceeds, so -race flagged the write/read on e.currentContext (TestImportReorgUnwindToGenesis, TestReorgBackAndForwardIntoCanonicalChain/ bg-prune). Snapshot e.currentContext under e.lock.RLock in the cleanup. --- execution/execmodule/forkchoice.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index e238f8862a8..5bb6482e0be 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -376,7 +376,15 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // The next semaphore acquirer must observe settled state, so the bg-commit/ // bg-prune paths run this eagerly before handing the semaphore to their goroutine. cleanupBeforeSemaRelease := sync.OnceFunc(func() { - e.currentContext.ResetPendingUpdates() + // e.currentContext is written by InsertBlocks under e.lock; read it under + // the same lock so this cleanup (which may run on the FCU goroutine while + // a later InsertBlocks proceeds) doesn't race that write. + e.lock.RLock() + cc := e.currentContext + e.lock.RUnlock() + if cc != nil { + cc.ResetPendingUpdates() + } e.forkValidator.ClearWithUnwind() }) defer cleanupBeforeSemaRelease() From 7ea6f806b4b34f1085c737651d92f1c485d870e4 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 18:09:55 +0000 Subject: [PATCH 164/167] execution, db, rpc: address FCU bg-commit review (P1/P2 blockers) - domain_shared: HasPrefix/IteratePrefix walk the whole generation chain and resolve each candidate key through GetLatest, so parent-generation storage is visible and parent-cleared storage is not resurrected; HistorySeek chains too. - memory_mutation: OverlayTemporalReadView GetLatest/HasPrefix return the installed reader's value+error authoritatively (no tombstone resurrection, no hidden error); GetAsOf/HistorySeek stop hiding reader errors. - getters/filters: build the published read view through BlockOverlayTemporalTx so it chains every ancestor generation's block overlay, not just the leaf. - committer/changeset_reconstruct: propagate changeset-reconstruct and storage- enumeration errors through publish instead of logging and discarding. - bg_commit: a failed durable commit triggers a clean node shutdown (once) rather than silently leaving the generation un-retired and hanging drain. - exec_module: WaitIdle stops the worker even when its ctx-acquire is cancelled; closeAllGens broadcasts fgIdle so drain waiters wake. - temporal_mem_batch: HistorySeek falls through to committed history when ts precedes the first in-memory write. - block: SetBlockAccessList copies the BAL so a tx-owned slice cannot alias. --- db/kv/membatchwithdb/memory_mutation.go | 30 ++++-- db/state/execctx/domain_shared.go | 93 ++++++++++++++----- db/state/execctx/domain_shared_test.go | 83 +++++++++++++++++ db/state/temporal_mem_batch.go | 12 +-- execution/execmodule/bg_commit.go | 18 +++- execution/execmodule/exec_module.go | 10 +- execution/execmodule/getters.go | 11 ++- execution/stagedsync/changeset_reconstruct.go | 12 ++- execution/stagedsync/committer.go | 20 ++-- execution/types/block.go | 5 +- rpc/rpchelper/filters.go | 16 +++- 11 files changed, 245 insertions(+), 65 deletions(-) diff --git a/db/kv/membatchwithdb/memory_mutation.go b/db/kv/membatchwithdb/memory_mutation.go index ffd92f38361..6789394e341 100644 --- a/db/kv/membatchwithdb/memory_mutation.go +++ b/db/kv/membatchwithdb/memory_mutation.go @@ -1143,19 +1143,21 @@ func (v *OverlayTemporalReadView) Apply(_ context.Context, f func(tx kv.Tx) erro // Temporal methods — check the installed SD chain before the committed tx. func (v *OverlayTemporalReadView) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { + // The installed reader walks the whole generation chain and falls through to + // v.temporalTx itself, so its value and error are authoritative — a nil here + // is a genuine tombstone, not a cue to re-read the committed tx. if dr, ok := v.MemoryMutation.DomainReader.(domainLatestReader); ok { - if val, step, err := dr.GetLatest(name, v.temporalTx, k); err == nil && val != nil { - return val, step, nil - } + return dr.GetLatest(name, v.temporalTx, k) } return v.temporalTx.GetLatest(name, k) } func (v *OverlayTemporalReadView) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + // Authoritative for the same reason as GetLatest: the reader already honours + // parent-generation tombstones and the committed tx, so a false result must + // not be overridden by re-checking committed storage the overlay cleared. if dr, ok := v.MemoryMutation.DomainReader.(domainLatestReader); ok { - if fk, fv, ok, err := dr.HasPrefix(name, prefix, v.temporalTx); ok || err != nil { - return fk, fv, ok, err - } + return dr.HasPrefix(name, prefix, v.temporalTx) } return v.temporalTx.HasPrefix(name, prefix) } @@ -1168,8 +1170,12 @@ func (v *OverlayTemporalReadView) GetAsOf(name kv.Domain, k []byte, ts uint64) ( // Check DomainReader independently — this method shadows MemoryMutation.GetAsOf // and falls through to v.temporalTx (not m.db), so the embedded check never fires. if v.MemoryMutation != nil && v.MemoryMutation.DomainReader != nil { - if val, ok, err := v.MemoryMutation.DomainReader.GetAsOf(name, k, ts); err == nil && ok { - return val, ok, nil + val, ok, err := v.MemoryMutation.DomainReader.GetAsOf(name, k, ts) + if err != nil { + return nil, false, err + } + if ok { + return val, true, nil } } return v.temporalTx.GetAsOf(name, k, ts) @@ -1187,8 +1193,12 @@ func (v *OverlayTemporalReadView) HistorySeek(name kv.Domain, k []byte, ts uint6 // Check DomainReader independently — this method shadows MemoryMutation.HistorySeek // and falls through to v.temporalTx (not m.db), so the embedded check never fires. if v.MemoryMutation != nil && v.MemoryMutation.DomainReader != nil { - if val, ok, err := v.MemoryMutation.DomainReader.HistorySeek(name, k, ts); err == nil && ok { - return val, ok, nil + val, ok, err := v.MemoryMutation.DomainReader.HistorySeek(name, k, ts) + if err != nil { + return nil, false, err + } + if ok { + return val, true, nil } } return v.temporalTx.HistorySeek(name, k, ts) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 06cd67bc205..a5467a20a24 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "runtime" + "slices" "sync" "sync/atomic" "time" @@ -877,28 +878,30 @@ func (sd *SharedDomains) InlineTouchKeyDisabled() bool { // clear in a nearer generation — e.g. a create-contract DomainDelPrefix // tombstone recorded only in this mem — overrides the ancestor's value rather // than falsely reporting the prefix as present. +// HasPrefix reports the first live key/value under prefix, chain-correctly: a +// candidate from the leaf batch (leaf RAM + committed DB) or any parent +// generation is confirmed through getLatest so a value cleared by a parent +// generation is not a false positive and a value written only in a parent is +// still found. Short-circuits on the first live key. func (sd *SharedDomains) HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]byte, []byte, bool, error) { - if k, v, ok, err := sd.mem.HasPrefix(domain, prefix, roTx); ok || err != nil { - return k, v, ok, err - } - tx, _ := roTx.(kv.TemporalTx) - if sd.parent == nil || tx == nil { - return nil, nil, false, nil + tx, ok := roTx.(kv.TemporalTx) + if sd.parent == nil || !ok { + return sd.mem.HasPrefix(domain, prefix, roTx) } var firstKey, firstVal []byte - for p := sd.parent; p != nil; p = p.parent { - err := p.mem.IteratePrefix(domain, prefix, roTx, func(k, v []byte) (bool, error) { - lv, _, e := sd.getLatestMetered(domain, tx, k, nil) - if e != nil { - return false, e - } - if len(lv) > 0 { - firstKey, firstVal = bytes.Clone(k), bytes.Clone(lv) - return false, nil - } - return true, nil - }) - if err != nil { + confirm := func(k, _ []byte) (bool, error) { + lv, _, e := sd.getLatestMetered(domain, tx, k, nil) + if e != nil { + return false, e + } + if len(lv) > 0 { + firstKey, firstVal = bytes.Clone(k), bytes.Clone(lv) + return false, nil + } + return true, nil + } + for p := sd; p != nil; p = p.parent { + if err := p.mem.IteratePrefix(domain, prefix, roTx, confirm); err != nil { return nil, nil, false, err } if firstKey != nil { @@ -908,8 +911,47 @@ func (sd *SharedDomains) HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) return nil, nil, false, nil } +// IteratePrefix emits every live key under prefix across the whole generation +// chain, newest-generation-wins with tombstone shadowing: it unions the +// candidate keys from the leaf batch (leaf RAM + committed DB) and every parent +// generation's RAM, resolves each through getLatest, and emits only the +// non-empty ones in key order. func (sd *SharedDomains) IteratePrefix(domain kv.Domain, prefix []byte, roTx kv.Tx, it func(k []byte, v []byte) (cont bool, err error)) error { - return sd.mem.IteratePrefix(domain, prefix, roTx, it) + tx, ok := roTx.(kv.TemporalTx) + if sd.parent == nil || !ok { + return sd.mem.IteratePrefix(domain, prefix, roTx, it) + } + seen := make(map[string]struct{}) + var keys []string + collect := func(k, _ []byte) (bool, error) { + ks := string(k) + if _, dup := seen[ks]; !dup { + seen[ks] = struct{}{} + keys = append(keys, ks) + } + return true, nil + } + for p := sd; p != nil; p = p.parent { + if err := p.mem.IteratePrefix(domain, prefix, roTx, collect); err != nil { + return err + } + } + slices.Sort(keys) + for _, ks := range keys { + k := []byte(ks) + v, _, err := sd.getLatestMetered(domain, tx, k, nil) + if err != nil { + return err + } + if len(v) == 0 { + continue + } + cont, err := it(k, v) + if err != nil || !cont { + return err + } + } + return nil } func (sd *SharedDomains) Close() { @@ -1643,9 +1685,16 @@ func (sd *SharedDomains) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs return sd.mem.IndexRange(name, k, fromTs, toTs, asc, limit, roTx) } -// HistorySeek returns the in-flight in-memory historical value of key as of ts. +// HistorySeek returns the in-flight in-memory historical value of key as of ts, +// walking the whole generation chain; (nil, false) means no generation has it, +// so the caller falls back to committed history. func (sd *SharedDomains) HistorySeek(domain kv.Domain, key []byte, ts uint64) ([]byte, bool, error) { - return sd.mem.HistorySeek(domain, key, ts) + for p := sd; p != nil; p = p.parent { + if v, ok, err := p.mem.HistorySeek(domain, key, ts); err != nil || ok { + return v, ok, err + } + } + return nil, false, nil } // DomainPut diff --git a/db/state/execctx/domain_shared_test.go b/db/state/execctx/domain_shared_test.go index 5472600e162..a3c8db9e5a0 100644 --- a/db/state/execctx/domain_shared_test.go +++ b/db/state/execctx/domain_shared_test.go @@ -1485,6 +1485,89 @@ func TestSharedDomain_HasPrefix_StorageDomain(t *testing.T) { } } +// TestSharedDomain_HasPrefix_ParentChain pins the background-commit generation +// chain: a child SD's prefix reads must see storage written only in an +// uncommitted parent generation, and must NOT resurrect committed storage that +// a parent generation cleared. +func TestSharedDomain_HasPrefix_ParentChain(t *testing.T) { + if testing.Short() { + t.Skip("slow test") + } + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + + db := newTestDb(t, uint64(1)) + + acc1 := common.HexToAddress("0x1234567890123456789012345678901234567890") + acc1slot := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001") + storageK1 := append(append([]byte{}, acc1[:]...), acc1slot[:]...) + acc2 := common.HexToAddress("0x1234567890123456789012345678901234567891") + acc2slot := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000002") + storageK2 := append(append([]byte{}, acc2[:]...), acc2slot[:]...) + + // Commit acc1's storage so it lives in the DB (the "already committed" state a + // parent generation may later clear). + rwTx0, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + t.Cleanup(rwTx0.Rollback) + base, err := execctx.NewSharedDomains(ctx, rwTx0, log.New()) + require.NoError(t, err) + require.NoError(t, base.DomainPut(kv.StorageDomain, rwTx0, storageK1, []byte{1}, 1, nil)) + require.NoError(t, base.Flush(ctx, rwTx0)) + require.NoError(t, rwTx0.Commit()) + base.Close() + + // Parent generation (uncommitted): write acc2's storage and clear acc1's. + rwTxP, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + t.Cleanup(rwTxP.Rollback) + parent, err := execctx.NewSharedDomains(ctx, rwTxP, log.New()) + require.NoError(t, err) + t.Cleanup(parent.Close) + require.NoError(t, parent.DomainPut(kv.StorageDomain, rwTxP, storageK2, []byte{2}, 2, nil)) + require.NoError(t, parent.DomainDelPrefix(kv.StorageDomain, rwTxP, acc1[:], 3)) + + // Child generation chained onto the parent, reading a committed snapshot. + roTx, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + t.Cleanup(roTx.Rollback) + child, err := execctx.NewSharedDomains(ctx, roTx, log.New()) + require.NoError(t, err) + t.Cleanup(child.Close) + child.SetParent(parent) + + // Item 3c: storage written only in the parent generation is visible. + firstKey, firstVal, ok, err := child.HasPrefix(kv.StorageDomain, acc2[:], roTx) + require.NoError(t, err) + require.True(t, ok, "parent-generation storage must be visible to the child") + require.Equal(t, storageK2, firstKey) + require.Equal(t, []byte{2}, firstVal) + + var iterated [][]byte + require.NoError(t, child.IteratePrefix(kv.StorageDomain, acc2[:], roTx, func(k, _ []byte) (bool, error) { + iterated = append(iterated, bytes.Clone(k)) + return true, nil + })) + require.Equal(t, [][]byte{storageK2}, iterated) + + // Item 3b: committed storage cleared by the parent generation must not be + // resurrected from the committed tx. + firstKey, firstVal, ok, err = child.HasPrefix(kv.StorageDomain, acc1[:], roTx) + require.NoError(t, err) + require.False(t, ok, "storage cleared by the parent generation must stay cleared") + require.Nil(t, firstKey) + require.Nil(t, firstVal) + + iterated = nil + require.NoError(t, child.IteratePrefix(kv.StorageDomain, acc1[:], roTx, func(k, _ []byte) (bool, error) { + iterated = append(iterated, bytes.Clone(k)) + return true, nil + })) + require.Empty(t, iterated, "IteratePrefix must not yield parent-cleared storage") +} + // TestDomainPut_HistoryCorrectness is a property test that verifies history invariants // after random sequences of DomainPut calls: // 1. GetAsOf returns the correct value at every txNum (history is complete and accurate) diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index a74ec0074b8..f4378f42075 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -422,10 +422,11 @@ func (sd *TemporalMemBatch) memRangeAsOf(domain kv.Domain, fromKey, toKey []byte } // HistorySeek returns the in-memory value in effect at ts: the value of the last -// recorded change with txNum < ts. ok is true when the key has in-memory history; -// a ts at or before the first change yields the empty creation-event marker -// (non-nil empty, true). Returns (nil, false) only when the key has no in-memory -// history, so the caller can fall back to committed history. +// recorded change with txNum < ts. Returns (nil, false) when there is no such +// change — either the key has no in-memory history, or ts precedes its first +// in-memory write — so the caller falls back to committed history, which holds +// the value for a key that existed before the overlay (and correctly reports +// absent for one created in it). func (sd *TemporalMemBatch) HistorySeek(domain kv.Domain, key []byte, ts uint64) ([]byte, bool, error) { if !sd.inMemHistoryReads { return nil, false, nil @@ -444,9 +445,6 @@ func (sd *TemporalMemBatch) HistorySeek(domain kv.Domain, key []byte, ts uint64) return e.data, true, nil } } - if len(entries) > 0 { - return []byte{}, true, nil - } return nil, false, nil } diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index a1632eb9685..f20b2893ab5 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -163,9 +163,19 @@ func (e *ExecModule) runCommit(gen *commitGen) { } // A durable commit failed for a non-shutdown reason: the block was already // reported VALID to the CL, so silently dropping its state would diverge - // from consensus. This is unrecoverable in-process — fail hard rather than - // retire the generation as if it committed. - e.logger.Crit("background commit failed; cannot retire generation", "block", gen.blockNum, "hash", gen.blockHash, "err", err) + // from consensus. Crit does not terminate this logger, and returning would + // leave the generation un-retired (FCU wedges Busy, drain/shutdown hangs). + // Trigger a clean node shutdown (cancels the root context) so the process + // restarts and re-derives the state. In a goroutine — stopNode drains this + // same worker — and once, so repeated drain failures don't respawn it. + e.logger.Crit("background commit failed; shutting down node", "block", gen.blockNum, "hash", gen.blockHash, "err", err) + e.commitFatalOnce.Do(func() { + go func() { + if stopErr := e.stopNode(); stopErr != nil { + e.logger.Error("Could not stop node after background commit failure", "err", stopErr) + } + }() + }) return } e.markGenCommitted(gen) @@ -299,4 +309,6 @@ func (e *ExecModule) closeAllGens() { } e.gens = nil e.uncommittedGens = 0 + // Wake any WaitCommitsDrained/WaitIdle waiter blocked on the generation count. + e.fgIdle.Broadcast() } diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index b1d08869139..52bf1385d7e 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -247,6 +247,9 @@ type ExecModule struct { commitWorkerStop chan struct{} commitStopOnce sync.Once commitWg sync.WaitGroup + // commitFatalOnce ensures a failed durable commit triggers node shutdown at + // most once, even as the worker drains further generations that also fail. + commitFatalOnce sync.Once // stateCache is a cache for state data (accounts, storage, code) stateCache *cache.StateCache @@ -351,10 +354,11 @@ func NewExecModule( // WaitIdle blocks until any in-flight updateForkChoice goroutine finishes. // Call before closing the database to avoid waitTxsAllDoneOnClose hangs. func (e *ExecModule) WaitIdle(ctx context.Context) { - if err := e.fgAcquire(ctx); err != nil { - return // context cancelled — best effort + // Best-effort foreground barrier; even if the ctx is cancelled we must still + // stop the worker below so it can never run against a closing DB. + if err := e.fgAcquire(ctx); err == nil { + e.fgRelease() } - e.fgRelease() // Stop the background-commit worker and wait for it to drain its queue // (including any in-flight commit tx) so DB-close does not race it. e.stopCommitWorker() diff --git a/execution/execmodule/getters.go b/execution/execmodule/getters.go index 17d84cca46a..e9eca1b8d65 100644 --- a/execution/execmodule/getters.go +++ b/execution/execmodule/getters.go @@ -66,15 +66,18 @@ func (e *ExecModule) beginOverlayOrRo(ctx context.Context) (kv.TemporalTx, func( } if sd != nil { if overlay := sd.BlockOverlay(); overlay != nil { - // Open a fresh RO tx while still holding the read lock so that - // the overlay cannot be closed between our check and the - // NewReadView call (TOCTOU avoidance). + // Open a fresh RO tx while still holding the read lock so the + // generation chain cannot change between the check and building the + // view (TOCTOU avoidance). Route through BlockOverlayTemporalTx so + // the view chains every ancestor generation's block overlay, not just + // the leaf — otherwise reads miss uncommitted block data from earlier + // FCUs. roTx, err := e.db.BeginTemporalRo(ctx) //nolint:gocritic if err != nil { e.lock.RUnlock() return nil, nil, err } - view := overlay.NewReadView(roTx) + view := sd.BlockOverlayTemporalTx(roTx) e.lock.RUnlock() return view, func() { roTx.Rollback() }, nil } diff --git a/execution/stagedsync/changeset_reconstruct.go b/execution/stagedsync/changeset_reconstruct.go index 48323a86bef..4c70277688e 100644 --- a/execution/stagedsync/changeset_reconstruct.go +++ b/execution/stagedsync/changeset_reconstruct.go @@ -2,6 +2,7 @@ package stagedsync import ( "bytes" + "fmt" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/empty" @@ -25,15 +26,15 @@ type csWrite struct { // now, from the post-tx state) so the changeset can be replayed and its prevs // resolved at settled compute time — matching exec's block-end flush timing rather // than the racy per-tx moment. Non-owned blocks need no changeset. -func (cc *commitmentCalculator) bufferChangesetWrites(r *txResult) { +func (cc *commitmentCalculator) bufferChangesetWrites(r *txResult) error { if !cc.ownsChangeset(r.blockNum) { - return + return nil } if cc.csBuilderBlock != r.blockNum { cc.csPending = cc.csPending[:0] cc.csBuilderBlock = r.blockNum } - cc.feedChangeset(r.writes, r.txNum) + return cc.feedChangeset(r.writes, r.txNum) } // asOfPrevReader supplies the pre-block value of an account/storage/code key as @@ -77,7 +78,7 @@ func (cc *commitmentCalculator) appendCS(domain kv.Domain, key, val []byte, txNu // storage/code puts carry their new bytes (empty → delete), and a self-destruct // cascades a storage-subtree wipe. Prevs are resolved later (at settled compute // time) so the replayed ChangeSets3 bytes match exec's. -func (cc *commitmentCalculator) feedChangeset(writes *state.WriteSet, txNum uint64) { +func (cc *commitmentCalculator) feedChangeset(writes *state.WriteSet, txNum uint64) error { touched := map[accounts.Address]struct{}{} for addr := range writes.Balances() { touched[addr] = struct{}{} @@ -162,9 +163,10 @@ func (cc *commitmentCalculator) feedChangeset(writes *state.WriteSet, txNum uint cc.appendCS(kv.StorageDomain, composite, nil, txNum) return nil }); err != nil { - cc.logger.Warn("["+cc.logPrefix+"] changeset reconstruct: storage enumeration error", "addr", addr, "err", err) + return fmt.Errorf("changeset reconstruct: storage enumeration for %x: %w", addr, err) } } + return nil } // buildResultLocalChangeset replays the buffered per-tx writes through a fresh diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 6a44a0aeb8c..2fa353c2ea1 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -346,7 +346,10 @@ func (cc *commitmentCalculator) handleMessage(ctx context.Context, msg applyResu if !r.writes.IsEmpty() { cc.asOfReader.txNum = r.txNum cc.state.ApplyWrites(r.writes, r.rules.IsAmsterdam) - cc.bufferChangesetWrites(r) + if err := cc.bufferChangesetWrites(r); err != nil { + cc.publish(ctx, commitmentResult{blockNum: r.blockNum, txNum: r.txNum, err: err}) + return + } } // A computed-ahead block already emitted its interior step checkpoints from @@ -411,7 +414,7 @@ func (cc *commitmentCalculator) handleMessage(ctx context.Context, msg applyResu // ahead of the tx-result stream; fill its account/storage/code diffs // now that the block's writes are buffered. if cc.ownsChangeset(r.BlockNum) { - cc.finalizeFoldedChangeset(r.BlockNum, r.BlockHash) + cc.finalizeFoldedChangeset(ctx, r) } } case cc.perBlockCompute(r.BlockNum): @@ -931,10 +934,13 @@ func (cc *commitmentCalculator) computeWithBlockAccumulator(ctx context.Context, // writes are complete — preserving the commitment branch deltas (Diffs[3]) the fold // already recorded. A fold computes the root (and Diffs[3]) ahead of the tx-result // stream, so 0..2 cannot be built at fold time; this closes that gap. -func (cc *commitmentCalculator) finalizeFoldedChangeset(blockNum uint64, blockHash common.Hash) { - localAcc, err := cc.buildResultLocalChangeset(blockNum) +func (cc *commitmentCalculator) finalizeFoldedChangeset(ctx context.Context, r *blockResult) { + localAcc, err := cc.buildResultLocalChangeset(r.BlockNum) if err != nil { - cc.logger.Warn("["+cc.logPrefix+"] changeset reconstruct: reader error", "block", blockNum, "err", err) + // A discarded reader error would save only the fold's commitment diffs + // (Diffs[3]), leaving account/storage/code state unrestorable on reorg — + // surface it so the apply loop fails the block instead. + cc.publish(ctx, commitmentResult{blockNum: r.BlockNum, txNum: r.lastTxNum, err: err}) return } if localAcc == nil { @@ -942,14 +948,14 @@ func (cc *commitmentCalculator) finalizeFoldedChangeset(blockNum uint64, blockHa } cc.doms.LockChangesetAccumulator() defer cc.doms.UnlockChangesetAccumulator() - cs := cc.doms.GetChangesetByHash(blockNum, blockHash) + cs := cc.doms.GetChangesetByHash(r.BlockNum, r.BlockHash) if cs == nil { cs = &changeset.StateChangeSet{} } cs.Diffs[kv.AccountsDomain] = localAcc.Diffs[kv.AccountsDomain] cs.Diffs[kv.StorageDomain] = localAcc.Diffs[kv.StorageDomain] cs.Diffs[kv.CodeDomain] = localAcc.Diffs[kv.CodeDomain] - cc.doms.SavePastChangesetAccumulator(blockHash, blockNum, cs) + cc.doms.SavePastChangesetAccumulator(r.BlockHash, r.BlockNum, cs) } // asOfStateReader reads account/storage/code at a specific txNum via diff --git a/execution/types/block.go b/execution/types/block.go index ba50e590e5b..e0b5d631d2d 100644 --- a/execution/types/block.go +++ b/execution/types/block.go @@ -1391,8 +1391,9 @@ func (b *Block) BlockAccessListHash() *common.Hash { return b.header.BlockAcce // payload (nil when absent). It is not part of the block's RLP encoding or hash. func (b *Block) BlockAccessList() []byte { return b.blockAccessList } -// SetBlockAccessList attaches the RLP-encoded BAL sidecar to the block. -func (b *Block) SetBlockAccessList(bal []byte) { b.blockAccessList = bal } +// SetBlockAccessList attaches the RLP-encoded BAL sidecar to the block, copying +// the input so a transaction-owned or later-mutated source cannot alias it. +func (b *Block) SetBlockAccessList(bal []byte) { b.blockAccessList = bytes.Clone(bal) } // Header returns a deep-copy of the entire block header using CopyHeader() func (b *Block) Header() *Header { return CopyHeader(b.header) } diff --git a/rpc/rpchelper/filters.go b/rpc/rpchelper/filters.go index 4645e6661f9..47e02a4a34b 100644 --- a/rpc/rpchelper/filters.go +++ b/rpc/rpchelper/filters.go @@ -1183,6 +1183,13 @@ func (ff *Filters) WithOverlay(tx kv.Tx) kv.Tx { return tx } if overlay := sd.BlockOverlay(); overlay != nil { + // Chain every ancestor generation's overlay when the base tx is temporal; + // a non-temporal tx cannot carry the chain, so fall back to the leaf. + if ttx, ok := tx.(kv.TemporalTx); ok { + if v := sd.BlockOverlayTemporalTx(ttx); v != nil { + return v + } + } return overlay.NewReadView(tx) } return tx @@ -1198,8 +1205,13 @@ func (ff *Filters) WithTemporalOverlay(tx kv.TemporalTx) kv.TemporalTx { if sd == nil { return tx } - if overlay := sd.BlockOverlay(); overlay != nil { - return overlay.NewReadView(tx) + // Route through BlockOverlayTemporalTx so the view chains every ancestor + // generation's block overlay, not just the leaf — otherwise reads miss + // uncommitted block data from earlier FCUs. + if sd.BlockOverlay() != nil { + if v := sd.BlockOverlayTemporalTx(tx); v != nil { + return v + } } return tx } From c747cfe811581c3a0ca8c9b7d98b1e3d3a60d6bf Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 18:32:17 +0000 Subject: [PATCH 165/167] db/state, execution, rpc: rename BlockOverlayTemporalTx -> OverlayTemporalTx The overlay is a general KV/temporal db, not block-specific, so drop the misleading Block prefix (and ParentBlockOverlayTemporalTx -> ParentOverlayTemporalTx). --- db/state/execctx/domain_shared.go | 18 +++++++++--------- db/state/execctx/domain_shared_test.go | 2 +- execution/builder/builder.go | 2 +- execution/exec/blocks_read_ahead.go | 2 +- execution/execmodule/bg_commit.go | 2 +- execution/execmodule/exec_module.go | 2 +- .../execmoduletester/exec_module_tester.go | 2 +- .../execmoduletester/overlay_read.go | 2 +- execution/execmodule/getters.go | 4 ++-- execution/execmodule/scoped_read.go | 2 +- execution/stagedsync/exec3.go | 2 +- rpc/jsonrpc/eth_call.go | 2 +- rpc/jsonrpc/graphql_api.go | 2 +- rpc/rpchelper/filters.go | 6 +++--- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a5467a20a24..df7b1a455a5 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -727,10 +727,10 @@ func (sd *SharedDomains) CloseBlockOverlay() { } } -// BlockOverlayTemporalTx returns a read-only temporal view of the block overlay. -// This allows consumers (RPC, shutter) to read uncommitted block data with +// OverlayTemporalTx returns a read-only temporal view of the overlay. This +// allows consumers (RPC, shutter) to read uncommitted overlay data with // temporal (state history) support. Returns nil if no overlay is active. -func (sd *SharedDomains) BlockOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { +func (sd *SharedDomains) OverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { // Chain the read view through the parent generations' block overlays, oldest // ancestor first, bottoming out at roTx — NewTemporalReadView rebinds the // fallthrough base, so the ancestor chain must be built explicitly here. This @@ -738,7 +738,7 @@ func (sd *SharedDomains) BlockOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalT // ancestor generation stays visible until its commit lands. base := roTx if sd.parent != nil { - if pv := sd.parent.BlockOverlayTemporalTx(roTx); pv != nil { + if pv := sd.parent.OverlayTemporalTx(roTx); pv != nil { base = pv } } @@ -752,15 +752,15 @@ func (sd *SharedDomains) BlockOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalT return overlay.NewTemporalReadView(base) } -// ParentBlockOverlayTemporalTx returns the block-overlay read view of the parent -// chain only (excluding this SD's own overlay), or nil when there is no parent. +// ParentOverlayTemporalTx returns the overlay read view of the parent chain +// only (excluding this SD's own overlay), or nil when there is no parent. // A reader derives its overlay base from this so the base and the chain it later -// walks via BlockOverlayTemporalTx are the same generations — they cannot diverge. -func (sd *SharedDomains) ParentBlockOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { +// walks via OverlayTemporalTx are the same generations — they cannot diverge. +func (sd *SharedDomains) ParentOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { if sd.parent == nil { return nil } - return sd.parent.BlockOverlayTemporalTx(roTx) + return sd.parent.OverlayTemporalTx(roTx) } // InitBlockOverlay creates (or replaces) the block-level metadata overlay backed by diff --git a/db/state/execctx/domain_shared_test.go b/db/state/execctx/domain_shared_test.go index a3c8db9e5a0..b61cb359617 100644 --- a/db/state/execctx/domain_shared_test.go +++ b/db/state/execctx/domain_shared_test.go @@ -2064,7 +2064,7 @@ func TestBlockOverlay_DomainReadsRegression(t *testing.T) { require.Equal(t, value, gotValHist) // --- Secondary path: overlay.NewTemporalReadView returns *OverlayTemporalReadView --- - overlayTx := sd.BlockOverlayTemporalTx(tx) + overlayTx := sd.OverlayTemporalTx(tx) require.NotNil(t, overlayTx) gotVal2, ok, err := overlayTx.GetAsOf(kv.ReceiptDomain, key, txNum+1) diff --git a/execution/builder/builder.go b/execution/builder/builder.go index 6181a1ec6ac..a640ff3ce41 100644 --- a/execution/builder/builder.go +++ b/execution/builder/builder.go @@ -138,7 +138,7 @@ func (b *Builder) Build(param *Parameters, interrupt *atomic.Bool) (result *type // generation chain, not just parentSD's local overlay — otherwise block // data held in an ancestor generation is missed and executionAt reads // stale, which desyncs the builder from the txpool's head. - if v := parentSD.BlockOverlayTemporalTx(tx); v != nil { + if v := parentSD.OverlayTemporalTx(tx); v != nil { compositeTx = v } } diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index b6fa24228d2..6d3388f5abf 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -166,7 +166,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t } else { var balSrc kv.TemporalTx = btx if sd != nil { - if ov := sd.BlockOverlayTemporalTx(btx); ov != nil { + if ov := sd.OverlayTemporalTx(btx); ov != nil { balSrc = ov } } diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index f20b2893ab5..623615f3dda 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -246,7 +246,7 @@ func (e *ExecModule) latestGen() *execctx.SharedDomains { // FCU's writes instead of a stale DB. Returns roTx unchanged when there is none. func (e *ExecModule) overlayBaseFor(roTx kv.TemporalTx) kv.TemporalTx { if parent := e.latestGen(); parent != nil { - if v := parent.BlockOverlayTemporalTx(roTx); v != nil { + if v := parent.OverlayTemporalTx(roTx); v != nil { return v } } diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 52bf1385d7e..4e48d805cae 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -651,7 +651,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // block-data reads cascade through the same generations that domain-state // reads do — never a separate, divergent capture. valOverlayBase := kv.TemporalTx(roTx) - if v := doms.ParentBlockOverlayTemporalTx(roTx); v != nil { + if v := doms.ParentOverlayTemporalTx(roTx); v != nil { valOverlayBase = v } if err := doms.InitBlockOverlay(valOverlayBase, roTx.Debug().Dirs().Tmp); err != nil { diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 5331a8fe652..e82c0852159 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -979,7 +979,7 @@ func (emt *ExecModuleTester) InsertChain(chain *blockgen.ChainPack) error { // it lands in the raw DB, so read through the overlay when one is published. var roTx kv.Tx = baseTx if sd := emt.PublishedSD(); sd != nil { - if v := sd.BlockOverlayTemporalTx(baseTx); v != nil { + if v := sd.OverlayTemporalTx(baseTx); v != nil { roTx = v } } diff --git a/execution/execmodule/execmoduletester/overlay_read.go b/execution/execmodule/execmoduletester/overlay_read.go index 1f8e798f416..f470b98e022 100644 --- a/execution/execmodule/execmoduletester/overlay_read.go +++ b/execution/execmodule/execmoduletester/overlay_read.go @@ -76,7 +76,7 @@ func wrapSDTx(base kv.TemporalTx, sd *execctx.SharedDomains) kv.TemporalTx { return base } read := base - if v := sd.BlockOverlayTemporalTx(base); v != nil { + if v := sd.OverlayTemporalTx(base); v != nil { read = v } return &sdRoTx{TemporalTx: read, base: base, sd: sd} diff --git a/execution/execmodule/getters.go b/execution/execmodule/getters.go index e9eca1b8d65..c5ff7c54146 100644 --- a/execution/execmodule/getters.go +++ b/execution/execmodule/getters.go @@ -68,7 +68,7 @@ func (e *ExecModule) beginOverlayOrRo(ctx context.Context) (kv.TemporalTx, func( if overlay := sd.BlockOverlay(); overlay != nil { // Open a fresh RO tx while still holding the read lock so the // generation chain cannot change between the check and building the - // view (TOCTOU avoidance). Route through BlockOverlayTemporalTx so + // view (TOCTOU avoidance). Route through OverlayTemporalTx so // the view chains every ancestor generation's block overlay, not just // the leaf — otherwise reads miss uncommitted block data from earlier // FCUs. @@ -77,7 +77,7 @@ func (e *ExecModule) beginOverlayOrRo(ctx context.Context) (kv.TemporalTx, func( e.lock.RUnlock() return nil, nil, err } - view := sd.BlockOverlayTemporalTx(roTx) + view := sd.OverlayTemporalTx(roTx) e.lock.RUnlock() return view, func() { roTx.Rollback() }, nil } diff --git a/execution/execmodule/scoped_read.go b/execution/execmodule/scoped_read.go index a37f68b1716..2a71933c935 100644 --- a/execution/execmodule/scoped_read.go +++ b/execution/execmodule/scoped_read.go @@ -52,7 +52,7 @@ type ScopedReadView struct { } // Tx returns the pinned base roTx. Readers pass it to sd.AsGetter / -// BlockOverlay().NewReadView / BlockOverlayTemporalTx. +// BlockOverlay().NewReadView / OverlayTemporalTx. func (v *ScopedReadView) Tx() kv.TemporalTx { return v.roTx } // HeadSD returns the head SharedDomains this view is pinned to, or nil when the diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 3931ccf8d71..095202d01d0 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -572,7 +572,7 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m // chain must be layered ahead of the tx so an uncommitted ancestor // generation's block data stays visible until its commit lands. var blockTx kv.Tx = execRoTx - if v := te.doms.BlockOverlayTemporalTx(execRoTx); v != nil { + if v := te.doms.OverlayTemporalTx(execRoTx); v != nil { blockTx = v } diff --git a/rpc/jsonrpc/eth_call.go b/rpc/jsonrpc/eth_call.go index 7d7cb447e2f..cd7030d2bff 100644 --- a/rpc/jsonrpc/eth_call.go +++ b/rpc/jsonrpc/eth_call.go @@ -87,7 +87,7 @@ func (api *APIImpl) Call(ctx context.Context, args ethapi2.CallArgs, requestedBl var tx kv.TemporalTx = roTx if api.filters != nil { if sd := api.filters.LatestSD(); sd != nil { - if overlayTx := sd.BlockOverlayTemporalTx(roTx); overlayTx != nil { + if overlayTx := sd.OverlayTemporalTx(roTx); overlayTx != nil { tx = overlayTx } } diff --git a/rpc/jsonrpc/graphql_api.go b/rpc/jsonrpc/graphql_api.go index e4dece748f1..1c1f65bad63 100644 --- a/rpc/jsonrpc/graphql_api.go +++ b/rpc/jsonrpc/graphql_api.go @@ -391,7 +391,7 @@ func (api *GraphQLAPIImpl) Call(ctx context.Context, blockNumber rpc.BlockNumber var tx kv.TemporalTx = roTx if api.filters != nil { if sd := api.filters.LatestSD(); sd != nil { - if overlayTx := sd.BlockOverlayTemporalTx(roTx); overlayTx != nil { + if overlayTx := sd.OverlayTemporalTx(roTx); overlayTx != nil { tx = overlayTx } } diff --git a/rpc/rpchelper/filters.go b/rpc/rpchelper/filters.go index 47e02a4a34b..2133db20f72 100644 --- a/rpc/rpchelper/filters.go +++ b/rpc/rpchelper/filters.go @@ -1186,7 +1186,7 @@ func (ff *Filters) WithOverlay(tx kv.Tx) kv.Tx { // Chain every ancestor generation's overlay when the base tx is temporal; // a non-temporal tx cannot carry the chain, so fall back to the leaf. if ttx, ok := tx.(kv.TemporalTx); ok { - if v := sd.BlockOverlayTemporalTx(ttx); v != nil { + if v := sd.OverlayTemporalTx(ttx); v != nil { return v } } @@ -1205,11 +1205,11 @@ func (ff *Filters) WithTemporalOverlay(tx kv.TemporalTx) kv.TemporalTx { if sd == nil { return tx } - // Route through BlockOverlayTemporalTx so the view chains every ancestor + // Route through OverlayTemporalTx so the view chains every ancestor // generation's block overlay, not just the leaf — otherwise reads miss // uncommitted block data from earlier FCUs. if sd.BlockOverlay() != nil { - if v := sd.BlockOverlayTemporalTx(tx); v != nil { + if v := sd.OverlayTemporalTx(tx); v != nil { return v } } From 868efdcf4e28aa8c1045d4ee142c7b5767d4760a Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Jul 2026 20:35:19 +0000 Subject: [PATCH 166/167] execution/execmodule: drop redundant ParentOverlayTemporalTx It was just OverlayTemporalTx on the parent, existing only to reach the unexported parent field from another package. The sole caller runs before InitBlockOverlay, so doms has no overlay of its own and OverlayTemporalTx already yields exactly the parent chain. --- db/state/execctx/domain_shared.go | 11 ----------- execution/execmodule/exec_module.go | 10 ++++++---- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index df7b1a455a5..165ddc9308c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -752,17 +752,6 @@ func (sd *SharedDomains) OverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { return overlay.NewTemporalReadView(base) } -// ParentOverlayTemporalTx returns the overlay read view of the parent chain -// only (excluding this SD's own overlay), or nil when there is no parent. -// A reader derives its overlay base from this so the base and the chain it later -// walks via OverlayTemporalTx are the same generations — they cannot diverge. -func (sd *SharedDomains) ParentOverlayTemporalTx(roTx kv.TemporalTx) kv.TemporalTx { - if sd.parent == nil { - return nil - } - return sd.parent.OverlayTemporalTx(roTx) -} - // InitBlockOverlay creates (or replaces) the block-level metadata overlay backed by // the given base transaction. Writes to the overlay are visible to subsequent reads // and are flushed atomically alongside domain state via Flush(). diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 4e48d805cae..d9dbe1ed4b8 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -647,11 +647,13 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b doms.SetParent(parent) } - // Back the validation overlay by the SD's OWN parent chain so - // block-data reads cascade through the same generations that domain-state - // reads do — never a separate, divergent capture. + // Back the validation overlay by the SD's OWN parent chain so block-data + // reads cascade through the same generations that domain-state reads do — + // never a separate, divergent capture. doms has no overlay of its own yet + // (InitBlockOverlay is next), so OverlayTemporalTx here yields exactly the + // parent chain. valOverlayBase := kv.TemporalTx(roTx) - if v := doms.ParentOverlayTemporalTx(roTx); v != nil { + if v := doms.OverlayTemporalTx(roTx); v != nil { valOverlayBase = v } if err := doms.InitBlockOverlay(valOverlayBase, roTx.Debug().Dirs().Tmp); err != nil { From 928dbef23c85caa1cdb6a955b62a96f6a823d04c Mon Sep 17 00:00:00 2001 From: mh0lt Date: Wed, 29 Jul 2026 13:35:07 +0000 Subject: [PATCH 167/167] execution, db: address FCU bg-commit review round 7 (P1/P2/P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - domain_shared: make HasPrefix/IteratePrefix two-phase (collect candidates under the mem read lock, resolve after releasing it) — the previous shape re-entered the same RWMutex via getLatest inside an IteratePrefix callback and could deadlock a queued DomainPut writer. HasPrefix now returns the first live key in global key order. - bg_commit: poison the commit stream synchronously on a failed durable commit so no already-queued descendant generation commits its delta over the hole before shutdown propagates; enqueueCommit rolls a generation back instead of parking it once poisoned. - exec_module: on WaitIdle timeout (foreground op still holding the semaphore) do not close the generation chain under the live reader — poison, stop the worker, and log; only close generations on a clean acquire. - forkchoice: run cleanupBeforeSemaRelease() before handing the semaphore to the background prune goroutine so pending-update/unwind cleanup can't race prune. - getters: guard beginOverlayOrRo against a nil view when the overlay is flushed concurrently — retry with a fresh roTx that sees the flushed blocks. - memory_mutation: MemoryMutation.GetAsOf/HistorySeek stop swallowing the DomainReader error. - un-skip TestNotificationDispatchBackgroundCommit (the SetParent chain fixed it). --- db/kv/membatchwithdb/memory_mutation.go | 16 +++-- db/state/execctx/domain_shared.go | 90 +++++++++++------------- execution/execmodule/bg_commit.go | 39 ++++++++-- execution/execmodule/exec_module.go | 29 +++++--- execution/execmodule/exec_module_test.go | 6 -- execution/execmodule/forkchoice.go | 6 ++ execution/execmodule/getters.go | 14 +++- 7 files changed, 129 insertions(+), 71 deletions(-) diff --git a/db/kv/membatchwithdb/memory_mutation.go b/db/kv/membatchwithdb/memory_mutation.go index 6789394e341..1c8df10b5c8 100644 --- a/db/kv/membatchwithdb/memory_mutation.go +++ b/db/kv/membatchwithdb/memory_mutation.go @@ -959,8 +959,12 @@ func (m *MemoryMutation) GetLatest(name kv.Domain, k []byte) (v []byte, step kv. func (m *MemoryMutation) GetAsOf(name kv.Domain, k []byte, ts uint64) (v []byte, ok bool, err error) { if m.DomainReader != nil { - if val, ok, err := m.DomainReader.GetAsOf(name, k, ts); err == nil && ok { - return val, ok, nil + val, ok, err := m.DomainReader.GetAsOf(name, k, ts) + if err != nil { + return nil, false, err + } + if ok { + return val, true, nil } } if m.db == nil { @@ -992,8 +996,12 @@ func (m *MemoryMutation) RangeAsOf(name kv.Domain, fromKey, toKey []byte, ts uin func (m *MemoryMutation) HistorySeek(name kv.Domain, k []byte, ts uint64) (v []byte, ok bool, err error) { if m.DomainReader != nil { - if val, ok, err := m.DomainReader.HistorySeek(name, k, ts); err == nil && ok { - return val, ok, nil + val, ok, err := m.DomainReader.HistorySeek(name, k, ts) + if err != nil { + return nil, false, err + } + if ok { + return val, true, nil } } if m.db == nil { diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 165ddc9308c..8cac5ed7a0a 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -859,73 +859,69 @@ func (sd *SharedDomains) InlineTouchKeyDisabled() bool { return sd.disableInlineTouchKey } -// HasPrefix reports whether any live key with the prefix exists. The local -// mem+files answer comes first; parent generations are then scanned so an -// uncommitted ancestor's keys are visible (a self-destruct in this generation -// must enumerate storage a not-yet-committed parent wrote). A key found in a -// parent is resolved through the child-first chain (getLatestMetered), so a -// clear in a nearer generation — e.g. a create-contract DomainDelPrefix -// tombstone recorded only in this mem — overrides the ancestor's value rather -// than falsely reporting the prefix as present. -// HasPrefix reports the first live key/value under prefix, chain-correctly: a -// candidate from the leaf batch (leaf RAM + committed DB) or any parent -// generation is confirmed through getLatest so a value cleared by a parent -// generation is not a false positive and a value written only in a parent is -// still found. Short-circuits on the first live key. +// collectPrefixCandidates unions the candidate keys under prefix from the leaf +// batch (leaf RAM + committed DB) and every parent generation's RAM, returned in +// key order. Collection runs inside each mem's read lock but only appends keys; +// callers resolve values afterwards (getLatest re-locks), so no read lock is +// ever held across a second acquisition — see the recursive-RLock hazard on +// sync.RWMutex. The whole candidate set is materialized before resolution, so a +// self-destruct enumerating a large contract's storage holds all its slot keys. +func (sd *SharedDomains) collectPrefixCandidates(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]string, error) { + seen := make(map[string]struct{}) + var keys []string + collect := func(k, _ []byte) (bool, error) { + ks := string(k) + if _, dup := seen[ks]; !dup { + seen[ks] = struct{}{} + keys = append(keys, ks) + } + return true, nil + } + for p := sd; p != nil; p = p.parent { + if err := p.mem.IteratePrefix(domain, prefix, roTx, collect); err != nil { + return nil, err + } + } + slices.Sort(keys) + return keys, nil +} + +// HasPrefix reports the first live key/value under prefix in key order, chain- +// correctly across the generation chain: a candidate cleared by a nearer +// generation is skipped and one written only in a parent is still found. func (sd *SharedDomains) HasPrefix(domain kv.Domain, prefix []byte, roTx kv.Tx) ([]byte, []byte, bool, error) { tx, ok := roTx.(kv.TemporalTx) if sd.parent == nil || !ok { return sd.mem.HasPrefix(domain, prefix, roTx) } - var firstKey, firstVal []byte - confirm := func(k, _ []byte) (bool, error) { - lv, _, e := sd.getLatestMetered(domain, tx, k, nil) - if e != nil { - return false, e - } - if len(lv) > 0 { - firstKey, firstVal = bytes.Clone(k), bytes.Clone(lv) - return false, nil - } - return true, nil + keys, err := sd.collectPrefixCandidates(domain, prefix, roTx) + if err != nil { + return nil, nil, false, err } - for p := sd; p != nil; p = p.parent { - if err := p.mem.IteratePrefix(domain, prefix, roTx, confirm); err != nil { + for _, ks := range keys { + k := []byte(ks) + v, _, err := sd.getLatestMetered(domain, tx, k, nil) + if err != nil { return nil, nil, false, err } - if firstKey != nil { - return firstKey, firstVal, true, nil + if len(v) > 0 { + return k, bytes.Clone(v), true, nil } } return nil, nil, false, nil } // IteratePrefix emits every live key under prefix across the whole generation -// chain, newest-generation-wins with tombstone shadowing: it unions the -// candidate keys from the leaf batch (leaf RAM + committed DB) and every parent -// generation's RAM, resolves each through getLatest, and emits only the -// non-empty ones in key order. +// chain in key order, newest-generation-wins with tombstone shadowing. func (sd *SharedDomains) IteratePrefix(domain kv.Domain, prefix []byte, roTx kv.Tx, it func(k []byte, v []byte) (cont bool, err error)) error { tx, ok := roTx.(kv.TemporalTx) if sd.parent == nil || !ok { return sd.mem.IteratePrefix(domain, prefix, roTx, it) } - seen := make(map[string]struct{}) - var keys []string - collect := func(k, _ []byte) (bool, error) { - ks := string(k) - if _, dup := seen[ks]; !dup { - seen[ks] = struct{}{} - keys = append(keys, ks) - } - return true, nil - } - for p := sd; p != nil; p = p.parent { - if err := p.mem.IteratePrefix(domain, prefix, roTx, collect); err != nil { - return err - } + keys, err := sd.collectPrefixCandidates(domain, prefix, roTx) + if err != nil { + return err } - slices.Sort(keys) for _, ks := range keys { k := []byte(ks) v, _, err := sd.getLatestMetered(domain, tx, k, nil) diff --git a/execution/execmodule/bg_commit.go b/execution/execmodule/bg_commit.go index 623615f3dda..d062a051e96 100644 --- a/execution/execmodule/bg_commit.go +++ b/execution/execmodule/bg_commit.go @@ -93,10 +93,17 @@ func (e *ExecModule) leaveForeground() { e.fgMu.Unlock() } -// enqueueCommit hands a completed generation to the background commit -// worker. Non-blocking by design (buffered channel) so the foreground FCU -// is never blocked handing off its commit. +// enqueueCommit hands a completed generation to the background commit worker. +// Non-blocking so the foreground FCU never blocks handing off its commit (the +// channel is buffered and the maxInFlightCommits backpressure keeps it from +// filling). Once the commit stream is poisoned (failed commit or shutdown) the +// worker no longer drains, so roll the generation's roTx back rather than park +// it forever — a leaked open reader would hang chainDB.Close. func (e *ExecModule) enqueueCommit(gen *commitGen) { + if e.commitIsPoisoned() { + gen.roTx.Rollback() + return + } e.commitCh <- gen } @@ -151,6 +158,12 @@ func (e *ExecModule) runCommit(gen *commitGen) { return } + // A prior generation's durable commit failed: do not advance the DB past the + // hole it left. Skip every remaining generation while shutdown propagates. + if e.commitIsPoisoned() { + return + } + err := e.runPostForkchoice(gen.sd, gen.roTx, gen.finishProgressBefore, gen.isSynced, gen.initialCycle) // roTx is rolled back inside runForkchoiceFlushCommit between Flush and // Commit; this is a safety net (Rollback is idempotent). @@ -163,11 +176,14 @@ func (e *ExecModule) runCommit(gen *commitGen) { } // A durable commit failed for a non-shutdown reason: the block was already // reported VALID to the CL, so silently dropping its state would diverge - // from consensus. Crit does not terminate this logger, and returning would + // from consensus. Poison the queue first (synchronously) so no already- + // enqueued descendant commits its delta over this hole before shutdown + // propagates. Crit does not terminate this logger, and returning would // leave the generation un-retired (FCU wedges Busy, drain/shutdown hangs). // Trigger a clean node shutdown (cancels the root context) so the process // restarts and re-derives the state. In a goroutine — stopNode drains this // same worker — and once, so repeated drain failures don't respawn it. + e.poisonCommits() e.logger.Crit("background commit failed; shutting down node", "block", gen.blockNum, "hash", gen.blockHash, "err", err) e.commitFatalOnce.Do(func() { go func() { @@ -273,6 +289,21 @@ func (e *ExecModule) genSuperseded(gen *commitGen) bool { return gen.epoch != e.genEpoch } +// poisonCommits marks the commit stream dead after a failed durable commit, so +// no queued or future generation commits its delta over the resulting hole. +func (e *ExecModule) poisonCommits() { + e.fgMu.Lock() + e.commitPoisoned = true + e.fgMu.Unlock() +} + +// commitIsPoisoned reports whether a durable commit has failed. +func (e *ExecModule) commitIsPoisoned() bool { + e.fgMu.Lock() + defer e.fgMu.Unlock() + return e.commitPoisoned +} + // addGen appends a new in-flight generation to the chain. func (e *ExecModule) addGen(gen *commitGen) { e.fgMu.Lock() diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index d9dbe1ed4b8..06ffd23f09f 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -250,6 +250,10 @@ type ExecModule struct { // commitFatalOnce ensures a failed durable commit triggers node shutdown at // most once, even as the worker drains further generations that also fail. commitFatalOnce sync.Once + // commitPoisoned (guarded by fgMu) is set when a durable commit fails: each + // generation flushes only its own delta, so a queued descendant must not + // commit over the hole left by a failed parent while shutdown propagates. + commitPoisoned bool // stateCache is a cache for state data (accounts, storage, code) stateCache *cache.StateCache @@ -354,18 +358,25 @@ func NewExecModule( // WaitIdle blocks until any in-flight updateForkChoice goroutine finishes. // Call before closing the database to avoid waitTxsAllDoneOnClose hangs. func (e *ExecModule) WaitIdle(ctx context.Context) { - // Best-effort foreground barrier; even if the ctx is cancelled we must still - // stop the worker below so it can never run against a closing DB. - if err := e.fgAcquire(ctx); err == nil { + if e.fgAcquire(ctx) == nil { + // Foreground is idle. Drain + commit any queued generations (the worker's + // stop path runs them), then release the generation chain now that no + // reader can be in flight — drainCommittedGens keeps the newest alive as + // Events.LatestSD until here. e.fgRelease() + e.stopCommitWorker() + e.closeAllGens() + return } - // Stop the background-commit worker and wait for it to drain its queue - // (including any in-flight commit tx) so DB-close does not race it. + // Timed out with a foreground op still holding the semaphore: closing the + // generation chain now would wipe SDs that op is still reading through, and + // parking its later commit in the stopped worker's channel would leak an open + // roTx and hang chainDB.Close. Poison so that late enqueueCommit rolls back + // instead of parking, stop the worker, leave the gens for the owning op (the + // process is exiting anyway), and log loudly. + e.poisonCommits() e.stopCommitWorker() - // Close the generation(s) still held — drainCommittedGens deliberately - // keeps the newest alive as Events.LatestSD; release it now that the - // worker has stopped and no foreground/RPC reader can be in flight. - e.closeAllGens() + e.logger.Warn("WaitIdle: foreground op still active at shutdown; not closing generations") } // stopCommitWorker signals the background-commit worker to drain and exit, diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 8fd2fd98775..af9b955f57a 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -1824,12 +1824,6 @@ func TestNotificationDispatchForegroundCommit(t *testing.T) { // test only processes the genesis → block 1 transition to verify that // notification dispatch works correctly in the background commit path. func TestNotificationDispatchBackgroundCommit(t *testing.T) { - // Background commit creates a race: FCU N returns before commit finishes, - // so FCU N+1 reads stale state from DB. This is the known limitation that - // the API-layer "latest head pointer" coordination is designed to solve. - // Once that's implemented, remove this skip and verify the full flow. - t.Skip("background commit requires API-layer coordination (latest head pointer) to work correctly") - m := execmoduletester.New(t, execmoduletester.WithFcuBackgroundCommit()) headerCh, unsub := m.Notifications.Events.AddHeaderSubscription() diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 5bb6482e0be..a191d9ad5da 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -793,6 +793,12 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // the pipeline Sync with the next FCU's RunLoop and opens its own RwTx, // which must not overlap either. if e.fcuBackgroundPrune { + // Settle pending updates + unwind state before handing the + // semaphore to the prune goroutine: a fast prune could otherwise + // fgRelease before the deferred cleanup runs, letting the next + // acquirer observe unsettled state. OnceFunc, so the outer defer + // is then a no-op. + cleanupBeforeSemaRelease() pruneHandedOff = true go func() { defer e.fgRelease() diff --git a/execution/execmodule/getters.go b/execution/execmodule/getters.go index c5ff7c54146..f696c0fb9a0 100644 --- a/execution/execmodule/getters.go +++ b/execution/execmodule/getters.go @@ -79,7 +79,19 @@ func (e *ExecModule) beginOverlayOrRo(ctx context.Context) (kv.TemporalTx, func( } view := sd.OverlayTemporalTx(roTx) e.lock.RUnlock() - return view, func() { roTx.Rollback() }, nil + if view != nil { + return view, func() { roTx.Rollback() }, nil + } + // The overlay was flushed to the DB (CloseBlockOverlay, e.g. a bulk + // InsertBlocks) between the check and the re-load, leaving no overlay + // in the chain. The flushed blocks are now committed, so a fresh roTx + // sees them; the one we opened predates the flush, so drop it. + roTx.Rollback() + tx, err := e.db.BeginTemporalRo(ctx) //nolint:gocritic + if err != nil { + return nil, nil, err + } + return tx, func() { tx.Rollback() }, nil } } e.lock.RUnlock()