From 12d67f65c0f1e55684bd8862f6e9fac52e0940be Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 2 Jul 2026 09:40:04 +0200 Subject: [PATCH 01/31] execution: read-ahead warmup must not clobber fresher StateCache entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warmBody prefetch goroutine reads committed (parent-block) state and Put it into the StateCache unconditionally. A laggard Put landing after the FCU flush's cache-apply replaced the flushed value with the pre-flush snapshot, stamped with the current epoch — permanently valid to the lazy unwind staleness check. The next block then executed against the stale value: wrong trie root / INVALID payload (the eest-spec-enginextests merge-queue flake first seen after #21386 merged). Fix in two layers: prefetch writes go through the new PutIfAbsent / PutCodeWithHashIfAbsent (live entries kept, stale ones replaced), so either interleaving converges to the authoritative value; and updateForkChoice drains in-flight warmup at entry — before the unwind epoch-bump and the flush cache-apply — subsuming the unwind-path-only drain. --- execution/cache/cache.go | 5 + execution/cache/cache_test.go | 85 +++++++++++++++ execution/cache/code_cache.go | 34 +++++- execution/cache/generic_cache.go | 15 +++ execution/cache/state_cache.go | 24 +++++ execution/exec/blocks_read_ahead.go | 7 +- execution/exec/blocks_read_ahead_test.go | 125 +++++++++++++++++++++++ execution/execmodule/exec_module.go | 5 +- execution/execmodule/forkchoice.go | 9 +- 9 files changed, 298 insertions(+), 11 deletions(-) create mode 100644 execution/exec/blocks_read_ahead_test.go diff --git a/execution/cache/cache.go b/execution/cache/cache.go index d797413c081..5fa38a6f4b8 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -30,6 +30,11 @@ type Cache interface { // reflects (used for txNum/epoch unwind invalidation). Put(key []byte, value []byte, txNum uint64) + // PutIfAbsent is Put except that a live entry for key is left untouched + // (a stale one is replaced) — for prefetch writers, whose snapshot may + // already be superseded by an authoritative Put. + PutIfAbsent(key []byte, value []byte, txNum uint64) + // Delete removes the data for the given key. Delete(key []byte) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 4587aa75440..dd4e1451a45 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/db/kv" ) @@ -771,3 +772,87 @@ func TestUnwind_FloorOnlyMovesDown(t *testing.T) { _, ok := c.Get(k) assert.False(t, ok, "deeper unwind's floor must not be raised by a later shallower one") } + +func TestDomainCache_PutIfAbsent(t *testing.T) { + c := NewDomainCacheMode(1*datasize.KB, ModeEvictLRU) + addr := makeAddr(1) + fresh := []byte("fresh") + stale := []byte("stale") + + // Absent → inserts. + c.PutIfAbsent(addr, stale, 10) + v, ok := c.Get(addr) + require.True(t, ok) + assert.Equal(t, stale, v) + + // Live entry → left untouched. + c.Put(addr, fresh, 20) + c.PutIfAbsent(addr, stale, 10) + v, ok = c.Get(addr) + require.True(t, ok) + assert.Equal(t, fresh, v, "PutIfAbsent must not replace a live entry") + + // Entry below the unwind floor survives the unwind and still blocks PutIfAbsent. + low := makeAddr(2) + c.Put(low, fresh, 3) + c.Unwind(5) + c.PutIfAbsent(low, stale, 4) + v, ok = c.Get(low) + require.True(t, ok) + assert.Equal(t, fresh, v) + + // Stale entry (at/above the floor, superseded epoch) → replaced. + c.PutIfAbsent(addr, stale, 10) // addr's entry was stamped txNum 20 >= floor 5 + v, ok = c.Get(addr) + require.True(t, ok) + assert.Equal(t, stale, v, "PutIfAbsent must replace a stale entry") +} + +func TestCodeCache_PutIfAbsentKeepsLiveAddrBinding(t *testing.T) { + cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + fresh := []byte{0xaa, 1, 2, 3} + stale := []byte{0xbb, 4, 5, 6} + + cc.PutIfAbsent(addr, stale, 10) + v, ok := cc.Get(addr) + require.True(t, ok) + assert.Equal(t, stale, v) + + cc.Put(addr, fresh, 20) + cc.PutIfAbsent(addr, stale, 10) + v, ok = cc.Get(addr) + require.True(t, ok) + assert.Equal(t, fresh, v, "PutIfAbsent must not rebind a live addr entry") + + // After an unwind marks the binding stale, PutIfAbsent may rebind. + cc.Unwind(5) + cc.PutIfAbsent(addr, stale, 4) + v, ok = cc.Get(addr) + require.True(t, ok) + assert.Equal(t, stale, v) +} + +func TestCodeCache_PutWithCodeHashIfAbsent(t *testing.T) { + cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + fresh := []byte{0xaa, 1, 2, 3} + stale := []byte{0xbb, 4, 5, 6} + freshHash := crypto.Keccak256(fresh) + staleHash := crypto.Keccak256(stale) + + cc.PutWithCodeHash(addr, fresh, freshHash, 20) + cc.PutWithCodeHashIfAbsent(addr, stale, staleHash, 10) + + v, ok := cc.Get(addr) + require.True(t, ok) + assert.Equal(t, fresh, v, "addr must stay bound to the fresher code") + + // The content-addressed layers are per-key-immutable and still populated. + v, ok = cc.GetByCodeHash(staleHash) + require.True(t, ok) + assert.Equal(t, stale, v) + size, ok := cc.GetCodeSizeByCodeHash(staleHash) + require.True(t, ok) + assert.Equal(t, len(stale), size) +} diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index f3bca5ae7c7..97774490721 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -310,7 +310,14 @@ func (c *CodeCache) GetWithTxNum(addr []byte) ([]byte, uint64, bool) { func (c *CodeCache) Put(addr []byte, code []byte, txNum uint64) { // No codeHash in hand here, so the entry is left unverified against maphash // collisions. The EVM read path uses PutWithCodeHash, which records it. - c.putCode(addr, code, [32]byte{}, txNum) + c.putCode(addr, code, [32]byte{}, txNum, true) +} + +// PutIfAbsent is Put except that a live addr→code binding is kept — for +// prefetch writers, whose snapshot may already be superseded. The +// content-addressed layers are per-key-immutable and skip live entries anyway. +func (c *CodeCache) PutIfAbsent(addr []byte, code []byte, txNum uint64) { + c.putCode(addr, code, [32]byte{}, txNum, false) } // putCode populates the addr→codeID and codeID→code layers. keyHash is the @@ -318,14 +325,22 @@ func (c *CodeCache) Put(addr []byte, code []byte, txNum uint64) { // a 64-bit maphash collision. Size is accounted only on the goroutine that // actually inserts (LoadOrStore is atomic), so concurrent Puts of the same // cold code can't both Add and permanently inflate codeSize. -func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum uint64) { +func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum uint64, overwriteAddr bool) { if len(code) == 0 { return } ep := c.coh.Epoch() codeID := maphash.Hash(code) - c.addrToHash.Add(common.BytesToAddress(addr), versionedAddressID{addrID: codeID, codeHash: keyHash, txNum: txNum, epoch: ep}) + a := common.BytesToAddress(addr) + bindAddr := overwriteAddr + if !bindAddr { + e, ok := c.addrToHash.Get(a) + bindAddr = !ok || c.isStale(e.txNum, e.epoch) + } + if bindAddr { + c.addrToHash.Add(a, versionedAddressID{addrID: codeID, codeHash: keyHash, txNum: txNum, epoch: ep}) + } hashKey := uint64AsBytes(&codeID) entry := codeEntry{code: code, keyHash: keyHash, txNum: txNum, epoch: ep} @@ -405,6 +420,17 @@ func (c *CodeCache) GetByCodeHash(codeHash []byte) ([]byte, bool) { // addr may be empty to populate only codeHashToCode (e.g. when populating from a // codehash-only path that hasn't seen the addr yet). func (c *CodeCache) PutWithCodeHash(addr []byte, code []byte, codeHash []byte, txNum uint64) { + c.putWithCodeHash(addr, code, codeHash, txNum, true) +} + +// PutWithCodeHashIfAbsent is PutWithCodeHash except that a live addr→code +// binding is kept — for prefetch writers, whose snapshot may already be +// superseded. +func (c *CodeCache) PutWithCodeHashIfAbsent(addr []byte, code []byte, codeHash []byte, txNum uint64) { + c.putWithCodeHash(addr, code, codeHash, txNum, false) +} + +func (c *CodeCache) putWithCodeHash(addr []byte, code []byte, codeHash []byte, txNum uint64, overwriteAddr bool) { if len(code) == 0 || len(codeHash) == 0 { return } @@ -412,7 +438,7 @@ func (c *CodeCache) PutWithCodeHash(addr []byte, code []byte, codeHash []byte, t kh := hash32(codeHash) if len(addr) > 0 { - c.putCode(addr, code, kh, txNum) + c.putCode(addr, code, kh, txNum, overwriteAddr) } // Populate the size-only layer alongside the bytes layer — every time diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index ce619376b6e..d0a81650610 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -192,6 +192,18 @@ func (c *GenericCache[T]) GetWithTxNum(key []byte) (T, uint64, bool) { // In ModeNoOp inserts that would overflow the byte budget are dropped // (and counted via the dropped metric). func (c *GenericCache[T]) Put(key []byte, value T, txNum uint64) { + c.put(key, value, txNum, true) +} + +// PutIfAbsent is Put except that a live entry for key is left untouched (a +// stale one is replaced). Prefetch writers must use this: they read an older +// snapshot, so an unconditional Put racing an authoritative one could pin +// superseded state in the cache. +func (c *GenericCache[T]) PutIfAbsent(key []byte, value T, txNum uint64) { + c.put(key, value, txNum, false) +} + +func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) { h := maphash.Hash(key) valBytes := c.sizeFunc(value) newSize := len(key) + valBytes + 24 @@ -203,6 +215,9 @@ func (c *GenericCache[T]) Put(key []byte, value T, txNum uint64) { // avoid an extra allocation; the freshly-decoded value replaces the // old one. if hasExisting && bytes.Equal(existing.key, key) { + if !overwrite && !c.coh.IsStale(existing.txNum, existing.epoch) { + return + } c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize - existing.size)) return diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index d94af694464..bc5a5bbf46d 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -165,6 +165,17 @@ func (c *StateCache) PutCodeWithHash(addr, code, codeHash []byte, txNum uint64) cc.PutWithCodeHash(addr, common.Copy(code), codeHash, txNum) } +// PutCodeWithHashIfAbsent is PutCodeWithHash except that a live addr→code +// binding is kept — for prefetch writers, whose snapshot may already be +// superseded. +func (c *StateCache) PutCodeWithHashIfAbsent(addr, code, codeHash []byte, txNum uint64) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutWithCodeHashIfAbsent(addr, common.Copy(code), codeHash, txNum) +} + // GetCodeSizeByHash returns the size of code by its Ethereum codeHash // without loading the bytes. Returns (0, false) when the size-only layer // is not populated for this hash. @@ -233,6 +244,19 @@ func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte, txNum uint6 cache.Put(key, common.Copy(value), txNum) } +// PutIfAbsent is Put except that a live entry is left untouched — for prefetch +// writers, whose snapshot may already be superseded by an authoritative Put. +func (c *StateCache) PutIfAbsent(domain kv.Domain, key []byte, value []byte, txNum uint64) { + cache := c.caches[domain] + if cache == nil { + return + } + if domain == kv.CommitmentDomain && bytes.Equal(key, commitmentdb.KeyCommitmentState) { + return + } + cache.PutIfAbsent(key, common.Copy(value), txNum) +} + // Delete removes the data for the given domain and key. func (c *StateCache) Delete(domain kv.Domain, key []byte) { cache := c.caches[domain] diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 438a1a7cefa..087c8693f5a 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -90,13 +90,16 @@ type cachePopulatingGetter struct { func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { v, step, err := cpg.g.GetLatest(name, k) if err == nil && cpg.sc != nil { + // If-absent writes only: this runs in a fire-and-forget goroutine over a + // committed snapshot, so an unconditional Put racing an FCU flush's + // cache-apply could replace the flushed value with the pre-flush one. if name == kv.CodeDomain && len(v) > 0 { // Key the content cache by the code's OWN hash, never a separately // read account codeHash: under parallel/speculative exec that hash // can be skewed or cross-account, and a (hash, code) pair that // doesn't satisfy keccak(code)==hash poisons every account sharing // the hash. keccak(v) makes each entry self-consistent. - cpg.sc.PutCodeWithHash(k, v, crypto.Keccak256(v), (uint64(step)+1)*cpg.stepSize-1) + cpg.sc.PutCodeWithHashIfAbsent(k, v, crypto.Keccak256(v), (uint64(step)+1)*cpg.stepSize-1) } else { // Cache including nil/empty results: a probe returning no // bytes is a valid negative answer (missing account, empty @@ -105,7 +108,7 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // { account: None, status: LoadedNotExisting } pattern. // Stamp with an upper bound on the value's write txNum (last txNum // of the step it came from) so unwind invalidation is correct. - cpg.sc.Put(name, k, v, (uint64(step)+1)*cpg.stepSize-1) + cpg.sc.PutIfAbsent(name, k, v, (uint64(step)+1)*cpg.stepSize-1) } } return v, step, err diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go new file mode 100644 index 00000000000..f3a88e4d15c --- /dev/null +++ b/execution/exec/blocks_read_ahead_test.go @@ -0,0 +1,125 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package exec + +import ( + "testing" + + "github.com/c2h5oh/datasize" + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/execution/cache" +) + +// stubTemporalGetter stands in for the committed-state snapshot a warmup +// goroutine reads: every GetLatest returns the same fixed value. +type stubTemporalGetter struct { + v []byte + step kv.Step +} + +func (s stubTemporalGetter) GetLatest(kv.Domain, []byte) ([]byte, kv.Step, error) { + return s.v, s.step, nil +} + +func (s stubTemporalGetter) HasPrefix(kv.Domain, []byte) ([]byte, []byte, bool, error) { + return nil, nil, false, nil +} + +func (s stubTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } + +func newTestStateCache() *cache.StateCache { + b := 1 * datasize.MB + return cache.NewStateCache(b, b, b, b) +} + +// A warmup read-through must never replace a fresher entry an authoritative +// writer (the FCU flush cache-apply) has already put: the warmup reads a +// pre-flush snapshot, so a laggard Put landing after the flush would pin stale +// state in the cache and corrupt the next block's execution. +func TestCachePopulatingGetterKeepsFresherEntry(t *testing.T) { + key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") + fresh := []byte("account-record-nonce-5") + stale := []byte("account-record-nonce-4") + for _, domain := range []kv.Domain{kv.AccountsDomain, kv.StorageDomain} { + sc := newTestStateCache() + sc.Put(domain, key, fresh, 54) + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: stale}, sc: sc, stepSize: 1_562_500} + + v, _, err := cpg.GetLatest(domain, key) + require.NoError(t, err) + require.Equal(t, stale, v, "read-through must still return the snapshot value") + + got, ok := sc.Get(domain, key) + require.True(t, ok, "domain %s", domain) + require.Equal(t, fresh, got, "domain %s: warmup must not clobber the fresher entry", domain) + } +} + +// Same invariant for the code addr→code binding, which is rebound when an +// account's code changes and is therefore just as clobber-able as accounts. +func TestCachePopulatingGetterKeepsFresherCodeBinding(t *testing.T) { + addr := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") + freshCode := []byte{0xaa, 0x01, 0x02, 0x03} + staleCode := []byte{0xbb, 0x04, 0x05, 0x06} + sc := newTestStateCache() + sc.PutCodeWithHash(addr, freshCode, crypto.Keccak256(freshCode), 54) + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: staleCode}, sc: sc, stepSize: 1_562_500} + + _, _, err := cpg.GetLatest(kv.CodeDomain, addr) + require.NoError(t, err) + + got, ok := sc.Get(kv.CodeDomain, addr) + require.True(t, ok) + require.Equal(t, freshCode, got, "warmup must not rebind addr to older code") +} + +// Cold keys must still be warmed — that is the prefetcher's purpose. +func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { + key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") + val := []byte("account-record") + code := []byte{0xaa, 0x01, 0x02, 0x03} + + for _, domain := range []kv.Domain{kv.AccountsDomain, kv.StorageDomain} { + sc := newTestStateCache() + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: val}, sc: sc, stepSize: 1_562_500} + _, _, err := cpg.GetLatest(domain, key) + require.NoError(t, err) + got, ok := sc.Get(domain, key) + require.True(t, ok, "domain %s", domain) + require.Equal(t, val, got, "domain %s", domain) + } + + sc := newTestStateCache() + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: code}, sc: sc, stepSize: 1_562_500} + _, _, err := cpg.GetLatest(kv.CodeDomain, key) + require.NoError(t, err) + got, ok := sc.Get(kv.CodeDomain, key) + require.True(t, ok) + require.Equal(t, code, got) + + // Negative results (missing account, empty slot) are cached as nil hits. + sc = newTestStateCache() + cpg = &cachePopulatingGetter{g: stubTemporalGetter{v: nil}, sc: sc, stepSize: 1_562_500} + _, _, err = cpg.GetLatest(kv.AccountsDomain, key) + require.NoError(t, err) + got, ok = sc.Get(kv.AccountsDomain, key) + require.True(t, ok) + require.Empty(t, got) +} diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 988dde84ae5..869b7fa8569 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -383,8 +383,9 @@ func (e *ExecModule) canonicalHash(ctx context.Context, tx kv.Tx, blockNumber ui // warmBody is fire-and-forget and populates the shared state/branch caches; if // it is still running when an unwind bumps the cache epoch, it can Put a // pre-unwind (dead-fork) value stamped with the post-unwind epoch — IsStale then -// returns false and the stale value is served as canonical (wrong root). Call -// before every UnwindTo that invalidates the cache. +// returns false and the stale value is served as canonical (wrong root). A +// laggard Put can likewise land after a flush's cache-apply and pin the +// pre-flush snapshot. Call before any unwind epoch-bump or flush cache-apply. func (e *ExecModule) drainReadAhead() { if e.readAheader == nil { return diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 3cd70394975..b266451e159 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -179,6 +179,12 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa }) defer cleanupBeforeSemaRelease() + // Drain any warmup a preceding newPayload spawned: its Puts reflect a + // pre-FCU snapshot and must land before this FCU's unwind epoch-bump and + // flush cache-apply, not after them (no new warmup starts while we hold + // the semaphore). + e.drainReadAhead() + var validationError string type canonicalEntry struct { hash common.Hash @@ -410,9 +416,6 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa }, false) } - // Drain in-flight warmup before the unwind bumps the cache epoch - // (cross-fork contamination — see drainReadAhead). - e.drainReadAhead() if err := e.pipelineExecutor.UnwindTo(unwindTarget, stagedsync.ForkChoice, tx); err != nil { return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, err, false) } From e7cbf189be43345dbe4d53af552ffe36324b0c72 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 2 Jul 2026 09:59:14 +0200 Subject: [PATCH 02/31] execution/cache: make PutIfAbsent atomic w.r.t. concurrent Put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The if-absent check+insert was Get-then-Add on a per-call-locked LRU, so a conditional writer could check (absent), lose the CPU to an authoritative Put, then clobber it. Serialize same-key writers: striped mutexes in GenericCache, a bind mutex around the CodeCache addr-to-code binding. Red-tested by hammering a single key with a concurrent Put/PutIfAbsent pair — pre-fix the stale value won within a few thousand rounds. --- execution/cache/cache_test.go | 23 +++++++++++++++++++ execution/cache/code_cache.go | 7 ++++++ .../cache/code_cache_concurrency_test.go | 22 ++++++++++++++++++ execution/cache/generic_cache.go | 13 +++++++++++ 4 files changed, 65 insertions(+) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index dd4e1451a45..ac1ed13b1c8 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -18,6 +18,7 @@ package cache import ( "bytes" + "sync" "testing" "github.com/c2h5oh/datasize" @@ -856,3 +857,25 @@ func TestCodeCache_PutWithCodeHashIfAbsent(t *testing.T) { require.True(t, ok) assert.Equal(t, len(stale), size) } + +// A conditional put must be atomic w.r.t. a concurrent unconditional Put of +// the same key: without a shared critical section the conditional writer can +// check (absent), lose the CPU to the authoritative writer's insert, then +// clobber it — the prefetch-vs-flush staleness this cache guards against. +func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { + c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) + addr := makeAddr(1) + fresh := []byte("fresh") + stale := []byte("stale") + for round := 0; round < 20000; round++ { + c.Delete(addr) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); c.Put(addr, fresh, 20) }() + go func() { defer wg.Done(); c.PutIfAbsent(addr, stale, 10) }() + wg.Wait() + v, ok := c.Get(addr) + require.True(t, ok) + require.Equal(t, fresh, v, "round %d: PutIfAbsent raced past a concurrent Put", round) + } +} diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 97774490721..97dba2e6c00 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -17,6 +17,7 @@ package cache import ( + "sync" "sync/atomic" "unsafe" @@ -150,6 +151,10 @@ type CodeCache struct { // the unwind floor. See execution/cache/coherence. coh coherence.Gen + // addrBindMu serializes addr→code binding writers so PutIfAbsent's + // check+bind is atomic w.r.t. a concurrent authoritative rebind. + addrBindMu sync.Mutex + // Stats counters (atomic for concurrent access) addrHits atomic.Uint64 addrMisses atomic.Uint64 @@ -333,6 +338,7 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui codeID := maphash.Hash(code) a := common.BytesToAddress(addr) + c.addrBindMu.Lock() bindAddr := overwriteAddr if !bindAddr { e, ok := c.addrToHash.Get(a) @@ -341,6 +347,7 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui if bindAddr { c.addrToHash.Add(a, versionedAddressID{addrID: codeID, codeHash: keyHash, txNum: txNum, epoch: ep}) } + c.addrBindMu.Unlock() hashKey := uint64AsBytes(&codeID) entry := codeEntry{code: code, keyHash: keyHash, txNum: txNum, epoch: ep} diff --git a/execution/cache/code_cache_concurrency_test.go b/execution/cache/code_cache_concurrency_test.go index 8d28219694c..089c5f05b88 100644 --- a/execution/cache/code_cache_concurrency_test.go +++ b/execution/cache/code_cache_concurrency_test.go @@ -118,3 +118,25 @@ func TestCodeCache_ConcurrentDistinctPuts_RespectCap(t *testing.T) { require.GreaterOrEqual(t, cc.codeHashCodeSize.Load(), int64(0), "codeHashToCode size must stay non-negative (no double back-out)") } + +// Same atomicity requirement for the addr→code binding: a concurrent +// authoritative Put must win over a conditional prefetch put in every +// interleaving. +func TestCodeCache_PutIfAbsentAtomicWithPut(t *testing.T) { + cc := NewCodeCache(64*datasize.MB, 16*datasize.MB) + addr := make([]byte, 20) + addr[0] = 0xcd + fresh := []byte{0xaa, 1, 2, 3} + stale := []byte{0xbb, 4, 5, 6} + for round := 0; round < 20000; round++ { + cc.Delete(addr) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); cc.Put(addr, fresh, 20) }() + go func() { defer wg.Done(); cc.PutIfAbsent(addr, stale, 10) }() + wg.Wait() + v, ok := cc.Get(addr) + require.True(t, ok) + require.Equal(t, fresh, v, "round %d: PutIfAbsent raced past a concurrent Put", round) + } +} diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index d0a81650610..1b0820c3149 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -18,6 +18,7 @@ package cache import ( "bytes" + "sync" "sync/atomic" "github.com/c2h5oh/datasize" @@ -29,6 +30,10 @@ import ( "github.com/erigontech/erigon/execution/cache/coherence" ) +// putStripeCount sizes the same-key write-serialization stripes; power of two +// so the stripe index is a mask of the key hash. +const putStripeCount = 256 + // avgBytesPerEntry is the assumption used to translate a byte budget into // the entry-count cap that freelru.ShardedLRU is sized against. 256 B // approximates account-record + key overhead and storage-slot value+key @@ -64,6 +69,10 @@ type GenericCache[T any] struct { // floor. See execution/cache/coherence. coh coherence.Gen + // putStripes serialize same-key writers so PutIfAbsent's check+insert is + // atomic w.r.t. a concurrent Put (freelru offers no conditional insert). + putStripes [putStripeCount]sync.Mutex + hits atomic.Uint64 misses atomic.Uint64 inserts atomic.Uint64 @@ -209,6 +218,10 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) newSize := len(key) + valBytes + 24 ep := c.coh.Epoch() + mu := &c.putStripes[h&(putStripeCount-1)] + mu.Lock() + defer mu.Unlock() + existing, hasExisting := c.data.Get(h) // Existing key — update in place. Reuse the stored key buffer to From d60a1f8d755fceff5e7b3085ca71b85e51c7112c Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 2 Jul 2026 12:04:05 +0200 Subject: [PATCH 03/31] execution/cache: dedup StateCache put wrappers, trim repeated PutIfAbsent rationale Route Put/PutIfAbsent and PutCodeWithHash/PutCodeWithHashIfAbsent through shared private cores so the nil-cache and KeyCommitmentState guards live in one place, matching the GenericCache/CodeCache shape. State the if-absent rationale once on Cache.PutIfAbsent; the other sites keep terse pointers. --- execution/cache/code_cache.go | 10 +++----- execution/cache/generic_cache.go | 6 ++--- execution/cache/state_cache.go | 43 +++++++++++++++++--------------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 97dba2e6c00..802038db1b9 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -318,9 +318,8 @@ func (c *CodeCache) Put(addr []byte, code []byte, txNum uint64) { c.putCode(addr, code, [32]byte{}, txNum, true) } -// PutIfAbsent is Put except that a live addr→code binding is kept — for -// prefetch writers, whose snapshot may already be superseded. The -// content-addressed layers are per-key-immutable and skip live entries anyway. +// PutIfAbsent implements Cache.PutIfAbsent for the addr→code binding; the +// content-addressed layers skip live entries regardless. func (c *CodeCache) PutIfAbsent(addr []byte, code []byte, txNum uint64) { c.putCode(addr, code, [32]byte{}, txNum, false) } @@ -430,9 +429,8 @@ func (c *CodeCache) PutWithCodeHash(addr []byte, code []byte, codeHash []byte, t c.putWithCodeHash(addr, code, codeHash, txNum, true) } -// PutWithCodeHashIfAbsent is PutWithCodeHash except that a live addr→code -// binding is kept — for prefetch writers, whose snapshot may already be -// superseded. +// PutWithCodeHashIfAbsent is PutWithCodeHash with if-absent binding semantics +// (see Cache.PutIfAbsent). func (c *CodeCache) PutWithCodeHashIfAbsent(addr []byte, code []byte, codeHash []byte, txNum uint64) { c.putWithCodeHash(addr, code, codeHash, txNum, false) } diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 1b0820c3149..86610d793b8 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -204,10 +204,8 @@ func (c *GenericCache[T]) Put(key []byte, value T, txNum uint64) { c.put(key, value, txNum, true) } -// PutIfAbsent is Put except that a live entry for key is left untouched (a -// stale one is replaced). Prefetch writers must use this: they read an older -// snapshot, so an unconditional Put racing an authoritative one could pin -// superseded state in the cache. +// PutIfAbsent implements Cache.PutIfAbsent (live entry kept, stale one +// replaced). func (c *GenericCache[T]) PutIfAbsent(key []byte, value T, txNum uint64) { c.put(key, value, txNum, false) } diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index bc5a5bbf46d..8be0258ca50 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -158,22 +158,25 @@ func (c *StateCache) GetCodeByHash(codeHash []byte) ([]byte, bool) { // codeHash-keyed codeHashToCode layer. Callers should prefer this over Put when they // have the codeHash from the account record — avoids a redundant keccak. func (c *StateCache) PutCodeWithHash(addr, code, codeHash []byte, txNum uint64) { - cc, ok := c.caches[kv.CodeDomain].(*CodeCache) - if !ok { - return - } - cc.PutWithCodeHash(addr, common.Copy(code), codeHash, txNum) + c.putCodeWithHash(addr, code, codeHash, txNum, true) } -// PutCodeWithHashIfAbsent is PutCodeWithHash except that a live addr→code -// binding is kept — for prefetch writers, whose snapshot may already be -// superseded. +// PutCodeWithHashIfAbsent is PutCodeWithHash with if-absent binding semantics +// (see Cache.PutIfAbsent). func (c *StateCache) PutCodeWithHashIfAbsent(addr, code, codeHash []byte, txNum uint64) { + c.putCodeWithHash(addr, code, codeHash, txNum, false) +} + +func (c *StateCache) putCodeWithHash(addr, code, codeHash []byte, txNum uint64, overwrite bool) { cc, ok := c.caches[kv.CodeDomain].(*CodeCache) if !ok { return } - cc.PutWithCodeHashIfAbsent(addr, common.Copy(code), codeHash, txNum) + if overwrite { + cc.PutWithCodeHash(addr, common.Copy(code), codeHash, txNum) + } else { + cc.PutWithCodeHashIfAbsent(addr, common.Copy(code), codeHash, txNum) + } } // GetCodeSizeByHash returns the size of code by its Ethereum codeHash @@ -234,19 +237,15 @@ func (c *StateCache) DeleteAddrCodeHash(addr []byte) { // Put stores data for the given domain and key, stamped with the txNum the // value reflects (for txNum/epoch unwind invalidation). func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte, txNum uint64) { - cache := c.caches[domain] - if cache == nil { - return - } - if domain == kv.CommitmentDomain && bytes.Equal(key, commitmentdb.KeyCommitmentState) { - return - } - cache.Put(key, common.Copy(value), txNum) + c.put(domain, key, value, txNum, true) } -// PutIfAbsent is Put except that a live entry is left untouched — for prefetch -// writers, whose snapshot may already be superseded by an authoritative Put. +// PutIfAbsent is Put with if-absent semantics (see Cache.PutIfAbsent). func (c *StateCache) PutIfAbsent(domain kv.Domain, key []byte, value []byte, txNum uint64) { + c.put(domain, key, value, txNum, false) +} + +func (c *StateCache) put(domain kv.Domain, key []byte, value []byte, txNum uint64, overwrite bool) { cache := c.caches[domain] if cache == nil { return @@ -254,7 +253,11 @@ func (c *StateCache) PutIfAbsent(domain kv.Domain, key []byte, value []byte, txN if domain == kv.CommitmentDomain && bytes.Equal(key, commitmentdb.KeyCommitmentState) { return } - cache.PutIfAbsent(key, common.Copy(value), txNum) + if overwrite { + cache.Put(key, common.Copy(value), txNum) + } else { + cache.PutIfAbsent(key, common.Copy(value), txNum) + } } // Delete removes the data for the given domain and key. From 2de03de0ef8b557a38d11dbe1af681bf3d6a51fd Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 2 Jul 2026 12:16:14 +0200 Subject: [PATCH 04/31] =?UTF-8?q?execution/exec:=20fix=20negative-caching?= =?UTF-8?q?=20comment=20=E2=80=94=20empty=20code=20is=20not=20cached?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeCache.putCode drops zero-length puts, so the comment's 'no code' claim was wrong (flagged by Copilot review; the wording predates this PR). --- execution/exec/blocks_read_ahead.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 087c8693f5a..57eb3a6fdb0 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -103,7 +103,8 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k } else { // Cache including nil/empty results: a probe returning no // bytes is a valid negative answer (missing account, empty - // storage slot, no code) and caching it lets repeated probes + // storage slot; empty code lands here too but CodeCache drops + // zero-length puts) and caching it lets repeated probes // skip the file accessor stack. Mirrors revm's CacheAccount // { account: None, status: LoadedNotExisting } pattern. // Stamp with an upper bound on the value's write txNum (last txNum From f1a99b5b5615b5a191dda6022549cf2bc3a537d7 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:19:08 +0200 Subject: [PATCH 05/31] execution/types/accounts: add targeted DeserialiseV3CodeHash extractor Extracts just the codeHash field from a SerialiseV3-encoded account record, skipping the full decode (balance parse, codeHash interning) that DeserialiseV3 pays. Bounds-checked against truncated records and faithful to CodeHash.IsEmpty for non-canonical records that spell out the empty/zero sentinel. Groundwork for #22120 finding 7: SharedDomains.codeHashForAddr runs this per mem-hit on the codeHash fast path. --- execution/types/accounts/account.go | 32 ++++++++++ execution/types/accounts/account_test.go | 80 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/execution/types/accounts/account.go b/execution/types/accounts/account.go index 69378516720..8b83e5f6dfd 100644 --- a/execution/types/accounts/account.go +++ b/execution/types/accounts/account.go @@ -17,6 +17,7 @@ package accounts import ( + "bytes" "fmt" "io" "math/bits" @@ -26,6 +27,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/empty" + "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/execution/rlp" ) @@ -641,6 +643,36 @@ func DeserialiseV3(a *Account, enc []byte) error { return nil } +// DeserialiseV3CodeHash extracts just the codeHash field from a +// SerialiseV3-encoded account, skipping the full decode (balance parse, +// codeHash interning) that DeserialiseV3 pays. Returns a subslice of enc — +// valid only while enc is — or nil for a malformed record or an account +// without code (including a non-canonical record spelling out the empty or +// zero sentinel, which CodeHash.IsEmpty treats as no-code). +func DeserialiseV3CodeHash(enc []byte) []byte { + pos := 0 + for range 2 { // skip the length-prefixed nonce and balance fields + if pos >= len(enc) { + return nil + } + pos += 1 + int(enc[pos]) + } + if pos >= len(enc) { + return nil + } + codeHashBytes := int(enc[pos]) + pos++ + if codeHashBytes != length.Hash || pos+codeHashBytes > len(enc) { + return nil + } + h := enc[pos : pos+codeHashBytes] + var zero common.Hash + if bytes.Equal(h, zero[:]) || bytes.Equal(h, empty.CodeHash[:]) { + return nil + } + return h +} + func SerialiseV3(a *Account) []byte { var l int l++ diff --git a/execution/types/accounts/account_test.go b/execution/types/accounts/account_test.go index eb25a8a8e64..57885f5a08f 100644 --- a/execution/types/accounts/account_test.go +++ b/execution/types/accounts/account_test.go @@ -17,6 +17,7 @@ package accounts import ( + "bytes" "testing" "github.com/holiman/uint256" @@ -76,6 +77,85 @@ func TestEmptyAccount_BufferStrangeBehaviour(t *testing.T) { isIncarnationEqual(t, a.Incarnation, decodedAcc.Incarnation) } +func TestDeserialiseV3CodeHash(t *testing.T) { + t.Parallel() + balances := []uint256.Int{{}, *uint256.NewInt(1), *uint256.NewInt(1e18), *new(uint256.Int).Lsh(uint256.NewInt(1), 200)} + nonces := []uint64{0, 1, 255, 1 << 40} + codeHashes := []CodeHash{EmptyCodeHash, InternCodeHash(common.BytesToHash(crypto.Keccak256([]byte{1, 2, 3})))} + incarnations := []uint64{0, 7} + + for _, nonce := range nonces { + for i := range balances { + for _, ch := range codeHashes { + for _, inc := range incarnations { + a := Account{Nonce: nonce, Balance: balances[i], CodeHash: ch, Incarnation: inc} + enc := SerialiseV3(&a) + + var full Account + if err := DeserialiseV3(&full, enc); err != nil { + t.Fatal(err) + } + got := DeserialiseV3CodeHash(enc) + if full.CodeHash.IsEmpty() { + if got != nil { + t.Fatalf("empty codeHash must extract as nil, got %x (acc %+v)", got, a) + } + } else { + want := full.CodeHash.Value() + if !bytes.Equal(got, want[:]) { + t.Fatalf("extracted %x, want %x (acc %+v)", got, want, a) + } + } + } + } + } + } +} + +func TestDeserialiseV3CodeHashMalformed(t *testing.T) { + t.Parallel() + a := Account{ + Nonce: 255, + Balance: *uint256.NewInt(1e18), + CodeHash: InternCodeHash(common.BytesToHash(crypto.Keccak256([]byte{1, 2, 3}))), + Incarnation: 4, + } + enc := SerialiseV3(&a) + // [1+nonce][1+balance][1+codeHash]... — the codeHash field is complete at: + codeHashEnd := 1 + int(enc[0]) + 1 + codeHashEnd += int(enc[codeHashEnd-1]) + 1 + codeHashEnd += int(enc[codeHashEnd-1]) + // Any truncation cutting into (or before) the codeHash must yield nil, + // never an out-of-bounds read; beyond it the codeHash is extractable. + for cut := 0; cut <= len(enc); cut++ { + got := DeserialiseV3CodeHash(enc[:cut]) + if cut < codeHashEnd && got != nil { + t.Fatalf("cut=%d (codeHash complete at %d): expected nil, got %x", cut, codeHashEnd, got) + } + if cut >= codeHashEnd && got == nil { + t.Fatalf("cut=%d (codeHash complete at %d): expected hash, got nil", cut, codeHashEnd) + } + } + if got := DeserialiseV3CodeHash(nil); got != nil { + t.Fatalf("nil input: expected nil, got %x", got) + } + // A record claiming a non-32-byte codeHash is malformed for extraction. + odd := append([]byte{0, 0, 31}, make([]byte, 31)...) + if got := DeserialiseV3CodeHash(odd); got != nil { + t.Fatalf("non-32-byte codeHash field: expected nil, got %x", got) + } + // Non-canonical records spelling out the no-code sentinels (canonical + // SerialiseV3 writes length 0 instead) must extract as nil, matching + // CodeHash.IsEmpty. + for _, sentinel := range [][]byte{make([]byte, 32), empty.CodeHash[:]} { + rec := append([]byte{0, 0, 32}, sentinel...) + rec = append(rec, 0) + if got := DeserialiseV3CodeHash(rec); got != nil { + t.Fatalf("sentinel codeHash %x: expected nil, got %x", sentinel, got) + } + } +} + func TestAccountEncodeWithCode(t *testing.T) { t.Parallel() a := Account{ From 5121282f774ace782b56079f4984b911d628829b Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:19:24 +0200 Subject: [PATCH 06/31] execution/cache: stripe Delete/stale-drop, liveness pre-checks, warmup-in-flight assert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three of the #22120 findings against the cache package itself: Finding 11: GenericCache.Delete and the lazy stale-drop inside GetWithTxNum removed entries outside the put stripes, so a removal interleaved between put's data.Get and data.Add double-subtracted the displaced entry's size (once via freelru's OnEvict, once via put's update delta) — currentSize drifted one entry per occurrence and never healed. Both removal paths now take the key's stripe, with a re-check under lock. Hammer tests reproduced the drift (exactly one entry size) before the fix and run race-clean after. Finding 12: new ContainsLive probes (Peek-based — no hit/miss counters, no LRU recency) let conditional writers skip preparing a put that a live entry would no-op: StateCache.put skips the value copy, and StateCache.HasLiveCode lets the read-ahead prefetcher skip the keccak+copy for an already-bound address. Advisory only — PutIfAbsent still decides under the key's stripe. Enforcement (issue recommendation A, cheap variant): StateCache gains a warmup-in-flight gauge (WarmupStarted/WarmupDone); Unwind panics under ASSERT_STATE_CACHE when a cache-populating warmup is still in flight, converting the drain-before-epoch-bump convention into a loud failure. The drain-free getter remains tracked in #22116. --- execution/cache/cache.go | 5 ++ execution/cache/cache_test.go | 101 +++++++++++++++++++++++++++++++ execution/cache/code_cache.go | 16 +++++ execution/cache/generic_cache.go | 31 +++++++++- execution/cache/state_cache.go | 43 ++++++++++++- 5 files changed, 192 insertions(+), 4 deletions(-) diff --git a/execution/cache/cache.go b/execution/cache/cache.go index 5fa38a6f4b8..45b17be1aba 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -35,6 +35,11 @@ type Cache interface { // already be superseded by an authoritative Put. PutIfAbsent(key []byte, value []byte, txNum uint64) + // ContainsLive reports whether key has a live (non-stale) entry, without + // touching hit/miss counters or LRU recency. A pre-check for conditional + // writers to skip preparing a put a live entry would no-op; advisory only. + ContainsLive(key []byte) bool + // Delete removes the data for the given key. Delete(key []byte) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index ac1ed13b1c8..fa2745e66e6 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -27,6 +27,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/db/kv" ) @@ -879,3 +880,103 @@ func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { require.Equal(t, fresh, v, "round %d: PutIfAbsent raced past a concurrent Put", round) } } + +// A Delete racing an update-in-place put must not double-subtract the +// displaced entry's size: freelru's OnEvict subtracts it for the Remove, and +// put's update delta subtracts it again unless the two writers share the +// key's stripe. +func TestDomainCache_DeleteAtomicWithPut_NoSizeDrift(t *testing.T) { + c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) + addr := makeAddr(1) + v1 := []byte("value-one") + v2 := []byte("value-two") + for round := 0; round < 20000; round++ { + c.Put(addr, v1, 10) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); c.Put(addr, v2, 20) }() + go func() { defer wg.Done(); c.Delete(addr) }() + wg.Wait() + c.Delete(addr) + require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) + } +} + +// Same invariant for the lazy stale-drop inside GetWithTxNum, the other +// unstriped Remove path. +func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { + c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) + addr := makeAddr(1) + v1 := []byte("value-one") + v2 := []byte("value-two") + for round := 0; round < 20000; round++ { + c.Put(addr, v1, 10) + c.Unwind(5) // epoch bump makes the entry above stale (txNum 10 >= floor 5) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); c.Put(addr, v2, 20) }() + go func() { defer wg.Done(); c.Get(addr) }() + wg.Wait() + c.Delete(addr) + require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) + } +} + +func TestDomainCache_ContainsLive(t *testing.T) { + c := NewDomainCacheMode(1*datasize.KB, ModeEvictLRU) + addr := makeAddr(1) + require.False(t, c.ContainsLive(addr), "absent key") + + c.Put(addr, []byte("v"), 10) + require.True(t, c.ContainsLive(addr)) + + c.Unwind(5) // entry txNum 10 >= floor 5, superseded epoch → stale + require.False(t, c.ContainsLive(addr), "stale entry must not read as live") + + // The probe is passive: the stale entry is left for PutIfAbsent to replace. + c.PutIfAbsent(addr, []byte("w"), 4) + v, ok := c.Get(addr) + require.True(t, ok) + assert.Equal(t, []byte("w"), v) + require.True(t, c.ContainsLive(addr)) +} + +func TestCodeCache_ContainsLive(t *testing.T) { + cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + code := []byte{0xaa, 1, 2, 3} + require.False(t, cc.ContainsLive(addr), "absent addr") + + cc.PutWithCodeHash(addr, code, crypto.Keccak256(code), 10) + require.True(t, cc.ContainsLive(addr)) + + cc.Unwind(5) + require.False(t, cc.ContainsLive(addr), "stale binding must not read as live") + + // A bound addr whose content layer refused the bytes (capacity) is not live. + tiny := NewCodeCache(2*datasize.B, 1*datasize.MB) + tiny.PutWithCodeHash(addr, code, crypto.Keccak256(code), 10) + require.False(t, tiny.ContainsLive(addr), "binding without content bytes is not servable") +} + +// The drain-before-unwind convention (drainReadAhead ordered before every +// epoch bump) is enforced here: a cache-populating warmup still in flight at +// Unwind time can stamp a dead-fork value with the post-unwind epoch. +func TestStateCache_UnwindAssertsWarmupInFlight(t *testing.T) { + old := dbg.AssertStateCache + dbg.AssertStateCache = true + t.Cleanup(func() { dbg.AssertStateCache = old }) + + b := 1 * datasize.MB + sc := NewStateCache(b, b, b, b) + sc.WarmupStarted() + require.Panics(t, func() { sc.Unwind(10) }, "epoch bump with a warmup in flight must fail loud") + sc.WarmupDone() + require.NotPanics(t, func() { sc.Unwind(10) }) + + // Without the assert flag the gauge is inert. + dbg.AssertStateCache = false + sc.WarmupStarted() + defer sc.WarmupDone() + require.NotPanics(t, func() { sc.Unwind(10) }) +} diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 802038db1b9..4f879422b93 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -355,6 +355,22 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui &c.coh, &c.codeSize, 8, int64(c.codeCapacityB)) } +// ContainsLive reports whether addr resolves to live code bytes through the +// addr→code binding, without touching hit/miss counters or LRU recency. +// Prefetchers probe it to skip the keccak+copy work of preparing a conditional +// put that a live binding would no-op; advisory only. +func (c *CodeCache) ContainsLive(addr []byte) bool { + vID, ok := c.addrToHash.Peek(common.BytesToAddress(addr)) + if !ok || c.isStale(vID.txNum, vID.epoch) { + return false + } + ce, ok := c.hashToCode.Get(uint64AsBytes(&vID.addrID)) + if !ok || len(ce.code) == 0 || c.isStale(ce.txNum, ce.epoch) { + return false + } + return vID.codeHash == ([32]byte{}) || ce.keyHash == vID.codeHash +} + // GetAddrCodeHash returns the Ethereum codeHash for addr if cached. Lets // SharedDomains.codeHashForAddr skip a cold AccountsDomain read when the // EVM-known codeHash is already known. Eviction is LRU; freshly seen addrs diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 86610d793b8..6fd748ed6a6 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -186,7 +186,7 @@ func (c *GenericCache[T]) GetWithTxNum(key []byte) (T, uint64, bool) { // tx — and must be dropped; >= not > (the surviving block's last txNum is // floor-1, so this never drops a live entry). if c.coh.IsStale(e.txNum, e.epoch) { - c.data.Remove(h) + c.dropStale(h, key) c.staleEvicted.Add(1) c.misses.Add(1) var zero T @@ -267,14 +267,41 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) c.inserts.Add(1) } -// Delete removes the data for the given key. +// Delete removes the data for the given key. Runs under the key's put stripe: +// an unstriped Remove racing put's read-modify-write would double-subtract the +// displaced entry's size (once via OnEvict, once via put's update delta). func (c *GenericCache[T]) Delete(key []byte) { h := maphash.Hash(key) + mu := &c.putStripes[h&(putStripeCount-1)] + mu.Lock() + defer mu.Unlock() if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { c.data.Remove(h) } } +// dropStale removes key's entry under its put stripe, re-checking that it is +// still the same key's stale entry — a concurrent put may have replaced it +// with a live one, and Remove must be striped for the same reason Delete is. +func (c *GenericCache[T]) dropStale(h uint64, key []byte) { + mu := &c.putStripes[h&(putStripeCount-1)] + mu.Lock() + defer mu.Unlock() + if e, ok := c.data.Get(h); ok && bytes.Equal(e.key, key) && c.coh.IsStale(e.txNum, e.epoch) { + c.data.Remove(h) + } +} + +// ContainsLive reports whether key has a live (non-stale) entry, without +// touching hit/miss counters or LRU recency. Prefetchers probe it to skip the +// copy work of preparing a conditional put that a live entry would no-op; +// advisory only — PutIfAbsent re-decides under the key's stripe. +func (c *GenericCache[T]) ContainsLive(key []byte) bool { + h := maphash.Hash(key) + e, ok := c.data.Peek(h) + return ok && bytes.Equal(e.key, key) && !c.coh.IsStale(e.txNum, e.epoch) +} + // Clear removes all entries from the cache. It also resets the (epoch, // unwindFloor) coherence pair: with no entries left, no stale (txNum, epoch) // can survive, so a fresh floor keeps subsequent Puts at the live epoch diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 8be0258ca50..8d0ac23830e 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -18,7 +18,9 @@ package cache import ( "bytes" + "fmt" "strings" + "sync/atomic" "github.com/c2h5oh/datasize" @@ -53,6 +55,12 @@ const ( // Code uses CodeCache (two-level for deduplication). type StateCache struct { caches [kv.DomainLen]Cache + + // warmupsInFlight counts fire-and-forget cache-populating prefetches + // (WarmupStarted/WarmupDone). Unwind asserts it is zero: a prefetch put + // racing the epoch bump could stamp a dead-fork value with the post-unwind + // epoch and have it served as canonical. + warmupsInFlight atomic.Int64 } // NewStateCache creates a new StateCache with the specified byte capacities. @@ -179,6 +187,15 @@ func (c *StateCache) putCodeWithHash(addr, code, codeHash []byte, txNum uint64, } } +// HasLiveCode reports whether addr resolves to live code bytes through the +// code cache's addr→code binding, without touching stats or LRU recency. +// Prefetchers probe it to skip the keccak+copy of preparing a conditional +// code put that a live binding would no-op; advisory only. +func (c *StateCache) HasLiveCode(addr []byte) bool { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + return ok && cc.ContainsLive(addr) +} + // GetCodeSizeByHash returns the size of code by its Ethereum codeHash // without loading the bytes. Returns (0, false) when the size-only layer // is not populated for this hash. @@ -255,9 +272,14 @@ func (c *StateCache) put(domain kv.Domain, key []byte, value []byte, txNum uint6 } if overwrite { cache.Put(key, common.Copy(value), txNum) - } else { - cache.PutIfAbsent(key, common.Copy(value), txNum) + return } + // Skip the copy a live entry would discard; advisory — PutIfAbsent + // re-decides under the key's stripe. + if cache.ContainsLive(key) { + return + } + cache.PutIfAbsent(key, common.Copy(value), txNum) } // Delete removes the data for the given domain and key. @@ -283,7 +305,15 @@ func (c *StateCache) Clear() { // GenericCaches and the CodeCache, all layers) bumps an epoch + lowers a floor // and drops stale entries lazily on read. This is the sole cache-invalidation // path on unwind — the executor never touches the cache during forward execution. +// +// Callers must drain any in-flight cache-populating warmup first (see +// WarmupStarted); the assert converts that convention into a loud failure. func (c *StateCache) Unwind(unwindToTxNum uint64) { + if dbg.AssertStateCache { + if n := c.warmupsInFlight.Load(); n != 0 { + panic(fmt.Sprintf("StateCache.Unwind with %d cache-populating warmup(s) in flight — missing drain before the epoch bump", n)) + } + } for _, cache := range c.caches { if cache != nil { cache.Unwind(unwindToTxNum) @@ -291,6 +321,15 @@ func (c *StateCache) Unwind(unwindToTxNum uint64) { } } +// WarmupStarted and WarmupDone bracket a fire-and-forget cache-populating +// prefetch. A prefetch put racing an unwind's epoch bump could stamp a +// dead-fork value with the post-unwind epoch, so Unwind asserts (under +// ASSERT_STATE_CACHE) that no warmup is in flight. +func (c *StateCache) WarmupStarted() { c.warmupsInFlight.Add(1) } + +// WarmupDone is the counterpart of WarmupStarted. +func (c *StateCache) WarmupDone() { c.warmupsInFlight.Add(-1) } + // GetCache returns the cache for the given domain. // Returns nil if the domain is not supported. func (c *StateCache) GetCache(domain kv.Domain) Cache { From ec75f7aa29696ffc40e5e4d2a543a57c89295c01 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:19:38 +0200 Subject: [PATCH 07/31] execution/exec: skip live-binding prefetch work, stamp negatives with domain progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-ahead warmup follow-ups from #22120: Finding 12: with if-absent semantics a live entry makes the conditional put a no-op, but the prefetcher had already paid the keccak over the full bytecode plus the value copy by then — the steady-state warm prefetch allocated and discarded ~1-3 MB of hash+copy work per block. The code branch now probes HasLiveCode before preparing the put; the accounts/storage copy elision lives in StateCache.put (previous commit). A live binding makes the prefetch a full no-op: it no longer populates content layers for its superseded snapshot code either. Latent-note fix: negative results (missing account, empty slot) carry no step to derive an unwind bound from, and the synthetic step-0 stamp ((0+1)*stepSize-1) sat below every realistic unwind floor, making cached negatives immortal. Correctness relied on the flush-callback overwrite always firing. Negatives are now stamped with the domain's progress at observation time (max committed txNum in the read snapshot), so they drop on any unwind that could matter instead of outliving the fact they cache. Also brackets the fire-and-forget warmBody goroutine with the new WarmupStarted/WarmupDone gauge; the release is ordered before warmWg.Done so a WaitForWarmup return implies the gauge is back to zero. --- execution/exec/blocks_read_ahead.go | 45 +++++++++++++++++------ execution/exec/blocks_read_ahead_test.go | 47 ++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 57eb3a6fdb0..fd145d1f64e 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -85,6 +85,10 @@ type cachePopulatingGetter struct { g kv.TemporalGetter sc *cache.StateCache stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) + // progress returns the domain's max committed txNum in the read snapshot; + // it stamps negative results, whose miss carries no step to derive a + // bound from. + progress func(kv.Domain) uint64 } func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { @@ -94,12 +98,16 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // committed snapshot, so an unconditional Put racing an FCU flush's // cache-apply could replace the flushed value with the pre-flush one. if name == kv.CodeDomain && len(v) > 0 { - // Key the content cache by the code's OWN hash, never a separately - // read account codeHash: under parallel/speculative exec that hash - // can be skewed or cross-account, and a (hash, code) pair that - // doesn't satisfy keccak(code)==hash poisons every account sharing - // the hash. keccak(v) makes each entry self-consistent. - cpg.sc.PutCodeWithHashIfAbsent(k, v, crypto.Keccak256(v), (uint64(step)+1)*cpg.stepSize-1) + // A live binding makes the conditional put a no-op — skip before + // paying the keccak+copy below. + if !cpg.sc.HasLiveCode(k) { + // Key the content cache by the code's OWN hash, never a separately + // read account codeHash: under parallel/speculative exec that hash + // can be skewed or cross-account, and a (hash, code) pair that + // doesn't satisfy keccak(code)==hash poisons every account sharing + // the hash. keccak(v) makes each entry self-consistent. + cpg.sc.PutCodeWithHashIfAbsent(k, v, crypto.Keccak256(v), (uint64(step)+1)*cpg.stepSize-1) + } } else { // Cache including nil/empty results: a probe returning no // bytes is a valid negative answer (missing account, empty @@ -107,9 +115,15 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // zero-length puts) and caching it lets repeated probes // skip the file accessor stack. Mirrors revm's CacheAccount // { account: None, status: LoadedNotExisting } pattern. - // Stamp with an upper bound on the value's write txNum (last txNum - // of the step it came from) so unwind invalidation is correct. - cpg.sc.PutIfAbsent(name, k, v, (uint64(step)+1)*cpg.stepSize-1) + // Stamp with an upper bound on the value's write txNum: the last + // txNum of the step it came from, or for a negative — which has no + // step — the domain's progress at observation time, so the entry + // drops on any unwind instead of outliving the fact it caches. + txNum := (uint64(step)+1)*cpg.stepSize - 1 + if len(v) == 0 { + txNum = cpg.progress(name) + } + cpg.sc.PutIfAbsent(name, k, v, txNum) } } return v, step, err @@ -132,9 +146,18 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h if !bra.warming.CompareAndSwap(false, true) { return } + // Registered before the goroutine spawns and released before warmWg.Done + // (defers run LIFO), so a WaitForWarmup return implies the gauge is back + // to zero — StateCache.Unwind asserts on it. + if bra.stateCache != nil { + bra.stateCache.WarmupStarted() + } bra.warmWg.Add(1) go func() { defer bra.warmWg.Done() + if bra.stateCache != nil { + defer bra.stateCache.WarmupDone() + } bra.warmBody(ctx, db, header, body, 8) // use 8 workers for warming }() } @@ -229,7 +252,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t } var getter kv.TemporalGetter = ttx if bra.stateCache != nil { - getter = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} + getter = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize(), progress: ttx.Debug().DomainProgress} } stateReader := state.NewReaderV3(getter) @@ -300,7 +323,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize(), progress: ttx.Debug().DomainProgress} getter = cpg } stateReader := state.NewReaderV3(getter) diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index f3a88e4d15c..71543c6fac7 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -98,7 +98,7 @@ func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { for _, domain := range []kv.Domain{kv.AccountsDomain, kv.StorageDomain} { sc := newTestStateCache() - cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: val}, sc: sc, stepSize: 1_562_500} + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: val}, sc: sc, stepSize: 1_562_500, progress: zeroProgress} _, _, err := cpg.GetLatest(domain, key) require.NoError(t, err) got, ok := sc.Get(domain, key) @@ -107,7 +107,7 @@ func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { } sc := newTestStateCache() - cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: code}, sc: sc, stepSize: 1_562_500} + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: code}, sc: sc, stepSize: 1_562_500, progress: zeroProgress} _, _, err := cpg.GetLatest(kv.CodeDomain, key) require.NoError(t, err) got, ok := sc.Get(kv.CodeDomain, key) @@ -116,10 +116,51 @@ func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { // Negative results (missing account, empty slot) are cached as nil hits. sc = newTestStateCache() - cpg = &cachePopulatingGetter{g: stubTemporalGetter{v: nil}, sc: sc, stepSize: 1_562_500} + cpg = &cachePopulatingGetter{g: stubTemporalGetter{v: nil}, sc: sc, stepSize: 1_562_500, progress: zeroProgress} _, _, err = cpg.GetLatest(kv.AccountsDomain, key) require.NoError(t, err) got, ok = sc.Get(kv.AccountsDomain, key) require.True(t, ok) require.Empty(t, got) } + +// With a live addr binding the prefetch must be a full no-op: not even the +// content layers may be populated for its (superseded) snapshot code, because +// the liveness pre-check exists to skip the keccak+copy for that code +// entirely. +func TestCachePopulatingGetterSkipsContentForLiveBinding(t *testing.T) { + addr := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") + freshCode := []byte{0xaa, 0x01, 0x02, 0x03} + staleCode := []byte{0xbb, 0x04, 0x05, 0x06} + sc := newTestStateCache() + sc.PutCodeWithHash(addr, freshCode, crypto.Keccak256(freshCode), 54) + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: staleCode}, sc: sc, stepSize: 1_562_500, progress: zeroProgress} + + _, _, err := cpg.GetLatest(kv.CodeDomain, addr) + require.NoError(t, err) + + _, ok := sc.GetCodeByHash(crypto.Keccak256(staleCode)) + require.False(t, ok, "live binding: prefetch must not populate content for the snapshot code") +} + +// Negative results are stamped with the domain's progress at observation time, +// not a synthetic step-0 bound — a synthetic stamp far below any real unwind +// floor would make the negative immortal. +func TestCachePopulatingGetterNegativeDropsOnUnwind(t *testing.T) { + key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") + sc := newTestStateCache() + cpg := &cachePopulatingGetter{ + g: stubTemporalGetter{v: nil}, sc: sc, stepSize: 1_562_500, + progress: func(kv.Domain) uint64 { return 10_000_000 }, + } + _, _, err := cpg.GetLatest(kv.AccountsDomain, key) + require.NoError(t, err) + _, ok := sc.Get(kv.AccountsDomain, key) + require.True(t, ok) + + sc.Unwind(5_000_000) + _, ok = sc.Get(kv.AccountsDomain, key) + require.False(t, ok, "a negative observed at txNum 10M must not survive an unwind to 5M") +} + +func zeroProgress(kv.Domain) uint64 { return 0 } From 5a143b8624000d50e9d38f82aaaf74215e874490 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:19:54 +0200 Subject: [PATCH 08/31] db/state/execctx: if-absent read-fill, in-flight-unwind assert fix, fold maxStep gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SharedDomains half of the #22120 findings: Finding 9: the getLatestMetered read-fill was a second unconditional-Put snapshot writer — an embedded-RPC eth_call straddling an FCU commit (or a bounded read during an in-flight unwind) could overwrite the flush-applied value with the pre-flush one, the same clobber shape #22146 fixes for the warmup. Read-fills never carry newer information than a flush-apply, so both fills (domain values and code) now use the if-absent puts. Finding 4: the ASSERT_STATE_CACHE divergence check compared a cache hit against a DB read bounded by the same maxStep the mem overlay published for a key an in-flight unwind re-bound. In that window MDBX still holds the not-yet-deleted dying row inside the bound, so the authoritative read returns dead-fork bytes and the assert panicked on a legitimate below-floor hit. The assert now runs only when no per-key bound is active. (The Value==nil delete-only diff that produces this signal comes from legacy V0-format persisted changesets; current V1 diffs are served from mem directly — see DomainRoTx.unwind.) Finding 7: the stateCache and branchCache maxStep gates were two copies of the same rule differing only in a subtle divide/don't-divide unit conversion; both now call one servableUnderBound helper carrying the shared rationale, with the unit conversion explicit at each site. codeHashForAddr's per-mem-hit full account decode is replaced by the targeted DeserialiseV3CodeHash extractor. Finding 5: ClearBranchCache and DetachBranchCache had no callers, and DetachBranchCache's docstring advertised a fork-validation guard that was never wired — correctness rests on the epoch-bumping sd.Unwind, which every unwind path already funnels through. Deleted rather than wired: detaching would cost fork validation its warm branch cache. ProbeReadLayers is kept — it gains a caller in #22154. Latent-note fix (SD side): negative read-fills are stamped with the domain's progress instead of the synthetic step-0 bound, mirroring the warmup getter. --- db/state/execctx/domain_shared.go | 116 ++++------ db/state/execctx/statecache_readfill_test.go | 211 +++++++++++++++++++ 2 files changed, 256 insertions(+), 71 deletions(-) create mode 100644 db/state/execctx/statecache_readfill_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index ea123c5a60d..0db8245ce57 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1025,30 +1025,6 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun return nil } -// ClearBranchCache empties the aggregator-scope commitment BranchCache. -// Use after operations that mutate commitment state outside the normal -// sd.Flush callback path — notably SetHead unwind, which truncates -// commitment domain history but does not re-write all the keys whose -// cached values are now stale. Without this call, the next FCU sees -// the pre-unwind KeyCommitmentState and trips ErrBehindCommitment. -func (sd *SharedDomains) ClearBranchCache() { - if sd.branchCache != nil { - sd.branchCache.Clear() - } -} - -// DetachBranchCache makes this SharedDomains ignore the aggregator-scope -// BranchCache: commitment branch reads go straight to sd.mem/overlay/MDBX and -// no read populates the shared cache. Used for fork-validation SDs, which read -// transient fork state — sharing the canonical BranchCache let them read stale -// committed branches (wrong trie root → INVALID payload) and pollute the cache -// with fork-transient branches. Only sd.branchCache (the read/populate path, -// domain_shared GetLatest) is consulted, so nil-ing it fully detaches; the -// canonical SDs keep their warm cache. -func (sd *SharedDomains) DetachBranchCache() { - sd.branchCache = nil -} - // TemporalDomain satisfaction. Collects no read metrics — see // temporalGetter.GetLatest for why there is no process-wide accumulator. func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { @@ -1064,6 +1040,18 @@ func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, return sd.getLatestMetered(domain, tx, k, kvmetrics.MetricsFromContext(ctx)) } +// servableUnderBound is the shared cache gate for reads of a key an in-flight +// unwind has re-bound: the mem overlay publishes a per-key maxStep while MDBX +// still holds the dying rows, and a cached entry reflecting a step above that +// bound would diverge from the bounded read the cache-disabled path takes. +// The (txNum, epoch) floor usually already drops such entries; this keeps the +// two read paths identical regardless. Callers convert their unit first — the +// StateCache stamps txNums (divide by step size), the BranchCache stores step +// indices (no divide). +func servableUnderBound(cStep, maxStep kv.Step) bool { + return cStep <= maxStep +} + // getLatestMetered is the read implementation. wm is the caller's lock-free // per-task/per-worker metrics accumulator (nil disables metrics for the call). // No global metrics lock is taken on this hot path — accumulators are combined @@ -1126,16 +1114,9 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k // that haven't been flushed to DB yet. Early return keeps correctness AND performance. if sd.stateCache != nil { v, cTxNum, ok := sd.stateCache.GetWithTxNum(domain, k) + // The cache stamps txNums — divide to get the step the entry reflects. cStep := kv.Step(cTxNum / sd.StepSize()) - // Respect maxStep, mirroring the BranchCache gate below. sd.mem / - // sd.parent.mem lowered maxStep above when an in-flight unwind re-bound - // this key to an earlier step (the per-key unwindChangeset signal). A - // cached entry from a higher step would diverge from the (maxStep-bounded) - // DB read the cache-disabled path takes, so treat it as a miss and fall - // through; the Put below refreshes it. For direct domains the (txNum,epoch) - // floor in Get usually already drops such entries — this keeps the two - // read paths identical regardless. - if ok && cStep > maxStep { + if ok && !servableUnderBound(cStep, maxStep) { ok = false } if dbg.KVReadLevelledMetrics { @@ -1146,7 +1127,11 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k } } if ok { - if dbg.AssertStateCache { + // The divergence assert is skipped while the mem overlay bounds this + // key (in-flight unwind): MDBX still holds the not-yet-deleted dying + // rows inside the bound, so the "authoritative" read can return + // dead-fork bytes and blame the cache for a legitimate hit. + if dbg.AssertStateCache && maxStep == kv.Step(math.MaxUint64) { // Fetch authoritative value from the backing tx and panic on any divergence. // sd.mem and sd.parent.mem were already checked above and missed, so the // backing tx is the single source of truth for this key at this point. @@ -1180,18 +1165,11 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k // with a newer MDBX state. if domain == kv.CommitmentDomain && sd.branchCache != nil { if cv, cStepU64, ok := sd.branchCache.Get(k); ok { - // Respect maxStep. sd.mem / sd.parent.mem lowered maxStep above when an - // unwound key reports its restored step via unwindChangeset (the per-key - // in-mem unwind signal). The cache's global epoch/floor is coarser than - // that per-key signal, so a cached entry below the floor can still belong - // to a step the unwind re-bound away. Serving it then diverges from the - // (maxStep-bounded) DB read the cache-disabled path takes. Fall through to - // the bounded read in that case; the Put below refreshes the entry. // Get returns the on-disk step index directly — do NOT divide by // StepSize (that double-division collapsed cStep to ~0, defeating the // gate). cStep := kv.Step(cStepU64) - if cStep <= maxStep { + if servableUnderBound(cStep, maxStep) { return cv, cStep, nil } } @@ -1212,11 +1190,17 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k return nil, 0, fmt.Errorf("storage %x read error: %w", k, err) } - // Populate state cache on successful read. Stamp with an upper bound on the - // value's write txNum (the read only gives us the file/step it came from): - // the last txNum of that step. An unwind below this bound can't leave the - // value stale, so frozen-step reads stay warm across unwinds while - // recent-step reads are dropped. + // Populate state cache on successful read, with if-absent semantics: a + // read-fill never carries newer information than a flush-apply, and an + // unconditional Put racing one (an embedded-RPC read straddling an FCU + // commit, or a bounded read during an in-flight unwind) would overwrite + // the fresher value. Stamp with an upper bound on the value's write txNum + // (the read only gives us the file/step it came from): the last txNum of + // that step. An unwind below this bound can't leave the value stale, so + // frozen-step reads stay warm across unwinds while recent-step reads are + // dropped. A negative result carries no step — stamp it with the domain's + // progress at observation time so it drops on any unwind instead of + // outliving the fact it caches. if sd.stateCache != nil { readTxNum := (uint64(step)+1)*sd.StepSize() - 1 if domain == kv.CodeDomain { @@ -1224,14 +1208,16 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k // This SD getter is the single place that populates the code cache // on a read. Key the content-addressed entry by the code's OWN hash, // keccak(v) — NEVER a separately-read account codeHash, which under - // parallel exec can be a skewed/cross-account value and would poison - // the shared codeHash->code map for every account sharing that hash. - // keccak(v) makes every cached entry self-consistent, so a skewed - // account read can never produce a bad entry. - sd.stateCache.PutCodeWithHash(k, v, crypto.Keccak256(v), readTxNum) + // parallel exec can be a skewed or cross-account value and would + // poison the shared codeHash→code map for every account sharing + // that hash. + sd.stateCache.PutCodeWithHashIfAbsent(k, v, crypto.Keccak256(v), readTxNum) } } else { - sd.stateCache.Put(domain, k, v, readTxNum) + if len(v) == 0 && sd.stateCache.GetCache(domain) != nil { + readTxNum = tx.Debug().DomainProgress(domain) + } + sd.stateCache.PutIfAbsent(domain, k, v, readTxNum) } } // Only cache a branch when the read's txN is known: a txN=0 entry would @@ -1411,25 +1397,13 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte, txNum ui } // decodeAccountCodeHash extracts the codeHash from an account's encoded -// (DecodeForStorage) bytes. Returns nil on decode error or when the account -// has no code (empty codeHash). +// bytes. Returns nil on a malformed record or when the account has no code. +// AccountsDomain values are SerialiseV3-encoded — DecodeForStorage is the +// legacy MDBX bitmask format with an incompatible layout that would silently +// misparse. The targeted extractor skips the full decode (balance parse, +// interning) this runs per mem-hit on the codeHash fast path. func decodeAccountCodeHash(enc []byte) []byte { - if len(enc) == 0 { - return nil - } - var acc accounts.Account - // AccountsDomain values are SerialiseV3-encoded, so they must be decoded - // with DeserialiseV3. DecodeForStorage is the legacy MDBX bitmask format - // with an incompatible binary layout; applied to V3 bytes it silently - // misparses and leaves CodeHash empty. - if err := accounts.DeserialiseV3(&acc, enc); err != nil { - return nil - } - if acc.CodeHash.IsEmpty() { - return nil - } - h := acc.CodeHash.Value() - return h[:] + return accounts.DeserialiseV3CodeHash(enc) } func (sd *SharedDomains) Metrics() *kvmetrics.DomainMetrics { diff --git a/db/state/execctx/statecache_readfill_test.go b/db/state/execctx/statecache_readfill_test.go new file mode 100644 index 00000000000..7209a1cac21 --- /dev/null +++ b/db/state/execctx/statecache_readfill_test.go @@ -0,0 +1,211 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execctx_test + +import ( + "encoding/binary" + "testing" + + "github.com/c2h5oh/datasize" + "github.com/holiman/uint256" + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/dbg" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" + "github.com/erigontech/erigon/execution/cache" + "github.com/erigontech/erigon/execution/types/accounts" +) + +func encAccount(nonce uint64) []byte { + a := accounts.Account{Nonce: nonce, Balance: *uint256.NewInt(nonce * 1000)} + return accounts.SerialiseV3(&a) +} + +// twoStepRows commits two versions of one account key so MDBX holds rows at +// step 0 (txNum 5, v1) and step 1 (txNum 20, v2), and returns a delete-only +// unwind diff for the step-1 row — the legacy-changeset shape that makes the +// mem overlay publish a per-key maxStep bound while MDBX still holds the +// dying row. +func twoStepRows(t *testing.T, db kv.TemporalRwDB, sc *cache.StateCache) (key, v1, v2 []byte, diffs [kv.DomainLen][]kv.DomainEntryDiff) { + t.Helper() + ctx := t.Context() + key = make([]byte, 20) + key[0] = 0xaa + v1, v2 = encAccount(1), encAccount(2) + + rwTx, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx.Rollback() + + sd, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) + require.NoError(t, err) + defer sd.Close() + sd.SetStateCacheForTest(sc) + + sd.SetTxNum(5) + require.NoError(t, sd.DomainPut(kv.AccountsDomain, rwTx, key, v1, 5, nil)) + sd.SetTxNum(20) + require.NoError(t, sd.DomainPut(kv.AccountsDomain, rwTx, key, v2, 20, v1)) + require.NoError(t, sd.Commit(ctx, rwTx)) + + stepBytes := make([]byte, 8) + binary.BigEndian.PutUint64(stepBytes, ^uint64(1)) + diffs[kv.AccountsDomain] = []kv.DomainEntryDiff{{Key: string(key) + string(stepBytes), Value: nil}} + return key, v1, v2, diffs +} + +func newSmallStateCache() *cache.StateCache { + b := 1 * datasize.MB + return cache.NewStateCache(b, b, b, b) +} + +// During an in-flight unwind the mem overlay bounds reads of an affected key +// by maxStep while MDBX still holds the not-yet-deleted dying row inside that +// bound. A cache hit legitimately below the unwind floor then diverges from +// the maxStep-bounded DB read, and the ASSERT_STATE_CACHE comparison must not +// blame the cache for it. +func TestAssertStateCache_NoFalsePanicDuringInFlightUnwind(t *testing.T) { + if testing.Short() { + t.Skip() + } + // Mutates dbg.AssertStateCache — must not run in parallel with tests that + // read it on the SD read path. + + const stepSize = uint64(16) + ctx := t.Context() + db := newTestDb(t, stepSize) + sc := newSmallStateCache() + key, v1, _, diffs := twoStepRows(t, db, sc) + + roTx, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTx.Rollback() + + sd, err := execctx.NewSharedDomains(ctx, roTx, log.New()) + require.NoError(t, err) + defer sd.Close() + sd.SetStateCacheForTest(sc) + + // A live cache entry below the unwind floor: the restored (correct) value. + sc.Put(kv.AccountsDomain, key, v1, 5) + sd.Unwind(10, &diffs) // in-flight: mem publishes maxStep=1; MDBX still holds the step-1 row + + old := dbg.AssertStateCache + dbg.AssertStateCache = true + t.Cleanup(func() { dbg.AssertStateCache = old }) + + var v []byte + require.NotPanics(t, func() { + v, _, err = sd.GetLatest(kv.AccountsDomain, roTx, key) + }, "assert must not fire on a legitimately-bounded cache hit during an in-flight unwind") + require.NoError(t, err) + require.Equal(t, v1, v, "the cache serves the restored value") +} + +// The read-fill after a fall-through read must not replace a live cache +// entry: it never carries newer information than a flush-apply, and during an +// in-flight unwind the bounded DB read can even return the not-yet-deleted +// dying row. +func TestReadFill_DoesNotClobberLiveEntry(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + + const stepSize = uint64(16) + ctx := t.Context() + db := newTestDb(t, stepSize) + sc := newSmallStateCache() + key, _, v2, diffs := twoStepRows(t, db, sc) + + roTx, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTx.Rollback() + + sd, err := execctx.NewSharedDomains(ctx, roTx, log.New()) + require.NoError(t, err) + defer sd.Close() + sd.SetStateCacheForTest(sc) + + sd.Unwind(10, &diffs) + // A live (current-epoch) entry above the read bound: the maxStep gate turns + // the hit into a miss, so the read falls through to the bounded DB read. + v3 := encAccount(3) + sc.Put(kv.AccountsDomain, key, v3, 40) + + v, _, err := sd.GetLatest(kv.AccountsDomain, roTx, key) + require.NoError(t, err) + require.Equal(t, v2, v, "fall-through read serves the maxStep-bounded DB row") + + got, ok := sc.Get(kv.AccountsDomain, key) + require.True(t, ok) + require.Equal(t, v3, got, "read-fill must not clobber the live entry") +} + +// Negative results (missing account) must be stamped with the domain's +// progress at observation time, not a synthetic step-0 bound that survives +// every unwind. +func TestReadFill_NegativeStampedWithProgress(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + + const stepSize = uint64(16) + ctx := t.Context() + db := newTestDb(t, stepSize) + sc := newSmallStateCache() + + rwTx, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx.Rollback() + + sd, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) + require.NoError(t, err) + defer sd.Close() + sd.SetStateCacheForTest(sc) + + written := make([]byte, 20) + written[0] = 0x01 + sd.SetTxNum(100) + require.NoError(t, sd.DomainPut(kv.AccountsDomain, rwTx, written, encAccount(7), 100, nil)) + require.NoError(t, sd.Commit(ctx, rwTx)) + + roTx, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTx.Rollback() + sd2, err := execctx.NewSharedDomains(ctx, roTx, log.New()) + require.NoError(t, err) + defer sd2.Close() + sd2.SetStateCacheForTest(sc) + + missing := make([]byte, 20) + missing[0] = 0x02 + v, _, err := sd2.GetLatest(kv.AccountsDomain, roTx, missing) + require.NoError(t, err) + require.Empty(t, v) + _, ok := sc.Get(kv.AccountsDomain, missing) + require.True(t, ok, "the negative result must be cached") + + // The domain's progress is 100 (the committed write), so any unwind at or + // below it must drop the negative instead of letting it outlive the fact. + sc.Unwind(50) + _, ok = sc.Get(kv.AccountsDomain, missing) + require.False(t, ok, "a negative observed at progress 100 must not survive an unwind to 50") +} From 2fe8e68083a6f106bae91f73b4767b72504b869e Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:20:04 +0200 Subject: [PATCH 09/31] execution/execmodule: drain and clear the state cache before ProcessFrozenBlocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 10 of #22120: engine servers are live before ExecModule.Start, and ValidateChain fires the read-ahead warmup before its too-far-away check, so a payload validated in the pre-Start window warms the cache with pre-catchup state — nil negatives included. Frozen-block processing advances state without touching the cache (its SDs are never wired to it) and neither clears nor epoch-bumps, so those entries stayed live through the whole catch-up and were served cache-before-aggTx afterwards: stale value, wrong root until restart. Start now drains any in-flight warmup and clears the cache under the semaphore before frozen-block processing; no new warmup can start until Start releases it. No isolated unit test: reproducing needs a live engine server racing PFB startup in the ms-scale pre-Start window. The change composes two already-tested primitives (drainReadAhead, StateCache.Clear) at a point where the semaphore excludes concurrent writers. --- execution/execmodule/exec_module.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 869b7fa8569..40f49474e27 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -680,6 +680,16 @@ func (e *ExecModule) Start(ctx context.Context, hook *stageloop.Hook) { } defer e.semaphore.Release(1) + // Engine servers are live before Start, so an early payload validation may + // already have warmed the state cache with pre-catchup state. Frozen-block + // processing advances state without touching the cache (its SDs are not + // wired to it), so such entries would be served stale afterwards — drain + // any in-flight warmup and clear before it runs. + e.drainReadAhead() + if e.stateCache != nil { + e.stateCache.Clear() + } + if err := e.pipelineExecutor.ProcessFrozenBlocks(ctx, hook, e.onlySnapDownloadOnStart); err != nil { if !errors.Is(err, context.Canceled) { e.logger.Error("Could not start execution service", "err", err) From 4f99e3806b143253a1161534be034468aca41c52 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:18:19 +0200 Subject: [PATCH 10/31] execution/exec: raise the warmup gauge only after warmWg.Add A WaitForWarmup landing between WarmupStarted and warmWg.Add saw a zero wg counter and returned while the gauge was already raised, so a drain-then-bump caller could trip the ASSERT_STATE_CACHE panic in StateCache.Unwind on a correctly-drained warmup. Today's callers cannot interleave there (the module semaphore excludes AddHeaderAndBody from every drain-then-bump path), but the assert exists to be trustworthy when that convention is violated, so its wiring must not manufacture false positives on its own. Raising the gauge after Add restores the invariant from both sides: a Wait either returns before Add (gauge still zero) or blocks until Done (gauge dropped first, defers run LIFO). Flagged by Copilot on #22159. --- execution/exec/blocks_read_ahead.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index fd145d1f64e..fa57bac532e 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -146,13 +146,15 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h if !bra.warming.CompareAndSwap(false, true) { return } - // Registered before the goroutine spawns and released before warmWg.Done - // (defers run LIFO), so a WaitForWarmup return implies the gauge is back - // to zero — StateCache.Unwind asserts on it. + // Gauge ordering makes "WaitForWarmup returned ⟹ gauge is zero" hold on + // its own: WarmupStarted only after warmWg.Add, so a Wait can't slip + // between them and return with the gauge raised; WarmupDone before + // warmWg.Done (defers run LIFO), so a Wait can't return before the + // gauge drops. StateCache.Unwind asserts on the gauge. + bra.warmWg.Add(1) if bra.stateCache != nil { bra.stateCache.WarmupStarted() } - bra.warmWg.Add(1) go func() { defer bra.warmWg.Done() if bra.stateCache != nil { From 464dde17928651f9c866afd451a8823994b2fd86 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 10:02:47 +0200 Subject: [PATCH 11/31] execution/cache: swap-then-migrate jump-grow so striped writes survive a resize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit maybeGrow copied entries and then swapped generations, so a striped writer that loaded the old generation before the swap landed its write in the abandoned copy — the migrated (older) value resurfaced as live: a stale serve, not the documented benign miss. Growth now publishes the new generation first, fences the put stripes, and migrates each key under its stripe with if-absent semantics; the grow trigger moves out of put's striped section (the fence would self-deadlock). growLRU keeps the old order: its content-addressed values are immutable per key, so a lost copy there really is a benign miss. --- execution/cache/generic_cache.go | 47 +++++++++--- .../cache/generic_cache_concurrency_test.go | 75 +++++++++++++++++++ 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index b1f4ede49c2..4655229817d 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -60,8 +60,10 @@ type entry[T any] struct { // data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { // data is the sharded LRU, replaced wholesale on a jump-grow. Load it once per - // operation; a write racing a resize may land in the LRU about to be replaced - // and be dropped — a benign miss (the value is re-read from the domain). + // operation, inside the stripe for writes: maybeGrow swaps generations first + // and then migrates under the same stripes, so a striped write is never + // silently undone by the migration (an update lost that way would resurface + // the older value as live — a stale serve, not a benign miss). data atomic.Pointer[freelru.ShardedLRU[uint64, entry[T]]] capacityB datasize.ByteSize mode Mode @@ -196,7 +198,16 @@ func (c *GenericCache[T]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, // maybeGrow jump-resizes the LRU one step larger when it is full, the ceiling // hasn't been reached, and the shared envelope can fund the step. Otherwise the -// LRU keeps its size and freelru evicts within it. Called with no lock held. +// LRU keeps its size and freelru evicts within it. Must be called with no +// stripe held (it sweeps every stripe). +// +// Swap first, then migrate: entries copied before the swap could be updated in +// the old generation by a concurrent striped writer, and the stale copy would +// resurface as live. After the swap, the stripe sweep fences the writers that +// loaded the old generation before it, freezing the old generation; migration +// then moves each key under its stripe, if-absent, so it never clobbers a +// fresher post-swap write. Until a key is migrated, reads of it miss — that +// (and only that) is the benign transient of a resize. func (c *GenericCache[T]) maybeGrow() { c.resizeMu.Lock() defer c.resizeMu.Unlock() @@ -215,12 +226,24 @@ func (c *GenericCache[T]) maybeGrow() { return } next := c.newShards(newCap) + c.data.Store(next) + for i := range c.putStripes { + c.putStripes[i].Lock() + c.putStripes[i].Unlock() //nolint:gocritic,staticcheck // empty critical section is the writer fence + } for _, k := range old.Keys() { + mu := &c.putStripes[k&(putStripeCount-1)] + mu.Lock() if v, ok := old.Get(k); ok { - next.Add(k, v) + if _, exists := next.Get(k); exists { + // A post-swap write superseded this entry; its residency ends here. + c.currentSize.Add(-int64(v.size)) + } else { + next.Add(k, v) + } } + mu.Unlock() } - c.data.Store(next) c.curCap.Store(newCap) c.reservedBytes += delta } @@ -313,6 +336,14 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) newSize := len(key) + valBytes + 24 ep := c.coh.Epoch() + // Grow toward the ceiling before taking the stripe — maybeGrow sweeps every + // stripe, so growing while holding one would deadlock. + if c.mode != ModeNoOp { + if curCap := c.curCap.Load(); curCap < c.maxCap && c.data.Load().Len() >= int(curCap) { + c.maybeGrow() + } + } + mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() defer mu.Unlock() @@ -341,12 +372,6 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) } } - // ModeEvictLRU: grow toward the ceiling before inserting into a full LRU, so a - // busy cache expands into its budget rather than evicting at the start size. - if curCap := c.curCap.Load(); c.mode != ModeNoOp && curCap < c.maxCap && lru.Len() >= int(curCap) { - c.maybeGrow() - lru = c.data.Load() - } // In ModeEvictLRU the byte budget is enforced through the entry-count cap, // not a separate currentSize check: capacityEntries is derived from // capacityB (capacityB/avgBytesPerEntry, see NewGenericCache / diff --git a/execution/cache/generic_cache_concurrency_test.go b/execution/cache/generic_cache_concurrency_test.go index 6e27e997385..6ad6227d95c 100644 --- a/execution/cache/generic_cache_concurrency_test.go +++ b/execution/cache/generic_cache_concurrency_test.go @@ -19,9 +19,11 @@ package cache import ( "encoding/binary" "sync" + "sync/atomic" "testing" "github.com/c2h5oh/datasize" + "github.com/stretchr/testify/require" ) // TestGenericCache_ConcurrentPutAcrossGrow guards the jump-grow data race: @@ -51,3 +53,76 @@ func TestGenericCache_ConcurrentPutAcrossGrow(t *testing.T) { } wg.Wait() } + +// A same-key put serialized by its stripe must never be undone by a grow: with +// copy-then-swap migration, a writer that loaded the old generation before the +// swap landed its write in the abandoned generation, and the migrated (older) +// value resurfaced as live — a stale serve, not a benign miss. The writer +// self-verifies each put and a reader checks the hot key's monotonically +// increasing value never goes backward. +func TestGenericCache_PutNotLostAcrossGrow(t *testing.T) { + value := func(n uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, n) + return b + } + for round := 0; round < 50; round++ { + c := NewGenericCache[[]byte](64*datasize.MB, func(v []byte) int { return len(v) }, ModeEvictLRU) + hot := []byte("hot-key") + c.Put(hot, value(0), 1) + + stop := make(chan struct{}) + var regressed atomic.Bool + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for n := uint64(1); ; n++ { + select { + case <-stop: + return + default: + } + c.Put(hot, value(n), n) + if v, ok := c.Get(hot); ok { + if got := binary.BigEndian.Uint64(v); got < n { + regressed.Store(true) + return + } + } + } + }() + go func() { + defer wg.Done() + last := uint64(0) + for { + select { + case <-stop: + return + default: + } + if v, ok := c.Get(hot); ok { + if n := binary.BigEndian.Uint64(v); n < last { + regressed.Store(true) + return + } else { + last = n + } + } + } + }() + + // Cross the grow threshold so maybeGrow swaps the generation while the + // hot-key writer runs. + key := make([]byte, 8) + for i := 0; i < 3*genericCacheStartCapacity; i++ { + binary.BigEndian.PutUint64(key, uint64(1+i)) + c.Put(key, []byte{1}, 1) + } + + close(stop) + wg.Wait() + c.Close() + require.False(t, regressed.Load(), "round %d: a striped put was lost across a grow (older value resurfaced)", round) + } +} From 561b29d8ff27a81898cb70b932a6d7c0089a6dde Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 10:03:01 +0200 Subject: [PATCH 12/31] db/state/execctx: tombstone deleted keys in the flush cache-apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting the entry left the key absent, so a read-fill from a reader straddling the deletion (a pre-delete snapshot) re-inserted the old value as live — PutIfAbsent only defers to live entries — and the deleted account or slot was served as canonical. A nil tombstone stamped with the delete's txNum defends the key and drops on an unwind at or below the deletion. The code layers keep the delete: they cannot represent negatives. --- db/state/execctx/domain_shared.go | 7 +- db/state/execctx/statecache_readfill_test.go | 72 ++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 464c12eb800..cac7f61a0da 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1043,7 +1043,10 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.AccountsDomain: if len(u.val) == 0 { - sd.stateCache.Delete(kv.AccountsDomain, u.key) + // Tombstone rather than delete: a live negative defends the key + // against a straddling pre-delete read-fill re-inserting the old + // value (PutIfAbsent only defers to live entries). + sd.stateCache.Put(kv.AccountsDomain, u.key, nil, u.txN) sd.stateCache.Delete(kv.CodeDomain, u.key) sd.stateCache.DeleteAddrCodeHash(u.key) } else { @@ -1052,7 +1055,7 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.StorageDomain: if len(u.val) == 0 { - sd.stateCache.Delete(kv.StorageDomain, u.key) + sd.stateCache.Put(kv.StorageDomain, u.key, nil, u.txN) } else { sd.stateCache.Put(kv.StorageDomain, u.key, u.val, u.txN) } diff --git a/db/state/execctx/statecache_readfill_test.go b/db/state/execctx/statecache_readfill_test.go index 7209a1cac21..21da1b63433 100644 --- a/db/state/execctx/statecache_readfill_test.go +++ b/db/state/execctx/statecache_readfill_test.go @@ -209,3 +209,75 @@ func TestReadFill_NegativeStampedWithProgress(t *testing.T) { _, ok = sc.Get(kv.AccountsDomain, missing) require.False(t, ok, "a negative observed at progress 100 must not survive an unwind to 50") } + +// A flush-apply for a deletion must leave a live tombstone, not remove the +// entry: with the key absent, a read-fill from a straddling pre-delete +// snapshot re-inserts the deleted value as live (PutIfAbsent only defers to +// live entries), and the resurrected account is served as canonical. +func TestReadFill_DoesNotResurrectDeletedKey(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + + const stepSize = uint64(16) + ctx := t.Context() + db := newTestDb(t, stepSize) + sc := newSmallStateCache() + + key := make([]byte, 20) + key[0] = 0xbb + v1 := encAccount(1) + + rwTx1, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx1.Rollback() + sd1, err := execctx.NewSharedDomains(ctx, rwTx1, log.New()) + require.NoError(t, err) + defer sd1.Close() + sd1.SetStateCacheForTest(sc) + sd1.SetTxNum(10) + require.NoError(t, sd1.DomainPut(kv.AccountsDomain, rwTx1, key, v1, 10, nil)) + require.NoError(t, sd1.Commit(ctx, rwTx1)) + + // A reader snapshot from before the deletion. + roTxOld, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTxOld.Rollback() + + rwTx2, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx2.Rollback() + sd2, err := execctx.NewSharedDomains(ctx, rwTx2, log.New()) + require.NoError(t, err) + defer sd2.Close() + sd2.SetStateCacheForTest(sc) + sd2.SetTxNum(20) + require.NoError(t, sd2.DomainDel(kv.AccountsDomain, rwTx2, key, 20, v1)) + require.NoError(t, sd2.Commit(ctx, rwTx2)) + + // The straddling reader: any fill it makes must not resurrect the account. + sdOld, err := execctx.NewSharedDomains(ctx, roTxOld, log.New()) + require.NoError(t, err) + defer sdOld.Close() + sdOld.SetStateCacheForTest(sc) + _, _, err = sdOld.GetLatest(kv.AccountsDomain, roTxOld, key) + require.NoError(t, err) + + roTxNew, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTxNew.Rollback() + sdNew, err := execctx.NewSharedDomains(ctx, roTxNew, log.New()) + require.NoError(t, err) + defer sdNew.Close() + sdNew.SetStateCacheForTest(sc) + v, _, err := sdNew.GetLatest(kv.AccountsDomain, roTxNew, key) + require.NoError(t, err) + require.Empty(t, v, "the straddling read-fill must not resurrect the deleted account") + + // The tombstone is stamped with the delete's txNum, so an unwind at or + // below it drops the negative instead of letting it outlive the deletion. + _, cTxNum, ok := sc.GetWithTxNum(kv.AccountsDomain, key) + require.True(t, ok) + require.Equal(t, uint64(20), cTxNum) +} From a8e81b5b02a302bb2fb5fc5d0a60ca43e48ca08c Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 10:03:14 +0200 Subject: [PATCH 13/31] db/state/execctx, execution/exec: trim code-negative warmup work, inline codeHash extraction, add negative-read benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code negatives in the warmup getter no longer pay a DomainProgress call and a liveness probe for a put CodeCache drops anyway. The decodeAccountCodeHash wrapper was a pure pass-through — call accounts.DeserialiseV3CodeHash directly. The new benchmarks bound the negative-stamp cost: the keys-table LastKey is ~290 ns and the whole cache-side fill ~0.5 us per cold negative (M2 Max), paid once per key between invalidations. --- db/state/execctx/domain_shared.go | 18 +--- .../execctx/statecache_readfill_bench_test.go | 99 +++++++++++++++++++ execution/exec/blocks_read_ahead.go | 10 +- 3 files changed, 108 insertions(+), 19 deletions(-) create mode 100644 db/state/execctx/statecache_readfill_bench_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index cac7f61a0da..95aae0e1b71 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1407,11 +1407,11 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte, txNum ui // on flush. Route mem-first; the LRU is a committed-state layer that may only // answer once mem has missed. if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + return accounts.DeserialiseV3CodeHash(v) } if sd.parent != nil { if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + return accounts.DeserialiseV3CodeHash(v) } } @@ -1432,14 +1432,14 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte, txNum ui resolve := func() []byte { if sd.stateCache != nil { if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + return accounts.DeserialiseV3CodeHash(v) } } v, _, err := tx.GetLatest(kv.AccountsDomain, addr) if err != nil || len(v) == 0 { return nil } - return decodeAccountCodeHash(v) + return accounts.DeserialiseV3CodeHash(v) } h := resolve() @@ -1457,16 +1457,6 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte, txNum ui return h } -// decodeAccountCodeHash extracts the codeHash from an account's encoded -// bytes. Returns nil on a malformed record or when the account has no code. -// AccountsDomain values are SerialiseV3-encoded — DecodeForStorage is the -// legacy MDBX bitmask format with an incompatible layout that would silently -// misparse. The targeted extractor skips the full decode (balance parse, -// interning) this runs per mem-hit on the codeHash fast path. -func decodeAccountCodeHash(enc []byte) []byte { - return accounts.DeserialiseV3CodeHash(enc) -} - func (sd *SharedDomains) Metrics() *kvmetrics.DomainMetrics { return &sd.metrics } diff --git a/db/state/execctx/statecache_readfill_bench_test.go b/db/state/execctx/statecache_readfill_bench_test.go new file mode 100644 index 00000000000..a5868cf2933 --- /dev/null +++ b/db/state/execctx/statecache_readfill_bench_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execctx_test + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/execctx" +) + +// benchSeedDb commits one account so the domain tables are non-empty for +// cold-negative probes. +func benchSeedDb(b *testing.B) kv.TemporalRwDB { + b.Helper() + const stepSize = uint64(16) + ctx := b.Context() + db := newTestDb(b, stepSize) + + rwTx, err := db.BeginTemporalRw(ctx) + require.NoError(b, err) + defer rwTx.Rollback() + sd, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) + require.NoError(b, err) + defer sd.Close() + written := make([]byte, 20) + written[0] = 0x01 + sd.SetTxNum(100) + require.NoError(b, sd.DomainPut(kv.AccountsDomain, rwTx, written, encAccount(7), 100, nil)) + require.NoError(b, sd.Commit(ctx, rwTx)) + return db +} + +// BenchmarkDomainProgress isolates the negative-stamp source: one +// files.EndTxNum read plus an MDBX LastKey on the domain's keys table. +func BenchmarkDomainProgress(b *testing.B) { + db := benchSeedDb(b) + roTx, err := db.BeginTemporalRo(b.Context()) + require.NoError(b, err) + defer roTx.Rollback() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = roTx.Debug().DomainProgress(kv.AccountsDomain) + } +} + +// benchColdNegativeReads drives the full cold-negative SD read: the whole +// miss stack, plus — when a cache is wired — the progress stamp and the +// if-absent fill. +func benchColdNegativeReads(b *testing.B, withCache bool) { + db := benchSeedDb(b) + ctx := b.Context() + roTx, err := db.BeginTemporalRo(ctx) + require.NoError(b, err) + defer roTx.Rollback() + sd, err := execctx.NewSharedDomains(ctx, roTx, log.New()) + require.NoError(b, err) + defer sd.Close() + if withCache { + sd.SetStateCacheForTest(newSmallStateCache()) + } + + key := make([]byte, 20) + key[0] = 0x02 + b.ResetTimer() + for i := 0; i < b.N; i++ { + binary.BigEndian.PutUint64(key[12:], uint64(i)+1) + v, _, err := sd.GetLatest(kv.AccountsDomain, roTx, key) + if err != nil { + b.Fatal(err) + } + if len(v) != 0 { + b.Fatalf("expected a negative, got %x", v) + } + } +} + +func BenchmarkGetLatestColdNegative(b *testing.B) { benchColdNegativeReads(b, true) } + +// The baseline the stamp+fill cost adds to. +func BenchmarkGetLatestColdNegativeNoCache(b *testing.B) { benchColdNegativeReads(b, false) } diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index fa57bac532e..c435ab12776 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -97,10 +97,11 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // If-absent writes only: this runs in a fire-and-forget goroutine over a // committed snapshot, so an unconditional Put racing an FCU flush's // cache-apply could replace the flushed value with the pre-flush one. - if name == kv.CodeDomain && len(v) > 0 { + if name == kv.CodeDomain { // A live binding makes the conditional put a no-op — skip before - // paying the keccak+copy below. - if !cpg.sc.HasLiveCode(k) { + // paying the keccak+copy below. Code negatives end here too: they + // are not cacheable (CodeCache drops zero-length puts). + if len(v) > 0 && !cpg.sc.HasLiveCode(k) { // Key the content cache by the code's OWN hash, never a separately // read account codeHash: under parallel/speculative exec that hash // can be skewed or cross-account, and a (hash, code) pair that @@ -111,8 +112,7 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k } else { // Cache including nil/empty results: a probe returning no // bytes is a valid negative answer (missing account, empty - // storage slot; empty code lands here too but CodeCache drops - // zero-length puts) and caching it lets repeated probes + // storage slot) and caching it lets repeated probes // skip the file accessor stack. Mirrors revm's CacheAccount // { account: None, status: LoadedNotExisting } pattern. // Stamp with an upper bound on the value's write txNum: the last From 044ea9d9d9a9f1793e5b099f93f65c8ebbce69b4 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 10:35:39 +0200 Subject: [PATCH 14/31] execution/cache: freeze writers for the jump-grow copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swap-then-migrate resize left each key absent from the primary generation until migrated, so a conditional put arriving mid-resize found a gap and filled it with a stale value — the writer class PutIfAbsent exists to close, reopened for the resize window. The pre-stripe grow trigger also paid a ShardedLRU.Len (a lock per shard) on every put, including warm updates. The copy now runs with every put stripe held and the swap publishes a fully-copied generation, so no operation ever sees a partial one; the fence sweep, per-key migration and superseded-size accounting go away with the window. The trigger returns to the insert path and the grow runs after the stripe is released (growth takes every stripe). --- execution/cache/generic_cache.go | 64 ++++++++++--------- .../cache/generic_cache_concurrency_test.go | 53 +++++++++++++++ 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 4655229817d..3b50000b477 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -59,11 +59,11 @@ type entry[T any] struct { // GenericCache is a sharded, LRU-evicting bounded cache for key-value // data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { - // data is the sharded LRU, replaced wholesale on a jump-grow. Load it once per - // operation, inside the stripe for writes: maybeGrow swaps generations first - // and then migrates under the same stripes, so a striped write is never - // silently undone by the migration (an update lost that way would resurface - // the older value as live — a stale serve, not a benign miss). + // data is the sharded LRU, replaced wholesale on a jump-grow. The swap + // happens with every put stripe held and the new generation fully copied, + // so a striped write can never land in a retired generation (a write lost + // that way would resurface the older value as live — a stale serve, not a + // benign miss) and no operation ever sees a partially-copied one. data atomic.Pointer[freelru.ShardedLRU[uint64, entry[T]]] capacityB datasize.ByteSize mode Mode @@ -198,16 +198,15 @@ func (c *GenericCache[T]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, // maybeGrow jump-resizes the LRU one step larger when it is full, the ceiling // hasn't been reached, and the shared envelope can fund the step. Otherwise the -// LRU keeps its size and freelru evicts within it. Must be called with no -// stripe held (it sweeps every stripe). +// LRU keeps its size and freelru evicts within it. Must not be called with a +// stripe held (it takes them all). // -// Swap first, then migrate: entries copied before the swap could be updated in -// the old generation by a concurrent striped writer, and the stale copy would -// resurface as live. After the swap, the stripe sweep fences the writers that -// loaded the old generation before it, freezing the old generation; migration -// then moves each key under its stripe, if-absent, so it never clobbers a -// fresher post-swap write. Until a key is migrated, reads of it miss — that -// (and only that) is the benign transient of a resize. +// The copy runs with every put stripe held: writers (and the striped +// stale-drop) are excluded, so no write can land in the generation being +// retired and a conditional put never sees a mid-resize gap it could fill +// with a stale value; readers stay on the retiring generation until the swap +// and never miss. Grows are a handful of steps per cache lifetime, so the +// stall is a bounded one-off, not a steady-state cost. func (c *GenericCache[T]) maybeGrow() { c.resizeMu.Lock() defer c.resizeMu.Unlock() @@ -225,26 +224,20 @@ func (c *GenericCache[T]) maybeGrow() { if !cachebudget.Global.Reserve(delta) { return } - next := c.newShards(newCap) - c.data.Store(next) + next := c.newShards(newCap) // allocate before excluding writers for i := range c.putStripes { c.putStripes[i].Lock() - c.putStripes[i].Unlock() //nolint:gocritic,staticcheck // empty critical section is the writer fence } for _, k := range old.Keys() { - mu := &c.putStripes[k&(putStripeCount-1)] - mu.Lock() if v, ok := old.Get(k); ok { - if _, exists := next.Get(k); exists { - // A post-swap write superseded this entry; its residency ends here. - c.currentSize.Add(-int64(v.size)) - } else { - next.Add(k, v) - } + next.Add(k, v) } - mu.Unlock() } + c.data.Store(next) c.curCap.Store(newCap) + for i := range c.putStripes { + c.putStripes[i].Unlock() + } c.reservedBytes += delta } @@ -336,13 +329,16 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) newSize := len(key) + valBytes + 24 ep := c.coh.Epoch() - // Grow toward the ceiling before taking the stripe — maybeGrow sweeps every - // stripe, so growing while holding one would deadlock. - if c.mode != ModeNoOp { - if curCap := c.curCap.Load(); curCap < c.maxCap && c.data.Load().Len() >= int(curCap) { + // Grow after the stripe is released (defers run LIFO): maybeGrow takes + // every stripe, so triggering it under one would deadlock. Detection stays + // on the insert path below — Len locks every shard, too costly per warm + // update. + needGrow := false + defer func() { + if needGrow { c.maybeGrow() } - } + }() mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() @@ -372,6 +368,12 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) } } + // This insert lands in a full LRU with ceiling headroom — grow one step + // (after the stripe is released, see above). + if curCap := c.curCap.Load(); c.mode != ModeNoOp && curCap < c.maxCap && lru.Len() >= int(curCap) { + needGrow = true + } + // In ModeEvictLRU the byte budget is enforced through the entry-count cap, // not a separate currentSize check: capacityEntries is derived from // capacityB (capacityB/avgBytesPerEntry, see NewGenericCache / diff --git a/execution/cache/generic_cache_concurrency_test.go b/execution/cache/generic_cache_concurrency_test.go index 6ad6227d95c..5668d19def6 100644 --- a/execution/cache/generic_cache_concurrency_test.go +++ b/execution/cache/generic_cache_concurrency_test.go @@ -126,3 +126,56 @@ func TestGenericCache_PutNotLostAcrossGrow(t *testing.T) { require.False(t, regressed.Load(), "round %d: a striped put was lost across a grow (older value resurfaced)", round) } } + +// A conditional put must keep deferring to a live entry across a grow: if the +// resize ever publishes a generation the entry hasn't reached yet, a +// PutIfAbsent arriving in that gap finds the key absent and inserts its +// (stale) value — the writer class the if-absent semantics exist to close. +// The prober watches for the generation swap and bursts conditional puts the +// moment it lands, mimicking a fill thread that starts a put mid-resize; its +// idle spin keeps the hot key MRU so LRU eviction cannot produce the stale +// value legitimately. +func TestGenericCache_PutIfAbsentDefersAcrossGrow(t *testing.T) { + fresh := []byte("fresh-value") + stale := []byte("stale-value") + for round := 0; round < 50; round++ { + c := NewGenericCache[[]byte](64*datasize.MB, func(v []byte) int { return len(v) }, ModeEvictLRU) + hot := []byte("hot-key") + c.Put(hot, fresh, 10) + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + before := c.data.Load() + for { + select { + case <-stop: + return + default: + } + if c.data.Load() != before { + for j := 0; j < 4096; j++ { + c.PutIfAbsent(hot, stale, 5) + } + return + } + c.Get(hot) + } + }() + + key := make([]byte, 8) + for i := 0; i < 3*genericCacheStartCapacity; i++ { + binary.BigEndian.PutUint64(key, uint64(1+i)) + c.Put(key, []byte{1}, 1) + } + + close(stop) + wg.Wait() + v, ok := c.Get(hot) + require.True(t, ok, "round %d: hot key missing", round) + require.Equal(t, fresh, v, "round %d: PutIfAbsent bypassed the live entry across a grow", round) + c.Close() + } +} From 1c85ee409619b1a683f47fa063ac43ed6b039bae Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 11:12:46 +0200 Subject: [PATCH 15/31] execution/cache, execution/exec: drop the generic ContainsLive probe, simplify put's grow trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cache.ContainsLive interface method had one caller — StateCache.put's pre-check — whose only real value (skipping the keccak for already-bound code) lives in HasLiveCode, which reaches CodeCache.ContainsLive concretely; for domain values the probe traded a tens-of-bytes copy against an extra Peek on the common miss path. putLocked now returns whether the caller should grow, replacing the defer-ordering trick, and warmBody builds its getter through one constructor. TestGenericCache_PutIfAbsentDefersAcrossGrow is rebuilt to be scheduling-proof: reaching Len >= startCap organically requires every freelru shard full, which makes the hot key evictable and a stale insert legitimate — a flake under CPU contention. The cache is now seeded below capacity pressure with the hot key inserted last (the LRU victim is always an older seed key) and the grow is forced by lowering curCap. --- execution/cache/cache.go | 5 -- execution/cache/cache_test.go | 19 -------- execution/cache/generic_cache.go | 46 +++++++------------ .../cache/generic_cache_concurrency_test.go | 25 ++++++---- execution/cache/state_cache.go | 9 +--- execution/exec/blocks_read_ahead.go | 8 +++- 6 files changed, 41 insertions(+), 71 deletions(-) diff --git a/execution/cache/cache.go b/execution/cache/cache.go index 1853f61aff0..fd59c16f2d5 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -35,11 +35,6 @@ type Cache interface { // already be superseded by an authoritative Put. PutIfAbsent(key []byte, value []byte, txNum uint64) - // ContainsLive reports whether key has a live (non-stale) entry, without - // touching hit/miss counters or LRU recency. A pre-check for conditional - // writers to skip preparing a put a live entry would no-op; advisory only. - ContainsLive(key []byte) bool - // Delete removes the data for the given key. Delete(key []byte) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 6016fef5215..2a188fd998c 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -924,25 +924,6 @@ func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { } } -func TestDomainCache_ContainsLive(t *testing.T) { - c := NewDomainCacheMode(1*datasize.KB, ModeEvictLRU) - addr := makeAddr(1) - require.False(t, c.ContainsLive(addr), "absent key") - - c.Put(addr, []byte("v"), 10) - require.True(t, c.ContainsLive(addr)) - - c.Unwind(5) // entry txNum 10 >= floor 5, superseded epoch → stale - require.False(t, c.ContainsLive(addr), "stale entry must not read as live") - - // The probe is passive: the stale entry is left for PutIfAbsent to replace. - c.PutIfAbsent(addr, []byte("w"), 4) - v, ok := c.Get(addr) - require.True(t, ok) - assert.Equal(t, []byte("w"), v) - require.True(t, c.ContainsLive(addr)) -} - func TestCodeCache_ContainsLive(t *testing.T) { cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) addr := makeAddr(1) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 3b50000b477..32069c96095 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -324,22 +324,22 @@ func (c *GenericCache[T]) PutIfAbsent(key []byte, value T, txNum uint64) { } func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) { + if c.putLocked(key, value, txNum, overwrite) { + // Grow outside the stripe — maybeGrow takes every stripe. + c.maybeGrow() + } +} + +// putLocked performs the write under the key's stripe and reports whether the +// insert landed in a full LRU with ceiling headroom, i.e. the caller should +// grow. Detection stays on the insert path — Len locks every shard, too costly +// per warm update. +func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite bool) bool { h := maphash.Hash(key) valBytes := c.sizeFunc(value) newSize := len(key) + valBytes + 24 ep := c.coh.Epoch() - // Grow after the stripe is released (defers run LIFO): maybeGrow takes - // every stripe, so triggering it under one would deadlock. Detection stays - // on the insert path below — Len locks every shard, too costly per warm - // update. - needGrow := false - defer func() { - if needGrow { - c.maybeGrow() - } - }() - mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() defer mu.Unlock() @@ -352,11 +352,11 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) // old one. if hasExisting && bytes.Equal(existing.key, key) { if !overwrite && !c.coh.IsStale(existing.txNum, existing.epoch) { - return + return false } lru.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize - existing.size)) - return + return false } if c.mode == ModeNoOp { @@ -364,15 +364,12 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) // entry-count cap, which ModeNoOp ("drop new keys when full") must not do. if c.currentSize.Load()+int64(newSize) > int64(c.capacityB) || lru.Len() >= int(c.maxCap) { c.dropped.Add(1) - return + return false } } - // This insert lands in a full LRU with ceiling headroom — grow one step - // (after the stripe is released, see above). - if curCap := c.curCap.Load(); c.mode != ModeNoOp && curCap < c.maxCap && lru.Len() >= int(curCap) { - needGrow = true - } + curCap := c.curCap.Load() + needGrow := c.mode != ModeNoOp && curCap < c.maxCap && lru.Len() >= int(curCap) // In ModeEvictLRU the byte budget is enforced through the entry-count cap, // not a separate currentSize check: capacityEntries is derived from @@ -397,6 +394,7 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) lru.Add(h, entry[T]{key: keyCopy, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize)) c.inserts.Add(1) + return needGrow } // Delete removes the data for the given key. Runs under the key's put stripe: @@ -426,16 +424,6 @@ func (c *GenericCache[T]) dropStale(h uint64, key []byte) { } } -// ContainsLive reports whether key has a live (non-stale) entry, without -// touching hit/miss counters or LRU recency. Prefetchers probe it to skip the -// copy work of preparing a conditional put that a live entry would no-op; -// advisory only — PutIfAbsent re-decides under the key's stripe. -func (c *GenericCache[T]) ContainsLive(key []byte) bool { - h := maphash.Hash(key) - e, ok := c.data.Load().Peek(h) - return ok && bytes.Equal(e.key, key) && !c.coh.IsStale(e.txNum, e.epoch) -} - // Clear removes all entries from the cache. It also resets the (epoch, // unwindFloor) coherence pair: with no entries left, no stale (txNum, epoch) // can survive, so a fresh floor keeps subsequent Puts at the live epoch diff --git a/execution/cache/generic_cache_concurrency_test.go b/execution/cache/generic_cache_concurrency_test.go index 5668d19def6..545d7221034 100644 --- a/execution/cache/generic_cache_concurrency_test.go +++ b/execution/cache/generic_cache_concurrency_test.go @@ -132,14 +132,24 @@ func TestGenericCache_PutNotLostAcrossGrow(t *testing.T) { // PutIfAbsent arriving in that gap finds the key absent and inserts its // (stale) value — the writer class the if-absent semantics exist to close. // The prober watches for the generation swap and bursts conditional puts the -// moment it lands, mimicking a fill thread that starts a put mid-resize; its -// idle spin keeps the hot key MRU so LRU eviction cannot produce the stale -// value legitimately. +// moment it lands, mimicking a fill thread that starts a put mid-resize. +// +// The cache is seeded below any capacity pressure with the hot key inserted +// last — the LRU victim is always an older seed key, so the hot key cannot be +// evicted and a stale value at the end can only have come through a resize +// gap. The grow is forced by lowering curCap: reaching Len >= startCap +// organically needs every freelru shard full, which would make the hot key +// evictable and the signal ambiguous. func TestGenericCache_PutIfAbsentDefersAcrossGrow(t *testing.T) { fresh := []byte("fresh-value") stale := []byte("stale-value") for round := 0; round < 50; round++ { c := NewGenericCache[[]byte](64*datasize.MB, func(v []byte) int { return len(v) }, ModeEvictLRU) + key := make([]byte, 8) + for i := 0; i < 512; i++ { + binary.BigEndian.PutUint64(key, uint64(1+i)) + c.Put(key, []byte{1}, 1) + } hot := []byte("hot-key") c.Put(hot, fresh, 10) @@ -161,15 +171,12 @@ func TestGenericCache_PutIfAbsentDefersAcrossGrow(t *testing.T) { } return } - c.Get(hot) } }() - key := make([]byte, 8) - for i := 0; i < 3*genericCacheStartCapacity; i++ { - binary.BigEndian.PutUint64(key, uint64(1+i)) - c.Put(key, []byte{1}, 1) - } + c.curCap.Store(uint32(c.Len())) + binary.BigEndian.PutUint64(key, 0) + c.Put(key, []byte{1}, 1) // insert at the lowered cap → triggers the grow close(stop) wg.Wait() diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index baf708abc35..bbf6f33088a 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -263,14 +263,9 @@ func (c *StateCache) put(domain kv.Domain, key []byte, value []byte, txNum uint6 } if overwrite { cache.Put(key, common.Copy(value), txNum) - return - } - // Skip the copy a live entry would discard; advisory — PutIfAbsent - // re-decides under the key's stripe. - if cache.ContainsLive(key) { - return + } else { + cache.PutIfAbsent(key, common.Copy(value), txNum) } - cache.PutIfAbsent(key, common.Copy(value), txNum) } // Delete removes the data for the given domain and key. diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index c435ab12776..5a77caf77f0 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -91,6 +91,10 @@ type cachePopulatingGetter struct { progress func(kv.Domain) uint64 } +func newCachePopulatingGetter(ttx kv.TemporalTx, sc *cache.StateCache) *cachePopulatingGetter { + return &cachePopulatingGetter{g: ttx, sc: sc, stepSize: ttx.Debug().StepSize(), progress: ttx.Debug().DomainProgress} +} + func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { v, step, err := cpg.g.GetLatest(name, k) if err == nil && cpg.sc != nil { @@ -254,7 +258,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t } var getter kv.TemporalGetter = ttx if bra.stateCache != nil { - getter = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize(), progress: ttx.Debug().DomainProgress} + getter = newCachePopulatingGetter(ttx, bra.stateCache) } stateReader := state.NewReaderV3(getter) @@ -325,7 +329,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize(), progress: ttx.Debug().DomainProgress} + cpg = newCachePopulatingGetter(ttx, bra.stateCache) getter = cpg } stateReader := state.NewReaderV3(getter) From 23bcf563e86d9fec08bbd667f6b4e8bf068234cf Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 11:36:49 +0200 Subject: [PATCH 16/31] execution/cache, db/state/execctx, execution/types/accounts: no-code deletion marker for the code cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A code deletion was applied to the cache as a raw Delete, so a read-fill from a reader straddling the deletion (a pre-delete snapshot) re-bound the deleted code as a live hit — the same resurrection the account and storage tombstones close, flagged by Copilot. An authoritative nil put now binds a live no-code marker (versionedAddressID.deleted, stamped with the delete's txNum): a valid negative on read, a binding conditional fills defer to, replaced by an authoritative rebind and dropped by an unwind at or below the deletion. The flush-apply writes Put(CodeDomain, key, nil, txN), symmetric with the other domains. Also tightens the DeserialiseV3CodeHash doc: it parses only up to and including the codeHash field; later fields are not validated. --- db/state/execctx/domain_shared.go | 7 +- db/state/execctx/statecache_readfill_test.go | 71 ++++++++++++++++++++ execution/cache/cache_test.go | 54 ++++++++++++++- execution/cache/code_cache.go | 27 ++++++-- execution/cache/state_cache.go | 9 +-- execution/types/accounts/account.go | 10 +-- 6 files changed, 161 insertions(+), 17 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 95aae0e1b71..21cd9852c42 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1045,9 +1045,10 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun if len(u.val) == 0 { // Tombstone rather than delete: a live negative defends the key // against a straddling pre-delete read-fill re-inserting the old - // value (PutIfAbsent only defers to live entries). + // value (PutIfAbsent only defers to live entries). Same for the + // code binding, via its no-code deletion marker. sd.stateCache.Put(kv.AccountsDomain, u.key, nil, u.txN) - sd.stateCache.Delete(kv.CodeDomain, u.key) + sd.stateCache.Put(kv.CodeDomain, u.key, nil, u.txN) sd.stateCache.DeleteAddrCodeHash(u.key) } else { sd.stateCache.Put(kv.AccountsDomain, u.key, u.val, u.txN) @@ -1061,7 +1062,7 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.CodeDomain: if len(u.val) == 0 { - sd.stateCache.Delete(kv.CodeDomain, u.key) + sd.stateCache.Put(kv.CodeDomain, u.key, nil, u.txN) } else { // Validated committed code: populate the addr layer AND the // content-addressed codeHash->code map, keyed by keccak(v) so each diff --git a/db/state/execctx/statecache_readfill_test.go b/db/state/execctx/statecache_readfill_test.go index 21da1b63433..85689293035 100644 --- a/db/state/execctx/statecache_readfill_test.go +++ b/db/state/execctx/statecache_readfill_test.go @@ -281,3 +281,74 @@ func TestReadFill_DoesNotResurrectDeletedKey(t *testing.T) { require.True(t, ok) require.Equal(t, uint64(20), cTxNum) } + +// The code-domain equivalent of the tombstone test: a code deletion must +// leave a live no-code marker on the addr binding, or a read-fill from a +// straddling pre-delete snapshot re-binds the deleted code as a live hit. +func TestReadFill_DoesNotResurrectDeletedCode(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + + const stepSize = uint64(16) + ctx := t.Context() + db := newTestDb(t, stepSize) + sc := newSmallStateCache() + + addr := make([]byte, 20) + addr[0] = 0xcc + code := []byte{0xaa, 1, 2, 3} + + rwTx1, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx1.Rollback() + sd1, err := execctx.NewSharedDomains(ctx, rwTx1, log.New()) + require.NoError(t, err) + defer sd1.Close() + sd1.SetStateCacheForTest(sc) + sd1.SetTxNum(10) + require.NoError(t, sd1.DomainPut(kv.CodeDomain, rwTx1, addr, code, 10, nil)) + require.NoError(t, sd1.Commit(ctx, rwTx1)) + + // A reader snapshot from before the deletion. + roTxOld, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTxOld.Rollback() + + rwTx2, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx2.Rollback() + sd2, err := execctx.NewSharedDomains(ctx, rwTx2, log.New()) + require.NoError(t, err) + defer sd2.Close() + sd2.SetStateCacheForTest(sc) + sd2.SetTxNum(20) + require.NoError(t, sd2.DomainDel(kv.CodeDomain, rwTx2, addr, 20, code)) + require.NoError(t, sd2.Commit(ctx, rwTx2)) + + // The straddling reader: any fill it makes must not resurrect the code. + sdOld, err := execctx.NewSharedDomains(ctx, roTxOld, log.New()) + require.NoError(t, err) + defer sdOld.Close() + sdOld.SetStateCacheForTest(sc) + _, _, err = sdOld.GetLatest(kv.CodeDomain, roTxOld, addr) + require.NoError(t, err) + + roTxNew, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer roTxNew.Rollback() + sdNew, err := execctx.NewSharedDomains(ctx, roTxNew, log.New()) + require.NoError(t, err) + defer sdNew.Close() + sdNew.SetStateCacheForTest(sc) + v, _, err := sdNew.GetLatest(kv.CodeDomain, roTxNew, addr) + require.NoError(t, err) + require.Empty(t, v, "the straddling read-fill must not resurrect the deleted code") + + // The marker is stamped with the delete's txNum, so an unwind at or below + // it drops the negative instead of letting it outlive the deletion. + _, cTxNum, ok := sc.GetWithTxNum(kv.CodeDomain, addr) + require.True(t, ok, "the deletion must be cached as a live no-code marker") + require.Equal(t, uint64(20), cTxNum) +} diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 2a188fd998c..d1f9bcd0fb3 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -243,9 +243,20 @@ func TestCodeCache_PutEmptyCode(t *testing.T) { addr := makeAddr(1) c.Put(addr, []byte{}, 0) - // Should not store empty code - assert.Equal(t, 0, c.Len()) + // An authoritative empty put is a deletion: it binds a no-code marker (a + // valid negative), never code bytes. + assert.Equal(t, 1, c.Len()) assert.Equal(t, 0, c.CodeLen()) + v, ok := c.Get(addr) + assert.True(t, ok) + assert.Empty(t, v) + + // A conditional empty put stays a no-op. + c2 := NewCodeCache(100, 200) + c2.PutIfAbsent(addr, []byte{}, 0) + assert.Equal(t, 0, c2.Len()) + _, ok = c2.Get(addr) + assert.False(t, ok) } func TestCodeCache_CodeDeduplication(t *testing.T) { @@ -924,6 +935,45 @@ func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { } } +// A code deletion (an overwrite put of nil) must leave a live no-code marker: +// it serves as a valid negative, a conditional bind defers to it, an +// authoritative rebind replaces it, and an unwind at or below the deletion +// drops it. +func TestCodeCache_DeletionMarker(t *testing.T) { + cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + code := []byte{0xaa, 1, 2, 3} + cc.PutWithCodeHash(addr, code, crypto.Keccak256(code), 10) + + cc.Put(addr, nil, 20) // the flush-apply shape of a deletion + + v, cTxNum, ok := cc.GetWithTxNum(addr) + require.True(t, ok, "a deletion is a valid negative, not a miss") + require.Empty(t, v) + require.Equal(t, uint64(20), cTxNum) + require.True(t, cc.ContainsLive(addr), "a conditional put would defer to the marker") + + // A straddling if-absent fill of the old code defers to the marker. + cc.PutWithCodeHashIfAbsent(addr, code, crypto.Keccak256(code), 15) + v, _, ok = cc.GetWithTxNum(addr) + require.True(t, ok) + require.Empty(t, v, "an if-absent fill must not resurrect deleted code") + + // An authoritative rebind (new deployment) replaces the marker. + fresh := []byte{0xbb, 4, 5, 6} + cc.PutWithCodeHash(addr, fresh, crypto.Keccak256(fresh), 30) + v, ok = cc.Get(addr) + require.True(t, ok) + require.Equal(t, fresh, v) + + // An unwind at or below the deletion drops the marker. + cc2 := NewCodeCache(1*datasize.MB, 1*datasize.MB) + cc2.Put(addr, nil, 20) + cc2.Unwind(15) + _, _, ok = cc2.GetWithTxNum(addr) + require.False(t, ok, "a marker from a rolled-back deletion must not survive the unwind") +} + func TestCodeCache_ContainsLive(t *testing.T) { cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) addr := makeAddr(1) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 3f275c85034..d746e8fdc26 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -84,6 +84,11 @@ type versionedAddressID struct { codeHash [32]byte txNum uint64 epoch uint32 + // deleted marks a live no-code binding: the addr's code was deleted at + // txNum. A valid negative on read, and a binding conditional puts defer + // to — without it, a fill from a pre-delete snapshot would resurrect the + // deleted code. + deleted bool } type addrCodeHashEntry struct { @@ -295,6 +300,9 @@ func (c *CodeCache) GetWithTxNum(addr []byte) ([]byte, uint64, bool) { return nil, 0, false } c.addrHits.Add(1) + if vID.deleted { + return nil, vID.txNum, true + } ce, ok := c.hashToCode.Get(vID.addrID) if !ok || len(ce.code) == 0 { @@ -341,6 +349,13 @@ func (c *CodeCache) PutIfAbsent(addr []byte, code []byte, txNum uint64) { // cold code can't both Add and permanently inflate codeSize. func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum uint64, overwriteAddr bool) { if len(code) == 0 { + // An authoritative nil put is a deletion — bind a live no-code marker. + // Conditional nil puts stay no-ops. + if overwriteAddr && len(addr) > 0 { + c.addrBindMu.Lock() + c.addrToHash.Add(common.BytesToAddress(addr), versionedAddressID{deleted: true, txNum: txNum, epoch: c.coh.Epoch()}) + c.addrBindMu.Unlock() + } return } ep := c.coh.Epoch() @@ -364,15 +379,19 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui &c.coh, &c.codeSize, 8, &c.putStripes[uint8(codeID)]) } -// ContainsLive reports whether addr resolves to live code bytes through the -// addr→code binding, without touching hit/miss counters or LRU recency. -// Prefetchers probe it to skip the keccak+copy work of preparing a conditional -// put that a live binding would no-op; advisory only. +// ContainsLive reports whether addr has a live binding a conditional put +// would defer to — servable code bytes or a no-code deletion marker — without +// touching hit/miss counters or LRU recency. Prefetchers probe it to skip the +// keccak+copy work of preparing a conditional put that such a binding would +// no-op; advisory only. func (c *CodeCache) ContainsLive(addr []byte) bool { vID, ok := c.addrToHash.Peek(common.BytesToAddress(addr)) if !ok || c.isStale(vID.txNum, vID.epoch) { return false } + if vID.deleted { + return true + } ce, ok := c.hashToCode.Peek(vID.addrID) if !ok || len(ce.code) == 0 || c.isStale(ce.txNum, ce.epoch) { return false diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index bbf6f33088a..ab5d075ffae 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -178,10 +178,11 @@ func (c *StateCache) putCodeWithHash(addr, code, codeHash []byte, txNum uint64, } } -// HasLiveCode reports whether addr resolves to live code bytes through the -// code cache's addr→code binding, without touching stats or LRU recency. -// Prefetchers probe it to skip the keccak+copy of preparing a conditional -// code put that a live binding would no-op; advisory only. +// HasLiveCode reports whether addr has a live code-cache binding a +// conditional put would defer to — servable code bytes or a no-code deletion +// marker — without touching stats or LRU recency. Prefetchers probe it to +// skip the keccak+copy of preparing a conditional code put that such a +// binding would no-op; advisory only. func (c *StateCache) HasLiveCode(addr []byte) bool { cc, ok := c.caches[kv.CodeDomain].(*CodeCache) return ok && cc.ContainsLive(addr) diff --git a/execution/types/accounts/account.go b/execution/types/accounts/account.go index 8b83e5f6dfd..587183afc7a 100644 --- a/execution/types/accounts/account.go +++ b/execution/types/accounts/account.go @@ -645,10 +645,12 @@ func DeserialiseV3(a *Account, enc []byte) error { // DeserialiseV3CodeHash extracts just the codeHash field from a // SerialiseV3-encoded account, skipping the full decode (balance parse, -// codeHash interning) that DeserialiseV3 pays. Returns a subslice of enc — -// valid only while enc is — or nil for a malformed record or an account -// without code (including a non-canonical record spelling out the empty or -// zero sentinel, which CodeHash.IsEmpty treats as no-code). +// codeHash interning) that DeserialiseV3 pays. It parses only up to and +// including the codeHash field — later fields are not validated. Returns a +// subslice of enc — valid only while enc is — or nil when the record is +// malformed up to that field or the account has no code (including a +// non-canonical record spelling out the empty or zero sentinel, which +// CodeHash.IsEmpty treats as no-code). func DeserialiseV3CodeHash(enc []byte) []byte { pos := 0 for range 2 { // skip the length-prefixed nonce and balance fields From 9072533ca6a6afb9abb4de8fe70934af26d0a0e2 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 11:43:15 +0200 Subject: [PATCH 17/31] =?UTF-8?q?execution/cache:=20remove=20Delete=20?= =?UTF-8?q?=E2=80=94=20deletions=20are=20authoritative=20nil=20puts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the flush-apply writing tombstones and no-code markers, Delete lost its last production caller — and any future call would reopen the resurrection window the markers close, since removing an entry leaves nothing for a straddling conditional fill to defer to. Drop it from the Cache interface and every implementation so the wrong operation is not expressible; the stale-drop keeps the stripe-serialized removal rationale. The hammer tests that used Delete as a reset now race on fresh keys per round, and the stale-drop drift check asserts the exact expected residency instead of zero-after-delete. --- execution/cache/cache.go | 8 +- execution/cache/cache_test.go | 81 ++----------------- execution/cache/code_cache.go | 5 -- .../cache/code_cache_concurrency_test.go | 3 +- execution/cache/generic_cache.go | 23 +----- execution/cache/state_cache.go | 9 --- 6 files changed, 18 insertions(+), 111 deletions(-) diff --git a/execution/cache/cache.go b/execution/cache/cache.go index fd59c16f2d5..10be2806f46 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -33,11 +33,13 @@ type Cache interface { // PutIfAbsent is Put except that a live entry for key is left untouched // (a stale one is replaced) — for prefetch writers, whose snapshot may // already be superseded by an authoritative Put. + // + // There is deliberately no Delete: a deletion is an authoritative Put of + // nil (a tombstone / no-code marker), so conditional fills from stale + // snapshots defer to it. Removing the entry instead would let such a fill + // resurrect the deleted value. PutIfAbsent(key []byte, value []byte, txNum uint64) - // Delete removes the data for the given key. - Delete(key []byte) - // Clear removes all mutable entries from the cache. Clear() diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index d1f9bcd0fb3..2a815084f38 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -155,20 +155,6 @@ func TestDomainCache_PutEvictsWhenFull_EvictMode(t *testing.T) { assert.Positive(t, missingCount, "ModeEvictLRU should have evicted some early entries") } -func TestDomainCache_Delete(t *testing.T) { - c := NewDomainCacheMode(100, ModeEvictLRU) - - addr := makeAddr(1) - c.Put(addr, makeValue(1), 0) - assert.Equal(t, 1, c.Len()) - - c.Delete(addr) - assert.Equal(t, 0, c.Len()) - - _, ok := c.Get(addr) - assert.False(t, ok) -} - func TestDomainCache_Clear(t *testing.T) { c := NewDomainCacheMode(100, ModeEvictLRU) @@ -353,21 +339,6 @@ func TestCodeCache_CodeCapacityLimit(t *testing.T) { assert.False(t, ok, "coldest code should have been evicted") } -func TestCodeCache_Delete(t *testing.T) { - c := NewCodeCache(100, 200) - - addr := makeAddr(1) - code := makeCode(1) - c.Put(addr, code, 0) - - c.Delete(addr) - assert.Equal(t, 0, c.Len()) - // Code should still exist (immutable) - assert.Equal(t, 1, c.CodeLen()) - - _, ok := c.Get(addr) - assert.False(t, ok) -} func TestCodeCache_Clear(t *testing.T) { c := NewCodeCache(100, 200) @@ -501,16 +472,6 @@ func TestStateCache_GetPut_UnsupportedDomain(t *testing.T) { assert.Nil(t, v) } -func TestStateCache_Delete(t *testing.T) { - c := NewStateCache(100, 100, 100, 100) - - addr := makeAddr(1) - c.Put(kv.AccountsDomain, addr, makeValue(1), 0) - c.Delete(kv.AccountsDomain, addr) - - _, ok := c.Get(kv.AccountsDomain, addr) - assert.False(t, ok) -} // Put(key, nil) must be a cache hit, not a miss. SharedDomains.GetLatest // caches deleted keys via Put(key, nil); if Get treats that as "not found", @@ -544,13 +505,6 @@ func TestStateCache_PutEmptySlice_ThenGet_IsCacheHit(t *testing.T) { assert.Empty(t, v) } -func TestStateCache_Delete_UnsupportedDomain(t *testing.T) { - c := NewStateCache(100, 100, 100, 100) - - // Should not panic - c.Delete(kv.ReceiptDomain, makeAddr(1)) -} - func TestStateCache_Clear(t *testing.T) { c := NewStateCache(100, 100, 100, 100) @@ -878,11 +832,10 @@ func TestCodeCache_PutWithCodeHashIfAbsent(t *testing.T) { // clobber it — the prefetch-vs-flush staleness this cache guards against. func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) - addr := makeAddr(1) fresh := []byte("fresh") stale := []byte("stale") for round := 0; round < 20000; round++ { - c.Delete(addr) + addr := makeAddr(round) // a fresh key each round, so both writers race on the insert path var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); c.Put(addr, fresh, 20) }() @@ -894,34 +847,17 @@ func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { } } -// A Delete racing an update-in-place put must not double-subtract the -// displaced entry's size: freelru's OnEvict subtracts it for the Remove, and -// put's update delta subtracts it again unless the two writers share the -// key's stripe. -func TestDomainCache_DeleteAtomicWithPut_NoSizeDrift(t *testing.T) { - c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) - addr := makeAddr(1) - v1 := []byte("value-one") - v2 := []byte("value-two") - for round := 0; round < 20000; round++ { - c.Put(addr, v1, 10) - var wg sync.WaitGroup - wg.Add(2) - go func() { defer wg.Done(); c.Put(addr, v2, 20) }() - go func() { defer wg.Done(); c.Delete(addr) }() - wg.Wait() - c.Delete(addr) - require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) - } -} - -// Same invariant for the lazy stale-drop inside GetWithTxNum, the other -// unstriped Remove path. +// The lazy stale-drop inside GetWithTxNum removes entries; an unstriped +// Remove racing put's read-modify-write double-subtracts the displaced +// entry's size (once via freelru's OnEvict, once via put's update delta). +// Exactly one live entry remains after every round, so drift shows as a size +// mismatch. func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) addr := makeAddr(1) v1 := []byte("value-one") v2 := []byte("value-two") + wantSize := int64(len(addr) + len(v1) + 24) // key + value + entry overhead for round := 0; round < 20000; round++ { c.Put(addr, v1, 10) c.Unwind(5) // epoch bump makes the entry above stale (txNum 10 >= floor 5) @@ -930,8 +866,7 @@ func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { go func() { defer wg.Done(); c.Put(addr, v2, 20) }() go func() { defer wg.Done(); c.Get(addr) }() wg.Wait() - c.Delete(addr) - require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) + require.Equal(t, wantSize, c.SizeBytes(), "round %d: size accounting drifted", round) } } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index d746e8fdc26..2077b302db4 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -543,11 +543,6 @@ func (c *CodeCache) PutCodeSizeByCodeHash(codeHash []byte, size int, txNum uint6 &c.coh, &c.codeSizeEntries, 1, &c.putStripes[uint8(hcs)]) } -// Delete removes the address → code mapping for addr. -func (c *CodeCache) Delete(addr []byte) { - c.addrToHash.Remove(common.BytesToAddress(addr)) -} - // Clear hard-resets every layer and the epoch/floor. Use on Reset / // fork-validation paths where no entry may carry over. func (c *CodeCache) Clear() { diff --git a/execution/cache/code_cache_concurrency_test.go b/execution/cache/code_cache_concurrency_test.go index 109a01874fa..bd37cd59437 100644 --- a/execution/cache/code_cache_concurrency_test.go +++ b/execution/cache/code_cache_concurrency_test.go @@ -17,6 +17,7 @@ package cache import ( + "encoding/binary" "sync" "testing" @@ -130,7 +131,7 @@ func TestCodeCache_PutIfAbsentAtomicWithPut(t *testing.T) { fresh := []byte{0xaa, 1, 2, 3} stale := []byte{0xbb, 4, 5, 6} for round := 0; round < 20000; round++ { - cc.Delete(addr) + binary.BigEndian.PutUint64(addr[1:], uint64(round)) // a fresh addr each round, so both writers race on the bind var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); cc.Put(addr, fresh, 20) }() diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 32069c96095..58333afe0ee 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -267,11 +267,6 @@ func (c *DomainCache) Put(key []byte, value []byte, txNum uint64) { c.GenericCache.Put(key, value, txNum) } -// Delete removes the data for the given key, delegating to GenericCache. -func (c *DomainCache) Delete(key []byte) { - c.GenericCache.Delete(key) -} - // Get retrieves data for the given key. func (c *GenericCache[T]) Get(key []byte) (T, bool) { v, _, ok := c.GetWithTxNum(key) @@ -397,23 +392,11 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite return needGrow } -// Delete removes the data for the given key. Runs under the key's put stripe: -// an unstriped Remove racing put's read-modify-write would double-subtract the -// displaced entry's size (once via OnEvict, once via put's update delta). -func (c *GenericCache[T]) Delete(key []byte) { - h := maphash.Hash(key) - mu := &c.putStripes[h&(putStripeCount-1)] - mu.Lock() - defer mu.Unlock() - lru := c.data.Load() - if existing, ok := lru.Get(h); ok && bytes.Equal(existing.key, key) { - lru.Remove(h) - } -} - // dropStale removes key's entry under its put stripe, re-checking that it is // still the same key's stale entry — a concurrent put may have replaced it -// with a live one, and Remove must be striped for the same reason Delete is. +// with a live one — and because an unstriped Remove racing put's +// read-modify-write would double-subtract the displaced entry's size (once +// via OnEvict, once via put's update delta). func (c *GenericCache[T]) dropStale(h uint64, key []byte) { mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index ab5d075ffae..86d64978404 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -269,15 +269,6 @@ func (c *StateCache) put(domain kv.Domain, key []byte, value []byte, txNum uint6 } } -// Delete removes the data for the given domain and key. -func (c *StateCache) Delete(domain kv.Domain, key []byte) { - cache := c.caches[domain] - if cache == nil { - return - } - cache.Delete(key) -} - // Clear removes all mutable entries from all caches. func (c *StateCache) Clear() { for _, cache := range c.caches { From 5324e0bba17607b7cf29329795d0082280171d22 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 11:45:44 +0200 Subject: [PATCH 18/31] execution/cache: gofmt --- execution/cache/cache_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 2a815084f38..728874d6fe6 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -339,7 +339,6 @@ func TestCodeCache_CodeCapacityLimit(t *testing.T) { assert.False(t, ok, "coldest code should have been evicted") } - func TestCodeCache_Clear(t *testing.T) { c := NewCodeCache(100, 200) @@ -472,7 +471,6 @@ func TestStateCache_GetPut_UnsupportedDomain(t *testing.T) { assert.Nil(t, v) } - // Put(key, nil) must be a cache hit, not a miss. SharedDomains.GetLatest // caches deleted keys via Put(key, nil); if Get treats that as "not found", // the caller unnecessarily falls through to the DB on every read. From 0b2eafce66721df15ae03f42bd3702325f1dacef Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 12:10:41 +0200 Subject: [PATCH 19/31] execution/cache, db/state/execctx, execution/exec, execution/types/accounts: dedup comment rationales to their canonical sites The dead-fork-warmup, keccak-poisoning, resurrection and freeze-copy rationales were each stated in full at two or three sites; they now live once at their canonical spot (warmupsInFlight, the SD code read-fill, the Cache no-Delete note, maybeGrow) with terse pointers elsewhere. Local comments tightened without dropping content. --- db/state/execctx/domain_shared.go | 39 ++++++++++++----------------- execution/cache/code_cache.go | 7 +++--- execution/cache/generic_cache.go | 19 ++++++-------- execution/cache/state_cache.go | 9 ++----- execution/exec/blocks_read_ahead.go | 32 ++++++++++------------- execution/types/accounts/account.go | 4 +-- 6 files changed, 44 insertions(+), 66 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 21cd9852c42..1c8d40b7d5e 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1043,10 +1043,9 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.AccountsDomain: if len(u.val) == 0 { - // Tombstone rather than delete: a live negative defends the key - // against a straddling pre-delete read-fill re-inserting the old - // value (PutIfAbsent only defers to live entries). Same for the - // code binding, via its no-code deletion marker. + // Deletions are authoritative nil puts — a tombstone here, the + // no-code marker for the code binding — so a straddling + // pre-delete read-fill defers instead of resurrecting the value. sd.stateCache.Put(kv.AccountsDomain, u.key, nil, u.txN) sd.stateCache.Put(kv.CodeDomain, u.key, nil, u.txN) sd.stateCache.DeleteAddrCodeHash(u.key) @@ -1092,14 +1091,12 @@ func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, return sd.getLatestMetered(domain, tx, k, kvmetrics.MetricsFromContext(ctx)) } -// servableUnderBound is the shared cache gate for reads of a key an in-flight -// unwind has re-bound: the mem overlay publishes a per-key maxStep while MDBX -// still holds the dying rows, and a cached entry reflecting a step above that -// bound would diverge from the bounded read the cache-disabled path takes. -// The (txNum, epoch) floor usually already drops such entries; this keeps the -// two read paths identical regardless. Callers convert their unit first — the -// StateCache stamps txNums (divide by step size), the BranchCache stores step -// indices (no divide). +// servableUnderBound gates a cached entry against an in-flight unwind's +// per-key maxStep: a hit above the bound would diverge from the bounded read +// the cache-disabled path takes (the epoch floor usually drops such entries +// already; the gate keeps the two paths identical regardless). Callers convert +// their unit first — the StateCache stamps txNums (divide by step size), the +// BranchCache stores step indices (no divide). func servableUnderBound(cStep, maxStep kv.Step) bool { return cStep <= maxStep } @@ -1242,17 +1239,13 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k return nil, 0, fmt.Errorf("storage %x read error: %w", k, err) } - // Populate state cache on successful read, with if-absent semantics: a - // read-fill never carries newer information than a flush-apply, and an - // unconditional Put racing one (an embedded-RPC read straddling an FCU - // commit, or a bounded read during an in-flight unwind) would overwrite - // the fresher value. Stamp with an upper bound on the value's write txNum - // (the read only gives us the file/step it came from): the last txNum of - // that step. An unwind below this bound can't leave the value stale, so - // frozen-step reads stay warm across unwinds while recent-step reads are - // dropped. A negative result carries no step — stamp it with the domain's - // progress at observation time so it drops on any unwind instead of - // outliving the fact it caches. + // Populate the cache with if-absent semantics: a read-fill never carries + // newer information than a flush-apply, so it must not overwrite one + // (e.g. an embedded-RPC read straddling an FCU commit). Stamp with the + // last txNum of the step the value came from — an upper bound on its + // write txNum — so an unwind below it can't leave the entry stale. A + // negative carries no step; stamp it with the domain's progress at + // observation time so any unwind drops it. if sd.stateCache != nil { readTxNum := (uint64(step)+1)*sd.StepSize() - 1 if domain == kv.CodeDomain { diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 2077b302db4..2b4bc808a53 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -84,10 +84,9 @@ type versionedAddressID struct { codeHash [32]byte txNum uint64 epoch uint32 - // deleted marks a live no-code binding: the addr's code was deleted at - // txNum. A valid negative on read, and a binding conditional puts defer - // to — without it, a fill from a pre-delete snapshot would resurrect the - // deleted code. + // deleted marks a live no-code binding — the code-cache analogue of a + // domain tombstone: a valid negative on read that conditional binds defer + // to (see the Cache interface's no-Delete note). deleted bool } diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 58333afe0ee..6f406219fc4 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -59,11 +59,9 @@ type entry[T any] struct { // GenericCache is a sharded, LRU-evicting bounded cache for key-value // data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { - // data is the sharded LRU, replaced wholesale on a jump-grow. The swap - // happens with every put stripe held and the new generation fully copied, - // so a striped write can never land in a retired generation (a write lost - // that way would resurface the older value as live — a stale serve, not a - // benign miss) and no operation ever sees a partially-copied one. + // data is the sharded LRU, replaced wholesale on a jump-grow with every + // put stripe held and the new generation fully copied — no write lands in + // a retired generation and no reader sees a partial one (see maybeGrow). data atomic.Pointer[freelru.ShardedLRU[uint64, entry[T]]] capacityB datasize.ByteSize mode Mode @@ -206,7 +204,7 @@ func (c *GenericCache[T]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, // retired and a conditional put never sees a mid-resize gap it could fill // with a stale value; readers stay on the retiring generation until the swap // and never miss. Grows are a handful of steps per cache lifetime, so the -// stall is a bounded one-off, not a steady-state cost. +// writer stall is a bounded one-off. func (c *GenericCache[T]) maybeGrow() { c.resizeMu.Lock() defer c.resizeMu.Unlock() @@ -392,11 +390,10 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite return needGrow } -// dropStale removes key's entry under its put stripe, re-checking that it is -// still the same key's stale entry — a concurrent put may have replaced it -// with a live one — and because an unstriped Remove racing put's -// read-modify-write would double-subtract the displaced entry's size (once -// via OnEvict, once via put's update delta). +// dropStale removes key's entry under its put stripe: the re-check keeps an +// entry a concurrent put revived, and striping the Remove stops it +// double-subtracting the displaced size against put's update delta (once via +// OnEvict, once via the delta). func (c *GenericCache[T]) dropStale(h uint64, key []byte) { mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 86d64978404..7d1ca839b20 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -179,10 +179,7 @@ func (c *StateCache) putCodeWithHash(addr, code, codeHash []byte, txNum uint64, } // HasLiveCode reports whether addr has a live code-cache binding a -// conditional put would defer to — servable code bytes or a no-code deletion -// marker — without touching stats or LRU recency. Prefetchers probe it to -// skip the keccak+copy of preparing a conditional code put that such a -// binding would no-op; advisory only. +// conditional put would defer to; see CodeCache.ContainsLive. func (c *StateCache) HasLiveCode(addr []byte) bool { cc, ok := c.caches[kv.CodeDomain].(*CodeCache) return ok && cc.ContainsLive(addr) @@ -310,9 +307,7 @@ func (c *StateCache) Unwind(unwindToTxNum uint64) { } // WarmupStarted and WarmupDone bracket a fire-and-forget cache-populating -// prefetch. A prefetch put racing an unwind's epoch bump could stamp a -// dead-fork value with the post-unwind epoch, so Unwind asserts (under -// ASSERT_STATE_CACHE) that no warmup is in flight. +// prefetch; see warmupsInFlight. func (c *StateCache) WarmupStarted() { c.warmupsInFlight.Add(1) } // WarmupDone is the counterpart of WarmupStarted. diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 5a77caf77f0..c8e1eca5241 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -106,23 +106,19 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // paying the keccak+copy below. Code negatives end here too: they // are not cacheable (CodeCache drops zero-length puts). if len(v) > 0 && !cpg.sc.HasLiveCode(k) { - // Key the content cache by the code's OWN hash, never a separately - // read account codeHash: under parallel/speculative exec that hash - // can be skewed or cross-account, and a (hash, code) pair that - // doesn't satisfy keccak(code)==hash poisons every account sharing - // the hash. keccak(v) makes each entry self-consistent. + // Key the content cache by keccak(v), the code's own hash — never + // a separately read account codeHash, which parallel exec can skew + // (see the code-domain read-fill in SharedDomains.getLatestMetered). cpg.sc.PutCodeWithHashIfAbsent(k, v, crypto.Keccak256(v), (uint64(step)+1)*cpg.stepSize-1) } } else { - // Cache including nil/empty results: a probe returning no - // bytes is a valid negative answer (missing account, empty - // storage slot) and caching it lets repeated probes - // skip the file accessor stack. Mirrors revm's CacheAccount - // { account: None, status: LoadedNotExisting } pattern. - // Stamp with an upper bound on the value's write txNum: the last - // txNum of the step it came from, or for a negative — which has no - // step — the domain's progress at observation time, so the entry - // drops on any unwind instead of outliving the fact it caches. + // Cache including nil/empty results: a probe returning no bytes is + // a valid negative answer (missing account, empty storage slot) and + // caching it lets repeated probes skip the file accessor stack — + // revm's CacheAccount { account: None, status: LoadedNotExisting } + // pattern. Stamp with the last txNum of the value's step; a + // negative has no step — use the domain's progress at observation + // time so any unwind drops it. txNum := (uint64(step)+1)*cpg.stepSize - 1 if len(v) == 0 { txNum = cpg.progress(name) @@ -150,11 +146,9 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h if !bra.warming.CompareAndSwap(false, true) { return } - // Gauge ordering makes "WaitForWarmup returned ⟹ gauge is zero" hold on - // its own: WarmupStarted only after warmWg.Add, so a Wait can't slip - // between them and return with the gauge raised; WarmupDone before - // warmWg.Done (defers run LIFO), so a Wait can't return before the - // gauge drops. StateCache.Unwind asserts on the gauge. + // Ordering makes "WaitForWarmup returned ⟹ gauge is zero" hold on its + // own: WarmupStarted only after warmWg.Add, WarmupDone before + // warmWg.Done (defers run LIFO). StateCache.Unwind asserts on the gauge. bra.warmWg.Add(1) if bra.stateCache != nil { bra.stateCache.WarmupStarted() diff --git a/execution/types/accounts/account.go b/execution/types/accounts/account.go index 587183afc7a..5251e376237 100644 --- a/execution/types/accounts/account.go +++ b/execution/types/accounts/account.go @@ -648,8 +648,8 @@ func DeserialiseV3(a *Account, enc []byte) error { // codeHash interning) that DeserialiseV3 pays. It parses only up to and // including the codeHash field — later fields are not validated. Returns a // subslice of enc — valid only while enc is — or nil when the record is -// malformed up to that field or the account has no code (including a -// non-canonical record spelling out the empty or zero sentinel, which +// malformed up to that field or the account has no code (including +// non-canonical spellings of the empty or zero sentinel, which // CodeHash.IsEmpty treats as no-code). func DeserialiseV3CodeHash(enc []byte) []byte { pos := 0 From 84b146e7e0f32657acb78c0cd112cfc5af8f3dd5 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Thu, 9 Jul 2026 12:17:35 +0200 Subject: [PATCH 20/31] execution/exec: skip caching negatives when the warmup getter has no progress oracle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cachePopulatingGetter built without progress nil-panicked on the first cold-negative read — inside the fire-and-forget warmup goroutine. Without the oracle there is no honest "true as of" stamp, so skip the fill: the alternatives are the crash or an unwind-immortal negative. --- execution/exec/blocks_read_ahead.go | 5 +++++ execution/exec/blocks_read_ahead_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index c8e1eca5241..98afd4c6acf 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -121,6 +121,11 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // time so any unwind drops it. txNum := (uint64(step)+1)*cpg.stepSize - 1 if len(v) == 0 { + if cpg.progress == nil { + // No progress oracle → no honest stamp; skip rather than + // cache an unwind-immortal negative. + return v, step, err + } txNum = cpg.progress(name) } cpg.sc.PutIfAbsent(name, k, v, txNum) diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 71543c6fac7..6fe6e09c721 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -163,4 +163,18 @@ func TestCachePopulatingGetterNegativeDropsOnUnwind(t *testing.T) { require.False(t, ok, "a negative observed at txNum 10M must not survive an unwind to 5M") } +// A getter constructed without a progress oracle must skip caching negatives +// (an honest stamp is impossible), not panic. +func TestCachePopulatingGetterNilProgressSkipsNegative(t *testing.T) { + key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") + sc := newTestStateCache() + cpg := &cachePopulatingGetter{g: stubTemporalGetter{v: nil}, sc: sc, stepSize: 1_562_500} + require.NotPanics(t, func() { + _, _, err := cpg.GetLatest(kv.AccountsDomain, key) + require.NoError(t, err) + }) + _, ok := sc.Get(kv.AccountsDomain, key) + require.False(t, ok, "no progress oracle — the negative must not be cached") +} + func zeroProgress(kv.Domain) uint64 { return 0 } From 972f75dfb1c689fe4515507096274871d5310e50 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Mon, 13 Jul 2026 22:11:09 +0200 Subject: [PATCH 21/31] =?UTF-8?q?Revert=20"execution/cache:=20remove=20Del?= =?UTF-8?q?ete=20=E2=80=94=20deletions=20are=20authoritative=20nil=20puts"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9072533ca6a6afb9abb4de8fe70934af26d0a0e2. --- execution/cache/cache.go | 8 +- execution/cache/cache_test.go | 83 +++++++++++++++++-- execution/cache/code_cache.go | 5 ++ .../cache/code_cache_concurrency_test.go | 3 +- execution/cache/generic_cache.go | 22 ++++- execution/cache/state_cache.go | 9 ++ 6 files changed, 113 insertions(+), 17 deletions(-) diff --git a/execution/cache/cache.go b/execution/cache/cache.go index 10be2806f46..fd59c16f2d5 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -33,13 +33,11 @@ type Cache interface { // PutIfAbsent is Put except that a live entry for key is left untouched // (a stale one is replaced) — for prefetch writers, whose snapshot may // already be superseded by an authoritative Put. - // - // There is deliberately no Delete: a deletion is an authoritative Put of - // nil (a tombstone / no-code marker), so conditional fills from stale - // snapshots defer to it. Removing the entry instead would let such a fill - // resurrect the deleted value. PutIfAbsent(key []byte, value []byte, txNum uint64) + // Delete removes the data for the given key. + Delete(key []byte) + // Clear removes all mutable entries from the cache. Clear() diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 728874d6fe6..d1f9bcd0fb3 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -155,6 +155,20 @@ func TestDomainCache_PutEvictsWhenFull_EvictMode(t *testing.T) { assert.Positive(t, missingCount, "ModeEvictLRU should have evicted some early entries") } +func TestDomainCache_Delete(t *testing.T) { + c := NewDomainCacheMode(100, ModeEvictLRU) + + addr := makeAddr(1) + c.Put(addr, makeValue(1), 0) + assert.Equal(t, 1, c.Len()) + + c.Delete(addr) + assert.Equal(t, 0, c.Len()) + + _, ok := c.Get(addr) + assert.False(t, ok) +} + func TestDomainCache_Clear(t *testing.T) { c := NewDomainCacheMode(100, ModeEvictLRU) @@ -339,6 +353,22 @@ func TestCodeCache_CodeCapacityLimit(t *testing.T) { assert.False(t, ok, "coldest code should have been evicted") } +func TestCodeCache_Delete(t *testing.T) { + c := NewCodeCache(100, 200) + + addr := makeAddr(1) + code := makeCode(1) + c.Put(addr, code, 0) + + c.Delete(addr) + assert.Equal(t, 0, c.Len()) + // Code should still exist (immutable) + assert.Equal(t, 1, c.CodeLen()) + + _, ok := c.Get(addr) + assert.False(t, ok) +} + func TestCodeCache_Clear(t *testing.T) { c := NewCodeCache(100, 200) @@ -471,6 +501,17 @@ func TestStateCache_GetPut_UnsupportedDomain(t *testing.T) { assert.Nil(t, v) } +func TestStateCache_Delete(t *testing.T) { + c := NewStateCache(100, 100, 100, 100) + + addr := makeAddr(1) + c.Put(kv.AccountsDomain, addr, makeValue(1), 0) + c.Delete(kv.AccountsDomain, addr) + + _, ok := c.Get(kv.AccountsDomain, addr) + assert.False(t, ok) +} + // Put(key, nil) must be a cache hit, not a miss. SharedDomains.GetLatest // caches deleted keys via Put(key, nil); if Get treats that as "not found", // the caller unnecessarily falls through to the DB on every read. @@ -503,6 +544,13 @@ func TestStateCache_PutEmptySlice_ThenGet_IsCacheHit(t *testing.T) { assert.Empty(t, v) } +func TestStateCache_Delete_UnsupportedDomain(t *testing.T) { + c := NewStateCache(100, 100, 100, 100) + + // Should not panic + c.Delete(kv.ReceiptDomain, makeAddr(1)) +} + func TestStateCache_Clear(t *testing.T) { c := NewStateCache(100, 100, 100, 100) @@ -830,10 +878,11 @@ func TestCodeCache_PutWithCodeHashIfAbsent(t *testing.T) { // clobber it — the prefetch-vs-flush staleness this cache guards against. func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) + addr := makeAddr(1) fresh := []byte("fresh") stale := []byte("stale") for round := 0; round < 20000; round++ { - addr := makeAddr(round) // a fresh key each round, so both writers race on the insert path + c.Delete(addr) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); c.Put(addr, fresh, 20) }() @@ -845,17 +894,34 @@ func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { } } -// The lazy stale-drop inside GetWithTxNum removes entries; an unstriped -// Remove racing put's read-modify-write double-subtracts the displaced -// entry's size (once via freelru's OnEvict, once via put's update delta). -// Exactly one live entry remains after every round, so drift shows as a size -// mismatch. +// A Delete racing an update-in-place put must not double-subtract the +// displaced entry's size: freelru's OnEvict subtracts it for the Remove, and +// put's update delta subtracts it again unless the two writers share the +// key's stripe. +func TestDomainCache_DeleteAtomicWithPut_NoSizeDrift(t *testing.T) { + c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) + addr := makeAddr(1) + v1 := []byte("value-one") + v2 := []byte("value-two") + for round := 0; round < 20000; round++ { + c.Put(addr, v1, 10) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); c.Put(addr, v2, 20) }() + go func() { defer wg.Done(); c.Delete(addr) }() + wg.Wait() + c.Delete(addr) + require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) + } +} + +// Same invariant for the lazy stale-drop inside GetWithTxNum, the other +// unstriped Remove path. func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) addr := makeAddr(1) v1 := []byte("value-one") v2 := []byte("value-two") - wantSize := int64(len(addr) + len(v1) + 24) // key + value + entry overhead for round := 0; round < 20000; round++ { c.Put(addr, v1, 10) c.Unwind(5) // epoch bump makes the entry above stale (txNum 10 >= floor 5) @@ -864,7 +930,8 @@ func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { go func() { defer wg.Done(); c.Put(addr, v2, 20) }() go func() { defer wg.Done(); c.Get(addr) }() wg.Wait() - require.Equal(t, wantSize, c.SizeBytes(), "round %d: size accounting drifted", round) + c.Delete(addr) + require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) } } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 06587bae4bc..4e36c53d0cb 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -539,6 +539,11 @@ func (c *CodeCache) PutCodeSizeByCodeHash(codeHash []byte, size int, txNum uint6 &c.coh, &c.codeSizeEntries, 1, &c.putStripes[uint8(hcs)]) } +// Delete removes the address → code mapping for addr. +func (c *CodeCache) Delete(addr []byte) { + c.addrToHash.Remove(common.BytesToAddress(addr)) +} + // Clear hard-resets every layer and the epoch/floor. Use on Reset / // fork-validation paths where no entry may carry over. func (c *CodeCache) Clear() { diff --git a/execution/cache/code_cache_concurrency_test.go b/execution/cache/code_cache_concurrency_test.go index bd37cd59437..109a01874fa 100644 --- a/execution/cache/code_cache_concurrency_test.go +++ b/execution/cache/code_cache_concurrency_test.go @@ -17,7 +17,6 @@ package cache import ( - "encoding/binary" "sync" "testing" @@ -131,7 +130,7 @@ func TestCodeCache_PutIfAbsentAtomicWithPut(t *testing.T) { fresh := []byte{0xaa, 1, 2, 3} stale := []byte{0xbb, 4, 5, 6} for round := 0; round < 20000; round++ { - binary.BigEndian.PutUint64(addr[1:], uint64(round)) // a fresh addr each round, so both writers race on the bind + cc.Delete(addr) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); cc.Put(addr, fresh, 20) }() diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index aa0c7083998..a3c406fcd38 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -253,6 +253,11 @@ func (c *DomainCache) Put(key []byte, value []byte, txNum uint64) { c.GenericCache.Put(key, value, txNum) } +// Delete removes the data for the given key, delegating to GenericCache. +func (c *DomainCache) Delete(key []byte) { + c.GenericCache.Delete(key) +} + // Get retrieves data for the given key. func (c *GenericCache[T]) Get(key []byte) (T, bool) { v, _, ok := c.GetWithTxNum(key) @@ -378,10 +383,23 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite return needGrow } +// Delete removes the data for the given key. Runs under the key's put stripe: +// an unstriped Remove racing put's read-modify-write would double-subtract the +// displaced entry's size (once via OnEvict, once via put's update delta). +func (c *GenericCache[T]) Delete(key []byte) { + h := maphash.Hash(key) + mu := &c.putStripes[h&(putStripeCount-1)] + mu.Lock() + defer mu.Unlock() + lru := c.data.Load() + if existing, ok := lru.Get(h); ok && bytes.Equal(existing.key, key) { + lru.Remove(h) + } +} + // dropStale removes key's entry under its put stripe: the re-check keeps an // entry a concurrent put revived, and striping the Remove stops it -// double-subtracting the displaced size against put's update delta (once via -// OnEvict, once via the delta). +// double-subtracting the displaced size against put's update delta. func (c *GenericCache[T]) dropStale(h uint64, key []byte) { mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 7d1ca839b20..b73c815bed2 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -266,6 +266,15 @@ func (c *StateCache) put(domain kv.Domain, key []byte, value []byte, txNum uint6 } } +// Delete removes the data for the given domain and key. +func (c *StateCache) Delete(domain kv.Domain, key []byte) { + cache := c.caches[domain] + if cache == nil { + return + } + cache.Delete(key) +} + // Clear removes all mutable entries from all caches. func (c *StateCache) Clear() { for _, cache := range c.caches { From 5317c020478b3d8e2ff95f94dd4b02ae51131bd9 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Mon, 13 Jul 2026 22:12:00 +0200 Subject: [PATCH 22/31] Revert "execution/cache, db/state/execctx, execution/types/accounts: no-code deletion marker for the code cache" This reverts commit 23bcf563e86d9fec08bbd667f6b4e8bf068234cf. --- db/state/execctx/domain_shared.go | 10 +-- db/state/execctx/statecache_readfill_test.go | 71 -------------------- execution/cache/cache_test.go | 54 +-------------- execution/cache/code_cache.go | 26 ++----- execution/cache/state_cache.go | 4 +- 5 files changed, 13 insertions(+), 152 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 1c8d40b7d5e..2bb84021624 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1043,11 +1043,11 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.AccountsDomain: if len(u.val) == 0 { - // Deletions are authoritative nil puts — a tombstone here, the - // no-code marker for the code binding — so a straddling - // pre-delete read-fill defers instead of resurrecting the value. + // Tombstone rather than delete: a live negative defends the key + // against a straddling pre-delete read-fill re-inserting the old + // value (PutIfAbsent only defers to live entries). sd.stateCache.Put(kv.AccountsDomain, u.key, nil, u.txN) - sd.stateCache.Put(kv.CodeDomain, u.key, nil, u.txN) + sd.stateCache.Delete(kv.CodeDomain, u.key) sd.stateCache.DeleteAddrCodeHash(u.key) } else { sd.stateCache.Put(kv.AccountsDomain, u.key, u.val, u.txN) @@ -1061,7 +1061,7 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.CodeDomain: if len(u.val) == 0 { - sd.stateCache.Put(kv.CodeDomain, u.key, nil, u.txN) + sd.stateCache.Delete(kv.CodeDomain, u.key) } else { // Validated committed code: populate the addr layer AND the // content-addressed codeHash->code map, keyed by keccak(v) so each diff --git a/db/state/execctx/statecache_readfill_test.go b/db/state/execctx/statecache_readfill_test.go index 85689293035..21da1b63433 100644 --- a/db/state/execctx/statecache_readfill_test.go +++ b/db/state/execctx/statecache_readfill_test.go @@ -281,74 +281,3 @@ func TestReadFill_DoesNotResurrectDeletedKey(t *testing.T) { require.True(t, ok) require.Equal(t, uint64(20), cTxNum) } - -// The code-domain equivalent of the tombstone test: a code deletion must -// leave a live no-code marker on the addr binding, or a read-fill from a -// straddling pre-delete snapshot re-binds the deleted code as a live hit. -func TestReadFill_DoesNotResurrectDeletedCode(t *testing.T) { - if testing.Short() { - t.Skip() - } - t.Parallel() - - const stepSize = uint64(16) - ctx := t.Context() - db := newTestDb(t, stepSize) - sc := newSmallStateCache() - - addr := make([]byte, 20) - addr[0] = 0xcc - code := []byte{0xaa, 1, 2, 3} - - rwTx1, err := db.BeginTemporalRw(ctx) - require.NoError(t, err) - defer rwTx1.Rollback() - sd1, err := execctx.NewSharedDomains(ctx, rwTx1, log.New()) - require.NoError(t, err) - defer sd1.Close() - sd1.SetStateCacheForTest(sc) - sd1.SetTxNum(10) - require.NoError(t, sd1.DomainPut(kv.CodeDomain, rwTx1, addr, code, 10, nil)) - require.NoError(t, sd1.Commit(ctx, rwTx1)) - - // A reader snapshot from before the deletion. - roTxOld, err := db.BeginTemporalRo(ctx) - require.NoError(t, err) - defer roTxOld.Rollback() - - rwTx2, err := db.BeginTemporalRw(ctx) - require.NoError(t, err) - defer rwTx2.Rollback() - sd2, err := execctx.NewSharedDomains(ctx, rwTx2, log.New()) - require.NoError(t, err) - defer sd2.Close() - sd2.SetStateCacheForTest(sc) - sd2.SetTxNum(20) - require.NoError(t, sd2.DomainDel(kv.CodeDomain, rwTx2, addr, 20, code)) - require.NoError(t, sd2.Commit(ctx, rwTx2)) - - // The straddling reader: any fill it makes must not resurrect the code. - sdOld, err := execctx.NewSharedDomains(ctx, roTxOld, log.New()) - require.NoError(t, err) - defer sdOld.Close() - sdOld.SetStateCacheForTest(sc) - _, _, err = sdOld.GetLatest(kv.CodeDomain, roTxOld, addr) - require.NoError(t, err) - - roTxNew, err := db.BeginTemporalRo(ctx) - require.NoError(t, err) - defer roTxNew.Rollback() - sdNew, err := execctx.NewSharedDomains(ctx, roTxNew, log.New()) - require.NoError(t, err) - defer sdNew.Close() - sdNew.SetStateCacheForTest(sc) - v, _, err := sdNew.GetLatest(kv.CodeDomain, roTxNew, addr) - require.NoError(t, err) - require.Empty(t, v, "the straddling read-fill must not resurrect the deleted code") - - // The marker is stamped with the delete's txNum, so an unwind at or below - // it drops the negative instead of letting it outlive the deletion. - _, cTxNum, ok := sc.GetWithTxNum(kv.CodeDomain, addr) - require.True(t, ok, "the deletion must be cached as a live no-code marker") - require.Equal(t, uint64(20), cTxNum) -} diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index d1f9bcd0fb3..2a188fd998c 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -243,20 +243,9 @@ func TestCodeCache_PutEmptyCode(t *testing.T) { addr := makeAddr(1) c.Put(addr, []byte{}, 0) - // An authoritative empty put is a deletion: it binds a no-code marker (a - // valid negative), never code bytes. - assert.Equal(t, 1, c.Len()) + // Should not store empty code + assert.Equal(t, 0, c.Len()) assert.Equal(t, 0, c.CodeLen()) - v, ok := c.Get(addr) - assert.True(t, ok) - assert.Empty(t, v) - - // A conditional empty put stays a no-op. - c2 := NewCodeCache(100, 200) - c2.PutIfAbsent(addr, []byte{}, 0) - assert.Equal(t, 0, c2.Len()) - _, ok = c2.Get(addr) - assert.False(t, ok) } func TestCodeCache_CodeDeduplication(t *testing.T) { @@ -935,45 +924,6 @@ func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { } } -// A code deletion (an overwrite put of nil) must leave a live no-code marker: -// it serves as a valid negative, a conditional bind defers to it, an -// authoritative rebind replaces it, and an unwind at or below the deletion -// drops it. -func TestCodeCache_DeletionMarker(t *testing.T) { - cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) - addr := makeAddr(1) - code := []byte{0xaa, 1, 2, 3} - cc.PutWithCodeHash(addr, code, crypto.Keccak256(code), 10) - - cc.Put(addr, nil, 20) // the flush-apply shape of a deletion - - v, cTxNum, ok := cc.GetWithTxNum(addr) - require.True(t, ok, "a deletion is a valid negative, not a miss") - require.Empty(t, v) - require.Equal(t, uint64(20), cTxNum) - require.True(t, cc.ContainsLive(addr), "a conditional put would defer to the marker") - - // A straddling if-absent fill of the old code defers to the marker. - cc.PutWithCodeHashIfAbsent(addr, code, crypto.Keccak256(code), 15) - v, _, ok = cc.GetWithTxNum(addr) - require.True(t, ok) - require.Empty(t, v, "an if-absent fill must not resurrect deleted code") - - // An authoritative rebind (new deployment) replaces the marker. - fresh := []byte{0xbb, 4, 5, 6} - cc.PutWithCodeHash(addr, fresh, crypto.Keccak256(fresh), 30) - v, ok = cc.Get(addr) - require.True(t, ok) - require.Equal(t, fresh, v) - - // An unwind at or below the deletion drops the marker. - cc2 := NewCodeCache(1*datasize.MB, 1*datasize.MB) - cc2.Put(addr, nil, 20) - cc2.Unwind(15) - _, _, ok = cc2.GetWithTxNum(addr) - require.False(t, ok, "a marker from a rolled-back deletion must not survive the unwind") -} - func TestCodeCache_ContainsLive(t *testing.T) { cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) addr := makeAddr(1) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 4e36c53d0cb..2c1c442bdc4 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -84,10 +84,6 @@ type versionedAddressID struct { codeHash [32]byte txNum uint64 epoch uint32 - // deleted marks a live no-code binding — the code-cache analogue of a - // domain tombstone: a valid negative on read that conditional binds defer - // to (see the Cache interface's no-Delete note). - deleted bool } type addrCodeHashEntry struct { @@ -296,9 +292,6 @@ func (c *CodeCache) GetWithTxNum(addr []byte) ([]byte, uint64, bool) { return nil, 0, false } c.addrHits.Add(1) - if vID.deleted { - return nil, vID.txNum, true - } ce, ok := c.hashToCode.Get(vID.addrID) if !ok || len(ce.code) == 0 { @@ -345,13 +338,6 @@ func (c *CodeCache) PutIfAbsent(addr []byte, code []byte, txNum uint64) { // cold code can't both Add and permanently inflate codeSize. func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum uint64, overwriteAddr bool) { if len(code) == 0 { - // An authoritative nil put is a deletion — bind a live no-code marker. - // Conditional nil puts stay no-ops. - if overwriteAddr && len(addr) > 0 { - c.addrBindMu.Lock() - c.addrToHash.Add(common.BytesToAddress(addr), versionedAddressID{deleted: true, txNum: txNum, epoch: c.coh.Epoch()}) - c.addrBindMu.Unlock() - } return } ep := c.coh.Epoch() @@ -375,19 +361,15 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui &c.coh, &c.codeSize, 8, &c.putStripes[uint8(codeID)]) } -// ContainsLive reports whether addr has a live binding a conditional put -// would defer to — servable code bytes or a no-code deletion marker — without -// touching hit/miss counters or LRU recency. Prefetchers probe it to skip the -// keccak+copy work of preparing a conditional put that such a binding would -// no-op; advisory only. +// ContainsLive reports whether addr resolves to live code bytes through the +// addr→code binding, without touching hit/miss counters or LRU recency. +// Prefetchers probe it to skip the keccak+copy work of preparing a conditional +// put that a live binding would no-op; advisory only. func (c *CodeCache) ContainsLive(addr []byte) bool { vID, ok := c.addrToHash.Peek(common.BytesToAddress(addr)) if !ok || c.isStale(vID.txNum, vID.epoch) { return false } - if vID.deleted { - return true - } ce, ok := c.hashToCode.Peek(vID.addrID) if !ok || len(ce.code) == 0 || c.isStale(ce.txNum, ce.epoch) { return false diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index b73c815bed2..28721b082ae 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -178,8 +178,8 @@ func (c *StateCache) putCodeWithHash(addr, code, codeHash []byte, txNum uint64, } } -// HasLiveCode reports whether addr has a live code-cache binding a -// conditional put would defer to; see CodeCache.ContainsLive. +// HasLiveCode reports whether addr resolves to live code bytes; see +// CodeCache.ContainsLive. func (c *StateCache) HasLiveCode(addr []byte) bool { cc, ok := c.caches[kv.CodeDomain].(*CodeCache) return ok && cc.ContainsLive(addr) From 69baf33ae1b3e2cee775a78d3be9e7cfc5fd6c0a Mon Sep 17 00:00:00 2001 From: yperbasis Date: Mon, 13 Jul 2026 22:12:15 +0200 Subject: [PATCH 23/31] Revert "db/state/execctx: tombstone deleted keys in the flush cache-apply" This reverts commit 561b29d8ff27a81898cb70b932a6d7c0089a6dde. --- db/state/execctx/domain_shared.go | 7 +- db/state/execctx/statecache_readfill_test.go | 72 -------------------- 2 files changed, 2 insertions(+), 77 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 2bb84021624..60b66f55c61 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1043,10 +1043,7 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.AccountsDomain: if len(u.val) == 0 { - // Tombstone rather than delete: a live negative defends the key - // against a straddling pre-delete read-fill re-inserting the old - // value (PutIfAbsent only defers to live entries). - sd.stateCache.Put(kv.AccountsDomain, u.key, nil, u.txN) + sd.stateCache.Delete(kv.AccountsDomain, u.key) sd.stateCache.Delete(kv.CodeDomain, u.key) sd.stateCache.DeleteAddrCodeHash(u.key) } else { @@ -1055,7 +1052,7 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } case kv.StorageDomain: if len(u.val) == 0 { - sd.stateCache.Put(kv.StorageDomain, u.key, nil, u.txN) + sd.stateCache.Delete(kv.StorageDomain, u.key) } else { sd.stateCache.Put(kv.StorageDomain, u.key, u.val, u.txN) } diff --git a/db/state/execctx/statecache_readfill_test.go b/db/state/execctx/statecache_readfill_test.go index 21da1b63433..7209a1cac21 100644 --- a/db/state/execctx/statecache_readfill_test.go +++ b/db/state/execctx/statecache_readfill_test.go @@ -209,75 +209,3 @@ func TestReadFill_NegativeStampedWithProgress(t *testing.T) { _, ok = sc.Get(kv.AccountsDomain, missing) require.False(t, ok, "a negative observed at progress 100 must not survive an unwind to 50") } - -// A flush-apply for a deletion must leave a live tombstone, not remove the -// entry: with the key absent, a read-fill from a straddling pre-delete -// snapshot re-inserts the deleted value as live (PutIfAbsent only defers to -// live entries), and the resurrected account is served as canonical. -func TestReadFill_DoesNotResurrectDeletedKey(t *testing.T) { - if testing.Short() { - t.Skip() - } - t.Parallel() - - const stepSize = uint64(16) - ctx := t.Context() - db := newTestDb(t, stepSize) - sc := newSmallStateCache() - - key := make([]byte, 20) - key[0] = 0xbb - v1 := encAccount(1) - - rwTx1, err := db.BeginTemporalRw(ctx) - require.NoError(t, err) - defer rwTx1.Rollback() - sd1, err := execctx.NewSharedDomains(ctx, rwTx1, log.New()) - require.NoError(t, err) - defer sd1.Close() - sd1.SetStateCacheForTest(sc) - sd1.SetTxNum(10) - require.NoError(t, sd1.DomainPut(kv.AccountsDomain, rwTx1, key, v1, 10, nil)) - require.NoError(t, sd1.Commit(ctx, rwTx1)) - - // A reader snapshot from before the deletion. - roTxOld, err := db.BeginTemporalRo(ctx) - require.NoError(t, err) - defer roTxOld.Rollback() - - rwTx2, err := db.BeginTemporalRw(ctx) - require.NoError(t, err) - defer rwTx2.Rollback() - sd2, err := execctx.NewSharedDomains(ctx, rwTx2, log.New()) - require.NoError(t, err) - defer sd2.Close() - sd2.SetStateCacheForTest(sc) - sd2.SetTxNum(20) - require.NoError(t, sd2.DomainDel(kv.AccountsDomain, rwTx2, key, 20, v1)) - require.NoError(t, sd2.Commit(ctx, rwTx2)) - - // The straddling reader: any fill it makes must not resurrect the account. - sdOld, err := execctx.NewSharedDomains(ctx, roTxOld, log.New()) - require.NoError(t, err) - defer sdOld.Close() - sdOld.SetStateCacheForTest(sc) - _, _, err = sdOld.GetLatest(kv.AccountsDomain, roTxOld, key) - require.NoError(t, err) - - roTxNew, err := db.BeginTemporalRo(ctx) - require.NoError(t, err) - defer roTxNew.Rollback() - sdNew, err := execctx.NewSharedDomains(ctx, roTxNew, log.New()) - require.NoError(t, err) - defer sdNew.Close() - sdNew.SetStateCacheForTest(sc) - v, _, err := sdNew.GetLatest(kv.AccountsDomain, roTxNew, key) - require.NoError(t, err) - require.Empty(t, v, "the straddling read-fill must not resurrect the deleted account") - - // The tombstone is stamped with the delete's txNum, so an unwind at or - // below it drops the negative instead of letting it outlive the deletion. - _, cTxNum, ok := sc.GetWithTxNum(kv.AccountsDomain, key) - require.True(t, ok) - require.Equal(t, uint64(20), cTxNum) -} From f3984be5c4229f4bd04b398afc779e2bf2c2f2af Mon Sep 17 00:00:00 2001 From: yperbasis Date: Mon, 13 Jul 2026 22:17:27 +0200 Subject: [PATCH 24/31] execution/cache: preserve non-tombstone test cleanups Keep fresh-key setup in the PutIfAbsent concurrency tests and exact residency accounting in the stale-drop test. Restore only the Delete-specific coverage needed after extracting tombstones from this PR. --- execution/cache/cache_test.go | 15 ++++++++------- execution/cache/code_cache_concurrency_test.go | 3 ++- execution/cache/generic_cache.go | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 2a188fd998c..3bd62e140a2 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -867,11 +867,10 @@ func TestCodeCache_PutWithCodeHashIfAbsent(t *testing.T) { // clobber it — the prefetch-vs-flush staleness this cache guards against. func TestDomainCache_PutIfAbsentAtomicWithPut(t *testing.T) { c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) - addr := makeAddr(1) fresh := []byte("fresh") stale := []byte("stale") for round := 0; round < 20000; round++ { - c.Delete(addr) + addr := makeAddr(round) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); c.Put(addr, fresh, 20) }() @@ -904,23 +903,25 @@ func TestDomainCache_DeleteAtomicWithPut_NoSizeDrift(t *testing.T) { } } -// Same invariant for the lazy stale-drop inside GetWithTxNum, the other -// unstriped Remove path. +// The lazy stale-drop inside GetWithTxNum removes entries; an unstriped +// Remove racing put's read-modify-write double-subtracts the displaced +// entry's size. Exactly one live entry remains after every round, so drift +// shows as a size mismatch. func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) addr := makeAddr(1) v1 := []byte("value-one") v2 := []byte("value-two") + wantSize := int64(len(addr) + len(v1) + 24) for round := 0; round < 20000; round++ { c.Put(addr, v1, 10) - c.Unwind(5) // epoch bump makes the entry above stale (txNum 10 >= floor 5) + c.Unwind(5) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); c.Put(addr, v2, 20) }() go func() { defer wg.Done(); c.Get(addr) }() wg.Wait() - c.Delete(addr) - require.Zero(t, c.SizeBytes(), "round %d: size accounting drifted", round) + require.Equal(t, wantSize, c.SizeBytes(), "round %d: size accounting drifted", round) } } diff --git a/execution/cache/code_cache_concurrency_test.go b/execution/cache/code_cache_concurrency_test.go index 109a01874fa..fb2a63fc12d 100644 --- a/execution/cache/code_cache_concurrency_test.go +++ b/execution/cache/code_cache_concurrency_test.go @@ -17,6 +17,7 @@ package cache import ( + "encoding/binary" "sync" "testing" @@ -130,7 +131,7 @@ func TestCodeCache_PutIfAbsentAtomicWithPut(t *testing.T) { fresh := []byte{0xaa, 1, 2, 3} stale := []byte{0xbb, 4, 5, 6} for round := 0; round < 20000; round++ { - cc.Delete(addr) + binary.BigEndian.PutUint64(addr[1:], uint64(round)) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done(); cc.Put(addr, fresh, 20) }() diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index a3c406fcd38..92ad1b66dbd 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -399,7 +399,8 @@ func (c *GenericCache[T]) Delete(key []byte) { // dropStale removes key's entry under its put stripe: the re-check keeps an // entry a concurrent put revived, and striping the Remove stops it -// double-subtracting the displaced size against put's update delta. +// double-subtracting the displaced size against put's update delta (once via +// OnEvict, once via the delta). func (c *GenericCache[T]) dropStale(h uint64, key []byte) { mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() From 975ecbbb030d09bc5e3ea85c3f6120a073f7dcea Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 11:26:04 +0200 Subject: [PATCH 25/31] execution/cache: fence GenericCache.Clear with the put stripes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A put racing Clear could load the retiring generation, land its entry where no reader sees it, and add the entry's size after Clear zeroed the counter — a phantom byte count that never drains. Clear now runs the counter reset, coherence re-init and generation swap with every put stripe held, mirroring maybeGrow's swap; lock order (resizeMu → stripes) is unchanged, so writers either complete before the swap or land in the fresh generation. --- execution/cache/cache_test.go | 23 +++++++++++++++++++++++ execution/cache/generic_cache.go | 24 +++++++++++++++++------- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 3bd62e140a2..a3abba6ac6c 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -925,6 +925,29 @@ func TestDomainCache_StaleDropAtomicWithPut_NoSizeDrift(t *testing.T) { } } +// A Clear racing a put must not leave phantom bytes: unless Clear excludes +// writers via the put stripes, a put that loaded the retiring generation +// lands its entry where no reader sees it and adds the entry's size after +// Clear zeroed the counter — inflating SizeBytes for an invisible entry. +func TestDomainCache_ClearAtomicWithPut_NoSizeDrift(t *testing.T) { + c := NewDomainCacheMode(1*datasize.MB, ModeEvictLRU) + addr := makeAddr(1) + v1 := []byte("value-one") + entrySize := int64(len(addr) + len(v1) + 24) + for round := 0; round < 20000; round++ { + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); c.Put(addr, v1, 10) }() + go func() { defer wg.Done(); c.Clear() }() + wg.Wait() + wantSize := int64(0) + if _, ok := c.Get(addr); ok { + wantSize = entrySize + } + require.Equal(t, wantSize, c.SizeBytes(), "round %d: size accounting drifted", round) + } +} + func TestCodeCache_ContainsLive(t *testing.T) { cc := NewCodeCache(1*datasize.MB, 1*datasize.MB) addr := makeAddr(1) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 92ad1b66dbd..84975eb0efb 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -59,9 +59,10 @@ type entry[T any] struct { // GenericCache is a sharded, LRU-evicting bounded cache for key-value // data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { - // data is the sharded LRU, replaced wholesale on a jump-grow with every - // put stripe held and the new generation fully copied — no write lands in - // a retired generation and no reader sees a partial one (see maybeGrow). + // data is the sharded LRU, replaced wholesale only with every put stripe + // held — on a jump-grow (fully copied generation) and on Clear (fresh + // empty one) — so no write lands in a retired generation and no reader + // sees a partial copy (see maybeGrow, Clear). data atomic.Pointer[freelru.ShardedLRU[uint64, entry[T]]] capacityB datasize.ByteSize mode Mode @@ -415,10 +416,10 @@ func (c *GenericCache[T]) dropStale(h uint64, key []byte) { // unwindFloor) coherence pair: with no entries left, no stale (txNum, epoch) // can survive, so a fresh floor keeps subsequent Puts at the live epoch // serviceable. Mirrors CodeCache.Clear (which already did this — the two had -// drifted). +// drifted). The counter reset and the generation swap run with every put +// stripe held — like maybeGrow's — so a racing put can neither land in the +// retired generation nor add its size after the reset. func (c *GenericCache[T]) Clear() { - c.currentSize.Store(0) - c.coh.Init() // Shrink back to the start size and return the grown budget to the envelope, // keeping the cache adaptive across fork-validation/reset (it regrows on // demand). A no-op Purge would leave the grown slot array resident. @@ -428,8 +429,17 @@ func (c *GenericCache[T]) Clear() { cachebudget.Global.Release(c.reservedBytes - int64(c.startCap)*c.avgEntryBytes) c.reservedBytes = int64(c.startCap) * c.avgEntryBytes } + next := c.newShards(c.startCap) // allocate before excluding writers + for i := range c.putStripes { + c.putStripes[i].Lock() + } + c.currentSize.Store(0) + c.coh.Init() c.curCap.Store(c.startCap) - c.data.Store(c.newShards(c.startCap)) + c.data.Store(next) + for i := range c.putStripes { + c.putStripes[i].Unlock() + } } // Close returns this cache's envelope reservation so later caches can grow into From f3f4e5752bd6aae9aa59b8a6088a2fc42cf2cd1b Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 12:01:55 +0200 Subject: [PATCH 26/31] execution/cache: account currentSize solely via OnEvict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A capacity eviction is a size-subtracting writer the put stripes cannot serialize: freelru picks its victim per shard (hash bits 16+), so an insert on one stripe can evict a key whose own update — on another stripe — sits between its Get and Add, and the update's newSize-existing.size delta then double-subtracts the victim's size. Replace the delta accounting (update and collision branches) with remove-then-add, making the OnEvict callback the sole subtractor, as putContent already does. Intentional removals (update, Delete, dropStale) are compensated in the evictions metric, which also stops stale drops counting in both staleEvicted and evictions. --- execution/cache/generic_cache.go | 46 ++++++++++++------- .../cache/generic_cache_concurrency_test.go | 34 ++++++++++++++ 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 84975eb0efb..62f6c5a2a33 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -173,7 +173,11 @@ func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntr } // newShards builds a sharded LRU of the given capacity with this cache's evict -// callback wired, so currentSize follows capacity-driven eviction and Remove. +// callback wired. The callback is the sole subtractor of currentSize — every +// removal (capacity eviction, Remove) accounts through it. Freelru picks +// eviction victims per shard (hash bits 16+), which the put stripes (bits 0-7) +// don't cover, so any subtraction computed outside the callback races a +// cross-stripe eviction of the same entry. func (c *GenericCache[T]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, entry[T]] { lru, err := freelru.NewSharded[uint64, entry[T]](capacity, u64identity) if err != nil { @@ -334,15 +338,16 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite lru := c.data.Load() existing, hasExisting := lru.Get(h) - // Existing key — update in place. Reuse the stored key buffer to - // avoid an extra allocation; the freshly-decoded value replaces the - // old one. + // Existing key — update by remove-then-add (see newShards for why a size + // delta would be wrong). Reuse the stored key buffer to avoid an extra + // allocation; the freshly-decoded value replaces the old one. if hasExisting && bytes.Equal(existing.key, key) { if !overwrite && !c.coh.IsStale(existing.txNum, existing.epoch) { return false } + c.removeLocked(lru, h) lru.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) - c.currentSize.Add(int64(newSize - existing.size)) + c.currentSize.Add(int64(newSize)) return false } @@ -371,11 +376,10 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite // balcache.go / db/state/cache.go accept. // hasExisting here means a 64-bit maphash collision (different key, same - // hash): freelru.Add replaces the colliding entry in place WITHOUT firing - // OnEvict, so subtract the displaced size now — otherwise currentSize drifts - // up by it permanently. + // hash): remove the colliding entry first so OnEvict accounts for it — + // freelru.Add would replace it in place without firing OnEvict. if hasExisting { - c.currentSize.Add(-int64(existing.size)) + c.removeLocked(lru, h) } keyCopy := common.Copy(key) lru.Add(h, entry[T]{key: keyCopy, val: value, size: newSize, txNum: txNum, epoch: ep}) @@ -384,9 +388,18 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite return needGrow } -// Delete removes the data for the given key. Runs under the key's put stripe: -// an unstriped Remove racing put's read-modify-write would double-subtract the -// displaced entry's size (once via OnEvict, once via put's update delta). +// removeLocked removes h under the caller-held stripe, deferring the size +// subtraction to OnEvict (see newShards). The evictions metric is compensated: +// an intentional removal is not a capacity eviction. +func (c *GenericCache[T]) removeLocked(lru *freelru.ShardedLRU[uint64, entry[T]], h uint64) { + if lru.Remove(h) { + c.evictions.Add(^uint64(0)) + } +} + +// Delete removes the data for the given key. Runs under the key's put stripe +// so the check-then-remove is atomic against same-key puts and excluded from +// generation swaps (maybeGrow, Clear), which fence via the stripes. func (c *GenericCache[T]) Delete(key []byte) { h := maphash.Hash(key) mu := &c.putStripes[h&(putStripeCount-1)] @@ -394,21 +407,20 @@ func (c *GenericCache[T]) Delete(key []byte) { defer mu.Unlock() lru := c.data.Load() if existing, ok := lru.Get(h); ok && bytes.Equal(existing.key, key) { - lru.Remove(h) + c.removeLocked(lru, h) } } // dropStale removes key's entry under its put stripe: the re-check keeps an -// entry a concurrent put revived, and striping the Remove stops it -// double-subtracting the displaced size against put's update delta (once via -// OnEvict, once via the delta). +// entry a concurrent put revived, and the stripe keeps the removal out of +// generation swaps. func (c *GenericCache[T]) dropStale(h uint64, key []byte) { mu := &c.putStripes[h&(putStripeCount-1)] mu.Lock() defer mu.Unlock() lru := c.data.Load() if e, ok := lru.Get(h); ok && bytes.Equal(e.key, key) && c.coh.IsStale(e.txNum, e.epoch) { - lru.Remove(h) + c.removeLocked(lru, h) } } diff --git a/execution/cache/generic_cache_concurrency_test.go b/execution/cache/generic_cache_concurrency_test.go index 545d7221034..e3e7a9d6d35 100644 --- a/execution/cache/generic_cache_concurrency_test.go +++ b/execution/cache/generic_cache_concurrency_test.go @@ -24,6 +24,8 @@ import ( "github.com/c2h5oh/datasize" "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/maphash" ) // TestGenericCache_ConcurrentPutAcrossGrow guards the jump-grow data race: @@ -186,3 +188,35 @@ func TestGenericCache_PutIfAbsentDefersAcrossGrow(t *testing.T) { c.Close() } } + +// A capacity eviction is a size-subtracting writer the put stripes cannot +// serialize: freelru picks its victim per shard (hash bits 16+), so an insert +// on one stripe can evict a key whose own update — on another stripe — is +// between its Get and Add; delta accounting against the pre-eviction size then +// double-subtracts. Capacity 1 collapses freelru to a single shard, making any +// two keys same-shard; the keys are chosen to differ in their put stripe. Each +// hit leaks negative size; drift accumulates and shows after the settle +// deletes. +func TestGenericCache_CapacityEvictionAtomicWithPut_NoSizeDrift(t *testing.T) { + c := newGenericCacheEntries(1*datasize.MB, 1, func(v []byte) int { return len(v) }, ModeEvictLRU) + a := makeAddr(1) + var b []byte + for i := 2; ; i++ { + b = makeAddr(i) + if maphash.Hash(a)&(putStripeCount-1) != maphash.Hash(b)&(putStripeCount-1) { + break + } + } + v := []byte("value-one") + for round := 0; round < 100000; round++ { + c.Put(b, v, 10) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); c.Put(a, v, 10) }() // insert → evicts b (cap 1) + go func() { defer wg.Done(); c.Put(b, v, 20) }() // same-key update path + wg.Wait() + } + c.Delete(a) + c.Delete(b) + require.Zero(t, c.SizeBytes(), "capacity eviction raced the update-path delta") +} From bebb2de2ec1bdc38f717b322ff9e16c8c6cfe195 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 12:12:11 +0200 Subject: [PATCH 27/31] execution/cache: document growLRU's unfenced-swap contract Generation swaps stay unfenced here on purpose: the layers it backs are content-addressed, so a lost write is a benign miss and a raced removal resurrects correct bytes that drop on the next stale read. State that, the counter approximation it implies, and that mutable-per-key values belong on GenericCache's fenced swap instead. --- execution/cache/grow_lru.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/execution/cache/grow_lru.go b/execution/cache/grow_lru.go index 68a430b4110..1e1b1897695 100644 --- a/execution/cache/grow_lru.go +++ b/execution/cache/grow_lru.go @@ -33,8 +33,14 @@ import ( // pre-commits its full configured capacity — the same demand-growth the state // caches use — reused across the CodeCache's content and size layers. // -// A write racing a resize may land in the LRU about to be replaced and be -// dropped; that is a benign cache miss (the value is re-read from the DB). +// Generation swaps (maybeGrow, Purge) are not fenced against writers — safe +// only for content-addressed layers, where a key's payload never changes: a +// write lost in a retired generation is a benign miss, and an entry whose +// removal a racing copy undid serves correct bytes until its stale stamp +// drops it on the next read. Do not reuse for mutable-per-key values — those +// need GenericCache's fenced swap. The onEvict-maintained counters are +// approximate across grow windows (a lost write is counted but never +// evicted; a raced removal can subtract twice). type growLRU[V any] struct { cur atomic.Pointer[freelru.ShardedLRU[uint64, V]] onEvict func(uint64, V) From 29654fd798b3cc7bce8ddb02516081d369270283 Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 12:38:53 +0200 Subject: [PATCH 28/31] execution/exec, execution/execmodule: treat an interrupted read-ahead drain as a failed precondition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WaitForWarmup could return on context cancellation with the warmup still running, so "WaitForWarmup returned ⟹ warmup gauge is zero" held only for the completed-wait path; a payload validation racing module shutdown could then bump the cache epoch un-drained (an open-tx unwind never re-checks the context) and trip the ASSERT_STATE_CACHE gauge assert. WaitForWarmup and drainReadAhead now report whether the warmup fully drained — false only when the module context is cancelled — and the epoch-bump/Clear call sites return instead of proceeding. The DB-close caller keeps ignoring the result: it only needs a bounded wait. --- execution/exec/blocks_read_ahead.go | 15 +++++++---- execution/exec/blocks_read_ahead_test.go | 17 ++++++++++++ execution/execmodule/exec_module.go | 33 ++++++++++++++---------- execution/execmodule/forkchoice.go | 6 +++-- execution/execmodule/set_head.go | 6 +++-- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 98afd4c6acf..0d22cb5fefb 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -151,7 +151,7 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h if !bra.warming.CompareAndSwap(false, true) { return } - // Ordering makes "WaitForWarmup returned ⟹ gauge is zero" hold on its + // Ordering makes "WaitForWarmup drained ⟹ gauge is zero" hold on its // own: WarmupStarted only after warmWg.Add, WarmupDone before // warmWg.Done (defers run LIFO). StateCache.Unwind asserts on the gauge. bra.warmWg.Add(1) @@ -168,10 +168,13 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h } } -// WaitForWarmup blocks until any in-flight warmBody goroutine finishes or -// the context is cancelled. Call before closing the database to avoid -// waitTxsAllDoneOnClose hangs. -func (bra *BlockReadAheader) WaitForWarmup(ctx context.Context) { +// WaitForWarmup blocks until any in-flight warmBody goroutine finishes or the +// context is cancelled, reporting whether the warmup fully drained. False +// means a warmup may still be running — callers about to bump the cache epoch +// or Clear must treat it as a failed precondition. Call before closing the +// database to avoid waitTxsAllDoneOnClose hangs (that caller may ignore the +// result: it only needs a bounded wait). +func (bra *BlockReadAheader) WaitForWarmup(ctx context.Context) bool { done := make(chan struct{}) go func() { bra.warmWg.Wait() @@ -179,7 +182,9 @@ func (bra *BlockReadAheader) WaitForWarmup(ctx context.Context) { }() select { case <-done: + return true case <-ctx.Done(): + return false } } diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 6fe6e09c721..6ecfcb74132 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -17,6 +17,7 @@ package exec import ( + "context" "testing" "github.com/c2h5oh/datasize" @@ -27,6 +28,22 @@ import ( "github.com/erigontech/erigon/execution/cache" ) +// A cancelled wait can return while a warmup is still in flight — the gauge +// convention only holds for a drained return, so callers about to bump the +// cache epoch (or Clear) must be able to tell the two apart. +func TestWaitForWarmupReportsDrained(t *testing.T) { + bra := &BlockReadAheader{} + require.True(t, bra.WaitForWarmup(context.Background()), "nothing in flight — drained") + + bra.warmWg.Add(1) + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + require.False(t, bra.WaitForWarmup(cancelled), "cancelled wait with a live warmup must report undrained") + + bra.warmWg.Done() + require.True(t, bra.WaitForWarmup(context.Background()), "drained after the warmup finished") +} + // stubTemporalGetter stands in for the committed-state snapshot a warmup // goroutine reads: every GetLatest returns the same fixed value. type stubTemporalGetter struct { diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index a85c4a77f95..96841069546 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -381,22 +381,24 @@ func (e *ExecModule) canonicalHash(ctx context.Context, tx kv.Tx, blockNumber ui return canonical, nil } -// drainReadAhead blocks until any in-flight block-assembly warmup finishes. -// warmBody is fire-and-forget and populates the shared state/branch caches; if -// it is still running when an unwind bumps the cache epoch, it can Put a -// pre-unwind (dead-fork) value stamped with the post-unwind epoch — IsStale then -// returns false and the stale value is served as canonical (wrong root). A -// laggard Put can likewise land after a flush's cache-apply and pin the -// pre-flush snapshot. Call before any unwind epoch-bump or flush cache-apply. -func (e *ExecModule) drainReadAhead() { +// drainReadAhead blocks until any in-flight block-assembly warmup finishes, +// reporting whether it fully drained — false only when the module context is +// cancelled (shutdown). warmBody is fire-and-forget and populates the shared +// state/branch caches; if it is still running when an unwind bumps the cache +// epoch, it can Put a pre-unwind (dead-fork) value stamped with the post-unwind +// epoch — IsStale then returns false and the stale value is served as canonical +// (wrong root). A laggard Put can likewise land after a flush's cache-apply and +// pin the pre-flush snapshot. Call before any unwind epoch-bump or flush +// cache-apply, and do not proceed to them on false. +func (e *ExecModule) drainReadAhead() bool { if e.readAheader == nil { - return + return true } ctx := e.bacgroundCtx if ctx == nil { ctx = context.Background() } - e.readAheader.WaitForWarmup(ctx) + return e.readAheader.WaitForWarmup(ctx) } func (e *ExecModule) unwindToCommonCanonical(sd *execctx.SharedDomains, tx kv.TemporalRwTx, header *types.Header) error { @@ -427,7 +429,9 @@ func (e *ExecModule) unwindToCommonCanonical(sd *execctx.SharedDomains, tx kv.Te return err } - e.drainReadAhead() + if !e.drainReadAhead() { + return fmt.Errorf("read-ahead drain interrupted before unwind: %w", e.bacgroundCtx.Err()) + } if err := e.pipelineExecutor.UnwindTo(unwindPoint, stagedsync.ExecUnwind, tx); err != nil { return err } @@ -691,8 +695,11 @@ func (e *ExecModule) Start(ctx context.Context, hook *stageloop.Hook) { // already have warmed the state cache with pre-catchup state. Frozen-block // processing advances state without touching the cache (its SDs are not // wired to it), so such entries would be served stale afterwards — drain - // any in-flight warmup and clear before it runs. - e.drainReadAhead() + // any in-flight warmup and clear before it runs. An interrupted drain + // means shutdown: return rather than Clear under a live warmup. + if !e.drainReadAhead() { + return + } if e.stateCache != nil { e.stateCache.Clear() } diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 280959cb907..16f1cfbfa6d 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -363,8 +363,10 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // Drain any warmup a preceding newPayload spawned: its Puts reflect a // pre-FCU snapshot and must land before this FCU's unwind epoch-bump and // flush cache-apply, not after them (no new warmup starts while we hold - // the semaphore). - e.drainReadAhead() + // the semaphore). An interrupted drain means shutdown — bail. + if !e.drainReadAhead() { + return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, e.bacgroundCtx.Err(), false) + } var validationError string diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 9a9c11c2004..e0ce6581297 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -113,8 +113,10 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { // Drain in-flight warmup before the unwind bumps the cache epoch, so a // fire-and-forget warmup can't Put a dead-fork value stamped with the new - // epoch (cross-fork contamination). - e.drainReadAhead() + // epoch (cross-fork contamination). An interrupted drain means shutdown. + if !e.drainReadAhead() { + return fmt.Errorf("read-ahead drain interrupted before unwind: %w", e.bacgroundCtx.Err()) + } // Set the unwind point and run the unwind if err := e.pipelineExecutor.UnwindTo(targetBlock, stagedsync.StagedUnwind, tx); err != nil { From 96e6572ee9890b3b94d0bdc0e2748bd2af3221fb Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 12:47:47 +0200 Subject: [PATCH 29/31] execution/exec: bind a warmup's gauge and puts to one launch-time cache snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AddHeaderAndBody read bra.stateCache separately for WarmupStarted, the deferred WarmupDone, and warmBody's getter wiring, so a SetStateCache racing the launch could split the pair (negative gauge) or tick one cache's gauge while the workers populate another — the one whose gauge Unwind asserts. Capture the pointer once and thread it through warmBody so the Started/Done bracket and the puts all bind to the same cache. --- execution/exec/blocks_read_ahead.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 0d22cb5fefb..c95b974df22 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -154,16 +154,20 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h // Ordering makes "WaitForWarmup drained ⟹ gauge is zero" hold on its // own: WarmupStarted only after warmWg.Add, WarmupDone before // warmWg.Done (defers run LIFO). StateCache.Unwind asserts on the gauge. + // The cache pointer is captured once so the Started/Done pair and the + // warmup's puts all bind to the same gauge even if SetStateCache races + // the launch. bra.warmWg.Add(1) - if bra.stateCache != nil { - bra.stateCache.WarmupStarted() + sc := bra.stateCache + if sc != nil { + sc.WarmupStarted() } go func() { defer bra.warmWg.Done() - if bra.stateCache != nil { - defer bra.stateCache.WarmupDone() + if sc != nil { + defer sc.WarmupDone() } - bra.warmBody(ctx, db, header, body, 8) // use 8 workers for warming + bra.warmBody(ctx, db, sc, header, body, 8) // use 8 workers for warming }() } } @@ -199,7 +203,9 @@ func (bra *BlockReadAheader) AddSenders(senders []byte, blockHash common.Hash) { // It reads: To accounts, To account code, To account storage from access lists, // and block-level access lists. Each worker creates its own transaction. // Only one warmBody can run at a time - concurrent calls are no-ops. -func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body, workers int) { +// sc is the launch-time cache snapshot (see AddHeaderAndBody), nil to warm the +// OS page cache only. +func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, sc *cache.StateCache, header *types.Header, body *types.Body, workers int) { defer bra.warming.Store(false) if !dbg.ReadAhead { @@ -261,8 +267,8 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t return nil } var getter kv.TemporalGetter = ttx - if bra.stateCache != nil { - getter = newCachePopulatingGetter(ttx, bra.stateCache) + if sc != nil { + getter = newCachePopulatingGetter(ttx, sc) } stateReader := state.NewReaderV3(getter) @@ -332,8 +338,8 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t } var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter - if bra.stateCache != nil { - cpg = newCachePopulatingGetter(ttx, bra.stateCache) + if sc != nil { + cpg = newCachePopulatingGetter(ttx, sc) getter = cpg } stateReader := state.NewReaderV3(getter) From 76f3799959cb1c262f6b260d4a614f9f4698049a Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 13:10:40 +0200 Subject: [PATCH 30/31] execution/cache: note the transient pre-grow-cap eviction at the growth threshold --- execution/cache/generic_cache.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 62f6c5a2a33..35642f0662f 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -361,6 +361,9 @@ func (c *GenericCache[T]) putLocked(key []byte, value T, txNum uint64, overwrite } curCap := c.curCap.Load() + // The insert lands before the grow (which must run outside the stripe), so + // it and any racers until the swap evict at the pre-grow cap — a transient + // bounded by the grow window, not a regression of the grow-first ordering. needGrow := c.mode != ModeNoOp && curCap < c.maxCap && lru.Len() >= int(curCap) // In ModeEvictLRU the byte budget is enforced through the entry-count cap, From d95229c5d772b13dfc7188fd5cad027dca77640a Mon Sep 17 00:00:00 2001 From: yperbasis Date: Tue, 14 Jul 2026 13:25:38 +0200 Subject: [PATCH 31/31] execution/cache: log jump-grow durations split by unfenced alloc and fenced copy Grows are silent, so the once-per-lifetime writer stall of the fenced copy (~150ms for the final 1GB-accounts step) surfaces as an unexplained FCU/execution latency blip. One Debug line per grow with the caps, copied count and the alloc/fenced split makes it self-explaining. --- execution/cache/generic_cache.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 35642f0662f..e3545978c72 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -20,6 +20,7 @@ import ( "bytes" "sync" "sync/atomic" + "time" "github.com/c2h5oh/datasize" "github.com/elastic/go-freelru" @@ -215,13 +216,17 @@ func (c *GenericCache[T]) maybeGrow() { if !cachebudget.Global.Reserve(delta) { return } + start := time.Now() next := c.newShards(newCap) // allocate before excluding writers + fenceStart := time.Now() for i := range c.putStripes { c.putStripes[i].Lock() } + copied := 0 for _, k := range old.Keys() { if v, ok := old.Get(k); ok { next.Add(k, v) + copied++ } } c.data.Store(next) @@ -230,6 +235,8 @@ func (c *GenericCache[T]) maybeGrow() { c.putStripes[i].Unlock() } c.reservedBytes += delta + log.Debug("[cache] jump-grow", "fromSlots", curCap, "toSlots", newCap, "copied", copied, + "alloc", fenceStart.Sub(start), "fenced", time.Since(fenceStart)) } // DomainCache wraps GenericCache[[]byte] to implement the Cache interface.