diff --git a/pkg/config/config.go b/pkg/config/config.go index 7420953..0d62795 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1282,6 +1282,14 @@ type BootstrapFCUConfig struct { MaxRetries int `yaml:"max_retries" mapstructure:"max_retries" json:"max_retries"` Backoff string `yaml:"backoff" mapstructure:"backoff" json:"backoff"` HeadBlockHash string `yaml:"head_block_hash" mapstructure:"head_block_hash" json:"head_block_hash,omitempty"` + + // RootAnchorBlockHash is the block safe/finalized point at for the run. It + // must sit strictly below the block the fixtures replay from (the datadir + // head at bootstrap), since a client will not move its head back to a block + // at or below the one it considers finalized. Empty lets the runner derive + // one. For a datadir built by advancing a snapshot, the snapshot block is + // the natural value: far below every anchor and the same for every client. + RootAnchorBlockHash string `yaml:"root_anchor_block_hash,omitempty" mapstructure:"root_anchor_block_hash" json:"root_anchor_block_hash,omitempty"` } // DefaultOpcodeExtractionTimeout is the per-block trace timeout applied diff --git a/pkg/eest/converter.go b/pkg/eest/converter.go index 0ee72f4..6f90820 100644 --- a/pkg/eest/converter.go +++ b/pkg/eest/converter.go @@ -68,9 +68,9 @@ func ConvertFixture(name string, fixture *Fixture) (*ConvertedTest, error) { // phase is the shared pre_run payloads (snapshot → start block, preRun may be // nil) followed by the fixture's own setupEngineNewPayloads (start block → // per-test pre-state). The fixture's engineNewPayloads (the benchmark block) -// become the measured test step. Each payload still emits an -// engine_newPayload + engine_forkchoiceUpdated pair, so the chain head -// advances naturally and no separate forkchoice injection is needed. +// become the measured test step. Each payload emits an engine_newPayload + +// engine_forkchoiceUpdated pair, preceded by one forkchoiceUpdated returning +// the head to the block the fixture starts from. func ConvertStatefulFixture(name string, fixture *Fixture, preRun *StatefulPreRun) (*ConvertedTest, error) { if fixture == nil { return nil, fmt.Errorf("fixture is nil") @@ -98,6 +98,21 @@ func ConvertStatefulFixture(name string, fixture *Fixture, preRun *StatefulPreRu PayloadCount: len(setupPayloads) + len(fixture.EngineNewPayloads), } + // Every fixture replays from the same anchor, but the previous test leaves + // the head wherever its last payload landed and nothing puts it back. Ask + // for the anchor rather than assuming we are on it: otherwise the first + // newPayload names a parent whose state the client may no longer hold and + // is answered ACCEPTED, orphaning every payload after it. + // + // Sent unconditionally — when the head already matches, the client says so + // for free, which beats tracking the head across tests. + anchorLine, err := buildAnchorForkchoiceCall(setupPayloads, fixture.EngineNewPayloads) + if err != nil { + return nil, fmt.Errorf("building anchor forkchoiceUpdated call: %w", err) + } + + result.SetupLines = append(result.SetupLines, anchorLine) + for i, payload := range setupPayloads { lines, err := convertPayload(payload, i+1) if err != nil { @@ -217,12 +232,45 @@ func buildNewPayloadCall(payload *EngineNewPayload, id int) (string, error) { // ZeroHash is the zero hash used for forkchoice state. const ZeroHash = "0x0000000000000000000000000000000000000000000000000000000000000000" -// buildForkchoiceUpdatedCall builds an engine_forkchoiceUpdatedVX JSON-RPC call. +// buildAnchorForkchoiceCall returns the chain head to the block this fixture +// replays from: the parent of its first payload. Taken from the payload rather +// than startBlockHash so it stays correct when a pre_run is prepended, whose +// first payload descends from the snapshot block instead. +func buildAnchorForkchoiceCall(setupPayloads, benchmarkPayloads []*EngineNewPayload) (string, error) { + payloads := setupPayloads + if len(payloads) == 0 { + payloads = benchmarkPayloads + } + + if len(payloads) == 0 || payloads[0].ExecutionPayload == nil { + return "", fmt.Errorf("no payload to derive the anchor from") + } + + anchor := payloads[0].ExecutionPayload.ParentHash + if anchor == "" { + return "", fmt.Errorf("first payload has no parentHash") + } + + // id 0 keeps the payload calls numbered from 1 as before. + return buildForkchoiceUpdatedCallForHash(anchor, payloads[0].ForkchoiceUpdatedVersion, 0) +} + +// buildForkchoiceUpdatedCall builds an engine_forkchoiceUpdatedVX JSON-RPC call +// setting the head to the payload's own block. func buildForkchoiceUpdatedCall(payload *EngineNewPayload, id int) (string, error) { - // Use the forkchoiceUpdated version from the fixture. - method := fmt.Sprintf("engine_forkchoiceUpdatedV%d", payload.ForkchoiceUpdatedVersion) + return buildForkchoiceUpdatedCallForHash( + payload.ExecutionPayload.BlockHash, payload.ForkchoiceUpdatedVersion, id) +} - blockHash := payload.ExecutionPayload.BlockHash +// buildForkchoiceUpdatedCallForHash builds an engine_forkchoiceUpdatedVX call +// setting the head to blockHash. +// +// safe and finalized stay zero, which clients read as "no update" — replaying a +// payload should not move either marker. Note the engine API only permits the +// zero hash "unless transition block is finalized", which these mainnet-fork +// datadirs do not satisfy; sending a real ancestor here is worth revisiting. +func buildForkchoiceUpdatedCallForHash(blockHash string, version, id int) (string, error) { + method := fmt.Sprintf("engine_forkchoiceUpdatedV%d", version) forkchoiceState := map[string]string{ "headBlockHash": blockHash, diff --git a/pkg/eest/converter_test.go b/pkg/eest/converter_test.go index ab85e78..c4ae204 100644 --- a/pkg/eest/converter_test.go +++ b/pkg/eest/converter_test.go @@ -285,21 +285,28 @@ func TestConvertStatefulFixture(t *testing.T) { assert.Equal(t, "0xbench", result.FinalHash) // 3 pre_run + 1 setup + 1 benchmark = 5 payloads. assert.Equal(t, 5, result.PayloadCount) - // Setup = (3 pre_run + 1 setup) * 2 lines (newPayload + fcU). - assert.Len(t, result.SetupLines, 8) + // Setup = 1 anchor fcU + (3 pre_run + 1 setup) * 2 lines (newPayload + fcU). + assert.Len(t, result.SetupLines, 9) // Test = 1 benchmark * 2 lines. assert.Len(t, result.TestLines, 2) + // The anchor forkchoiceUpdated leads, pointing at the parent of the first + // payload. With a pre_run prepended that is the snapshot block, not the + // fixture's startBlockHash. + assert.Equal(t, "engine_forkchoiceUpdatedV3", rpcMethod(t, result.SetupLines[0])) + assert.Equal(t, "0xsnapshot", forkchoiceHeadHash(t, result.SetupLines[0]), + "the anchor must be the parent of the first pre_run block") + // Ordering by CONTENT (not just method name): the shared pre_run blocks must - // precede the fixture's own setup block. SetupLines are (newPayload, fcU) - // pairs, so newPayload lines sit at even indices: [0]=pre_run#1, [2]=pre_run#2, - // [4]=pre_run#3 (start), [6]=setup. - assert.Equal(t, "engine_newPayloadV4", rpcMethod(t, result.SetupLines[0])) - assert.Equal(t, "0xb1", newPayloadBlockHash(t, result.SetupLines[0]), + // precede the fixture's own setup block. After the anchor line, SetupLines are + // (newPayload, fcU) pairs, so newPayload lines sit at odd indices: + // [1]=pre_run#1, [3]=pre_run#2, [5]=pre_run#3 (start), [7]=setup. + assert.Equal(t, "engine_newPayloadV4", rpcMethod(t, result.SetupLines[1])) + assert.Equal(t, "0xb1", newPayloadBlockHash(t, result.SetupLines[1]), "first setup line must replay the first pre_run block, not the fixture's setup") - assert.Equal(t, "0xstart", newPayloadBlockHash(t, result.SetupLines[4]), + assert.Equal(t, "0xstart", newPayloadBlockHash(t, result.SetupLines[5]), "third newPayload is the last pre_run block (start)") - assert.Equal(t, "0xsetup", newPayloadBlockHash(t, result.SetupLines[6]), + assert.Equal(t, "0xsetup", newPayloadBlockHash(t, result.SetupLines[7]), "the fixture's own setup block comes AFTER the pre_run blocks") // The benchmark newPayload is the test step. @@ -338,6 +345,25 @@ func newPayloadBlockHash(t *testing.T, line string) string { return hash } +// forkchoiceHeadHash decodes a forkchoiceUpdated line and returns its +// headBlockHash. +func forkchoiceHeadHash(t *testing.T, line string) string { + t.Helper() + + var call map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &call)) + + params, ok := call["params"].([]any) + require.True(t, ok && len(params) > 0, "forkchoiceUpdated line must carry params") + + state, ok := params[0].(map[string]any) + require.True(t, ok, "first param must be the forkchoice state object") + + hash, _ := state["headBlockHash"].(string) + + return hash +} + func TestConvertStatefulFixture_NilPreRun(t *testing.T) { fixture := &Fixture{ Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, @@ -349,10 +375,70 @@ func TestConvertStatefulFixture_NilPreRun(t *testing.T) { result, err := ConvertStatefulFixture("test_stateful", fixture, nil) require.NoError(t, err) - // Without pre_run, only the fixture's own setup payload is replayed. - assert.Len(t, result.SetupLines, 2) + // Without pre_run: 1 anchor fcU + the fixture's own setup payload's 2 lines. + assert.Len(t, result.SetupLines, 3) assert.Len(t, result.TestLines, 2) assert.Equal(t, 2, result.PayloadCount) + + // With no pre_run the anchor is the fixture's start block. + assert.Equal(t, "0xstart", forkchoiceHeadHash(t, result.SetupLines[0])) +} + +// Every fixture replays from the same anchor, but nothing rewinds the client +// between tests, so the replay has to ask for the anchor itself. Without this +// line the first newPayload names a parent that is not the head, the client +// answers ACCEPTED, and every later payload is orphaned. +func TestConvertStatefulFixture_AnchorForkchoicePrecedesReplay(t *testing.T) { + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + SnapshotBlockHash: "0xsnapshot", + StartBlockHash: "0xstart", + SetupEngineNewPayloads: []*EngineNewPayload{statefulPayload("0x4", "0xsetup", "0xstart")}, + EngineNewPayloads: []*EngineNewPayload{statefulPayload("0x5", "0xbench", "0xsetup")}, + } + + result, err := ConvertStatefulFixture("test_stateful", fixture, nil) + require.NoError(t, err) + + require.NotEmpty(t, result.SetupLines) + + assert.Equal(t, "engine_forkchoiceUpdatedV3", rpcMethod(t, result.SetupLines[0]), + "the replay must open with a forkchoiceUpdated, not a newPayload") + assert.Equal(t, "0xstart", forkchoiceHeadHash(t, result.SetupLines[0]), + "it must point at the parent of the first payload") + + // safe and finalized stay zero: replaying a payload must not move either + // marker. + var call map[string]any + require.NoError(t, json.Unmarshal([]byte(result.SetupLines[0]), &call)) + state, _ := call["params"].([]any)[0].(map[string]any) + assert.Equal(t, ZeroHash, state["safeBlockHash"]) + assert.Equal(t, ZeroHash, state["finalizedBlockHash"]) +} + +// With a pre_run prepended the anchor must come from the first payload actually +// replayed, which descends from the snapshot block, not startBlockHash. +func TestConvertStatefulFixture_AnchorIsFirstPayloadParent(t *testing.T) { + preRun := &StatefulPreRun{ + EngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x1", "0xb1", "0xsnapshot"), + statefulPayload("0x2", "0xstart", "0xb1"), + }, + } + + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + SnapshotBlockHash: "0xsnapshot", + StartBlockHash: "0xstart", + SetupEngineNewPayloads: []*EngineNewPayload{statefulPayload("0x4", "0xsetup", "0xstart")}, + EngineNewPayloads: []*EngineNewPayload{statefulPayload("0x5", "0xbench", "0xsetup")}, + } + + result, err := ConvertStatefulFixture("test_stateful", fixture, preRun) + require.NoError(t, err) + + assert.Equal(t, "0xsnapshot", forkchoiceHeadHash(t, result.SetupLines[0]), + "with a pre_run the anchor is the snapshot block, not startBlockHash") } func TestConvertStatefulFixture_NoBenchmarkPayloads(t *testing.T) { diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 3970104..04a6fa8 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -395,7 +395,7 @@ func (e *executor) RunPreRunSteps(ctx context.Context, opts *ExecuteOptions) (in log.Info("Running pre-run step") preRunResult := NewTestResult(step.Name) - if err := e.runStepFile(ctx, opts, step, preRunResult, false, opts.PreRunStepSleep); err != nil { + if err := e.runStepFile(ctx, opts, step, preRunResult, false, opts.PreRunStepSleep, StepTypePreRun); err != nil { // FailFast: surface the error to the caller without writing partial results. if opts.FailFast { return 0, fmt.Errorf("pre-run step %q failed: %w", step.Name, err) @@ -493,7 +493,7 @@ func (e *executor) ExecuteTests(ctx context.Context, opts *ExecuteOptions) (*Exe log.Info("Running pre-run step") preRunResult := NewTestResult(step.Name) - if err := e.runStepFile(ctx, opts, step, preRunResult, false, opts.PreRunStepSleep); err != nil { + if err := e.runStepFile(ctx, opts, step, preRunResult, false, opts.PreRunStepSleep, StepTypePreRun); err != nil { log.WithError(err).Warn("Pre-run step failed") // Check if the failure was due to context cancellation. @@ -576,7 +576,7 @@ func (e *executor) ExecuteTests(ctx context.Context, opts *ExecuteOptions) (*Exe setupResult := NewTestResult(test.Name) - if err := e.runStepFile(ctx, opts, test.Setup, setupResult, false, 0); err != nil { + if err := e.runStepFile(ctx, opts, test.Setup, setupResult, false, 0, StepTypeSetup); err != nil { log.WithError(err).Error("Setup step failed") testPassed = false @@ -612,7 +612,7 @@ func (e *executor) ExecuteTests(ctx context.Context, opts *ExecuteOptions) (*Exe testResult := NewTestResult(test.Name) - if err := e.runStepFile(ctx, opts, test.Test, testResult, true, 0); err != nil { + if err := e.runStepFile(ctx, opts, test.Test, testResult, true, 0, StepTypeTest); err != nil { log.WithError(err).Error("Test step failed") testPassed = false @@ -668,7 +668,7 @@ func (e *executor) ExecuteTests(ctx context.Context, opts *ExecuteOptions) (*Exe cleanupResult := NewTestResult(test.Name) - if err := e.runStepFile(ctx, opts, test.Cleanup, cleanupResult, false, 0); err != nil { + if err := e.runStepFile(ctx, opts, test.Cleanup, cleanupResult, false, 0, StepTypeCleanup); err != nil { log.WithError(err).Error("Cleanup step failed") testPassed = false @@ -814,6 +814,7 @@ writeResults: // runStepFile executes a single step file or provider. // If captureBlockLogs is true, blockHashes from engine_newPayload calls are registered for log matching. // betweenLineSleep, when > 0, sleeps for that duration between each RPC call. +// stepType decides whether resume skipping applies; see runStepLines. func (e *executor) runStepFile( ctx context.Context, opts *ExecuteOptions, @@ -821,13 +822,15 @@ func (e *executor) runStepFile( result *TestResult, captureBlockLogs bool, betweenLineSleep time.Duration, + stepType StepType, ) error { // Use provider if available, otherwise read from file. if step.Provider != nil { - return e.runStepLines(ctx, opts, step.Name, step.Provider.Lines(), result, captureBlockLogs, betweenLineSleep) + return e.runStepLines(ctx, opts, step.Name, step.Provider.Lines(), result, + captureBlockLogs, betweenLineSleep, stepType) } - return e.runStepFromFile(ctx, opts, step, result, captureBlockLogs, betweenLineSleep) + return e.runStepFromFile(ctx, opts, step, result, captureBlockLogs, betweenLineSleep, stepType) } // runStepFromFile reads and executes lines from a file. @@ -838,6 +841,7 @@ func (e *executor) runStepFromFile( result *TestResult, captureBlockLogs bool, betweenLineSleep time.Duration, + stepType StepType, ) error { file, err := os.Open(step.Path) if err != nil { @@ -867,7 +871,8 @@ func (e *executor) runStepFromFile( } } - return e.runStepLines(ctx, opts, step.Name, lines, result, captureBlockLogs, betweenLineSleep) + return e.runStepLines(ctx, opts, step.Name, lines, result, captureBlockLogs, + betweenLineSleep, stepType) } // runStepLines executes JSON-RPC lines. @@ -882,6 +887,7 @@ func (e *executor) runStepLines( result *TestResult, captureBlockLogs bool, betweenLineSleep time.Duration, + stepType StepType, ) error { stepStart := time.Now() @@ -897,7 +903,12 @@ func (e *executor) runStepLines( // skipping is true when we're dropping already-applied lines at the // start of the file (resume scenario). Cleared once we encounter the // first engine_newPayload whose blockNumber > SkipUntilBlockNumber. - skipping := opts.SkipUntilBlockNumber > 0 + // + // Only the pre-run replay resumes; it alone may be partly applied. A test's + // steps always start from the replay anchor and must be sent whole — + // skipping them silently dropped any leading line that is not a newPayload, + // such as a forkchoiceUpdated returning the head to the anchor. + skipping := opts.SkipUntilBlockNumber > 0 && stepType == StepTypePreRun skippedCount := 0 for lineNum, line := range lines { diff --git a/pkg/runner/lifecycle.go b/pkg/runner/lifecycle.go index 2ece01e..c11175e 100644 --- a/pkg/runner/lifecycle.go +++ b/pkg/runner/lifecycle.go @@ -1078,6 +1078,28 @@ func (r *runner) runContainerLifecycle( } } + // Move safe/finalized below the head before anything replays. Prestates + // arrive with their head block already finalized, which would make the + // replay anchor permanently unreachable. Best-effort: a client that will + // not lower its finalized marker still runs, it just keeps the old + // failure mode. + if blockHash != "" { + var configuredAnchor string + if r.cfg.FullConfig != nil { + if fcuCfg := r.cfg.FullConfig.GetBootstrapFCU(instance); fcuCfg != nil { + configuredAnchor = fcuCfg.RootAnchorBlockHash + } + } + + if anchorErr := r.resetForkchoiceAnchor( + execCtx, log, containerIP, spec.EnginePort(), spec.RPCPort(), + blockHash, configuredAnchor, + ); anchorErr != nil { + log.WithError(anchorErr).Warn( + "Could not reset safe/finalized; a deep rewind to the replay anchor may fail") + } + } + // Send bootstrap FCU if configured. if r.cfg.FullConfig != nil { if fcuCfg := r.cfg.FullConfig.GetBootstrapFCU(instance); fcuCfg != nil && fcuCfg.Enabled { @@ -1092,7 +1114,7 @@ func (r *runner) runContainerLifecycle( if fcuHash != "" { if fcuErr := r.sendBootstrapFCU( - execCtx, log, containerIP, spec.EnginePort(), fcuHash, fcuCfg, + execCtx, log, containerIP, spec.EnginePort(), fcuHash, "", fcuCfg, ); fcuErr != nil { log.WithError(fcuErr).Error("Bootstrap FCU failed") diff --git a/pkg/runner/root_anchor_test.go b/pkg/runner/root_anchor_test.go new file mode 100644 index 0000000..4cd683e --- /dev/null +++ b/pkg/runner/root_anchor_test.go @@ -0,0 +1,155 @@ +package runner + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeChain answers eth_getBlockByNumber for "latest", "finalized" and hex +// numbers. A tag mapped to 0 is reported as a null result, the way a client +// with no finalized block answers. +type fakeChain struct { + latest uint64 + finalized uint64 // 0 means "no finalized block" +} + +func (f fakeChain) server(t *testing.T) *httptest.Server { + t.Helper() + + hash := func(n uint64) string { + return fmt.Sprintf("0x%064x", n) + } + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Params []any `json:"params"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + + tag, _ := req.Params[0].(string) + + var num uint64 + + switch tag { + case "latest": + num = f.latest + case "finalized": + num = f.finalized + default: + parsed, err := strconv.ParseUint(strings.TrimPrefix(tag, "0x"), 16, 64) + require.NoError(t, err) + num = parsed + } + + if num == 0 { + _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":null}`)) + + return + } + + _, _ = fmt.Fprintf(w, + `{"jsonrpc":"2.0","id":1,"result":{"number":"0x%x","hash":%q,"stateRoot":%q}}`, + num, hash(num), hash(num)) + })) +} + +func hostPort(t *testing.T, srv *httptest.Server) (string, int) { + t.Helper() + + host, port, err := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":") + require.True(t, err) + + p, convErr := strconv.Atoi(port) + require.NoError(t, convErr) + + return host, p +} + +// The root anchor must land strictly below the head — the block every fixture +// replays from — since a client will not move its head back to a block at or +// below the one it considers finalized. +func TestResolveRootAnchor(t *testing.T) { + log := logrus.New() + log.SetLevel(logrus.PanicLevel) + + tests := []struct { + name string + chain fakeChain + configured string + want string + reason string + }{ + { + name: "configured wins", + chain: fakeChain{latest: 200, finalized: 100}, + configured: "0xdeadbeef", + want: "0xdeadbeef", + reason: "an explicit hash is used verbatim, without consulting the client", + }, + { + name: "reuses an already-low finalized block", + chain: fakeChain{latest: 200, finalized: 100}, + want: fmt.Sprintf("0x%064x", 100), + reason: "a finalized block below the head is already a usable anchor", + }, + { + name: "falls back when finalized is the head itself", + chain: fakeChain{latest: 200, finalized: 200}, + want: fmt.Sprintf("0x%064x", 199), + reason: "this is the broken datadir: keeping it would pin the replay anchor", + }, + { + name: "falls back when finalized is above the head", + chain: fakeChain{latest: 200, finalized: 250}, + want: fmt.Sprintf("0x%064x", 199), + reason: "a finalized block above the head is never a valid anchor", + }, + { + name: "falls back when there is no finalized block", + chain: fakeChain{latest: 200, finalized: 0}, + want: fmt.Sprintf("0x%064x", 199), + reason: "a client that has never finalized still needs an anchor", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := tc.chain.server(t) + defer srv.Close() + + host, port := hostPort(t, srv) + r := &runner{} + + got, err := r.resolveRootAnchor( + context.Background(), log, host, port, tc.configured) + require.NoError(t, err) + assert.Equal(t, tc.want, got, tc.reason) + }) + } +} + +// Genesis has nothing below it: the caller must be told, not handed zero. +func TestResolveRootAnchorAtGenesis(t *testing.T) { + log := logrus.New() + log.SetLevel(logrus.PanicLevel) + + srv := fakeChain{latest: 0, finalized: 0}.server(t) + defer srv.Close() + + host, port := hostPort(t, srv) + r := &runner{} + + _, err := r.resolveRootAnchor(context.Background(), log, host, port, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "genesis") +} diff --git a/pkg/runner/rpc.go b/pkg/runner/rpc.go index 1565d11..7839baa 100644 --- a/pkg/runner/rpc.go +++ b/pkg/runner/rpc.go @@ -80,11 +80,24 @@ func (r *runner) checkRPCHealth(ctx context.Context, url string) (string, bool) // getLatestBlock fetches the latest block number, hash, and state root from the RPC endpoint. func (r *runner) getLatestBlock(ctx context.Context, host string, port int) (uint64, string, string, error) { + return r.getBlockByTag(ctx, host, port, "latest") +} + +// getBlockByTag fetches a block by tag, passed to eth_getBlockByNumber verbatim +// ("latest", "finalized", "safe" or a hex number). A tag with no block behind it +// yields a zero hash and no error, so callers can treat "not set" as ordinary. +func (r *runner) getBlockByTag( + ctx context.Context, + host string, + port int, + tag string, +) (uint64, string, string, error) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() url := fmt.Sprintf("http://%s:%d", host, port) - body := `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}` + body := fmt.Sprintf( + `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":[%q,false],"id":1}`, tag) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body)) if err != nil { @@ -120,6 +133,11 @@ func (r *runner) getLatestBlock(ctx context.Context, host string, port int) (uin return 0, "", "", fmt.Errorf("parsing response: %w", err) } + // A tag with no block behind it comes back as null, not an error. + if rpcResp.Result.Hash == "" { + return 0, "", "", nil + } + // Parse hex block number. blockNum, err := strconv.ParseUint(strings.TrimPrefix(rpcResp.Result.Number, "0x"), 16, 64) if err != nil { @@ -129,17 +147,123 @@ func (r *runner) getLatestBlock(ctx context.Context, host string, port int) (uin return blockNum, rpcResp.Result.Hash, rpcResp.Result.StateRoot, nil } +// resolveRootAnchor picks the block safe/finalized point at for the whole run: +// a configured hash, else the client's own finalized block when it already sits +// below the head, else the head's parent. +// +// It must sit strictly below the block the fixtures replay from — the datadir +// head at bootstrap. geth will not move its head to a block at or below the one +// it considers finalized; it answers VALID and does nothing, so the anchor +// becomes unreachable. The zero hash is no help: clients read it as "no update", +// and the engine API only permits it "unless transition block is finalized". +func (r *runner) resolveRootAnchor( + ctx context.Context, + log logrus.FieldLogger, + host string, + rpcPort int, + configured string, +) (string, error) { + if configured != "" { + log.WithField("root_anchor", configured).Info( + "Using configured root anchor for safe/finalized") + + return configured, nil + } + + headNum, _, _, err := r.getBlockByTag(ctx, host, rpcPort, "latest") + if err != nil { + return "", fmt.Errorf("fetching the head for root anchor resolution: %w", err) + } + + if finNum, finHash, _, err := r.getBlockByTag(ctx, host, rpcPort, "finalized"); err == nil && + finHash != "" && finNum < headNum { + log.WithFields(logrus.Fields{"root_anchor": finHash, "block": finNum}).Info( + "Reusing the client's finalized block as the root anchor") + + return finHash, nil + } + + if headNum == 0 { + return "", fmt.Errorf("head is the genesis block; no room for a root anchor below it") + } + + parentNum := headNum - 1 + + _, parentHash, _, err := r.getBlockByTag(ctx, host, rpcPort, fmt.Sprintf("0x%x", parentNum)) + if err != nil { + return "", fmt.Errorf("fetching block %d for the root anchor: %w", parentNum, err) + } + + if parentHash == "" { + return "", fmt.Errorf("block %d not available for the root anchor", parentNum) + } + + log.WithFields(logrus.Fields{"root_anchor": parentHash, "block": parentNum}).Info( + "Using the head's parent as the root anchor") + + return parentHash, nil +} + +// resetForkchoiceAnchor points safe/finalized at a block strictly below the +// datadir head, once per instance before any test runs. +// +// Prestates are built by advancing a snapshot with forkchoiceUpdated calls +// setting head = safe = finalized on every block, so the block every fixture +// replays from arrives already finalized — and geth will not move its head +// back to it. That only bites once a test leaves the head far away +// (test_blockhash builds 258 blocks): the anchor's state is no longer retained +// and the call that would rebuild it is refused, stranding every later test. +// +// Unconditional, not part of bootstrap_fcu, which most configs leave unset. +func (r *runner) resetForkchoiceAnchor( + ctx context.Context, + log logrus.FieldLogger, + host string, + enginePort int, + rpcPort int, + headBlockHash string, + configured string, +) error { + anchor, err := r.resolveRootAnchor(ctx, log, host, rpcPort, configured) + if err != nil { + return fmt.Errorf("resolving root anchor: %w", err) + } + + payload := fmt.Sprintf( + `{"jsonrpc":"2.0","method":"engine_forkchoiceUpdatedV3",`+ + `"params":[{"headBlockHash":"%s","safeBlockHash":"%s",`+ + `"finalizedBlockHash":"%s"},null],"id":1}`, + headBlockHash, anchor, anchor, + ) + + url := fmt.Sprintf("http://%s:%d", host, enginePort) + if err := r.doBootstrapFCURequest(ctx, url, payload); err != nil { + return fmt.Errorf("sending forkchoice anchor reset: %w", err) + } + + log.WithFields(logrus.Fields{ + "head": headBlockHash, + "root_anchor": anchor, + }).Info("Reset safe/finalized to the root anchor") + + return nil +} + // sendBootstrapFCU sends an engine_forkchoiceUpdatedV3 call to confirm the // client is fully synced and ready for test execution. The call is retried // up to cfg.MaxRetries times with cfg.Backoff between attempts — some clients // (e.g., Erigon) may still be performing internal initialization after RPC // becomes available. A VALID response confirms the client is ready. +// +// rootAnchorBlockHash is the block safe/finalized point at; "" keeps the +// previous behaviour of sending the zero hash. func (r *runner) sendBootstrapFCU( ctx context.Context, log logrus.FieldLogger, host string, enginePort int, headBlockHash string, + rootAnchorBlockHash string, cfg *config.BootstrapFCUConfig, ) error { const zeroHash = "0x0000000000000000000000000000000000000000000000000000000000000000" @@ -149,12 +273,19 @@ func (r *runner) sendBootstrapFCU( return fmt.Errorf("parsing backoff duration: %w", err) } + // The zero hash cannot move a stale marker down: clients read it as + // "no update". + anchor := rootAnchorBlockHash + if anchor == "" { + anchor = zeroHash + } + // Build the forkchoiceUpdatedV3 payload. payload := fmt.Sprintf( `{"jsonrpc":"2.0","method":"engine_forkchoiceUpdatedV3",`+ `"params":[{"headBlockHash":"%s","safeBlockHash":"%s",`+ `"finalizedBlockHash":"%s"},null],"id":1}`, - headBlockHash, zeroHash, zeroHash, + headBlockHash, anchor, anchor, ) url := fmt.Sprintf("http://%s:%d", host, enginePort) diff --git a/pkg/runner/strategy_container.go b/pkg/runner/strategy_container.go index 4346d80..c1dd8d0 100644 --- a/pkg/runner/strategy_container.go +++ b/pkg/runner/strategy_container.go @@ -550,9 +550,18 @@ func (r *runner) runTestsWithContainerStrategy( } if blkHash != "" { + rootAnchor, anchorErr := r.resolveRootAnchor( + ctx, testLog, currentContainerIP, spec.RPCPort(), + fcuCfg.RootAnchorBlockHash, + ) + if anchorErr != nil { + testLog.WithError(anchorErr).Warn( + "Could not resolve a root anchor; leaving safe/finalized unset") + } + if fcuErr := r.sendBootstrapFCU( ctx, testLog, currentContainerIP, - spec.EnginePort(), blkHash, fcuCfg, + spec.EnginePort(), blkHash, rootAnchor, fcuCfg, ); fcuErr != nil { testLog.WithError(fcuErr).Error( "Bootstrap FCU failed", @@ -786,9 +795,18 @@ func (r *runner) runTestsWithContainerStrategy( } if blkHash != "" { + rootAnchor, anchorErr := r.resolveRootAnchor( + ctx, testLog, currentContainerIP, spec.RPCPort(), + fcuCfg.RootAnchorBlockHash, + ) + if anchorErr != nil { + testLog.WithError(anchorErr).Warn( + "Could not resolve a root anchor; leaving safe/finalized unset") + } + if fcuErr := r.sendBootstrapFCU( ctx, testLog, currentContainerIP, - spec.EnginePort(), blkHash, fcuCfg, + spec.EnginePort(), blkHash, rootAnchor, fcuCfg, ); fcuErr != nil { testLog.WithError(fcuErr).Error( "Bootstrap FCU failed",