From dc8d4c55d6cb9b61ba350f15c44870510d002335 Mon Sep 17 00:00:00 2001 From: stavrosvl7 Date: Thu, 2 Jul 2026 01:20:52 +0300 Subject: [PATCH] feat: known-failures (xfail) list for expected failures Add known-failures.json (repo root) listing tests expected to fail while a fix is pending, each linked to its PR. Listed tests still run, but: - a failure is reported as KNOWN_FAIL and does not fail the run; - an unexpected pass is reported as UNEXPECTED_PASS (warn), or fails the run with --strict-known-failures to force cleanup. Keeps the job green while upstream/NM fixes are in flight, preserves coverage, and flags the moment a fix lands. Exit code stays keyed on real (non-known) failures. Seeded with the 6 currently-pending diffs (NM #12003; geth #35271/#35129/#35228). Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 19 ++++++++ cmd/integration/main.go | 10 ++++ internal/config/config.go | 7 +++ internal/config/knownfailures.go | 48 ++++++++++++++++++ internal/config/knownfailures_test.go | 64 ++++++++++++++++++++++++ internal/runner/runner.go | 70 ++++++++++++++++++--------- internal/runner/stats.go | 30 ++++++++++-- known-failures.json | 26 ++++++++++ 8 files changed, 245 insertions(+), 29 deletions(-) create mode 100644 internal/config/knownfailures.go create mode 100644 internal/config/knownfailures_test.go create mode 100644 known-failures.json diff --git a/README.md b/README.md index cb1753ac..5e299b57 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,25 @@ make Check out the dedicated guide in [Integration Tests](./integration/README.md). +### Known (expected) failures + +`known-failures.json` (repo root) lists tests that are expected to fail while a fix +is pending, keyed by `"/"` (extension-insensitive) with a linked PR: + +```json +{ + "eth_getBalance/test_40": { "pr": "ethereum/go-ethereum#35271", "note": "geth -32000 vs NM -32602; geth-side fix" } +} +``` + +Listed tests still **run**, but: + +- a listed test that **fails** is reported as `KNOWN_FAIL ()` and does **not** fail the run; +- a listed test that **passes** is reported as `UNEXPECTED_PASS` (a warning to remove the entry) — with `--strict-known-failures` this fails the run to force the cleanup. + +Flags: `--known-failures ` (default `known-failures.json`), `--strict-known-failures`. +Prefer this over excluding/skipping a test: coverage is preserved, and you're told the moment the fix lands. + ## Performance Testing Check out the dedicated guide in [Performance Tests](./perf/README.md). diff --git a/cmd/integration/main.go b/cmd/integration/main.go index 898cdaaa..1848bb00 100644 --- a/cmd/integration/main.go +++ b/cmd/integration/main.go @@ -118,6 +118,10 @@ func parseFlags(cfg *config.Config) error { reportFile := flag.String("R", "", "write CSV summary report to file") flag.StringVar(reportFile, "report-file", "", "write CSV summary report to file") + knownFailuresPath := flag.String("known-failures", "known-failures.json", "path to known-failures JSON file; listed tests that fail are reported as KNOWN_FAIL and do not fail the run") + + strictKnownFailures := flag.Bool("strict-known-failures", false, "fail the run when a known failure unexpectedly passes (forces cleanup of known-failures.json)") + cpuProfile := flag.String("cpuprofile", "", "write cpu profile to file") memProfile := flag.String("memprofile", "", "write memory profile to file") traceFile := flag.String("trace", "", "write execution trace to file") @@ -153,6 +157,12 @@ func parseFlags(cfg *config.Config) error { cfg.ArchiveNode = *archiveNode cfg.PrunedNode = *prunedNode cfg.MaxFailures = *maxFailures + cfg.StrictKnownFailures = *strictKnownFailures + knownFailures, err := config.LoadKnownFailures(*knownFailuresPath) + if err != nil { + return err + } + cfg.KnownFailures = knownFailures cfg.ReportFile = *reportFile cfg.CpuProfile = *cpuProfile cfg.MemProfile = *memProfile diff --git a/internal/config/config.go b/internal/config/config.go index 6109c690..826291f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -119,6 +119,13 @@ type Config struct { // Failure cap MaxFailures int // stop after this many failures (0 = unlimited) + // Known (expected) failures: tests here still run, but a failure is reported + // as KNOWN_FAIL and does not fail the run; a listed test that passes is reported + // as UNEXPECTED_PASS (warn, or fail the run when StrictKnownFailures is set). + // Keyed by "/" without file extension (e.g. "eth_getBalance/test_40"). + KnownFailures map[string]KnownFailure + StrictKnownFailures bool + // Report ReportFile string diff --git a/internal/config/knownfailures.go b/internal/config/knownfailures.go new file mode 100644 index 00000000..1eb4aff5 --- /dev/null +++ b/internal/config/knownfailures.go @@ -0,0 +1,48 @@ +package config + +import ( + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// KnownFailure describes a test that is expected to fail, with a link to the PR +// tracking the fix and a short human-readable reason. +type KnownFailure struct { + PR string `json:"pr"` + Note string `json:"note"` +} + +// NormalizeTestKey returns the extension-less "/" key used to match a +// test against the known-failures map (e.g. "eth_getBalance/test_40.json" -> +// "eth_getBalance/test_40"). +func NormalizeTestKey(name string) string { + return strings.TrimSuffix(name, filepath.Ext(name)) +} + +// LoadKnownFailures reads the known-failures JSON file at path. A missing file is +// not an error (returns an empty map), so the list is entirely optional. Keys are +// normalized to be extension-insensitive. +func LoadKnownFailures(path string) (map[string]KnownFailure, error) { + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return map[string]KnownFailure{}, nil + } + return nil, err + } + + var raw map[string]KnownFailure + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + + out := make(map[string]KnownFailure, len(raw)) + for k, v := range raw { + out[NormalizeTestKey(k)] = v + } + return out, nil +} diff --git a/internal/config/knownfailures_test.go b/internal/config/knownfailures_test.go new file mode 100644 index 00000000..97c1cf93 --- /dev/null +++ b/internal/config/knownfailures_test.go @@ -0,0 +1,64 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadKnownFailures_MissingFileIsEmpty(t *testing.T) { + m, err := LoadKnownFailures(filepath.Join(t.TempDir(), "does-not-exist.json")) + if err != nil { + t.Fatalf("missing file should not error, got %v", err) + } + if len(m) != 0 { + t.Fatalf("expected empty map, got %d entries", len(m)) + } +} + +func TestLoadKnownFailures_ParsesAndNormalizesKeys(t *testing.T) { + path := filepath.Join(t.TempDir(), "known-failures.json") + content := `{ + "eth_getBalance/test_40.json": {"pr": "geth#35271", "note": "empty {} block param"}, + "eth_sendRawTransaction/test_23": {"pr": "geth#35129", "note": "empty raw tx"} + }` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + m, err := LoadKnownFailures(path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // The .json extension in a key must be normalized away so it matches the + // runtime test name (which carries the extension). + if kf, ok := m["eth_getBalance/test_40"]; !ok || kf.PR != "geth#35271" { + t.Fatalf("expected normalized key eth_getBalance/test_40 with PR geth#35271, got %+v (ok=%v)", kf, ok) + } + if kf, ok := m["eth_sendRawTransaction/test_23"]; !ok || kf.PR != "geth#35129" { + t.Fatalf("expected eth_sendRawTransaction/test_23 with PR geth#35129, got %+v (ok=%v)", kf, ok) + } +} + +func TestLoadKnownFailures_InvalidJSONErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.json") + if err := os.WriteFile(path, []byte("{not json"), 0644); err != nil { + t.Fatal(err) + } + if _, err := LoadKnownFailures(path); err == nil { + t.Fatal("expected error for invalid JSON, got nil") + } +} + +func TestNormalizeTestKey(t *testing.T) { + cases := map[string]string{ + "eth_getBalance/test_40.json": "eth_getBalance/test_40", + "eth_getBalance/test_40": "eth_getBalance/test_40", + "eth_x/test_1.tar": "eth_x/test_1", + } + for in, want := range cases { + if got := NormalizeTestKey(in); got != want { + t.Errorf("NormalizeTestKey(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e920fd1f..e1af4c41 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -321,25 +321,57 @@ func printResult(w *bufio.Writer, result *testdata.TestResult, stats *Stats, cfg tt := fmt.Sprintf("%-15s", result.Test.TransportType) fmt.Fprintf(w, "%04d. %s::%s ", result.Test.Number, tt, file) - if result.Outcome.Success { - stats.AddSuccess(result.Outcome.Metrics) - if cfg.VerboseLevel > 0 { - fmt.Fprintln(w, "OK") - } else { - fmt.Fprint(w, "OK\r") - } + kf, isKnownFailure := cfg.KnownFailures[config.NormalizeTestKey(result.Test.Name)] + + addReport := func(res string, errField any) { if cfg.VerboseLevel == 1 || cfg.ReportFile != "" { reportMu.Lock() *reportEntries = append(*reportEntries, reportEntry{ TestNumber: result.Test.Number, TransportType: result.Test.TransportType, TestName: result.Test.Name, - Result: "OK", - ErrorMessage: "", + Result: res, + ErrorMessage: errField, }) reportMu.Unlock() } - } else { + } + + switch { + case result.Outcome.Success && isKnownFailure: + // A known failure that now passes: flag it so the entry gets removed. + // In strict mode it fails the run to force the cleanup. + stats.AddUnexpectedPass() + msg := fmt.Sprintf("known failure %s appears resolved — remove it from known-failures.json", kf.PR) + if cfg.StrictKnownFailures { + stats.AddFailure() + fmt.Fprintf(w, "failed: UNEXPECTED PASS (%s)\n", msg) + } else { + stats.AddSuccess(result.Outcome.Metrics) + fmt.Fprintf(w, "OK (UNEXPECTED PASS: %s)\n", msg) + } + addReport("UNEXPECTED_PASS", kf.PR) + + case result.Outcome.Success: + stats.AddSuccess(result.Outcome.Metrics) + if cfg.VerboseLevel > 0 { + fmt.Fprintln(w, "OK") + } else { + fmt.Fprint(w, "OK\r") + } + addReport("OK", "") + + case isKnownFailure: + // Expected failure: report but do not fail the run. + stats.AddKnownFailure() + errMsg := "no error" + if result.Outcome.Error != nil { + errMsg = result.Outcome.Error.Error() + } + fmt.Fprintf(w, "KNOWN FAIL (%s)\n", kf.PR) + addReport("KNOWN_FAIL", kf.PR+": "+errMsg) + + default: stats.AddFailure() errMsg := "no error" if result.Outcome.Error != nil { @@ -351,21 +383,11 @@ func printResult(w *bufio.Writer, result *testdata.TestResult, stats *Stats, cfg } else { fmt.Fprintf(w, "failed: %s\n", errMsg) } - if cfg.VerboseLevel == 1 || cfg.ReportFile != "" { - var errField any = errMsg - if result.Outcome.ErrorDetails != nil { - errField = result.Outcome.ErrorDetails - } - reportMu.Lock() - *reportEntries = append(*reportEntries, reportEntry{ - TestNumber: result.Test.Number, - TransportType: result.Test.TransportType, - TestName: result.Test.Name, - Result: "FAILED", - ErrorMessage: errField, - }) - reportMu.Unlock() + var errField any = errMsg + if result.Outcome.ErrorDetails != nil { + errField = result.Outcome.ErrorDetails } + addReport("FAILED", errField) if maxFailuresReached(cfg, stats) { fmt.Fprintf(w, "\nABORTED: too many failures (%d), test sequence stopped early\n", cfg.MaxFailures) w.Flush() diff --git a/internal/runner/stats.go b/internal/runner/stats.go index a475113e..6d5a52d2 100644 --- a/internal/runner/stats.go +++ b/internal/runner/stats.go @@ -9,11 +9,13 @@ import ( // Stats aggregates metrics and counts across all tests. type Stats struct { - SuccessTests int - FailedTests int - ExecutedTests int - SkippedTests int - ScheduledTests int + SuccessTests int + FailedTests int + KnownFailures int + UnexpectedPasses int + ExecutedTests int + SkippedTests int + ScheduledTests int TotalRoundTripTime time.Duration TotalMarshallingTime time.Duration @@ -39,6 +41,18 @@ func (s *Stats) AddFailure() { s.ExecutedTests++ } +// AddKnownFailure records a failure that is listed in the known-failures file. +// It counts as executed but NOT as a failure, so it does not fail the run. +func (s *Stats) AddKnownFailure() { + s.KnownFailures++ + s.ExecutedTests++ +} + +// AddUnexpectedPass records a test that passed but is listed as a known failure. +func (s *Stats) AddUnexpectedPass() { + s.UnexpectedPasses++ +} + // PrintSummary prints the v1-compatible summary output. func (s *Stats) PrintSummary(startTime time.Time, elapsed time.Duration, iterations, totalAPIs, totalTests int) { fmt.Println("\n ") @@ -57,4 +71,10 @@ func (s *Stats) PrintSummary(startTime time.Time, elapsed time.Duration, iterati fmt.Printf("Number of executed tests: %d\n", s.ExecutedTests) fmt.Printf("Number of success tests: %d\n", s.SuccessTests) fmt.Printf("Number of failed tests: %d\n", s.FailedTests) + if s.KnownFailures > 0 { + fmt.Printf("Number of known failures: %d\n", s.KnownFailures) + } + if s.UnexpectedPasses > 0 { + fmt.Printf("Number of unexpected passes: %d\n", s.UnexpectedPasses) + } } diff --git a/known-failures.json b/known-failures.json new file mode 100644 index 00000000..606e0093 --- /dev/null +++ b/known-failures.json @@ -0,0 +1,26 @@ +{ + "eth_sendRawTransaction/test_05": { + "pr": "NethermindEth/nethermind#12003", + "note": "NM returns -32000 'Invalid RLP' for malformed hex; should be -32602 (invalid params). NM-side fix." + }, + "eth_sendRawTransaction/test_06": { + "pr": "NethermindEth/nethermind#12003", + "note": "Same as test_05 (no-0x-prefix hex). NM-side fix." + }, + "eth_sendRawTransaction/test_21": { + "pr": "NethermindEth/nethermind#12003", + "note": "Same as test_05 (odd-length hex). NM-side fix." + }, + "eth_getBalance/test_40": { + "pr": "ethereum/go-ethereum#35271", + "note": "geth returns -32000 for an empty {} EIP-1898 block param; NM's -32602 is spec-correct. geth-side fix (awaiting geth release)." + }, + "eth_sendRawTransaction/test_23": { + "pr": "ethereum/go-ethereum#35129", + "note": "geth returns -32000 for an empty raw tx; NM's -32602 is spec-correct. geth-side fix (awaiting geth release)." + }, + "eth_estimateGas/test_28": { + "pr": "ethereum/go-ethereum#35228", + "note": "geth does not apply movePrecompileToAddress in the gas estimator; NM's 0x5208 is correct. geth-side fix (awaiting geth release)." + } +}