feat: known-failures (xfail) list for expected failures

README.md

@@ -28,6 +28,25 @@ make
 
 Check out the dedicated guide in [Integration Tests](./integration/README.md).
 
+### Known (expected) failures
+
+`known-failures.json` (repo root) lists tests that are expected to fail while a fix
+is pending, keyed by `"<api>/<test>"` (extension-insensitive) with a linked PR:
+
+```json
+{
+  "eth_getBalance/test_40": { "pr": "ethereum/go-ethereum#35271", "note": "geth -32000 vs NM -32602; geth-side fix" }
+}
+```
+
+Listed tests still **run**, but:
+
+- a listed test that **fails** is reported as `KNOWN_FAIL (<pr>)` and does **not** fail the run;
+- a listed test that **passes** is reported as `UNEXPECTED_PASS` (a warning to remove the entry) — with `--strict-known-failures` this fails the run to force the cleanup.
+
+Flags: `--known-failures <path>` (default `known-failures.json`), `--strict-known-failures`.
+Prefer this over excluding/skipping a test: coverage is preserved, and you're told the moment the fix lands.
+
 ## Performance Testing
 
 Check out the dedicated guide in [Performance Tests](./perf/README.md).

cmd/integration/main.go

@@ -118,6 +118,10 @@ func parseFlags(cfg *config.Config) error {
 	reportFile := flag.String("R", "", "write CSV summary report to file")
 	flag.StringVar(reportFile, "report-file", "", "write CSV summary report to file")
 
+	knownFailuresPath := flag.String("known-failures", "known-failures.json", "path to known-failures JSON file; listed tests that fail are reported as KNOWN_FAIL and do not fail the run")
+
+	strictKnownFailures := flag.Bool("strict-known-failures", false, "fail the run when a known failure unexpectedly passes (forces cleanup of known-failures.json)")
+
 	cpuProfile := flag.String("cpuprofile", "", "write cpu profile to file")
 	memProfile := flag.String("memprofile", "", "write memory profile to file")
 	traceFile := flag.String("trace", "", "write execution trace to file")
@@ -153,6 +157,12 @@ func parseFlags(cfg *config.Config) error {
 	cfg.ArchiveNode = *archiveNode
 	cfg.PrunedNode = *prunedNode
 	cfg.MaxFailures = *maxFailures
+	cfg.StrictKnownFailures = *strictKnownFailures
+	knownFailures, err := config.LoadKnownFailures(*knownFailuresPath)
+	if err != nil {
+		return err
+	}
+	cfg.KnownFailures = knownFailures
 	cfg.ReportFile = *reportFile
 	cfg.CpuProfile = *cpuProfile
 	cfg.MemProfile = *memProfile

internal/config/config.go

@@ -119,6 +119,13 @@ type Config struct {
 	// Failure cap
 	MaxFailures int // stop after this many failures (0 = unlimited)
 
+	// Known (expected) failures: tests here still run, but a failure is reported
+	// as KNOWN_FAIL and does not fail the run; a listed test that passes is reported
+	// as UNEXPECTED_PASS (warn, or fail the run when StrictKnownFailures is set).
+	// Keyed by "<api>/<test>" without file extension (e.g. "eth_getBalance/test_40").
+	KnownFailures       map[string]KnownFailure
+	StrictKnownFailures bool
+
 	// Report
 	ReportFile string
 

internal/config/knownfailures.go

@@ -0,0 +1,48 @@
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// KnownFailure describes a test that is expected to fail, with a link to the PR
+// tracking the fix and a short human-readable reason.
+type KnownFailure struct {
+	PR   string `json:"pr"`
+	Note string `json:"note"`
+}
+
+// NormalizeTestKey returns the extension-less "<api>/<test>" key used to match a
+// test against the known-failures map (e.g. "eth_getBalance/test_40.json" ->
+// "eth_getBalance/test_40").
+func NormalizeTestKey(name string) string {
+	return strings.TrimSuffix(name, filepath.Ext(name))
+}
+
+// LoadKnownFailures reads the known-failures JSON file at path. A missing file is
+// not an error (returns an empty map), so the list is entirely optional. Keys are
+// normalized to be extension-insensitive.
+func LoadKnownFailures(path string) (map[string]KnownFailure, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return map[string]KnownFailure{}, nil
+		}
+		return nil, err
+	}
+
+	var raw map[string]KnownFailure
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return nil, err
+	}
+
+	out := make(map[string]KnownFailure, len(raw))
+	for k, v := range raw {
+		out[NormalizeTestKey(k)] = v
+	}
+	return out, nil
+}

internal/config/knownfailures_test.go

@@ -0,0 +1,64 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestLoadKnownFailures_MissingFileIsEmpty(t *testing.T) {
+	m, err := LoadKnownFailures(filepath.Join(t.TempDir(), "does-not-exist.json"))
+	if err != nil {
+		t.Fatalf("missing file should not error, got %v", err)
+	}
+	if len(m) != 0 {
+		t.Fatalf("expected empty map, got %d entries", len(m))
+	}
+}
+
+func TestLoadKnownFailures_ParsesAndNormalizesKeys(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "known-failures.json")
+	content := `{
+	  "eth_getBalance/test_40.json": {"pr": "geth#35271", "note": "empty {} block param"},
+	  "eth_sendRawTransaction/test_23": {"pr": "geth#35129", "note": "empty raw tx"}
+	}`
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	m, err := LoadKnownFailures(path)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	// The .json extension in a key must be normalized away so it matches the
+	// runtime test name (which carries the extension).
+	if kf, ok := m["eth_getBalance/test_40"]; !ok || kf.PR != "geth#35271" {
+		t.Fatalf("expected normalized key eth_getBalance/test_40 with PR geth#35271, got %+v (ok=%v)", kf, ok)
+	}
+	if kf, ok := m["eth_sendRawTransaction/test_23"]; !ok || kf.PR != "geth#35129" {
+		t.Fatalf("expected eth_sendRawTransaction/test_23 with PR geth#35129, got %+v (ok=%v)", kf, ok)
+	}
+}
+
+func TestLoadKnownFailures_InvalidJSONErrors(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "bad.json")
+	if err := os.WriteFile(path, []byte("{not json"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := LoadKnownFailures(path); err == nil {
+		t.Fatal("expected error for invalid JSON, got nil")
+	}
+}
+
+func TestNormalizeTestKey(t *testing.T) {
+	cases := map[string]string{
+		"eth_getBalance/test_40.json": "eth_getBalance/test_40",
+		"eth_getBalance/test_40":      "eth_getBalance/test_40",
+		"eth_x/test_1.tar":            "eth_x/test_1",
+	}
+	for in, want := range cases {
+		if got := NormalizeTestKey(in); got != want {
+			t.Errorf("NormalizeTestKey(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

internal/runner/runner.go

@@ -321,25 +321,57 @@ func printResult(w *bufio.Writer, result *testdata.TestResult, stats *Stats, cfg
 	tt := fmt.Sprintf("%-15s", result.Test.TransportType)
 	fmt.Fprintf(w, "%04d. %s::%s   ", result.Test.Number, tt, file)
 
-	if result.Outcome.Success {
-		stats.AddSuccess(result.Outcome.Metrics)
-		if cfg.VerboseLevel > 0 {
-			fmt.Fprintln(w, "OK")
-		} else {
-			fmt.Fprint(w, "OK\r")
-		}
+	kf, isKnownFailure := cfg.KnownFailures[config.NormalizeTestKey(result.Test.Name)]
+
+	addReport := func(res string, errField any) {
 		if cfg.VerboseLevel == 1 || cfg.ReportFile != "" {
 			reportMu.Lock()
 			*reportEntries = append(*reportEntries, reportEntry{
 				TestNumber:    result.Test.Number,
 				TransportType: result.Test.TransportType,
 				TestName:      result.Test.Name,
-				Result:        "OK",
-				ErrorMessage:  "",
+				Result:        res,
+				ErrorMessage:  errField,
 			})
 			reportMu.Unlock()
 		}
-	} else {
+	}
+
+	switch {
+	case result.Outcome.Success && isKnownFailure:
+		// A known failure that now passes: flag it so the entry gets removed.
+		// In strict mode it fails the run to force the cleanup.
+		stats.AddUnexpectedPass()
+		msg := fmt.Sprintf("known failure %s appears resolved — remove it from known-failures.json", kf.PR)
+		if cfg.StrictKnownFailures {
+			stats.AddFailure()
+			fmt.Fprintf(w, "failed: UNEXPECTED PASS (%s)\n", msg)
+		} else {
+			stats.AddSuccess(result.Outcome.Metrics)
+			fmt.Fprintf(w, "OK (UNEXPECTED PASS: %s)\n", msg)
+		}
+		addReport("UNEXPECTED_PASS", kf.PR)
+
+	case result.Outcome.Success:
+		stats.AddSuccess(result.Outcome.Metrics)
+		if cfg.VerboseLevel > 0 {
+			fmt.Fprintln(w, "OK")
+		} else {
+			fmt.Fprint(w, "OK\r")
+		}
+		addReport("OK", "")
+
+	case isKnownFailure:
+		// Expected failure: report but do not fail the run.
+		stats.AddKnownFailure()
+		errMsg := "no error"
+		if result.Outcome.Error != nil {
+			errMsg = result.Outcome.Error.Error()
+		}
+		fmt.Fprintf(w, "KNOWN FAIL (%s)\n", kf.PR)
+		addReport("KNOWN_FAIL", kf.PR+": "+errMsg)
+
+	default:
 		stats.AddFailure()
 		errMsg := "no error"
 		if result.Outcome.Error != nil {
@@ -351,21 +383,11 @@ func printResult(w *bufio.Writer, result *testdata.TestResult, stats *Stats, cfg
 		} else {
 			fmt.Fprintf(w, "failed: %s\n", errMsg)
 		}
-		if cfg.VerboseLevel == 1 || cfg.ReportFile != "" {
-			var errField any = errMsg
-			if result.Outcome.ErrorDetails != nil {
-				errField = result.Outcome.ErrorDetails
-			}
-			reportMu.Lock()
-			*reportEntries = append(*reportEntries, reportEntry{
-				TestNumber:    result.Test.Number,
-				TransportType: result.Test.TransportType,
-				TestName:      result.Test.Name,
-				Result:        "FAILED",
-				ErrorMessage:  errField,
-			})
-			reportMu.Unlock()
+		var errField any = errMsg
+		if result.Outcome.ErrorDetails != nil {
+			errField = result.Outcome.ErrorDetails
 		}
+		addReport("FAILED", errField)
 		if maxFailuresReached(cfg, stats) {
 			fmt.Fprintf(w, "\nABORTED: too many failures (%d), test sequence stopped early\n", cfg.MaxFailures)
 			w.Flush()

internal/runner/stats.go

@@ -9,11 +9,13 @@ import (
 
 // Stats aggregates metrics and counts across all tests.
 type Stats struct {
-	SuccessTests   int
-	FailedTests    int
-	ExecutedTests  int
-	SkippedTests   int
-	ScheduledTests int
+	SuccessTests     int
+	FailedTests      int
+	KnownFailures    int
+	UnexpectedPasses int
+	ExecutedTests    int
+	SkippedTests     int
+	ScheduledTests   int
 
 	TotalRoundTripTime     time.Duration
 	TotalMarshallingTime   time.Duration
@@ -39,6 +41,18 @@ func (s *Stats) AddFailure() {
 	s.ExecutedTests++
 }
 
+// AddKnownFailure records a failure that is listed in the known-failures file.
+// It counts as executed but NOT as a failure, so it does not fail the run.
+func (s *Stats) AddKnownFailure() {
+	s.KnownFailures++
+	s.ExecutedTests++
+}
+
+// AddUnexpectedPass records a test that passed but is listed as a known failure.
+func (s *Stats) AddUnexpectedPass() {
+	s.UnexpectedPasses++
+}
+
 // PrintSummary prints the v1-compatible summary output.
 func (s *Stats) PrintSummary(startTime time.Time, elapsed time.Duration, iterations, totalAPIs, totalTests int) {
 	fmt.Println("\n                                                                                                                  ")
@@ -57,4 +71,10 @@ func (s *Stats) PrintSummary(startTime time.Time, elapsed time.Duration, iterati
 	fmt.Printf("Number of executed tests:     %d\n", s.ExecutedTests)
 	fmt.Printf("Number of success tests:      %d\n", s.SuccessTests)
 	fmt.Printf("Number of failed tests:       %d\n", s.FailedTests)
+	if s.KnownFailures > 0 {
+		fmt.Printf("Number of known failures:     %d\n", s.KnownFailures)
+	}
+	if s.UnexpectedPasses > 0 {
+		fmt.Printf("Number of unexpected passes:  %d\n", s.UnexpectedPasses)
+	}
 }

known-failures.json

@@ -0,0 +1,26 @@
+{
+  "eth_sendRawTransaction/test_05": {
+    "pr": "NethermindEth/nethermind#12003",
+    "note": "NM returns -32000 'Invalid RLP' for malformed hex; should be -32602 (invalid params). NM-side fix."
+  },
+  "eth_sendRawTransaction/test_06": {
+    "pr": "NethermindEth/nethermind#12003",
+    "note": "Same as test_05 (no-0x-prefix hex). NM-side fix."
+  },
+  "eth_sendRawTransaction/test_21": {
+    "pr": "NethermindEth/nethermind#12003",
+    "note": "Same as test_05 (odd-length hex). NM-side fix."
+  },
+  "eth_getBalance/test_40": {
+    "pr": "ethereum/go-ethereum#35271",
+    "note": "geth returns -32000 for an empty {} EIP-1898 block param; NM's -32602 is spec-correct. geth-side fix (awaiting geth release)."
+  },
+  "eth_sendRawTransaction/test_23": {
+    "pr": "ethereum/go-ethereum#35129",
+    "note": "geth returns -32000 for an empty raw tx; NM's -32602 is spec-correct. geth-side fix (awaiting geth release)."
+  },
+  "eth_estimateGas/test_28": {
+    "pr": "ethereum/go-ethereum#35228",
+    "note": "geth does not apply movePrecompileToAddress in the gas estimator; NM's 0x5208 is correct. geth-side fix (awaiting geth release)."
+  }
+}