From 6e4c7ecdc0a8af593d71baa05302a15affbc54fa Mon Sep 17 00:00:00 2001 From: Stefan Date: Tue, 11 Aug 2026 10:00:48 +0200 Subject: [PATCH 1/4] feat(suite): cap the pre-run payloads kept in a suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The jochemnet bloatnet pre-run bundle is a single 9.4 GiB pre-run.request — ~8k blocks of setup. CreateSuiteOutput copies it into the suite directory on every job and the runner then uploads it, so it costs a 9.4 GiB local write plus a 9.4 GiB transfer per completed run. Nothing consumes it: the runner replays pre-runs from the fixtures cache, not from the suite, and the UI has no use for a bundle that size. Cap it. A pre-run step over max_pre_run_step_size (default 512MB) is recorded in summary.json with its size and "omitted": true, but not copied — so it is never uploaded either. The size is checked with a stat before the copy, so an oversized bundle costs neither the write nor the transfer. "0" disables the limit. Recording the omission rather than silently dropping the entry keeps the suite honest about what the run replayed, and gives the UI something to check before offering the file for viewing. The bytes remain in the fixtures artifact the step came from. --- cmd/benchmarkoor/run.go | 1 + config.example.yaml | 7 +++ pkg/config/config.go | 27 +++++++++++ pkg/executor/executor.go | 8 +++- pkg/executor/suite.go | 59 +++++++++++++++++++++-- pkg/executor/suite_test.go | 98 +++++++++++++++++++++++++++++++++++--- 6 files changed, 188 insertions(+), 12 deletions(-) diff --git a/cmd/benchmarkoor/run.go b/cmd/benchmarkoor/run.go index 76b062c..37e1dfd 100644 --- a/cmd/benchmarkoor/run.go +++ b/cmd/benchmarkoor/run.go @@ -256,6 +256,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { ResultsOwner: resultsOwner, SystemResourceCollectionEnabled: *cfg.Runner.Benchmark.SystemResourceCollectionEnabled, GitHubToken: cfg.Runner.GitHubToken, + MaxPreRunStepSize: cfg.Runner.Benchmark.Tests.GetMaxPreRunStepSize(), } exec = executor.NewExecutor(log, execCfg) diff --git a/config.example.yaml b/config.example.yaml index 9255f78..fcd728a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -227,6 +227,13 @@ runner: # # local_genesis_tarball: /home/user/eest-output/benchmark_genesis.tar.gz # # # fixtures_subdir also works with local tarballs # + # # Optional: cap on pre-run step files kept in the suite directory, and + # # so on what gets uploaded with it. A bundle over the limit is recorded + # # in summary.json with "omitted": true but not stored — it is a replay + # # script for the runner, not something the UI needs, and it stays in the + # # fixtures artifact it came from. Default "512MB"; "0" keeps every one. + # # max_pre_run_step_size: 512MB + # # Optional: External opcode metadata for the test suite. # # A JSON file mapping test names to opcode counts: {"test_name": {"OPCODE": count, ...}} # # Two modes: diff --git a/pkg/config/config.go b/pkg/config/config.go index fb15a80..dc20580 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -819,6 +819,33 @@ type TestsConfig struct { Metadata MetadataConfig `yaml:"metadata,omitempty" mapstructure:"metadata"` Source SourceConfig `yaml:"source,omitempty" mapstructure:"source"` OpcodeSource *OpcodeSourceConfig `yaml:"opcode_source,omitempty" mapstructure:"opcode_source"` + // MaxPreRunStepSize caps the pre-run step files kept in the suite + // directory, e.g. "512MB". A bundle over the limit is recorded in + // summary.json but not copied, so it is never uploaded: it is a replay + // script for the runner, not something the UI has any use for, and it + // remains in the fixtures artifact it came from. "0" keeps every bundle. + // Empty uses DefaultMaxPreRunStepSize. + MaxPreRunStepSize string `yaml:"max_pre_run_step_size,omitempty" mapstructure:"max_pre_run_step_size"` +} + +// DefaultMaxPreRunStepSize is the default cap on pre-run step files kept in a +// suite directory. Sized to admit ordinary bundles while excluding the +// multi-GB ones a bloatnet-style setup produces. +const DefaultMaxPreRunStepSize = 512 * 1024 * 1024 + +// GetMaxPreRunStepSize returns the cap in bytes, with the default applied. +// A zero value means no limit. +func (t *TestsConfig) GetMaxPreRunStepSize() int64 { + if t == nil || t.MaxPreRunStepSize == "" { + return DefaultMaxPreRunStepSize + } + + size, err := ParseByteSize(t.MaxPreRunStepSize) + if err != nil { + return DefaultMaxPreRunStepSize + } + + return int64(size) } // SourceConfig defines where to find test files. diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index d19983e..ccfcd15 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -143,6 +143,9 @@ type Config struct { ResultsOwner *fsutil.OwnerConfig // Optional file ownership for results directory SystemResourceCollectionEnabled bool // Enable system resource collection (cgroups/Docker Stats) GitHubToken string // Optional GitHub token for API-based artifact downloads + // MaxPreRunStepSize caps the pre-run payloads kept in the suite directory, + // and so the ones uploaded with it. Zero or less keeps every one. + MaxPreRunStepSize int64 } // NewExecutor creates a new executor instance. @@ -260,7 +263,10 @@ func (e *executor) createSuiteOutput() error { } // Create suite output directory. - if err := CreateSuiteOutput(e.log, e.cfg.ResultsDir, hash, suiteInfo, e.prepared, e.cfg.ResultsOwner); err != nil { + if err := CreateSuiteOutput( + e.log, e.cfg.ResultsDir, hash, suiteInfo, e.prepared, e.cfg.ResultsOwner, + e.cfg.MaxPreRunStepSize, + ); err != nil { return fmt.Errorf("creating suite output: %w", err) } diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 9b6a10a..6c1a4b2 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -72,6 +72,13 @@ type SourceStepsGlobs struct { // SuiteFile represents a file in the suite output. type SuiteFile struct { OgPath string `json:"og_path"` // original relative path + // SizeBytes is the size of the source file, recorded even when the file + // itself was left out of the suite. + SizeBytes int64 `json:"size_bytes,omitempty"` + // Omitted marks a step whose payload was too large to keep in the suite + // directory, so it was never uploaded. The UI must not offer it for + // viewing; the bytes live in the fixtures artifact the step came from. + Omitted bool `json:"omitted,omitempty"` } // SuiteTestEEST contains EEST-specific metadata for a test. @@ -159,12 +166,15 @@ func getStepContent(step *StepFile) ([]byte, error) { } // CreateSuiteOutput creates the suite directory structure with copied files and summary. +// maxPreRunStepSize caps the pre-run payloads kept in the suite; see +// config.TestsConfig.MaxPreRunStepSize. Zero or less keeps every one. func CreateSuiteOutput( log logrus.FieldLogger, resultsDir, hash string, info *SuiteInfo, prepared *PreparedSource, owner *fsutil.OwnerConfig, + maxPreRunStepSize int64, ) error { suiteDir := filepath.Join(resultsDir, "suites", hash) @@ -203,7 +213,7 @@ func CreateSuiteOutput( // Copy pre-run steps. // Structure: //pre_run.request (same pattern as tests). for _, f := range prepared.PreRunSteps { - suiteFile, err := copyPreRunStepFile(suiteDir, f, owner) + suiteFile, err := copyPreRunStepFile(log, suiteDir, f, owner, maxPreRunStepSize) if err != nil { return fmt.Errorf("copying pre-run step: %w", err) } @@ -419,7 +429,15 @@ func copyTestStepFile(testDir, stepType string, file *StepFile, owner *fsutil.Ow // copyPreRunStepFile copies a pre-run step file to the suite directory. // Files are stored as //pre_run.request (same pattern as tests). -func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConfig) (*SuiteFile, error) { +// A file over maxSize is described in the summary but not copied, so it never +// reaches the bucket either; maxSize <= 0 disables the limit. +func copyPreRunStepFile( + log logrus.FieldLogger, + suiteDir string, + file *StepFile, + owner *fsutil.OwnerConfig, + maxSize int64, +) (*SuiteFile, error) { // Create step directory using the step name (relative path). stepDir := filepath.Join(suiteDir, file.Name) if err := fsutil.MkdirAll(stepDir, 0755, owner); err != nil { @@ -430,11 +448,19 @@ func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConf // Handle provider-based steps. if file.Provider != nil { - if err := fsutil.WriteFile(dstPath, file.Provider.Content(), 0644, owner); err != nil { + content := file.Provider.Content() + + if size := int64(len(content)); maxSize > 0 && size > maxSize { + logOmittedPreRun(log, file.Name, size, maxSize) + + return &SuiteFile{OgPath: file.Name, SizeBytes: size, Omitted: true}, nil + } + + if err := fsutil.WriteFile(dstPath, content, 0644, owner); err != nil { return nil, fmt.Errorf("writing content: %w", err) } - return &SuiteFile{OgPath: file.Name}, nil + return &SuiteFile{OgPath: file.Name, SizeBytes: int64(len(content))}, nil } // Handle file-based steps. @@ -445,6 +471,19 @@ func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConf defer func() { _ = srcFile.Close() }() + stat, err := srcFile.Stat() + if err != nil { + return nil, fmt.Errorf("stating source: %w", err) + } + + // Checked before the copy, so an oversized bundle costs neither the local + // write nor the upload. + if maxSize > 0 && stat.Size() > maxSize { + logOmittedPreRun(log, file.Name, stat.Size(), maxSize) + + return &SuiteFile{OgPath: file.Name, SizeBytes: stat.Size(), Omitted: true}, nil + } + dstFile, err := fsutil.Create(dstPath, owner) if err != nil { return nil, fmt.Errorf("creating destination: %w", err) @@ -456,7 +495,17 @@ func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConf return nil, fmt.Errorf("copying content: %w", err) } - return &SuiteFile{OgPath: file.Name}, nil + return &SuiteFile{OgPath: file.Name, SizeBytes: stat.Size()}, nil +} + +func logOmittedPreRun(log logrus.FieldLogger, name string, size, maxSize int64) { + log.WithFields(logrus.Fields{ + "step": name, + "bytes": size, + "max": maxSize, + "og_path": name, + "artifact": "still available in the fixtures the step came from", + }).Info("Pre-run bundle over the size limit; describing it in the suite without storing it") } // GetGitCommitSHA retrieves the current commit SHA from a git repository. diff --git a/pkg/executor/suite_test.go b/pkg/executor/suite_test.go index 7e2e45f..befb672 100644 --- a/pkg/executor/suite_test.go +++ b/pkg/executor/suite_test.go @@ -44,7 +44,7 @@ func TestCreateSuiteOutput_WritesPayloadSizes(t *testing.T) { Hash: "deadbeef", } log := logrus.New() - err := CreateSuiteOutput(log, tmp, "deadbeef", info, prepared, nil) + err := CreateSuiteOutput(log, tmp, "deadbeef", info, prepared, nil, 0) require.NoError(t, err) summaryPath := filepath.Join(tmp, "suites", "deadbeef", "summary.json") @@ -84,7 +84,7 @@ func TestCreateSuiteOutput_AggregatesMetadataOpcodeCounts(t *testing.T) { }, } info := &SuiteInfo{Hash: "cafe"} - err := CreateSuiteOutput(logrus.New(), tmp, "cafe", info, prepared, nil) + err := CreateSuiteOutput(logrus.New(), tmp, "cafe", info, prepared, nil, 0) require.NoError(t, err) data, err := os.ReadFile(filepath.Join(tmp, "suites", "cafe", "summary.json")) @@ -167,7 +167,7 @@ func TestCreateSuiteOutput_CopiesEESTMeta(t *testing.T) { } info := &SuiteInfo{Hash: "abc123"} - require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "abc123", info, prepared, nil)) + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "abc123", info, prepared, nil, 0)) suiteMeta := filepath.Join(tmp, "suites", "abc123", ".eest-meta") @@ -203,7 +203,7 @@ func TestCreateSuiteOutput_NoEESTMetaWhenAbsent(t *testing.T) { } info := &SuiteInfo{Hash: "nometa01"} - require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "nometa01", info, prepared, nil)) + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "nometa01", info, prepared, nil, 0)) _, err := os.Stat(filepath.Join(tmp, "suites", "nometa01", ".eest-meta")) assert.True(t, os.IsNotExist(err)) @@ -234,7 +234,7 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { // First run — creates the suite and writes initial sizes. log := logrus.New() info1 := &SuiteInfo{Hash: "cafef00d"} - require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info1, prepared, nil)) + require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info1, prepared, nil, 0)) // Simulate a legacy summary: rewrite the file with payload_sizes cleared. summaryPath := filepath.Join(tmp, "suites", "cafef00d", "summary.json") @@ -251,7 +251,7 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { // Second run — should detect suite exists, read on-disk test.request, and merge. info2 := &SuiteInfo{Hash: "cafef00d"} - require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info2, prepared, nil)) + require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info2, prepared, nil, 0)) final, err := os.ReadFile(summaryPath) require.NoError(t, err) @@ -263,3 +263,89 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { require.Len(t, parsed.Tests[0].PayloadSizes.Test.SSZFull, 1) assert.Greater(t, parsed.Tests[0].PayloadSizes.Test.SSZFull[0], uint64(100), "merge path should backfill sizes") } + +// A pre-run bundle is a replay script for the runner, not something the UI +// needs — the jochemnet bloatnet one is 9.4 GiB of setup blocks. Over the +// limit it gets described in the summary but never written, so it is never +// uploaded either. +func TestCreateSuiteOutput_OmitsOversizedPreRunSteps(t *testing.T) { + log := logrus.New() + tmp := t.TempDir() + + bundle := filepath.Join(t.TempDir(), "pre-run.request") + require.NoError(t, os.WriteFile(bundle, []byte("0123456789"), 0o600)) + + prepared := &PreparedSource{ + PreRunSteps: []*StepFile{{Name: "pre_run/pre-run.request", Path: bundle}}, + Tests: []*TestWithSteps{ + { + Name: "test_with_prerun", + Test: &StepFile{ + Name: "test_with_prerun", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + + info := &SuiteInfo{Hash: "b16b16"} + require.NoError(t, CreateSuiteOutput(log, tmp, "b16b16", info, prepared, nil, 5)) + + suiteDir := filepath.Join(tmp, "suites", "b16b16") + assert.NoFileExists(t, filepath.Join(suiteDir, "pre_run", "pre-run.request", "pre_run.request")) + + var parsed SuiteInfo + data, err := os.ReadFile(filepath.Join(suiteDir, "summary.json")) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &parsed)) + + // The suite still describes the step, so nothing about the run is lost. + require.Len(t, parsed.PreRunSteps, 1) + assert.Equal(t, "pre_run/pre-run.request", parsed.PreRunSteps[0].OgPath) + assert.True(t, parsed.PreRunSteps[0].Omitted) + assert.Equal(t, int64(10), parsed.PreRunSteps[0].SizeBytes) + + // Tests are untouched by the pre-run limit. + require.Len(t, parsed.Tests, 1) + assert.FileExists(t, filepath.Join(suiteDir, "test_with_prerun", "test.request")) +} + +// Under the limit, and with the limit disabled, the bundle is stored as before. +func TestCreateSuiteOutput_KeepsPreRunStepsWithinLimit(t *testing.T) { + log := logrus.New() + + for name, limit := range map[string]int64{"under limit": 1024, "no limit": 0} { + t.Run(name, func(t *testing.T) { + tmp := t.TempDir() + + bundle := filepath.Join(t.TempDir(), "pre-run.request") + require.NoError(t, os.WriteFile(bundle, []byte("0123456789"), 0o600)) + + prepared := &PreparedSource{ + PreRunSteps: []*StepFile{{Name: "pre_run/pre-run.request", Path: bundle}}, + Tests: []*TestWithSteps{ + { + Name: "test_with_prerun", + Test: &StepFile{ + Name: "test_with_prerun", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + + info := &SuiteInfo{Hash: "5ma11"} + require.NoError(t, CreateSuiteOutput(log, tmp, "5ma11", info, prepared, nil, limit)) + + stored := filepath.Join(tmp, "suites", "5ma11", "pre_run", "pre-run.request", "pre_run.request") + assert.FileExists(t, stored) + + var parsed SuiteInfo + data, err := os.ReadFile(filepath.Join(tmp, "suites", "5ma11", "summary.json")) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &parsed)) + require.Len(t, parsed.PreRunSteps, 1) + assert.False(t, parsed.PreRunSteps[0].Omitted) + }) + } +} From 94a94fd8ff4936737b934254cb2b8c875feefac2 Mon Sep 17 00:00:00 2001 From: Stefan Date: Tue, 11 Aug 2026 10:08:25 +0200 Subject: [PATCH 2/4] refactor(suite): check the pre-run size before creating anything Stat the step first so an omitted bundle leaves no empty step directory implying a file that was never stored, and so the provider path does not materialise content it is about to discard. --- pkg/executor/suite.go | 65 +++++++++++++++++++------------------- pkg/executor/suite_test.go | 23 ++++++++++++++ 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 6c1a4b2..841badc 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -438,6 +438,24 @@ func copyPreRunStepFile( owner *fsutil.OwnerConfig, maxSize int64, ) (*SuiteFile, error) { + size, err := stepSize(file) + if err != nil { + return nil, err + } + + // Checked before anything is created, so an oversized bundle costs neither + // the local write nor the upload — and leaves no empty directory behind to + // suggest a file that was never stored. + if maxSize > 0 && size > maxSize { + log.WithFields(logrus.Fields{ + "step": file.Name, + "bytes": size, + "max": maxSize, + }).Info("Pre-run bundle over the size limit; describing it in the suite without storing it") + + return &SuiteFile{OgPath: file.Name, SizeBytes: size, Omitted: true}, nil + } + // Create step directory using the step name (relative path). stepDir := filepath.Join(suiteDir, file.Name) if err := fsutil.MkdirAll(stepDir, 0755, owner); err != nil { @@ -448,19 +466,11 @@ func copyPreRunStepFile( // Handle provider-based steps. if file.Provider != nil { - content := file.Provider.Content() - - if size := int64(len(content)); maxSize > 0 && size > maxSize { - logOmittedPreRun(log, file.Name, size, maxSize) - - return &SuiteFile{OgPath: file.Name, SizeBytes: size, Omitted: true}, nil - } - - if err := fsutil.WriteFile(dstPath, content, 0644, owner); err != nil { + if err := fsutil.WriteFile(dstPath, file.Provider.Content(), 0644, owner); err != nil { return nil, fmt.Errorf("writing content: %w", err) } - return &SuiteFile{OgPath: file.Name, SizeBytes: int64(len(content))}, nil + return &SuiteFile{OgPath: file.Name, SizeBytes: size}, nil } // Handle file-based steps. @@ -471,19 +481,6 @@ func copyPreRunStepFile( defer func() { _ = srcFile.Close() }() - stat, err := srcFile.Stat() - if err != nil { - return nil, fmt.Errorf("stating source: %w", err) - } - - // Checked before the copy, so an oversized bundle costs neither the local - // write nor the upload. - if maxSize > 0 && stat.Size() > maxSize { - logOmittedPreRun(log, file.Name, stat.Size(), maxSize) - - return &SuiteFile{OgPath: file.Name, SizeBytes: stat.Size(), Omitted: true}, nil - } - dstFile, err := fsutil.Create(dstPath, owner) if err != nil { return nil, fmt.Errorf("creating destination: %w", err) @@ -495,17 +492,21 @@ func copyPreRunStepFile( return nil, fmt.Errorf("copying content: %w", err) } - return &SuiteFile{OgPath: file.Name, SizeBytes: stat.Size()}, nil + return &SuiteFile{OgPath: file.Name, SizeBytes: size}, nil } -func logOmittedPreRun(log logrus.FieldLogger, name string, size, maxSize int64) { - log.WithFields(logrus.Fields{ - "step": name, - "bytes": size, - "max": maxSize, - "og_path": name, - "artifact": "still available in the fixtures the step came from", - }).Info("Pre-run bundle over the size limit; describing it in the suite without storing it") +// stepSize reports a step's payload size without reading a file-backed one. +func stepSize(file *StepFile) (int64, error) { + if file.Provider != nil { + return int64(len(file.Provider.Content())), nil + } + + stat, err := os.Stat(file.Path) + if err != nil { + return 0, fmt.Errorf("stating source: %w", err) + } + + return stat.Size(), nil } // GetGitCommitSHA retrieves the current commit SHA from a git repository. diff --git a/pkg/executor/suite_test.go b/pkg/executor/suite_test.go index befb672..53dc4c0 100644 --- a/pkg/executor/suite_test.go +++ b/pkg/executor/suite_test.go @@ -349,3 +349,26 @@ func TestCreateSuiteOutput_KeepsPreRunStepsWithinLimit(t *testing.T) { }) } } + +// An omitted bundle must not leave an empty directory behind, which would +// imply a file that was never stored. +func TestCreateSuiteOutput_OmittedPreRunLeavesNoStepDir(t *testing.T) { + tmp := t.TempDir() + + bundle := filepath.Join(t.TempDir(), "pre-run.request") + require.NoError(t, os.WriteFile(bundle, []byte("0123456789"), 0o600)) + + prepared := &PreparedSource{ + PreRunSteps: []*StepFile{{Name: "pre_run/pre-run.request", Path: bundle}}, + Tests: []*TestWithSteps{ + { + Name: "test_x", + Test: &StepFile{Name: "test_x", Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}}, + }, + }, + } + + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "n0d1r", &SuiteInfo{Hash: "n0d1r"}, prepared, nil, 5)) + + assert.NoDirExists(t, filepath.Join(tmp, "suites", "n0d1r", "pre_run")) +} From 4a53a60cc932df650cf538f42cf920168e92e6d4 Mon Sep 17 00:00:00 2001 From: Stefan Date: Tue, 11 Aug 2026 10:12:56 +0200 Subject: [PATCH 3/4] refactor(config): move the pre-run cap under results_upload and name it for what it does max_pre_run_step_size under tests read like a test-execution setting; nothing in the name said it governs what gets uploaded with a suite. It is now results_upload.max_pre_run_upload_size, next to the rest of the upload config, where both the path and the name say what it affects. Also documents it in docs/configuration.md, with a section covering why the cap exists, what an omitted bundle looks like in summary.json, and where to find the bytes when one is skipped. --- cmd/benchmarkoor/run.go | 2 +- config.example.yaml | 13 +++++----- docs/configuration.md | 30 ++++++++++++++++++++++ pkg/config/config.go | 54 ++++++++++++++++++++-------------------- pkg/executor/executor.go | 8 +++--- pkg/executor/suite.go | 9 ++++--- 6 files changed, 73 insertions(+), 43 deletions(-) diff --git a/cmd/benchmarkoor/run.go b/cmd/benchmarkoor/run.go index 37e1dfd..a2dfe42 100644 --- a/cmd/benchmarkoor/run.go +++ b/cmd/benchmarkoor/run.go @@ -256,7 +256,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { ResultsOwner: resultsOwner, SystemResourceCollectionEnabled: *cfg.Runner.Benchmark.SystemResourceCollectionEnabled, GitHubToken: cfg.Runner.GitHubToken, - MaxPreRunStepSize: cfg.Runner.Benchmark.Tests.GetMaxPreRunStepSize(), + MaxPreRunUploadSize: cfg.Runner.Benchmark.ResultsUpload.GetMaxPreRunUploadSize(), } exec = executor.NewExecutor(log, execCfg) diff --git a/config.example.yaml b/config.example.yaml index fcd728a..9698b7b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -106,6 +106,12 @@ runner: # # Path-style addressing: required for MinIO and Cloudflare R2. # force_path_style: false # # parallel_uploads: 50 # Number of concurrent file uploads + # # Cap on pre-run bundles uploaded with a suite. One over the limit is + # # recorded in summary.json with "omitted": true but never stored, so it + # # is never uploaded — it is a replay script for the runner, not + # # something the UI needs, and it stays in the fixtures artifact it came + # # from. Default "512MB"; "0" uploads every bundle. + # # max_pre_run_upload_size: 512MB # # Cap on the whole post-run upload (run dir + suite dir). Defaults to # # 60m; a stateful suite ships its pre-run bundle, which can be tens of GB. # # timeout: 60m @@ -227,13 +233,6 @@ runner: # # local_genesis_tarball: /home/user/eest-output/benchmark_genesis.tar.gz # # # fixtures_subdir also works with local tarballs # - # # Optional: cap on pre-run step files kept in the suite directory, and - # # so on what gets uploaded with it. A bundle over the limit is recorded - # # in summary.json with "omitted": true but not stored — it is a replay - # # script for the runner, not something the UI needs, and it stays in the - # # fixtures artifact it came from. Default "512MB"; "0" keeps every one. - # # max_pre_run_step_size: 512MB - # # Optional: External opcode metadata for the test suite. # # A JSON file mapping test names to opcode counts: {"test_name": {"OPCODE": count, ...}} # # Two modes: diff --git a/docs/configuration.md b/docs/configuration.md index 1f06a1e..9eedadc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -656,6 +656,7 @@ The `runner.benchmark.results_upload` section configures automatic uploading of runner: benchmark: results_upload: + # max_pre_run_upload_size: 512MB s3: enabled: true endpoint_url: https://s3.amazonaws.com @@ -669,6 +670,12 @@ runner: force_path_style: false ``` +| Option | Type | Required | Default | Description | +|--------|------|----------|---------|-------------| +| `max_pre_run_upload_size` | string | No | `512MB` | Cap on pre-run bundles uploaded with a suite (see [Pre-Run Upload Size](#pre-run-upload-size)). `0` uploads every bundle | + +**`s3` options:** + | Option | Type | Required | Default | Description | |--------|------|----------|---------|-------------| | `enabled` | bool | Yes | `false` | Enable S3 upload | @@ -683,6 +690,29 @@ runner: | `force_path_style` | bool | No | `false` | Use path-style addressing (required for MinIO and Cloudflare R2) | | `parallel_uploads` | int | No | `50` | Number of concurrent file uploads | +#### Pre-Run Upload Size + +A suite directory holds a copy of every step file so the UI can display it, and that includes the pre-run bundle. Those bundles can be enormous — a bloatnet-style setup that deploys 100k contracts produces a single `pre-run.request` of around 9.4 GiB — and nothing reads them from there: the runner replays pre-runs from its fixtures cache, not from the suite. + +`max_pre_run_upload_size` caps what gets kept, and therefore what gets uploaded. A bundle over the limit is still described in `summary.json`, so the suite stays honest about what the run replayed: + +```json +"pre_run_steps": [ + { "og_path": "pre_run/pre-run.request", "size_bytes": 10062313486, "omitted": true } +] +``` + +The size is checked before the copy, so an oversized bundle costs neither the local write nor the transfer. The bytes remain in the fixtures artifact the step came from, which is where to look if you need to inspect one. + +Set `0` to upload every bundle regardless of size. + +```yaml +runner: + benchmark: + results_upload: + max_pre_run_upload_size: 512MB +``` + **Important:** The `endpoint_url` must be the base URL without any path component. Do not include the bucket name in the URL — the SDK handles that separately via the `bucket` field. For example, use `https://.r2.cloudflarestorage.com`, not `https://.r2.cloudflarestorage.com/my-bucket`. When enabled, a preflight check runs before any benchmarks to verify S3 connectivity. Each instance's results directory is uploaded after the run completes (including on failure, for partial results). diff --git a/pkg/config/config.go b/pkg/config/config.go index dc20580..5653ec4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -774,6 +774,33 @@ type BenchmarkConfig struct { // ResultsUploadConfig contains configuration for uploading results. type ResultsUploadConfig struct { S3 *S3UploadConfig `yaml:"s3,omitempty" mapstructure:"s3"` + // MaxPreRunUploadSize caps the pre-run bundles uploaded with a suite, e.g. + // "512MB". A bundle over the limit is recorded in summary.json with + // "omitted": true but never copied into the suite directory, so it is + // never uploaded: it is a replay script for the runner, not something the + // UI has any use for, and it stays in the fixtures artifact it came from. + // "0" uploads every bundle. Empty uses DefaultMaxPreRunUploadSize. + MaxPreRunUploadSize string `yaml:"max_pre_run_upload_size,omitempty" mapstructure:"max_pre_run_upload_size"` +} + +// DefaultMaxPreRunUploadSize is the default cap on uploaded pre-run bundles. +// Sized to admit ordinary ones while excluding the multi-GB bundles a +// bloatnet-style setup produces. +const DefaultMaxPreRunUploadSize = 512 * 1024 * 1024 + +// GetMaxPreRunUploadSize returns the cap in bytes, with the default applied. +// A zero value means no limit. +func (r *ResultsUploadConfig) GetMaxPreRunUploadSize() int64 { + if r == nil || r.MaxPreRunUploadSize == "" { + return DefaultMaxPreRunUploadSize + } + + size, err := ParseByteSize(r.MaxPreRunUploadSize) + if err != nil { + return DefaultMaxPreRunUploadSize + } + + return int64(size) } // S3UploadConfig contains S3-compatible storage upload settings. @@ -819,33 +846,6 @@ type TestsConfig struct { Metadata MetadataConfig `yaml:"metadata,omitempty" mapstructure:"metadata"` Source SourceConfig `yaml:"source,omitempty" mapstructure:"source"` OpcodeSource *OpcodeSourceConfig `yaml:"opcode_source,omitempty" mapstructure:"opcode_source"` - // MaxPreRunStepSize caps the pre-run step files kept in the suite - // directory, e.g. "512MB". A bundle over the limit is recorded in - // summary.json but not copied, so it is never uploaded: it is a replay - // script for the runner, not something the UI has any use for, and it - // remains in the fixtures artifact it came from. "0" keeps every bundle. - // Empty uses DefaultMaxPreRunStepSize. - MaxPreRunStepSize string `yaml:"max_pre_run_step_size,omitempty" mapstructure:"max_pre_run_step_size"` -} - -// DefaultMaxPreRunStepSize is the default cap on pre-run step files kept in a -// suite directory. Sized to admit ordinary bundles while excluding the -// multi-GB ones a bloatnet-style setup produces. -const DefaultMaxPreRunStepSize = 512 * 1024 * 1024 - -// GetMaxPreRunStepSize returns the cap in bytes, with the default applied. -// A zero value means no limit. -func (t *TestsConfig) GetMaxPreRunStepSize() int64 { - if t == nil || t.MaxPreRunStepSize == "" { - return DefaultMaxPreRunStepSize - } - - size, err := ParseByteSize(t.MaxPreRunStepSize) - if err != nil { - return DefaultMaxPreRunStepSize - } - - return int64(size) } // SourceConfig defines where to find test files. diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index ccfcd15..9ed582b 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -143,9 +143,9 @@ type Config struct { ResultsOwner *fsutil.OwnerConfig // Optional file ownership for results directory SystemResourceCollectionEnabled bool // Enable system resource collection (cgroups/Docker Stats) GitHubToken string // Optional GitHub token for API-based artifact downloads - // MaxPreRunStepSize caps the pre-run payloads kept in the suite directory, - // and so the ones uploaded with it. Zero or less keeps every one. - MaxPreRunStepSize int64 + // MaxPreRunUploadSize caps the pre-run payloads kept in the suite + // directory, and so the ones uploaded with it. Zero or less keeps every one. + MaxPreRunUploadSize int64 } // NewExecutor creates a new executor instance. @@ -265,7 +265,7 @@ func (e *executor) createSuiteOutput() error { // Create suite output directory. if err := CreateSuiteOutput( e.log, e.cfg.ResultsDir, hash, suiteInfo, e.prepared, e.cfg.ResultsOwner, - e.cfg.MaxPreRunStepSize, + e.cfg.MaxPreRunUploadSize, ); err != nil { return fmt.Errorf("creating suite output: %w", err) } diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 841badc..5b4c40e 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -166,15 +166,16 @@ func getStepContent(step *StepFile) ([]byte, error) { } // CreateSuiteOutput creates the suite directory structure with copied files and summary. -// maxPreRunStepSize caps the pre-run payloads kept in the suite; see -// config.TestsConfig.MaxPreRunStepSize. Zero or less keeps every one. +// maxPreRunUploadSize caps the pre-run payloads kept in the suite, and so the +// ones uploaded with it; see config.ResultsUploadConfig.MaxPreRunUploadSize. +// Zero or less keeps every one. func CreateSuiteOutput( log logrus.FieldLogger, resultsDir, hash string, info *SuiteInfo, prepared *PreparedSource, owner *fsutil.OwnerConfig, - maxPreRunStepSize int64, + maxPreRunUploadSize int64, ) error { suiteDir := filepath.Join(resultsDir, "suites", hash) @@ -213,7 +214,7 @@ func CreateSuiteOutput( // Copy pre-run steps. // Structure: //pre_run.request (same pattern as tests). for _, f := range prepared.PreRunSteps { - suiteFile, err := copyPreRunStepFile(log, suiteDir, f, owner, maxPreRunStepSize) + suiteFile, err := copyPreRunStepFile(log, suiteDir, f, owner, maxPreRunUploadSize) if err != nil { return fmt.Errorf("copying pre-run step: %w", err) } From 4f8e620663a97fd4e89f4a885743b9fecf485e3e Mon Sep 17 00:00:00 2001 From: Stefan Date: Tue, 11 Aug 2026 10:16:08 +0200 Subject: [PATCH 4/4] style: trim the comments added by this PR to two lines --- config.example.yaml | 7 ++----- pkg/config/config.go | 14 ++++---------- pkg/executor/executor.go | 4 ++-- pkg/executor/suite.go | 22 +++++++++------------- pkg/executor/suite_test.go | 9 +++------ 5 files changed, 20 insertions(+), 36 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 9698b7b..1ce0d68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -106,11 +106,8 @@ runner: # # Path-style addressing: required for MinIO and Cloudflare R2. # force_path_style: false # # parallel_uploads: 50 # Number of concurrent file uploads - # # Cap on pre-run bundles uploaded with a suite. One over the limit is - # # recorded in summary.json with "omitted": true but never stored, so it - # # is never uploaded — it is a replay script for the runner, not - # # something the UI needs, and it stays in the fixtures artifact it came - # # from. Default "512MB"; "0" uploads every bundle. + # # Cap on pre-run bundles uploaded with a suite; over it they are + # # recorded in summary.json but not stored. Default "512MB", "0" = all. # # max_pre_run_upload_size: 512MB # # Cap on the whole post-run upload (run dir + suite dir). Defaults to # # 60m; a stateful suite ships its pre-run bundle, which can be tens of GB. diff --git a/pkg/config/config.go b/pkg/config/config.go index 5653ec4..7420953 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -775,21 +775,15 @@ type BenchmarkConfig struct { type ResultsUploadConfig struct { S3 *S3UploadConfig `yaml:"s3,omitempty" mapstructure:"s3"` // MaxPreRunUploadSize caps the pre-run bundles uploaded with a suite, e.g. - // "512MB". A bundle over the limit is recorded in summary.json with - // "omitted": true but never copied into the suite directory, so it is - // never uploaded: it is a replay script for the runner, not something the - // UI has any use for, and it stays in the fixtures artifact it came from. - // "0" uploads every bundle. Empty uses DefaultMaxPreRunUploadSize. + // "512MB". Over it they are recorded in summary.json but not stored. MaxPreRunUploadSize string `yaml:"max_pre_run_upload_size,omitempty" mapstructure:"max_pre_run_upload_size"` } -// DefaultMaxPreRunUploadSize is the default cap on uploaded pre-run bundles. -// Sized to admit ordinary ones while excluding the multi-GB bundles a -// bloatnet-style setup produces. +// DefaultMaxPreRunUploadSize admits ordinary pre-run bundles while excluding +// the multi-GB ones a bloatnet-style setup produces. const DefaultMaxPreRunUploadSize = 512 * 1024 * 1024 -// GetMaxPreRunUploadSize returns the cap in bytes, with the default applied. -// A zero value means no limit. +// GetMaxPreRunUploadSize returns the cap in bytes; zero means no limit. func (r *ResultsUploadConfig) GetMaxPreRunUploadSize() int64 { if r == nil || r.MaxPreRunUploadSize == "" { return DefaultMaxPreRunUploadSize diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 9ed582b..3970104 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -143,8 +143,8 @@ type Config struct { ResultsOwner *fsutil.OwnerConfig // Optional file ownership for results directory SystemResourceCollectionEnabled bool // Enable system resource collection (cgroups/Docker Stats) GitHubToken string // Optional GitHub token for API-based artifact downloads - // MaxPreRunUploadSize caps the pre-run payloads kept in the suite - // directory, and so the ones uploaded with it. Zero or less keeps every one. + // MaxPreRunUploadSize caps the pre-run payloads kept in the suite, and so + // the ones uploaded with it. Zero or less keeps every one. MaxPreRunUploadSize int64 } diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 5b4c40e..24e5efb 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -72,12 +72,10 @@ type SourceStepsGlobs struct { // SuiteFile represents a file in the suite output. type SuiteFile struct { OgPath string `json:"og_path"` // original relative path - // SizeBytes is the size of the source file, recorded even when the file - // itself was left out of the suite. + // SizeBytes is the source file's size, recorded even when it was omitted. SizeBytes int64 `json:"size_bytes,omitempty"` - // Omitted marks a step whose payload was too large to keep in the suite - // directory, so it was never uploaded. The UI must not offer it for - // viewing; the bytes live in the fixtures artifact the step came from. + // Omitted marks a payload too large to keep, so never uploaded. The UI + // must not offer it; the bytes stay in the fixtures artifact. Omitted bool `json:"omitted,omitempty"` } @@ -166,9 +164,8 @@ func getStepContent(step *StepFile) ([]byte, error) { } // CreateSuiteOutput creates the suite directory structure with copied files and summary. -// maxPreRunUploadSize caps the pre-run payloads kept in the suite, and so the -// ones uploaded with it; see config.ResultsUploadConfig.MaxPreRunUploadSize. -// Zero or less keeps every one. +// maxPreRunUploadSize caps the pre-run payloads kept, and so uploaded; see +// config.ResultsUploadConfig.MaxPreRunUploadSize. Zero or less keeps every one. func CreateSuiteOutput( log logrus.FieldLogger, resultsDir, hash string, @@ -430,8 +427,8 @@ func copyTestStepFile(testDir, stepType string, file *StepFile, owner *fsutil.Ow // copyPreRunStepFile copies a pre-run step file to the suite directory. // Files are stored as //pre_run.request (same pattern as tests). -// A file over maxSize is described in the summary but not copied, so it never -// reaches the bucket either; maxSize <= 0 disables the limit. +// Over maxSize a file is described in the summary but not copied, so never +// uploaded either; maxSize <= 0 disables the limit. func copyPreRunStepFile( log logrus.FieldLogger, suiteDir string, @@ -444,9 +441,8 @@ func copyPreRunStepFile( return nil, err } - // Checked before anything is created, so an oversized bundle costs neither - // the local write nor the upload — and leaves no empty directory behind to - // suggest a file that was never stored. + // Checked before anything is created: no wasted write, and no empty + // directory implying a file that was never stored. if maxSize > 0 && size > maxSize { log.WithFields(logrus.Fields{ "step": file.Name, diff --git a/pkg/executor/suite_test.go b/pkg/executor/suite_test.go index 53dc4c0..e8c3290 100644 --- a/pkg/executor/suite_test.go +++ b/pkg/executor/suite_test.go @@ -264,10 +264,8 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { assert.Greater(t, parsed.Tests[0].PayloadSizes.Test.SSZFull[0], uint64(100), "merge path should backfill sizes") } -// A pre-run bundle is a replay script for the runner, not something the UI -// needs — the jochemnet bloatnet one is 9.4 GiB of setup blocks. Over the -// limit it gets described in the summary but never written, so it is never -// uploaded either. +// A pre-run bundle is a runner replay script, not something the UI needs — +// the jochemnet bloatnet one is 9.4 GiB. Over the limit it is described only. func TestCreateSuiteOutput_OmitsOversizedPreRunSteps(t *testing.T) { log := logrus.New() tmp := t.TempDir() @@ -350,8 +348,7 @@ func TestCreateSuiteOutput_KeepsPreRunStepsWithinLimit(t *testing.T) { } } -// An omitted bundle must not leave an empty directory behind, which would -// imply a file that was never stored. +// An omitted bundle must leave no empty directory implying a stored file. func TestCreateSuiteOutput_OmittedPreRunLeavesNoStepDir(t *testing.T) { tmp := t.TempDir()