diff --git a/cmd/benchmarkoor/run.go b/cmd/benchmarkoor/run.go index 76b062c..a2dfe42 100644 --- a/cmd/benchmarkoor/run.go +++ b/cmd/benchmarkoor/run.go @@ -256,6 +256,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { ResultsOwner: resultsOwner, SystemResourceCollectionEnabled: *cfg.Runner.Benchmark.SystemResourceCollectionEnabled, GitHubToken: cfg.Runner.GitHubToken, + MaxPreRunUploadSize: cfg.Runner.Benchmark.ResultsUpload.GetMaxPreRunUploadSize(), } exec = executor.NewExecutor(log, execCfg) diff --git a/config.example.yaml b/config.example.yaml index 9255f78..1ce0d68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -106,6 +106,9 @@ runner: # # Path-style addressing: required for MinIO and Cloudflare R2. # force_path_style: false # # parallel_uploads: 50 # Number of concurrent file uploads + # # Cap on pre-run bundles uploaded with a suite; over it they are + # # recorded in summary.json but not stored. Default "512MB", "0" = all. + # # max_pre_run_upload_size: 512MB # # Cap on the whole post-run upload (run dir + suite dir). Defaults to # # 60m; a stateful suite ships its pre-run bundle, which can be tens of GB. # # timeout: 60m diff --git a/docs/configuration.md b/docs/configuration.md index 1f06a1e..9eedadc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -656,6 +656,7 @@ The `runner.benchmark.results_upload` section configures automatic uploading of runner: benchmark: results_upload: + # max_pre_run_upload_size: 512MB s3: enabled: true endpoint_url: https://s3.amazonaws.com @@ -669,6 +670,12 @@ runner: force_path_style: false ``` +| Option | Type | Required | Default | Description | +|--------|------|----------|---------|-------------| +| `max_pre_run_upload_size` | string | No | `512MB` | Cap on pre-run bundles uploaded with a suite (see [Pre-Run Upload Size](#pre-run-upload-size)). `0` uploads every bundle | + +**`s3` options:** + | Option | Type | Required | Default | Description | |--------|------|----------|---------|-------------| | `enabled` | bool | Yes | `false` | Enable S3 upload | @@ -683,6 +690,29 @@ runner: | `force_path_style` | bool | No | `false` | Use path-style addressing (required for MinIO and Cloudflare R2) | | `parallel_uploads` | int | No | `50` | Number of concurrent file uploads | +#### Pre-Run Upload Size + +A suite directory holds a copy of every step file so the UI can display it, and that includes the pre-run bundle. Those bundles can be enormous — a bloatnet-style setup that deploys 100k contracts produces a single `pre-run.request` of around 9.4 GiB — and nothing reads them from there: the runner replays pre-runs from its fixtures cache, not from the suite. + +`max_pre_run_upload_size` caps what gets kept, and therefore what gets uploaded. A bundle over the limit is still described in `summary.json`, so the suite stays honest about what the run replayed: + +```json +"pre_run_steps": [ + { "og_path": "pre_run/pre-run.request", "size_bytes": 10062313486, "omitted": true } +] +``` + +The size is checked before the copy, so an oversized bundle costs neither the local write nor the transfer. The bytes remain in the fixtures artifact the step came from, which is where to look if you need to inspect one. + +Set `0` to upload every bundle regardless of size. + +```yaml +runner: + benchmark: + results_upload: + max_pre_run_upload_size: 512MB +``` + **Important:** The `endpoint_url` must be the base URL without any path component. Do not include the bucket name in the URL — the SDK handles that separately via the `bucket` field. For example, use `https://.r2.cloudflarestorage.com`, not `https://.r2.cloudflarestorage.com/my-bucket`. When enabled, a preflight check runs before any benchmarks to verify S3 connectivity. Each instance's results directory is uploaded after the run completes (including on failure, for partial results). diff --git a/pkg/config/config.go b/pkg/config/config.go index fb15a80..7420953 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -774,6 +774,27 @@ type BenchmarkConfig struct { // ResultsUploadConfig contains configuration for uploading results. type ResultsUploadConfig struct { S3 *S3UploadConfig `yaml:"s3,omitempty" mapstructure:"s3"` + // MaxPreRunUploadSize caps the pre-run bundles uploaded with a suite, e.g. + // "512MB". Over it they are recorded in summary.json but not stored. + MaxPreRunUploadSize string `yaml:"max_pre_run_upload_size,omitempty" mapstructure:"max_pre_run_upload_size"` +} + +// DefaultMaxPreRunUploadSize admits ordinary pre-run bundles while excluding +// the multi-GB ones a bloatnet-style setup produces. +const DefaultMaxPreRunUploadSize = 512 * 1024 * 1024 + +// GetMaxPreRunUploadSize returns the cap in bytes; zero means no limit. +func (r *ResultsUploadConfig) GetMaxPreRunUploadSize() int64 { + if r == nil || r.MaxPreRunUploadSize == "" { + return DefaultMaxPreRunUploadSize + } + + size, err := ParseByteSize(r.MaxPreRunUploadSize) + if err != nil { + return DefaultMaxPreRunUploadSize + } + + return int64(size) } // S3UploadConfig contains S3-compatible storage upload settings. diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index d19983e..3970104 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -143,6 +143,9 @@ type Config struct { ResultsOwner *fsutil.OwnerConfig // Optional file ownership for results directory SystemResourceCollectionEnabled bool // Enable system resource collection (cgroups/Docker Stats) GitHubToken string // Optional GitHub token for API-based artifact downloads + // MaxPreRunUploadSize caps the pre-run payloads kept in the suite, and so + // the ones uploaded with it. Zero or less keeps every one. + MaxPreRunUploadSize int64 } // NewExecutor creates a new executor instance. @@ -260,7 +263,10 @@ func (e *executor) createSuiteOutput() error { } // Create suite output directory. - if err := CreateSuiteOutput(e.log, e.cfg.ResultsDir, hash, suiteInfo, e.prepared, e.cfg.ResultsOwner); err != nil { + if err := CreateSuiteOutput( + e.log, e.cfg.ResultsDir, hash, suiteInfo, e.prepared, e.cfg.ResultsOwner, + e.cfg.MaxPreRunUploadSize, + ); err != nil { return fmt.Errorf("creating suite output: %w", err) } diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 9b6a10a..24e5efb 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -72,6 +72,11 @@ type SourceStepsGlobs struct { // SuiteFile represents a file in the suite output. type SuiteFile struct { OgPath string `json:"og_path"` // original relative path + // SizeBytes is the source file's size, recorded even when it was omitted. + SizeBytes int64 `json:"size_bytes,omitempty"` + // Omitted marks a payload too large to keep, so never uploaded. The UI + // must not offer it; the bytes stay in the fixtures artifact. + Omitted bool `json:"omitted,omitempty"` } // SuiteTestEEST contains EEST-specific metadata for a test. @@ -159,12 +164,15 @@ func getStepContent(step *StepFile) ([]byte, error) { } // CreateSuiteOutput creates the suite directory structure with copied files and summary. +// maxPreRunUploadSize caps the pre-run payloads kept, and so uploaded; see +// config.ResultsUploadConfig.MaxPreRunUploadSize. Zero or less keeps every one. func CreateSuiteOutput( log logrus.FieldLogger, resultsDir, hash string, info *SuiteInfo, prepared *PreparedSource, owner *fsutil.OwnerConfig, + maxPreRunUploadSize int64, ) error { suiteDir := filepath.Join(resultsDir, "suites", hash) @@ -203,7 +211,7 @@ func CreateSuiteOutput( // Copy pre-run steps. // Structure: //pre_run.request (same pattern as tests). for _, f := range prepared.PreRunSteps { - suiteFile, err := copyPreRunStepFile(suiteDir, f, owner) + suiteFile, err := copyPreRunStepFile(log, suiteDir, f, owner, maxPreRunUploadSize) if err != nil { return fmt.Errorf("copying pre-run step: %w", err) } @@ -419,7 +427,32 @@ func copyTestStepFile(testDir, stepType string, file *StepFile, owner *fsutil.Ow // copyPreRunStepFile copies a pre-run step file to the suite directory. // Files are stored as //pre_run.request (same pattern as tests). -func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConfig) (*SuiteFile, error) { +// Over maxSize a file is described in the summary but not copied, so never +// uploaded either; maxSize <= 0 disables the limit. +func copyPreRunStepFile( + log logrus.FieldLogger, + suiteDir string, + file *StepFile, + owner *fsutil.OwnerConfig, + maxSize int64, +) (*SuiteFile, error) { + size, err := stepSize(file) + if err != nil { + return nil, err + } + + // Checked before anything is created: no wasted write, and no empty + // directory implying a file that was never stored. + if maxSize > 0 && size > maxSize { + log.WithFields(logrus.Fields{ + "step": file.Name, + "bytes": size, + "max": maxSize, + }).Info("Pre-run bundle over the size limit; describing it in the suite without storing it") + + return &SuiteFile{OgPath: file.Name, SizeBytes: size, Omitted: true}, nil + } + // Create step directory using the step name (relative path). stepDir := filepath.Join(suiteDir, file.Name) if err := fsutil.MkdirAll(stepDir, 0755, owner); err != nil { @@ -434,7 +467,7 @@ func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConf return nil, fmt.Errorf("writing content: %w", err) } - return &SuiteFile{OgPath: file.Name}, nil + return &SuiteFile{OgPath: file.Name, SizeBytes: size}, nil } // Handle file-based steps. @@ -456,7 +489,21 @@ func copyPreRunStepFile(suiteDir string, file *StepFile, owner *fsutil.OwnerConf return nil, fmt.Errorf("copying content: %w", err) } - return &SuiteFile{OgPath: file.Name}, nil + return &SuiteFile{OgPath: file.Name, SizeBytes: size}, nil +} + +// stepSize reports a step's payload size without reading a file-backed one. +func stepSize(file *StepFile) (int64, error) { + if file.Provider != nil { + return int64(len(file.Provider.Content())), nil + } + + stat, err := os.Stat(file.Path) + if err != nil { + return 0, fmt.Errorf("stating source: %w", err) + } + + return stat.Size(), nil } // GetGitCommitSHA retrieves the current commit SHA from a git repository. diff --git a/pkg/executor/suite_test.go b/pkg/executor/suite_test.go index 7e2e45f..e8c3290 100644 --- a/pkg/executor/suite_test.go +++ b/pkg/executor/suite_test.go @@ -44,7 +44,7 @@ func TestCreateSuiteOutput_WritesPayloadSizes(t *testing.T) { Hash: "deadbeef", } log := logrus.New() - err := CreateSuiteOutput(log, tmp, "deadbeef", info, prepared, nil) + err := CreateSuiteOutput(log, tmp, "deadbeef", info, prepared, nil, 0) require.NoError(t, err) summaryPath := filepath.Join(tmp, "suites", "deadbeef", "summary.json") @@ -84,7 +84,7 @@ func TestCreateSuiteOutput_AggregatesMetadataOpcodeCounts(t *testing.T) { }, } info := &SuiteInfo{Hash: "cafe"} - err := CreateSuiteOutput(logrus.New(), tmp, "cafe", info, prepared, nil) + err := CreateSuiteOutput(logrus.New(), tmp, "cafe", info, prepared, nil, 0) require.NoError(t, err) data, err := os.ReadFile(filepath.Join(tmp, "suites", "cafe", "summary.json")) @@ -167,7 +167,7 @@ func TestCreateSuiteOutput_CopiesEESTMeta(t *testing.T) { } info := &SuiteInfo{Hash: "abc123"} - require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "abc123", info, prepared, nil)) + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "abc123", info, prepared, nil, 0)) suiteMeta := filepath.Join(tmp, "suites", "abc123", ".eest-meta") @@ -203,7 +203,7 @@ func TestCreateSuiteOutput_NoEESTMetaWhenAbsent(t *testing.T) { } info := &SuiteInfo{Hash: "nometa01"} - require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "nometa01", info, prepared, nil)) + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "nometa01", info, prepared, nil, 0)) _, err := os.Stat(filepath.Join(tmp, "suites", "nometa01", ".eest-meta")) assert.True(t, os.IsNotExist(err)) @@ -234,7 +234,7 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { // First run — creates the suite and writes initial sizes. log := logrus.New() info1 := &SuiteInfo{Hash: "cafef00d"} - require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info1, prepared, nil)) + require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info1, prepared, nil, 0)) // Simulate a legacy summary: rewrite the file with payload_sizes cleared. summaryPath := filepath.Join(tmp, "suites", "cafef00d", "summary.json") @@ -251,7 +251,7 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { // Second run — should detect suite exists, read on-disk test.request, and merge. info2 := &SuiteInfo{Hash: "cafef00d"} - require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info2, prepared, nil)) + require.NoError(t, CreateSuiteOutput(log, tmp, "cafef00d", info2, prepared, nil, 0)) final, err := os.ReadFile(summaryPath) require.NoError(t, err) @@ -263,3 +263,109 @@ func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { require.Len(t, parsed.Tests[0].PayloadSizes.Test.SSZFull, 1) assert.Greater(t, parsed.Tests[0].PayloadSizes.Test.SSZFull[0], uint64(100), "merge path should backfill sizes") } + +// A pre-run bundle is a runner replay script, not something the UI needs — +// the jochemnet bloatnet one is 9.4 GiB. Over the limit it is described only. +func TestCreateSuiteOutput_OmitsOversizedPreRunSteps(t *testing.T) { + log := logrus.New() + tmp := t.TempDir() + + bundle := filepath.Join(t.TempDir(), "pre-run.request") + require.NoError(t, os.WriteFile(bundle, []byte("0123456789"), 0o600)) + + prepared := &PreparedSource{ + PreRunSteps: []*StepFile{{Name: "pre_run/pre-run.request", Path: bundle}}, + Tests: []*TestWithSteps{ + { + Name: "test_with_prerun", + Test: &StepFile{ + Name: "test_with_prerun", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + + info := &SuiteInfo{Hash: "b16b16"} + require.NoError(t, CreateSuiteOutput(log, tmp, "b16b16", info, prepared, nil, 5)) + + suiteDir := filepath.Join(tmp, "suites", "b16b16") + assert.NoFileExists(t, filepath.Join(suiteDir, "pre_run", "pre-run.request", "pre_run.request")) + + var parsed SuiteInfo + data, err := os.ReadFile(filepath.Join(suiteDir, "summary.json")) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &parsed)) + + // The suite still describes the step, so nothing about the run is lost. + require.Len(t, parsed.PreRunSteps, 1) + assert.Equal(t, "pre_run/pre-run.request", parsed.PreRunSteps[0].OgPath) + assert.True(t, parsed.PreRunSteps[0].Omitted) + assert.Equal(t, int64(10), parsed.PreRunSteps[0].SizeBytes) + + // Tests are untouched by the pre-run limit. + require.Len(t, parsed.Tests, 1) + assert.FileExists(t, filepath.Join(suiteDir, "test_with_prerun", "test.request")) +} + +// Under the limit, and with the limit disabled, the bundle is stored as before. +func TestCreateSuiteOutput_KeepsPreRunStepsWithinLimit(t *testing.T) { + log := logrus.New() + + for name, limit := range map[string]int64{"under limit": 1024, "no limit": 0} { + t.Run(name, func(t *testing.T) { + tmp := t.TempDir() + + bundle := filepath.Join(t.TempDir(), "pre-run.request") + require.NoError(t, os.WriteFile(bundle, []byte("0123456789"), 0o600)) + + prepared := &PreparedSource{ + PreRunSteps: []*StepFile{{Name: "pre_run/pre-run.request", Path: bundle}}, + Tests: []*TestWithSteps{ + { + Name: "test_with_prerun", + Test: &StepFile{ + Name: "test_with_prerun", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + + info := &SuiteInfo{Hash: "5ma11"} + require.NoError(t, CreateSuiteOutput(log, tmp, "5ma11", info, prepared, nil, limit)) + + stored := filepath.Join(tmp, "suites", "5ma11", "pre_run", "pre-run.request", "pre_run.request") + assert.FileExists(t, stored) + + var parsed SuiteInfo + data, err := os.ReadFile(filepath.Join(tmp, "suites", "5ma11", "summary.json")) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, &parsed)) + require.Len(t, parsed.PreRunSteps, 1) + assert.False(t, parsed.PreRunSteps[0].Omitted) + }) + } +} + +// An omitted bundle must leave no empty directory implying a stored file. +func TestCreateSuiteOutput_OmittedPreRunLeavesNoStepDir(t *testing.T) { + tmp := t.TempDir() + + bundle := filepath.Join(t.TempDir(), "pre-run.request") + require.NoError(t, os.WriteFile(bundle, []byte("0123456789"), 0o600)) + + prepared := &PreparedSource{ + PreRunSteps: []*StepFile{{Name: "pre_run/pre-run.request", Path: bundle}}, + Tests: []*TestWithSteps{ + { + Name: "test_x", + Test: &StepFile{Name: "test_x", Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}}, + }, + }, + } + + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "n0d1r", &SuiteInfo{Hash: "n0d1r"}, prepared, nil, 5)) + + assert.NoDirExists(t, filepath.Join(tmp, "suites", "n0d1r", "pre_run")) +}