From 5c4d44b02ae560d8361b397306beeb3b093137f9 Mon Sep 17 00:00:00 2001 From: CYJiang <86391540+googs1025@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:26:22 +0800 Subject: [PATCH] feat(cli): add baseline run flag --- docs/guide/cli-reference.md | 4 ++ docs/guide/writing-evals.md | 2 + docs/zh/guide/cli-reference.md | 4 ++ docs/zh/guide/writing-evals.md | 2 + internal/cli/run.go | 20 ++++++ internal/cli/run_test.go | 83 ++++++++++++++++++++++ skills/skill-upper/references/cli.md | 2 + skills/skill-upper/references/eval-yaml.md | 1 + 8 files changed, 118 insertions(+) diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 640d1e8c..a4851069 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -31,6 +31,7 @@ skill-up run [path] [flags] | `--engine` | From config | Override engine name | | `--model` | From config | Override model (format: `provider/name`) | | `--parallelism` | From config | Override `cases.parallelism`. Allowed range: 1–256 | +| `--baseline` | From config | Override `benchmark.enabled` to `true` for this run | | `--api-key` | — | Pass an API key (higher precedence than env vars) | | `-v, --verbose` | `0` | Increase log verbosity. Default `info`; `-v` / `--verbose` / `--verbose=true` → `debug`; `-vv` / `--verbose=2` → `trace`; `--verbose=false` disables extra detail | @@ -59,6 +60,9 @@ skill-up run ./evals/eval.yaml --engine codex --model openai/gpt-4 # Temporarily override case parallelism skill-up run ./evals/eval.yaml --parallelism 4 +# Run with baseline comparison +skill-up run ./evals/eval.yaml --baseline + # Multiple report formats skill-up run ./evals/eval.yaml --format json --format html --format junit diff --git a/docs/guide/writing-evals.md b/docs/guide/writing-evals.md index 73383fda..c449b425 100644 --- a/docs/guide/writing-evals.md +++ b/docs/guide/writing-evals.md @@ -760,6 +760,8 @@ Setting `benchmark.enabled: true` runs every case **twice**: 2. **without_skill** — Skill removed (baseline) The diff highlights the value the Skill adds (pass-rate uplift, time/token deltas). +For one-off comparisons, `skill-up run ./evals/eval.yaml --baseline` enables the +same mode without changing `eval.yaml`. ```yaml benchmark: diff --git a/docs/zh/guide/cli-reference.md b/docs/zh/guide/cli-reference.md index b57b454d..c73aee16 100644 --- a/docs/zh/guide/cli-reference.md +++ b/docs/zh/guide/cli-reference.md @@ -31,6 +31,7 @@ skill-up run [path] [flags] | `--engine` | 配置文件中的值 | 覆盖 Engine 名称 | | `--model` | 配置文件中的值 | 覆盖模型（格式：`provider/name`） | | `--parallelism` | 配置文件中的值 | 覆盖 `cases.parallelism`，用于临时调整用例并行数，取值范围为 1 到 256 | +| `--baseline` | 配置文件中的值 | 为本次运行覆盖 `benchmark.enabled` 为 `true` | | `--api-key` | — | 传入 API Key（优先级高于环境变量） | | `-v, --verbose` | `0` | 增加日志详细程度。默认输出 `info`；`-v`/`--verbose`/`--verbose=true` 输出 `debug`；`-vv`/`--verbose=2` 输出 `trace`；`--verbose=false` 关闭附加详细日志 | @@ -52,6 +53,9 @@ skill-up run ./evals/eval.yaml --engine codex --model openai/gpt-4 # 临时覆盖用例并行数 skill-up run ./evals/eval.yaml --parallelism 4 +# 启用基线对比 +skill-up run ./evals/eval.yaml --baseline + # 生成多种格式报告 skill-up run ./evals/eval.yaml --format json --format html --format junit diff --git a/docs/zh/guide/writing-evals.md b/docs/zh/guide/writing-evals.md index cfd5c01e..4414fc94 100644 --- a/docs/zh/guide/writing-evals.md +++ b/docs/zh/guide/writing-evals.md @@ -732,6 +732,8 @@ judge: 2. **without_skill** — 不安装 Skill 执行（基线组） 对比结果会体现 Skill 带来的增量价值（通过率提升、时间和 token 消耗差异）。 +临时对比时，也可以运行 `skill-up run ./evals/eval.yaml --baseline`， +无需修改 `eval.yaml`。 ```yaml benchmark: diff --git a/internal/cli/run.go b/internal/cli/run.go index 486e5b5a..efc4c117 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -110,6 +110,7 @@ func init() { runCmd.Flags().String("model", "", "Override model (accepts either a bare model name or provider/name)") runCmd.Flags().String("api-key", "", "API key for the model provider") runCmd.Flags().Int("parallelism", 0, "Override cases.parallelism. Must be between 1 and 256 when specified") + runCmd.Flags().Bool("baseline", false, "Override benchmark.enabled to true for this run") runCmd.Flags().SetNormalizeFunc(normalizeRunFlagName) runCmd.Flags().StringArray(runtimeKwargFlagName, nil, "Environment kwarg in key=value format (can be used multiple times; --rk is accepted as an alias)") runCmd.Flags().StringArray(engineKwargFlagName, nil, "Engine kwarg in key=value format (can be used multiple times; --ek is accepted as an alias). Recognised keys are per-agent (e.g. codex honours bypass_sandbox=true)") @@ -524,6 +525,9 @@ func applyRunConfigOverrides(evalCfg *config.EvalConfig, cmd *cobra.Command) err if err := applyRuntimeTypeOverride(evalCfg, cmd); err != nil { return err } + if err := applyBaselineOverride(evalCfg, cmd); err != nil { + return err + } applyUserConfigKwargs(cmd.Context(), evalCfg) parallelismFlag := cmd.Flags().Lookup("parallelism") @@ -546,6 +550,22 @@ func applyRunConfigOverrides(evalCfg *config.EvalConfig, cmd *cobra.Command) err return nil } +func applyBaselineOverride(evalCfg *config.EvalConfig, cmd *cobra.Command) error { + flag := cmd.Flags().Lookup("baseline") + if flag == nil || !flag.Changed { + return nil + } + enabled, err := cmd.Flags().GetBool("baseline") + if err != nil { + return fmt.Errorf("read --baseline: %w", err) + } + if !enabled { + return nil + } + evalCfg.Benchmark.Enabled = true + return nil +} + // applyUserConfigKwargs fills in missing environment kwargs from user-config // defaults. CLI --runtime-kwarg and eval.yaml entries are preserved. func applyUserConfigKwargs(ctx context.Context, evalCfg *config.EvalConfig) { diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index cd5bf081..e8438acd 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -57,6 +57,7 @@ func newRunPhaseTestCommand(t *testing.T) *cobra.Command { cmd.Flags().String("model", "", "") cmd.Flags().String("api-key", "", "") cmd.Flags().Int("parallelism", 0, "") + cmd.Flags().Bool("baseline", false, "") cmd.Flags().SetNormalizeFunc(normalizeRunFlagName) cmd.Flags().StringArray(runtimeKwargFlagName, nil, "") var verbose verbosityValue @@ -1281,6 +1282,88 @@ func TestApplyRunConfigOverrides_ParallelismUnsetPreservesConfig(t *testing.T) { } } +func TestApplyRunConfigOverrides_BaselineEnablesBenchmark(t *testing.T) { + t.Parallel() + + cmd := &cobra.Command{} + cmd.Flags().Bool("baseline", false, "") + if err := cmd.Flags().Set("baseline", testFlagBoolTrue); err != nil { + t.Fatalf("set baseline: %v", err) + } + + cfg := config.DefaultEvalConfig() + cfg.Benchmark.Enabled = false + if err := applyRunConfigOverrides(cfg, cmd); err != nil { + t.Fatalf("applyRunConfigOverrides: %v", err) + } + if !cfg.Benchmark.Enabled { + t.Fatal("Benchmark.Enabled = false, want true") + } +} + +func TestApplyRunConfigOverrides_BaselineUnsetPreservesConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + initial bool + }{ + {name: "disabled", initial: false}, + {name: "enabled", initial: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cmd := &cobra.Command{} + cmd.Flags().Bool("baseline", false, "") + + cfg := config.DefaultEvalConfig() + cfg.Benchmark.Enabled = tt.initial + if err := applyRunConfigOverrides(cfg, cmd); err != nil { + t.Fatalf("applyRunConfigOverrides: %v", err) + } + if got := cfg.Benchmark.Enabled; got != tt.initial { + t.Fatalf("Benchmark.Enabled = %t, want %t", got, tt.initial) + } + }) + } +} + +func TestApplyRunConfigOverrides_BaselineFalsePreservesConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + initial bool + }{ + {name: "disabled", initial: false}, + {name: "enabled", initial: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cmd := &cobra.Command{} + cmd.Flags().Bool("baseline", false, "") + if err := cmd.Flags().Set("baseline", "false"); err != nil { + t.Fatalf("set baseline: %v", err) + } + + cfg := config.DefaultEvalConfig() + cfg.Benchmark.Enabled = tt.initial + if err := applyRunConfigOverrides(cfg, cmd); err != nil { + t.Fatalf("applyRunConfigOverrides: %v", err) + } + if got := cfg.Benchmark.Enabled; got != tt.initial { + t.Fatalf("Benchmark.Enabled = %t, want %t", got, tt.initial) + } + }) + } +} + func TestApplyRunConfigOverrides_RuntimeKwargs(t *testing.T) { t.Parallel() diff --git a/skills/skill-upper/references/cli.md b/skills/skill-upper/references/cli.md index 35c0ee60..cd63f104 100644 --- a/skills/skill-upper/references/cli.md +++ b/skills/skill-upper/references/cli.md @@ -32,6 +32,7 @@ skill-up run [path] [flags] | `--runtime` | 配置中的值 | 覆盖 `environment.type`（`none`、`opensandbox`、`docker`） | | `--model` | 配置中的值 | 覆盖模型（格式：`provider/name`） | | `--parallelism` | 配置中的值 | 覆盖 `cases.parallelism`，临时调整用例并行数，取值 1–256 | +| `--baseline` | 配置中的值 | 为本次运行覆盖 `benchmark.enabled` 为 `true` | | `--api-key` | — | 传入 API Key（优先级高于环境变量） | | `-v, --verbose` | `0` | 日志详细程度：`info` 默认；`-v` 为 `debug`；`-vv` / `--verbose=2` 为 `trace` | @@ -50,6 +51,7 @@ skill-up run ./evals/eval.yaml --include-case-name "basic-*" skill-up run ./evals/eval.yaml --exclude-case-name "*-old" skill-up run ./evals/eval.yaml --engine codex --model openai/gpt-4 skill-up run ./evals/eval.yaml --parallelism 4 +skill-up run ./evals/eval.yaml --baseline skill-up run ./evals/eval.yaml --format html --format junit skill-up run ./evals/eval.yaml --iteration 3 skill-up run ./evals/eval.yaml diff --git a/skills/skill-upper/references/eval-yaml.md b/skills/skill-upper/references/eval-yaml.md index f648c349..b6ec2a15 100644 --- a/skills/skill-upper/references/eval-yaml.md +++ b/skills/skill-upper/references/eval-yaml.md @@ -60,6 +60,7 @@ report: ``` `cases.parallelism` 可被 `skill-up run --parallelism N`（1–256）临时覆盖。 +临时启用基线对比时，可以使用 `skill-up run --baseline`，等价于为本次运行设置 `benchmark.enabled: true`。 `collect_artifacts`（`cases.defaults` 级，或单个 `case.yaml` 内追加）用 [doublestar](https://github.com/bmatcuk/doublestar) glob（`*` 单层、`**` 跨目录）声明要采集的 workspace 文件。无论 Agent 成功/失败/超时，命中文件都会保留相对路径下载到 `///outputs/workspace/`。两层按并集去重合并。它与 `report.artifacts`（产物*类型*）、`agent_judge` 的 git diff（字符串）正交。