Skip to content

feat(paired): same-runner base-vs-PR comparison #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pedromvgomes merged 1 commit into from
Jul 19, 2026
Merged

feat(paired): same-runner base-vs-PR comparison #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 58 additions & 12 deletions action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ inputs:
"auto" (default) compares on pull_request and records on a push to the
default branch. Force with "compare" or "record".
default: auto
paired:
description: >
"true" runs a paired same-runner comparison on pull_request: bench-command
runs on BOTH the PR head and the base commit on the same runner, and the
two are compared directly — cancelling machine-to-machine variance that a
stored baseline can't. Requires `bench-command` and the caller's checkout
to use `fetch-depth: 0` (so the base commit is available). Ignored for
record runs.
default: "false"
github-token:
description: token used to post the PR comment and push the state branch
default: ${{ github.token }}
Expand Down Expand Up @@ -83,8 +92,11 @@ runs:
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
EVENT: ${{ github.event_name }}
PR_BASE: ${{ github.event.pull_request.base.ref }}
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
PAIRED: ${{ inputs.paired }}
BENCH_COMMAND: ${{ inputs.bench-command }}
run: |
set -euo pipefail
report="${RUNNER_TEMP}/cubit-report.md"
Expand All @@ -111,19 +123,53 @@ runs:
dash_arg=()
[[ -n "$DASH" ]] && dash_arg=(--dashboard-url "$DASH")

fallback_report() {
printf '## <img src="https://raw.githubusercontent.com/wardnet/cubit/main/assets/cubit-logo.png" alt="" width="16" align="top"> cubit — performance\n\n⚠️ cubit could not produce a report — no benchmark output was found at `%s` (did the benchmark step run?).\n' "$DIR" > "$report"
}

if [[ "$mode" == "compare" ]]; then
# latest:<base> yields an empty baseline (all "new") the first time,
# which is fine — cubit does not fail on a missing baseline.
#
# This is a light gate: a compare error (e.g. no benchmark output was
# produced) must NOT fail the consuming workflow. `if ! ...` is exempt
# from `set -e`, so we fall back to a short note and keep the job green.
if ! cubit compare --repo . --criterion-dir "$DIR" \
--baseline-ref "latest:${PR_BASE}" \
--commit "$PR_HEAD_SHA" --branch "$PR_HEAD_REF" \
--threshold "$THRESHOLD" "${dash_arg[@]}" \
--out "$report"; then
printf '## <img src="https://raw.githubusercontent.com/wardnet/cubit/main/assets/cubit-logo.png" alt="" width="16" align="top"> cubit — performance\n\n⚠️ cubit could not produce a report — no benchmark output was found at `%s` (did the benchmark step run?).\n' "$DIR" > "$report"
# This is a light gate: a compare error must NOT fail the consuming
# workflow. `if ! ...` is exempt from `set -e`, so we fall back to a
# short note and keep the job green.
if [[ "$PAIRED" == "true" && -n "${PR_BASE_SHA:-}" && -n "${BENCH_COMMAND:-}" ]]; then
# Paired same-runner comparison. The "Run benchmarks" step already
# produced the PR head's criterion in $DIR; snapshot it, then bench the
# BASE commit on this same runner and compare the two directly —
# cancelling machine-to-machine variance a stored baseline can't.
pr_crit="${RUNNER_TEMP}/cubit-crit-pr"
base_crit="${RUNNER_TEMP}/cubit-crit-base"
rm -rf "$pr_crit" "$base_crit"
cp -r "$DIR" "$pr_crit" 2>/dev/null || mkdir -p "$pr_crit"

orig=$(git rev-parse HEAD)
git fetch --no-tags origin "$PR_BASE_SHA" 2>/dev/null || true
if git checkout -q --detach "$PR_BASE_SHA" 2>/dev/null; then
rm -rf "$DIR"
eval "$BENCH_COMMAND" || true
cp -r "$DIR" "$base_crit" 2>/dev/null || mkdir -p "$base_crit"
git checkout -q --detach "$orig"
else
echo "cubit: could not check out base ${PR_BASE_SHA} — set the caller's checkout to fetch-depth: 0. Comparing against an empty baseline." >&2
mkdir -p "$base_crit"
fi

if ! cubit compare --repo . --criterion-dir "$pr_crit" \
--baseline-criterion-dir "$base_crit" \
--commit "$PR_HEAD_SHA" --branch "$PR_HEAD_REF" \
--threshold "$THRESHOLD" "${dash_arg[@]}" \
--out "$report"; then
fallback_report
fi
else
# Single run vs the stored cubit-state baseline. latest:<base> yields
# an empty baseline (all "new") the first time, which is fine.
if ! cubit compare --repo . --criterion-dir "$DIR" \
--baseline-ref "latest:${PR_BASE}" \
--commit "$PR_HEAD_SHA" --branch "$PR_HEAD_REF" \
--threshold "$THRESHOLD" "${dash_arg[@]}" \
--out "$report"; then
fallback_report
fi
fi
# Render the standalone trend dashboard for the run artifact
# (best-effort — a missing state branch just yields an empty page).
Expand Down
52 changes: 34 additions & 18 deletions cmd/cubit/compare.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,27 @@ import (

func newCompareCmd() *cobra.Command {
var (
criterionDir string
baselinePath string
baselineRef string
repo string
stateBranch string
commit string
branch string
threshold float64
dashboardURL string
out string
criterionDir string
baselinePath string
baselineCritDir string
baselineRef string
repo string
stateBranch string
commit string
branch string
threshold float64
dashboardURL string
out string
)
cmd := &cobra.Command{
Use: "compare",
Short: "Compare a benchmark run against a baseline and render a report",
Long: "Ingests a criterion output directory, diffs it against a baseline, and " +
"writes a Markdown report suitable for a PR comment. The baseline comes " +
"from --baseline (a file) or --baseline-ref (a record on the cubit-state " +
"branch). Advisory only — it never fails the build.",
"from --baseline (a file), --baseline-criterion-dir (a second criterion " +
"output, e.g. a paired same-runner run of the base commit), or " +
"--baseline-ref (a record on the cubit-state branch). Advisory only — it " +
"never fails the build.",
Args: cobra.NoArgs,
SilenceUsage: true,
SilenceErrors: true,
Expand All @@ -39,7 +42,7 @@ func newCompareCmd() *cobra.Command {
if err != nil {
return err
}
baseline, err := resolveBaseline(baselinePath, baselineRef, repo, stateBranch)
baseline, err := resolveBaseline(baselinePath, baselineCritDir, baselineRef, repo, stateBranch)
if err != nil {
return err
}
Expand All @@ -50,7 +53,8 @@ func newCompareCmd() *cobra.Command {
}
f := cmd.Flags()
f.StringVar(&criterionDir, "criterion-dir", "target/criterion", "criterion output directory to ingest")
f.StringVar(&baselinePath, "baseline", "", "baseline run JSON file (takes precedence over --baseline-ref)")
f.StringVar(&baselinePath, "baseline", "", "baseline run JSON file (highest precedence)")
f.StringVar(&baselineCritDir, "baseline-criterion-dir", "", "criterion output dir to use as the baseline (a paired same-runner run of the base commit); takes precedence over --baseline-ref")
f.StringVar(&baselineRef, "baseline-ref", "", "baseline from the cubit-state branch: a commit SHA, \"latest\", or \"latest:<branch>\"")
f.StringVar(&repo, "repo", ".", "git repository holding the cubit-state branch")
f.StringVar(&stateBranch, "state-branch", gitstore.DefaultBranch, "branch storing recorded runs")
Expand All @@ -62,13 +66,25 @@ func newCompareCmd() *cobra.Command {
return cmd
}

// resolveBaseline loads the baseline run from an explicit file, or from a
// reference on the cubit-state branch, or returns an empty run (everything
// reported as new) when neither is given or the reference is not found yet.
func resolveBaseline(path, ref, repo, stateBranch string) (model.Run, error) {
// resolveBaseline loads the baseline run from (in precedence order) an explicit
// file, a second criterion output directory (the paired same-runner base run),
// or a reference on the cubit-state branch. It returns an empty run (everything
// reported as new) when none is given, or when the paired baseline dir produced
// no benchmarks — a base bench that failed to run must degrade to "all new"
// rather than erroring, so the comment still posts.
func resolveBaseline(path, critDir, ref, repo, stateBranch string) (model.Run, error) {
if path != "" {
return loadRun(path)
}
if critDir != "" {
run, err := buildRun(critDir, "", "", "")
if err != nil {
// No benchmarks in the base run (e.g. the base bench errored) — treat
// as no baseline rather than failing the comparison.
return model.Run{}, nil
}
return run, nil
}
if ref == "" {
return model.Run{}, nil
}
Expand Down
Loading