diff --git a/.github/scripts/metrics b/.github/scripts/metrics index e3c1fa48..051a6f82 100755 --- a/.github/scripts/metrics +++ b/.github/scripts/metrics @@ -28,6 +28,31 @@ SIZE_COLUMNS = [ "file_kb", ] +# Churn is only worth measuring where our defects come from: tests echo the change +# they cover, third_party/ is vendored, examples/ builds other people's source, +# .github/ is CI plumbing, and prose is not code. +CHURN_EXCLUDE_PREFIXES = (".github/", "test/", "third_party/", "examples/", "docs/") + +CHURN_EXCLUDE_SUFFIXES = (".md",) + +# Generated and committed: `make bootstrap` regenerates these from the Tupfiles, so +# their churn is a mechanical echo, at ~10x the volume, of a change already counted +# at its source. Ranked 2nd and 3rd before they were excluded. +CHURN_EXCLUDE_FILES = ("bootstrap-linux.sh", "bootstrap-macos.sh") + +CHURN_HOT_FILES = 10 + +CHURN_COLUMNS = [ + "window_days", + "files", + "added", + "removed", + "surviving", + "churned", + "churn_pct", + "hot_files", +] + COVERAGE_COLUMNS = [ "overall", "median", @@ -301,6 +326,132 @@ def cmd_coverage(args): return 0 +def git(*args): + run = subprocess.run( + ["git", *args], capture_output=True, text=True, check=False + ) + return run.stdout if run.returncode == 0 else "" + + +BLAME_HEADER = re.compile(r"^\^?([0-9a-f]{40}) \d+ (\d+)") + + +def blame_line_times(rev, path): + """Commit time of the last change to each line of path, as of rev. + + Porcelain emits a commit's metadata only the first time that commit appears, + so the time is resolved at the content line, by which point it is either just + parsed or already cached. + """ + times = {} + sha_time = {} + sha = None + lineno = None + for line in git("blame", "--porcelain", rev, "--", path).splitlines(): + if line.startswith("\t"): + if sha is not None: + times[lineno] = sha_time.get(sha) + sha = lineno = None + continue + header = BLAME_HEADER.match(line) + if header: + sha = header.group(1) + lineno = int(header.group(2)) + elif line.startswith("committer-time ") and sha is not None: + sha_time[sha] = int(line.split(" ", 1)[1]) + return times + + +def removed_line_numbers(base, head, path): + """Line numbers in base's copy of path that this diff deletes.""" + removed = [] + for line in git("diff", "-U0", "-M", f"{base}..{head}", "--", path).splitlines(): + if not line.startswith("@@"): + continue + old = line.split(" ")[1] # "-a,b" + start, _, count = old[1:].partition(",") + count = int(count) if count else 1 + removed.extend(range(int(start), int(start) + count)) + return removed + + +def cmd_churn(args): + """Churn across the whole codebase over a window, not the diff of one change. + + Of the lines written in the window, how many are already gone? Work that was + written and then discarded inside the same window is the codebase telling you + where it is being rewritten rather than built. + """ + # Every git call here degrades to empty output, so a missing history would be + # reported as zero churn rather than as a failure. Refuse instead. + if git("rev-parse", "--is-shallow-repository").strip() == "true": + print("churn needs full history (fetch-depth: 0)", file=sys.stderr) + return 1 + + rev = git("rev-parse", args.rev).strip() + if not rev: + print(f"unknown revision {args.rev}", file=sys.stderr) + return 1 + # Dated against the analyzed commit, not wall-clock now, so a re-run agrees. + cutoff = int(git("show", "-s", "--format=%ct", rev).strip() or 0) - args.window_days * 86400 + + added_by_file = {} + added = removed = 0 + for line in git( + "log", "--numstat", "-M", "--format=", f"--since={args.window_days} days ago", rev + ).splitlines(): + cells = line.split("\t") + if len(cells) != 3 or cells[0] == "-": # binary + continue + path = cells[2] + if (path.startswith(CHURN_EXCLUDE_PREFIXES) + or path.endswith(CHURN_EXCLUDE_SUFFIXES) + or path in CHURN_EXCLUDE_FILES): + continue + added += int(cells[0]) + removed += int(cells[1]) + added_by_file[path] = added_by_file.get(path, 0) + int(cells[0]) + + if not added: + print(f"no production changes in the last {args.window_days} days", file=sys.stderr) + return 1 + + # A line written in the window and still present at rev survived; the rest of what + # was written in the window has since been deleted or rewritten. + surviving = 0 + churn_by_file = [] + for path, wrote in added_by_file.items(): + lived = sum( + 1 for when in blame_line_times(rev, path).values() + if when is not None and when >= cutoff + ) + surviving += min(lived, wrote) + gone = wrote - min(lived, wrote) + if gone: + churn_by_file.append((gone, path)) + + churned = added - surviving + churn_pct = churned / added * 100 + + churn_by_file.sort(key=lambda e: (-e[0], e[1])) + hot = "; ".join( + f"{p} ({n})" for n, p in churn_by_file[:CHURN_HOT_FILES] + ) or "none" + + print("\t".join(CHURN_COLUMNS)) + print("\t".join(str(v) for v in ( + args.window_days, + len(added_by_file), + added, + removed, + surviving, + churned, + f"{churn_pct:.1f}", + hot, + ))) + return 0 + + def read_tsv(path): if not path: return [] @@ -439,6 +590,45 @@ def coverage_section(cur, base): return lines +def churn_section(cur): + window = cur["window_days"] + lines = [ + f"### Code churn (whole codebase, last {window}d)", + "", + "| Files | Lines written | Still present | Churned | Churn rate |", + "|---|---|---|---|---|", + "| {} | {} | {} | {} | {} |".format( + cur["files"], + cur["added"], + cur["surviving"], + cur["churned"], + f"{num(cur['churn_pct']):g}%", + ), + "", + f"Of the lines written across the codebase in the last {window} days, how many " + "are already gone — work that was written and then discarded or rewritten " + "inside the same window. This is the state of the tree including this PR, not " + "a measure of the PR itself. Only code we write is counted: tests, examples, " + "vendored and generated files, CI plumbing and prose are excluded. " + f"{cur['removed']} lines were deleted in the window in total, most of them " + "older than it.", + ] + if cur["hot_files"] != "none": + lines += [ + "", + "
Where the churn is", + "", + "| File | Lines written then discarded |", + "|---|---|", + ] + for entry in cur["hot_files"].split("; "): + match = re.match(r"^(.*) \((\d+)\)$", entry) + if match: + lines.append(f"| `{match.group(1)}` | {match.group(2)} |") + lines += ["", "
"] + return lines + + def cmd_render(args): sections = [] baseline_used = False @@ -461,6 +651,10 @@ def cmd_render(args): baseline_used |= bool(base) sections.append(size_section(size, base)) + churn = read_tsv(args.churn) + if churn: + sections.append(churn_section(churn[0])) + cov = read_tsv(args.coverage) if cov: base_rows = read_tsv(args.baseline_coverage) @@ -506,6 +700,11 @@ def main() -> int: coverage.add_argument("summary_json") coverage.set_defaults(func=cmd_coverage) + churn = sub.add_parser("churn") + churn.add_argument("rev", nargs="?", default="HEAD") + churn.add_argument("--window-days", type=int, default=30) + churn.set_defaults(func=cmd_churn) + size = sub.add_parser("size") size.add_argument("binaries", nargs="+") size.set_defaults(func=cmd_size) @@ -515,6 +714,7 @@ def main() -> int: render.add_argument("--stat") render.add_argument("--size") render.add_argument("--coverage") + render.add_argument("--churn") render.add_argument("--baseline-perf") render.add_argument("--baseline-stat") render.add_argument("--baseline-size") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 850e3457..90da8f37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -285,6 +285,12 @@ jobs: actions: read steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 # churn blames removed lines; a shallow clone has no history + - name: Code churn + env: + HEAD: ${{ github.event.pull_request.head.sha }} + run: .github/scripts/metrics churn "$HEAD" > churn-metrics.tsv - name: Download perf metrics uses: actions/download-artifact@v4 with: @@ -330,6 +336,7 @@ jobs: --perf perf-metrics.tsv --stat stat-metrics.tsv \ --size size-metrics.tsv \ --coverage coverage-metrics.tsv \ + --churn churn-metrics.tsv \ --baseline-perf baseline/perf/perf-metrics.tsv \ --baseline-stat baseline/stat/stat-metrics.tsv \ --baseline-size baseline/size/size-metrics.tsv \