diff --git a/Cargo.lock b/Cargo.lock index 0f4ec29de2..65513c1fbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -398,6 +398,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -1003,6 +1012,7 @@ dependencies = [ "num-traits", "oorandom", "page_size", + "plotters", "regex", "serde", "serde_json", @@ -1788,6 +1798,7 @@ dependencies = [ "bolero", "bytes", "chrono", + "criterion", "dataplane-args", "dataplane-cli", "dataplane-clock", @@ -1799,10 +1810,12 @@ dependencies = [ "dataplane-lifecycle", "dataplane-lpm", "dataplane-net", + "dataplane-routing", "dataplane-tracectl", "derive_builder", "dplane-rpc", "futures-util", + "iai-callgrind", "inotify", "ipnet", "left-right 0.11.7", @@ -2880,6 +2893,42 @@ dependencies = [ "tower-service", ] +[[package]] +name = "iai-callgrind" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b1e4910d3a9137442723dfb772c32dc10674c4181ca078d2fd227cd5dce9db0" +dependencies = [ + "bincode", + "derive_more", + "iai-callgrind-macros", + "iai-callgrind-runner", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d03775318d3f9f01b39ac6612b01464006dc397a654a89dd57df2fd34fb68c3" +dependencies = [ + "derive_more", + "proc-macro-error2", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.119", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74c9743c00c3bca4aaffc69c87cae56837796cd362438daf354a3f785788c68" +dependencies = [ + "serde", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -4337,6 +4386,34 @@ dependencies = [ "time", ] +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" version = "1.15.0" diff --git a/Cargo.toml b/Cargo.toml index 38d0071a8a..f0783857a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,7 +133,12 @@ chrono = { version = "0.4.45", default-features = false, features = [] } clap = { version = "4.6.6", default-features = true, features = [] } color-eyre = { version = "0.6.5", default-features = false, features = [] } colored = { version = "3.1.1", default-features = false, features = [] } -criterion = { version = "0.8.2", default-features = false, features = [] } +criterion = { version = "0.8.2", default-features = false, features = [ + "cargo_bench_support", + "html_reports", + "plotters", +] } +iai-callgrind = { version = "0.16.1" } crossbeam-utils = { version = "0.8.22", default-features = false, features = [] } dashmap = { version = "6.2.1", default-features = false, features = [] } derive_builder = { version = "0.20.2", default-features = false, features = [] } diff --git a/acl/Cargo.toml b/acl/Cargo.toml index f9d01cb8a9..b4e9ad0f69 100644 --- a/acl/Cargo.toml +++ b/acl/Cargo.toml @@ -27,7 +27,7 @@ clock = { workspace = true, features = ["virtual"] } # differential-test against it, so make it available whenever test/bench targets are built. dataplane-acl = { path = ".", features = ["reference"] } bolero = { workspace = true, features = ["std"] } -criterion = { workspace = true, features = ["cargo_bench_support"] } +criterion = { workspace = true } dpdk = { workspace = true, features = ["test"] } match-action = { workspace = true, features = ["derive", "bolero"] } net = { workspace = true, features = ["test_buffer", "builder"] } diff --git a/default.nix b/default.nix index 4f8964ec67..c2a06e9055 100644 --- a/default.nix +++ b/default.nix @@ -170,6 +170,7 @@ let duvet gateway-crd gettext + iai-callgrind-runner jq just kopium @@ -188,6 +189,8 @@ let rust-toolchain shellcheck skopeo + static-web-server + valgrind wasmtime wget yq diff --git a/deny.toml b/deny.toml index cc4c4356e6..bd06179b4c 100644 --- a/deny.toml +++ b/deny.toml @@ -25,6 +25,7 @@ ignore = [ "RUSTSEC-2024-0436", # proc-macro-error2 is unmaintained but is needed by multi_index_map_derive, and our own fixin; ignore until both dependencies have migrated away from it. "RUSTSEC-2026-0173", + "RUSTSEC-2025-0141", ] [licenses] diff --git a/justfile b/justfile index d099795855..f3af12acac 100644 --- a/justfile +++ b/justfile @@ -41,6 +41,9 @@ kernel := if platform == "wasm32-wasip1" { "wasip1" } else { "linux" } # cargo build profile (debug/release/fuzz) profile := "debug" +export callgrind_package := "dataplane-routing" +export callgrind_bench := "fib_lookup_callgrind" + # sanitizer to use (address/thread/safe-stack/cfi/"") sanitize := "" @@ -199,14 +202,66 @@ fuzz target time="60s" *args="": {{ if sanitize == "thread" { "--build-std" } else { "" } }} \ {{ _cargo_feature_flags }} {{ args }} -# Build and run the criterion benches. The rte_acl benches are gated behind the -# `dpdk` feature, so run `just features=dpdk bench` to exercise them; a plain -# `just bench` builds them as empty `main()` and only runs the reference benches. +[private] +[script] +_bench-release-only: + {{ _just_debuggable_ }} + if [ '{{ profile }}' != "release" ]; then + echo "error: benchmarks want profile=release, not '{{ profile }}'" >&2 + echo " run: just profile=release bench" >&2 + exit 1 + fi + +[doc("Wall-clock time, via criterion")] [script] -bench: (build "benches") +bench *args: _bench-release-only (build "benches") {{ _just_debuggable_ }} shopt -s nullglob - for bench in ./results/benches/bin/*; do "$bench" --bench; done + for bench in ./results/benches/bin/*; do + case "${bench}" in + *_callgrind) continue ;; + esac + "${bench}" --bench {{ args }} + done + if [ -f target/criterion/report/index.html ]; then + echo + echo "html report: target/criterion/report/index.html" + fi + +[doc("Instructions and cache traffic, via iai-callgrind")] +[script] +bench-callgrind *args: + {{ _just_debuggable_ }} + cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" {{ args }} + +[doc("Compare against a baseline and print a markdown report")] +[script] +bench-compare baseline="base" *args: + {{ _just_debuggable_ }} + mkdir -p results/bench + if find target/iai -type f -name '*base@{{ baseline }}*' -print -quit 2>/dev/null | grep -q . \ + && cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" -- \ + --baseline='{{ baseline }}' --output-format=json > results/bench/run.jsonl 2>/dev/null; then + ./scripts/bench-report.ts results/bench/run.jsonl {{ args }} + else + cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" -- \ + --save-baseline='{{ baseline }}' --output-format=json > results/bench/run.jsonl + ./scripts/bench-report.ts results/bench/run.jsonl {{ args }} + fi + +[doc("Record a baseline for `bench-compare`, without reporting")] +[script] +bench-baseline name="base": + {{ _just_debuggable_ }} + cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" -- \ + --save-baseline='{{ name }}' > /dev/null + echo "recorded baseline '{{ name }}'" + +[doc("Serve the criterion html report over http")] +[script] +bench-serve port="8080": + {{ _just_debuggable_ }} + just serve ./target/criterion '{{ port }}' report/index.html [script] build-each *args: (build "workspace" args) @@ -754,6 +809,31 @@ coverage *args: cargo llvm-cov report --branch --lcov --output-path="${out}/lcov.info" cargo llvm-cov report --branch --codecov --output-path="${out}/codecov.json" cargo llvm-cov report --branch --summary-only + echo + echo "html report: ${out}/html/index.html (\`just serve-coverage\` to browse it)" + +serve_host := "127.0.0.1" + +[doc("Serve a directory of generated html over http")] +[script] +serve dir port="8080" index="index.html": + {{ _just_debuggable_ }} + if [ ! -d '{{ dir }}' ]; then + echo "error: no such directory: {{ dir }}" >&2 + exit 1 + fi + server="$(command -v static-web-server || true)" + if [ -z "${server}" ] && [ -x ./devroot/bin/static-web-server ]; then + server="$(pwd)/devroot/bin/static-web-server" + fi + if [ -z "${server}" ]; then + echo "error: static-web-server not found; re-enter the dev shell, or \`just setup-roots\`" >&2 + exit 1 + fi + echo "serving {{ dir }} at http://{{ serve_host }}:{{ port }}/{{ index }} (ctrl-c to stop)" + "${server}" --root '{{ dir }}' --host '{{ serve_host }}' --port '{{ port }}' --log-level warn + +serve-coverage port="8080": (serve "./target/nextest/coverage/html" port) [script] duvet *args: diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index 16e84046d8..b9313768d6 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -25,6 +25,10 @@ in opengrep = final.callPackage ../pkgs/opengrep { src = sources.opengrep; }; + iai-callgrind-runner = final.callPackage ../pkgs/iai-callgrind-runner { + inherit (override-packages) rustPlatform; + version = "0.16.1"; + }; cargo-bolero = prev.cargo-bolero.override { inherit (override-packages) rustPlatform; }; cargo-deny = prev.cargo-deny.override { inherit (override-packages) rustPlatform; }; cargo-edit = prev.cargo-edit.override { inherit (override-packages) rustPlatform; }; diff --git a/nix/pkgs/iai-callgrind-runner/default.nix b/nix/pkgs/iai-callgrind-runner/default.nix new file mode 100644 index 0000000000..7033648528 --- /dev/null +++ b/nix/pkgs/iai-callgrind-runner/default.nix @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Open Network Fabric Authors +{ + fetchCrate, + rustPlatform, + version, + ... +}: +rustPlatform.buildRustPackage (final: { + pname = "iai-callgrind-runner"; + inherit version; + src = fetchCrate { + inherit (final) pname version; + hash = "sha256-wJTwaqAz8GWCJ/l9GRXYBVBkpPYrWxN4VQ7GdRFXmzM="; + }; + cargoHash = "sha256-4N7P23bCeeJee/Cm3sSORByh+HzflOENqYqpu629mpA="; + doCheck = false; +}) diff --git a/routing/Cargo.toml b/routing/Cargo.toml index 90e72b2c48..d7c2376794 100644 --- a/routing/Cargo.toml +++ b/routing/Cargo.toml @@ -56,11 +56,22 @@ procfs = { workspace = true } netdev = { workspace = true } [dev-dependencies] +dataplane-routing = { path = ".", features = ["testing"] } +lpm = { workspace = true, features = ["testing"] } clock = { workspace = true, features = ["virtual"] } +criterion = { workspace = true } +iai-callgrind = { workspace = true } bolero = { workspace = true, default-features = false } concurrency = { workspace = true } -lpm = { workspace = true, features = ["testing"] } net = { workspace = true, features = ["test_buffer"] } rand = { workspace = true, default-features = false, features = ["thread_rng"] } tokio = { workspace = true, features = ["time", "test-util"] } tracing-test = { workspace = true, features = [] } + +[[bench]] +name = "fib_lookup" +harness = false + +[[bench]] +name = "fib_lookup_callgrind" +harness = false diff --git a/routing/benches/common/mod.rs b/routing/benches/common/mod.rs new file mode 100644 index 0000000000..506f8c3e9f --- /dev/null +++ b/routing/benches/common/mod.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +#![allow(dead_code)] + +use lpm::prefix::Prefix; +use net::buffer::TestBuffer; +use net::interface::InterfaceIndex; +use net::ip::NextHeader; +use net::packet::Packet; +use net::packet::test_utils::build_test_ipv4_packet_with_transport; + +use dataplane_routing::testing::{Fib, FibGroup, FibWriter, FwAction, NhopKey, RouteOrigin}; +use dataplane_routing::{EgressObject, FibEntry, PktInstruction}; + +pub const ROUTE_ADDR: (&str, u8) = ("5.0.0.0", 8); + +#[macro_export] +macro_rules! for_each_shape { + ($expand:ident) => { + $expand! { + g1_e1 = (1, 1), + g1_e4 = (1, 4), + g4_e1 = (4, 1), + g4_e4 = (4, 4), + g8_e1 = (8, 1), + g16_e1 = (16, 1), + g16_e4 = (16, 4), + } + }; +} + +pub struct Fixture { + pub writer: FibWriter, + pub packet: Packet, +} + +fn nhop_key(n: u8) -> NhopKey { + NhopKey::new( + RouteOrigin::default(), + Some(format!("10.0.{n}.1").parse().expect("valid address")), + InterfaceIndex::try_new(u32::from(n) + 1).ok(), + None, + FwAction::Forward, + ) +} + +fn fib_group(n: u8, entries: u8) -> FibGroup { + let mut group = FibGroup::new(); + for e in 0..entries { + group.add(FibEntry::with_inst(PktInstruction::Egress( + EgressObject::new( + InterfaceIndex::try_new(u32::from(n) * 256 + u32::from(e) + 1).ok(), + Some(format!("10.{n}.{e}.1").parse().expect("valid address")), + ), + ))); + } + group +} + +pub fn packet() -> Packet { + build_test_ipv4_packet_with_transport(64, Some(NextHeader::UDP)) + .expect("a well-formed test packet") +} + +pub fn fixture(groups: u8, entries_per_group: u8) -> &'static Fixture { + let (mut writer, _reader) = FibWriter::new(0); + let keys: Vec = (0..groups).map(nhop_key).collect(); + for (n, key) in keys.iter().enumerate() { + let n = u8::try_from(n).expect("group count fits a byte"); + writer.register_fibgroup(key, &fib_group(n, entries_per_group), false); + } + writer.add_fibroute(Prefix::expect_from(ROUTE_ADDR), keys, true); + + let packet = packet(); + + { + let fib = writer.enter().expect("fib is readable"); + let (hit, _) = Fib::lpm_entry_prefix(&fib, &packet); + assert_eq!( + hit, + Prefix::expect_from(ROUTE_ADDR), + "{groups}g x{entries_per_group}e: lookup missed the installed route" + ); + } + + Box::leak(Box::new(Fixture { writer, packet })) +} + +#[inline(always)] +pub fn lookup(fixture: &Fixture) { + let fib = fixture.writer.enter().expect("fib is readable"); + let (prefix, entry) = Fib::lpm_entry_prefix(&fib, std::hint::black_box(&fixture.packet)); + std::hint::black_box((prefix, entry)); +} diff --git a/routing/benches/fib_lookup.rs b/routing/benches/fib_lookup.rs new file mode 100644 index 0000000000..c4cb562a8b --- /dev/null +++ b/routing/benches/fib_lookup.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +use std::hint::black_box; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; + +mod common; +use common::{Fixture, fixture, lookup, packet}; + +macro_rules! shape_table { + ($($id:ident = ($groups:expr, $entries:expr)),* $(,)?) => { + &[$((stringify!($id), $groups, $entries)),*] + }; +} +const SHAPES: &[(&str, u8, u8)] = for_each_shape!(shape_table); + +fn bench_lookup(c: &mut Criterion) { + let mut group = c.benchmark_group("fib_lpm_entry_prefix"); + for &(id, groups, entries) in SHAPES { + let fixture: &'static Fixture = fixture(groups, entries); + let total = u64::from(groups) * u64::from(entries); + group.throughput(Throughput::Elements(1)); + group.bench_with_input( + BenchmarkId::from_parameter(format!("{id} ({total} entries)")), + &fixture, + |b, fixture| b.iter(|| lookup(black_box(fixture))), + ); + } + group.finish(); +} + +fn bench_trie_floor(c: &mut Criterion) { + let mut group = c.benchmark_group("fib_lpm_floor"); + let packet = packet(); + let destination = packet + .ip_destination() + .expect("the test packet has a destination"); + + for &(groups, entries) in &[(1u8, 1u8), (16, 1)] { + let fixture = fixture(groups, entries); + group.bench_function( + BenchmarkId::from_parameter(format!("{groups}g x{entries}e")), + |b| { + let fib = fixture.writer.enter().expect("fib is readable"); + b.iter(|| black_box(fib.lpm_with_prefix(black_box(&destination)))); + }, + ); + } + group.finish(); +} + +fn bench_destination(c: &mut Criterion) { + let packet = packet(); + c.bench_function("packet_ip_destination", |b| { + b.iter(|| black_box(black_box(&packet).ip_destination())); + }); +} + +criterion_group!(benches, bench_lookup, bench_trie_floor, bench_destination); +criterion_main!(benches); diff --git a/routing/benches/fib_lookup_callgrind.rs b/routing/benches/fib_lookup_callgrind.rs new file mode 100644 index 0000000000..5bcee45331 --- /dev/null +++ b/routing/benches/fib_lookup_callgrind.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +use std::hint::black_box; + +use iai_callgrind::{ + Cachegrind, Dhat, LibraryBenchmarkConfig, library_benchmark, library_benchmark_group, main, +}; + +mod common; +use common::{Fixture, fixture, lookup}; + +#[library_benchmark] +#[bench::guard_only(args = (1, 1), setup = fixture)] +fn enter_only(fixture: &'static Fixture) { + black_box(fixture.writer.enter().expect("fib is readable")); +} + +macro_rules! shape_benches { + ($($id:ident = ($groups:expr, $entries:expr)),* $(,)?) => { + #[library_benchmark] + $(#[bench::$id(args = ($groups, $entries), setup = fixture)])* + fn lpm_entry_prefix(fixture: &'static Fixture) { + lookup(fixture); + } + }; +} +for_each_shape!(shape_benches); + +#[library_benchmark( + config = LibraryBenchmarkConfig::default() + .tool(Cachegrind::default().args([ + "--D1=32768,8,64", + "--I1=32768,8,64", + "--LL=33554432,16,64", + ])) + .tool(Dhat::default()) +)] +#[bench::g1_e1(args = (1, 1), setup = fixture)] +#[bench::g16_e4(args = (16, 4), setup = fixture)] +fn under_other_tools(fixture: &'static Fixture) { + lookup(fixture); +} + +library_benchmark_group!( + name = fib_lookup; + benchmarks = enter_only, lpm_entry_prefix, under_other_tools +); +main!(library_benchmark_groups = fib_lookup); diff --git a/routing/src/lib.rs b/routing/src/lib.rs index f5d1b3b82a..82496d120b 100644 --- a/routing/src/lib.rs +++ b/routing/src/lib.rs @@ -45,6 +45,14 @@ pub use rib::encapsulation::{ }; pub use rib::vrf::{RouterVrfConfig, VrfId}; +#[cfg(any(test, feature = "testing"))] +pub mod testing { + pub use crate::fib::fibobjects::FibGroup; + pub use crate::fib::fibtype::{Fib, FibReader, FibWriter}; + pub use crate::rib::nexthop::{FwAction, NhopKey}; + pub use crate::rib::vrf::RouteOrigin; +} + pub use bmp::spawn_bmp_server; pub use router::ctl::RouterCtlSender; pub use router::{BmpServerParams, CliSources, Router, RouterParams, RouterParamsBuilder}; diff --git a/routing/src/rib/nexthop.rs b/routing/src/rib/nexthop.rs index 14a74abb62..efa2f137d6 100644 --- a/routing/src/rib/nexthop.rs +++ b/routing/src/rib/nexthop.rs @@ -93,7 +93,7 @@ impl NhopKey { } } #[cfg(test)] - pub fn from_address(address: &str) -> Self { + pub(crate) fn from_address(address: &str) -> Self { Self { address: Some(IpAddr::from_str(address).expect("Bad address")), ..Default::default() @@ -101,7 +101,7 @@ impl NhopKey { } #[cfg(test)] #[must_use] - pub fn with_addr_ifindex(address: &str, ifindex: u32) -> Self { + pub(crate) fn with_addr_ifindex(address: &str, ifindex: u32) -> Self { Self { address: Some(IpAddr::from_str(address).expect("Bad address")), ifindex: Some(InterfaceIndex::try_new(ifindex).expect("Bad ifindex")), @@ -110,7 +110,7 @@ impl NhopKey { } #[cfg(test)] #[must_use] - pub fn with_address(address: &IpAddr) -> Self { + pub(crate) fn with_address(address: &IpAddr) -> Self { Self { address: Some(*address), ..Default::default() @@ -118,7 +118,7 @@ impl NhopKey { } #[cfg(test)] #[must_use] - pub fn with_ifindex(ifindex: u32) -> Self { + pub(crate) fn with_ifindex(ifindex: u32) -> Self { Self { ifindex: Some(InterfaceIndex::try_new(ifindex).unwrap()), ..Default::default() diff --git a/scripts/bench-report.ts b/scripts/bench-report.ts new file mode 100755 index 0000000000..4a59184363 --- /dev/null +++ b/scripts/bench-report.ts @@ -0,0 +1,204 @@ +#!/usr/bin/env -S deno run --allow-read +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + + +type Metric = { Int: number } | { Float: number }; + +type Metrics = { Both: [Metric, Metric] } | { Left: Metric } | { + Right: Metric; +}; + +interface Record_ { + id: string | null; + function_name: string; + module_path: string; + profiles: Array<{ + tool: string; + summaries: { + total: { + summary: Record< + string, + Record< + string, + { + diffs?: { diff_pct?: string }; + metrics: Metrics; + } + > + >; + }; + }; + }>; +} + +const HEADLINE: Record = { + Callgrind: ["Ir", "EstimatedCycles"], + Cachegrind: ["Ir", "EstimatedCycles"], + DHAT: ["TotalBytes", "TotalBlocks"], + Memcheck: ["Errors"], + Massif: ["PeakBytes"], +}; + +function value(m: Metric): number { + return "Int" in m ? m.Int : m.Float; +} + +interface Row { + bench: string; + tool: string; + metric: string; + now: number; + before: number | null; + pct: number | null; +} + +function rows(records: Record_[]): Row[] { + const out: Row[] = []; + for (const rec of records) { + const bench = rec.id ? `${rec.function_name} ${rec.id}` : rec.function_name; + for (const profile of rec.profiles ?? []) { + const summaries = profile.summaries?.total?.summary ?? {}; + const summary = summaries[profile.tool] ?? Object.values(summaries)[0]; + if (!summary) continue; + for (const metric of HEADLINE[profile.tool] ?? []) { + const entry = summary[metric]; + if (!entry) continue; + const m = entry.metrics; + const now = "Both" in m + ? value(m.Both[0]) + : "Left" in m + ? value(m.Left) + : value(m.Right); + const before = "Both" in m ? value(m.Both[1]) : null; + const pct = before === null || before === 0 + ? null + : ((now - before) / before) * 100; + out.push({ bench, tool: profile.tool, metric, now, before, pct }); + } + } + } + return out; +} + +const fmt = (n: number) => n.toLocaleString("en-US"); +const pct = (p: number | null) => + p === null ? "—" : `${p >= 0 ? "+" : ""}${p.toFixed(2)}%`; + +function bar(p: number | null, worst: number): string { + if (p === null || worst === 0 || Math.abs(p) < 0.005) return ""; + const width = Math.min(10, Math.round((Math.abs(p) / worst) * 10)); + return (p >= 0 ? "▰" : "▱").repeat(Math.max(1, width)); +} + +function table(rs: Row[]): string { + const worst = Math.max(0, ...rs.map((r) => Math.abs(r.pct ?? 0))); + const head = + "| benchmark | tool | metric | before | after | change | |\n|---|---|---|---:|---:|---:|---|"; + const body = rs.map( + (r) => + `| ${r.bench} | ${r.tool} | ${r.metric} | ${ + r.before === null ? "—" : fmt(r.before) + } | ${fmt(r.now)} | ${pct(r.pct)} | ${bar(r.pct, worst)} |`, + ); + return [head, ...body].join("\n"); +} + +function chart(rs: Row[]): string { + const shown = rs.filter( + (r) => r.tool === "Callgrind" && r.metric === "Ir" && r.pct !== null, + ); + if (shown.length === 0) return ""; + const labels = shown.map((r) => `"${r.bench.replace(/"/g, "")}"`).join(", "); + const values = shown.map((r) => (r.pct ?? 0).toFixed(2)).join(", "); + const span = Math.max(5, ...shown.map((r) => Math.abs(r.pct ?? 0))) * 1.2; + return [ + "```mermaid", + "xychart-beta", + ' title "Instructions retired, change vs baseline (%)"', + ` x-axis [${labels}]`, + ` y-axis "change (%)" ${(-span).toFixed(0)} --> ${span.toFixed(0)}`, + ` bar [${values}]`, + "```", + ].join("\n"); +} + +function main() { + const args = Deno.args.filter((a) => !a.startsWith("--")); + const flags = Deno.args.filter((a) => a.startsWith("--")); + const threshold = Number( + flags.find((f) => f.startsWith("--threshold="))?.split("=")[1] ?? "5", + ); + const headlineOnly = flags.includes("--headline-only"); + + if (args.length !== 1) { + console.error( + "usage: bench-report.ts [--threshold=N] [--headline-only]", + ); + Deno.exit(2); + } + + const records: Record_[] = Deno.readTextFileSync(args[0]) + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line)); + + const rs = rows(records); + const compared = rs.filter((r) => r.pct !== null); + + const worstOf = (subset: Row[]) => + subset.reduce( + (acc: Row | null, r) => + acc === null || Math.abs(r.pct!) > Math.abs(acc.pct!) ? r : acc, + null, + ); + const worst = worstOf( + compared.filter((r) => r.tool === "Callgrind" && r.metric === "Ir"), + ) ?? worstOf(compared); + const stark = worst !== null && Math.abs(worst.pct!) >= threshold; + + const alloc = worstOf( + compared.filter( + (r) => + r.tool === "DHAT" && r.metric === "TotalBytes" && Math.abs(r.pct!) >= 1, + ), + ); + const allocNote = alloc === null + ? "" + : `; bytes allocated ${pct(alloc.pct)} (DHAT)`; + + if (compared.length === 0) { + console.log( + "**Benchmarks**: no baseline to compare against; recorded a new one.", + ); + } else if (!stark) { + console.log( + `**Benchmarks**: no change beyond ${threshold}% (largest: ${ + worst!.bench + } ${worst!.metric} ${pct(worst!.pct)})${allocNote}.`, + ); + } else { + const dir = worst!.pct! > 0 ? "more" : "less"; + console.log( + `**Benchmarks**: ${worst!.bench} does ${pct(worst!.pct)} ${dir} work (${ + worst!.metric + })${allocNote}.`, + ); + } + if (headlineOnly) return; + + console.log(""); + console.log(table(rs)); + if (stark) { + console.log(""); + console.log(chart(rs)); + } + console.log(""); + console.log( + "> Instruction counts from callgrind, not timings: repeatable across runners, and blind to " + + "anything that changes data layout rather than instruction count. See " + + "`development/code/benchmarking.md`.", + ); +} + +main();