diff --git a/.gitignore b/.gitignore index 81f716f..89e8d69 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,10 @@ AGENTS.md ### Examples ### examples/*/*.class + +### Reproducer ### +reproducer/runs/ + +### Python ### +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 50dc5ef..e1f0eff 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,14 @@ CSV rows also include `JIT log file` and `JMH result file` path columns. - The target method should preferably return a value. `void` methods are more likely to be removed by JIT optimization as dead code. - The target method should preferably execute at least `1000` instructions. Around `50-100` instructions may come from non-optimized JMH wrapper overhead (`Method.invoke()`). By default this optimization should work and overhead will be around `10-20` instructions. +## Reproducer + +The `reproducer/` directory contains a runner for curated baseline/variant cases. +It compiles each case, runs Comparator repeatedly, preserves raw JIT/JMH artifacts, +and writes per-case aggregate CSV files. + +See [reproducer/README.md](reproducer/README.md) for usage. + ## API usage ### Run an analysis @@ -41,15 +49,19 @@ import comparator.Analysis; import comparator.method.TargetMethod; import java.nio.file.Path; -final Path classpath = Path.of("examples", "loop-computations"); -new Analysis(new TargetMethod(classpath, "PlainForExample", "run")) +final Path classpath = Path.of( + "reproducer", "cases", "case00_primitive_loop_examples", "baseline" +); +new Analysis(new TargetMethod(classpath, "PrimitiveLoopExample", "run")) .results() .print(System.out); ``` The classpath argument must point to a directory or JAR that contains compiled classes. -The `examples/` folders do not include `.class` files, so compile them before running the examples. +The `reproducer/cases/` folders do not include `.class` files, so compile the selected +case role before running API examples. For `case00_primitive_loop_examples`, each role +has its own classpath because all variants use the same `PrimitiveLoopExample` class name. ### Run an analysis with label @@ -58,9 +70,11 @@ import comparator.Analysis; import comparator.method.TargetMethod; import java.nio.file.Path; -final Path classpath = Path.of("examples", "loop-computations"); +final Path classpath = Path.of( + "reproducer", "cases", "case00_primitive_loop_examples", "baseline" +); new Analysis( - new TargetMethod(classpath, "PlainForExample", "run"), + new TargetMethod(classpath, "PrimitiveLoopExample", "run"), "baseline-for-loop" ).results().print(System.out); ``` @@ -77,17 +91,18 @@ import comparator.method.Classpath; import comparator.method.TargetMethod; import java.nio.file.Path; -final Classpath classpath = new Classpath(Path.of("examples", "loop-computations")); +final Path caseRoot = Path.of("reproducer", "cases", "case00_primitive_loop_examples"); +final Classpath baseline = new Classpath(caseRoot.resolve("baseline")); +final Classpath plainArray = new Classpath(caseRoot.resolve("variants").resolve("plain_array")); +final Classpath indexedLoop = new Classpath(caseRoot.resolve("variants").resolve("indexed_loop")); +final Classpath streamBoxed = new Classpath(caseRoot.resolve("variants").resolve("stream_boxed")); new CsvComparisons( new CsvComparison( - new Analysis(new TargetMethod(classpath, "PlainForExample", "run")), - new Analysis(new TargetMethod(classpath, "StreamBoxedExample", "run")), - new Analysis(new TargetMethod(classpath, "PlainForIndexedExample", "run")) - ), - new CsvComparison( - new Analysis(new TargetMethod(classpath, "PlainForExample", "run")), - new Analysis(new TargetMethod(classpath, "PlainForReplaceAllExample", "run")) + new Analysis(new TargetMethod(baseline, "PrimitiveLoopExample", "run"), "baseline"), + new Analysis(new TargetMethod(plainArray, "PrimitiveLoopExample", "run"), "plain_array"), + new Analysis(new TargetMethod(indexedLoop, "PrimitiveLoopExample", "run"), "indexed_loop"), + new Analysis(new TargetMethod(streamBoxed, "PrimitiveLoopExample", "run"), "stream_boxed") ) ).saveAsCsv(Path.of("comparisons.csv")); ``` @@ -96,28 +111,20 @@ new CsvComparisons( Example of `comparisons.csv` content in table form. File path columns are shortened for readability: -Comparison 1 - -| Target | JMH primary score, us/op | JMH primary score relative error, ratio | Allocations, B/op | Allocations relative error, ratio | Instructions, #/op | Memory loads, #/op | Memory stores, #/op | Native code size, B | JIT log file | JMH result file | JIT metrics mean dissimilarity score | JIT metrics max dissimilarity score | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- | -| PlainForExample::run | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PlainForExample-jit-log-...xml` | `.../PlainForExample-jmh-result-...json` | Original | Original | -| PlainForPlainArrayExample::run | 4.55 | 0.11 | 8040.03 | 0.00 | 25719.34 | 3281.44 | 1151.00 | 1552.00 | `.../PlainForPlainArrayExample-jit-log-...xml` | `.../PlainForPlainArrayExample-jmh-result-...json` | 1.40 | 1.79 | -| PlainForIndexedExample::run | 25.64 | 0.12 | 38016.18 | 0.00 | 149624.24 | 41981.45 | 19787.82 | 1960.00 | `.../PlainForIndexedExample-jit-log-...xml` | `.../PlainForIndexedExample-jmh-result-...json` | 0.09 | 0.15 | - -Comparison 2 - | Target | JMH primary score, us/op | JMH primary score relative error, ratio | Allocations, B/op | Allocations relative error, ratio | Instructions, #/op | Memory loads, #/op | Memory stores, #/op | Native code size, B | JIT log file | JMH result file | JIT metrics mean dissimilarity score | JIT metrics max dissimilarity score | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- | -| PlainForExample::run | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PlainForExample-jit-log-...xml` | `.../PlainForExample-jmh-result-...json` | Original | Original | -| PlainForReplaceAllExample::run | 36.08 | 0.11 | 69952.25 | 0.00 | 240191.93 | 65149.11 | 32818.88 | 2336.00 | `.../PlainForReplaceAllExample-jit-log-...xml` | `.../PlainForReplaceAllExample-jmh-result-...json` | 0.44 | 0.59 | +| baseline | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | Original | Original | +| plain_array | 4.55 | 0.11 | 8040.03 | 0.00 | 25719.34 | 3281.44 | 1151.00 | 1552.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 1.40 | 1.79 | +| indexed_loop | 25.64 | 0.12 | 38016.18 | 0.00 | 149624.24 | 41981.45 | 19787.82 | 1960.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 0.09 | 0.15 | +| stream_boxed | 46.05 | 0.10 | 70232.32 | 0.00 | 243005.89 | 74453.67 | 32071.86 | 3528.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 0.54 | 0.70 | ### Labeled comparison example ```java new CsvComparisons( new CsvComparison( - new Analysis(new TargetMethod(classpath, "PlainForExample", "run"), "Baseline"), - new Analysis(new TargetMethod(classpath, "StreamBoxedExample", "run"), "Stream") + new Analysis(new TargetMethod(baseline, "PrimitiveLoopExample", "run"), "Baseline"), + new Analysis(new TargetMethod(streamBoxed, "PrimitiveLoopExample", "run"), "Stream") ) ).saveAsCsv(Path.of("labels-demo.csv")); ``` @@ -126,8 +133,8 @@ Example of `labels-demo.csv` content in table form: | Target | JMH primary score, us/op | JMH primary score relative error, ratio | Allocations, B/op | Allocations relative error, ratio | Instructions, #/op | Memory loads, #/op | Memory stores, #/op | Native code size, B | JIT log file | JMH result file | JIT metrics mean dissimilarity score | JIT metrics max dissimilarity score | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- | -| Baseline | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PlainForExample-jit-log-...xml` | `.../PlainForExample-jmh-result-...json` | Original | Original | -| Stream | 46.05 | 0.10 | 70232.32 | 0.00 | 243005.89 | 74453.67 | 32071.86 | 3528.00 | `.../StreamBoxedExample-jit-log-...xml` | `.../StreamBoxedExample-jmh-result-...json` | 0.54 | 0.70 | +| Baseline | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | Original | Original | +| Stream | 46.05 | 0.10 | 70232.32 | 0.00 | 243005.89 | 74453.67 | 32071.86 | 3528.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 0.54 | 0.70 | ## Comparison metrics diff --git a/build.gradle b/build.gradle index 17b20b3..1fee707 100644 --- a/build.gradle +++ b/build.gradle @@ -48,3 +48,10 @@ jacocoTestReport { application { mainClass = 'comparator.Main' } + +tasks.register('printRuntimeClasspath') { + dependsOn classes + doLast { + println sourceSets.main.runtimeClasspath.asPath + } +} diff --git a/examples/loop-computations/PlainForExample.java b/examples/loop-computations/PlainForExample.java deleted file mode 100644 index 33ba69f..0000000 --- a/examples/loop-computations/PlainForExample.java +++ /dev/null @@ -1,24 +0,0 @@ -import java.util.ArrayList; -import java.util.List; - -public class PlainForExample { - private static final int N = 2_000; - - private static int compute(int x) { - int y = x * 31; - y ^= (y >>> 16); - return y + 7; - } - - public static long run() { - List list = new ArrayList<>(N); - for (int i = 0; i < N; i++) { - list.add(i); - } - long sum = 0; - for (int v : list) { - sum += compute(v); - } - return sum; - } -} diff --git a/reproducer/README.md b/reproducer/README.md new file mode 100644 index 0000000..7f28f09 --- /dev/null +++ b/reproducer/README.md @@ -0,0 +1,155 @@ +# Reproducer + +The reproducer runs curated JIT instability cases from `reproducer/cases`. +Each case contains a `baseline` Java source and one or more equivalent variant +Java sources. Existing single-variant cases may use `variant/`; multi-variant +cases use `variants//`. + +It can: + +- discover valid baseline/variant pairs; +- compile each pair into isolated class directories; +- run Comparator on each pair through JMH; +- collect JIT logs, JMH JSON results, and perf-backed metrics; +- repeat whole-case executions with `--runs`; +- write per-run CSV files and per-case aggregate CSV files. + +## Requirements + +- Linux with working `perf`; +- JDK with `java` and `javac` on `PATH`; +- Gradle wrapper from this repository. + +The runner checks `perf --version` and `perf stat -e instructions -- sleep 0.1` +before starting expensive work. + +## Usage + +Run all cases once: + +```bash +python3 reproducer/run.py --runs 1 +``` + +Run selected cases by prefix: + +```bash +python3 reproducer/run.py --runs 3 --include-cases case01,case03 +``` + +Run the primitive loop demonstration case: + +```bash +python3 reproducer/run.py --runs 3 --include-cases case00 +``` + +Useful options: + +- `--runs N` - required number of whole-case repeats; +- `--include-cases case01,case03` - comma-separated case-id prefixes; +- `--session-id NAME` - fixed output session name; +- `--cases-root PATH` - custom cases directory; +- `--runs-root PATH` - custom output directory. + +## Case layout + +Single-variant cases can use the compatibility layout: + +```text +case01_example/ + baseline/Example.java + variant/Example.java +``` + +Multi-variant cases use named variant roles: + +```text +case00_primitive_loop_examples/ + baseline/PrimitiveLoopExample.java + variants/plain_array/PrimitiveLoopExample.java + variants/indexed_loop/PrimitiveLoopExample.java + variants/replace_all/PrimitiveLoopExample.java + variants/stream_boxed/PrimitiveLoopExample.java +``` + +Each role must contain exactly one package-less Java source with the same file +name as the baseline and a static no-argument `run` method. + +## Output + +Each run creates a session under `reproducer/runs/` and updates +`reproducer/runs/latest` when symlinks are supported. + +Important files: + +- `metadata.json` - tools, environment, selected cases, Gradle classpath output; +- `index.csv` - one row per case run; +- `cases//runs/run-001/status.json` - run status or captured failure; +- `cases//runs/run-001/logs/*.log` - stdout and stderr for `javac` and Comparator commands; +- `cases//runs/run-001/comparisons.csv` - raw Comparator CSV; +- `cases//runs/run-001/artifacts/*` - JIT logs and JMH results; +- `cases//all_runs.csv` - concatenated rows across runs; +- `cases//summary.csv` - count, mean, stdev, min, and max per role. + +## Plotting + +**Generate a mean-difference PDF for one case:** + +```bash +python3 reproducer/plotting/mean_difference.py \ + reproducer/runs/example-session/cases/example-case +``` + +By default this reads the case `summary.csv` and creates: + +```text +reproducer/runs/example-session/cases/example-case/mean_difference.pdf +``` + +Example plot: + +![Mean difference plot](plotting/examples/mean_difference.png) + +The chart includes five aggregated metrics: JMH score, allocations, +instructions, memory loads, and memory stores. + +**Generate a per-run relative-difference PDF for one case:** + +```bash +python3 reproducer/plotting/run_difference.py \ + reproducer/runs/example-session/cases/example-case +``` + +By default this reads the case `all_runs.csv` and creates: + +```text +reproducer/runs/example-session/cases/example-case/run_difference.pdf +``` + +Example plot: + +![Run difference plot](plotting/examples/run_difference.png) + +The chart includes six per-run metrics: JMH score, allocations, instructions, +memory loads, memory stores, and native code size. + +**Generate raw per-metric PDFs for one case:** + +```bash +python3 reproducer/plotting/metric_difference.py \ + reproducer/runs/example-session/cases/example-case +``` + +By default this reads the case `all_runs.csv` and creates one PDF per metric +under: + +```text +reproducer/runs/example-session/cases/example-case/metric_difference/ +``` + +Example plot: + +![Native code size by run plot](plotting/examples/native_code_size_by_run.png) + +The charts include six raw metrics: JMH score, allocations, instructions, +memory loads, memory stores, and native code size. diff --git a/reproducer/cases/case00_primitive_loop_examples/baseline/PrimitiveLoopExample.java b/reproducer/cases/case00_primitive_loop_examples/baseline/PrimitiveLoopExample.java new file mode 100644 index 0000000..09645c5 --- /dev/null +++ b/reproducer/cases/case00_primitive_loop_examples/baseline/PrimitiveLoopExample.java @@ -0,0 +1,33 @@ +/* + * Mechanism: primitive loop control examples. + * Hypothesis: equivalent ways to build, traverse, and transform a small + * integer collection can produce different optimized code shapes even when + * they return the same result. + * Expected symptom: plain-array and indexed-loop variants should stay close to + * the baseline, while replaceAll and boxed-stream variants are expected to show + * larger instruction/load/store and native-code-size movement. + */ +import java.util.ArrayList; +import java.util.List; + +public class PrimitiveLoopExample { + private static final int N = 2_000; + + private static int compute(final int x) { + int y = x * 31; + y ^= y >>> 16; + return y + 7; + } + + public static long run() { + final List list = new ArrayList<>(N); + for (int i = 0; i < N; i++) { + list.add(i); + } + long sum = 0; + for (final int value : list) { + sum += compute(value); + } + return sum; + } +} diff --git a/examples/loop-computations/PlainForIndexedExample.java b/reproducer/cases/case00_primitive_loop_examples/variants/indexed_loop/PrimitiveLoopExample.java similarity index 88% rename from examples/loop-computations/PlainForIndexedExample.java rename to reproducer/cases/case00_primitive_loop_examples/variants/indexed_loop/PrimitiveLoopExample.java index a9292f6..da2b142 100644 --- a/examples/loop-computations/PlainForIndexedExample.java +++ b/reproducer/cases/case00_primitive_loop_examples/variants/indexed_loop/PrimitiveLoopExample.java @@ -1,12 +1,12 @@ import java.util.ArrayList; import java.util.List; -public class PlainForIndexedExample { +public class PrimitiveLoopExample { private static final int N = 2_000; private static int compute(final int x) { int y = x * 31; - y ^= (y >>> 16); + y ^= y >>> 16; return y + 7; } diff --git a/examples/loop-computations/PlainForPlainArrayExample.java b/reproducer/cases/case00_primitive_loop_examples/variants/plain_array/PrimitiveLoopExample.java similarity index 71% rename from examples/loop-computations/PlainForPlainArrayExample.java rename to reproducer/cases/case00_primitive_loop_examples/variants/plain_array/PrimitiveLoopExample.java index 26fdae9..119b247 100644 --- a/examples/loop-computations/PlainForPlainArrayExample.java +++ b/reproducer/cases/case00_primitive_loop_examples/variants/plain_array/PrimitiveLoopExample.java @@ -1,9 +1,9 @@ -public class PlainForPlainArrayExample { +public class PrimitiveLoopExample { private static final int N = 2_000; private static int compute(final int x) { int y = x * 31; - y ^= (y >>> 16); + y ^= y >>> 16; return y + 7; } @@ -13,8 +13,8 @@ public static long run() { values[i] = i; } long sum = 0; - for (final int v : values) { - sum += compute(v); + for (final int value : values) { + sum += compute(value); } return sum; } diff --git a/examples/loop-computations/PlainForReplaceAllExample.java b/reproducer/cases/case00_primitive_loop_examples/variants/replace_all/PrimitiveLoopExample.java similarity index 69% rename from examples/loop-computations/PlainForReplaceAllExample.java rename to reproducer/cases/case00_primitive_loop_examples/variants/replace_all/PrimitiveLoopExample.java index f356f7e..d2fd52d 100644 --- a/examples/loop-computations/PlainForReplaceAllExample.java +++ b/reproducer/cases/case00_primitive_loop_examples/variants/replace_all/PrimitiveLoopExample.java @@ -1,12 +1,12 @@ import java.util.ArrayList; import java.util.List; -public class PlainForReplaceAllExample { +public class PrimitiveLoopExample { private static final int N = 2_000; private static int compute(final int x) { int y = x * 31; - y ^= (y >>> 16); + y ^= y >>> 16; return y + 7; } @@ -15,10 +15,10 @@ public static long run() { for (int i = 0; i < N; i++) { list.add(i); } - list.replaceAll(PlainForReplaceAllExample::compute); + list.replaceAll(PrimitiveLoopExample::compute); long sum = 0; - for (int v : list) { - sum += v; + for (final int value : list) { + sum += value; } return sum; } diff --git a/reproducer/cases/case00_primitive_loop_examples/variants/stream_boxed/PrimitiveLoopExample.java b/reproducer/cases/case00_primitive_loop_examples/variants/stream_boxed/PrimitiveLoopExample.java new file mode 100644 index 0000000..983e546 --- /dev/null +++ b/reproducer/cases/case00_primitive_loop_examples/variants/stream_boxed/PrimitiveLoopExample.java @@ -0,0 +1,23 @@ +import java.util.ArrayList; +import java.util.List; + +public class PrimitiveLoopExample { + private static final int N = 2_000; + + private static int compute(final int x) { + int y = x * 31; + y ^= y >>> 16; + return y + 7; + } + + public static long run() { + final List list = new ArrayList<>(N); + for (int i = 0; i < N; i++) { + list.add(i); + } + return list.stream() + .map(PrimitiveLoopExample::compute) + .mapToLong(Integer::longValue) + .sum(); + } +} diff --git a/reproducer/cases/case01_use_string_builder_concat_loop/baseline/UseStringBuilderConcatCase.java b/reproducer/cases/case01_use_string_builder_concat_loop/baseline/UseStringBuilderConcatCase.java new file mode 100644 index 0000000..b81187d --- /dev/null +++ b/reproducer/cases/case01_use_string_builder_concat_loop/baseline/UseStringBuilderConcatCase.java @@ -0,0 +1,28 @@ +/* + * Mechanism: use-string-builder. + * Aggregate provenance: FuzzInput09_PushDownAndConcat/1-use-string-builder. + * Hypothesis: replacing repeated string concatenation with an explicit + * StringBuilder changes the optimized code shape enough that HotSpot does + * not converge to the same machine code. + * Expected symptom: allocation and supporting-counter drift in the + * StringBuilder variant. + * Loop note: the outer run() loop is intentionally large enough to keep the + * coarse comparator run comfortably above ~10 us/op. + */ +public class UseStringBuilderConcatCase { + private static int render(int seed) { + String text = ""; + for (int i = 0; i < 48; i++) { + text += seed + i; + } + return text.length(); + } + + public static int run() { + int res = 0; + for (int i = 0; i < 800; i++) { + res += render(i); + } + return res; + } +} diff --git a/reproducer/cases/case01_use_string_builder_concat_loop/variant/UseStringBuilderConcatCase.java b/reproducer/cases/case01_use_string_builder_concat_loop/variant/UseStringBuilderConcatCase.java new file mode 100644 index 0000000..eb572e8 --- /dev/null +++ b/reproducer/cases/case01_use_string_builder_concat_loop/variant/UseStringBuilderConcatCase.java @@ -0,0 +1,17 @@ +public class UseStringBuilderConcatCase { + private static int render(int seed) { + StringBuilder text = new StringBuilder(); + for (int i = 0; i < 48; i++) { + text.append(seed + i); + } + return text.length(); + } + + public static int run() { + int res = 0; + for (int i = 0; i < 800; i++) { + res += render(i); + } + return res; + } +} diff --git a/reproducer/cases/case02_use_arrays_stream_primitives/baseline/UseArraysStreamPrimitiveCase.java b/reproducer/cases/case02_use_arrays_stream_primitives/baseline/UseArraysStreamPrimitiveCase.java new file mode 100644 index 0000000..a60de86 --- /dev/null +++ b/reproducer/cases/case02_use_arrays_stream_primitives/baseline/UseArraysStreamPrimitiveCase.java @@ -0,0 +1,29 @@ +/* + * Mechanism: use-arrays-stream. + * Aggregate provenance: FuzzInput02_ArraysStreamAndBoxing/1-use-arrays-stream-3. + * Hypothesis: switching from a boxed List stream to a primitive IntStream + * changes the stream pipeline enough to perturb JIT counters. + * Expected symptom: instruction/load/store drift in addition to any JMH score + * movement. + * Loop note: the outer run() loop is intentionally large enough to keep the + * coarse comparator run comfortably above ~10 us/op. + */ +import java.util.Arrays; + +public class UseArraysStreamPrimitiveCase { + private static int sumEven(int seed) { + return Arrays.asList(seed, seed + 1, seed + 2, seed + 3, seed + 4, seed + 5) + .stream() + .mapToInt(Integer::intValue) + .filter(value -> (value & 1) == 0) + .sum(); + } + + public static int run() { + int res = 0; + for (int i = 0; i < 12_000; i++) { + res += sumEven(i); + } + return res; + } +} diff --git a/reproducer/cases/case02_use_arrays_stream_primitives/variant/UseArraysStreamPrimitiveCase.java b/reproducer/cases/case02_use_arrays_stream_primitives/variant/UseArraysStreamPrimitiveCase.java new file mode 100644 index 0000000..af34168 --- /dev/null +++ b/reproducer/cases/case02_use_arrays_stream_primitives/variant/UseArraysStreamPrimitiveCase.java @@ -0,0 +1,17 @@ +import java.util.Arrays; + +public class UseArraysStreamPrimitiveCase { + private static int sumEven(int seed) { + return Arrays.stream(new int[] {seed, seed + 1, seed + 2, seed + 3, seed + 4, seed + 5}) + .filter(value -> (value & 1) == 0) + .sum(); + } + + public static int run() { + int res = 0; + for (int i = 0; i < 12_000; i++) { + res += sumEven(i); + } + return res; + } +} diff --git a/reproducer/cases/case03_make_method_field_final/baseline/MakeMethodFieldFinalCase.java b/reproducer/cases/case03_make_method_field_final/baseline/MakeMethodFieldFinalCase.java new file mode 100644 index 0000000..42eb94a --- /dev/null +++ b/reproducer/cases/case03_make_method_field_final/baseline/MakeMethodFieldFinalCase.java @@ -0,0 +1,49 @@ +/* + * Mechanism: make-fields-and-variables-final. + * Aggregate provenance: FuzzInput04_FieldMethod/1-make-fields-and-variables-final. + * Hypothesis: making the reflective Method field final gives HotSpot a more + * stable field shape around the reflective call path. + * Expected symptom: counter and generated-code-size movement in the final-field + * variant. + * Loop note: the outer run() loop is intentionally large enough to keep the + * coarse comparator run comfortably above ~10 us/op. + */ +import java.lang.reflect.Method; + +public class MakeMethodFieldFinalCase { + private static Method METHOD = getSum0Method(); + + private static Method getSum0Method() { + try { + Method method = MakeMethodFieldFinalCase.class.getDeclaredMethod( + "sum0", + int.class, + int.class + ); + method.setAccessible(true); + return method; + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Failed to initialize METHOD", e); + } + } + + private static int sum0(int a, int b) { + return a + b; + } + + public static int sum(int a, int b) { + try { + return (Integer) METHOD.invoke(null, a, b); + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Failed to invoke METHOD", e); + } + } + + public static int run() { + int res = 0; + for (int i = 0; i < 900; i++) { + res = sum(res, i); + } + return res; + } +} diff --git a/reproducer/cases/case03_make_method_field_final/variant/MakeMethodFieldFinalCase.java b/reproducer/cases/case03_make_method_field_final/variant/MakeMethodFieldFinalCase.java new file mode 100644 index 0000000..d3c0a5d --- /dev/null +++ b/reproducer/cases/case03_make_method_field_final/variant/MakeMethodFieldFinalCase.java @@ -0,0 +1,39 @@ +import java.lang.reflect.Method; + +public class MakeMethodFieldFinalCase { + private static final Method METHOD = getSum0Method(); + + private static Method getSum0Method() { + try { + Method method = MakeMethodFieldFinalCase.class.getDeclaredMethod( + "sum0", + int.class, + int.class + ); + method.setAccessible(true); + return method; + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Failed to initialize METHOD", e); + } + } + + private static int sum0(int a, int b) { + return a + b; + } + + public static int sum(int a, int b) { + try { + return (Integer) METHOD.invoke(null, a, b); + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Failed to invoke METHOD", e); + } + } + + public static int run() { + int res = 0; + for (int i = 0; i < 900; i++) { + res = sum(res, i); + } + return res; + } +} diff --git a/reproducer/cases/case04_use_arrays_stream_inline_budget/baseline/UseArraysStreamInlineBudgetCase.java b/reproducer/cases/case04_use_arrays_stream_inline_budget/baseline/UseArraysStreamInlineBudgetCase.java new file mode 100644 index 0000000..e222ddc --- /dev/null +++ b/reproducer/cases/case04_use_arrays_stream_inline_budget/baseline/UseArraysStreamInlineBudgetCase.java @@ -0,0 +1,50 @@ +/* + * Mechanism: use-arrays-stream as an inlining-budget cliff. + * Aggregate provenance: FuzzInput05_MixedInlineArraysAndFinal/ + * 1-inline-method-1/2-use-arrays-stream. + * Hypothesis: replacing one boxed stream count with Arrays.stream(new int[]) + * should not destabilize unrelated helper inlining, but in practice it can + * change HotSpot's compile plan enough to perturb whether the run1 -> run0 + * edge is inlined. + * Expected symptom: instruction/load/store drift with smaller JMH movement; the + * JIT log may show a run0 inlining divergence. + * Minimality note: runBody() preserves the original 100-iteration hot loop. + * The outer run() loop only repeats that shape to keep the coarse comparator + * above ~10 us/op without rebuilding the whole fuzzed class. + */ +import java.util.Arrays; + +public class UseArraysStreamInlineBudgetCase { + private int sq(int x) { + return x * x; + } + + public long run0() { + int base = 10; + return Arrays.asList(1, 2, 3).stream().count() + + Arrays.asList("x", "y").stream().count() + + sq(base) + + sq(3) + + 3; + } + + public static int run1() { + return (int) new UseArraysStreamInlineBudgetCase().run0(); + } + + private static int runBody() { + int res = 0; + for (int i = 0; i < 100; i++) { + res += run1(); + } + return res; + } + + public static int run() { + int total = 0; + for (int r = 0; r < 4; r++) { + total += runBody(); + } + return total; + } +} diff --git a/reproducer/cases/case04_use_arrays_stream_inline_budget/variant/UseArraysStreamInlineBudgetCase.java b/reproducer/cases/case04_use_arrays_stream_inline_budget/variant/UseArraysStreamInlineBudgetCase.java new file mode 100644 index 0000000..5e070ec --- /dev/null +++ b/reproducer/cases/case04_use_arrays_stream_inline_budget/variant/UseArraysStreamInlineBudgetCase.java @@ -0,0 +1,36 @@ +import java.util.Arrays; + +public class UseArraysStreamInlineBudgetCase { + private int sq(int x) { + return x * x; + } + + public long run0() { + int base = 10; + return Arrays.stream(new int[] {1, 2, 3}).count() + + Arrays.asList("x", "y").stream().count() + + sq(base) + + sq(3) + + 3; + } + + public static int run1() { + return (int) new UseArraysStreamInlineBudgetCase().run0(); + } + + private static int runBody() { + int res = 0; + for (int i = 0; i < 100; i++) { + res += run1(); + } + return res; + } + + public static int run() { + int total = 0; + for (int r = 0; r < 4; r++) { + total += runBody(); + } + return total; + } +} diff --git a/reproducer/cases/case05_loop_computations_stream_boxed/baseline/LoopComputationsStreamBoxedCase.java b/reproducer/cases/case05_loop_computations_stream_boxed/baseline/LoopComputationsStreamBoxedCase.java new file mode 100644 index 0000000..c7da068 --- /dev/null +++ b/reproducer/cases/case05_loop_computations_stream_boxed/baseline/LoopComputationsStreamBoxedCase.java @@ -0,0 +1,36 @@ +/* + * Mechanism: plain counted loop vs boxed stream pipeline. + * External provenance: case00_primitive_loop_examples baseline -> stream_boxed. + * Hypothesis: replacing a direct accumulation loop with a boxed stream + * pipeline adds enough library and lambda machinery to change the optimized + * code shape in a compact, easy-to-explain control example. + * Expected symptom: higher JMH score and supporting instruction/load/store + * drift in the stream version. + * Minimality note: this is already the control example itself; only the class + * name was unified so the comparator can treat the pair as one refactoring. + */ +import java.util.ArrayList; +import java.util.List; + +public class LoopComputationsStreamBoxedCase { + private static final int N = 2_000; + + private static int compute(int x) { + int y = x * 31; + y ^= y >>> 16; + return y + 7; + } + + public static long run() { + List list = new ArrayList<>(N); + for (int i = 0; i < N; i++) { + list.add(i); + } + + long sum = 0; + for (int value : list) { + sum += compute(value); + } + return sum; + } +} diff --git a/examples/loop-computations/StreamBoxedExample.java b/reproducer/cases/case05_loop_computations_stream_boxed/variant/LoopComputationsStreamBoxedCase.java similarity index 66% rename from examples/loop-computations/StreamBoxedExample.java rename to reproducer/cases/case05_loop_computations_stream_boxed/variant/LoopComputationsStreamBoxedCase.java index fa050f1..6799c94 100644 --- a/examples/loop-computations/StreamBoxedExample.java +++ b/reproducer/cases/case05_loop_computations_stream_boxed/variant/LoopComputationsStreamBoxedCase.java @@ -1,12 +1,12 @@ import java.util.ArrayList; import java.util.List; -public class StreamBoxedExample { +public class LoopComputationsStreamBoxedCase { private static final int N = 2_000; private static int compute(int x) { int y = x * 31; - y ^= (y >>> 16); + y ^= y >>> 16; return y + 7; } @@ -15,9 +15,10 @@ public static long run() { for (int i = 0; i < N; i++) { list.add(i); } + return list.stream() - .map(x -> compute(x)) - .mapToLong(Integer::longValue) - .sum(); + .map(LoopComputationsStreamBoxedCase::compute) + .mapToLong(Integer::longValue) + .sum(); } } diff --git a/reproducer/plotting/examples/mean_difference.png b/reproducer/plotting/examples/mean_difference.png new file mode 100644 index 0000000..eb38d24 Binary files /dev/null and b/reproducer/plotting/examples/mean_difference.png differ diff --git a/reproducer/plotting/examples/native_code_size_by_run.png b/reproducer/plotting/examples/native_code_size_by_run.png new file mode 100644 index 0000000..a9b2fdf Binary files /dev/null and b/reproducer/plotting/examples/native_code_size_by_run.png differ diff --git a/reproducer/plotting/examples/run_difference.png b/reproducer/plotting/examples/run_difference.png new file mode 100644 index 0000000..62cfc5e Binary files /dev/null and b/reproducer/plotting/examples/run_difference.png differ diff --git a/reproducer/plotting/mean_difference.py b/reproducer/plotting/mean_difference.py new file mode 100644 index 0000000..9169409 --- /dev/null +++ b/reproducer/plotting/mean_difference.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Mean relative-difference chart for one reproducer case.""" + +from __future__ import annotations + +import argparse +import csv +import math +import os +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +MATPLOTLIB_CACHE = Path(tempfile.gettempdir()) / "comparator-matplotlib" +MATPLOTLIB_CACHE.mkdir(parents=True, exist_ok=True) +os.environ.setdefault("MPLCONFIGDIR", str(MATPLOTLIB_CACHE)) +os.environ.setdefault("XDG_CACHE_HOME", str(MATPLOTLIB_CACHE / "xdg")) + +try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.ticker import FuncFormatter +except ModuleNotFoundError as error: + MATPLOTLIB_IMPORT_ERROR: ModuleNotFoundError | None = error +else: + MATPLOTLIB_IMPORT_ERROR = None + + +REQUIRED_COLUMNS = ("role", "metric", "count", "mean", "stdev") +ROLES = ("baseline", "variant") + + +@dataclass(frozen=True) +class MetricSpec: + """Fixed reproducer metric included in the mean-difference chart.""" + + label: str + csv_name: str + + +@dataclass(frozen=True) +class RoleSummary: + """Aggregated measurement row for one metric role.""" + + role: str + count: int + mean: float + stdev: float + + @property + def cv_percent(self) -> float: + return self.stdev / self.mean * 100.0 + + +@dataclass(frozen=True) +class MetricSummary: + """Mean comparison for one metric across baseline and variant roles.""" + + spec: MetricSpec + baseline: RoleSummary + variant: RoleSummary + + @property + def count(self) -> int: + return self.baseline.count + + @property + def relative_difference_percent(self) -> float: + return (self.variant.mean - self.baseline.mean) / self.baseline.mean * 100.0 + + @property + def cv_label(self) -> str: + return ( + f"{self.spec.label}\n" + f"CV(orig): {self.baseline.cv_percent:.2f}%\n" + f"CV(ref): {self.variant.cv_percent:.2f}%" + ) + + +class PlottingError(RuntimeError): + """Fatal mean-difference plotting error.""" + + +class SummaryCsv: + """Validated summary.csv source for one reproducer case.""" + + def __init__(self, case_dir: Path) -> None: + self.case_dir = case_dir + self.path = case_dir / "summary.csv" + + def metric_summaries(self) -> list[MetricSummary]: + self._require_case_directory() + rows = self._rows() + summaries = [self._summary_for(metric, rows) for metric in METRICS] + self._require_same_count(summaries) + return summaries + + def _require_case_directory(self) -> None: + if not self.case_dir.is_dir(): + raise PlottingError(f"Case directory does not exist: {self.case_dir}") + if not self.path.is_file(): + raise PlottingError(f"Missing summary.csv: {self.path}") + + def _rows(self) -> list[dict[str, str]]: + with self.path.open(newline="", encoding="utf-8") as source: + reader = csv.DictReader(source) + if reader.fieldnames is None: + raise PlottingError(f"summary.csv is empty: {self.path}") + missing = [name for name in REQUIRED_COLUMNS if name not in reader.fieldnames] + if missing: + raise PlottingError( + f"summary.csv is missing required columns {', '.join(missing)}: {self.path}" + ) + return list(reader) + + def _summary_for(self, metric: MetricSpec, rows: list[dict[str, str]]) -> MetricSummary: + by_role = { + role: self._single_role_summary(metric, role, rows) + for role in ROLES + } + baseline = by_role["baseline"] + variant = by_role["variant"] + if baseline.count != variant.count: + raise PlottingError( + f"Inconsistent run count in summary.csv for metric {metric.csv_name}: {self.case_dir}" + ) + if baseline.mean == 0.0: + raise PlottingError( + f"Cannot compute relative difference because baseline mean is zero " + f"for metric {metric.csv_name}: {self.case_dir}" + ) + if variant.mean == 0.0: + raise PlottingError( + f"Cannot compute CV because variant mean is zero for metric {metric.csv_name}: {self.case_dir}" + ) + return MetricSummary(metric, baseline, variant) + + def _single_role_summary( + self, + metric: MetricSpec, + role: str, + rows: list[dict[str, str]], + ) -> RoleSummary: + matches = [ + row + for row in rows + if row.get("role") == role and row.get("metric") == metric.csv_name + ] + if len(matches) == 0: + raise PlottingError(f"Missing metric for role {role}: {metric.csv_name}: {self.case_dir}") + if len(matches) > 1: + raise PlottingError(f"Duplicate metric for role {role}: {metric.csv_name}: {self.case_dir}") + row = matches[0] + count = self._positive_count(row["count"], metric, role) + mean = self._finite_float(row["mean"], "mean", metric, role) + stdev = self._finite_float(row["stdev"], "stdev", metric, role) + return RoleSummary(role, count, mean, stdev) + + def _positive_count(self, raw: str, metric: MetricSpec, role: str) -> int: + value = self._finite_float(raw, "count", metric, role) + if not value.is_integer() or value < 1: + raise PlottingError( + f"Metric has invalid count for role {role}: {metric.csv_name}: {self.case_dir}" + ) + return int(value) + + def _finite_float(self, raw: str, field: str, metric: MetricSpec, role: str) -> float: + try: + value = float(raw) + except ValueError as error: + raise PlottingError( + f"Metric has non-numeric {field} for role {role}: {metric.csv_name}: {self.case_dir}" + ) from error + if not math.isfinite(value): + raise PlottingError( + f"Metric has non-finite {field} for role {role}: {metric.csv_name}: {self.case_dir}" + ) + return value + + def _require_same_count(self, summaries: list[MetricSummary]) -> None: + expected = summaries[0].count + for summary in summaries: + if summary.baseline.count != expected or summary.variant.count != expected: + raise PlottingError( + f"Inconsistent run count in summary.csv for metric {summary.spec.csv_name}: {self.case_dir}" + ) + + +class MeanDifferencePlot: + """PDF chart for mean relative differences in one reproducer case.""" + + def __init__(self, case_dir: Path, summaries: list[MetricSummary], output: Path) -> None: + self.case_dir = case_dir + self.summaries = summaries + self.output = output + + def save(self) -> None: + values = [summary.relative_difference_percent for summary in self.summaries] + labels = [summary.cv_label for summary in self.summaries] + colors = ["#1f9a7a" if value < 0.0 else "#c95f4a" for value in values] + + figure, axis = plt.subplots(figsize=(11.5, 7.0)) + x_positions = list(range(len(self.summaries))) + bars = axis.bar(x_positions, values, color=colors, width=0.62, edgecolor="none") + + axis.axhline(0.0, color="#4f6273", linewidth=1.5) + axis.grid(axis="y", color="#e5eaef", linewidth=1.0) + axis.set_axisbelow(True) + axis.set_ylabel("Difference, %", fontsize=13) + axis.set_xticks(x_positions) + axis.set_xticklabels(labels, fontsize=11, fontweight="bold") + axis.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{value:.0f}%")) + axis.tick_params(axis="x", length=0, pad=14) + axis.tick_params(axis="y", labelsize=11) + axis.margins(x=0.04) + + for side in ("top", "right", "left", "bottom"): + axis.spines[side].set_visible(False) + + lower, upper, label_offset = self._limits(values) + axis.set_ylim(lower, upper) + for bar, value in zip(bars, values): + y = value + label_offset if value >= 0.0 else value - label_offset + vertical_alignment = "bottom" if value >= 0.0 else "top" + axis.text( + bar.get_x() + bar.get_width() / 2.0, + y, + f"{value:.1f}%", + ha="center", + va=vertical_alignment, + fontsize=12, + fontweight="bold", + color="#1f2933", + ) + + count = self.summaries[0].count + title = ( + f"{self.case_dir.name}\n" + f"Metric difference between original and modified variant (N = {count})" + ) + axis.set_title(title, fontsize=16, pad=20) + figure.subplots_adjust(left=0.08, right=0.99, top=0.82, bottom=0.24) + + self.output.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(self.output, format="pdf", bbox_inches="tight") + plt.close(figure) + + def _limits(self, values: list[float]) -> tuple[float, float, float]: + minimum = min(values + [0.0]) + maximum = max(values + [0.0]) + span = maximum - minimum + if span == 0.0: + span = max(abs(maximum), 1.0) + + padding = span * 0.18 + lower = minimum - padding + upper = maximum + padding + + if minimum >= 0.0: + lower = -max(span * 0.15, 1.0) + if maximum <= 0.0: + upper = max(span * 0.15, 1.0) + + label_offset = max(span * 0.025, 0.25) + return lower, upper, label_offset + + +METRICS = ( + MetricSpec("JMH score", "JMH primary score, us/op"), + MetricSpec("Allocations", "Allocations, B/op"), + MetricSpec("Instructions", "Instructions, #/op"), + MetricSpec("Memory loads", "Memory loads, #/op"), + MetricSpec("Memory stores", "Memory stores, #/op"), +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Plot mean relative differences for one reproducer case.") + parser.add_argument("case_dir", help="Path to one reproducer case directory.") + parser.add_argument("--output", help="Output PDF path. Defaults to /mean_difference.pdf.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + case_dir = Path(args.case_dir) + output = Path(args.output) if args.output else case_dir / "mean_difference.pdf" + try: + if MATPLOTLIB_IMPORT_ERROR is not None: + raise PlottingError("Missing Python dependency: matplotlib") + summaries = SummaryCsv(case_dir).metric_summaries() + MeanDifferencePlot(case_dir, summaries, output).save() + return 0 + except PlottingError as error: + print(str(error), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/reproducer/plotting/metric_difference.py b/reproducer/plotting/metric_difference.py new file mode 100644 index 0000000..61f5758 --- /dev/null +++ b/reproducer/plotting/metric_difference.py @@ -0,0 +1,553 @@ +#!/usr/bin/env python3 +"""Raw per-run metric charts for one reproducer case.""" + +from __future__ import annotations + +import argparse +import csv +import math +import os +import statistics +import sys +import tempfile +import textwrap +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar + +MATPLOTLIB_CACHE = Path(tempfile.gettempdir()) / "comparator-matplotlib" +MATPLOTLIB_CACHE.mkdir(parents=True, exist_ok=True) +os.environ.setdefault("MPLCONFIGDIR", str(MATPLOTLIB_CACHE)) +os.environ.setdefault("XDG_CACHE_HOME", str(MATPLOTLIB_CACHE / "xdg")) + +try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.ticker import FuncFormatter, MaxNLocator +except ModuleNotFoundError as error: + MATPLOTLIB_IMPORT_ERROR: ModuleNotFoundError | None = error +else: + MATPLOTLIB_IMPORT_ERROR = None + + +REQUIRED_COLUMNS = ("role", "run_index") +ROLES = ("baseline", "variant") +ROLE_LABELS = {"baseline": "orig", "variant": "ref"} + + +class PlottingError(RuntimeError): + """Fatal raw metric plotting error.""" + + +@dataclass(frozen=True) +class MetricSpec: + """Fixed reproducer metric included in raw per-run charts.""" + + label: str + csv_name: str + y_label: str + output_name: str + value_kind: str + + +@dataclass(frozen=True) +class RoleSeries: + """Ordered raw values for one metric role.""" + + role: str + run_indices: tuple[int, ...] + values: tuple[float, ...] + + @property + def label(self) -> str: + return ROLE_LABELS[self.role] + + @property + def by_run_index(self) -> dict[int, float]: + return dict(zip(self.run_indices, self.values)) + + +@dataclass(frozen=True) +class MetricSeries: + """Raw baseline and variant series for one reproducer metric.""" + + spec: MetricSpec + baseline: RoleSeries + variant: RoleSeries + + @property + def run_indices(self) -> tuple[int, ...]: + return self.baseline.run_indices + + @property + def roles(self) -> tuple[RoleSeries, RoleSeries]: + return self.baseline, self.variant + + +@dataclass(frozen=True) +class MetricStatistics: + """Calculated stability summary for one metric role.""" + + spec: MetricSpec + series: RoleSeries + count: int + mean: float + stdev: float + cv_percent: float + value_range: float + distinct_values_count: int + frequency: tuple[tuple[float, int], ...] + MAX_FULL_FREQUENCY_ITEMS: ClassVar[int] = 12 + + @classmethod + def from_series( + cls, + spec: MetricSpec, + series: RoleSeries, + case_dir: Path, + ) -> MetricStatistics: + values = list(series.values) + mean = statistics.mean(values) + if mean == 0.0: + raise PlottingError( + f"Cannot compute CV because mean is zero for metric {spec.csv_name} " + f"role {series.role}: {case_dir}" + ) + stdev = statistics.stdev(values) if len(values) > 1 else 0.0 + counts = Counter(values) + return cls( + spec=spec, + series=series, + count=len(values), + mean=mean, + stdev=stdev, + cv_percent=stdev / mean * 100.0, + value_range=max(values) - min(values), + distinct_values_count=len(counts), + frequency=tuple(sorted(counts.items(), key=lambda item: item[0])), + ) + + def panel_text(self) -> str: + formatter = MetricValueFormatter(self.spec) + return "\n".join( + ( + self.series.label, + f"CV: {self.cv_percent:.2f}%", + f"Range: {formatter.with_unit(self.value_range)}", + f"Distinct values: {self.distinct_values_count}", + f"Mean: {formatter.with_unit(self.mean)}", + f"Freq: {self._frequency_text(formatter)}", + ) + ) + + def _frequency_text(self, formatter: MetricValueFormatter) -> str: + if len(self.frequency) > self.MAX_FULL_FREQUENCY_ITEMS: + if all(count == 1 for _, count in self.frequency): + first = formatter.compact(self.frequency[0][0]) + last = formatter.compact(self.frequency[-1][0]) + return f"{len(self.frequency)} unique values ({first}..{last})" + return self._abbreviated_frequency_text(formatter) + items = [ + f"{formatter.compact(value)}x{count}" + for value, count in self.frequency + ] + return ", ".join(items) + + def _abbreviated_frequency_text(self, formatter: MetricValueFormatter) -> str: + head_size = 6 + tail_size = 3 + head = self.frequency[:head_size] + tail = self.frequency[-tail_size:] + omitted = len(self.frequency) - head_size - tail_size + head_text = [f"{formatter.compact(value)}x{count}" for value, count in head] + tail_text = [f"{formatter.compact(value)}x{count}" for value, count in tail] + return ", ".join([*head_text, f"... +{omitted} more", *tail_text]) + + +class MetricValueFormatter: + """Compact numeric formatter for one metric unit.""" + + SI_UNITS = ( + (1_000_000_000.0, "G"), + (1_000_000.0, "M"), + (1_000.0, "K"), + ) + + def __init__(self, spec: MetricSpec) -> None: + self.spec = spec + + def axis_tick(self, value: float) -> str: + number, suffix = self._scaled(value) + if self.spec.value_kind in ("bytes", "bytes_per_op"): + return f"{number}{suffix}B" + return f"{number}{suffix}" + + def with_unit(self, value: float) -> str: + number, suffix = self._scaled(value) + if self.spec.value_kind == "bytes": + return f"{number} {suffix}B" + if self.spec.value_kind == "bytes_per_op": + return f"{number} {suffix}B/op" + if self.spec.value_kind == "jmh": + return f"{number} us/op" + return f"{number} #/op" + + def compact(self, value: float) -> str: + number, suffix = self._scaled(value) + return f"{number}{suffix}" + + def _scaled(self, value: float) -> tuple[str, str]: + absolute = abs(value) + for threshold, suffix in self.SI_UNITS: + if absolute >= threshold: + return self._plain(value / threshold), suffix + return self._plain(value), "" + + def _plain(self, value: float) -> str: + if math.isclose(value, round(value), rel_tol=0.0, abs_tol=1e-9): + return str(int(round(value))) + formatted = f"{value:.4g}" + if "e" in formatted or "E" in formatted: + return formatted + return formatted.rstrip("0").rstrip(".") + + +class AllRunsCsv: + """Validated all_runs.csv source for one reproducer case.""" + + def __init__(self, case_dir: Path) -> None: + self.case_dir = case_dir + self.path = case_dir / "all_runs.csv" + + def metric_series(self) -> list[MetricSeries]: + self._require_case_directory() + rows, fieldnames = self._rows() + self._require_metric_columns(fieldnames) + self._require_roles(rows) + return [self._series_for(metric, rows) for metric in METRICS] + + def _require_case_directory(self) -> None: + if not self.case_dir.is_dir(): + raise PlottingError(f"Case directory does not exist: {self.case_dir}") + if not self.path.is_file(): + raise PlottingError(f"Missing all_runs.csv: {self.path}") + + def _rows(self) -> tuple[list[dict[str, str]], list[str]]: + with self.path.open(newline="", encoding="utf-8") as source: + reader = csv.DictReader(source) + if reader.fieldnames is None: + raise PlottingError(f"all_runs.csv is empty: {self.path}") + missing = [name for name in REQUIRED_COLUMNS if name not in reader.fieldnames] + if missing: + raise PlottingError( + f"all_runs.csv is missing required columns {', '.join(missing)}: {self.path}" + ) + return list(reader), list(reader.fieldnames) + + def _require_metric_columns(self, fieldnames: list[str]) -> None: + for metric in METRICS: + if metric.csv_name not in fieldnames: + raise PlottingError(f"Missing metric column: {metric.csv_name}: {self.case_dir}") + + def _require_roles(self, rows: list[dict[str, str]]) -> None: + roles = {row.get("role", "") for row in rows} + for role in ROLES: + if role not in roles: + raise PlottingError(f"Missing role {role} in all_runs.csv: {self.case_dir}") + + def _series_for(self, metric: MetricSpec, rows: list[dict[str, str]]) -> MetricSeries: + by_role = {role: self._role_series(metric, role, rows) for role in ROLES} + baseline = by_role["baseline"] + variant = by_role["variant"] + self._require_matching_run_indices(metric, baseline, variant) + return MetricSeries(metric, baseline, variant) + + def _role_series( + self, + metric: MetricSpec, + role: str, + rows: list[dict[str, str]], + ) -> RoleSeries: + values: dict[int, float] = {} + for row_number, row in enumerate(rows, start=2): + if row.get("role") != role: + continue + run_index = self._positive_run_index(row.get("run_index", ""), metric, role, row_number) + if run_index in values: + raise PlottingError( + f"Duplicate value for metric {metric.csv_name} " + f"role {role} at run {run_index}: {self.case_dir}" + ) + values[run_index] = self._finite_float(row[metric.csv_name], metric, role, run_index) + + if not values: + raise PlottingError( + f"Missing values for metric {metric.csv_name} role {role}: {self.case_dir}" + ) + run_indices = tuple(sorted(values)) + return RoleSeries(role, run_indices, tuple(values[index] for index in run_indices)) + + def _positive_run_index( + self, + raw: str, + metric: MetricSpec, + role: str, + row_number: int, + ) -> int: + try: + value = int(raw) + except ValueError as error: + raise PlottingError( + f"Invalid run_index for metric {metric.csv_name} " + f"role {role} at CSV row {row_number}: {raw!r}: {self.case_dir}" + ) from error + if value < 1: + raise PlottingError( + f"Invalid run_index for metric {metric.csv_name} " + f"role {role} at CSV row {row_number}: {raw!r}: {self.case_dir}" + ) + return value + + def _finite_float(self, raw: str, metric: MetricSpec, role: str, run_index: int) -> float: + try: + value = float(raw) + except ValueError as error: + raise PlottingError( + f"Metric has non-numeric value for role {role} " + f"at run {run_index}: {metric.csv_name}: {self.case_dir}" + ) from error + if not math.isfinite(value): + raise PlottingError( + f"Metric has non-finite value for role {role} " + f"at run {run_index}: {metric.csv_name}: {self.case_dir}" + ) + return value + + def _require_matching_run_indices( + self, + metric: MetricSpec, + baseline: RoleSeries, + variant: RoleSeries, + ) -> None: + baseline_runs = set(baseline.run_indices) + variant_runs = set(variant.run_indices) + if baseline_runs != variant_runs: + raise PlottingError( + f"Run index mismatch for metric {metric.csv_name}: " + f"baseline={sorted(baseline_runs)}, variant={sorted(variant_runs)}: {self.case_dir}" + ) + + +class MetricDifferencePlot: + """PDF line chart for raw values of one reproducer metric.""" + + COLORS = {"baseline": "#3f7ee8", "variant": "#dd7433"} + MARKERS = {"baseline": "o", "variant": "s"} + + def __init__(self, case_dir: Path, metric: MetricSeries, output: Path) -> None: + self.case_dir = case_dir + self.metric = metric + self.output = output + + def save(self) -> None: + figure, axis = plt.subplots(figsize=(14.8, 7.6)) + formatter = MetricValueFormatter(self.metric.spec) + statistics_by_role = [ + MetricStatistics.from_series(self.metric.spec, series, self.case_dir) + for series in self.metric.roles + ] + + for series, stats in zip(self.metric.roles, statistics_by_role): + color = self.COLORS[series.role] + axis.plot( + series.run_indices, + series.values, + label=series.label, + color=color, + marker=self.MARKERS[series.role], + linewidth=2.2, + markersize=6.8, + markeredgecolor="white", + markeredgewidth=1.1, + ) + axis.axhline( + stats.mean, + color=color, + linestyle=(0, (4, 4)), + linewidth=1.6, + alpha=0.72, + ) + + axis.grid(axis="y", color="#e5eaef", linewidth=1.0) + axis.grid(axis="x", color="#edf2f7", linewidth=0.8) + axis.set_axisbelow(True) + axis.set_xlabel("Run index", fontsize=13) + axis.set_ylabel(self.metric.spec.y_label, fontsize=13) + axis.yaxis.set_major_formatter(FuncFormatter(lambda value, _: formatter.axis_tick(value))) + axis.xaxis.set_major_locator(MaxNLocator(integer=True)) + axis.tick_params(axis="both", labelsize=11, colors="#4f6273") + axis.margins(x=0.02) + + for side in ("top", "right", "left", "bottom"): + axis.spines[side].set_visible(False) + + axis.set_ylim(*self._limits()) + axis.legend( + loc="upper left", + bbox_to_anchor=(1.025, 1.0), + frameon=False, + fontsize=12, + borderaxespad=0.0, + handlelength=2.6, + ) + axis.text( + 1.025, + 0.84, + self._panel_text(statistics_by_role), + transform=axis.transAxes, + ha="left", + va="top", + fontsize=10.8, + color="#34495e", + linespacing=1.35, + bbox={ + "boxstyle": "round,pad=1.0", + "facecolor": "white", + "edgecolor": "#dce3eb", + "linewidth": 1.0, + }, + ) + + title = ( + f"{self.case_dir.name}\n" + f"Metric \"{self.metric.spec.label}\" value for each experiment" + ) + axis.set_title(title, fontsize=16, pad=18) + figure.subplots_adjust(left=0.08, right=0.72, top=0.84, bottom=0.13) + + self.output.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(self.output, format="pdf", bbox_inches="tight") + plt.close(figure) + + def _limits(self) -> tuple[float, float]: + values = [value for series in self.metric.roles for value in series.values] + minimum = min(values) + maximum = max(values) + span = maximum - minimum + if span == 0.0: + span = max(abs(maximum), 1.0) + padding = span * 0.10 + return minimum - padding, maximum + padding + + def _panel_text(self, statistics_by_role: list[MetricStatistics]) -> str: + sections = [stats.panel_text() for stats in statistics_by_role] + wrapped = [self._wrap_frequency(section) for section in sections] + return "\n\n".join(wrapped) + + def _wrap_frequency(self, section: str) -> str: + lines = [] + for line in section.splitlines(): + if not line.startswith("Freq: "): + lines.append(line) + continue + wrapped = textwrap.wrap( + line, + width=38, + subsequent_indent=" ", + break_long_words=False, + break_on_hyphens=False, + ) + lines.extend(wrapped) + return "\n".join(lines) + + +METRICS = ( + MetricSpec( + "JMH score", + "JMH primary score, us/op", + "JMH score, us/op", + "jmh_score_by_run.pdf", + "jmh", + ), + MetricSpec( + "Allocations", + "Allocations, B/op", + "Allocations, B/op", + "allocations_by_run.pdf", + "bytes_per_op", + ), + MetricSpec( + "Instructions", + "Instructions, #/op", + "Instructions, #/op", + "instructions_by_run.pdf", + "number_per_op", + ), + MetricSpec( + "Memory loads", + "Memory loads, #/op", + "Memory loads, #/op", + "memory_loads_by_run.pdf", + "number_per_op", + ), + MetricSpec( + "Memory stores", + "Memory stores, #/op", + "Memory stores, #/op", + "memory_stores_by_run.pdf", + "number_per_op", + ), + MetricSpec( + "Native code size", + "Native code size, B", + "Native code size, B", + "native_code_size_by_run.pdf", + "bytes", + ), +) + + +def load_metric_series(case_dir: Path) -> list[MetricSeries]: + return AllRunsCsv(case_dir).metric_series() + + +def plot_metric(case_dir: Path, metric: MetricSeries, output: Path) -> None: + MetricDifferencePlot(case_dir, metric, output).save() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Plot raw per-run metric values for one reproducer case." + ) + parser.add_argument("case_dir", help="Path to one reproducer case directory.") + parser.add_argument( + "--output-dir", + help="Output directory. Defaults to /metric_difference/.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + case_dir = Path(args.case_dir) + output_dir = Path(args.output_dir) if args.output_dir else case_dir / "metric_difference" + try: + if MATPLOTLIB_IMPORT_ERROR is not None: + raise PlottingError("Missing Python dependency: matplotlib") + metrics = load_metric_series(case_dir) + for metric in metrics: + for series in metric.roles: + MetricStatistics.from_series(metric.spec, series, case_dir) + for metric in metrics: + plot_metric(case_dir, metric, output_dir / metric.spec.output_name) + return 0 + except PlottingError as error: + print(str(error), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/reproducer/plotting/run_difference.py b/reproducer/plotting/run_difference.py new file mode 100644 index 0000000..aa6d4a0 --- /dev/null +++ b/reproducer/plotting/run_difference.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""Per-run relative-difference chart for one reproducer case.""" + +from __future__ import annotations + +import argparse +import csv +import math +import os +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +MATPLOTLIB_CACHE = Path(tempfile.gettempdir()) / "comparator-matplotlib" +MATPLOTLIB_CACHE.mkdir(parents=True, exist_ok=True) +os.environ.setdefault("MPLCONFIGDIR", str(MATPLOTLIB_CACHE)) +os.environ.setdefault("XDG_CACHE_HOME", str(MATPLOTLIB_CACHE / "xdg")) + +try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.ticker import FuncFormatter, MaxNLocator +except ModuleNotFoundError as error: + MATPLOTLIB_IMPORT_ERROR: ModuleNotFoundError | None = error +else: + MATPLOTLIB_IMPORT_ERROR = None + + +REQUIRED_COLUMNS = ("role", "run_index") +ROLES = ("baseline", "variant") + + +@dataclass(frozen=True) +class MetricSpec: + """Fixed reproducer metric included in the run-difference chart.""" + + label: str + csv_name: str + color: str + marker: str + + +@dataclass(frozen=True) +class MetricRunDifferences: + """Ordered relative differences for one metric across reproducer runs.""" + + spec: MetricSpec + run_indices: tuple[int, ...] + percentages: tuple[float, ...] + + +class PlottingError(RuntimeError): + """Fatal run-difference plotting error.""" + + +class AllRunsCsv: + """Validated all_runs.csv source for one reproducer case.""" + + def __init__(self, case_dir: Path) -> None: + self.case_dir = case_dir + self.path = case_dir / "all_runs.csv" + + def run_differences(self) -> list[MetricRunDifferences]: + self._require_case_directory() + rows = self._rows() + self._require_roles(rows) + return [self._differences_for(metric, rows) for metric in METRICS] + + def _require_case_directory(self) -> None: + if not self.case_dir.is_dir(): + raise PlottingError(f"Case directory does not exist: {self.case_dir}") + if not self.path.is_file(): + raise PlottingError(f"Missing all_runs.csv: {self.path}") + + def _rows(self) -> list[dict[str, str]]: + with self.path.open(newline="", encoding="utf-8") as source: + reader = csv.DictReader(source) + if reader.fieldnames is None: + raise PlottingError(f"all_runs.csv is empty: {self.path}") + missing_required = [name for name in REQUIRED_COLUMNS if name not in reader.fieldnames] + if missing_required: + raise PlottingError( + f"all_runs.csv is missing required columns " + f"{', '.join(missing_required)}: {self.path}" + ) + for metric in METRICS: + if metric.csv_name not in reader.fieldnames: + raise PlottingError( + f"Missing metric column: {metric.csv_name}: {self.case_dir}" + ) + return list(reader) + + def _require_roles(self, rows: list[dict[str, str]]) -> None: + roles = {row.get("role", "") for row in rows} + for role in ROLES: + if role not in roles: + raise PlottingError(f"Missing role {role} in all_runs.csv: {self.case_dir}") + + def _differences_for( + self, + metric: MetricSpec, + rows: list[dict[str, str]], + ) -> MetricRunDifferences: + by_role = { + role: self._values_for(metric, role, rows) + for role in ROLES + } + baseline = by_role["baseline"] + variant = by_role["variant"] + self._require_matching_run_indices(metric, baseline, variant) + + run_indices = tuple(sorted(baseline)) + percentages = tuple( + self._relative_difference(metric, run_index, baseline[run_index], variant[run_index]) + for run_index in run_indices + ) + return MetricRunDifferences(metric, run_indices, percentages) + + def _values_for( + self, + metric: MetricSpec, + role: str, + rows: list[dict[str, str]], + ) -> dict[int, float]: + values: dict[int, float] = {} + for row_number, row in enumerate(rows, start=2): + if row.get("role") != role: + continue + + run_index = self._positive_run_index(row.get("run_index", ""), metric, role, row_number) + if run_index in values: + raise PlottingError( + f"Duplicate value for metric {metric.csv_name} " + f"role {role} at run {run_index}: {self.case_dir}" + ) + values[run_index] = self._finite_float(row[metric.csv_name], metric, role, run_index) + + if not values: + raise PlottingError( + f"Missing values for metric {metric.csv_name} role {role}: {self.case_dir}" + ) + return values + + def _positive_run_index( + self, + raw: str, + metric: MetricSpec, + role: str, + row_number: int, + ) -> int: + try: + value = int(raw) + except ValueError as error: + raise PlottingError( + f"Invalid run_index for metric {metric.csv_name} " + f"role {role} at CSV row {row_number}: {raw!r}: {self.case_dir}" + ) from error + if value < 1: + raise PlottingError( + f"Invalid run_index for metric {metric.csv_name} " + f"role {role} at CSV row {row_number}: {raw!r}: {self.case_dir}" + ) + return value + + def _finite_float(self, raw: str, metric: MetricSpec, role: str, run_index: int) -> float: + try: + value = float(raw) + except ValueError as error: + raise PlottingError( + f"Metric has non-numeric value for role {role} " + f"at run {run_index}: {metric.csv_name}: {self.case_dir}" + ) from error + if not math.isfinite(value): + raise PlottingError( + f"Metric has non-finite value for role {role} " + f"at run {run_index}: {metric.csv_name}: {self.case_dir}" + ) + return value + + def _require_matching_run_indices( + self, + metric: MetricSpec, + baseline: dict[int, float], + variant: dict[int, float], + ) -> None: + baseline_runs = set(baseline) + variant_runs = set(variant) + if baseline_runs != variant_runs: + raise PlottingError( + f"Run index mismatch for metric {metric.csv_name}: " + f"baseline={sorted(baseline_runs)}, variant={sorted(variant_runs)}: {self.case_dir}" + ) + + def _relative_difference( + self, + metric: MetricSpec, + run_index: int, + baseline_value: float, + variant_value: float, + ) -> float: + if baseline_value == 0.0: + raise PlottingError( + f"Cannot compute relative difference because baseline value is zero " + f"for metric {metric.csv_name} at run {run_index}: {self.case_dir}" + ) + return (variant_value - baseline_value) / baseline_value * 100.0 + + +class RunDifferencePlot: + """PDF line chart for per-run relative differences in one reproducer case.""" + + def __init__( + self, + case_dir: Path, + differences: list[MetricRunDifferences], + output: Path, + ) -> None: + self.case_dir = case_dir + self.differences = differences + self.output = output + + def save(self) -> None: + figure, axis = plt.subplots(figsize=(13.8, 7.2)) + + for difference in self.differences: + axis.plot( + difference.run_indices, + difference.percentages, + label=difference.spec.label, + color=difference.spec.color, + marker=difference.spec.marker, + linewidth=2.2, + markersize=6.8, + markeredgecolor="white", + markeredgewidth=1.1, + ) + + axis.axhline(0.0, color="#b8c2cc", linewidth=1.3) + axis.grid(axis="y", color="#e5eaef", linewidth=1.0) + axis.grid(axis="x", color="#edf2f7", linewidth=0.8) + axis.set_axisbelow(True) + axis.set_xlabel("Run index", fontsize=13) + axis.set_ylabel("Difference, %", fontsize=13) + axis.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{value:.0f}%")) + axis.xaxis.set_major_locator(MaxNLocator(integer=True)) + axis.tick_params(axis="both", labelsize=11, colors="#4f6273") + axis.margins(x=0.02) + + for side in ("top", "right", "left", "bottom"): + axis.spines[side].set_visible(False) + + axis.set_ylim(*self._limits()) + axis.legend( + loc="upper left", + bbox_to_anchor=(1.02, 1.0), + frameon=False, + fontsize=12, + borderaxespad=0.0, + handlelength=2.6, + ) + + title = ( + f"{self.case_dir.name}\n" + "Metric difference between original " + "and modified variant " + "for each experiment" + ) + axis.set_title(title, fontsize=16, pad=18) + figure.subplots_adjust(left=0.08, right=0.78, top=0.84, bottom=0.13) + + self.output.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(self.output, format="pdf", bbox_inches="tight") + plt.close(figure) + + def _limits(self) -> tuple[float, float]: + values = [ + value + for difference in self.differences + for value in difference.percentages + ] + minimum = min(values + [0.0]) + maximum = max(values + [0.0]) + span = maximum - minimum + if span == 0.0: + span = max(abs(maximum), 1.0) + + padding = max(span * 0.12, 1.0) + lower = minimum - padding + upper = maximum + padding + + if minimum >= 0.0: + lower = -padding + if maximum <= 0.0: + upper = padding + return lower, upper + + +METRICS = ( + MetricSpec("JMH score", "JMH primary score, us/op", "#3f7ee8", "o"), + MetricSpec("Allocations", "Allocations, B/op", "#dd7433", "o"), + MetricSpec("Instructions", "Instructions, #/op", "#249b68", "o"), + MetricSpec("Memory loads", "Memory loads, #/op", "#9b5de5", "o"), + MetricSpec("Memory stores", "Memory stores, #/op", "#d94f83", "o"), + MetricSpec("Native code size", "Native code size, B", "#8a6f3f", "o"), +) + + +def load_run_differences(case_dir: Path) -> list[MetricRunDifferences]: + return AllRunsCsv(case_dir).run_differences() + + +def plot_run_differences( + case_dir: Path, + differences: list[MetricRunDifferences], + output: Path, +) -> None: + RunDifferencePlot(case_dir, differences, output).save() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Plot per-run relative differences for one reproducer case." + ) + parser.add_argument("case_dir", help="Path to one reproducer case directory.") + parser.add_argument( + "--output", + help="Output PDF path. Defaults to /run_difference.pdf.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + case_dir = Path(args.case_dir) + output = Path(args.output) if args.output else case_dir / "run_difference.pdf" + try: + if MATPLOTLIB_IMPORT_ERROR is not None: + raise PlottingError("Missing Python dependency: matplotlib") + differences = load_run_differences(case_dir) + plot_run_differences(case_dir, differences, output) + return 0 + except PlottingError as error: + print(str(error), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/reproducer/run.py b/reproducer/run.py new file mode 100644 index 0000000..56a5274 --- /dev/null +++ b/reproducer/run.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Curated JIT instability reproducer runner.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import platform +import re +import shutil +import statistics +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + + +MANIFEST_HEADER = [ + "case_id", + "role", + "classpath", + "class_name", + "method_name", + "jit_log", + "jmh_result", + "label", +] + + +@dataclass(frozen=True) +class CaseVariant: + """Variant source for one reproducer case role.""" + + role: str + source: Path + + +@dataclass(frozen=True) +class Case: + """Curated reproducer case.""" + + case_id: str + baseline_source: Path + variants: tuple[CaseVariant, ...] + class_name: str + method_name: str = "run" + + def roles(self) -> list[str]: + return ["baseline"] + [variant.role for variant in self.variants] + + +class ReproducerError(RuntimeError): + """Fatal reproducer error.""" + + +def main() -> int: + args = parse_args() + repo_root = Path(__file__).resolve().parents[1] + cases_root = resolve_root(repo_root, args.cases_root) + runs_root = resolve_root(repo_root, args.runs_root) + try: + require_positive_runs(args.runs) + preflight_perf() + cases = selected_cases(discover_cases(cases_root), include_prefixes(args.include_cases)) + session = Session(repo_root, cases_root, runs_root, args.runs, args.include_cases, cases, args.session_id) + session.run() + return 0 + except ReproducerError as error: + print(str(error), file=sys.stderr) + return 1 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run curated JIT instability reproducer cases.") + parser.add_argument("--runs", type=int, required=True) + parser.add_argument("--include-cases") + parser.add_argument("--session-id") + parser.add_argument("--cases-root", default="reproducer/cases") + parser.add_argument("--runs-root", default="reproducer/runs") + return parser.parse_args() + + +def resolve_root(repo_root: Path, raw: str) -> Path: + path = Path(raw) + if path.is_absolute(): + return path + return repo_root / path + + +def require_positive_runs(runs: int) -> None: + if runs < 1: + raise ReproducerError("--runs must be a positive integer") + + +def now() -> str: + return datetime.now().astimezone().isoformat(timespec="seconds") + + +def session_id() -> str: + return datetime.now().strftime("%Y%m%d_%H%M%S") + + +def log(message: str) -> None: + print(f"[{now()}] {message}", flush=True) + + +def run_command(command: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, capture_output=True, text=True, check=False) + + +def combined_output(process: subprocess.CompletedProcess[str]) -> str: + return process.stdout + process.stderr + + +def command_path(name: str) -> str: + resolved = shutil.which(name) + if resolved is None: + return name + return resolved + + +def classpath_line(line: str) -> bool: + return line.startswith("/") and "build/classes/java/main" in line + + +def preflight_perf() -> None: + version = run_command(["perf", "--version"]) + if version.returncode != 0: + raise ReproducerError( + "perf is required but was not found in PATH.\nInstall Linux perf and rerun reproducer." + ) + probe = run_command(["perf", "stat", "-e", "instructions", "--", "sleep", "0.1"]) + if probe.returncode != 0: + raise ReproducerError( + "perf is installed but cannot collect required events.\n" + "Tried: perf stat -e instructions -- sleep 0.1\n" + "This usually means perf_event_paranoid is too restrictive or the environment does not expose perf " + "counters.\nFix Linux perf permissions, then rerun reproducer." + ) + + +def include_prefixes(raw: str | None) -> list[str]: + if raw is None: + return [] + prefixes = [part.strip() for part in raw.split(",")] + if any(part == "" for part in prefixes): + raise ReproducerError("--include-cases contains an empty item") + return prefixes + + +def discover_cases(cases_root: Path) -> list[Case]: + if not cases_root.is_dir(): + raise ReproducerError(f"Cases root does not exist: {cases_root}") + cases = [case_from_directory(path) for path in sorted(cases_root.iterdir()) if path.is_dir()] + if not cases: + raise ReproducerError(f"No cases found in {cases_root}") + return cases + + +def case_from_directory(path: Path) -> Case: + baseline = single_java_file(path / "baseline", path.name, "baseline") + variants = variant_sources(path, baseline.name) + source_contract(baseline) + for variant in variants: + source_contract(variant.source) + return Case(path.name, baseline, tuple(variants), baseline.stem) + + +def variant_sources(case_root: Path, baseline_name: str) -> list[CaseVariant]: + variants: list[CaseVariant] = [] + legacy = case_root / "variant" + if legacy.exists(): + variants.append(CaseVariant("variant", matching_java_file(legacy, case_root.name, "variant", baseline_name))) + modern = case_root / "variants" + if modern.exists(): + if not modern.is_dir(): + raise ReproducerError(f"Case {case_root.name} variants path is not a directory: {modern}") + for role_dir in sorted(path for path in modern.iterdir() if path.is_dir()): + role = role_dir.name + require_role(role, case_root.name) + variants.append( + CaseVariant(role, matching_java_file(role_dir, case_root.name, role, baseline_name)) + ) + if not variants: + raise ReproducerError(f"Case {case_root.name} must contain variant/ or variants// directories") + roles = [variant.role for variant in variants] + if len(roles) != len(set(roles)): + raise ReproducerError(f"Case {case_root.name} contains duplicate variant roles: {','.join(roles)}") + return variants + + +def matching_java_file(directory: Path, case_id: str, role: str, baseline_name: str) -> Path: + source = single_java_file(directory, case_id, role) + if source.name != baseline_name: + raise ReproducerError( + f"Case {case_id} role {role} class file must match baseline file name: {baseline_name}" + ) + return source + + +def require_role(role: str, case_id: str) -> None: + if not re.fullmatch(r"[a-z][a-z0-9_]*", role): + raise ReproducerError(f"Case {case_id} variant role must be a lowercase identifier: {role}") + if role == "baseline": + raise ReproducerError(f"Case {case_id} variant role is reserved: {role}") + + +def single_java_file(directory: Path, case_id: str, role: str) -> Path: + if not directory.is_dir(): + raise ReproducerError(f"Case {case_id} is missing {role} directory: {directory}") + files = sorted(directory.glob("*.java")) + if len(files) != 1: + raise ReproducerError(f"Case {case_id} {role} must contain exactly one .java file") + return files[0] + + +def source_contract(source: Path) -> None: + text = source.read_text(encoding="utf-8") + if re.search(r"^\s*package\s+", text, flags=re.MULTILINE): + raise ReproducerError(f"Source must not declare a package: {source}") + class_pattern = r"\bclass\s+" + re.escape(source.stem) + r"\b" + if re.search(class_pattern, text) is None: + raise ReproducerError(f"Class name must match file name: {source}") + if re.search(r"\bstatic\b[\s\S]*?\brun\s*\(\s*\)", text) is None: + raise ReproducerError(f"Source must expose a no-argument static run method: {source}") + + +def selected_cases(cases: list[Case], prefixes: list[str]) -> list[Case]: + if not prefixes: + return sorted(cases, key=lambda case: case.case_id) + selected: dict[str, Case] = {} + for prefix in prefixes: + matches = [case for case in cases if case.case_id.startswith(prefix)] + if not matches: + raise ReproducerError(f"--include-cases item matches no cases: {prefix}") + for case in matches: + selected[case.case_id] = case + return [selected[key] for key in sorted(selected)] + + +class Session: + """One reproducer execution session.""" + + def __init__( + self, + repo_root: Path, + cases_root: Path, + runs_root: Path, + runs: int, + include_cases: str | None, + cases: list[Case], + requested_session_id: str | None, + ) -> None: + self.repo_root = repo_root + self.cases_root = cases_root + self.runs_root = runs_root + self.runs = runs + self.include_cases = include_prefixes(include_cases) + self.cases = cases + self.session_id = requested_session_id or session_id() + self.root = runs_root / self.session_id + self.index_rows: list[dict[str, str]] = [] + self.metadata = self.initial_metadata() + + def run(self) -> None: + self.root.mkdir(parents=True, exist_ok=False) + self.write_metadata() + try: + runtime_classpath = self.runtime_classpath() + self.metadata["gradle"]["runtime_classpath"] = runtime_classpath + self.write_case_metadata() + for case in self.cases: + self.run_case(case, runtime_classpath) + aggregate_case(self.root, self.session_id, case) + self.metadata["finished_at"] = now() + self.write_index() + self.write_metadata() + self.update_latest() + except ReproducerError: + self.metadata["finished_at"] = now() + self.write_index() + self.write_metadata() + raise + + def initial_metadata(self) -> dict: + return { + "session_id": self.session_id, + "started_at": now(), + "finished_at": "", + "repo_root": str(self.repo_root), + "cases_root": str(self.cases_root), + "runs_root": str(self.runs_root), + "runs": self.runs, + "include_cases": self.include_cases, + "selected_cases": [case.case_id for case in self.cases], + "tools": tools_metadata(), + "environment": environment_metadata(), + "git": git_metadata(self.repo_root), + "gradle": { + "classpath_command": ["./gradlew", "classes", "printRuntimeClasspath"], + "runtime_classpath": "", + "command_output": "", + }, + } + + def runtime_classpath(self) -> str: + command = ["./gradlew", "classes", "printRuntimeClasspath"] + process = run_command(command, self.repo_root) + self.metadata["gradle"]["command_output"] = combined_output(process) + self.write_metadata() + if process.returncode != 0: + raise ReproducerError("Gradle runtime classpath command failed:\n" + combined_output(process)) + lines = [line.strip() for line in process.stdout.splitlines() if classpath_line(line.strip())] + if not lines: + raise ReproducerError("Gradle runtime classpath command produced no classpath") + return lines[-1] + + def write_case_metadata(self) -> None: + for case in self.cases: + case_root = self.root / "cases" / case.case_id + case_root.mkdir(parents=True, exist_ok=True) + write_json( + case_root / "case_metadata.json", + { + "case_id": case.case_id, + "baseline_source": str(case.baseline_source), + "variants": [ + {"role": variant.role, "source": str(variant.source)} + for variant in case.variants + ], + "class_name": case.class_name, + "method_name": case.method_name, + "runs": self.runs, + }, + ) + + def run_case(self, case: Case, runtime_classpath: str) -> None: + for index in range(1, self.runs + 1): + run_name = f"run-{index:03d}" + run_root = self.root / "cases" / case.case_id / "runs" / run_name + try: + self.run_once(case, index, run_name, run_root, runtime_classpath) + self.index_rows.append(index_row(self.session_id, case, index, run_name, "success", run_root)) + except ReproducerError: + self.index_rows.append(index_row(self.session_id, case, index, run_name, "failed", run_root)) + raise + + def run_once(self, case: Case, index: int, run_name: str, run_root: Path, runtime_classpath: str) -> None: + started = now() + run_root.mkdir(parents=True, exist_ok=True) + paths = RunPaths(run_root) + try: + log(f"Starting {case.case_id} {run_name}") + paths.create(case.roles()) + compile_source( + case.baseline_source, paths.classes("baseline"), case.class_name, "compile-baseline", paths + ) + for variant in case.variants: + compile_source( + variant.source, paths.classes(variant.role), case.class_name, f"compile-{variant.role}", paths + ) + write_manifest(case, paths) + log(f"Starting comparison for {case.case_id} {run_name}") + compare(case, paths, runtime_classpath, self.repo_root) + log(f"Finished comparison for {case.case_id} {run_name}") + write_json(paths.status, {"status": "success", "stage": "done", "started_at": started, "finished_at": now()}) + except RunFailure as failure: + log(f"Failed {case.case_id} {run_name} at {failure.stage}") + write_json(paths.status, failure.status(started)) + raise ReproducerError(f"Case {case.case_id} {run_name} failed at {failure.stage}") from failure + + def write_index(self) -> None: + write_csv( + self.root / "index.csv", + ["session_id", "case_id", "run_index", "run_name", "status", "comparisons_csv"], + self.index_rows, + ) + + def write_metadata(self) -> None: + write_json(self.root / "metadata.json", self.metadata) + + def update_latest(self) -> None: + latest = self.runs_root / "latest" + try: + if latest.is_symlink() or latest.exists(): + latest.unlink() + latest.symlink_to(self.root, target_is_directory=True) + except OSError as error: + print(f"Warning: unable to update latest symlink: {error}", file=sys.stderr) + + +class RunFailure(Exception): + """Failed run stage with captured command output.""" + + def __init__(self, stage: str, command: list[str], process: subprocess.CompletedProcess[str]) -> None: + super().__init__(stage) + self.stage = stage + self.command = command + self.process = process + + def status(self, started: str) -> dict: + return { + "status": "failed", + "stage": self.stage, + "command": self.command, + "exit_code": self.process.returncode, + "stdout": self.process.stdout, + "stderr": self.process.stderr, + "started_at": started, + "finished_at": now(), + } + + +class RunPaths: + """Filesystem paths for one case run.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.status = root / "status.json" + self.logs = root / "logs" + self.pairs = root / "pairs.csv" + self.comparisons = root / "comparisons.csv" + + def classes(self, role: str) -> Path: + return self.root / "classes" / role + + def artifacts(self, role: str) -> Path: + return self.root / "artifacts" / role + + def create(self, roles: list[str]) -> None: + for role in roles: + self.classes(role).mkdir(parents=True, exist_ok=True) + self.artifacts(role).mkdir(parents=True, exist_ok=True) + for directory in [self.logs]: + directory.mkdir(parents=True, exist_ok=True) + + +def compile_source(source: Path, classes: Path, class_name: str, stage: str, paths: RunPaths) -> None: + command = ["javac", "-d", str(classes), str(source)] + process = run_command(command) + save_process_logs(process, paths.logs, stage) + if process.returncode != 0: + raise RunFailure(stage, command, process) + expected = classes / f"{class_name}.class" + if not expected.is_file(): + failed = subprocess.CompletedProcess(command, 1, "", f"Expected class file was not created: {expected}") + raise RunFailure(stage, command, failed) + + +def write_manifest(case: Case, paths: RunPaths) -> None: + rows = [manifest_row(case, "baseline", paths.classes("baseline"), paths.artifacts("baseline"))] + for variant in case.variants: + rows.append(manifest_row(case, variant.role, paths.classes(variant.role), paths.artifacts(variant.role))) + write_csv(paths.pairs, MANIFEST_HEADER, rows) + + +def manifest_row(case: Case, role: str, classes: Path, artifacts: Path) -> dict[str, str]: + return { + "case_id": case.case_id, + "role": role, + "classpath": str(classes.resolve()), + "class_name": case.class_name, + "method_name": case.method_name, + "jit_log": str((artifacts / "jit-log.xml").resolve()), + "jmh_result": str((artifacts / "jmh-result.json").resolve()), + "label": f"{case.case_id}/{role}", + } + + +def compare(case: Case, paths: RunPaths, runtime_classpath: str, repo_root: Path) -> None: + command = [ + "java", + "-cp", + runtime_classpath, + "comparator.reproducer.ComparePairs", + "--manifest", + str(paths.pairs), + "--output", + str(paths.comparisons), + ] + process = run_command(command, repo_root) + save_process_logs(process, paths.logs, "compare") + if process.returncode != 0: + raise RunFailure("compare", command, process) + if not paths.comparisons.is_file(): + failed = subprocess.CompletedProcess(command, 1, "", f"Comparisons CSV was not created for {case.case_id}") + raise RunFailure("compare", command, failed) + + +def save_process_logs(process: subprocess.CompletedProcess[str], logs: Path, stage: str) -> None: + logs.mkdir(parents=True, exist_ok=True) + (logs / f"{stage}.stdout.log").write_text(process.stdout, encoding="utf-8") + (logs / f"{stage}.stderr.log").write_text(process.stderr, encoding="utf-8") + + +def aggregate_case(session_root: Path, sid: str, case: Case) -> None: + case_root = session_root / "cases" / case.case_id + rows: list[dict[str, str]] = [] + header: list[str] = [] + for run_dir in sorted((case_root / "runs").glob("run-*")): + run_index = int(run_dir.name.removeprefix("run-")) + with (run_dir / "comparisons.csv").open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + header = reader.fieldnames or header + for row in reader: + updated = dict(row) + updated["session_id"] = sid + updated["case_id"] = case.case_id + updated["run_index"] = str(run_index) + updated["run_name"] = run_dir.name + updated["role"] = role_from_target(row.get("Target", "")) + rows.append(updated) + if not rows: + raise ReproducerError(f"No comparison rows found for aggregation: {case.case_id}") + all_header = header + ["session_id", "case_id", "run_index", "run_name", "role"] + write_csv(case_root / "all_runs.csv", all_header, rows) + write_csv(case_root / "summary.csv", ["role", "metric", "count", "mean", "stdev", "min", "max"], summary_rows(rows)) + + +def role_from_target(target: str) -> str: + if "/" in target: + return target.rsplit("/", maxsplit=1)[1] + return "" + + +def summary_rows(rows: list[dict[str, str]]) -> list[dict[str, str]]: + grouped: dict[str, list[dict[str, str]]] = {} + for row in rows: + grouped.setdefault(row["role"], []).append(row) + result: list[dict[str, str]] = [] + for role in sorted(grouped): + columns = numeric_columns(grouped[role]) + for column in columns: + values = [float(row[column]) for row in grouped[role] if is_number(row.get(column, ""))] + result.append(summary_row(role, column, values)) + return result + + +def numeric_columns(rows: list[dict[str, str]]) -> list[str]: + columns = rows[0].keys() + return [column for column in columns if any(is_number(row.get(column, "")) for row in rows)] + + +def summary_row(role: str, metric: str, values: list[float]) -> dict[str, str]: + return { + "role": role, + "metric": metric, + "count": str(len(values)), + "mean": str(statistics.fmean(values)), + "stdev": str(statistics.stdev(values) if len(values) > 1 else 0.0), + "min": str(min(values)), + "max": str(max(values)), + } + + +def is_number(value: str) -> bool: + try: + number = float(value) + return math.isfinite(number) + except ValueError: + return False + + +def index_row(sid: str, case: Case, index: int, run_name: str, status: str, run_root: Path) -> dict[str, str]: + return { + "session_id": sid, + "case_id": case.case_id, + "run_index": str(index), + "run_name": run_name, + "status": status, + "comparisons_csv": str(run_root / "comparisons.csv"), + } + + +def tools_metadata() -> dict: + return { + "java": tool_metadata("java", ["java", "-version"]), + "javac": tool_metadata("javac", ["javac", "-version"]), + "perf": tool_metadata("perf", ["perf", "--version"]), + } + + +def tool_metadata(name: str, command: list[str]) -> dict: + process = run_command(command) + return {"command": command_path(name), "version_output": combined_output(process)} + + +def environment_metadata() -> dict: + return { + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + "python_version": platform.python_version(), + } + + +def git_metadata(repo_root: Path) -> dict: + commit = run_command(["git", "rev-parse", "HEAD"], repo_root) + status = run_command(["git", "status", "--porcelain"], repo_root) + return {"commit": commit.stdout.strip(), "dirty": bool(status.stdout.strip())} + + +def write_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_csv(path: Path, header: list[str], rows: list[dict[str, str]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=header) + writer.writeheader() + writer.writerows(rows) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/reproducer/test_run.py b/reproducer/test_run.py new file mode 100644 index 0000000..36f67fe --- /dev/null +++ b/reproducer/test_run.py @@ -0,0 +1,103 @@ +"""Contract tests for the reproducer Python runner.""" + +from __future__ import annotations + +import csv +import subprocess +import tempfile +import unittest +from pathlib import Path + +import run + + +class DiscoveryTest(unittest.TestCase): + """Case discovery contract tests.""" + + def test_discovers_one_valid_case(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_case(root / "case01_example") + cases = run.discover_cases(root) + self.assertEqual(["case01_example"], [case.case_id for case in cases]) + self.assertEqual(["variant"], [variant.role for variant in cases[0].variants]) + + def test_discovers_multi_variant_case(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case_root = root / "case00_example" + source = case_root / "baseline" / "Example.java" + source.parent.mkdir(parents=True) + source.write_text("class Example { public static int run() { return 1; } }", encoding="utf-8") + for role in ["plain_array", "stream_boxed"]: + variant = case_root / "variants" / role / "Example.java" + variant.parent.mkdir(parents=True) + variant.write_text("class Example { public static int run() { return 1; } }", encoding="utf-8") + cases = run.discover_cases(root) + self.assertEqual(["plain_array", "stream_boxed"], [variant.role for variant in cases[0].variants]) + + def test_rejects_multiple_baseline_sources(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.write_case(root / "case01_example") + (root / "case01_example" / "baseline" / "Other.java").write_text( + "class Other { public static int run() { return 1; } }", + encoding="utf-8", + ) + with self.assertRaises(run.ReproducerError): + run.discover_cases(root) + + def write_case(self, case_root: Path) -> None: + for role in ["baseline", "variant"]: + source = case_root / role / "Example.java" + source.parent.mkdir(parents=True) + source.write_text("class Example { public static int run() { return 1; } }", encoding="utf-8") + + +class AggregationTest(unittest.TestCase): + """CSV aggregation contract tests.""" + + def test_aggregates_synthetic_comparisons(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case = run.Case( + "case01_example", + root / "baseline.java", + ( + run.CaseVariant("variant", root / "variant.java"), + run.CaseVariant("stream_boxed", root / "stream_boxed.java"), + ), + "Example", + ) + run_dir = root / "cases" / case.case_id / "runs" / "run-001" + run_dir.mkdir(parents=True) + self.write_comparisons(run_dir / "comparisons.csv") + run.aggregate_case(root, "session01", case) + all_rows = self.read_csv(root / "cases" / case.case_id / "all_runs.csv") + summary_rows = self.read_csv(root / "cases" / case.case_id / "summary.csv") + self.assertEqual(["baseline", "variant", "stream_boxed"], [row["role"] for row in all_rows]) + self.assertTrue(any(row["metric"] == "score" for row in summary_rows)) + + def test_writes_process_logs(self) -> None: + with tempfile.TemporaryDirectory() as directory: + logs = Path(directory) / "logs" + process = subprocess.CompletedProcess(["example"], 0, "out", "err") + run.save_process_logs(process, logs, "compare") + self.assertEqual("out", (logs / "compare.stdout.log").read_text(encoding="utf-8")) + self.assertEqual("err", (logs / "compare.stderr.log").read_text(encoding="utf-8")) + + def write_comparisons(self, path: Path) -> None: + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=["Target", "score"]) + writer.writeheader() + writer.writerow({"Target": "case01_example/baseline", "score": "1.0"}) + writer.writerow({"Target": "case01_example/variant", "score": "2.0"}) + writer.writerow({"Target": "case01_example/stream_boxed", "score": "3.0"}) + + def read_csv(self, path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8") as stream: + return list(csv.DictReader(stream)) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main/java/comparator/Main.java b/src/main/java/comparator/Main.java index 074cf25..f68960f 100644 --- a/src/main/java/comparator/Main.java +++ b/src/main/java/comparator/Main.java @@ -11,6 +11,15 @@ public final class Main { /** Run method. */ private static final String RUN_METHOD = "run"; + /** Case root. */ + private static final Path CASE_ROOT = Path.of("reproducer", "cases", "case00_primitive_loop_examples"); + + /** Class name. */ + private static final String CLASS_NAME = "PrimitiveLoopExample"; + + /** Variants directory. */ + private static final String VARIANTS = "variants"; + private Main() { // Intentionally empty. } @@ -21,29 +30,25 @@ private Main() { * @param args command line arguments */ public static void main(final String[] args) { - final Classpath loopComputationsClasspath = new Classpath(Path.of("examples", "loop-computations")); new CsvComparisons( new CsvComparison( - new Analysis( - new TargetMethod(loopComputationsClasspath, "PlainForExample", Main.RUN_METHOD) - ), - new Analysis( - new TargetMethod( - loopComputationsClasspath, "PlainForPlainArrayExample", Main.RUN_METHOD - ) - ), - new Analysis( - new TargetMethod(loopComputationsClasspath, "PlainForIndexedExample", Main.RUN_METHOD) - ), - new Analysis( - new TargetMethod( - loopComputationsClasspath, "PlainForReplaceAllExample", Main.RUN_METHOD - ) - ), - new Analysis( - new TargetMethod(loopComputationsClasspath, "StreamBoxedExample", Main.RUN_METHOD) - ) + Main.analysis(Main.CASE_ROOT.resolve("baseline"), "baseline"), + Main.analysis(Main.variant("plain_array"), "plain_array"), + Main.analysis(Main.variant("indexed_loop"), "indexed_loop"), + Main.analysis(Main.variant("replace_all"), "replace_all"), + Main.analysis(Main.variant("stream_boxed"), "stream_boxed") ) ).saveAsCsv(Path.of("comparisons.csv")); } + + private static Path variant(final String role) { + return Main.CASE_ROOT.resolve(Main.VARIANTS).resolve(role); + } + + private static Analysis analysis(final Path classpath, final String label) { + return new Analysis( + new TargetMethod(new Classpath(classpath), Main.CLASS_NAME, Main.RUN_METHOD), + label + ); + } } diff --git a/src/main/java/comparator/reproducer/ComparePairs.java b/src/main/java/comparator/reproducer/ComparePairs.java new file mode 100644 index 0000000..831334b --- /dev/null +++ b/src/main/java/comparator/reproducer/ComparePairs.java @@ -0,0 +1,101 @@ +package comparator.reproducer; + +import comparator.Analysis; +import comparator.comparison.CsvComparison; +import comparator.method.Classpath; +import comparator.method.TargetMethod; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +/** + * Batch comparator command line entry point for reproducer manifests. + */ +@SuppressWarnings("PMD.SystemPrintln") +public final class ComparePairs { + private ComparePairs() { + } + + /** + * Main method. + * + * @param args + * command line arguments + */ + public static void main(final String[] args) { + try { + final ComparePairsOptions options = ComparePairsOptions.fromArgs(args); + if (options.help()) { + System.out.println("Usage: ComparePairs --manifest --output "); + } else { + new ComparePairs().run(options); + } + } catch (final IllegalArgumentException | IllegalStateException exception) { + System.err.println(exception.getMessage()); + System.exit(1); + } + } + + private void run(final ComparePairsOptions options) { + try { + Files.createDirectories(options.output().toAbsolutePath().getParent()); + try (BufferedWriter writer = Files.newBufferedWriter(options.output(), StandardCharsets.UTF_8)) { + boolean headerWritten = false; + for (final ManifestCase comparison : ComparisonManifest.fromFile(options.manifest()).cases().values()) { + headerWritten = this.writeComparison(comparison, writer, headerWritten); + } + } + } catch (final IOException exception) { + throw new IllegalStateException("Unable to write comparisons: " + options.output(), exception); + } + } + + private boolean writeComparison(final ManifestCase comparison, final BufferedWriter writer, + final boolean headerWritten) + throws IOException { + final String csv = new CsvComparison( + this.analysis(comparison.baseline()), + comparison.variants().stream().map(this::analysis).toList() + ).asCsv(); + final List lines = new BufferedReaderLines(csv).asList(); + int start = 0; + if (headerWritten) { + start = 1; + } + for (int index = start; index < lines.size(); index += 1) { + if (headerWritten || index > start) { + writer.newLine(); + } + writer.write(lines.get(index)); + } + return true; + } + + private Analysis analysis(final ManifestEntry entry) { + return new Analysis( + new TargetMethod(new Classpath(entry.classpath()), entry.className(), entry.methodName()), + entry.jitLog(), + entry.jmhResult(), + entry.label() + ); + } + + /** + * Line list extracted from a generated CSV string. + */ + private static final class BufferedReaderLines { + /** CSV content. */ + private final String csv; + + private BufferedReaderLines(final String csv) { + this.csv = csv; + } + + private List asList() { + return new java.io.BufferedReader(new StringReader(this.csv)).lines().toList(); + } + } +} diff --git a/src/main/java/comparator/reproducer/ComparePairsOptions.java b/src/main/java/comparator/reproducer/ComparePairsOptions.java new file mode 100644 index 0000000..8f6168d --- /dev/null +++ b/src/main/java/comparator/reproducer/ComparePairsOptions.java @@ -0,0 +1,85 @@ +package comparator.reproducer; + +import java.nio.file.Path; +import java.util.Arrays; + +/** + * Command line options for reproducer pair comparison. + */ +@SuppressWarnings({ "PMD.ProhibitPublicStaticMethods", "PMD.CyclomaticComplexity" }) +public final class ComparePairsOptions { + /** Help flag. */ + private final boolean help; + + /** Manifest file. */ + private final Path manifest; + + /** Output file. */ + private final Path output; + + private ComparePairsOptions(final boolean help, final Path manifest, final Path output) { + this.help = help; + this.manifest = manifest; + this.output = output; + } + + /** + * Parses command line arguments. + * + * @param args + * command line arguments + * @return parsed options + */ + public static ComparePairsOptions fromArgs(final String... args) { + if (args.length == 1 && "--help".equals(args[0])) { + return new ComparePairsOptions(true, Path.of("."), Path.of(".")); + } + Path manifest = Path.of(""); + Path output = Path.of(""); + int index = 0; + while (index < args.length) { + final String arg = args[index]; + if ("--manifest".equals(arg)) { + manifest = ComparePairsOptions.value(args, index + 1, arg); + index += 2; + } else if ("--output".equals(arg)) { + output = ComparePairsOptions.value(args, index + 1, arg); + index += 2; + } else { + throw new IllegalArgumentException("Unknown argument: " + arg + " in " + Arrays.toString(args)); + } + } + if (manifest.toString().isEmpty() || output.toString().isEmpty()) { + throw new IllegalArgumentException("Both --manifest and --output are required"); + } + return new ComparePairsOptions(false, manifest, output); + } + + /** + * @return help flag + */ + public boolean help() { + return this.help; + } + + /** + * @return manifest file + */ + public Path manifest() { + return this.manifest; + } + + /** + * @return output file + */ + public Path output() { + return this.output; + } + + private static Path value(final String[] args, final int index, final String option) { + if (index >= args.length) { + throw new IllegalArgumentException("Missing value for " + option); + } + return Path.of(args[index]); + } +} diff --git a/src/main/java/comparator/reproducer/ComparisonManifest.java b/src/main/java/comparator/reproducer/ComparisonManifest.java new file mode 100644 index 0000000..ae86f68 --- /dev/null +++ b/src/main/java/comparator/reproducer/ComparisonManifest.java @@ -0,0 +1,147 @@ +package comparator.reproducer; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Strict CSV manifest of reproducer comparison cases. + */ +@SuppressWarnings({ "PMD.ProhibitPublicStaticMethods", "PMD.CyclomaticComplexity" }) +public final class ComparisonManifest { + /** Cases grouped by case identifier. */ + private final Map cases; + + private ComparisonManifest(final Map cases) { + this.cases = Collections.unmodifiableMap(new LinkedHashMap<>(cases)); + } + + /** + * Reads a manifest file. + * + * @param file + * manifest file + * @return parsed manifest + */ + public static ComparisonManifest fromFile(final Path file) { + try { + return ComparisonManifest.fromLines(Files.readAllLines(file, StandardCharsets.UTF_8)); + } catch (final IOException exception) { + throw new IllegalStateException("Unable to read manifest: " + file, exception); + } + } + + /** + * Reads manifest lines. + * + * @param lines + * manifest lines + * @return parsed manifest + */ + public static ComparisonManifest fromLines(final List lines) { + final List> rows = lines.stream() + .filter(line -> !line.isBlank()) + .filter(line -> !line.stripLeading().startsWith("#")) + .map(ComparisonManifest::parseCsvLine) + .toList(); + if (rows.isEmpty()) { + throw new IllegalArgumentException("Manifest must contain a header"); + } + if (!ManifestEntry.HEADER.equals(rows.get(0))) { + throw new IllegalArgumentException("Manifest header must be exactly: " + ManifestEntry.HEADER); + } + final Map grouped = new LinkedHashMap<>(); + rows.stream().skip(1).map(ManifestEntry::new).forEach(entry -> { + final MutableCase pair = grouped.computeIfAbsent(entry.caseId(), key -> new MutableCase()); + pair.add(entry); + }); + return new ComparisonManifest(ComparisonManifest.freeze(grouped)); + } + + /** + * @return manifest cases + */ + public Map cases() { + return this.cases; + } + + private static Map freeze(final Map grouped) { + final Map result = new LinkedHashMap<>(); + grouped.forEach((caseId, pair) -> result.put(caseId, pair.freeze(caseId))); + return result; + } + + private static List parseCsvLine(final String line) { + final List values = new ArrayList<>(); + final StringBuilder current = new StringBuilder(); + boolean quoted = false; + int index = 0; + while (index < line.length()) { + final char chr = line.charAt(index); + if (chr == '"') { + if (quoted && index + 1 < line.length() && line.charAt(index + 1) == '"') { + current.append(chr); + index += 1; + } else { + quoted = !quoted; + } + } else if (chr == ',' && !quoted) { + values.add(current.toString()); + current.setLength(0); + } else { + current.append(chr); + } + index += 1; + } + if (quoted) { + throw new IllegalArgumentException("Unclosed quote in manifest line: " + line); + } + values.add(current.toString()); + return List.copyOf(values); + } + + /** + * Mutable case under construction. + */ + private static final class MutableCase { + /** Baseline entry. */ + private ManifestEntry baseline; + + /** Variant entries. */ + private final List variants = new ArrayList<>(0); + + /** Seen roles. */ + private final List roles = new ArrayList<>(0); + + private void add(final ManifestEntry entry) { + if (this.roles.contains(entry.role())) { + throw new IllegalArgumentException( + "Duplicate manifest role for case: " + entry.caseId() + "/" + + entry.role() + ); + } + this.roles.add(entry.role()); + if ("baseline".equals(entry.role())) { + this.baseline = entry; + } else { + this.variants.add(entry); + } + } + + private ManifestCase freeze(final String caseId) { + if (this.baseline == null || this.variants.isEmpty()) { + throw new IllegalArgumentException( + "Manifest case must contain one baseline and at least one variant: " + + caseId + ); + } + return new ManifestCase(this.baseline, this.variants); + } + } +} diff --git a/src/main/java/comparator/reproducer/ManifestCase.java b/src/main/java/comparator/reproducer/ManifestCase.java new file mode 100644 index 0000000..5d9e087 --- /dev/null +++ b/src/main/java/comparator/reproducer/ManifestCase.java @@ -0,0 +1,41 @@ +package comparator.reproducer; + +import java.util.List; + +/** + * Manifest entries for one reproducer case. + */ +public final class ManifestCase { + /** Baseline entry. */ + private final ManifestEntry baseline; + + /** Variant entries. */ + private final List variants; + + /** + * Ctor. + * + * @param baseline + * baseline entry + * @param variants + * variant entries + */ + public ManifestCase(final ManifestEntry baseline, final List variants) { + this.baseline = baseline; + this.variants = List.copyOf(variants); + } + + /** + * @return baseline entry + */ + public ManifestEntry baseline() { + return this.baseline; + } + + /** + * @return variant entries + */ + public List variants() { + return this.variants; + } +} diff --git a/src/main/java/comparator/reproducer/ManifestEntry.java b/src/main/java/comparator/reproducer/ManifestEntry.java new file mode 100644 index 0000000..2f787b9 --- /dev/null +++ b/src/main/java/comparator/reproducer/ManifestEntry.java @@ -0,0 +1,144 @@ +package comparator.reproducer; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Single manifest row for one compared target role. + */ +@SuppressWarnings({ "PMD.DataClass", "PMD.ConstructorOnlyInitializesOrCallOtherConstructors" }) +public final class ManifestEntry { + /** Valid manifest header. */ + public static final List HEADER = List.of( + "case_id", + "role", + "classpath", + "class_name", + "method_name", + "jit_log", + "jmh_result", + "label" + ); + + /** Case identifier. */ + private final String caseId; + + /** Case role. */ + private final String role; + + /** Classpath root. */ + private final Path classpath; + + /** Class name. */ + private final String className; + + /** Method name. */ + private final String methodName; + + /** JIT log file. */ + private final Path jitLog; + + /** JMH result file. */ + private final Path jmhResult; + + /** Stable target label. */ + private final String label; + + /** + * Ctor. + * + * @param values + * manifest row values + */ + public ManifestEntry(final List values) { + if (values.size() != ManifestEntry.HEADER.size()) { + throw new IllegalArgumentException( + "Manifest row must contain exactly " + ManifestEntry.HEADER.size() + + " columns: " + values + ); + } + this.caseId = ManifestEntry.required(values.get(0), "case_id"); + this.role = ManifestEntry.required(values.get(1), "role"); + this.classpath = Path.of(ManifestEntry.required(values.get(2), "classpath")); + this.className = ManifestEntry.required(values.get(3), "class_name"); + this.methodName = ManifestEntry.required(values.get(4), "method_name"); + this.jitLog = Path.of(ManifestEntry.required(values.get(5), "jit_log")); + this.jmhResult = Path.of(ManifestEntry.required(values.get(6), "jmh_result")); + this.label = ManifestEntry.required(values.get(7), "label"); + this.validate(); + } + + /** + * @return case identifier + */ + public String caseId() { + return this.caseId; + } + + /** + * @return case role + */ + public String role() { + return this.role; + } + + /** + * @return classpath root + */ + public Path classpath() { + return this.classpath; + } + + /** + * @return class name + */ + public String className() { + return this.className; + } + + /** + * @return method name + */ + public String methodName() { + return this.methodName; + } + + /** + * @return JIT log file + */ + public Path jitLog() { + return this.jitLog; + } + + /** + * @return JMH result file + */ + public Path jmhResult() { + return this.jmhResult; + } + + /** + * @return stable target label + */ + public String label() { + return this.label; + } + + private void validate() { + if (!this.role.matches("[a-z][a-z0-9_]*")) { + throw new IllegalArgumentException("Manifest role must be a lowercase identifier: " + this.role); + } + if (!Files.isDirectory(this.classpath)) { + throw new IllegalArgumentException("Manifest classpath must be an existing directory: " + this.classpath); + } + } + + private static String required(final String value, final String column) { + final String trimmed = value.trim(); + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("Manifest column must not be blank: " + column); + } + return trimmed; + } +} diff --git a/src/main/java/comparator/reproducer/package-info.java b/src/main/java/comparator/reproducer/package-info.java new file mode 100644 index 0000000..02f4893 --- /dev/null +++ b/src/main/java/comparator/reproducer/package-info.java @@ -0,0 +1,4 @@ +/** + * Reproducer batch comparison command line objects. + */ +package comparator.reproducer; diff --git a/src/test/java/comparator/reproducer/ComparePairsOptionsTest.java b/src/test/java/comparator/reproducer/ComparePairsOptionsTest.java new file mode 100644 index 0000000..5521ef4 --- /dev/null +++ b/src/test/java/comparator/reproducer/ComparePairsOptionsTest.java @@ -0,0 +1,26 @@ +package comparator.reproducer; + +import java.nio.file.Path; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** CLI option parser contract tests. */ +final class ComparePairsOptionsTest { + @Test + void parsesManifestAndOutput() { + final ComparePairsOptions options = ComparePairsOptions.fromArgs( + "--manifest", "pairs.csv", "--output", "comparisons.csv" + ); + Assertions.assertEquals(Path.of("pairs.csv"), options.manifest(), "Manifest option should be parsed"); + Assertions.assertEquals(Path.of("comparisons.csv"), options.output(), "Output option should be parsed"); + } + + @Test + void rejectsMissingOutput() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ComparePairsOptions.fromArgs("--manifest", "pairs.csv"), + "Missing required options should fail" + ); + } +} diff --git a/src/test/java/comparator/reproducer/ComparisonManifestTest.java b/src/test/java/comparator/reproducer/ComparisonManifestTest.java new file mode 100644 index 0000000..156e56e --- /dev/null +++ b/src/test/java/comparator/reproducer/ComparisonManifestTest.java @@ -0,0 +1,79 @@ +package comparator.reproducer; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Manifest parser contract tests. */ +@SuppressWarnings("PMD.SignatureDeclareThrowsException") +final class ComparisonManifestTest { + /** Target columns. */ + private static final String TARGET = ",Example,run,"; + + @Test + void parsesValidBaselineVariantCase(@TempDir final Path tempDir) throws Exception { + final Path baseline = Files.createDirectories(tempDir.resolve("baseline")); + final Path variant = Files.createDirectories(tempDir.resolve("variant")); + final ComparisonManifest manifest = ComparisonManifest.fromLines( + List.of( + String.join(",", ManifestEntry.HEADER), + "case01,baseline," + baseline + ComparisonManifestTest.TARGET + tempDir.resolve("baseline.xml") + + "," + tempDir.resolve("baseline.json") + ",case01/baseline", + "case01,variant," + variant + ComparisonManifestTest.TARGET + tempDir.resolve("variant.xml") + + "," + tempDir.resolve("variant.json") + ",case01/variant" + ) + ); + final ManifestCase comparison = manifest.cases().get("case01"); + Assertions.assertEquals("case01/baseline", comparison.baseline().label(), "Baseline label should be parsed"); + Assertions.assertEquals( + "case01/variant", + comparison.variants().get(0).label(), + "Variant label should be parsed" + ); + } + + @Test + void parsesMultipleVariants(@TempDir final Path tempDir) throws Exception { + final Path baseline = Files.createDirectories(tempDir.resolve("baseline")); + final Path plainArray = Files.createDirectories(tempDir.resolve("plain_array")); + final Path streamBoxed = Files.createDirectories(tempDir.resolve("stream_boxed")); + final ComparisonManifest manifest = ComparisonManifest.fromLines( + List.of( + String.join(",", ManifestEntry.HEADER), + "case00,baseline," + baseline + ComparisonManifestTest.TARGET + tempDir.resolve("baseline.xml") + + "," + tempDir.resolve("baseline.json") + ",case00/baseline", + "case00,plain_array," + plainArray + ComparisonManifestTest.TARGET + tempDir.resolve("plain_array.xml") + + "," + tempDir.resolve("plain_array.json") + ",case00/plain_array", + "case00,stream_boxed," + streamBoxed + ComparisonManifestTest.TARGET + + tempDir.resolve("stream_boxed.xml") + + "," + tempDir.resolve("stream_boxed.json") + ",case00/stream_boxed" + ) + ); + final ManifestCase comparison = manifest.cases().get("case00"); + Assertions.assertEquals(2, comparison.variants().size(), "All variants should be parsed"); + Assertions.assertEquals( + List.of("plain_array", "stream_boxed"), + comparison.variants().stream().map(ManifestEntry::role).toList(), + "Variant roles should be preserved" + ); + } + + @Test + void rejectsMissingVariant(@TempDir final Path tempDir) throws Exception { + final Path baseline = Files.createDirectories(tempDir.resolve("baseline")); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ComparisonManifest.fromLines( + List.of( + String.join(",", ManifestEntry.HEADER), + "case01,baseline," + baseline + ComparisonManifestTest.TARGET + tempDir.resolve("baseline.xml") + + "," + tempDir.resolve("baseline.json") + ",case01/baseline" + ) + ), + "One-sided manifest cases should be rejected" + ); + } +} diff --git a/src/test/java/comparator/reproducer/package-info.java b/src/test/java/comparator/reproducer/package-info.java new file mode 100644 index 0000000..0d03c49 --- /dev/null +++ b/src/test/java/comparator/reproducer/package-info.java @@ -0,0 +1,4 @@ +/** + * Reproducer command line tests. + */ +package comparator.reproducer;