Reproducer

.gitignore

@@ -45,3 +45,10 @@ AGENTS.md
 
 ### Examples ###
 examples/*/*.class
+
+### Reproducer ###
+reproducer/runs/
+
+### Python ###
+__pycache__/
+*.pyc

README.md

@@ -32,6 +32,14 @@ CSV rows also include `JIT log file` and `JMH result file` path columns.
 - The target method should preferably return a value. `void` methods are more likely to be removed by JIT optimization as dead code.
 - The target method should preferably execute at least `1000` instructions. Around `50-100` instructions may come from non-optimized JMH wrapper overhead (`Method.invoke()`). By default this optimization should work and overhead will be around `10-20` instructions.
 
+## Reproducer
+
+The `reproducer/` directory contains a runner for curated baseline/variant cases.
+It compiles each case, runs Comparator repeatedly, preserves raw JIT/JMH artifacts,
+and writes per-case aggregate CSV files.
+
+See [reproducer/README.md](reproducer/README.md) for usage.
+
 ## API usage
 
 ### Run an analysis
@@ -41,15 +49,19 @@ import comparator.Analysis;
 import comparator.method.TargetMethod;
 import java.nio.file.Path;
 
-final Path classpath = Path.of("examples", "loop-computations");
-new Analysis(new TargetMethod(classpath, "PlainForExample", "run"))
+final Path classpath = Path.of(
+        "reproducer", "cases", "case00_primitive_loop_examples", "baseline"
+);
+new Analysis(new TargetMethod(classpath, "PrimitiveLoopExample", "run"))
         .results()
         .print(System.out);
 ```
 
 The classpath argument must point to a directory or JAR that contains compiled classes.
 
-The `examples/` folders do not include `.class` files, so compile them before running the examples.
+The `reproducer/cases/` folders do not include `.class` files, so compile the selected
+case role before running API examples. For `case00_primitive_loop_examples`, each role
+has its own classpath because all variants use the same `PrimitiveLoopExample` class name.
 
 ### Run an analysis with label
 
@@ -58,9 +70,11 @@ import comparator.Analysis;
 import comparator.method.TargetMethod;
 import java.nio.file.Path;
 
-final Path classpath = Path.of("examples", "loop-computations");
+final Path classpath = Path.of(
+        "reproducer", "cases", "case00_primitive_loop_examples", "baseline"
+);
 new Analysis(
-        new TargetMethod(classpath, "PlainForExample", "run"),
+        new TargetMethod(classpath, "PrimitiveLoopExample", "run"),
         "baseline-for-loop"
 ).results().print(System.out);
 ```
@@ -77,17 +91,18 @@ import comparator.method.Classpath;
 import comparator.method.TargetMethod;
 import java.nio.file.Path;
 
-final Classpath classpath = new Classpath(Path.of("examples", "loop-computations"));
+final Path caseRoot = Path.of("reproducer", "cases", "case00_primitive_loop_examples");
+final Classpath baseline = new Classpath(caseRoot.resolve("baseline"));
+final Classpath plainArray = new Classpath(caseRoot.resolve("variants").resolve("plain_array"));
+final Classpath indexedLoop = new Classpath(caseRoot.resolve("variants").resolve("indexed_loop"));
+final Classpath streamBoxed = new Classpath(caseRoot.resolve("variants").resolve("stream_boxed"));
 
 new CsvComparisons(
         new CsvComparison(
-                new Analysis(new TargetMethod(classpath, "PlainForExample", "run")),
-                new Analysis(new TargetMethod(classpath, "StreamBoxedExample", "run")),
-                new Analysis(new TargetMethod(classpath, "PlainForIndexedExample", "run"))
-        ),
-        new CsvComparison(
-                new Analysis(new TargetMethod(classpath, "PlainForExample", "run")),
-                new Analysis(new TargetMethod(classpath, "PlainForReplaceAllExample", "run"))
+                new Analysis(new TargetMethod(baseline, "PrimitiveLoopExample", "run"), "baseline"),
+                new Analysis(new TargetMethod(plainArray, "PrimitiveLoopExample", "run"), "plain_array"),
+                new Analysis(new TargetMethod(indexedLoop, "PrimitiveLoopExample", "run"), "indexed_loop"),
+                new Analysis(new TargetMethod(streamBoxed, "PrimitiveLoopExample", "run"), "stream_boxed")
         )
 ).saveAsCsv(Path.of("comparisons.csv"));
 ```
@@ -96,28 +111,20 @@ new CsvComparisons(
 
 Example of `comparisons.csv` content in table form. File path columns are shortened for readability:
 
-Comparison 1
-
-| Target | JMH primary score, us/op | JMH primary score relative error, ratio | Allocations, B/op | Allocations relative error, ratio | Instructions, #/op | Memory loads, #/op | Memory stores, #/op | Native code size, B | JIT log file | JMH result file | JIT metrics mean dissimilarity score | JIT metrics max dissimilarity score |
-| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- |
-| PlainForExample::run | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PlainForExample-jit-log-...xml` | `.../PlainForExample-jmh-result-...json` | Original | Original |
-| PlainForPlainArrayExample::run | 4.55 | 0.11 | 8040.03 | 0.00 | 25719.34 | 3281.44 | 1151.00 | 1552.00 | `.../PlainForPlainArrayExample-jit-log-...xml` | `.../PlainForPlainArrayExample-jmh-result-...json` | 1.40 | 1.79 |
-| PlainForIndexedExample::run | 25.64 | 0.12 | 38016.18 | 0.00 | 149624.24 | 41981.45 | 19787.82 | 1960.00 | `.../PlainForIndexedExample-jit-log-...xml` | `.../PlainForIndexedExample-jmh-result-...json` | 0.09 | 0.15 |
-
-Comparison 2
-
 | Target | JMH primary score, us/op | JMH primary score relative error, ratio | Allocations, B/op | Allocations relative error, ratio | Instructions, #/op | Memory loads, #/op | Memory stores, #/op | Native code size, B | JIT log file | JMH result file | JIT metrics mean dissimilarity score | JIT metrics max dissimilarity score |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- |
-| PlainForExample::run | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PlainForExample-jit-log-...xml` | `.../PlainForExample-jmh-result-...json` | Original | Original |
-| PlainForReplaceAllExample::run | 36.08 | 0.11 | 69952.25 | 0.00 | 240191.93 | 65149.11 | 32818.88 | 2336.00 | `.../PlainForReplaceAllExample-jit-log-...xml` | `.../PlainForReplaceAllExample-jmh-result-...json` | 0.44 | 0.59 |
+| baseline | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | Original | Original |
+| plain_array | 4.55 | 0.11 | 8040.03 | 0.00 | 25719.34 | 3281.44 | 1151.00 | 1552.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 1.40 | 1.79 |
+| indexed_loop | 25.64 | 0.12 | 38016.18 | 0.00 | 149624.24 | 41981.45 | 19787.82 | 1960.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 0.09 | 0.15 |
+| stream_boxed | 46.05 | 0.10 | 70232.32 | 0.00 | 243005.89 | 74453.67 | 32071.86 | 3528.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 0.54 | 0.70 |
 
 ### Labeled comparison example
 
 ```java
 new CsvComparisons(
         new CsvComparison(
-                new Analysis(new TargetMethod(classpath, "PlainForExample", "run"), "Baseline"),
-                new Analysis(new TargetMethod(classpath, "StreamBoxedExample", "run"), "Stream")
+                new Analysis(new TargetMethod(baseline, "PrimitiveLoopExample", "run"), "Baseline"),
+                new Analysis(new TargetMethod(streamBoxed, "PrimitiveLoopExample", "run"), "Stream")
         )
 ).saveAsCsv(Path.of("labels-demo.csv"));
 ```
@@ -126,8 +133,8 @@ Example of `labels-demo.csv` content in table form:
 
 | Target | JMH primary score, us/op | JMH primary score relative error, ratio | Allocations, B/op | Allocations relative error, ratio | Instructions, #/op | Memory loads, #/op | Memory stores, #/op | Native code size, B | JIT log file | JMH result file | JIT metrics mean dissimilarity score | JIT metrics max dissimilarity score |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- |
-| Baseline | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PlainForExample-jit-log-...xml` | `.../PlainForExample-jmh-result-...json` | Original | Original |
-| Stream | 46.05 | 0.10 | 70232.32 | 0.00 | 243005.89 | 74453.67 | 32071.86 | 3528.00 | `.../StreamBoxedExample-jit-log-...xml` | `.../StreamBoxedExample-jmh-result-...json` | 0.54 | 0.70 |
+| Baseline | 22.09 | 0.10 | 38016.15 | 0.00 | 149402.64 | 43000.08 | 20805.29 | 2256.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | Original | Original |
+| Stream | 46.05 | 0.10 | 70232.32 | 0.00 | 243005.89 | 74453.67 | 32071.86 | 3528.00 | `.../PrimitiveLoopExample-jit-log-...xml` | `.../PrimitiveLoopExample-jmh-result-...json` | 0.54 | 0.70 |
 
 ## Comparison metrics
 

build.gradle

@@ -48,3 +48,10 @@ jacocoTestReport {
 application {
     mainClass = 'comparator.Main'
 }
+
+tasks.register('printRuntimeClasspath') {
+    dependsOn classes
+    doLast {
+        println sourceSets.main.runtimeClasspath.asPath
+    }
+}

reproducer/README.md

@@ -0,0 +1,155 @@
+# Reproducer
+
+The reproducer runs curated JIT instability cases from `reproducer/cases`.
+Each case contains a `baseline` Java source and one or more equivalent variant
+Java sources. Existing single-variant cases may use `variant/`; multi-variant
+cases use `variants/<role>/`.
+
+It can:
+
+- discover valid baseline/variant pairs;
+- compile each pair into isolated class directories;
+- run Comparator on each pair through JMH;
+- collect JIT logs, JMH JSON results, and perf-backed metrics;
+- repeat whole-case executions with `--runs`;
+- write per-run CSV files and per-case aggregate CSV files.
+
+## Requirements
+
+- Linux with working `perf`;
+- JDK with `java` and `javac` on `PATH`;
+- Gradle wrapper from this repository.
+
+The runner checks `perf --version` and `perf stat -e instructions -- sleep 0.1`
+before starting expensive work.
+
+## Usage
+
+Run all cases once:
+
+```bash
+python3 reproducer/run.py --runs 1
+```
+
+Run selected cases by prefix:
+
+```bash
+python3 reproducer/run.py --runs 3 --include-cases case01,case03
+```
+
+Run the primitive loop demonstration case:
+
+```bash
+python3 reproducer/run.py --runs 3 --include-cases case00
+```
+
+Useful options:
+
+- `--runs N` - required number of whole-case repeats;
+- `--include-cases case01,case03` - comma-separated case-id prefixes;
+- `--session-id NAME` - fixed output session name;
+- `--cases-root PATH` - custom cases directory;
+- `--runs-root PATH` - custom output directory.
+
+## Case layout
+
+Single-variant cases can use the compatibility layout:
+
+```text
+case01_example/
+  baseline/Example.java
+  variant/Example.java
+```
+
+Multi-variant cases use named variant roles:
+
+```text
+case00_primitive_loop_examples/
+  baseline/PrimitiveLoopExample.java
+  variants/plain_array/PrimitiveLoopExample.java
+  variants/indexed_loop/PrimitiveLoopExample.java
+  variants/replace_all/PrimitiveLoopExample.java
+  variants/stream_boxed/PrimitiveLoopExample.java
+```
+
+Each role must contain exactly one package-less Java source with the same file
+name as the baseline and a static no-argument `run` method.
+
+## Output
+
+Each run creates a session under `reproducer/runs/<session_id>` and updates
+`reproducer/runs/latest` when symlinks are supported.
+
+Important files:
+
+- `metadata.json` - tools, environment, selected cases, Gradle classpath output;
+- `index.csv` - one row per case run;
+- `cases/<case_id>/runs/run-001/status.json` - run status or captured failure;
+- `cases/<case_id>/runs/run-001/logs/*.log` - stdout and stderr for `javac` and Comparator commands;
+- `cases/<case_id>/runs/run-001/comparisons.csv` - raw Comparator CSV;
+- `cases/<case_id>/runs/run-001/artifacts/*` - JIT logs and JMH results;
+- `cases/<case_id>/all_runs.csv` - concatenated rows across runs;
+- `cases/<case_id>/summary.csv` - count, mean, stdev, min, and max per role.
+
+## Plotting
+
+**Generate a mean-difference PDF for one case:**
+
+```bash
+python3 reproducer/plotting/mean_difference.py \
+  reproducer/runs/example-session/cases/example-case
+```
+
+By default this reads the case `summary.csv` and creates:
+
+```text
+reproducer/runs/example-session/cases/example-case/mean_difference.pdf
+```
+
+Example plot:
+
+![Mean difference plot](plotting/examples/mean_difference.png)
+
+The chart includes five aggregated metrics: JMH score, allocations,
+instructions, memory loads, and memory stores.
+
+**Generate a per-run relative-difference PDF for one case:**
+
+```bash
+python3 reproducer/plotting/run_difference.py \
+  reproducer/runs/example-session/cases/example-case
+```
+
+By default this reads the case `all_runs.csv` and creates:
+
+```text
+reproducer/runs/example-session/cases/example-case/run_difference.pdf
+```
+
+Example plot:
+
+![Run difference plot](plotting/examples/run_difference.png)
+
+The chart includes six per-run metrics: JMH score, allocations, instructions,
+memory loads, memory stores, and native code size.
+
+**Generate raw per-metric PDFs for one case:**
+
+```bash
+python3 reproducer/plotting/metric_difference.py \
+  reproducer/runs/example-session/cases/example-case
+```
+
+By default this reads the case `all_runs.csv` and creates one PDF per metric
+under:
+
+```text
+reproducer/runs/example-session/cases/example-case/metric_difference/
+```
+
+Example plot:
+
+![Native code size by run plot](plotting/examples/native_code_size_by_run.png)
+
+The charts include six raw metrics: JMH score, allocations, instructions,
+memory loads, memory stores, and native code size.

reproducer/cases/case00_primitive_loop_examples/baseline/PrimitiveLoopExample.java

@@ -0,0 +1,33 @@
+/*
+ * Mechanism: primitive loop control examples.
+ * Hypothesis: equivalent ways to build, traverse, and transform a small
+ * integer collection can produce different optimized code shapes even when
+ * they return the same result.
+ * Expected symptom: plain-array and indexed-loop variants should stay close to
+ * the baseline, while replaceAll and boxed-stream variants are expected to show
+ * larger instruction/load/store and native-code-size movement.
+ */
+import java.util.ArrayList;
+import java.util.List;
+
+public class PrimitiveLoopExample {
+    private static final int N = 2_000;
+
+    private static int compute(final int x) {
+        int y = x * 31;
+        y ^= y >>> 16;
+        return y + 7;
+    }
+
+    public static long run() {
+        final List<Integer> list = new ArrayList<>(N);
+        for (int i = 0; i < N; i++) {
+            list.add(i);
+        }
+        long sum = 0;
+        for (final int value : list) {
+            sum += compute(value);
+        }
+        return sum;
+    }
+}

examples/loop-computations/PlainForIndexedExample.java → reproducer/cases/case00_primitive_loop_examples/variants/indexed_loop/PrimitiveLoopExample.java

@@ -1,12 +1,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class PlainForIndexedExample {
+public class PrimitiveLoopExample {
     private static final int N = 2_000;
 
     private static int compute(final int x) {
         int y = x * 31;
-        y ^= (y >>> 16);
+        y ^= y >>> 16;
         return y + 7;
     }
 

examples/loop-computations/PlainForPlainArrayExample.java → reproducer/cases/case00_primitive_loop_examples/variants/plain_array/PrimitiveLoopExample.java

@@ -1,9 +1,9 @@
-public class PlainForPlainArrayExample {
+public class PrimitiveLoopExample {
     private static final int N = 2_000;
 
     private static int compute(final int x) {
         int y = x * 31;
-        y ^= (y >>> 16);
+        y ^= y >>> 16;
         return y + 7;
     }
 
@@ -13,8 +13,8 @@ public static long run() {
             values[i] = i;
         }
         long sum = 0;
-        for (final int v : values) {
-            sum += compute(v);
+        for (final int value : values) {
+            sum += compute(value);
         }
         return sum;
     }