ci: add flaky-test management (#489)

.config/nextest.toml

@@ -0,0 +1,32 @@
+# cargo-nextest configuration.
+#
+# CI runs the suite under `cargo nextest run` (flaky-test management, #489)
+# instead of `cargo test`. The two runners parallelize differently, and that
+# difference broke one test:
+#
+#   The heavy integration tests in digstore-cli AND digstore-compiler exercise
+#   the real chia:// read/compile path, which JIT-compiles the guest wasm in
+#   wasmtime — many seconds of CPU per test (nextest even marks several SLOW,
+#   >60s). `cargo test` runs test BINARIES one at a time, so only one such heavy
+#   test compiles at a time. nextest instead runs num-cpus tests from a single
+#   global pool, so on the 4-vCPU CI runners several of these fully saturate the
+#   runner at once. cli_dev's `dev` child process (a separate process, not a
+#   nextest thread) then can't get its tokio serve loop scheduled to accept `/`
+#   within the test's readiness poll — failing
+#   `dev_serves_real_read_path_with_injected_shims` ("dev server did not come
+#   up") deterministically on both ubuntu-latest and windows-latest.
+#
+# Cap ALL of these heavy integration tests (across both crates) to 2 concurrent
+# via one shared group, so the runner always keeps scheduler slack for the dev
+# child while the fast in-process unit tests keep full parallelism. This mirrors
+# why `cargo test` was green (a local `--test-threads 2` full-suite run
+# reproduces green) without giving up nextest's flaky-run detection.
+
+[test-groups]
+heavy-integration = { max-threads = 2 }
+
+[[profile.default.overrides]]
+# `kind(test)` selects the integration tests under tests/ (not the crates'
+# in-process unit tests), scoped to the two wasmtime-heavy crates.
+filter = 'kind(test) & (package(digstore-cli) | package(digstore-compiler))'
+test-group = 'heavy-integration'

.github/workflows/ci.yml

@@ -98,8 +98,29 @@ jobs:
       - name: Build
         run: cargo build --workspace --locked
 
-      - name: Test
-        run: cargo test --workspace --locked
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
+      # cargo-nextest natively detects+reports flaky tests (retried-but-passed
+      # surfaces as "FLAKY" in the run summary, not silently green) instead of
+      # papering over intermittent failures with a bare re-run (#489).
+      #
+      # Intentionally a bare `cargo nextest run`, NOT `cargo llvm-cov nextest`:
+      # main has no coverage measurement/gate today, so this PR (flaky-test
+      # management only) must not introduce one. Coverage instrumentation also
+      # measurably slows the instrumented `digstore` binary that CLI
+      # integration tests spawn as a child process (e.g. `cli_dev`'s dev-server
+      # startup poll), which blew existing CI-tuned timeouts under nextest's
+      # parallelism and made an otherwise-passing test fail consistently. A
+      # future PR can add `cargo llvm-cov` + an explicit `--fail-under-lines`
+      # gate as its own, reviewable change.
+      - name: Test (nextest, flaky-aware)
+        run: cargo nextest run --workspace --locked --retries 2
+
+      # nextest does not execute doctests (https://github.com/nextest-rs/nextest/issues/16) —
+      # run them separately so they still gate CI.
+      - name: Doctests
+        run: cargo test --doc --workspace --locked
 
   # The read-crypto wasm crate (crates/dig-client-wasm) is EXCLUDED from the
   # workspace (it must resolve without chia-bls/blst, which don't build for
@@ -134,9 +155,23 @@ jobs:
         working-directory: crates/dig-client-wasm
         run: cargo clippy --all-targets -- -D warnings
 
-      - name: Parity tests (native, vs digstore-crypto)
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
+      # This crate is excluded from the `--workspace` job above (it must
+      # resolve without chia-bls/blst for wasm32); its thin wasm-bindgen glue
+      # in src/lib.rs is exercised end-to-end by the "Verify assembled
+      # package" step below, not by these native parity tests. Flaky-aware
+      # nextest only (no coverage gate anywhere in this workflow — see the
+      # note on the workspace job's Test step).
+      - name: Parity tests (native, vs digstore-crypto; flaky-aware)
+        working-directory: crates/dig-client-wasm
+        run: cargo nextest run --locked --retries 2
+
+      # nextest does not execute doctests — run separately so they still gate CI.
+      - name: Parity doctests
         working-directory: crates/dig-client-wasm
-        run: cargo test --locked
+        run: cargo test --doc --locked
 
       - name: Build + assemble package (web + node -> pkg/)
         working-directory: crates/dig-client-wasm

Cargo.toml

@@ -5,7 +5,7 @@ exclude = ["crates/digstore-prover/guest", "crates/dig-client-wasm"]
 
 [workspace.package]
 edition = "2021"
-version = "0.13.4"
+version = "0.13.5"
 license = "GPL-2.0-only"
 
 [workspace.dependencies]

crates/digstore-cli/tests/cli_dev.rs

@@ -26,7 +26,11 @@ use std::time::{Duration, Instant};
 fn http_get(port: u16, path: &str) -> Option<String> {
     let mut stream = TcpStream::connect(("127.0.0.1", port)).ok()?;
     stream
-        .set_read_timeout(Some(Duration::from_secs(10)))
+        // Generous per-request read timeout: on a CPU-saturated CI runner the
+        // child's tokio serve loop can be slow to get scheduled to answer, and a
+        // single poll attempt must outlast that scheduling delay rather than give
+        // up early and force a fresh connection.
+        .set_read_timeout(Some(Duration::from_secs(30)))
         .ok()?;
     let req = format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n");
     stream.write_all(req.as_bytes()).ok()?;
@@ -96,7 +100,10 @@ fn http_get_retry(port: u16, path: &str) -> Option<String> {
 
 /// Poll until the dev server answers `/` (or time out).
 fn wait_for_server(port: u16) -> Option<String> {
-    let deadline = Instant::now() + Duration::from_secs(40);
+    // Generous overall window: under nextest the child competes with the rest of
+    // the suite for CPU, so it may take considerably longer than on an idle
+    // machine before its serve loop is scheduled to answer `/`.
+    let deadline = Instant::now() + Duration::from_secs(90);
     while Instant::now() < deadline {
         if let Some(body) = http_get(port, "/") {
             if !body.is_empty() {