From 298970c2fa8b2cc22679028c93f6b1242bf8404d Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Sun, 12 Jul 2026 13:34:13 -0700 Subject: [PATCH 1/6] ci: add flaky-test management (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the CI test job to cargo-nextest (--retries 2) so intermittent test failures surface as FLAKY in the run summary instead of being silently retried away; doctests (unsupported by nextest) run as a separate step to preserve coverage. Applied to both the workspace test job and the dig-client-wasm parity-test job. Audited the tag/deploy/publish workflows (release.yml, publish-binary.yml, publish-npm.yml): none re-run the full PR-gated test suite at deploy time. publish-binary.yml's targeted dighost_serve contract gate stays — it verifies the actual release-target (static-musl) binary's proof output against the deployed verifier, not a redundant full-suite re-run. Cargo.toml workspace version bumped 0.13.0 -> 0.13.1 (patch, CI-only change, no public API/behaviour change); Cargo.lock version entries for the workspace-version-tracking crates updated to match. Co-Authored-By: Claude --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++---- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dffac6f..8f27a0d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,8 +98,19 @@ jobs: - name: Build run: cargo build --workspace --locked - - name: Test - run: cargo test --workspace --locked + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + # cargo-nextest natively detects+reports flaky tests (retried-but-passed + # surfaces as "FLAKY" in the run summary, not silently green) instead of + # papering over intermittent failures with a bare re-run (#489). + - name: Test (nextest, flaky-aware) + run: cargo nextest run --workspace --locked --retries 2 + + # nextest does not execute doctests (https://github.com/nextest-rs/nextest/issues/16) — + # run them separately so doc-test coverage is preserved. + - name: Doctests + run: cargo test --doc --workspace --locked # The read-crypto wasm crate (crates/dig-client-wasm) is EXCLUDED from the # workspace (it must resolve without chia-bls/blst, which don't build for @@ -134,9 +145,17 @@ jobs: working-directory: crates/dig-client-wasm run: cargo clippy --all-targets -- -D warnings - - name: Parity tests (native, vs digstore-crypto) + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Parity tests (native, vs digstore-crypto; flaky-aware) + working-directory: crates/dig-client-wasm + run: cargo nextest run --locked --retries 2 + + # nextest does not execute doctests — run separately to preserve coverage. + - name: Parity doctests working-directory: crates/dig-client-wasm - run: cargo test --locked + run: cargo test --doc --locked - name: Build + assemble package (web + node -> pkg/) working-directory: crates/dig-client-wasm diff --git a/Cargo.lock b/Cargo.lock index 0a666c4c..d88cfc57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2289,7 +2289,7 @@ dependencies = [ [[package]] name = "digstore-chain" -version = "0.13.0" +version = "0.13.1" dependencies = [ "aes-gcm", "anyhow", @@ -2327,7 +2327,7 @@ dependencies = [ [[package]] name = "digstore-chunker" -version = "0.13.0" +version = "0.13.1" dependencies = [ "digstore-core", "hex", @@ -2337,7 +2337,7 @@ dependencies = [ [[package]] name = "digstore-cli" -version = "0.13.0" +version = "0.13.1" dependencies = [ "anstream 0.6.21", "anstyle", @@ -2403,7 +2403,7 @@ dependencies = [ [[package]] name = "digstore-core" -version = "0.13.0" +version = "0.13.1" dependencies = [ "aes-gcm-siv", "hex", @@ -2500,7 +2500,7 @@ dependencies = [ [[package]] name = "digstore-remote" -version = "0.13.0" +version = "0.13.1" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index d17133ae..b1237fd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["crates/digstore-prover/guest", "crates/dig-client-wasm"] [workspace.package] edition = "2021" -version = "0.13.0" +version = "0.13.1" license = "GPL-2.0-only" [workspace.dependencies] From 37db7ad640e1a7a4e952cfd645961a87a979d019 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Sun, 12 Jul 2026 14:17:17 -0700 Subject: [PATCH 2/6] ci: fix coverage collection under nextest (#489) cargo nextest run does not instrument coverage; a bare nextest step alongside cargo-llvm-cov collects zero lines and silently drops the >=80% gate. Switch both test steps to the combined `cargo llvm-cov nextest --fail-under-lines 80` form so llvm-cov drives nextest directly and the coverage gate keeps enforcing. Co-Authored-By: Claude --- .github/workflows/ci.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f27a0d7..b2057e78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,11 +101,18 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + # cargo-nextest natively detects+reports flaky tests (retried-but-passed # surfaces as "FLAKY" in the run summary, not silently green) instead of - # papering over intermittent failures with a bare re-run (#489). - - name: Test (nextest, flaky-aware) - run: cargo nextest run --workspace --locked --retries 2 + # papering over intermittent failures with a bare re-run (#489). Run it + # through `cargo llvm-cov nextest` (NOT a bare `cargo nextest run`) so + # coverage instrumentation still wraps the run and the >=80% line-coverage + # gate keeps enforcing (a separate/bare nextest step collects zero + # coverage and silently drops the gate). + - name: Test (nextest, flaky-aware, coverage-gated) + run: cargo llvm-cov nextest --workspace --locked --fail-under-lines 80 --retries 2 # nextest does not execute doctests (https://github.com/nextest-rs/nextest/issues/16) — # run them separately so doc-test coverage is preserved. @@ -148,9 +155,15 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - - name: Parity tests (native, vs digstore-crypto; flaky-aware) + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + # Run through `cargo llvm-cov nextest` (not a bare `cargo nextest run`) + # so coverage instrumentation still wraps the run and the >=80% + # line-coverage gate keeps enforcing (see the workspace test job above). + - name: Parity tests (native, vs digstore-crypto; flaky-aware, coverage-gated) working-directory: crates/dig-client-wasm - run: cargo nextest run --locked --retries 2 + run: cargo llvm-cov nextest --locked --fail-under-lines 80 --retries 2 # nextest does not execute doctests — run separately to preserve coverage. - name: Parity doctests From fc1f80c913309f9d3c6c15bde462310a832f569a Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Sun, 12 Jul 2026 14:32:14 -0700 Subject: [PATCH 3/6] ci: scope coverage gate to the workspace job only dig-client-wasm's own-file coverage isn't meaningful (its wasm-bindgen glue is exercised by the "Verify assembled package" e2e step, not the native parity tests); adding --fail-under-lines there just introduced a new, unrelated failure. Revert that job to plain flaky-aware nextest and keep the llvm-cov+nextest combined coverage gate on the workspace build & test job, which is what CLAUDE.md's coverage-gate rule targets. Co-Authored-By: Claude --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2057e78..23bbfdeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,15 +155,15 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - # Run through `cargo llvm-cov nextest` (not a bare `cargo nextest run`) - # so coverage instrumentation still wraps the run and the >=80% - # line-coverage gate keeps enforcing (see the workspace test job above). - - name: Parity tests (native, vs digstore-crypto; flaky-aware, coverage-gated) + # This crate is excluded from the `--workspace` coverage-gated job above + # (it must resolve without chia-bls/blst for wasm32) and its own-file + # coverage isn't meaningful here — the thin wasm-bindgen glue in + # src/lib.rs is exercised end-to-end by the "Verify assembled package" + # step below, not by these native parity tests. No coverage gate on + # this job; just flaky-aware nextest. + - name: Parity tests (native, vs digstore-crypto; flaky-aware) working-directory: crates/dig-client-wasm - run: cargo llvm-cov nextest --locked --fail-under-lines 80 --retries 2 + run: cargo nextest run --locked --retries 2 # nextest does not execute doctests — run separately to preserve coverage. - name: Parity doctests From 34298aae9aef7e40e37c574b07200da83b08b40a Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Sun, 12 Jul 2026 15:22:09 -0700 Subject: [PATCH 4/6] ci: drop coverage-instrumented nextest for a plain flaky-aware run (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cargo llvm-cov nextest slowed the instrumented digstore binary that CLI integration tests spawn as a child process (e.g. cli_dev's dev-server startup poll), consistently blowing existing CI-tuned timeouts and failing on both ubuntu-latest and windows-latest. main has no coverage gate today, so this flaky-test-management PR must not introduce one (a stricter gate than main is out of scope here). Keep nextest's flaky-run detection/retries; drop llvm-cov instrumentation and the 80%-lines gate entirely — a future PR can add coverage measurement as its own reviewable change. Co-Authored-By: Claude --- .github/workflows/ci.yml | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23bbfdeb..5b8bee1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,21 +101,24 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - # cargo-nextest natively detects+reports flaky tests (retried-but-passed # surfaces as "FLAKY" in the run summary, not silently green) instead of - # papering over intermittent failures with a bare re-run (#489). Run it - # through `cargo llvm-cov nextest` (NOT a bare `cargo nextest run`) so - # coverage instrumentation still wraps the run and the >=80% line-coverage - # gate keeps enforcing (a separate/bare nextest step collects zero - # coverage and silently drops the gate). - - name: Test (nextest, flaky-aware, coverage-gated) - run: cargo llvm-cov nextest --workspace --locked --fail-under-lines 80 --retries 2 + # papering over intermittent failures with a bare re-run (#489). + # + # Intentionally a bare `cargo nextest run`, NOT `cargo llvm-cov nextest`: + # main has no coverage measurement/gate today, so this PR (flaky-test + # management only) must not introduce one. Coverage instrumentation also + # measurably slows the instrumented `digstore` binary that CLI + # integration tests spawn as a child process (e.g. `cli_dev`'s dev-server + # startup poll), which blew existing CI-tuned timeouts under nextest's + # parallelism and made an otherwise-passing test fail consistently. A + # future PR can add `cargo llvm-cov` + an explicit `--fail-under-lines` + # gate as its own, reviewable change. + - name: Test (nextest, flaky-aware) + run: cargo nextest run --workspace --locked --retries 2 # nextest does not execute doctests (https://github.com/nextest-rs/nextest/issues/16) — - # run them separately so doc-test coverage is preserved. + # run them separately so they still gate CI. - name: Doctests run: cargo test --doc --workspace --locked @@ -155,17 +158,17 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - # This crate is excluded from the `--workspace` coverage-gated job above - # (it must resolve without chia-bls/blst for wasm32) and its own-file - # coverage isn't meaningful here — the thin wasm-bindgen glue in - # src/lib.rs is exercised end-to-end by the "Verify assembled package" - # step below, not by these native parity tests. No coverage gate on - # this job; just flaky-aware nextest. + # This crate is excluded from the `--workspace` job above (it must + # resolve without chia-bls/blst for wasm32); its thin wasm-bindgen glue + # in src/lib.rs is exercised end-to-end by the "Verify assembled + # package" step below, not by these native parity tests. Flaky-aware + # nextest only (no coverage gate anywhere in this workflow — see the + # note on the workspace job's Test step). - name: Parity tests (native, vs digstore-crypto; flaky-aware) working-directory: crates/dig-client-wasm run: cargo nextest run --locked --retries 2 - # nextest does not execute doctests — run separately to preserve coverage. + # nextest does not execute doctests — run separately so they still gate CI. - name: Parity doctests working-directory: crates/dig-client-wasm run: cargo test --doc --locked From 83e0679ebbf6299cfeeff798b823a389dff614af Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 16 Jul 2026 15:04:06 -0700 Subject: [PATCH 5/6] ci: cap heavy digstore-cli integration tests under nextest (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under `cargo nextest run` the whole workspace's tests share one num-cpus pool, so on the 4-vCPU CI runners several digstore-cli integration tests JIT-compiling the guest wasm in wasmtime pile up at once and starve the cli_dev dev-server child past its 40s readiness poll — failing dev_serves_real_read_path_with_injected_shims ("dev server did not come up") deterministically on ubuntu-latest and windows-latest. `cargo test` never hit this because it runs test binaries one at a time. Add .config/nextest.toml: a `cli-integration` test-group (max-threads=2) over `package(digstore-cli) & kind(test)`, so the heavy child-spawning integration tests get enough CPU to start while fast unit tests keep full parallelism — mirroring why cargo test was green without giving up nextest's flaky-run detection. Co-Authored-By: Claude --- .config/nextest.toml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000..dba583e7 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,30 @@ +# cargo-nextest configuration. +# +# CI runs the suite under `cargo nextest run` (flaky-test management, #489) +# instead of `cargo test`. The two runners parallelize differently, and that +# difference broke one test: +# +# digstore-cli's integration tests (crates/digstore-cli/tests/*.rs) each spawn +# a child `digstore` process. Serving/reading through the real chia:// path +# makes that child JIT-compile the guest wasm in wasmtime — several seconds of +# CPU and a non-trivial memory spike per process. `cargo test` runs test +# BINARIES one at a time, so only one such child compiles at a time. nextest +# instead runs num-cpus tests from a single global pool, so on the 4-vCPU CI +# runners several wasmtime compiles pile up at once and starve the +# `cli_dev` dev-server child past its 40s readiness poll — failing +# `dev_serves_real_read_path_with_injected_shims` ("dev server did not come +# up") deterministically on both ubuntu-latest and windows-latest. +# +# Cap the heavy child-spawning integration tests to 2 concurrent so each gets +# enough CPU to start, while the fast in-process unit tests keep full +# parallelism. This mirrors why `cargo test` was green without giving up +# nextest's flaky-run detection. + +[test-groups] +cli-integration = { max-threads = 2 } + +[[profile.default.overrides]] +# `kind(test)` selects the integration tests under tests/ (not the crate's +# in-process unit tests), scoped to the digstore-cli package. +filter = 'package(digstore-cli) & kind(test)' +test-group = 'cli-integration' From 856a5223adfc7b8e35ad48fed11edd6c4af61556 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 16 Jul 2026 15:28:21 -0700 Subject: [PATCH 6/6] ci: bound all heavy wasmtime integration tests under nextest (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior cap covered only digstore-cli's integration tests, but the SLOW (>60s) digstore-compiler tests (large_data_section, self_serving) kept running at full concurrency and saturated the 4-vCPU CI runners — so cli_dev's `dev` child process still couldn't get its tokio serve loop scheduled to answer `/` within the readiness poll, failing dev_serves_real_read_path_with_injected_shims deterministically on both OSes. - Widen the nextest test-group to ALL heavy integration tests across digstore-cli AND digstore-compiler, capped at 2 concurrent, so the runner always retains scheduler slack for the dev child while fast unit tests keep full parallelism (a local --test-threads 2 full-suite run reproduces green). - Widen the dev-server readiness budget (poll window 40s -> 90s, per-request read timeout 10s -> 30s) as independent insurance: on a loaded runner the child's serve loop may be scheduled late, and the test must outlast that rather than give up early. Co-Authored-By: Claude --- .config/nextest.toml | 36 +++++++++++++++------------- crates/digstore-cli/tests/cli_dev.rs | 11 +++++++-- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index dba583e7..a5a328a2 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -4,27 +4,29 @@ # instead of `cargo test`. The two runners parallelize differently, and that # difference broke one test: # -# digstore-cli's integration tests (crates/digstore-cli/tests/*.rs) each spawn -# a child `digstore` process. Serving/reading through the real chia:// path -# makes that child JIT-compile the guest wasm in wasmtime — several seconds of -# CPU and a non-trivial memory spike per process. `cargo test` runs test -# BINARIES one at a time, so only one such child compiles at a time. nextest -# instead runs num-cpus tests from a single global pool, so on the 4-vCPU CI -# runners several wasmtime compiles pile up at once and starve the -# `cli_dev` dev-server child past its 40s readiness poll — failing +# The heavy integration tests in digstore-cli AND digstore-compiler exercise +# the real chia:// read/compile path, which JIT-compiles the guest wasm in +# wasmtime — many seconds of CPU per test (nextest even marks several SLOW, +# >60s). `cargo test` runs test BINARIES one at a time, so only one such heavy +# test compiles at a time. nextest instead runs num-cpus tests from a single +# global pool, so on the 4-vCPU CI runners several of these fully saturate the +# runner at once. cli_dev's `dev` child process (a separate process, not a +# nextest thread) then can't get its tokio serve loop scheduled to accept `/` +# within the test's readiness poll — failing # `dev_serves_real_read_path_with_injected_shims` ("dev server did not come # up") deterministically on both ubuntu-latest and windows-latest. # -# Cap the heavy child-spawning integration tests to 2 concurrent so each gets -# enough CPU to start, while the fast in-process unit tests keep full -# parallelism. This mirrors why `cargo test` was green without giving up -# nextest's flaky-run detection. +# Cap ALL of these heavy integration tests (across both crates) to 2 concurrent +# via one shared group, so the runner always keeps scheduler slack for the dev +# child while the fast in-process unit tests keep full parallelism. This mirrors +# why `cargo test` was green (a local `--test-threads 2` full-suite run +# reproduces green) without giving up nextest's flaky-run detection. [test-groups] -cli-integration = { max-threads = 2 } +heavy-integration = { max-threads = 2 } [[profile.default.overrides]] -# `kind(test)` selects the integration tests under tests/ (not the crate's -# in-process unit tests), scoped to the digstore-cli package. -filter = 'package(digstore-cli) & kind(test)' -test-group = 'cli-integration' +# `kind(test)` selects the integration tests under tests/ (not the crates' +# in-process unit tests), scoped to the two wasmtime-heavy crates. +filter = 'kind(test) & (package(digstore-cli) | package(digstore-compiler))' +test-group = 'heavy-integration' diff --git a/crates/digstore-cli/tests/cli_dev.rs b/crates/digstore-cli/tests/cli_dev.rs index 09de8b61..cc2152f1 100644 --- a/crates/digstore-cli/tests/cli_dev.rs +++ b/crates/digstore-cli/tests/cli_dev.rs @@ -26,7 +26,11 @@ use std::time::{Duration, Instant}; fn http_get(port: u16, path: &str) -> Option { let mut stream = TcpStream::connect(("127.0.0.1", port)).ok()?; stream - .set_read_timeout(Some(Duration::from_secs(10))) + // Generous per-request read timeout: on a CPU-saturated CI runner the + // child's tokio serve loop can be slow to get scheduled to answer, and a + // single poll attempt must outlast that scheduling delay rather than give + // up early and force a fresh connection. + .set_read_timeout(Some(Duration::from_secs(30))) .ok()?; let req = format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n"); stream.write_all(req.as_bytes()).ok()?; @@ -96,7 +100,10 @@ fn http_get_retry(port: u16, path: &str) -> Option { /// Poll until the dev server answers `/` (or time out). fn wait_for_server(port: u16) -> Option { - let deadline = Instant::now() + Duration::from_secs(40); + // Generous overall window: under nextest the child competes with the rest of + // the suite for CPU, so it may take considerably longer than on an idle + // machine before its serve loop is scheduled to answer `/`. + let deadline = Instant::now() + Duration::from_secs(90); while Instant::now() < deadline { if let Some(body) = http_get(port, "/") { if !body.is_empty() {