From f8baafd2bf13d53124566f965172e7692932d800 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Tue, 26 May 2026 09:04:21 -0500 Subject: [PATCH 1/2] test: add run_full_fuzz.sh deep-campaign helper for devs Adds fuzz/run_full_fuzz.sh: runs every cargo-fuzz target in libFuzzer fork mode for a per-target budget, persisting the (resumable) corpus and collecting crash artifacts, with a clean/dirty exit status. Safe-by-default for any machine, not just a big workstation: - FORKS defaults to cores-2 (machine stays responsive; FORKS=$(nproc) for all), and the run prints estimated total time + peak RAM (~FORKS x RSS_LIMIT_MB) up front. - When run interactively it waits 5s so a heavy run can be aborted and re-launched with smaller knobs; skipped under nohup/redirection. - Tunables: SECS_PER_TARGET, FORKS, RSS_LIMIT_MB, TARGETS. Quick run: SECS_PER_TARGET=120 FORKS=2 ./fuzz/run_full_fuzz.sh (~14 min, 2 cores). Also gitignores fuzz/*.log (campaign logs) and adds a CONTRIBUTING "Fuzzing" section pointing devs at the CI smoke (fuzz.yml), ad-hoc cargo-fuzz, and this helper. Validated with `bash -n` and a config-arithmetic simulation across machine profiles; not run live (leaving the CPU free). Signed-off-by: Nelson Spence --- CONTRIBUTING.md | 18 +++++++ fuzz/.gitignore | 3 ++ fuzz/run_full_fuzz.sh | 110 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100755 fuzz/run_full_fuzz.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a2781369..739322c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,6 +52,24 @@ cargo clippy -p ordvec-python --all-targets -- -D warnings maturin develop && pytest ordvec-python/tests # in a virtualenv ``` +### Fuzzing + +The loader, write→load round-trip, and FastScan paths have seven cargo-fuzz +targets in `fuzz/`. CI runs a bounded smoke on every PR (`fuzz.yml`); locally +you need a nightly toolchain and `cargo install cargo-fuzz`: + +```sh +cargo +nightly fuzz build # compile all targets +cargo +nightly fuzz run load_rank # one target, ad hoc +./fuzz/run_full_fuzz.sh # full deep campaign (all targets) +``` + +`run_full_fuzz.sh` runs each target in libFuzzer fork mode, persists the corpus +(resumable across runs), and collects crash artifacts. **It is heavy by +default** (~3h × 7 targets, `cores − 2` forks) — dial it down on a laptop via +env knobs, e.g. `SECS_PER_TARGET=120 FORKS=2 ./fuzz/run_full_fuzz.sh`. See the +script header for all knobs. + ## Workflow - **Branches:** `/` (feat, fix, refactor, docs, test, chore, diff --git a/fuzz/.gitignore b/fuzz/.gitignore index 1a45eee7..3e8eab7e 100644 --- a/fuzz/.gitignore +++ b/fuzz/.gitignore @@ -2,3 +2,6 @@ target corpus artifacts coverage + +# Campaign logs from run_full_fuzz.sh (e.g. full_fuzz_run.log) +*.log diff --git a/fuzz/run_full_fuzz.sh b/fuzz/run_full_fuzz.sh new file mode 100755 index 00000000..230f18bf --- /dev/null +++ b/fuzz/run_full_fuzz.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Full deep fuzz campaign across every ordvec cargo-fuzz target. +# +# Tuned for a high-core workstation (e.g. Ryzen 9950X / 128 GB DDR5). Runs each +# target in libFuzzer *fork* mode across all cores for a per-target wall-clock +# budget, persisting the corpus (runs are cumulative / resumable) and collecting +# any crash artifacts. Fork mode is resilient: a crashing/OOMing child records +# its artifact and the campaign keeps going, so one bad input never ends the run. +# +# Requires: a nightly toolchain and cargo-fuzz (`cargo install cargo-fuzz`). +# +# HEAVY BY DEFAULT. The defaults are a long, many-core campaign (~3h x 7 +# targets ~= 21h total; FORKS = cores - 2; peak RAM ~= FORKS x RSS_LIMIT_MB) +# tuned for a big workstation. On a laptop or smaller box, DIAL IT DOWN with the +# env knobs below so you don't peg every core or exhaust RAM. A quick run: +# +# SECS_PER_TARGET=120 FORKS=2 ./fuzz/run_full_fuzz.sh # ~14 min on 2 cores +# +# The script prints the estimated total time + RAM up front and, when run +# interactively, waits 5s so you can Ctrl-C and re-run with smaller knobs. +# +# Launch it detached so it survives the terminal/session: +# +# setsid nohup ./fuzz/run_full_fuzz.sh > fuzz/full_fuzz_run.log 2>&1 & +# tail -f fuzz/full_fuzz_run.log # watch +# pkill -f 'cargo.*fuzz run' # stop early (corpus is kept) +# +# Tunables (env): +# SECS_PER_TARGET per-target wall-clock budget (default 10800 = 3h) +# FORKS concurrent fork workers (default = nproc - 2) +# RSS_LIMIT_MB per-process RSS cap (default 3072) +# TARGETS space-separated target list (default = all seven) +# +# Examples: +# SECS_PER_TARGET=43200 ./fuzz/run_full_fuzz.sh # 12h per target +# TARGETS="fastscan_b2 search_rankquant" ./fuzz/run_full_fuzz.sh +set -u + +cd "$(dirname "$0")/.." || exit 1 + +SECS_PER_TARGET="${SECS_PER_TARGET:-10800}" +# All cores but two, so the machine stays responsive and a small box is not +# pegged. Use FORKS=$(nproc) for every core. Peak RAM ~= FORKS x RSS_LIMIT_MB. +NCPU="$(nproc)" +FORKS="${FORKS:-$(( NCPU > 2 ? NCPU - 2 : 1 ))}" +RSS_LIMIT_MB="${RSS_LIMIT_MB:-3072}" +TARGETS="${TARGETS:-load_rank load_rankquant load_bitmap load_sign_bitmap roundtrip_rankquant search_rankquant fastscan_b2}" + +read -ra _targets <<<"${TARGETS}" +n_targets=${#_targets[@]} +total_secs=$(( SECS_PER_TARGET * n_targets )) +echo "=== ordvec full fuzz campaign ===" +echo "start: $(date -Is)" +echo "secs/target: ${SECS_PER_TARGET} (~$(( SECS_PER_TARGET / 60 ))m each)" +echo "targets: ${n_targets} — ${TARGETS}" +echo "est. total: ~$(( total_secs / 3600 ))h $(( total_secs % 3600 / 60 ))m (targets run sequentially)" +echo "forks: ${FORKS} (of ${NCPU} cores)" +echo "rss limit (MB): ${RSS_LIMIT_MB} → peak RAM ~$(( FORKS * RSS_LIMIT_MB / 1024 )) GB" +echo "host: $(uname -srm)" +echo +# Interactive abort window: when stdout is a terminal, pause so a heavy run can +# be cancelled and re-launched with smaller knobs. Skipped under redirection +# (e.g. nohup ... > log) so detached campaigns start immediately. +if [ -t 1 ]; then + echo "Heavy run — Ctrl-C within 5s to abort (or re-run with SECS_PER_TARGET=… FORKS=…)." + sleep 5 + echo +fi + +# Build once up front so a compile error fails fast (not mid-campaign). +echo "=== building all fuzz targets (release) ===" +if ! cargo +nightly fuzz build; then + echo "FATAL: fuzz build failed; aborting campaign." >&2 + exit 1 +fi +echo + +mkdir -p fuzz/corpus fuzz/artifacts + +for t in ${TARGETS}; do + echo "############################################################" + echo "### target: ${t} started $(date -Is)" + echo "############################################################" + mkdir -p "fuzz/corpus/${t}" "fuzz/artifacts/${t}" + cargo +nightly fuzz run "${t}" -- \ + -fork="${FORKS}" \ + -ignore_crashes=1 \ + -rss_limit_mb="${RSS_LIMIT_MB}" \ + -max_total_time="${SECS_PER_TARGET}" \ + -print_final_stats=1 + echo "### target ${t} finished $(date -Is) (libfuzzer rc=$?)" + echo +done + +echo "============================================================" +echo "=== campaign complete $(date -Is) — crash artifact summary ===" +crashes=$(find fuzz/artifacts -type f \ + \( -name 'crash-*' -o -name 'oom-*' -o -name 'timeout-*' -o -name 'leak-*' \) 2>/dev/null) +if [ -z "${crashes}" ]; then + echo "CLEAN: no crash / oom / timeout / leak artifacts across any target." + status=0 +else + echo "ARTIFACTS FOUND — investigate before publishing:" + echo "${crashes}" + status=1 +fi +echo +echo "corpus sizes:" +du -sh fuzz/corpus/* 2>/dev/null +exit "${status}" From 6d5a35de7fd1c948da7847333f4b355cbada1826 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Tue, 26 May 2026 09:47:45 -0500 Subject: [PATCH 2/2] fix(fuzz): portability + correctness in run_full_fuzz.sh Addresses PR #74 review (gemini/qodo/copilot/codex): - Portable CPU count: nproc -> getconf -> BSD/macOS sysctl -> 1, so a missing nproc on macOS no longer leaves NCPU empty under set -u. - Portable timestamps: GNU 'date -Is' -> 'date -u +%Y-%m-%dT%H:%M:%SZ' (a now() helper), which BSD/macOS date supports. - Ctrl-C trap (SIGINT): stops the whole campaign instead of killing one target's fuzzer and marching to the next. - Propagate exit codes: each target's rc is captured; a non-zero run sets any_fail and the final status is non-zero (was only logged before, Codex P1). - Iterate the parsed array (for t in "${_targets[@]}") instead of unquoted word-splitting over $TARGETS. Validated: bash -n, now(), nproc fallback, and the propagate-on-failure logic. Signed-off-by: Nelson Spence --- fuzz/run_full_fuzz.sh | 44 +++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/fuzz/run_full_fuzz.sh b/fuzz/run_full_fuzz.sh index 230f18bf..a6d7bdcf 100755 --- a/fuzz/run_full_fuzz.sh +++ b/fuzz/run_full_fuzz.sh @@ -38,10 +38,19 @@ set -u cd "$(dirname "$0")/.." || exit 1 +# Portable UTC timestamp — GNU `date -Is` / `-I` isn't available on BSD/macOS. +now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# Ctrl-C stops the whole campaign, not just the current target's fuzzer — +# without this, killing one `cargo fuzz` lets the loop march on to the next. +trap 'echo; echo "interrupted — stopping campaign (corpus kept)."; exit 130' INT + SECS_PER_TARGET="${SECS_PER_TARGET:-10800}" -# All cores but two, so the machine stays responsive and a small box is not -# pegged. Use FORKS=$(nproc) for every core. Peak RAM ~= FORKS x RSS_LIMIT_MB. -NCPU="$(nproc)" +# Default to all cores but two, so the machine stays responsive and a small box +# isn't pegged; override with FORKS=. Peak RAM is roughly FORKS x +# RSS_LIMIT_MB. CPU count is detected portably (Linux nproc, then getconf, then +# BSD/macOS sysctl, else 1) so `set -u` never sees an empty NCPU. +NCPU="$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)" FORKS="${FORKS:-$(( NCPU > 2 ? NCPU - 2 : 1 ))}" RSS_LIMIT_MB="${RSS_LIMIT_MB:-3072}" TARGETS="${TARGETS:-load_rank load_rankquant load_bitmap load_sign_bitmap roundtrip_rankquant search_rankquant fastscan_b2}" @@ -50,7 +59,7 @@ read -ra _targets <<<"${TARGETS}" n_targets=${#_targets[@]} total_secs=$(( SECS_PER_TARGET * n_targets )) echo "=== ordvec full fuzz campaign ===" -echo "start: $(date -Is)" +echo "start: $(now)" echo "secs/target: ${SECS_PER_TARGET} (~$(( SECS_PER_TARGET / 60 ))m each)" echo "targets: ${n_targets} — ${TARGETS}" echo "est. total: ~$(( total_secs / 3600 ))h $(( total_secs % 3600 / 60 ))m (targets run sequentially)" @@ -77,9 +86,10 @@ echo mkdir -p fuzz/corpus fuzz/artifacts -for t in ${TARGETS}; do +any_fail=0 +for t in "${_targets[@]}"; do echo "############################################################" - echo "### target: ${t} started $(date -Is)" + echo "### target: ${t} started $(now)" echo "############################################################" mkdir -p "fuzz/corpus/${t}" "fuzz/artifacts/${t}" cargo +nightly fuzz run "${t}" -- \ @@ -88,22 +98,32 @@ for t in ${TARGETS}; do -rss_limit_mb="${RSS_LIMIT_MB}" \ -max_total_time="${SECS_PER_TARGET}" \ -print_final_stats=1 - echo "### target ${t} finished $(date -Is) (libfuzzer rc=$?)" + rc=$? + echo "### target ${t} finished $(now) (libfuzzer rc=${rc})" + if [ "${rc}" -ne 0 ]; then + echo "### WARNING: ${t} exited non-zero (rc=${rc}) — recorded as a campaign failure." + any_fail=1 + fi echo done echo "============================================================" -echo "=== campaign complete $(date -Is) — crash artifact summary ===" +echo "=== campaign complete $(now) — summary ===" +status=0 crashes=$(find fuzz/artifacts -type f \ \( -name 'crash-*' -o -name 'oom-*' -o -name 'timeout-*' -o -name 'leak-*' \) 2>/dev/null) -if [ -z "${crashes}" ]; then - echo "CLEAN: no crash / oom / timeout / leak artifacts across any target." - status=0 -else +if [ -n "${crashes}" ]; then echo "ARTIFACTS FOUND — investigate before publishing:" echo "${crashes}" status=1 fi +if [ "${any_fail}" -ne 0 ]; then + echo "One or more fuzz targets exited non-zero (see WARNING lines above)." + status=1 +fi +if [ "${status}" -eq 0 ]; then + echo "CLEAN: no crash / oom / timeout / leak artifacts, every target exited 0." +fi echo echo "corpus sizes:" du -sh fuzz/corpus/* 2>/dev/null