From 9119af3e320f6162dc7515ea77edadb01cf1c6aa Mon Sep 17 00:00:00 2001
From: MadivB <yuxuan@caltech.edu>
Date: Sun, 28 Jun 2026 20:56:08 -0700
Subject: [PATCH 1/3] Add run-cl-matching production step (ND sim + 2x2 sim +
 2x2 data)

New step that runs charge-light matching after run-ndlar-flow. Drives the
external CLMatching_AlphaRelease repo via three wrappers following the
ND_Production convention (sources util/init.inc.sh, reads ND_PRODUCTION_*
env vars, writes outputs under <stepname>/<OUT_NAME>/{FLOW,PT}/<subDir>/):

  run_cl_matching_ND_sim.sh    in-place HDF5 writeback (Mode A: flow file
                               reserves t_0/t_cluster_id on calib_prompt_hits
                               and calib_final_hits). Output is a modified
                               .FLOW.hdf5, not a .pt. Falls back to .pt for
                               older flow files without those fields.
  run_cl_matching_2x2_sim.sh   2x2 simulation -> .qlmatch2x2.pt
  run_cl_matching_2x2_data.sh  2x2 real DAQ data -> .qlmatch2x2.pt
                               (input via ND_PRODUCTION_CLMATCH_DATA_FILE,
                               no upstream chain)

install_cl_matching.sh clones https://github.com/MadivB/CLMatching_AlphaRelease.git
into this directory (CLMatching_AlphaRelease/) and runs check_install.py
to surface any missing assets. Uses the shared NERSC nersc-python (torch
already installed there) -- no dedicated venv built here.

admin/install_everything.sh: chain run-cl-matching install in alongside
the other steps.

The cloned CLMatching_AlphaRelease/ is .gitignored (mirrors what
run-larnd-sim/ and run-ndlar-flow/ do for their cloned sources).

CLMatching requires a 4-GPU node per task and 8 workers. Standard
fw_gpu.slurm.sh in fireworks4dune uses --ntasks-per-node=4 --gpus-per-task=1
which would give each fw task only 1 GPU; when a cl_matching spec yaml is
added later it should pair with a new slurm wrapper (--ntasks-per-node=1
--gpus-per-node=4) so each fw task owns the whole node.
---
 admin/install_everything.sh                 |  4 ++
 run-cl-matching/.gitignore                  |  1 +
 run-cl-matching/install_cl_matching.sh      | 50 ++++++++++++++
 run-cl-matching/run_cl_matching_2x2_data.sh | 72 +++++++++++++++++++++
 run-cl-matching/run_cl_matching_2x2_sim.sh  | 70 ++++++++++++++++++++
 run-cl-matching/run_cl_matching_ND_sim.sh   | 72 +++++++++++++++++++++
 6 files changed, 269 insertions(+)
 create mode 100644 run-cl-matching/.gitignore
 create mode 100755 run-cl-matching/install_cl_matching.sh
 create mode 100755 run-cl-matching/run_cl_matching_2x2_data.sh
 create mode 100755 run-cl-matching/run_cl_matching_2x2_sim.sh
 create mode 100755 run-cl-matching/run_cl_matching_ND_sim.sh

diff --git a/admin/install_everything.sh b/admin/install_everything.sh
index 82e150af..2dcd529d 100755
--- a/admin/install_everything.sh
+++ b/admin/install_everything.sh
@@ -28,6 +28,10 @@ pushd run-ndlar-flow
 ./install_ndlar_flow.sh
 popd
 
+pushd run-cl-matching
+./install_cl_matching.sh
+popd
+
 pushd run-pandora
 ./install_pandora.sh "$detector"
 popd
diff --git a/run-cl-matching/.gitignore b/run-cl-matching/.gitignore
new file mode 100644
index 00000000..7d171f11
--- /dev/null
+++ b/run-cl-matching/.gitignore
@@ -0,0 +1 @@
+/CLMatching_AlphaRelease
diff --git a/run-cl-matching/install_cl_matching.sh b/run-cl-matching/install_cl_matching.sh
new file mode 100755
index 00000000..b7e5e671
--- /dev/null
+++ b/run-cl-matching/install_cl_matching.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Install the CLMatching_AlphaRelease repo into this directory.
+#
+# Usage:
+#   ./install_cl_matching.sh              # clones into ./CLMatching_AlphaRelease
+#   ./install_cl_matching.sh /some/dir    # clones into /some/dir/CLMatching_AlphaRelease
+#
+# Notes:
+#   * We do NOT build a dedicated venv here. The CL matching code uses the
+#     shared NERSC nersc-python (which already includes torch + cuda). The
+#     runtime scripts pin the python via the PY env var.
+#   * On a fresh clone the 2x2 perceiver / pulse / variance assets ship
+#     in-repo. The ND-LAr perceiver and a few pulse/variance files live on
+#     GitHub Releases and need to be downloaded; check_install.py reports
+#     what's missing and prints the download commands.
+
+source ../util/prelude.inc.sh
+
+installDir=${1:-.}
+repoName=CLMatching_AlphaRelease
+repoURL=https://github.com/MadivB/CLMatching_AlphaRelease.git
+repoBranch=${ND_PRODUCTION_CLMATCH_BRANCH:-main}
+
+if [[ -e "$installDir/$repoName" ]]; then
+  echo "$installDir/$repoName already exists; delete it then run me again"
+  exit 1
+fi
+
+mkdir -p "$installDir"
+cd "$installDir"
+
+git clone -b "$repoBranch" "$repoURL" "$repoName"
+
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+
+cd "$repoName"
+
+echo
+echo "--- Validating required assets via scripts/check_install.py ---"
+"$PY" scripts/check_install.py || {
+    echo
+    echo "ERROR: one or more required CL matching assets are missing."
+    echo "       Run the download command(s) printed above, then re-run:"
+    echo "         $PY scripts/check_install.py"
+    echo
+    echo "       The 2x2 assets are bundled in-repo (should always resolve)."
+    echo "       The ND-LAr perceiver is hosted on GitHub Releases."
+    exit 1
+}
+echo "--- CLMatching install OK ---"
diff --git a/run-cl-matching/run_cl_matching_2x2_data.sh b/run-cl-matching/run_cl_matching_2x2_data.sh
new file mode 100755
index 00000000..66558e7f
--- /dev/null
+++ b/run-cl-matching/run_cl_matching_2x2_data.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# 2x2 charge-light matching (real DATA).
+#
+# Real 2x2 DAQ flow files are NOT produced by ND_Production's run-ndlar-flow
+# step; they live on the dune cfs area (or wherever the user points us). The
+# input file is therefore identified by env var rather than the production
+# IN_NAME / INDEX convention used by sim steps.
+#
+# Input  : $ND_PRODUCTION_CLMATCH_DATA_FILE (absolute path to a 2x2 reflow file)
+#          default = /global/cfs/cdirs/dune/www/data/2x2/reflows/v10/flow/beam/
+#                    july10_2024/nominal_hv/packet-0050018-2024_07_10_09_36_12_CDT.FLOW.hdf5
+# Output : run-cl-matching/<OUT_NAME>/PT/<subDir>/<outName>.qlmatch2x2.pt
+#
+# Algorithm version (env ND_PRODUCTION_CLMATCH_VERSION):
+#   v1.0 (default) = error-matrix small-cluster association
+#   v2.0           = region-grow + learned-variance tiebreaker
+#
+# Requires a 4-GPU node. On Perlmutter:
+#   salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \
+#     srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_2x2_data.sh
+
+source ../util/reload_in_container.inc.sh
+source ../util/init.inc.sh
+
+CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"}
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+VERSION=${ND_PRODUCTION_CLMATCH_VERSION:-v1.0}
+
+DEFAULT_DATA_FILE=/global/cfs/cdirs/dune/www/data/2x2/reflows/v10/flow/beam/july10_2024/nominal_hv/packet-0050018-2024_07_10_09_36_12_CDT.FLOW.hdf5
+inFile=${ND_PRODUCTION_CLMATCH_DATA_FILE:-$DEFAULT_DATA_FILE}
+
+if [[ ! -d "$CLMATCH_REPO" ]]; then
+    echo "ERROR: CLMatching repo not found at $CLMATCH_REPO"
+    echo "       Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2
+    exit 1
+fi
+if [[ ! -f "$inFile" ]]; then
+    echo "ERROR: input data file not found: $inFile"
+    echo "       Set ND_PRODUCTION_CLMATCH_DATA_FILE to point at a 2x2 reflow .FLOW.hdf5." >&2
+    exit 2
+fi
+
+workDir=$tmpOutDir/${outName}_work
+ptName=${outName}.qlmatch2x2.pt
+rm -rf "$workDir"
+
+set -o errexit
+mkdir -p "$workDir"
+
+cd "$CLMATCH_REPO"
+
+run env FILE="$inFile" \
+    VERSION="$VERSION" \
+    OUT_DIR="$workDir" \
+    PT_DIR="$workDir/pt_outputs" \
+    LOG_DIR="$workDir/logs" \
+    PY="$PY" HERE="$CLMATCH_REPO" \
+    bash scripts/run_2x2_data.sh
+
+srcBasename=$(basename "$inFile" .hdf5)
+producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt"
+if [[ ! -f "$producedPt" ]]; then
+    echo "ERROR: expected .pt not found at $producedPt" >&2
+    echo "       contents of $workDir/pt_outputs:" >&2
+    ls -la "$workDir/pt_outputs" >&2 || true
+    exit 3
+fi
+
+mkdir -p "$outDir/PT/$subDir"
+mv "$producedPt" "$outDir/PT/$subDir/$ptName"
+rm -f "$workDir"/*.npz "$workDir"/*.json
diff --git a/run-cl-matching/run_cl_matching_2x2_sim.sh b/run-cl-matching/run_cl_matching_2x2_sim.sh
new file mode 100755
index 00000000..2fc429fb
--- /dev/null
+++ b/run-cl-matching/run_cl_matching_2x2_sim.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# 2x2 charge-light matching (simulation).
+#
+# Input  : run-ndlar-flow/<IN_NAME>/FLOW/<subDir>/<inName>.FLOW.hdf5
+#          (a 2x2-configured flow file)
+# Output : run-cl-matching/<OUT_NAME>/PT/<subDir>/<outName>.qlmatch2x2.pt
+#
+# The 2x2 workflow always produces a .pt (the 2x2 calib_*_hits dtypes do not
+# yet reserve t_0/t_cluster_id, so we cannot do in-place HDF5 writeback).
+#
+# Algorithm version (env ND_PRODUCTION_CLMATCH_VERSION):
+#   v1.0 (default) = error-matrix small-cluster association (greedy, unit-var)
+#   v2.0           = region-grow + learned-variance tiebreaker
+#
+# Requires a 4-GPU node. On Perlmutter:
+#   salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \
+#     srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_2x2_sim.sh
+
+source ../util/reload_in_container.inc.sh
+source ../util/init.inc.sh
+
+CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"}
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+VERSION=${ND_PRODUCTION_CLMATCH_VERSION:-v1.0}
+
+if [[ ! -d "$CLMATCH_REPO" ]]; then
+    echo "ERROR: CLMatching repo not found at $CLMATCH_REPO"
+    echo "       Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2
+    exit 1
+fi
+
+inDir=${ND_PRODUCTION_OUTDIR_BASE}/run-ndlar-flow/$ND_PRODUCTION_IN_NAME
+inName=$ND_PRODUCTION_IN_NAME.$globalIdx
+inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5)
+
+# 2x2 produces a .pt, not a modified HDF5. Stage workers/shards in a per-file
+# tmp dir; mv the final .pt into the canonical outDir at the end.
+workDir=$tmpOutDir/${outName}_work
+ptName=${outName}.qlmatch2x2.pt
+rm -rf "$workDir"
+
+set -o errexit
+mkdir -p "$workDir"
+
+cd "$CLMATCH_REPO"
+
+run env FILE="$inFile" \
+    VERSION="$VERSION" \
+    OUT_DIR="$workDir" \
+    PT_DIR="$workDir/pt_outputs" \
+    LOG_DIR="$workDir/logs" \
+    PY="$PY" HERE="$CLMATCH_REPO" \
+    bash scripts/run_2x2_sim.sh
+
+# The aggregator names the .pt after the input file's basename (with .hdf5
+# stripped, then '.qlmatch2x2.pt' appended).
+srcBasename=$(basename "$inFile" .hdf5)
+producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt"
+if [[ ! -f "$producedPt" ]]; then
+    echo "ERROR: expected .pt not found at $producedPt" >&2
+    echo "       contents of $workDir/pt_outputs:" >&2
+    ls -la "$workDir/pt_outputs" >&2 || true
+    exit 2
+fi
+
+mkdir -p "$outDir/PT/$subDir"
+mv "$producedPt" "$outDir/PT/$subDir/$ptName"
+# Keep worker logs for debugging; drop the per-event NPZ shards (large, transient).
+rm -f "$workDir"/*.npz "$workDir"/*.json
diff --git a/run-cl-matching/run_cl_matching_ND_sim.sh b/run-cl-matching/run_cl_matching_ND_sim.sh
new file mode 100755
index 00000000..f4b00ba6
--- /dev/null
+++ b/run-cl-matching/run_cl_matching_ND_sim.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# ND-LAr charge-light matching (simulation).
+#
+# Input  : run-ndlar-flow/<IN_NAME>/FLOW/<subDir>/<inName>.FLOW.hdf5
+# Output : run-cl-matching/<OUT_NAME>/FLOW/<subDir>/<outName>.FLOW.hdf5
+#
+# This step rewrites t_0 and t_cluster_id INSIDE the FLOW.hdf5 (Mode A; the
+# new flow dtype reserves those fields). The output is therefore another
+# .FLOW.hdf5, not a .pt. For older flow files without those fields the
+# underlying CLMatching aggregator falls back to .pt under
+# <CLMatching repo>/output/QLmatchingvAlpha/ -- but for ND production we
+# assume Mode A.
+#
+# Requires a 4-GPU node. On Perlmutter:
+#   salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \
+#     srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_ND_sim.sh
+#
+# (Or invoke inside an sbatch script that grabs a GPU node.)
+
+source ../util/reload_in_container.inc.sh
+source ../util/init.inc.sh
+
+CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"}
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+
+if [[ ! -d "$CLMATCH_REPO" ]]; then
+    echo "ERROR: CLMatching repo not found at $CLMATCH_REPO"
+    echo "       Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2
+    exit 1
+fi
+
+inDir=${ND_PRODUCTION_OUTDIR_BASE}/run-ndlar-flow/$ND_PRODUCTION_IN_NAME
+inName=$ND_PRODUCTION_IN_NAME.$globalIdx
+inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5)
+
+# CL matching modifies the flow file IN-PLACE for Mode A. To avoid scribbling
+# on the upstream step's output, copy to tmpOutDir first, run there, then mv
+# the modified file to the canonical outDir.
+outFile=$tmpOutDir/${outName}.FLOW.hdf5
+rm -f "$outFile"
+
+set -o errexit
+echo "Copying input flow file to tmp work area:"
+echo "  $inFile -> $outFile"
+cp "$inFile" "$outFile"
+
+cd "$CLMATCH_REPO"
+
+# The single-file driver auto-detects Mode A (in-place HDF5) vs Mode B (.pt)
+# from the source dtype and runs the 8-worker pipeline + aggregation.
+run env PY="$PY" REPO="$CLMATCH_REPO" \
+    bash scripts/process_one_flow_file.sh "$outFile"
+
+# Sanity-check that Mode A populated t_0/t_cluster_id.
+"$PY" - <<PY
+import h5py, sys
+with h5py.File("$outFile", "r") as h:
+    for path in ("charge/calib_prompt_hits/data", "charge/calib_final_hits/data"):
+        d = h[path]
+        if "t_0" not in d.dtype.names or "t_cluster_id" not in d.dtype.names:
+            print(f"WARN: {path} dtype lacks t_0/t_cluster_id (Mode B file). Skipping HDF5-populated check.")
+            sys.exit(0)
+        nz = int((d["t_cluster_id"][:] != 0).any() or (d["t_0"][:] != 0).any())
+        if not nz:
+            print(f"ERROR: {path} t_0 and t_cluster_id are still all zero after CL matching.", file=sys.stderr)
+            sys.exit(2)
+print("Mode A writeback verified: t_0 and t_cluster_id populated in HDF5.")
+PY
+
+mkdir -p "$outDir/FLOW/$subDir"
+mv "$outFile" "$outDir/FLOW/$subDir"

From 2aeef1b99ea19be56cb99951984dfeaa2d797058 Mon Sep 17 00:00:00 2001
From: MadivB <yuxuan@caltech.edu>
Date: Mon, 29 Jun 2026 10:47:28 -0700
Subject: [PATCH 2/3] run-cl-matching: fix .pt basename + route ND worker
 shards to per-task tmpdir

Two bugs surfaced when running the wrappers fresh on a real GPU node:

1. 2x2 wrappers stripped '.hdf5' when computing the expected .pt name, but
   the underlying aggregate_2x2_to_pt.py keeps '.hdf5' in the output name
   ('<basename incl. .hdf5>.qlmatch2x2.pt'). The wrappers therefore failed
   the post-run existence check even though the .pt was produced correctly.
   Fix: use 'basename "\$inFile"' instead of 'basename "\$inFile" .hdf5'.

2. run_cl_matching_ND_sim.sh invoked scripts/process_one_flow_file.sh
   without an explicit out_dir, so worker NPZ shards landed at
   <CLMatching_repo>/output/QLmatchingvAlpha/<basename>/. Over 10k+ files
   this would explode the cloned repo's output/ dir.
   Fix: pass per-task \$workDir under tmpOutDir as the 2nd positional arg
   to process_one_flow_file.sh; preserve the worker logs under the
   canonical LOGS dir, then rm -rf \$workDir after the .FLOW.hdf5 is moved.

Verified Bug 1 fix against the .pt produced by the previous failing run
(file exists at fixed path, not at the buggy path).
---
 run-cl-matching/run_cl_matching_2x2_data.sh |  3 ++-
 run-cl-matching/run_cl_matching_2x2_sim.sh  |  6 +++---
 run-cl-matching/run_cl_matching_ND_sim.sh   | 15 ++++++++++++++-
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/run-cl-matching/run_cl_matching_2x2_data.sh b/run-cl-matching/run_cl_matching_2x2_data.sh
index 66558e7f..bbec18f5 100755
--- a/run-cl-matching/run_cl_matching_2x2_data.sh
+++ b/run-cl-matching/run_cl_matching_2x2_data.sh
@@ -58,7 +58,8 @@ run env FILE="$inFile" \
     PY="$PY" HERE="$CLMATCH_REPO" \
     bash scripts/run_2x2_data.sh
 
-srcBasename=$(basename "$inFile" .hdf5)
+# The aggregator names the .pt as '<full basename incl. .hdf5>.qlmatch2x2.pt'.
+srcBasename=$(basename "$inFile")
 producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt"
 if [[ ! -f "$producedPt" ]]; then
     echo "ERROR: expected .pt not found at $producedPt" >&2
diff --git a/run-cl-matching/run_cl_matching_2x2_sim.sh b/run-cl-matching/run_cl_matching_2x2_sim.sh
index 2fc429fb..8aea440c 100755
--- a/run-cl-matching/run_cl_matching_2x2_sim.sh
+++ b/run-cl-matching/run_cl_matching_2x2_sim.sh
@@ -53,9 +53,9 @@ run env FILE="$inFile" \
     PY="$PY" HERE="$CLMATCH_REPO" \
     bash scripts/run_2x2_sim.sh
 
-# The aggregator names the .pt after the input file's basename (with .hdf5
-# stripped, then '.qlmatch2x2.pt' appended).
-srcBasename=$(basename "$inFile" .hdf5)
+# The aggregator names the .pt as '<full basename incl. .hdf5>.qlmatch2x2.pt'
+# (yes, it keeps the .hdf5 in the .pt filename).
+srcBasename=$(basename "$inFile")
 producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt"
 if [[ ! -f "$producedPt" ]]; then
     echo "ERROR: expected .pt not found at $producedPt" >&2
diff --git a/run-cl-matching/run_cl_matching_ND_sim.sh b/run-cl-matching/run_cl_matching_ND_sim.sh
index f4b00ba6..4554a226 100755
--- a/run-cl-matching/run_cl_matching_ND_sim.sh
+++ b/run-cl-matching/run_cl_matching_ND_sim.sh
@@ -38,9 +38,12 @@ inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5)
 # on the upstream step's output, copy to tmpOutDir first, run there, then mv
 # the modified file to the canonical outDir.
 outFile=$tmpOutDir/${outName}.FLOW.hdf5
+workDir=$tmpOutDir/${outName}_work
 rm -f "$outFile"
+rm -rf "$workDir"
 
 set -o errexit
+mkdir -p "$workDir"
 echo "Copying input flow file to tmp work area:"
 echo "  $inFile -> $outFile"
 cp "$inFile" "$outFile"
@@ -49,8 +52,10 @@ cd "$CLMATCH_REPO"
 
 # The single-file driver auto-detects Mode A (in-place HDF5) vs Mode B (.pt)
 # from the source dtype and runs the 8-worker pipeline + aggregation.
+# Explicitly route worker shards/logs through our per-task $workDir so they
+# do not accumulate inside the cloned CLMatching repo across production runs.
 run env PY="$PY" REPO="$CLMATCH_REPO" \
-    bash scripts/process_one_flow_file.sh "$outFile"
+    bash scripts/process_one_flow_file.sh "$outFile" "$workDir"
 
 # Sanity-check that Mode A populated t_0/t_cluster_id.
 "$PY" - <<PY
@@ -70,3 +75,11 @@ PY
 
 mkdir -p "$outDir/FLOW/$subDir"
 mv "$outFile" "$outDir/FLOW/$subDir"
+
+# Worker shards are large and transient; preserve the worker logs (small) under
+# the canonical LOGS dir so failures stay debuggable, then drop the rest.
+if [[ -d "$workDir/worker_logs" ]]; then
+    mkdir -p "$logDir"
+    cp -r "$workDir/worker_logs" "$logDir/${outName}_worker_logs" 2>/dev/null || true
+fi
+rm -rf "$workDir"

From 6c80d26a43fdcdb640f443f7d4f01df8d46ac650 Mon Sep 17 00:00:00 2001
From: MadivB <yuxuan@caltech.edu>
Date: Wed, 1 Jul 2026 16:13:31 -0700
Subject: [PATCH 3/3] run_cl_matching_ND_sim: don't trust process_one_flow_file
 rc for Mode A

The ND sim wrapper's `run env ... process_one_flow_file.sh` was tripping
`set -o errexit` because the underlying CLMatching launcher's trailing
`ls .../*.v_alpha_test.pt | head` fails rc=2 in Mode A (no .pt produced),
which `set -o pipefail` propagates. Even though the HDF5 writeback fully
succeeded, the wrapper aborted before running its Python sanity-check or
mv'ing the file to outDir/FLOW/<subDir>/.

Bracket the `run` call with `set +/-o errexit` so the exit code cannot
prematurely kill us; the Python sanity check below is the authoritative
Mode A success criterion. A companion fix in CLMatching_AlphaRelease
(commit 056026d) removes the underlying spurious exit for future runs,
but this belt-and-suspenders is cheap and lets the wrapper stay correct
even against pre-fix CLMatching checkouts.
---
 run-cl-matching/run_cl_matching_ND_sim.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/run-cl-matching/run_cl_matching_ND_sim.sh b/run-cl-matching/run_cl_matching_ND_sim.sh
index 4554a226..0006a998 100755
--- a/run-cl-matching/run_cl_matching_ND_sim.sh
+++ b/run-cl-matching/run_cl_matching_ND_sim.sh
@@ -54,8 +54,15 @@ cd "$CLMATCH_REPO"
 # from the source dtype and runs the 8-worker pipeline + aggregation.
 # Explicitly route worker shards/logs through our per-task $workDir so they
 # do not accumulate inside the cloned CLMatching repo across production runs.
+#
+# Bracket the `run` call in `set +/-o errexit`: process_one_flow_file.sh (and
+# its underlying launcher) may exit non-zero from spurious tail commands (e.g.
+# ls-ing pt_outputs that Mode A never populates). The Python sanity-check
+# below is the authoritative Mode A success criterion; do not trust rc here.
+set +o errexit
 run env PY="$PY" REPO="$CLMATCH_REPO" \
     bash scripts/process_one_flow_file.sh "$outFile" "$workDir"
+set -o errexit
 
 # Sanity-check that Mode A populated t_0/t_cluster_id.
 "$PY" - <<PY