Feature/yuxuan cl matching integration

admin/install_everything.sh

@@ -21,14 +21,19 @@ popd
 if [ "$detector" = "ndlar" ] || [ "$detector" = "2x2" ]; then
   echo "Installation of $detector specific tools"
 
-  pushd run-larnd-sim 
+  pushd run-larnd-sim
   ./install_larnd_sim.sh
   popd
 
   pushd run-ndlar-flow
   ./install_ndlar_flow.sh
   popd
 
+  pushd run-cl-matching
+  echo "Installing CL matching tools"
+  ./install_cl_matching.sh
+  popd
+
   pushd run-pandora
   ./install_pandora.sh "$detector"
   popd

run-cl-matching/.gitignore

@@ -0,0 +1 @@
+/CLMatching_AlphaRelease

run-cl-matching/install_cl_matching.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Install the CLMatching_AlphaRelease repo into this directory.
+#
+# Usage:
+#   ./install_cl_matching.sh              # clones into ./CLMatching_AlphaRelease
+#   ./install_cl_matching.sh /some/dir    # clones into /some/dir/CLMatching_AlphaRelease
+#
+# Notes:
+#   * We do NOT build a dedicated venv here. The CL matching code uses the
+#     shared NERSC nersc-python (which already includes torch + cuda). The
+#     runtime scripts pin the python via the PY env var.
+#   * On a fresh clone the 2x2 perceiver / pulse / variance assets ship
+#     in-repo. The ND-LAr perceiver and a few pulse/variance files live on
+#     GitHub Releases and need to be downloaded; check_install.py reports
+#     what's missing and prints the download commands.
+
+source ../util/prelude.inc.sh
+
+installDir=${1:-.}
+repoName=CLMatching_AlphaRelease
+repoURL=https://github.com/MadivB/CLMatching_AlphaRelease.git
+repoBranch=${ND_PRODUCTION_CLMATCH_BRANCH:-main}
+
+if [[ -e "$installDir/$repoName" ]]; then
+  echo "$installDir/$repoName already exists; delete it then run me again"
+  exit 1
+fi
+
+mkdir -p "$installDir"
+cd "$installDir"
+
+git clone -b "$repoBranch" "$repoURL" "$repoName"
+
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+
+cd "$repoName"
+
+echo
+echo "--- Validating required assets via scripts/check_install.py ---"
+"$PY" scripts/check_install.py || {
+    echo
+    echo "ERROR: one or more required CL matching assets are missing."
+    echo "       Run the download command(s) printed above, then re-run:"
+    echo "         $PY scripts/check_install.py"
+    echo
+    echo "       The 2x2 assets are bundled in-repo (should always resolve)."
+    echo "       The ND-LAr perceiver is hosted on GitHub Releases."
+    exit 1
+}
+echo "--- CLMatching install OK ---"

run-cl-matching/run_cl_matching_2x2_data.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#
+# 2x2 charge-light matching (real DATA).
+#
+# Real 2x2 DAQ flow files are NOT produced by ND_Production's run-ndlar-flow
+# step; they live on the dune cfs area (or wherever the user points us). The
+# input file is therefore identified by env var rather than the production
+# IN_NAME / INDEX convention used by sim steps.
+#
+# Input  : $ND_PRODUCTION_CLMATCH_DATA_FILE (absolute path to a 2x2 reflow file)
+#          default = /global/cfs/cdirs/dune/www/data/2x2/reflows/v10/flow/beam/
+#                    july10_2024/nominal_hv/packet-0050018-2024_07_10_09_36_12_CDT.FLOW.hdf5
+# Output : run-cl-matching/<OUT_NAME>/PT/<subDir>/<outName>.qlmatch2x2.pt
+#
+# Algorithm version (env ND_PRODUCTION_CLMATCH_VERSION):
+#   v1.0 (default) = error-matrix small-cluster association
+#   v2.0           = region-grow + learned-variance tiebreaker
+#
+# Requires a 4-GPU node. On Perlmutter:
+#   salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \
+#     srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_2x2_data.sh
+
+source ../util/reload_in_container.inc.sh
+source ../util/init.inc.sh
+
+CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"}
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+VERSION=${ND_PRODUCTION_CLMATCH_VERSION:-v1.0}
+
+DEFAULT_DATA_FILE=/global/cfs/cdirs/dune/www/data/2x2/reflows/v10/flow/beam/july10_2024/nominal_hv/packet-0050018-2024_07_10_09_36_12_CDT.FLOW.hdf5
+inFile=${ND_PRODUCTION_CLMATCH_DATA_FILE:-$DEFAULT_DATA_FILE}
+
+if [[ ! -d "$CLMATCH_REPO" ]]; then
+    echo "ERROR: CLMatching repo not found at $CLMATCH_REPO"
+    echo "       Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2
+    exit 1
+fi
+if [[ ! -f "$inFile" ]]; then
+    echo "ERROR: input data file not found: $inFile"
+    echo "       Set ND_PRODUCTION_CLMATCH_DATA_FILE to point at a 2x2 reflow .FLOW.hdf5." >&2
+    exit 2
+fi
+
+workDir=$tmpOutDir/${outName}_work
+ptName=${outName}.qlmatch2x2.pt
+rm -rf "$workDir"
+
+set -o errexit
+mkdir -p "$workDir"
+
+cd "$CLMATCH_REPO"
+
+run env FILE="$inFile" \
+    VERSION="$VERSION" \
+    OUT_DIR="$workDir" \
+    PT_DIR="$workDir/pt_outputs" \
+    LOG_DIR="$workDir/logs" \
+    PY="$PY" HERE="$CLMATCH_REPO" \
+    bash scripts/run_2x2_data.sh
+
+# The aggregator names the .pt as '<full basename incl. .hdf5>.qlmatch2x2.pt'.
+srcBasename=$(basename "$inFile")
+producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt"
+if [[ ! -f "$producedPt" ]]; then
+    echo "ERROR: expected .pt not found at $producedPt" >&2
+    echo "       contents of $workDir/pt_outputs:" >&2
+    ls -la "$workDir/pt_outputs" >&2 || true
+    exit 3
+fi
+
+mkdir -p "$outDir/PT/$subDir"
+mv "$producedPt" "$outDir/PT/$subDir/$ptName"
+rm -f "$workDir"/*.npz "$workDir"/*.json

run-cl-matching/run_cl_matching_2x2_sim.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# 2x2 charge-light matching (simulation).
+#
+# Input  : run-ndlar-flow/<IN_NAME>/FLOW/<subDir>/<inName>.FLOW.hdf5
+#          (a 2x2-configured flow file)
+# Output : run-cl-matching/<OUT_NAME>/PT/<subDir>/<outName>.qlmatch2x2.pt
+#
+# The 2x2 workflow always produces a .pt (the 2x2 calib_*_hits dtypes do not
+# yet reserve t_0/t_cluster_id, so we cannot do in-place HDF5 writeback).
+#
+# Algorithm version (env ND_PRODUCTION_CLMATCH_VERSION):
+#   v1.0 (default) = error-matrix small-cluster association (greedy, unit-var)
+#   v2.0           = region-grow + learned-variance tiebreaker
+#
+# Requires a 4-GPU node. On Perlmutter:
+#   salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \
+#     srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_2x2_sim.sh
+
+source ../util/reload_in_container.inc.sh
+source ../util/init.inc.sh
+
+CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"}
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+VERSION=${ND_PRODUCTION_CLMATCH_VERSION:-v1.0}
+
+if [[ ! -d "$CLMATCH_REPO" ]]; then
+    echo "ERROR: CLMatching repo not found at $CLMATCH_REPO"
+    echo "       Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2
+    exit 1
+fi
+
+inDir=${ND_PRODUCTION_OUTDIR_BASE}/run-ndlar-flow/$ND_PRODUCTION_IN_NAME
+inName=$ND_PRODUCTION_IN_NAME.$globalIdx
+inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5)
+
+# 2x2 produces a .pt, not a modified HDF5. Stage workers/shards in a per-file
+# tmp dir; mv the final .pt into the canonical outDir at the end.
+workDir=$tmpOutDir/${outName}_work
+ptName=${outName}.qlmatch2x2.pt
+rm -rf "$workDir"
+
+set -o errexit
+mkdir -p "$workDir"
+
+cd "$CLMATCH_REPO"
+
+run env FILE="$inFile" \
+    VERSION="$VERSION" \
+    OUT_DIR="$workDir" \
+    PT_DIR="$workDir/pt_outputs" \
+    LOG_DIR="$workDir/logs" \
+    PY="$PY" HERE="$CLMATCH_REPO" \
+    bash scripts/run_2x2_sim.sh
+
+# The aggregator names the .pt as '<full basename incl. .hdf5>.qlmatch2x2.pt'
+# (yes, it keeps the .hdf5 in the .pt filename).
+srcBasename=$(basename "$inFile")
+producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt"
+if [[ ! -f "$producedPt" ]]; then
+    echo "ERROR: expected .pt not found at $producedPt" >&2
+    echo "       contents of $workDir/pt_outputs:" >&2
+    ls -la "$workDir/pt_outputs" >&2 || true
+    exit 2
+fi
+
+mkdir -p "$outDir/PT/$subDir"
+mv "$producedPt" "$outDir/PT/$subDir/$ptName"
+# Keep worker logs for debugging; drop the per-event NPZ shards (large, transient).
+rm -f "$workDir"/*.npz "$workDir"/*.json

run-cl-matching/run_cl_matching_ND_sim.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# ND-LAr charge-light matching (simulation).
+#
+# Input  : run-ndlar-flow/<IN_NAME>/FLOW/<subDir>/<inName>.FLOW.hdf5
+# Output : run-cl-matching/<OUT_NAME>/FLOW/<subDir>/<outName>.FLOW.hdf5
+#
+# This step rewrites t_0 and t_cluster_id INSIDE the FLOW.hdf5 (Mode A; the
+# new flow dtype reserves those fields). The output is therefore another
+# .FLOW.hdf5, not a .pt. For older flow files without those fields the
+# underlying CLMatching aggregator falls back to .pt under
+# <CLMatching repo>/output/QLmatchingvAlpha/ -- but for ND production we
+# assume Mode A.
+#
+# Requires a 4-GPU node. On Perlmutter:
+#   salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \
+#     srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_ND_sim.sh
+#
+# (Or invoke inside an sbatch script that grabs a GPU node.)
+
+source ../util/reload_in_container.inc.sh
+source ../util/init.inc.sh
+
+CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"}
+PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python}
+
+if [[ ! -d "$CLMATCH_REPO" ]]; then
+    echo "ERROR: CLMatching repo not found at $CLMATCH_REPO"
+    echo "       Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2
+    exit 1
+fi
+
+inDir=${ND_PRODUCTION_OUTDIR_BASE}/run-ndlar-flow/$ND_PRODUCTION_IN_NAME
+inName=$ND_PRODUCTION_IN_NAME.$globalIdx
+inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5)
+
+# CL matching modifies the flow file IN-PLACE for Mode A. To avoid scribbling
+# on the upstream step's output, copy to tmpOutDir first, run there, then mv
+# the modified file to the canonical outDir.
+outFile=$tmpOutDir/${outName}.FLOW.hdf5
+workDir=$tmpOutDir/${outName}_work
+rm -f "$outFile"
+rm -rf "$workDir"
+
+set -o errexit
+mkdir -p "$workDir"
+echo "Copying input flow file to tmp work area:"
+echo "  $inFile -> $outFile"
+cp "$inFile" "$outFile"
+
+cd "$CLMATCH_REPO"
+
+# The single-file driver auto-detects Mode A (in-place HDF5) vs Mode B (.pt)
+# from the source dtype and runs the 8-worker pipeline + aggregation.
+# Explicitly route worker shards/logs through our per-task $workDir so they
+# do not accumulate inside the cloned CLMatching repo across production runs.
+#
+# Bracket the `run` call in `set +/-o errexit`: process_one_flow_file.sh (and
+# its underlying launcher) may exit non-zero from spurious tail commands (e.g.
+# ls-ing pt_outputs that Mode A never populates). The Python sanity-check
+# below is the authoritative Mode A success criterion; do not trust rc here.
+set +o errexit
+run env PY="$PY" REPO="$CLMATCH_REPO" \
+    bash scripts/process_one_flow_file.sh "$outFile" "$workDir"
+set -o errexit
+
+# Sanity-check that Mode A populated t_0/t_cluster_id.
+"$PY" - <<PY
+import h5py, sys
+with h5py.File("$outFile", "r") as h:
+    for path in ("charge/calib_prompt_hits/data", "charge/calib_final_hits/data"):
+        d = h[path]
+        if "t_0" not in d.dtype.names or "t_cluster_id" not in d.dtype.names:
+            print(f"WARN: {path} dtype lacks t_0/t_cluster_id (Mode B file). Skipping HDF5-populated check.")
+            sys.exit(0)
+        nz = int((d["t_cluster_id"][:] != 0).any() or (d["t_0"][:] != 0).any())
+        if not nz:
+            print(f"ERROR: {path} t_0 and t_cluster_id are still all zero after CL matching.", file=sys.stderr)
+            sys.exit(2)
+print("Mode A writeback verified: t_0 and t_cluster_id populated in HDF5.")
+PY
+
+mkdir -p "$outDir/FLOW/$subDir"
+mv "$outFile" "$outDir/FLOW/$subDir"
+
+# Worker shards are large and transient; preserve the worker logs (small) under
+# the canonical LOGS dir so failures stay debuggable, then drop the rest.
+if [[ -d "$workDir/worker_logs" ]]; then
+    mkdir -p "$logDir"
+    cp -r "$workDir/worker_logs" "$logDir/${outName}_worker_logs" 2>/dev/null || true
+fi
+rm -rf "$workDir"