diff --git a/admin/install_everything.sh b/admin/install_everything.sh index d85b786..a3ae627 100755 --- a/admin/install_everything.sh +++ b/admin/install_everything.sh @@ -21,7 +21,7 @@ popd if [ "$detector" = "ndlar" ] || [ "$detector" = "2x2" ]; then echo "Installation of $detector specific tools" - pushd run-larnd-sim + pushd run-larnd-sim ./install_larnd_sim.sh popd @@ -29,6 +29,11 @@ if [ "$detector" = "ndlar" ] || [ "$detector" = "2x2" ]; then ./install_ndlar_flow.sh popd + pushd run-cl-matching + echo "Installing CL matching tools" + ./install_cl_matching.sh + popd + pushd run-pandora ./install_pandora.sh "$detector" popd diff --git a/run-cl-matching/.gitignore b/run-cl-matching/.gitignore new file mode 100644 index 0000000..7d171f1 --- /dev/null +++ b/run-cl-matching/.gitignore @@ -0,0 +1 @@ +/CLMatching_AlphaRelease diff --git a/run-cl-matching/install_cl_matching.sh b/run-cl-matching/install_cl_matching.sh new file mode 100755 index 0000000..b7e5e67 --- /dev/null +++ b/run-cl-matching/install_cl_matching.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Install the CLMatching_AlphaRelease repo into this directory. +# +# Usage: +# ./install_cl_matching.sh # clones into ./CLMatching_AlphaRelease +# ./install_cl_matching.sh /some/dir # clones into /some/dir/CLMatching_AlphaRelease +# +# Notes: +# * We do NOT build a dedicated venv here. The CL matching code uses the +# shared NERSC nersc-python (which already includes torch + cuda). The +# runtime scripts pin the python via the PY env var. +# * On a fresh clone the 2x2 perceiver / pulse / variance assets ship +# in-repo. The ND-LAr perceiver and a few pulse/variance files live on +# GitHub Releases and need to be downloaded; check_install.py reports +# what's missing and prints the download commands. + +source ../util/prelude.inc.sh + +installDir=${1:-.} +repoName=CLMatching_AlphaRelease +repoURL=https://github.com/MadivB/CLMatching_AlphaRelease.git +repoBranch=${ND_PRODUCTION_CLMATCH_BRANCH:-main} + +if [[ -e "$installDir/$repoName" ]]; then + echo "$installDir/$repoName already exists; delete it then run me again" + exit 1 +fi + +mkdir -p "$installDir" +cd "$installDir" + +git clone -b "$repoBranch" "$repoURL" "$repoName" + +PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python} + +cd "$repoName" + +echo +echo "--- Validating required assets via scripts/check_install.py ---" +"$PY" scripts/check_install.py || { + echo + echo "ERROR: one or more required CL matching assets are missing." + echo " Run the download command(s) printed above, then re-run:" + echo " $PY scripts/check_install.py" + echo + echo " The 2x2 assets are bundled in-repo (should always resolve)." + echo " The ND-LAr perceiver is hosted on GitHub Releases." + exit 1 +} +echo "--- CLMatching install OK ---" diff --git a/run-cl-matching/run_cl_matching_2x2_data.sh b/run-cl-matching/run_cl_matching_2x2_data.sh new file mode 100755 index 0000000..bbec18f --- /dev/null +++ b/run-cl-matching/run_cl_matching_2x2_data.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# 2x2 charge-light matching (real DATA). +# +# Real 2x2 DAQ flow files are NOT produced by ND_Production's run-ndlar-flow +# step; they live on the dune cfs area (or wherever the user points us). The +# input file is therefore identified by env var rather than the production +# IN_NAME / INDEX convention used by sim steps. +# +# Input : $ND_PRODUCTION_CLMATCH_DATA_FILE (absolute path to a 2x2 reflow file) +# default = /global/cfs/cdirs/dune/www/data/2x2/reflows/v10/flow/beam/ +# july10_2024/nominal_hv/packet-0050018-2024_07_10_09_36_12_CDT.FLOW.hdf5 +# Output : run-cl-matching//PT//.qlmatch2x2.pt +# +# Algorithm version (env ND_PRODUCTION_CLMATCH_VERSION): +# v1.0 (default) = error-matrix small-cluster association +# v2.0 = region-grow + learned-variance tiebreaker +# +# Requires a 4-GPU node. On Perlmutter: +# salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \ +# srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_2x2_data.sh + +source ../util/reload_in_container.inc.sh +source ../util/init.inc.sh + +CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"} +PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python} +VERSION=${ND_PRODUCTION_CLMATCH_VERSION:-v1.0} + +DEFAULT_DATA_FILE=/global/cfs/cdirs/dune/www/data/2x2/reflows/v10/flow/beam/july10_2024/nominal_hv/packet-0050018-2024_07_10_09_36_12_CDT.FLOW.hdf5 +inFile=${ND_PRODUCTION_CLMATCH_DATA_FILE:-$DEFAULT_DATA_FILE} + +if [[ ! -d "$CLMATCH_REPO" ]]; then + echo "ERROR: CLMatching repo not found at $CLMATCH_REPO" + echo " Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2 + exit 1 +fi +if [[ ! -f "$inFile" ]]; then + echo "ERROR: input data file not found: $inFile" + echo " Set ND_PRODUCTION_CLMATCH_DATA_FILE to point at a 2x2 reflow .FLOW.hdf5." >&2 + exit 2 +fi + +workDir=$tmpOutDir/${outName}_work +ptName=${outName}.qlmatch2x2.pt +rm -rf "$workDir" + +set -o errexit +mkdir -p "$workDir" + +cd "$CLMATCH_REPO" + +run env FILE="$inFile" \ + VERSION="$VERSION" \ + OUT_DIR="$workDir" \ + PT_DIR="$workDir/pt_outputs" \ + LOG_DIR="$workDir/logs" \ + PY="$PY" HERE="$CLMATCH_REPO" \ + bash scripts/run_2x2_data.sh + +# The aggregator names the .pt as '.qlmatch2x2.pt'. +srcBasename=$(basename "$inFile") +producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt" +if [[ ! -f "$producedPt" ]]; then + echo "ERROR: expected .pt not found at $producedPt" >&2 + echo " contents of $workDir/pt_outputs:" >&2 + ls -la "$workDir/pt_outputs" >&2 || true + exit 3 +fi + +mkdir -p "$outDir/PT/$subDir" +mv "$producedPt" "$outDir/PT/$subDir/$ptName" +rm -f "$workDir"/*.npz "$workDir"/*.json diff --git a/run-cl-matching/run_cl_matching_2x2_sim.sh b/run-cl-matching/run_cl_matching_2x2_sim.sh new file mode 100755 index 0000000..8aea440 --- /dev/null +++ b/run-cl-matching/run_cl_matching_2x2_sim.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# 2x2 charge-light matching (simulation). +# +# Input : run-ndlar-flow//FLOW//.FLOW.hdf5 +# (a 2x2-configured flow file) +# Output : run-cl-matching//PT//.qlmatch2x2.pt +# +# The 2x2 workflow always produces a .pt (the 2x2 calib_*_hits dtypes do not +# yet reserve t_0/t_cluster_id, so we cannot do in-place HDF5 writeback). +# +# Algorithm version (env ND_PRODUCTION_CLMATCH_VERSION): +# v1.0 (default) = error-matrix small-cluster association (greedy, unit-var) +# v2.0 = region-grow + learned-variance tiebreaker +# +# Requires a 4-GPU node. On Perlmutter: +# salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \ +# srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_2x2_sim.sh + +source ../util/reload_in_container.inc.sh +source ../util/init.inc.sh + +CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"} +PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python} +VERSION=${ND_PRODUCTION_CLMATCH_VERSION:-v1.0} + +if [[ ! -d "$CLMATCH_REPO" ]]; then + echo "ERROR: CLMatching repo not found at $CLMATCH_REPO" + echo " Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2 + exit 1 +fi + +inDir=${ND_PRODUCTION_OUTDIR_BASE}/run-ndlar-flow/$ND_PRODUCTION_IN_NAME +inName=$ND_PRODUCTION_IN_NAME.$globalIdx +inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5) + +# 2x2 produces a .pt, not a modified HDF5. Stage workers/shards in a per-file +# tmp dir; mv the final .pt into the canonical outDir at the end. +workDir=$tmpOutDir/${outName}_work +ptName=${outName}.qlmatch2x2.pt +rm -rf "$workDir" + +set -o errexit +mkdir -p "$workDir" + +cd "$CLMATCH_REPO" + +run env FILE="$inFile" \ + VERSION="$VERSION" \ + OUT_DIR="$workDir" \ + PT_DIR="$workDir/pt_outputs" \ + LOG_DIR="$workDir/logs" \ + PY="$PY" HERE="$CLMATCH_REPO" \ + bash scripts/run_2x2_sim.sh + +# The aggregator names the .pt as '.qlmatch2x2.pt' +# (yes, it keeps the .hdf5 in the .pt filename). +srcBasename=$(basename "$inFile") +producedPt="$workDir/pt_outputs/${srcBasename}.qlmatch2x2.pt" +if [[ ! -f "$producedPt" ]]; then + echo "ERROR: expected .pt not found at $producedPt" >&2 + echo " contents of $workDir/pt_outputs:" >&2 + ls -la "$workDir/pt_outputs" >&2 || true + exit 2 +fi + +mkdir -p "$outDir/PT/$subDir" +mv "$producedPt" "$outDir/PT/$subDir/$ptName" +# Keep worker logs for debugging; drop the per-event NPZ shards (large, transient). +rm -f "$workDir"/*.npz "$workDir"/*.json diff --git a/run-cl-matching/run_cl_matching_ND_sim.sh b/run-cl-matching/run_cl_matching_ND_sim.sh new file mode 100755 index 0000000..0006a99 --- /dev/null +++ b/run-cl-matching/run_cl_matching_ND_sim.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# ND-LAr charge-light matching (simulation). +# +# Input : run-ndlar-flow//FLOW//.FLOW.hdf5 +# Output : run-cl-matching//FLOW//.FLOW.hdf5 +# +# This step rewrites t_0 and t_cluster_id INSIDE the FLOW.hdf5 (Mode A; the +# new flow dtype reserves those fields). The output is therefore another +# .FLOW.hdf5, not a .pt. For older flow files without those fields the +# underlying CLMatching aggregator falls back to .pt under +# /output/QLmatchingvAlpha/ -- but for ND production we +# assume Mode A. +# +# Requires a 4-GPU node. On Perlmutter: +# salloc -A dune -q interactive -C gpu --gpus-per-node=4 -N 1 -t 60 \ +# srun -N1 -n1 --gpus-per-node=4 ./run_cl_matching_ND_sim.sh +# +# (Or invoke inside an sbatch script that grabs a GPU node.) + +source ../util/reload_in_container.inc.sh +source ../util/init.inc.sh + +CLMATCH_REPO=${ND_PRODUCTION_CLMATCH_REPO:-"$ND_PRODUCTION_INSTALL_DIR/CLMatching_AlphaRelease"} +PY=${PY:-/global/common/software/nersc/pe/conda-envs/26.1.0/python-3.13/nersc-python/bin/python} + +if [[ ! -d "$CLMATCH_REPO" ]]; then + echo "ERROR: CLMatching repo not found at $CLMATCH_REPO" + echo " Run ./install_cl_matching.sh first, or set ND_PRODUCTION_CLMATCH_REPO." >&2 + exit 1 +fi + +inDir=${ND_PRODUCTION_OUTDIR_BASE}/run-ndlar-flow/$ND_PRODUCTION_IN_NAME +inName=$ND_PRODUCTION_IN_NAME.$globalIdx +inFile=$(realpath $inDir/FLOW/$subDir/${inName}.FLOW.hdf5) + +# CL matching modifies the flow file IN-PLACE for Mode A. To avoid scribbling +# on the upstream step's output, copy to tmpOutDir first, run there, then mv +# the modified file to the canonical outDir. +outFile=$tmpOutDir/${outName}.FLOW.hdf5 +workDir=$tmpOutDir/${outName}_work +rm -f "$outFile" +rm -rf "$workDir" + +set -o errexit +mkdir -p "$workDir" +echo "Copying input flow file to tmp work area:" +echo " $inFile -> $outFile" +cp "$inFile" "$outFile" + +cd "$CLMATCH_REPO" + +# The single-file driver auto-detects Mode A (in-place HDF5) vs Mode B (.pt) +# from the source dtype and runs the 8-worker pipeline + aggregation. +# Explicitly route worker shards/logs through our per-task $workDir so they +# do not accumulate inside the cloned CLMatching repo across production runs. +# +# Bracket the `run` call in `set +/-o errexit`: process_one_flow_file.sh (and +# its underlying launcher) may exit non-zero from spurious tail commands (e.g. +# ls-ing pt_outputs that Mode A never populates). The Python sanity-check +# below is the authoritative Mode A success criterion; do not trust rc here. +set +o errexit +run env PY="$PY" REPO="$CLMATCH_REPO" \ + bash scripts/process_one_flow_file.sh "$outFile" "$workDir" +set -o errexit + +# Sanity-check that Mode A populated t_0/t_cluster_id. +"$PY" - </dev/null || true +fi +rm -rf "$workDir"