Crash handler core dedup

.github/workflows/ci.yml

@@ -54,14 +54,23 @@ jobs:
         run: |
           docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests'
 
+      - name: Configure host core_pattern for coredumps
+        if: ${{ !env.ACT }}
+        run: |
+          sudo sysctl -w kernel.core_pattern='core.%p'
+
+      - name: Run spindle-serial-ubuntu crash tests
+        run: |
+          docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test'
+
       - name: Bring spindle-serial-ubuntu down
         id: serial-ubuntu-down
         if: ${{ always() }}
         continue-on-error: true
         run: |
           cd containers/spindle-serial-ubuntu
           docker compose down
-          
+
   spindle-serial-ubuntu_arm64:
     name: Testsuite (Serial, Ubuntu, arm64)
     environment: Spindle CI
@@ -98,6 +107,15 @@ jobs:
         run: |
           docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests'
 
+      - name: Configure host core_pattern for coredumps
+        if: ${{ !env.ACT }}
+        run: |
+          sudo sysctl -w kernel.core_pattern='core.%p'
+
+      - name: Run spindle-serial-ubuntu crash tests
+        run: |
+          docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test'
+
       - name: Bring spindle-serial-ubuntu down
         id: serial-ubuntu-down
         if: ${{ always() }}
@@ -142,6 +160,15 @@ jobs:
         run: |
           docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3'
 
+      - name: Configure host core_pattern for coredumps
+        if: ${{ !env.ACT }}
+        run: |
+          sudo sysctl -w kernel.core_pattern='core.%p'
+
+      - name: Run spindle-flux-ubuntu crash tests
+        run: |
+          docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test'
+
       - name: Bring spindle-flux-ubuntu down
         id: flux-ubuntu-down
         if: ${{ always() }}
@@ -200,6 +227,15 @@ jobs:
         run: |
           docker exec slurm-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}'
 
+      - name: Configure host core_pattern for coredumps
+        if: ${{ !env.ACT }}
+        run: |
+          sudo sysctl -w kernel.core_pattern='core.%p'
+
+      - name: Run spindle-slurm-ubuntu crash tests
+        run: |
+          docker exec slurm-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm --nodes=${workers} --scratch=/shared/spindle_crash_test'
+
       - name: Bring spindle-slurm-ubuntu down
         id: slurm-ubuntu-down
         if: ${{ always() }}
@@ -321,6 +357,15 @@ jobs:
         run: |
           docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} --spindle-session ./runTests ${workers}'
 
+      - name: Configure host core_pattern for coredumps
+        if: ${{ !env.ACT }}
+        run: |
+          sudo sysctl -w kernel.core_pattern='core.%p'
+
+      - name: Run spindle-slurm-plugin-ubuntu crash tests
+        run: |
+          docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test'
+
       - name: Bring spindle-slurm-plugin-ubuntu down
         id: slurm-ubuntu-down
         if: ${{ always() }}

config.h.in

@@ -27,6 +27,9 @@
 /* Define if were using sockets for client/server communication */
 #undef COMM_SOCKET
 
+/* Enable --crash-dedup by default */
+#undef CRASH_DEDUP_ENABLED_BY_DEFAULT
+
 /* Defined if GLIBC has an allocation bug */
 #undef DTV_ALLOCATION_BUG
 

configure

@@ -879,6 +879,7 @@ enable_sec_munge
 enable_sec_keydir
 enable_sec_launchmon
 enable_sec_none
+enable_crash_dedup
 with_gcrypt_dir
 with_munge_dir
 with_glibc_be_dir
@@ -1576,6 +1577,7 @@ Optional Features:
   --enable-sec-launchmon  Enable LaunchmMON to propagate authentication key
                           for connections
   --enable-sec-none       Disable security authentication of connections
+  --enable-crash-dedup    Enable --crash-dedup by default (default: disabled)
   --enable-remap-pageone  Controls mmap behavior when replacing executables.
                           Should be false on older linux systems
   --enable-testsuite      Chooses whether to build the Spindle testsuite.
@@ -17658,6 +17660,20 @@ if test "${enable_sec_none+set}" = set; then :
 fi
 
 
+# Check whether --enable-crash-dedup was given.
+if test "${enable_crash_dedup+set}" = set; then :
+  enableval=$enable_crash_dedup; enable_crash_dedup=$enableval
+else
+  enable_crash_dedup=no
+fi
+
+
+if test "x$enable_crash_dedup" = xyes; then :
+
+$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h
+
+fi
+
 
 # Check whether --with-gcrypt-dir was given.
 if test "${with_gcrypt_dir+set}" = set; then :

configure.common.ac

@@ -427,6 +427,16 @@ AC_ARG_ENABLE(sec-none,
               [AS_HELP_STRING([--enable-sec-none],[Disable security authentication of connections])],
               [WANT_NOSEC="true";EXPLICIT_SEC="true"],)
 
+AC_ARG_ENABLE([crash-dedup],
+  [AS_HELP_STRING([--enable-crash-dedup],
+     [Enable --crash-dedup by default (default: disabled)])],
+  [enable_crash_dedup=$enableval],
+  [enable_crash_dedup=no])
+
+AS_IF([test "x$enable_crash_dedup" = xyes],
+  [AC_DEFINE([CRASH_DEDUP_ENABLED_BY_DEFAULT], [1],
+             [Enable --crash-dedup by default])])
+
 AC_ARG_WITH(gcrypt-dir,
             [AS_HELP_STRING([--with-gcrypt-dir=DIR],[Look in DIR for gcrypt installation])],
             [GCRYPT_CFLAGS="-I${withval}/include";GCRYPT_LIBS="-L${withval}/lib"],)

containers/spindle-flux-ubuntu/Dockerfile

@@ -7,6 +7,7 @@ ENV workers=${replicas}
 USER root
 ENV TMPDIR=/tmp
 RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
     && apt-get -qq install -y --no-install-recommends \
@@ -26,6 +27,7 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
         libgcrypt20 \
         libgcrypt20-dev \
         gdb \
+        libc6-dbg \
         software-properties-common
 
 ARG USER=fluxuser

containers/spindle-flux-ubuntu/docker-compose.yml

@@ -31,12 +31,19 @@ networks:
   flux:
     driver: bridge
 
+# Shared scratch volume so run_crash_tests.sh can count coredumps
+# produced by any rank on any node from a single location.
+volumes:
+  crash_scratch:
+
 # Common parameters for all nodes.
 x-shared-node-parameters: &shared-node-parameters
   build: *shared-build-context
   networks:
     - flux
   environment: *shared-environment
+  volumes:
+    - crash_scratch:/shared
   cap_add:
     - SYS_NICE # Required for libnuma
 

containers/spindle-flux-ubuntu/scripts/build_spindle.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-flux-ubuntu/scripts/entrypoint.sh

@@ -22,6 +22,11 @@ thisHost=${thisHost[0]}
 echo $thisHost
 export FLUX_FAKE_HOSTNAME=$thisHost
 
+if [ -d /shared ]; then
+    sudo chown -R "$(id -un):$(id -gn)" /shared
+    sudo chmod 755 /shared
+fi
+
 # Start munged
 sudo -u munge /usr/sbin/munged
 

containers/spindle-serial-ubuntu/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:${ubuntu_version}
 USER root
 ENV TMPDIR=/tmp
 RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
 # install latest pkg utils:
@@ -26,6 +27,8 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recomme
         gfortran-13 \
         gcc-13 \
         g++-13 \
+        gdb \
+        libc6-dbg \
         munge \
         libmunge-dev \
         libhwloc-dev \

containers/spindle-serial-ubuntu/scripts/build_spindle.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-slurm-ubuntu/base/Dockerfile

@@ -27,6 +27,7 @@ RUN apt-get update \
         gfortran-13 \
         gcc-13 \
         g++-13 \
+        gdb \
         munge \
         libmunge-dev \
         libhwloc-dev \

containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile

@@ -4,9 +4,15 @@ ARG replicas=4
 ENV workers=${replicas}
 ENV TMPDIR=/tmp
 RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin
 
+RUN apt-get update \
+    && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
+    gdb \
+    libc6-dbg
+
 # Slurm daemons run as $SLURM_USER
 ARG SLURM_USER=slurm
 

containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml

@@ -35,13 +35,21 @@ networks:
   slurm:
     driver: bridge
 
+# Shared scratch volume so run_crash_tests.sh can count coredumps
+# written by ranks on any node from the head node.
+volumes:
+  crash_scratch:
+
 # Common parameters for all nodes.
 x-shared-node-parameters: &shared-node-parameters
   build: *shared-build-context
   networks:
     - slurm
+  volumes:
+    - crash_scratch:/shared
   cap_add:
     - SYS_NICE # Required for libnuma
+    - SYS_RESOURCE # Allow changing ulimit
 
 x-healthcheck-parameters: &healthcheck-parameters
   start_period: 3s

containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh

@@ -3,9 +3,14 @@
 echo "SLURM_ROLE: ${SLURM_ROLE}"
 
 echo "Starting sshd..."
-sudo service ssh start
+sudo bash -c 'ulimit -c unlimited; service ssh start'
 echo "Starting munged..."
-sudo -u munge /usr/sbin/munged 
+sudo -u munge /usr/sbin/munged
+
+if [ -d /shared ]; then
+    sudo chown -R "$(id -un):$(id -gn)" /shared
+    sudo chmod 755 /shared
+fi
 
 if [ "${SLURM_ROLE}" = "db" ]; then
     echo "Starting slurmdbd..."
@@ -15,7 +20,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then
     sudo -u slurm /usr/sbin/slurmctld -i -Dvvv
 elif [ "${SLURM_ROLE}" = "worker" ] ; then
     echo "Starting slurmd..."
-    sudo /usr/sbin/slurmd -Dvvv
+    sudo bash -c 'ulimit -c unlimited; exec env -u PWD /usr/sbin/slurmd -Dvvv'
 fi
 
 sleep inf

containers/spindle-slurm-ubuntu/testing/Dockerfile

@@ -4,9 +4,15 @@ ARG replicas=4
 ENV workers=${replicas}
 ENV TMPDIR=/tmp
 RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing
 
+RUN apt-get update \
+    && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
+    gdb \
+    libc6-dbg
+
 # Slurm daemons run as $SLURM_USER
 ARG SLURM_USER=slurm
 

containers/spindle-slurm-ubuntu/testing/docker-compose.yml

@@ -35,13 +35,21 @@ networks:
   slurm:
     driver: bridge
 
+# Shared scratch volume so run_crash_tests.sh can count coredumps
+# written by ranks on any node from the head node.
+volumes:
+  crash_scratch:
+
 # Common parameters for all nodes.
 x-shared-node-parameters: &shared-node-parameters
   build: *shared-build-context
   networks:
     - slurm
+  volumes:
+    - crash_scratch:/shared
   cap_add:
     - SYS_NICE # Required for libnuma
+    - SYS_RESOURCE # Allow changing ulimit
 
 x-healthcheck-parameters: &healthcheck-parameters
   start_period: 3s

containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh

@@ -3,9 +3,14 @@
 echo "SLURM_ROLE: ${SLURM_ROLE}"
 
 echo "Starting sshd..."
-sudo service ssh start
+sudo bash -c 'ulimit -c unlimited; service ssh start'
 echo "Starting munged..."
-sudo -u munge /usr/sbin/munged 
+sudo -u munge /usr/sbin/munged
+
+if [ -d /shared ]; then
+    sudo chown -R "$(id -un):$(id -gn)" /shared
+    sudo chmod 755 /shared
+fi
 
 if [ "${SLURM_ROLE}" = "db" ]; then
     echo "Starting slurmdbd..."
@@ -15,7 +20,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then
     sudo -u slurm /usr/sbin/slurmctld -i -Dvvv
 elif [ "${SLURM_ROLE}" = "worker" ] ; then
     echo "Starting slurmd..."
-    sudo /usr/sbin/slurmd -Dvvv
+    sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv'
 fi
 
 sleep inf